diff --git a/.env.dev b/.env.dev index cdd6af510..fd7aa3ba4 100644 --- a/.env.dev +++ b/.env.dev @@ -1,3 +1,7 @@ +# Non-secret development overlays used while generating the Axum config blob. +# Sourcing this file alone does not configure the Axum server: also export the +# blob and referenced secret-store values as shown in docs/guide/getting-started.md. + # [publisher] TRUSTED_SERVER__PUBLISHER__ORIGIN_URL=http://localhost:9090 diff --git a/.env.example b/.env.example index c2ac88e3a..a7f5973cd 100644 --- a/.env.example +++ b/.env.example @@ -1,6 +1,14 @@ -# Trusted Server Environment Variables -# Copy this file to .env.dev, .env.staging, or .env.production and fill in values -# See docs/guide/configuration.md for details +# Trusted Server development environment variables +# Copy this file to .env.dev, .env.staging, or .env.production and fill in +# non-secret values. App-config secrets are key names in the pushed blob and +# their values belong in the platform secret store; see the configuration guide. +# For Axum runtime loading, export the config blob as: +# TRUSTED_SERVER_CONFIG_TRUSTED_SERVER_CONFIG_TRUSTED_SERVER_CONFIG= +# and export one secret per key name as: +# TRUSTED_SERVER_SECRET_TRUSTED_SERVER_SECRETS_= +# The commented examples below are CLI overlays for ordinary fields only. +# Fastly example: map logical app-config secrets to physical `ts_secrets`. +# EDGEZERO__STORES__SECRETS__TRUSTED_SERVER_SECRETS__NAME=ts_secrets # ============================================================================= # Publisher Settings @@ -8,14 +16,12 @@ TRUSTED_SERVER__PUBLISHER__DOMAIN=publisher.com TRUSTED_SERVER__PUBLISHER__COOKIE_DOMAIN=.publisher.com TRUSTED_SERVER__PUBLISHER__ORIGIN_URL=https://origin.publisher.com -TRUSTED_SERVER__PUBLISHER__PROXY_SECRET= # ============================================================================= # Synthetic ID Settings # ============================================================================= TRUSTED_SERVER__SYNTHETIC__COUNTER_STORE=counter_store TRUSTED_SERVER__SYNTHETIC__OPID_STORE=opid_store -TRUSTED_SERVER__SYNTHETIC__SECRET_KEY= # Template variables: client_ip, user_agent, first_party_id, auth_user_id, publisher_domain, accept_language TRUSTED_SERVER__SYNTHETIC__TEMPLATE={{ client_ip }}:{{ user_agent }}:{{ first_party_id }} diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b7a145e02..1f1bbe27a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -12,6 +12,7 @@ jobs: test-rust: name: cargo test runs-on: ubuntu-latest + timeout-minutes: 45 steps: - uses: actions/checkout@v4 @@ -230,9 +231,14 @@ jobs: run: | cargo clippy --manifest-path crates/trusted-server-cli/Cargo.toml --target "$(rustc -vV | sed -n 's/host: //p')" --all-targets -- -D warnings + - name: Set up Chrome for browser fixture tests + id: setup-chrome + uses: browser-actions/setup-chrome@v1 + - name: cargo test - run: | - cargo test --manifest-path crates/trusted-server-cli/Cargo.toml --target "$(rustc -vV | sed -n 's/host: //p')" + run: ./scripts/test-cli.sh + env: + CHROME: ${{ steps.setup-chrome.outputs.chrome-path }} test-typescript: name: vitest diff --git a/.gitignore b/.gitignore index 24b9e06aa..96ffa2a5c 100644 --- a/.gitignore +++ b/.gitignore @@ -63,3 +63,4 @@ src/*.html # leftover local build artifacts (node_modules, target, dist) that remain on disk. /crates/js/ /crates/integration-tests/ +wrangler.integration.generated.toml diff --git a/CHANGELOG.md b/CHANGELOG.md index b6c1aee9b..083d6e746 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed +- **Breaking:** Auction providers and bidder routes now use the configuration-first `[auction.providers.]` and `[auction.bidders.]` maps. The removed `[auction].providers = [...]` list and removed server fields under `[integrations.prebid]` and `[integrations.aps]` are rejected even when those integrations are disabled, and `ts config push` rejects the old shape before publication. Move PBS `server_url` to provider `endpoint`, server timeout to provider `timeout_ms`, request controls and bidder-parameter overrides to the `prebid-server` `profile_config`, notification suppression to `notifications`, and each former server bidder to an `[auction.bidders.]` route. Move APS endpoint, timeout, account, inventory, debug, and creative controls to an `aps` provider and its `profile_config`. Browser Prebid settings remain under `[integrations.prebid]`; values such as timeout and debug that previously affected both browser and server behavior must now be configured for each owner. Provider endpoints must be absolute HTTPS URLs. Only bidder codes present in `[auction.bidders]` are folded into Trusted Server requests; unlisted publisher bids remain native browser demand. Provider response names now use the configured provider ID, such as `pbs-main`, instead of the legacy literal `prebid`; audit consumers that match `AuctionResponse.provider`. This schema has no mixed-version-safe deployment order: old binaries reject the maps and new binaries reject the retired fields, so activate the new binary and config blob together. Rollbacks must restore an old-schema blob together with the old binary. - **Breaking** — Admin Basic-auth coverage now includes `GET /_ts/admin/ec`, `GET /_ts/admin/ec/{id}`, and `GET /_ts/admin/eids`. Existing configurations whose `[[handlers]]` patterns protect only the key-management endpoints now fail startup; broaden coverage before deploying, preferably with a namespace-boundary pattern such as `^/_ts/admin(?:/|$)`. Coverage of the dynamic `/_ts/admin/ec/{id}` route is no longer inferred from ID-shaped samples: the router accepts any segment after `/_ts/admin/ec/` and Basic Auth runs on the raw path before routing, so patterns anchored to the EC ID grammar (for example `^/_ts/admin/ec/[a-f0-9]{64}[.][A-Za-z0-9]{6}$`) are rejected in favor of a prefix-level matcher. Placeholder and well-known weak handler passwords (`changeme`, `password`, `admin`, `replace-with-…`) now fail startup on every handler rather than only on handlers inferred to cover an admin endpoint, because first-match-wins handler selection lets a narrow handler shadow the admin namespace. +- Prebid Server provider endpoints now normalize origin-only legacy `server_url` values to `/openrtb2/auction`. Query parameters are preserved, the canonical path loses a trailing slash, and configured non-root custom paths remain exact. - Publisher HTML uses the browser-only `Cache-Control: private, max-age=60` policy for successful GET document responses and their `304 Not Modified` revalidations when server-side ad templates are structurally inactive, while preserving origin `private`/`no-store` policies and request-scoped bot, prefetch, or consent-denied responses. The `private` directive prevents shared caches that use `Cache-Control` from storing the document. Cookie-bearing responses using the generated inactive policy are finalized as `private, max-age=0`; CDN-specific cache headers remain unchanged and continue to control supporting CDNs independently. Set `[creative_opportunities].enabled = false` to disable publisher HTML and SPA template delivery without disabling direct `POST /auction` callers; an absent configuration, an unmatched slot, or a disabled auction also make the stack structurally inactive. An explicit `enabled = false` is not compatible with older binaries: restore the default, re-push and finalize the config before rolling back. - **Breaking** — Replaced the legacy APS contextual integration with APS OpenRTB at `/e/pb/bid`. APS configuration now uses canonical `account_id` (`pub_id` remains a compatibility alias), no longer requires APS-specific slot IDs, and defaults script creative eligibility off. Operators must update the endpoint, disable native APS demand for Trusted Server cohorts, and prepare GAM/Universal Creative targeting for `hb_bidder=aps` before rollout. `aps` entries in Prebid bidder lists are logged and stripped. APS renderer winners now preserve the upstream bid `id`, omit `crid` when APS omits it, and carry `ext.trusted_server.renderer` instead of `adm`; external `/auction` consumers must support this response shape. - **Breaking** — All auction paths now forward only a validated publisher-owned page URL as `site.page`, removing query and fragment data. APS OpenRTB omits `site.ref`; the existing Prebid Server path continues to forward the browser `Referer` as `site.ref`. Query-driven sites may lose contextual targeting and per-page reporting signals that previously came from query parameters. diff --git a/Cargo.lock b/Cargo.lock index 8e1ceb1f3..19d6f7788 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -787,7 +787,7 @@ version = "3.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "faf9468729b8cbcea668e36183cb69d317348c2e08e994829fb56ebfdfbaac34" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.48.0", ] [[package]] @@ -1427,7 +1427,7 @@ dependencies = [ [[package]] name = "edgezero-adapter" version = "0.1.0" -source = "git+https://github.com/stackpop/edgezero?tag=v0.0.7#5c9886e51d17e6969531356bacdf27f144ac8a2e" +source = "git+https://github.com/stackpop/edgezero?rev=055f7e94e53f3b7d9551ae078db7f015930ba953#055f7e94e53f3b7d9551ae078db7f015930ba953" dependencies = [ "toml", ] @@ -1435,7 +1435,7 @@ dependencies = [ [[package]] name = "edgezero-adapter-axum" version = "0.1.0" -source = "git+https://github.com/stackpop/edgezero?tag=v0.0.7#5c9886e51d17e6969531356bacdf27f144ac8a2e" +source = "git+https://github.com/stackpop/edgezero?rev=055f7e94e53f3b7d9551ae078db7f015930ba953#055f7e94e53f3b7d9551ae078db7f015930ba953" dependencies = [ "anyhow", "async-trait", @@ -1463,7 +1463,7 @@ dependencies = [ [[package]] name = "edgezero-adapter-cloudflare" version = "0.1.0" -source = "git+https://github.com/stackpop/edgezero?tag=v0.0.7#5c9886e51d17e6969531356bacdf27f144ac8a2e" +source = "git+https://github.com/stackpop/edgezero?rev=055f7e94e53f3b7d9551ae078db7f015930ba953#055f7e94e53f3b7d9551ae078db7f015930ba953" dependencies = [ "anyhow", "async-trait", @@ -1486,7 +1486,7 @@ dependencies = [ [[package]] name = "edgezero-adapter-fastly" version = "0.1.0" -source = "git+https://github.com/stackpop/edgezero?tag=v0.0.7#5c9886e51d17e6969531356bacdf27f144ac8a2e" +source = "git+https://github.com/stackpop/edgezero?rev=055f7e94e53f3b7d9551ae078db7f015930ba953#055f7e94e53f3b7d9551ae078db7f015930ba953" dependencies = [ "anyhow", "async-stream", @@ -1515,7 +1515,7 @@ dependencies = [ [[package]] name = "edgezero-adapter-spin" version = "0.1.0" -source = "git+https://github.com/stackpop/edgezero?tag=v0.0.7#5c9886e51d17e6969531356bacdf27f144ac8a2e" +source = "git+https://github.com/stackpop/edgezero?rev=055f7e94e53f3b7d9551ae078db7f015930ba953#055f7e94e53f3b7d9551ae078db7f015930ba953" dependencies = [ "anyhow", "async-trait", @@ -1542,7 +1542,7 @@ dependencies = [ [[package]] name = "edgezero-cli" version = "0.1.0" -source = "git+https://github.com/stackpop/edgezero?tag=v0.0.7#5c9886e51d17e6969531356bacdf27f144ac8a2e" +source = "git+https://github.com/stackpop/edgezero?rev=055f7e94e53f3b7d9551ae078db7f015930ba953#055f7e94e53f3b7d9551ae078db7f015930ba953" dependencies = [ "chrono", "clap", @@ -1567,7 +1567,7 @@ dependencies = [ [[package]] name = "edgezero-core" version = "0.1.0" -source = "git+https://github.com/stackpop/edgezero?tag=v0.0.7#5c9886e51d17e6969531356bacdf27f144ac8a2e" +source = "git+https://github.com/stackpop/edgezero?rev=055f7e94e53f3b7d9551ae078db7f015930ba953#055f7e94e53f3b7d9551ae078db7f015930ba953" dependencies = [ "anyhow", "async-compression", @@ -1598,7 +1598,7 @@ dependencies = [ [[package]] name = "edgezero-macros" version = "0.1.0" -source = "git+https://github.com/stackpop/edgezero?tag=v0.0.7#5c9886e51d17e6969531356bacdf27f144ac8a2e" +source = "git+https://github.com/stackpop/edgezero?rev=055f7e94e53f3b7d9551ae078db7f015930ba953#055f7e94e53f3b7d9551ae078db7f015930ba953" dependencies = [ "log", "proc-macro2", @@ -3676,7 +3676,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "be769465445e8c1474e9c5dac2018218498557af32d9ed057325ec9a41ae81bf" dependencies = [ "heck", - "itertools 0.13.0", + "itertools 0.10.5", "log", "multimap", "once_cell", @@ -3696,7 +3696,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d" dependencies = [ "anyhow", - "itertools 0.13.0", + "itertools 0.10.5", "proc-macro2", "quote", "syn 2.0.118", @@ -3709,7 +3709,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b570b25f7617e43d59005d0990ccb79e950a423952cea19671b7a876da390adf" dependencies = [ "anyhow", - "itertools 0.13.0", + "itertools 0.10.5", "proc-macro2", "quote", "syn 2.0.118", @@ -5396,9 +5396,10 @@ dependencies = [ "futures", "log", "log-fastly", + "rand 0.8.6", "serde", "serde_json", - "sha2 0.10.9", + "toml", "trusted-server-core", "url", "urlencoding", @@ -5437,12 +5438,16 @@ dependencies = [ "derive_more", "directories", "edgezero-cli", + "edgezero-core", "error-stack", "futures", + "glob", + "http", "http-body-util", "hyper", "hyper-util", "log", + "rand 0.8.6", "rcgen", "regex", "rustls", @@ -5450,12 +5455,15 @@ dependencies = [ "scraper", "serde", "serde_json", + "similar", + "temp-env", "tempfile", "time", "tokio", "tokio-rustls", "toml", "toml_edit 0.23.10+spec-1.0.0", + "tracing", "trusted-server-core", "url", "webpki-roots", @@ -6012,7 +6020,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.48.0", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 7faba7553..2cf94b02a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -54,12 +54,12 @@ criterion = { version = "0.5", default-features = false, features = ["cargo_benc derive_more = { version = "2.0", features = ["display", "error"] } directories = "5" ed25519-dalek = { version = "2.2", features = ["rand_core"] } -edgezero-adapter-axum = { git = "https://github.com/stackpop/edgezero", tag = "v0.0.7", default-features = false } -edgezero-adapter-cloudflare = { git = "https://github.com/stackpop/edgezero", tag = "v0.0.7", default-features = false } -edgezero-adapter-fastly = { git = "https://github.com/stackpop/edgezero", tag = "v0.0.7", default-features = false } -edgezero-adapter-spin = { git = "https://github.com/stackpop/edgezero", tag = "v0.0.7", default-features = false } -edgezero-cli = { git = "https://github.com/stackpop/edgezero", tag = "v0.0.7" } -edgezero-core = { git = "https://github.com/stackpop/edgezero", tag = "v0.0.7", default-features = false } +edgezero-adapter-axum = { git = "https://github.com/stackpop/edgezero", rev = "055f7e94e53f3b7d9551ae078db7f015930ba953", default-features = false } +edgezero-adapter-cloudflare = { git = "https://github.com/stackpop/edgezero", rev = "055f7e94e53f3b7d9551ae078db7f015930ba953", default-features = false } +edgezero-adapter-fastly = { git = "https://github.com/stackpop/edgezero", rev = "055f7e94e53f3b7d9551ae078db7f015930ba953", default-features = false } +edgezero-adapter-spin = { git = "https://github.com/stackpop/edgezero", rev = "055f7e94e53f3b7d9551ae078db7f015930ba953", default-features = false } +edgezero-cli = { git = "https://github.com/stackpop/edgezero", rev = "055f7e94e53f3b7d9551ae078db7f015930ba953" } +edgezero-core = { git = "https://github.com/stackpop/edgezero", rev = "055f7e94e53f3b7d9551ae078db7f015930ba953", default-features = false } env_logger = "0.11" error-stack = "0.6" esi = "0.7.2" @@ -96,6 +96,7 @@ scraper = "0.24.0" serde = { version = "1.0", features = ["derive"] } serde_json = "1.0.149" sha2 = "0.10.9" +similar = "2.7" simple_logger = "5" spin-sdk = { version = "~6.0", default-features = false, features = ["http", "key-value", "variables"] } subtle = "2.6" @@ -108,6 +109,7 @@ tokio-rustls = "0.26" toml = "1.1" toml_edit = "0.23.10" tower = "0.4" +tracing = "0.1" trusted-server-core = { path = "crates/trusted-server-core" } trusted-server-js = { path = "crates/trusted-server-js" } trusted-server-openrtb = { path = "crates/trusted-server-openrtb" } diff --git a/README.md b/README.md index b87fe61ad..0ad5b0351 100644 --- a/README.md +++ b/README.md @@ -37,11 +37,12 @@ ts --help # Create local config, then edit placeholders before validation ts config init -# Edit trusted-server.toml +# Edit trusted-server.toml. Server auctions use map-shaped +# [auction.providers.] and [auction.bidders.] tables. ts config validate # Audit a public page with Chrome/Chromium to bootstrap a draft config -ts audit https://publisher.example +ts audit generate https://publisher.example # Run tests (Fastly/WASM crates — requires Viceroy) cargo test-fastly diff --git a/TESTING.md b/TESTING.md index e5ccba4cf..c50d10a67 100644 --- a/TESTING.md +++ b/TESTING.md @@ -1,22 +1,57 @@ -# Testing the Auction Orchestration System +# Testing auction orchestration -## Quick Test Summary +## Start the local server -The auction orchestration system has been integrated into the existing Prebid endpoints. You can test it right away using the Fastly local server! - -## How to Test - -### 1. Start the Local Server +Configure at least one reachable provider in `trusted-server.toml`, then start the +Fastly development server: ```bash fastly compute serve ``` -### 2. Test with Existing Endpoint +Provider endpoints must use HTTPS. Fastly and Viceroy also need a backend that +matches the provider host and TLS settings. For a deterministic local bidder, +use `scripts/template-cache-local-test.sh`, which creates a temporary CA and +registers the matching backend. + +## Example configuration + +```toml +[auction] +enabled = true +timeout_ms = 2000 +mediator = "adserver_mock" + +[auction.providers.pbs-main] +protocol = "openrtb-2.6" +profile = "prebid-server" +endpoint = "https://prebid.example.com/openrtb2/auction" +routing = "explicit" -The `/auction` endpoint now uses the orchestrator when `auction.enabled = true` in config. +[auction.providers.aps-main] +protocol = "openrtb-2.6" +profile = "aps" +endpoint = "https://aps.example.com/e/pb/bid" +routing = "all_eligible" +profile_config = { account_id = "example-aps-account", debug = false } + +[auction.bidders.example-server] +provider = "pbs-main" + +[integrations.adserver_mock] +enabled = true +endpoint = "https://mediator.example.com/mediate" +timeout_ms = 500 +``` + +Replace the example endpoints and profile values before running the server. +Omit `mediator` to test local highest-bid selection without mediation. + +## Send a routed request + +The PBS provider uses explicit routing, so the request must include params for a +bidder listed in `[auction.bidders]`: -**Test Request:** ```bash curl -X POST http://localhost:7676/auction \ -H "Content-Type: application/json" \ @@ -28,7 +63,15 @@ curl -X POST http://localhost:7676/auction \ "banner": { "sizes": [[728, 90], [970, 250]] } - } + }, + "bids": [ + { + "bidder": "example-server", + "params": { + "placement": "example-header-placement" + } + } + ] }, { "code": "sidebar", @@ -42,144 +85,70 @@ curl -X POST http://localhost:7676/auction \ }' ``` -### 3. What You'll See - -**With Orchestrator Enabled** (`auction.enabled = true`): -- Logs showing: `"Using auction orchestrator"` -- Parallel execution of APS OpenRTB and Prebid Server -- Optional mock-adserver mediation selecting winning bids -- Final response with winning creatives +The first impression routes to `pbs-main` and `aps-main`. The second routes only +to `aps-main` because APS uses `all_eligible` and PBS uses `explicit`. -**With Orchestrator Disabled** (`auction.enabled = false`): -- Logs showing: `"Using legacy Prebid flow"` -- Direct Prebid Server call (backward compatible) +## Check current logs -##Configuration - -Edit `trusted-server.toml` to customize the auction: - -```toml -# Enable/disable orchestrator -[auction] -enabled = true -providers = ["prebid", "aps"] -mediator = "adserver_mock" # If set: mediation, if omitted: highest bid wins -timeout_ms = 2000 +Startup logs report plan-backed construction and the provider count: -# APS OpenRTB provider. The built-in production endpoint is used when -# endpoint is omitted; use only an account authorized for test traffic. -[integrations.aps] -enabled = true -account_id = "example-account" -timeout_ms = 800 -debug = false - -[integrations.adserver_mock] -enabled = true -endpoint = "http://localhost:6767/adserver/mediate" -timeout_ms = 500 +```text +Building plan-backed auction orchestrator +Auction orchestrator built with 2 bidder providers ``` -## Test Scenarios +A launched request logs the configured provider ID, predicted backend, and +budget. Collection logs the pending and immediate response counts: -### Scenario 1: Parallel + Mediation (Default) -**Config:** -```toml -[auction] -enabled = true -providers = ["prebid", "aps"] -mediator = "adserver_mock" # Mediator configured = parallel mediation strategy +```text +Dispatching bid request to 'pbs-main' (backend: ..., budget: ...ms) +Dispatching bid request to 'aps-main' (backend: ..., budget: ...ms) +Dispatched 2 SSP request(s) with 0 immediate response(s) (timeout: ...ms) ``` -**Expected Flow:** -1. Prebid queries its configured bidders through Prebid Server -2. APS sends an OpenRTB request for eligible banner impressions -3. AdServer Mock mediates the provider responses -4. The winning creative or typed APS renderer is returned +Exact backend names and budgets depend on the adapter and remaining auction +deadline. Provider failures are isolated and appear in response metadata under +the configured provider ID. -### Scenario 2: Parallel Only (No Mediation) -**Config:** -```toml -[auction] -enabled = true -providers = ["prebid", "aps"] -# No mediator = parallel only strategy -``` +## Disabled auction -**Expected Flow:** -1. Prebid and APS run in parallel -2. Highest bid wins automatically -3. No mediation +Set: -### Scenario 3: Legacy Mode (Backward Compatible) -**Config:** ```toml [auction] enabled = false ``` -**Expected Flow:** -- Original Prebid-only behavior -- No orchestration overhead - -## Debugging +`POST /auction` returns an immediate no-bid response, emits an +`auction_disabled` skipped telemetry event, and performs no provider or mediator +work. The request log is: -### Check Logs -The orchestrator logs extensively: +```text +/auction: auction is disabled; returning no-bid response ``` -INFO: Using auction orchestrator -INFO: Running auction with strategy: parallel_mediation -INFO: Running 2 bidders in parallel -INFO: Requesting bids from: prebid -INFO: Prebid returned 2 bids (time: 120ms) -INFO: Requesting bids from: aps -INFO: APS requests bids for 2 impressions -INFO: APS returns 2 accepted bids in 80ms -INFO: GAM mediation: slot 'header-banner' won by 'aps' at $2.50 CPM -``` - -### Verify Provider Registration -Look for these log messages on startup: -``` -INFO: Registering auction provider: prebid -INFO: Registering auction provider: aps -INFO: Registering auction provider: adserver_mock -``` - -### Common Issues - -**Issue:** `"Provider 'aps' not registered"` -**Fix:** Make sure `[integrations.aps]` is configured in `trusted-server.toml` - -**Issue:** `"No providers configured"` -**Fix:** Make sure `providers = ["prebid", "aps"]` is set in `[auction]` -**Issue:** Tests fail with WASM errors -**Explanation:** Async tests don't work in WASM test environment. Integration tests via HTTP work fine! +## Automated checks -## Next Steps +Use the repository aliases instead of bare `cargo test --workspace`: -1. **Verify Prebid Server demand** - Confirm configured bidders return expected test bids -2. **Verify APS eligibility** - Confirm the test account, inventory identity, and `/e/pb/bid` endpoint are authorized -3. **Exercise renderer security** - Run the APS browser integration suite for iframe and script creatives -4. **Add metrics** - Track bid rates, win rates, latency, and aggregate drop reasons per provider +```bash +cargo test-fastly +cargo test-axum +cargo test-cloudflare +cargo test-spin +``` -## Provider Behavior +For browser integration tests: -### APS (Amazon) -- Sends real OpenRTB requests for eligible banner slots -- Safely drops malformed, unsupported, or unrenderable bids and reports aggregate reasons -- Reduces multiple APS candidates to one winner per impression -- Returns typed renderer descriptors rather than exposing `adm` outside the sandbox -- Automated tests intercept upstream traffic and use fictional response fixtures +```bash +cd crates/trusted-server-js/lib +npx vitest run +``` -### AdServer Mock -- Acts as mediator by calling mocktioneer's mediation endpoint -- Selects winning bids based on highest CPM -- Response time varies based on mocktioneer instance +The template-cache harness exercises plan compilation, HTTPS backend naming, +provider dispatch, mediation, and both ESI and inline delivery modes: -### Prebid -- **Real implementation** - makes actual HTTP calls -- Queries configured SSPs -- Returns real bids from real bidders -- Response time: varies (network dependent) +```bash +./scripts/template-cache-local-test.sh esi +./scripts/template-cache-local-test.sh inline +``` diff --git a/crates/trusted-server-adapter-axum/Cargo.toml b/crates/trusted-server-adapter-axum/Cargo.toml index 15b6ee59d..09e8c77d2 100644 --- a/crates/trusted-server-adapter-axum/Cargo.toml +++ b/crates/trusted-server-adapter-axum/Cargo.toml @@ -20,6 +20,7 @@ path = "src/main.rs" [dependencies] async-trait = { workspace = true } +axum = { workspace = true } edgezero-adapter-axum = { workspace = true, features = ["axum"] } edgezero-core = { workspace = true } error-stack = { workspace = true } @@ -27,12 +28,11 @@ futures = { workspace = true } log = { workspace = true } reqwest = { workspace = true } simple_logger = { workspace = true } -tokio = { workspace = true, features = ["rt-multi-thread", "macros", "sync", "time"] } +tokio = { workspace = true, features = ["rt-multi-thread", "macros", "net", "signal", "sync", "time"] } +tower = { workspace = true, features = ["util"] } trusted-server-core = { workspace = true } [dev-dependencies] -axum = { workspace = true } base64 = { workspace = true } temp-env = { workspace = true } tokio = { workspace = true, features = ["rt-multi-thread", "macros"] } -tower = { workspace = true, features = ["util"] } diff --git a/crates/trusted-server-adapter-axum/src/app.rs b/crates/trusted-server-adapter-axum/src/app.rs index 9a371f805..fcf8bf98f 100644 --- a/crates/trusted-server-adapter-axum/src/app.rs +++ b/crates/trusted-server-adapter-axum/src/app.rs @@ -1,6 +1,7 @@ use core::future::Future; use std::sync::Arc; +use edgezero_adapter_axum::service::EdgeZeroAxumService; use edgezero_core::app::Hooks; use edgezero_core::context::RequestContext; use edgezero_core::error::EdgeError; @@ -10,7 +11,9 @@ use edgezero_core::http::{ use edgezero_core::router::RouterService; use error_stack::Report; use trusted_server_core::auction::endpoints::handle_auction; -use trusted_server_core::auction::{AuctionOrchestrator, build_orchestrator}; +use trusted_server_core::auction::{ + AuctionOrchestrator, build_orchestrator_with_plan, compile_auction_plan, +}; use trusted_server_core::cache_policy::EdgeCacheHeader; use trusted_server_core::ec::EcContext; use trusted_server_core::ec::admin::{ @@ -38,7 +41,7 @@ use trusted_server_core::settings_data::{ use trusted_server_core::platform::RuntimeServices; use crate::middleware::{AuthMiddleware, FinalizeResponseMiddleware, SanitizeRequestMiddleware}; -use crate::platform::{AxumPlatformConfigStore, build_runtime_services}; +use crate::platform::{AxumPlatformConfigStore, AxumPlatformSecretStore, build_runtime_services}; // --------------------------------------------------------------------------- // AppState @@ -60,8 +63,13 @@ pub struct AppState { fn build_state() -> Result, Report> { let store_name = default_config_store_name(); let config_key = default_config_key(); - let settings = - get_settings_from_config_store(&AxumPlatformConfigStore, &store_name, &config_key)?; + let settings = get_settings_from_config_store( + &AxumPlatformConfigStore, + &AxumPlatformSecretStore, + &store_name, + &config_key, + &trusted_server_core::settings_data::default_secret_store_name(), + )?; build_state_with_settings(settings) } @@ -74,8 +82,10 @@ fn build_state() -> Result, Report> { fn build_state_with_settings( settings: Settings, ) -> Result, Report> { - let orchestrator = build_orchestrator(&settings)?; - let registry = IntegrationRegistry::new(&settings)?; + let plan = Arc::new(compile_auction_plan(&settings)?); + plan.validate_for_target(trusted_server_core::platform::AuctionTargetId::Axum)?; + let orchestrator = build_orchestrator_with_plan(Arc::clone(&plan), &settings)?; + let registry = IntegrationRegistry::with_plan(&settings, plan)?; Ok(Arc::new(AppState { settings: Arc::new(settings), @@ -454,13 +464,13 @@ fn named_route_handler( // Build the geo-aware EC context so the auction consent // gate sees the caller's jurisdiction — `EcContext::default()` // fails it closed for consented users. - let ec_context = build_ec_context(&state, &services, &req); + let mut ec_context = build_ec_context(&state, &services, &req); handle_auction( &state.settings, &state.orchestrator, None, None, - &ec_context, + &mut ec_context, &services, req, ) @@ -473,7 +483,7 @@ fn named_route_handler( if req.method() == Method::OPTIONS { Ok(page_bids_preflight_denied()) } else { - let ec_context = build_ec_context(&state, &services, &req); + let mut ec_context = build_ec_context(&state, &services, &req); let auction = AuctionDispatch { orchestrator: &state.orchestrator, slots: state.settings.creative_opportunity_slots(), @@ -484,7 +494,7 @@ fn named_route_handler( &services, None, auction, - &ec_context, + &mut ec_context, req, ) .await @@ -565,15 +575,7 @@ impl Hooks for TrustedServerApp { } fn routes() -> RouterService { - let state = match build_state() { - Ok(s) => s, - Err(ref e) => { - log::error!("failed to build application state: {:?}", e); - return startup_error_router(e); - } - }; - - build_router(&state) + Self::routes_with_server_timing_flag().0 } } @@ -594,6 +596,44 @@ impl TrustedServerApp { let state = build_state_with_settings(settings)?; Ok(build_router(&state)) } + + /// The dev server's fully configured tower service: the application + /// router wrapped in the terminal timing layer + /// ([`crate::timing::TimingService`]), with `server_timing_enabled` + /// read from the same settings snapshot that built the router. + /// + /// This is the standard construction path for serving this adapter. + /// [`Hooks::routes`] satisfies the `Hooks` trait contract and returns + /// the bare router without the timing layer; callers who serve traffic + /// should use this instead so `server_timing_enabled` is never + /// silently discarded. + #[must_use] + pub fn dev_server_service() -> crate::timing::TimingService { + let (router, server_timing_enabled) = Self::routes_with_server_timing_flag(); + crate::timing::TimingService::new(EdgeZeroAxumService::new(router), server_timing_enabled) + } + + /// Build the router alongside whether `Server-Timing` emission is + /// enabled, read from the same settings snapshot used to build the + /// router. + /// + /// The Axum dev server's terminal timing layer ([`crate::timing`]) needs + /// this flag once at startup: unlike the Fastly adapter, which rebuilds + /// `Settings` per request, the Axum dev server builds its application + /// state once and reuses the same [`RouterService`] for every request. + #[must_use] + fn routes_with_server_timing_flag() -> (RouterService, bool) { + let state = match build_state() { + Ok(s) => s, + Err(ref e) => { + log::error!("failed to build application state: {:?}", e); + return (startup_error_router(e), false); + } + }; + + let server_timing_enabled = state.settings.observability.server_timing_enabled; + (build_router(&state), server_timing_enabled) + } } fn build_router(state: &Arc) -> RouterService { diff --git a/crates/trusted-server-adapter-axum/src/lib.rs b/crates/trusted-server-adapter-axum/src/lib.rs index 2f15e566d..b1d4c3dd8 100644 --- a/crates/trusted-server-adapter-axum/src/lib.rs +++ b/crates/trusted-server-adapter-axum/src/lib.rs @@ -10,3 +10,6 @@ pub mod app; pub mod middleware; /// Platform-trait implementations backed by env vars and `reqwest`. pub mod platform; +/// Terminal timing layer wrapping the Axum dev server's tower `Service` +/// boundary with the request-phase `Server-Timing` freeze point. +pub mod timing; diff --git a/crates/trusted-server-adapter-axum/src/main.rs b/crates/trusted-server-adapter-axum/src/main.rs index 960982176..4e360ea41 100644 --- a/crates/trusted-server-adapter-axum/src/main.rs +++ b/crates/trusted-server-adapter-axum/src/main.rs @@ -1,6 +1,15 @@ -use edgezero_adapter_axum::dev_server::{AxumDevServer, AxumDevServerConfig}; -use edgezero_core::app::Hooks as _; +use std::net::SocketAddr; + +use axum::Router; +use edgezero_adapter_axum::dev_server::AxumDevServerConfig; +use edgezero_adapter_axum::service::EdgeZeroAxumService; +use tokio::net::TcpListener; +use tokio::runtime::Builder as RuntimeBuilder; +use tokio::signal; +use tower::Service as _; +use tower::service_fn; use trusted_server_adapter_axum::app::TrustedServerApp; +use trusted_server_adapter_axum::timing::TimingService; #[allow(clippy::print_stderr)] fn main() { @@ -20,13 +29,63 @@ fn main() { }; log::info!("Listening on http://{}", config.addr); - let router = TrustedServerApp::routes(); - if let Err(err) = AxumDevServer::with_config(router, config).run() { + let service = TrustedServerApp::dev_server_service(); + if let Err(err) = run(service, config) { log::error!("trusted-server-adapter-axum failed: {err}"); std::process::exit(1); } } +/// Runs the Axum dev server with the request-phase timing terminal layer +/// ([`trusted_server_adapter_axum::timing::TimingService`]) wrapped around +/// `EdgeZeroAxumService`, ahead of `axum::serve`. +/// +/// This does not use `edgezero_adapter_axum::dev_server::AxumDevServer::run`: +/// that helper only accepts a bare [`RouterService`] and builds its own +/// `EdgeZeroAxumService` and `axum::Router` internally, with no seam for an +/// outer service wrapper. Router-generated 404/405 responses bypass +/// `RouterBuilder::middleware` (see `trusted_server_adapter_axum::timing`), +/// so the freeze point has to wrap the tower `Service` boundary itself. +/// Driving `axum::serve` directly here mirrors that helper's own internal +/// bind/wrap/serve/shutdown sequence closely enough to keep behavior +/// identical for callers (`PORT` env var, ctrl-c graceful shutdown). +/// +/// # Errors +/// +/// Returns an error if the Tokio runtime fails to start, the listener fails +/// to bind, or the underlying serve loop errors. +fn run( + service: TimingService, + config: AxumDevServerConfig, +) -> std::io::Result<()> { + let runtime = RuntimeBuilder::new_multi_thread().enable_all().build()?; + runtime.block_on(serve(service, config)) +} + +async fn serve( + service: TimingService, + config: AxumDevServerConfig, +) -> std::io::Result<()> { + let listener = TcpListener::bind(config.addr).await?; + + let axum_router = Router::new().fallback_service(service_fn(move |req| { + let mut svc = service.clone(); + async move { svc.call(req).await } + })); + let make_service = axum_router.into_make_service_with_connect_info::(); + + let server = axum::serve(listener, make_service); + if config.enable_ctrl_c { + server + .with_graceful_shutdown(async { + let _ctrl_c = signal::ctrl_c().await; + }) + .await + } else { + server.await + } +} + /// Read a port number from the `PORT` environment variable. /// /// Returns `None` when the variable is unset. Exits non-zero if the value diff --git a/crates/trusted-server-adapter-axum/src/platform.rs b/crates/trusted-server-adapter-axum/src/platform.rs index a511daab2..7dcdd53d8 100644 --- a/crates/trusted-server-adapter-axum/src/platform.rs +++ b/crates/trusted-server-adapter-axum/src/platform.rs @@ -9,9 +9,10 @@ use async_trait::async_trait; use edgezero_core::http::{HeaderMap, HeaderName, HeaderValue, header}; use error_stack::{Report, ResultExt as _}; use trusted_server_core::platform::{ - ClientInfo, GeoInfo, PlatformBackend, PlatformBackendSpec, PlatformConfigStore, PlatformError, - PlatformGeo, PlatformHttpClient, PlatformHttpRequest, PlatformPendingRequest, PlatformResponse, - PlatformSecretStore, PlatformSelectResult, RuntimeServices, StoreId, StoreName, + BackendNamingPolicy, ClientInfo, GeoInfo, PlatformBackend, PlatformBackendSpec, + PlatformConfigStore, PlatformError, PlatformGeo, PlatformHttpClient, PlatformHttpRequest, + PlatformPendingRequest, PlatformResponse, PlatformSecretStore, PlatformSelectResult, + RuntimeServices, StoreId, StoreName, }; // --------------------------------------------------------------------------- @@ -154,24 +155,15 @@ impl PlatformSecretStore for AxumPlatformSecretStore { pub struct AxumPlatformBackend; impl PlatformBackend for AxumPlatformBackend { + fn naming_policy(&self) -> BackendNamingPolicy { + BackendNamingPolicy::Axum + } + fn predict_name(&self, spec: &PlatformBackendSpec) -> Result> { - let port = spec - .port - .unwrap_or(if spec.scheme == "https" { 443 } else { 80 }); - // Keep two providers that share an origin on distinct names so auction - // response correlation cannot cross providers. - let discriminator = spec - .discriminator - .as_deref() - .map(|d| format!("_p_{}", normalize_env_segment(d))) - .unwrap_or_default(); - Ok(format!( - "{}_{}_{}{}", - normalize_env_segment(&spec.scheme), - normalize_env_segment(&spec.host), - port, - discriminator, - )) + self.naming_policy() + .predict(spec) + .map(|prediction| prediction.name) + .change_context(PlatformError::Backend) } fn ensure(&self, spec: &PlatformBackendSpec) -> Result> { @@ -601,6 +593,21 @@ mod tests { use std::time::Duration; use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _}; + #[test] + fn auction_http_capabilities_are_explicit() { + let client = AxumPlatformHttpClient::new(); + let capabilities = trusted_server_core::platform::AuctionTargetId::Axum + .descriptor() + .capabilities(); + assert!(client.supports_concurrent_fanout()); + assert!(capabilities.supports_concurrent_provider_fanout()); + assert!(!client.has_enforceable_total_request_deadline()); + assert!( + !capabilities.has_enforceable_total_request_deadline(), + "reqwest's transport timeout is not an adapter-enforced auction deadline" + ); + } + #[test] fn config_store_reads_from_env_var() { temp_env::with_var( @@ -693,6 +700,33 @@ mod tests { assert!(with_ip.is_none(), "should return None for any IP"); } + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn http_client_surfaces_redirect_without_following() { + let url = serve_raw_response( + b"HTTP/1.1 302 Found\r\nLocation: https://redirect.example/next\r\nContent-Length: 0\r\n\r\n", + ) + .await; + let request = edgezero_core::http::request_builder() + .uri(url) + .body(EdgeBody::empty()) + .expect("should build outbound request"); + + let response = AxumPlatformHttpClient::new() + .send(PlatformHttpRequest::new(request, "test_backend")) + .await + .expect("should surface redirect") + .response; + + assert_eq!(response.status().as_u16(), 302); + assert_eq!( + response + .headers() + .get(edgezero_core::http::header::LOCATION) + .and_then(|value| value.to_str().ok()), + Some("https://redirect.example/next") + ); + } + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn http_client_strips_hop_by_hop_response_headers() { let url = serve_raw_response( diff --git a/crates/trusted-server-adapter-axum/src/timing.rs b/crates/trusted-server-adapter-axum/src/timing.rs new file mode 100644 index 000000000..832823c15 --- /dev/null +++ b/crates/trusted-server-adapter-axum/src/timing.rs @@ -0,0 +1,242 @@ +//! Terminal timing layer for the Axum dev server. +//! +//! [`TimingService`](crate::timing::TimingService) wraps the tower `Service` +//! boundary the Axum dev server's router sits behind: it creates a +//! [`RequestTimings`](trusted_server_core::request_timing::RequestTimings) +//! collector per request, threads it through request extensions so +//! downstream core handlers can record into it, and on the way back stamps +//! `mark_headers_ready` and appends the `Server-Timing` header via +//! [`append_server_timing_if_private`](trusted_server_core::request_timing::append_server_timing_if_private). +//! +//! This wraps *outside* `RouterService` rather than registering as +//! `RouterBuilder::middleware`. A router-generated 404/405 short-circuits +//! `RouterInner::dispatch` before its middleware chain ever runs, so +//! middleware never sees those responses. By the time a response reaches +//! this layer -- after `RouterService::oneshot` inside +//! `EdgeZeroAxumService::call` has already converted any dispatch error into +//! a plain response -- every response is covered uniformly, router-generated +//! or not. +//! +//! `/health` is excluded by path match before a +//! [`RequestTimings`](trusted_server_core::request_timing::RequestTimings) +//! collector is even created: health checks never carry timing data on any +//! adapter. +//! +//! Unlike the Fastly adapter (state built per request, adding +//! `Phase::AppBuild` to the rendered header), the Axum dev server builds its +//! application state once at startup. There is no per-request app-build +//! interval to measure, so `ts-appbuild` never appears in the header here. + +use std::convert::Infallible; +use std::future::Future; +use std::pin::Pin; +use std::task::{Context, Poll}; + +use axum::body::Body as AxumBody; +use axum::http::{Request, Response}; +use tower::Service; +use trusted_server_core::request_timing::{RequestTimings, append_server_timing_if_private}; + +/// Path excluded from timing collection and `Server-Timing` emission: health +/// checks never carry timing data on any adapter. +const HEALTH_PATH: &str = "/health"; + +/// Wraps an inner Axum tower service with the request-phase timing freeze +/// point described in the module docs. +#[derive(Clone)] +pub struct TimingService { + inner: S, + server_timing_enabled: bool, +} + +impl TimingService { + /// Wraps `inner`, appending `Server-Timing` when `server_timing_enabled` + /// is set and the response is conclusively private. + #[must_use] + pub fn new(inner: S, server_timing_enabled: bool) -> Self { + Self { + inner, + server_timing_enabled, + } + } +} + +impl Service> for TimingService +where + S: Service, Response = Response, Error = Infallible> + + Clone + + Send + + 'static, + S::Future: Send + 'static, +{ + type Error = Infallible; + type Future = Pin> + Send>>; + type Response = Response; + + fn call(&mut self, mut req: Request) -> Self::Future { + let mut inner = self.inner.clone(); + + // Excluded before a collector is even created: `/health` never + // carries timing data, on any adapter. + if req.uri().path() == HEALTH_PATH { + return Box::pin(async move { inner.call(req).await }); + } + + let server_timing_enabled = self.server_timing_enabled; + let timings = RequestTimings::new(); + req.extensions_mut().insert(timings.clone()); + + Box::pin(async move { + let mut response = inner.call(req).await?; + append_server_timing_if_private(&mut response, &timings, server_timing_enabled); + Ok(response) + }) + } + + fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll> { + self.inner.poll_ready(cx) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use axum::http::header::CACHE_CONTROL; + use axum::http::{HeaderValue, StatusCode}; + use edgezero_adapter_axum::service::EdgeZeroAxumService; + use edgezero_core::body::Body as EdgeBody; + use edgezero_core::context::RequestContext; + use edgezero_core::error::EdgeError; + use edgezero_core::http::response_builder; + use edgezero_core::router::RouterService; + use tower::{ServiceExt as _, service_fn}; + + /// Builds a private (`cache-control: private, no-store`) response for a + /// handler under test. + fn private_ok_response() -> Result { + Ok(response_builder() + .status(StatusCode::OK) + .header("cache-control", "private, no-store") + .body(EdgeBody::from("ok")) + .expect("should build a private response fixture")) + } + + /// Reads a response header as a UTF-8 string, or `None` if absent. + fn header(response: &Response, name: &str) -> Option { + response + .headers() + .get(name) + .and_then(|value| value.to_str().ok()) + .map(ToOwned::to_owned) + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn axum_emits_header_on_private_response() { + let router = RouterService::builder() + .get("/private", |_ctx: RequestContext| async { + private_ok_response() + }) + .build(); + let mut service = TimingService::new(EdgeZeroAxumService::new(router), true); + + let request = Request::builder() + .uri("/private") + .body(AxumBody::empty()) + .expect("should build request"); + let response = service + .ready() + .await + .expect("should be ready") + .call(request) + .await + .expect("should not fail"); + + let server_timing = header(&response, "server-timing").expect("should emit header"); + assert!( + server_timing.contains("ts-total;dur="), + "should carry the collected total: {server_timing}" + ); + assert!( + !server_timing.contains("ts-appbuild"), + "the Axum dev server builds state once at startup, so there is no \ + per-request app-build interval to render: {server_timing}" + ); + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn axum_404_carries_header_when_private() { + // An empty router has no routes at all, so any path dispatches + // through `RouterInner::dispatch`'s `NotFound` branch -- exactly the + // path that bypasses `RouterBuilder::middleware`. The router's own + // `EdgeError::into_response` does not attach `Cache-Control`, so a + // small wrapping service forces the response private here, standing + // in for whatever upstream layer would normally mark a genuinely + // private 404. This proves the freeze point still runs for a + // router-generated response without weakening + // `append_server_timing_if_private`'s real gating logic. + let empty_router = RouterService::builder().build(); + let inner = EdgeZeroAxumService::new(empty_router); + let force_private = service_fn(move |req: Request| { + let mut svc = inner.clone(); + async move { + let mut response = svc.call(req).await?; + response + .headers_mut() + .insert(CACHE_CONTROL, HeaderValue::from_static("private, no-store")); + Ok::<_, Infallible>(response) + } + }); + let mut service = TimingService::new(force_private, true); + + let request = Request::builder() + .uri("/does-not-exist") + .body(AxumBody::empty()) + .expect("should build request"); + let response = service + .ready() + .await + .expect("should be ready") + .call(request) + .await + .expect("should not fail"); + + assert_eq!( + response.status(), + StatusCode::NOT_FOUND, + "should still be the router's own not-found response" + ); + let server_timing = header(&response, "server-timing") + .expect("a router-generated 404 must still carry the header when private"); + assert!( + server_timing.contains("ts-total;dur="), + "should carry the collected total: {server_timing}" + ); + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn axum_health_is_excluded() { + let router = RouterService::builder() + .get("/health", |_ctx: RequestContext| async { + private_ok_response() + }) + .build(); + let mut service = TimingService::new(EdgeZeroAxumService::new(router), true); + + let request = Request::builder() + .uri("/health") + .body(AxumBody::empty()) + .expect("should build request"); + let response = service + .ready() + .await + .expect("should be ready") + .call(request) + .await + .expect("should not fail"); + + assert!( + header(&response, "server-timing").is_none(), + "/health must never carry a server-timing header" + ); + } +} diff --git a/crates/trusted-server-adapter-axum/tests/routes.rs b/crates/trusted-server-adapter-axum/tests/routes.rs index ed199e6bf..6812b7421 100644 --- a/crates/trusted-server-adapter-axum/tests/routes.rs +++ b/crates/trusted-server-adapter-axum/tests/routes.rs @@ -18,8 +18,8 @@ const LEGACY_ADMIN_DENY_METHODS: &[&str] = /// The settings baked into the binary contain placeholder secrets that /// `get_settings()` rejects by design, which would turn every route into a /// startup error page (and its route table into the fallback-only set). -fn test_router() -> edgezero_core::router::RouterService { - let settings = trusted_server_core::settings::Settings::from_toml( +fn test_settings() -> trusted_server_core::settings::Settings { + trusted_server_core::settings::Settings::from_toml( r#" [[handlers]] path = "^/_ts/admin" @@ -36,9 +36,11 @@ fn test_router() -> edgezero_core::router::RouterService { passphrase = "test-secret-key-32-bytes-minimum" "#, ) - .expect("should parse route test settings"); + .expect("should parse route test settings") +} - TrustedServerApp::routes_with_settings(settings) +fn test_router() -> edgezero_core::router::RouterService { + TrustedServerApp::routes_with_settings(test_settings()) .expect("should build router from test settings") } @@ -62,6 +64,42 @@ fn assert_route_registered(method: &str, path: &str) { ); } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn aps_profile_serves_renderer_through_adapter_fallback() { + let mut settings = test_settings(); + settings.auction.providers.insert( + "aps-main".parse().expect("should parse APS provider ID"), + trusted_server_core::auction::ProviderConfig { + protocol: "openrtb-2.6".to_string(), + profile: "aps".to_string(), + endpoint: "https://aps.example/e/pb/bid".to_string(), + timeout_ms: None, + routing: trusted_server_core::auction::RoutingMode::AllEligible, + notifications: trusted_server_core::auction::NotificationConfig::default(), + profile_config: "{\"account_id\":\"example-account\"}" + .parse() + .expect("should parse APS profile config"), + }, + ); + let router = TrustedServerApp::routes_with_settings(settings) + .expect("should build router with APS profile"); + let mut service = EdgeZeroAxumService::new(router); + let request = Request::builder() + .method("GET") + .uri("/integrations/aps/renderer") + .body(AxumBody::empty()) + .expect("should build APS renderer request"); + + let response = service + .ready() + .await + .expect("should be ready") + .call(request) + .await + .expect("should serve APS renderer"); + assert_eq!(response.status().as_u16(), 200); +} + /// Verify that every expected explicit route is registered in the route table. /// /// Uses [`RouterService::routes()`] for introspection rather than checking diff --git a/crates/trusted-server-adapter-cloudflare/src/app.rs b/crates/trusted-server-adapter-cloudflare/src/app.rs index 6ce0a5ee3..7ea582ee8 100644 --- a/crates/trusted-server-adapter-cloudflare/src/app.rs +++ b/crates/trusted-server-adapter-cloudflare/src/app.rs @@ -9,10 +9,12 @@ use edgezero_core::http::{HeaderValue, Method, Request, Response, StatusCode, he use edgezero_core::router::RouterService; use error_stack::Report; use trusted_server_core::auction::endpoints::handle_auction; -use trusted_server_core::auction::{AuctionOrchestrator, build_orchestrator}; +use trusted_server_core::auction::{ + AuctionOrchestrator, build_orchestrator_with_plan, compile_auction_plan, +}; use trusted_server_core::cache_policy::EdgeCacheHeader; #[cfg(target_arch = "wasm32")] -use trusted_server_core::config_payload::settings_from_config_blob; +use trusted_server_core::config_payload::{DEFAULT_SECRET_STORE_ID, settings_from_config_blob}; use trusted_server_core::ec::EcContext; use trusted_server_core::ec::admin::{ admin_ec_lookup_not_supported as core_admin_ec_lookup_not_supported, @@ -22,6 +24,8 @@ use trusted_server_core::ec::registry::PartnerRegistry; use trusted_server_core::error::{IntoHttpResponse as _, TrustedServerError}; use trusted_server_core::integrations::{IntegrationRegistry, ProxyDispatchInput}; use trusted_server_core::platform::RuntimeServices; +#[cfg(target_arch = "wasm32")] +use trusted_server_core::platform::StoreName; use trusted_server_core::proxy::{ handle_first_party_click, handle_first_party_proxy, handle_first_party_proxy_rebuild, handle_first_party_proxy_sign, @@ -44,11 +48,23 @@ use crate::platform::build_runtime_services; // --------------------------------------------------------------------------- #[cfg(target_arch = "wasm32")] -static CLOUDFLARE_CONFIG_JSON: std::sync::OnceLock = std::sync::OnceLock::new(); +thread_local! { + static CLOUDFLARE_CONFIG_JSON: std::cell::OnceCell = const { std::cell::OnceCell::new() }; + static CLOUDFLARE_ENV: std::cell::OnceCell = const { std::cell::OnceCell::new() }; +} #[cfg(target_arch = "wasm32")] pub fn set_cloudflare_config_json(value: String) { - let _ = CLOUDFLARE_CONFIG_JSON.set(value); + CLOUDFLARE_CONFIG_JSON.with(|slot| { + let _ = slot.set(value); + }); +} + +#[cfg(target_arch = "wasm32")] +pub fn set_cloudflare_env(env: worker::Env) { + CLOUDFLARE_ENV.with(|slot| { + let _ = slot.set(env); + }); } /// Application state built once at startup and shared across all requests. @@ -76,18 +92,22 @@ fn load_startup_settings() -> Result> { #[cfg(not(target_arch = "wasm32"))] fn load_startup_settings() -> Result> { - Settings::from_toml(include_str!("../../../trusted-server.example.toml")) + Err(Report::new(TrustedServerError::Configuration { + message: "Cloudflare startup settings require a Worker config binding".to_string(), + }) + .attach("use TrustedServerApp::routes_with_settings for host tests")) } #[cfg(target_arch = "wasm32")] fn settings_from_cloudflare_config_json() -> Result> { - let raw_config = CLOUDFLARE_CONFIG_JSON.get().ok_or_else(|| { + let raw_config = CLOUDFLARE_CONFIG_JSON.with(|slot| slot.get().cloned()); + let raw_config = raw_config.ok_or_else(|| { Report::new(TrustedServerError::Configuration { message: "Cloudflare TRUSTED_SERVER_CONFIG is required".to_string(), }) .attach("set TRUSTED_SERVER_CONFIG to JSON containing the app_config blob envelope") })?; - let value: serde_json::Value = serde_json::from_str(raw_config).map_err(|error| { + let value: serde_json::Value = serde_json::from_str(&raw_config).map_err(|error| { Report::new(TrustedServerError::Configuration { message: "invalid Cloudflare TRUSTED_SERVER_CONFIG JSON".to_string(), }) @@ -101,7 +121,16 @@ fn settings_from_cloudflare_config_json() -> Result Result Result, Report> { - let orchestrator = build_orchestrator(&settings)?; - let registry = IntegrationRegistry::new(&settings)?; + let plan = Arc::new(compile_auction_plan(&settings)?); + plan.validate_for_target(trusted_server_core::platform::AuctionTargetId::Cloudflare)?; + let orchestrator = build_orchestrator_with_plan(Arc::clone(&plan), &settings)?; + let registry = IntegrationRegistry::with_plan(&settings, plan)?; Ok(Arc::new(AppState { settings: Arc::new(settings), @@ -523,13 +554,13 @@ fn build_router(state: &Arc) -> RouterService { // Build the geo-aware EC context so the auction consent gate // sees the caller's jurisdiction — `EcContext::default()` // fails it closed for consented users. - let ec_context = build_ec_context(&s.settings, &services, &req); + let mut ec_context = build_ec_context(&s.settings, &services, &req); handle_auction( &s.settings, &s.orchestrator, None, None, - &ec_context, + &mut ec_context, &services, req, ) @@ -587,13 +618,13 @@ fn build_router(state: &Arc) -> RouterService { // preflight fall through to a permissive origin would reopen exactly // the cross-site hole the canonical path closes. let page_bids = make_handler(Arc::clone(&state), |s, services, req| async move { - let ec_context = build_ec_context(&s.settings, &services, &req); + let mut ec_context = build_ec_context(&s.settings, &services, &req); let auction = AuctionDispatch { orchestrator: &s.orchestrator, slots: s.settings.creative_opportunity_slots(), registry: None, }; - handle_page_bids(&s.settings, &services, None, auction, &ec_context, req).await + handle_page_bids(&s.settings, &services, None, auction, &mut ec_context, req).await }); let page_bids_preflight = make_handler(Arc::clone(&state), |_s, _services, _req| async move { @@ -625,3 +656,149 @@ fn build_router(state: &Arc) -> RouterService { router.build() } } + +#[cfg(test)] +mod tests { + use super::*; + + fn aps_profile_settings() -> Settings { + let mut settings = Settings::from_toml( + r#" + [[handlers]] + path = "^/_ts/admin" + username = "admin" + password = "admin-password" + + [publisher] + domain = "publisher.example" + cookie_domain = ".publisher.example" + origin_url = "https://origin.publisher.example" + proxy_secret = "fictional-proxy-secret" + + [ec] + passphrase = "fictional-secret-key-32-bytes-minimum" + "#, + ) + .expect("should parse startup test settings"); + settings.auction.providers.insert( + "aps-main".parse().expect("should parse APS provider ID"), + trusted_server_core::auction::ProviderConfig { + protocol: "openrtb-2.6".to_string(), + profile: "aps".to_string(), + endpoint: "https://aps.example/e/pb/bid".to_string(), + timeout_ms: None, + routing: trusted_server_core::auction::RoutingMode::AllEligible, + notifications: trusted_server_core::auction::NotificationConfig::default(), + profile_config: serde_json::json!({"account_id":"example-account"}), + }, + ); + settings + } + + #[test] + fn startup_registers_aps_renderer_route() { + let state = build_state_with_settings(aps_profile_settings()) + .expect("Cloudflare startup should register APS renderer"); + assert!( + state.registry.has_route( + &edgezero_core::http::Method::GET, + "/integrations/aps/renderer" + ), + "Cloudflare startup registry should expose the APS renderer" + ); + } + + #[test] + fn disabled_startup_accepts_dormant_multi_provider_auction_plan() { + let mut settings = Settings::from_toml( + r#" + [[handlers]] + path = "^/_ts/admin" + username = "admin" + password = "admin-password" + + [publisher] + domain = "publisher.example" + cookie_domain = ".publisher.example" + origin_url = "https://origin.publisher.example" + proxy_secret = "fictional-proxy-secret" + + [ec] + passphrase = "fictional-secret-key-32-bytes-minimum" + "#, + ) + .expect("should parse startup test settings"); + settings.auction.enabled = false; + settings.auction.providers = + std::iter::IntoIterator::into_iter(["provider-a", "provider-b"]) + .map(|id| { + ( + id.parse().expect("should parse provider ID"), + trusted_server_core::auction::ProviderConfig { + protocol: "openrtb-2.6".to_string(), + profile: "standard".to_string(), + endpoint: format!("https://{id}.example/openrtb"), + timeout_ms: None, + routing: trusted_server_core::auction::RoutingMode::AllEligible, + notifications: + trusted_server_core::auction::NotificationConfig::default(), + profile_config: serde_json::json!({}), + }, + ) + }) + .collect(); + + build_state_with_settings(settings) + .expect("disabled Cloudflare auction should accept dormant fanout"); + } + + #[test] + fn startup_rejects_multi_provider_auction_plan() { + let mut settings = Settings::from_toml( + r#" + [[handlers]] + path = "^/_ts/admin" + username = "admin" + password = "admin-password" + + [publisher] + domain = "publisher.example" + cookie_domain = ".publisher.example" + origin_url = "https://origin.publisher.example" + proxy_secret = "fictional-proxy-secret" + + [ec] + passphrase = "fictional-secret-key-32-bytes-minimum" + "#, + ) + .expect("should parse startup test settings"); + settings.auction.enabled = true; + settings.auction.providers = + std::iter::IntoIterator::into_iter(["provider-a", "provider-b"]) + .map(|id| { + ( + id.parse().expect("should parse provider ID"), + trusted_server_core::auction::ProviderConfig { + protocol: "openrtb-2.6".to_string(), + profile: "standard".to_string(), + endpoint: format!("https://{id}.example/openrtb"), + timeout_ms: None, + routing: trusted_server_core::auction::RoutingMode::AllEligible, + notifications: + trusted_server_core::auction::NotificationConfig::default(), + profile_config: serde_json::json!({}), + }, + ) + }) + .collect(); + + let error = match build_state_with_settings(settings) { + Ok(_) => panic!("Cloudflare startup should reject multi-provider fanout"), + Err(error) => error, + }; + assert!( + format!("{error:?}").contains("concurrent provider fanout"), + "should identify unsupported fanout: {error:?}" + ); + } +} diff --git a/crates/trusted-server-adapter-cloudflare/src/lib.rs b/crates/trusted-server-adapter-cloudflare/src/lib.rs index 2ce435b17..980a9b209 100644 --- a/crates/trusted-server-adapter-cloudflare/src/lib.rs +++ b/crates/trusted-server-adapter-cloudflare/src/lib.rs @@ -27,6 +27,7 @@ pub async fn main(req: Request, env: Env, ctx: Context) -> Result { if let Ok(config) = env.var("TRUSTED_SERVER_CONFIG") { app::set_cloudflare_config_json(config.to_string()); } + app::set_cloudflare_env(env.clone()); match edgezero_adapter_cloudflare::run_app::(req, env, ctx).await { Ok(resp) => Ok(resp), diff --git a/crates/trusted-server-adapter-cloudflare/src/platform.rs b/crates/trusted-server-adapter-cloudflare/src/platform.rs index fff0bfed1..cded42a0d 100644 --- a/crates/trusted-server-adapter-cloudflare/src/platform.rs +++ b/crates/trusted-server-adapter-cloudflare/src/platform.rs @@ -5,18 +5,16 @@ use std::time::Duration; use bytes::Bytes; use edgezero_core::config_store::ConfigStoreHandle; use edgezero_core::key_value_store::{KvHandle, KvPage, KvStore}; -use error_stack::Report; +use error_stack::{Report, ResultExt as _}; use trusted_server_core::platform::{ - ClientInfo, GeoInfo, KvError, PlatformBackend, PlatformBackendSpec, PlatformConfigStore, - PlatformError, PlatformGeo, PlatformHttpClient, PlatformKvStore, PlatformSecretStore, - RuntimeServices, StoreId, StoreName, UnavailableKvStore, + BackendNamingPolicy, ClientInfo, GeoInfo, KvError, PlatformBackend, PlatformBackendSpec, + PlatformConfigStore, PlatformError, PlatformGeo, PlatformHttpClient, PlatformKvStore, + PlatformSecretStore, RuntimeServices, StoreId, StoreName, UnavailableKvStore, }; #[cfg(not(target_arch = "wasm32"))] use trusted_server_core::platform::UnavailableHttpClient; -#[cfg(target_arch = "wasm32")] -use error_stack::ResultExt as _; #[cfg(target_arch = "wasm32")] use trusted_server_core::platform::{ PlatformHttpRequest, PlatformPendingRequest, PlatformResponse, PlatformSelectResult, @@ -61,27 +59,15 @@ impl PlatformSecretStore for NoopSecretStore { struct NoopBackend; impl PlatformBackend for NoopBackend { + fn naming_policy(&self) -> BackendNamingPolicy { + BackendNamingPolicy::Cloudflare + } + fn predict_name(&self, spec: &PlatformBackendSpec) -> Result> { - let port = spec - .port - .unwrap_or(if spec.scheme == "https" { 443 } else { 80 }); - let timeout_ms = spec.first_byte_timeout.as_millis(); - let cert_suffix = if spec.certificate_check { - "" - } else { - "_nocert" - }; - // Keep two providers that share an origin on distinct names so auction - // response correlation cannot cross providers. - let discriminator = spec - .discriminator - .as_deref() - .map(|d| format!("_p_{d}")) - .unwrap_or_default(); - Ok(format!( - "{}_{}_{}_{timeout_ms}ms{cert_suffix}{discriminator}", - spec.scheme, spec.host, port - )) + self.naming_policy() + .predict(spec) + .map(|prediction| prediction.name) + .change_context(PlatformError::Backend) } fn ensure(&self, spec: &PlatformBackendSpec) -> Result> { @@ -284,13 +270,22 @@ fn outbound_cache_mode(bypass_cache: bool) -> OutboundCacheMode { } } +#[cfg(target_arch = "wasm32")] +fn outbound_request_init(method: worker::Method, headers: worker::Headers) -> worker::RequestInit { + let mut init = worker::RequestInit::new(); + init.with_method(method) + .with_headers(headers) + .with_redirect(worker::RequestRedirect::Manual); + init +} + #[cfg(target_arch = "wasm32")] impl CloudflareHttpClient { async fn execute( &self, request: PlatformHttpRequest, ) -> Result> { - use worker::{CacheMode, Fetch, Headers, Method, Request, RequestInit, RequestRedirect}; + use worker::{CacheMode, Fetch, Headers, Method, Request}; // The Cloudflare fetch path cannot honor Fastly-style Image Optimizer // metadata, and it always buffers the response body (see below). The @@ -340,7 +335,6 @@ impl CloudflareHttpClient { } }; - let mut init = RequestInit::new(); // Force manual redirect handling: the Workers runtime otherwise defaults // to `RequestRedirect::Follow` and transparently chases 3xx responses to // any host inside `Fetch::send()`. Core's `proxy_with_redirects` does its @@ -348,9 +342,7 @@ impl CloudflareHttpClient { // `allowed_domains`; auto-following here would bypass that allowlist // (SSRF). `Manual` surfaces the 3xx + Location back to core unfollowed, // matching the Axum adapter's `redirect::Policy::none()`. - init.with_method(method) - .with_headers(headers) - .with_redirect(RequestRedirect::Manual); + let mut init = outbound_request_init(method, headers); // Setting the `cache` field requires the `cache_option_enabled` // compatibility flag, which is only on by default from compatibility // date 2024-11-11. `wrangler.toml`/`wrangler.ci.toml` pin an earlier @@ -547,8 +539,8 @@ impl PlatformHttpClient for CloudflareHttpClient { /// Bridges [`worker::Env`] secrets to [`PlatformSecretStore`] by calling /// `env.secret(key)` synchronously. Writes and deletes return errors. #[cfg(target_arch = "wasm32")] -struct CloudflareSecretStoreAdapter { - env: worker::Env, +pub(crate) struct CloudflareSecretStoreAdapter { + pub(crate) env: worker::Env, } #[cfg(target_arch = "wasm32")] @@ -762,6 +754,25 @@ fn reject_multi_provider_fanout(len: usize) -> Result<(), Report> mod tests { use super::*; use edgezero_core::context::RequestContext; + + #[cfg(target_arch = "wasm32")] + #[test] + fn outbound_request_creation_sets_manual_redirect_mode() { + let init = outbound_request_init(worker::Method::Get, worker::Headers::new()); + assert!(matches!(init.redirect, worker::RequestRedirect::Manual)); + } + + #[test] + fn auction_http_capabilities_are_explicit() { + let capabilities = trusted_server_core::platform::AuctionTargetId::Cloudflare + .descriptor() + .capabilities(); + assert!(!capabilities.supports_concurrent_provider_fanout()); + assert!( + !capabilities.has_enforceable_total_request_deadline(), + "Workers fetch does not expose an enforceable hard total request deadline" + ); + } use edgezero_core::http::{HeaderValue, request_builder}; use edgezero_core::params::PathParams; diff --git a/crates/trusted-server-adapter-cloudflare/wrangler.ci.toml b/crates/trusted-server-adapter-cloudflare/wrangler.ci.toml index e6891eb79..9992db712 100644 --- a/crates/trusted-server-adapter-cloudflare/wrangler.ci.toml +++ b/crates/trusted-server-adapter-cloudflare/wrangler.ci.toml @@ -14,3 +14,12 @@ id = "ci-local-kv" # Placeholder replaced by the integration test harness with a JSON object that # contains the runtime Trusted Server app-config blob envelope. TRUSTED_SERVER_CONFIG = "{}" + +# Fictitious integration-only secret values. `worker::Env::secret` reads these +# string bindings in local Wrangler runs; production values are provisioned with +# `wrangler secret put` instead of being committed to a manifest. +integration_admin_password = "integration-admin-password-32-bytes-ok" +integration_proxy_secret = "integration-test-proxy-secret-32-bytes-ok" +integration_ec_passphrase = "integration-test-ec-secret-padded-32" +integration_partner_token_alpha = "integration-test-token-alpha-32-bytes-ok" +integration_partner_token_bravo = "integration-test-token-bravo-32-bytes-ok" diff --git a/crates/trusted-server-adapter-cloudflare/wrangler.toml b/crates/trusted-server-adapter-cloudflare/wrangler.toml index 7c91173fc..48eb2db8d 100644 --- a/crates/trusted-server-adapter-cloudflare/wrangler.toml +++ b/crates/trusted-server-adapter-cloudflare/wrangler.toml @@ -26,3 +26,7 @@ id = "REPLACE_WITH_YOUR_KV_NAMESPACE_ID" # invalid placeholder with JSON containing an `app_config` blob envelope before # deploying or running `wrangler dev` against real traffic. TRUSTED_SERVER_CONFIG = '{"app_config":""}' + +# App-config secret values are provisioned as Worker secrets with +# `wrangler secret put `. The pushed blob contains only those key +# names; never add secret values to this file. diff --git a/crates/trusted-server-adapter-fastly/Cargo.toml b/crates/trusted-server-adapter-fastly/Cargo.toml index 47cc609b2..584085e76 100644 --- a/crates/trusted-server-adapter-fastly/Cargo.toml +++ b/crates/trusted-server-adapter-fastly/Cargo.toml @@ -27,12 +27,13 @@ log = { workspace = true } log-fastly = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } -sha2 = { workspace = true } trusted-server-core = { workspace = true } +rand = { workspace = true } url = { workspace = true } urlencoding = { workspace = true } [dev-dependencies] bytes = { workspace = true } edgezero-core = { workspace = true, features = ["test-utils"] } +toml = { workspace = true } trusted-server-core = { workspace = true, features = ["test-utils"] } diff --git a/crates/trusted-server-adapter-fastly/src/app.rs b/crates/trusted-server-adapter-fastly/src/app.rs index 4ba61f603..f3746f6f4 100644 --- a/crates/trusted-server-adapter-fastly/src/app.rs +++ b/crates/trusted-server-adapter-fastly/src/app.rs @@ -99,10 +99,14 @@ use edgezero_core::http::{ }; use edgezero_core::router::RouterService; use error_stack::Report; +use trusted_server_core::access_telemetry::{RouteClass, RouteMetadata, publisher_route_template}; use trusted_server_core::auction::AuctionTelemetrySink; use trusted_server_core::auction::endpoints::handle_auction; -use trusted_server_core::auction::{AuctionOrchestrator, build_orchestrator}; +use trusted_server_core::auction::{ + AuctionOrchestrator, build_orchestrator_with_plan, compile_auction_plan, +}; use trusted_server_core::cache_policy::EdgeCacheHeader; +use trusted_server_core::config_payload::DEFAULT_SECRET_STORE_ID; use trusted_server_core::constants::{COOKIE_SHAREDID, COOKIE_TS_EIDS}; use trusted_server_core::ec::EcContext; use trusted_server_core::ec::admin::{ @@ -115,12 +119,15 @@ use trusted_server_core::ec::identify::{cors_preflight_identify, handle_identify use trusted_server_core::ec::kv::KvIdentityGraph; use trusted_server_core::ec::registry::PartnerRegistry; use trusted_server_core::error::{IntoHttpResponse as _, TrustedServerError}; +use trusted_server_core::geo::GeoLookupState; use trusted_server_core::http_util::is_navigation_request; use trusted_server_core::integrations::{ IntegrationRegistry, ProxyDispatchInput, RequestFilterEffects, RequestFilterRegistryInput, RequestFilterRegistryOutcome, }; -use trusted_server_core::platform::{ClientInfo, GeoInfo, PlatformKvStore, RuntimeServices}; +use trusted_server_core::platform::{ + ClientInfo, GeoInfo, PlatformKvStore, RuntimeServices, StoreName, TimedKvStore, +}; use trusted_server_core::proxy::{ AssetProxyCachePolicy, handle_asset_proxy_request, handle_first_party_click, handle_first_party_proxy, handle_first_party_proxy_rebuild, handle_first_party_proxy_sign, @@ -134,9 +141,10 @@ use trusted_server_core::request_signing::{ handle_deactivate_key, handle_rotate_key, handle_trusted_server_discovery, handle_verify_signature, }; +use trusted_server_core::request_timing::{Phase, RequestTimings}; use trusted_server_core::settings::{ProxyAssetRoute, Settings}; use trusted_server_core::settings_data::{ - config_key, config_store_name, get_settings_from_config_store, + DEFAULT_CONFIG_STORE_ID, config_key, config_store_name, get_settings_from_config_store, }; use trusted_server_core::tester_cookie::{handle_clear_tester, handle_set_tester}; @@ -150,6 +158,23 @@ use crate::platform::{ // AppState // --------------------------------------------------------------------------- +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct RuntimeStoreConfig { + pub(crate) config_store_name: StoreName, + pub(crate) config_key: String, + pub(crate) secret_store_name: StoreName, +} + +impl RuntimeStoreConfig { + pub(crate) fn from_env(env: &EnvConfig) -> Self { + Self { + config_store_name: config_store_name(env), + config_key: config_key(env), + secret_store_name: StoreName::from(env.store_name("secrets", DEFAULT_SECRET_STORE_ID)), + } + } +} + /// Application state built once per Wasm instance and shared for its lifetime. /// /// In Fastly Compute each request spawns a new Wasm instance, so this struct is @@ -168,16 +193,22 @@ pub(crate) struct AppState { /// /// Returns an error when settings, the auction orchestrator, or the integration /// registry fail to initialise. -pub(crate) fn build_state(env: &EnvConfig) -> Result, Report> { - build_state_from_settings(load_settings_from_config_store(env)?) +pub(crate) fn build_state( + stores: &RuntimeStoreConfig, +) -> Result, Report> { + build_state_from_settings(load_settings_from_config_store(stores)?) } pub(crate) fn load_settings_from_config_store( - env: &EnvConfig, + stores: &RuntimeStoreConfig, ) -> Result> { - let store_name = config_store_name(env); - let key = config_key(env); - get_settings_from_config_store(&FastlyPlatformConfigStore, &store_name, &key) + get_settings_from_config_store( + &FastlyPlatformConfigStore, + &FastlyPlatformSecretStore, + &stores.config_store_name, + &stores.config_key, + &stores.secret_store_name, + ) } pub(crate) fn build_state_from_settings( @@ -185,8 +216,10 @@ pub(crate) fn build_state_from_settings( ) -> Result, Report> { warn_if_certificate_check_disabled(&settings); - let orchestrator = build_orchestrator(&settings)?; - let registry = IntegrationRegistry::new(&settings)?; + let plan = Arc::new(compile_auction_plan(&settings)?); + plan.validate_for_target(trusted_server_core::platform::AuctionTargetId::Fastly)?; + let orchestrator = build_orchestrator_with_plan(Arc::clone(&plan), &settings)?; + let registry = IntegrationRegistry::with_plan(&settings, plan)?; let auction_telemetry_sink = crate::tinybird::auction_sink_from_settings(&settings); let default_kv_store = Arc::new(UnavailableKvStore) as Arc; @@ -223,13 +256,18 @@ fn warn_if_certificate_check_disabled(settings: &Settings) { pub(crate) fn runtime_services_for_consent_route( settings: &Settings, runtime_services: &RuntimeServices, + timings: &RequestTimings, ) -> Result> { let Some(store_name) = settings.consent.consent_store.as_deref() else { return Ok(runtime_services.clone()); }; open_kv_store(store_name) - .map(|store| runtime_services.clone().with_kv_store(store)) + .map(|store| { + let timed_store = + Arc::new(TimedKvStore::new(store, timings.clone())) as Arc; + runtime_services.clone().with_kv_store(timed_store) + }) .map_err(|e| { Report::new(TrustedServerError::KvStore { store_name: store_name.to_string(), @@ -297,6 +335,12 @@ fn uses_dynamic_tsjs_fallback(method: &Method, path: &str) -> bool { *method == Method::GET && path.starts_with("/static/tsjs=") } +/// Coarse route template for every `tsjs` bundle request, used as the +/// `route_template` in the [`RouteMetadata`] attached by the tsjs branch of +/// [`dispatch_fallback`]. Actual filenames vary by module/hash; the prefix +/// alone is the route identity that matters for access telemetry. +const TSJS_ROUTE_TEMPLATE: &str = "/static/tsjs=*"; + // --------------------------------------------------------------------------- // EC request state // --------------------------------------------------------------------------- @@ -358,6 +402,17 @@ impl EcRequestState { services: self.services, } } + + /// Derives the carried [`GeoLookupState`] from this request's geo lookup + /// outcome, so response-phase finalize can reuse it instead of repeating + /// the lookup. `build_ec_request_state` always attempts the lookup, so + /// `None` here means the lookup ran and failed, not that it was skipped. + fn geo_lookup_state(&self) -> GeoLookupState { + match &self.geo_info { + Some(info) => GeoLookupState::Resolved(info.clone()), + None => GeoLookupState::Attempted, + } + } } /// Derives device signals from the request's `User-Agent` header. @@ -413,18 +468,33 @@ fn build_ec_request_state( let eids_cookie = crate::extract_cookie_value(req, COOKIE_TS_EIDS); let sharedid_cookie = crate::extract_cookie_value(req, COOKIE_SHAREDID); - let geo_info = services - .geo() - .lookup(services.client_info().client_ip) - .unwrap_or_else(|e| { - log::warn!("geo lookup failed during EC setup: {e}"); - None - }); + let timings = req + .extensions() + .get::() + .cloned() + .unwrap_or_default(); + let geo_info = { + let _span = timings.span(Phase::Geo); + services + .geo() + .lookup(services.client_info().client_ip) + .unwrap_or_else(|e| { + log::warn!("geo lookup failed during EC setup: {e}"); + None + }) + }; let (ec_context, setup_error) = match EcContext::read_from_request_with_geo(settings, req, services, geo_info.as_ref()) { Ok(mut context) => { context.set_device_signals(device_signals); + // Orphan-recovery eligibility is intentionally left false here. + // Authorizing it during generic pre-routing would let named + // routes and request-filter short circuits (e.g. a DataDome + // challenge) reach EC finalization and rotate an identity off a + // non-publisher response. It is granted only inside the + // publisher fallback, after filters pass and the origin start + // succeeds — see `dispatch_fallback`. (context, None) } Err(report) => (EcContext::default(), Some(report)), @@ -433,7 +503,7 @@ fn build_ec_request_state( // Bot gate: suppress KV-backed EC writes for unrecognized clients, except // consent withdrawals. Revocations keep the write path so tombstones stay // authoritative even for privacy-extension-heavy clients. - let kv_graph = crate::maybe_identity_graph(settings); + let kv_graph = crate::identity_graph_with_timing(settings, &timings); let finalize_kv_graph = if setup_error.is_none() && (is_real_browser || ec_consent_withdrawn(ec_context.consent())) { @@ -485,6 +555,18 @@ async fn run_pre_route_filters( req: &mut Request, geo_info: Option<&GeoInfo>, ) -> PreRoute { + // Only recorded when a filter is actually registered, so unconfigured + // deployments omit ts-filter from the Server-Timing header entirely. + let timings = req + .extensions() + .get::() + .cloned() + .unwrap_or_default(); + let _span = state + .registry + .has_request_filters() + .then(|| timings.span(Phase::Filter)); + match state .registry .filter_request(RequestFilterRegistryInput { @@ -518,6 +600,7 @@ fn attach_dispatch_extensions( ec: EcRequestState, effects: RequestFilterEffects, ) -> Response { + response.extensions_mut().insert(ec.geo_lookup_state()); response.extensions_mut().insert(ec.into_finalize_state()); if !effects.response_headers.is_empty() { response.extensions_mut().insert(effects); @@ -554,7 +637,12 @@ async fn execute_named( // Deliberately do not use an EC request-state graph: that // copy is bot-gated, while operators use curl for this // authenticated diagnostic. - let kv = crate::maybe_identity_graph(&state.settings); + let timings = req + .extensions() + .get::() + .cloned() + .unwrap_or_default(); + let kv = crate::identity_graph_with_timing(&state.settings, &timings); handle_admin_ec_lookup(kv.as_ref(), ®istry, &req) } NamedRouteHandler::AdminEidsLookup => handle_admin_eids_lookup(®istry, &req), @@ -625,7 +713,12 @@ async fn run_named_route( if req.method() == Method::OPTIONS { cors_preflight_identify(&state.settings, &req) } else { - let kv = crate::require_identity_graph(&state.settings)?; + let timings = req + .extensions() + .get::() + .cloned() + .unwrap_or_default(); + let kv = crate::require_identity_graph_with_timing(&state.settings, &timings)?; let partner_registry = PartnerRegistry::from_config(&state.settings.ec.partners)?; handle_identify( &state.settings, @@ -642,7 +735,13 @@ async fn run_named_route( // The auction reads consent data, so the consent KV store must be // available — fail closed with 503 when it is configured but // cannot be opened, matching legacy behavior. - let consent_services = runtime_services_for_consent_route(&state.settings, services)?; + let timings = req + .extensions() + .get::() + .cloned() + .unwrap_or_default(); + let consent_services = + runtime_services_for_consent_route(&state.settings, services, &timings)?; let partner_registry = PartnerRegistry::from_config(&state.settings.ec.partners)?; let registry_ref = if partner_registry.is_empty() { None @@ -654,7 +753,7 @@ async fn run_named_route( &state.orchestrator, ec.kv_graph.as_ref(), registry_ref, - &ec.ec_context, + &mut ec.ec_context, &consent_services, req, ) @@ -670,7 +769,13 @@ async fn run_named_route( // Like the auction, page-bids reads consent data, so the consent KV // store must be available — fail closed with 503 when configured but // unopenable, matching legacy. - let consent_services = runtime_services_for_consent_route(&state.settings, services)?; + let timings = req + .extensions() + .get::() + .cloned() + .unwrap_or_default(); + let consent_services = + runtime_services_for_consent_route(&state.settings, services, &timings)?; let partner_registry = PartnerRegistry::from_config(&state.settings.ec.partners)?; let registry_ref = if partner_registry.is_empty() { None @@ -687,7 +792,7 @@ async fn run_named_route( &consent_services, ec.kv_graph.as_ref(), auction, - &ec.ec_context, + &mut ec.ec_context, req, ) .await @@ -715,12 +820,18 @@ fn run_batch_sync(state: &AppState, services: &RuntimeServices, req: Request) -> let is_real_browser = device_signals.looks_like_browser(); let eids_cookie = crate::extract_cookie_value(&req, COOKIE_TS_EIDS); let sharedid_cookie = crate::extract_cookie_value(&req, COOKIE_SHAREDID); + let timings = req + .extensions() + .get::() + .cloned() + .unwrap_or_default(); - let result = crate::require_identity_graph(&state.settings).and_then(|kv| { - let partner_registry = PartnerRegistry::from_config(&state.settings.ec.partners)?; - let limiter = FastlyRateLimiter::new(RATE_COUNTER_NAME); - handle_batch_sync(&kv, &partner_registry, &limiter, req) - }); + let result = + crate::require_identity_graph_with_timing(&state.settings, &timings).and_then(|kv| { + let partner_registry = PartnerRegistry::from_config(&state.settings.ec.partners)?; + let limiter = FastlyRateLimiter::new(RATE_COUNTER_NAME); + handle_batch_sync(&kv, &partner_registry, &limiter, req) + }); let mut response = result.unwrap_or_else(|e| http_error(&e)); // Legacy parity: batch-sync responses still pass through @@ -780,12 +891,28 @@ async fn dispatch_fallback( PreRoute::Continue { effects } => effects, }; + // Assigned exactly once, per branch below, alongside the routing + // decision itself, so the access-telemetry route identity always + // reflects which branch actually dispatched the request — including + // when that branch's handler errors. The asset-route sub-branch is an + // early return handled separately by `dispatch_asset_fallback`, so it + // never reaches (or needs to assign) this binding. + let route_metadata: Option; + let result = if uses_dynamic_tsjs_fallback(&method, &path) { + route_metadata = Some(RouteMetadata { + route_class: RouteClass::Tsjs, + route_template: TSJS_ROUTE_TEMPLATE.to_owned(), + }); handle_tsjs_dynamic(&req, &state.registry, EdgeCacheHeader::SurrogateControl) } else if state.registry.has_route(&method, &path) { // Integration-proxy responses are not bounded by // publisher.max_buffered_body_bytes. Publisher fallback below uses the // publisher-specific streaming finalizer instead. + route_metadata = Some(RouteMetadata { + route_class: RouteClass::IntegrationProxy, + route_template: publisher_route_template(&path), + }); state .registry .handle_proxy(ProxyDispatchInput { @@ -813,14 +940,36 @@ async fn dispatch_fallback( .then(|| state.settings.asset_route_for_path(&path)) .flatten(); if let Some(asset_route) = matched_asset_route { - return dispatch_asset_fallback(state, services, req, asset_route, &effects).await; + // The template is the operator-configured route prefix, so it + // is bounded and content-free by construction (unlike request + // paths, which need `publisher_route_template`). + let asset_metadata = RouteMetadata { + route_class: RouteClass::Asset, + route_template: format!("{}/*", asset_route.prefix.trim_end_matches('/')), + }; + let mut response = dispatch_asset_fallback( + state, + services, + req, + asset_route, + &effects, + ec.geo_lookup_state(), + ) + .await; + response.extensions_mut().insert(asset_metadata); + return response; } + route_metadata = Some(RouteMetadata { + route_class: RouteClass::PublisherHtml, + route_template: publisher_route_template(&path), + }); + // Generate an EC ID if needed — mirrors the legacy catch-all arm. // Only for document navigations by recognised browsers; subresource // requests may lack consent signals such as Sec-GPC. - if ec.is_real_browser - && is_navigation_request(&req) + let is_publisher_navigation = ec.is_real_browser && is_navigation_request(&req); + if is_publisher_navigation && let Err(err) = ec .ec_context .generate_if_needed(&state.settings, ec.kv_graph.as_ref()) @@ -831,7 +980,12 @@ async fn dispatch_fallback( // Publisher pages read consent data, so the consent KV store must be // available — fail closed with 503 when it is configured but cannot // be opened, matching legacy behavior. - match runtime_services_for_consent_route(&state.settings, services) { + let timings = req + .extensions() + .get::() + .cloned() + .unwrap_or_default(); + match runtime_services_for_consent_route(&state.settings, services, &timings) { Ok(publisher_services) => { // Run the server-side auction with the configured creative- // opportunity slots and collect dispatched bids from the lazy @@ -859,6 +1013,14 @@ async fn dispatch_fallback( .await { Ok(pub_response) => { + // Origin start succeeded on the sole publisher- + // page path: authorize orphan recovery now, and + // only for real-browser document navigations. + // Restricting it here keeps identity rotation + // within the publisher-navigation boundary — + // named routes, integration proxies, and filter + // short circuits never reach this point. + ec.ec_context.set_recovery_eligible(is_publisher_navigation); publisher_response_into_streaming_response( pub_response, &method, @@ -879,7 +1041,10 @@ async fn dispatch_fallback( } }; - let response = result.unwrap_or_else(|e| http_error(&e)); + let mut response = result.unwrap_or_else(|e| http_error(&e)); + if let Some(metadata) = route_metadata { + response.extensions_mut().insert(metadata); + } attach_dispatch_extensions(response, ec, effects) } @@ -903,7 +1068,10 @@ fn asset_response_carries_body(method: &Method, status: StatusCode) -> bool { /// [`AssetProxyCachePolicy`] out via response extensions so `edgezero_main` /// can reapply protected cache directives after finalization. EC finalization /// is intentionally skipped: no [`EcFinalizeState`] is attached, matching the -/// legacy `should_finalize_ec = false` behavior for asset responses. +/// legacy `should_finalize_ec = false` behavior for asset responses. The +/// caller's [`GeoLookupState`] is still attached, since `build_ec_request_state` +/// already attempted the lookup before the asset route was matched — this is +/// the one exit path that carries geo state without an `EcFinalizeState`. /// /// Like legacy `route_request`, asset bodies are streamed straight to the client /// with no cap: the origin stream is attached to the response and `edgezero_main` @@ -918,6 +1086,7 @@ async fn dispatch_asset_fallback( req: Request, asset_route: &ProxyAssetRoute, effects: &RequestFilterEffects, + geo_state: GeoLookupState, ) -> Response { log::info!("No explicit route matched; proxying via configured asset route"); @@ -939,6 +1108,7 @@ async fn dispatch_asset_fallback( } response.extensions_mut().insert(cache_policy); + response.extensions_mut().insert(geo_state); attach_request_filter_effects(&mut response, effects); response } @@ -947,6 +1117,7 @@ async fn dispatch_asset_fallback( response .extensions_mut() .insert(AssetProxyCachePolicy::NoStorePrivate); + response.extensions_mut().insert(geo_state); attach_request_filter_effects(&mut response, effects); response } @@ -1069,6 +1240,10 @@ struct NamedRoute { path: &'static str, primary_methods: &'static [Method], handler: NamedRouteHandler, + /// Access-telemetry traffic category for this row. Attached verbatim + /// alongside `path` (the route-table pattern) to every response this + /// route produces — see [`named_route_handler`]. + route_class: RouteClass, } const LEGACY_ADMIN_DENY_METHODS: &[Method] = &[ @@ -1086,21 +1261,25 @@ const NAMED_ROUTES: &[NamedRoute] = &[ path: "/.well-known/trusted-server.json", primary_methods: &[Method::GET], handler: NamedRouteHandler::TrustedServerDiscovery, + route_class: RouteClass::Other, }, NamedRoute { path: "/verify-signature", primary_methods: &[Method::POST], handler: NamedRouteHandler::VerifySignature, + route_class: RouteClass::Ec, }, NamedRoute { path: "/_ts/admin/keys/rotate", primary_methods: &[Method::POST], handler: NamedRouteHandler::RotateKey, + route_class: RouteClass::Ec, }, NamedRoute { path: "/_ts/admin/keys/deactivate", primary_methods: &[Method::POST], handler: NamedRouteHandler::DeactivateKey, + route_class: RouteClass::Ec, }, // Admin EC lookup: the bare route reads the EC ID from the caller's // `ts-ec` cookie; the parameterized route takes an explicit EC ID. @@ -1108,11 +1287,13 @@ const NAMED_ROUTES: &[NamedRoute] = &[ path: "/_ts/admin/ec", primary_methods: &[Method::GET], handler: NamedRouteHandler::AdminEcLookup, + route_class: RouteClass::Ec, }, NamedRoute { path: "/_ts/admin/ec/{id}", primary_methods: &[Method::GET], handler: NamedRouteHandler::AdminEcLookup, + route_class: RouteClass::Ec, }, // Admin EIDs echo: decodes the request's ts-eids/sharedId cookies with // an ingestion preview. Pure request inspection — no KV access. @@ -1120,6 +1301,7 @@ const NAMED_ROUTES: &[NamedRoute] = &[ path: "/_ts/admin/eids", primary_methods: &[Method::GET], handler: NamedRouteHandler::AdminEidsLookup, + route_class: RouteClass::Ec, }, // The legacy non-`/_ts` aliases (`/admin/keys/*`) are denied locally with a // 404 instead of executing key operations: the production basic-auth handler @@ -1131,36 +1313,43 @@ const NAMED_ROUTES: &[NamedRoute] = &[ path: "/admin/keys/rotate", primary_methods: LEGACY_ADMIN_DENY_METHODS, handler: NamedRouteHandler::LegacyAdminDenied, + route_class: RouteClass::Other, }, NamedRoute { path: "/admin/keys/deactivate", primary_methods: LEGACY_ADMIN_DENY_METHODS, handler: NamedRouteHandler::LegacyAdminDenied, + route_class: RouteClass::Other, }, NamedRoute { path: "/_ts/api/v1/batch-sync", primary_methods: &[Method::POST], handler: NamedRouteHandler::BatchSync, + route_class: RouteClass::Ec, }, NamedRoute { path: "/_ts/api/v1/identify", primary_methods: &[Method::GET, Method::OPTIONS], handler: NamedRouteHandler::Identify, + route_class: RouteClass::Ec, }, NamedRoute { path: "/_ts/set-tester", primary_methods: &[Method::GET], handler: NamedRouteHandler::SetTester, + route_class: RouteClass::Other, }, NamedRoute { path: "/_ts/clear-tester", primary_methods: &[Method::GET], handler: NamedRouteHandler::ClearTester, + route_class: RouteClass::Other, }, NamedRoute { path: "/auction", primary_methods: &[Method::POST], handler: NamedRouteHandler::Auction, + route_class: RouteClass::AuctionApi, }, // GET runs the SPA re-auction; OPTIONS is denied in-handler as a CORS // preflight guard for this side-effecting endpoint. @@ -1168,6 +1357,7 @@ const NAMED_ROUTES: &[NamedRoute] = &[ path: PAGE_BIDS_PATH, primary_methods: &[Method::GET, Method::OPTIONS], handler: NamedRouteHandler::PageBids, + route_class: RouteClass::AuctionApi, }, // Deprecated double-underscore alias. tsjs bundles served before the // `/_ts/page-bids` rename keep requesting this path from already-loaded @@ -1178,21 +1368,29 @@ const NAMED_ROUTES: &[NamedRoute] = &[ path: PAGE_BIDS_LEGACY_PATH, primary_methods: &[Method::GET, Method::OPTIONS], handler: NamedRouteHandler::PageBids, + route_class: RouteClass::AuctionApi, }, + // Classified `Other` rather than `IntegrationProxy`: that class is + // reserved for `state.registry.handle_proxy` (the js-integration proxy + // dispatch in `dispatch_fallback`), which these first-party proxy routes + // do not go through. NamedRoute { path: "/first-party/proxy", primary_methods: &[Method::GET], handler: NamedRouteHandler::FirstPartyProxy, + route_class: RouteClass::Other, }, NamedRoute { path: "/first-party/click", primary_methods: &[Method::GET], handler: NamedRouteHandler::FirstPartyClick, + route_class: RouteClass::Other, }, NamedRoute { path: "/first-party/sign", primary_methods: &[Method::GET, Method::POST], handler: NamedRouteHandler::FirstPartySign, + route_class: RouteClass::Other, }, NamedRoute { path: "/first-party/proxy-rebuild", @@ -1201,16 +1399,35 @@ const NAMED_ROUTES: &[NamedRoute] = &[ // POST is blocked by CORS and the guard navigates here for a 302 instead. primary_methods: &[Method::GET, Method::POST], handler: NamedRouteHandler::FirstPartyProxyRebuild, + route_class: RouteClass::Other, }, ]; +/// Wraps [`execute_named`], attaching a [`RouteMetadata`] extension carrying +/// `route_class` and the route-table pattern (`route_template`, verbatim, +/// with parameters left as placeholders) to every response the handler +/// produces — including its early-return diagnostic and setup-error arms, +/// since the attachment happens once around the whole future rather than in +/// each branch. fn named_route_handler( state: Arc, handler: NamedRouteHandler, + route_class: RouteClass, + route_template: &'static str, ) -> impl Fn(RequestContext) -> HandlerFuture + Clone + Send + Sync + 'static { move |ctx: RequestContext| { let state = Arc::clone(&state); - Box::pin(execute_named(state, ctx, handler)) + Box::pin(async move { + execute_named(state, ctx, handler) + .await + .map(|mut response| { + response.extensions_mut().insert(RouteMetadata { + route_class, + route_template: route_template.to_owned(), + }); + response + }) + }) } } @@ -1231,15 +1448,17 @@ fn fallback_route_handler( pub struct TrustedServerApp; impl TrustedServerApp { - pub(crate) fn build_app_with_state(env: &EnvConfig) -> (App, Option>) { - let (router, state) = Self::router_with_state(env); + pub(crate) fn build_app_with_state( + stores: &RuntimeStoreConfig, + ) -> (App, Option>) { + let (router, state) = Self::router_with_state(stores); let mut app = App::with_name(router, Self::name()); Self::configure(&mut app); (app, state) } - fn router_with_state(env: &EnvConfig) -> (RouterService, Option>) { - let state = match build_state(env) { + fn router_with_state(stores: &RuntimeStoreConfig) -> (RouterService, Option>) { + let state = match build_state(stores) { Ok(state) => state, Err(ref e) => { log::error!("failed to build application state: {:?}", e); @@ -1269,7 +1488,12 @@ impl TrustedServerApp { router = router.route( route.path, method.clone(), - named_route_handler(Arc::clone(state), route.handler), + named_route_handler( + Arc::clone(state), + route.handler, + route.route_class, + route.path, + ), ); } @@ -1297,16 +1521,25 @@ impl Hooks for TrustedServerApp { } fn routes() -> RouterService { - Self::router_with_state(&EnvConfig::from_env()).0 + let runtime_env = EnvConfig::from_env(); + let stores = RuntimeStoreConfig::from_env(&runtime_env); + Self::router_with_state(&stores).0 } fn stores() -> StoresMetadata { StoresMetadata { config: Some(StoreMetadata { - default: "trusted_server_config", - ids: &["trusted_server_config"], + default: DEFAULT_CONFIG_STORE_ID, + ids: &[DEFAULT_CONFIG_STORE_ID], + }), + kv: Some(StoreMetadata { + default: "trusted_server_kv", + ids: &["trusted_server_kv"], + }), + secrets: Some(StoreMetadata { + default: DEFAULT_SECRET_STORE_ID, + ids: &[DEFAULT_SECRET_STORE_ID], }), - ..StoresMetadata::default() } } } @@ -1320,16 +1553,20 @@ mod tests { use super::{ AppState, AuctionDispatch, EcContext, EdgeCacheHeader, HandlerFuture, NAMED_ROUTES, - NamedRouteHandler, PAGE_BIDS_LEGACY_PATH, PAGE_BIDS_PATH, TrustedServerApp, - build_per_request_services, build_state_from_settings, handle_publisher_request, - publisher_response_into_streaming_response, startup_error_router, + NamedRouteHandler, PAGE_BIDS_LEGACY_PATH, PAGE_BIDS_PATH, RouteClass, RouteMetadata, + RuntimeStoreConfig, TSJS_ROUTE_TEMPLATE, TrustedServerApp, build_per_request_services, + build_state_from_settings, handle_publisher_request, + publisher_response_into_streaming_response, publisher_route_template, startup_error_router, }; use base64::Engine as _; use bytes::Bytes; use edgezero_core::app::{Hooks as _, StoreMetadata}; use edgezero_core::body::Body; use edgezero_core::context::RequestContext; - use edgezero_core::http::{Method, Response, StatusCode, header, request_builder}; + use edgezero_core::env_config::EnvConfig; + use edgezero_core::http::{ + Method, Response, StatusCode, header, request_builder, response_builder, + }; use edgezero_core::key_value_store::NoopKvStore; use edgezero_core::params::PathParams; use edgezero_core::router::RouterService; @@ -1338,22 +1575,101 @@ mod tests { use error_stack::Report; use futures::executor::block_on; use serde_json::json; - use trusted_server_core::constants::HEADER_X_GEO_INFO_AVAILABLE; + use trusted_server_core::constants::{HEADER_X_GEO_COUNTRY, HEADER_X_GEO_INFO_AVAILABLE}; use trusted_server_core::ec::device::DeviceSignals; use trusted_server_core::error::TrustedServerError; + use trusted_server_core::geo::GeoLookupState; use trusted_server_core::integrations::{ HeaderMutation, IntegrationRegistry, IntegrationRequestFilter, RequestFilterDecision, RequestFilterEffects, RequestFilterInput, }; use trusted_server_core::platform::{ - ClientInfo, PlatformBackend, PlatformBackendSpec, PlatformError, PlatformHttpClient, - PlatformHttpRequest, PlatformKvStore, PlatformPendingRequest, PlatformResponse, - PlatformSelectResult, PlatformTemplateCache, PlatformTemplateCacheReservation, - RuntimeServices, TemplateCacheError, TemplateCacheKey, TemplateCacheLookup, - TemplateCacheMiss, TemplateCacheReservation, TemplateEntry, TemplateMetadata, + ClientInfo, GeoInfo, PlatformBackend, PlatformBackendSpec, PlatformError, PlatformGeo, + PlatformHttpClient, PlatformHttpRequest, PlatformKvStore, PlatformPendingRequest, + PlatformResponse, PlatformSelectResult, PlatformTemplateCache, + PlatformTemplateCacheReservation, RuntimeServices, TemplateCacheError, TemplateCacheKey, + TemplateCacheLookup, TemplateCacheMiss, TemplateCacheReservation, TemplateEntry, + TemplateMetadata, }; + use trusted_server_core::request_timing::RequestTimings; use trusted_server_core::settings::Settings; + #[test] + fn hooks_store_metadata_matches_edgezero_manifest() { + let manifest: toml::Value = toml::from_str(include_str!("../../../edgezero.toml")) + .expect("should parse edgezero manifest"); + let manifest_stores = manifest + .get("stores") + .and_then(toml::Value::as_table) + .expect("manifest should declare stores"); + let metadata = TrustedServerApp::stores(); + + for (kind, runtime_store) in [ + ( + "config", + metadata.config.expect("should declare config stores"), + ), + ("kv", metadata.kv.expect("should declare KV stores")), + ( + "secrets", + metadata.secrets.expect("should declare secret stores"), + ), + ] { + let manifest_store = manifest_stores + .get(kind) + .and_then(toml::Value::as_table) + .unwrap_or_else(|| panic!("manifest should declare {kind} stores")); + let manifest_default = manifest_store + .get("default") + .and_then(toml::Value::as_str) + .unwrap_or_else(|| panic!("manifest {kind} stores should declare a default")); + let manifest_ids = manifest_store + .get("ids") + .and_then(toml::Value::as_array) + .unwrap_or_else(|| panic!("manifest {kind} stores should declare ids")) + .iter() + .map(toml::Value::as_str) + .collect::>>() + .unwrap_or_else(|| panic!("manifest {kind} store ids should be strings")); + + assert_eq!(runtime_store.default, manifest_default); + assert_eq!(runtime_store.ids, manifest_ids); + } + } + + #[test] + fn runtime_store_config_maps_logical_store_names_and_config_key() { + let env = EnvConfig::from_vars([ + ( + "EDGEZERO__STORES__CONFIG__TRUSTED_SERVER_CONFIG__NAME", + "physical_config", + ), + ( + "EDGEZERO__STORES__CONFIG__TRUSTED_SERVER_CONFIG__KEY", + "active_config", + ), + ( + "EDGEZERO__STORES__SECRETS__TRUSTED_SERVER_SECRETS__NAME", + "ts_secrets", + ), + ]); + + let stores = RuntimeStoreConfig::from_env(&env); + + assert_eq!(stores.config_store_name.as_ref(), "physical_config"); + assert_eq!(stores.config_key, "active_config"); + assert_eq!(stores.secret_store_name.as_ref(), "ts_secrets"); + } + + #[test] + fn runtime_store_config_uses_logical_defaults_without_overrides() { + let stores = RuntimeStoreConfig::from_env(&EnvConfig::default()); + + assert_eq!(stores.config_store_name.as_ref(), "trusted_server_config"); + assert_eq!(stores.config_key, "trusted_server_config"); + assert_eq!(stores.secret_store_name.as_ref(), "trusted_server_secrets"); + } + fn settings_with_missing_consent_store() -> Settings { Settings::from_toml( r#" @@ -1384,7 +1700,6 @@ mod tests { [integrations.prebid] enabled = true - server_url = "https://test-prebid.com/openrtb2/auction" external_bundle_url = "https://assets.example/prebid/trusted-prebid.js" [integrations.datadome] @@ -1392,7 +1707,10 @@ mod tests { [auction] enabled = true - providers = ["prebid"] + [auction.providers.prebid] + protocol = "openrtb-2.6" + profile = "prebid-server" + endpoint = "https://test-prebid.com/openrtb2/auction" timeout_ms = 2000 "#, ) @@ -1450,12 +1768,14 @@ mod tests { [integrations.prebid] enabled = true - server_url = "https://test-prebid.com/openrtb2/auction" external_bundle_url = "https://assets.example/prebid/trusted-prebid.js" [auction] enabled = true - providers = ["prebid"] + [auction.providers.prebid] + protocol = "openrtb-2.6" + profile = "prebid-server" + endpoint = "https://test-prebid.com/openrtb2/auction" timeout_ms = 2000 "#, ) @@ -1468,7 +1788,7 @@ mod tests { } #[test] - fn trusted_server_app_declares_config_store_metadata() { + fn trusted_server_app_declares_runtime_store_metadata() { let stores = TrustedServerApp::stores(); assert_eq!( @@ -1478,8 +1798,20 @@ mod tests { ids: &["trusted_server_config"], }) ); - assert_eq!(stores.kv, None); - assert_eq!(stores.secrets, None); + assert_eq!( + stores.kv, + Some(StoreMetadata { + default: "trusted_server_kv", + ids: &["trusted_server_kv"], + }) + ); + assert_eq!( + stores.secrets, + Some(StoreMetadata { + default: "trusted_server_secrets", + ids: &["trusted_server_secrets"], + }) + ); } #[test] @@ -1512,19 +1844,24 @@ mod tests { ); } - /// Builds a router whose `AppState` uses a registry containing the given - /// request filters (and no routes), so dispatch-level request-filter - /// behavior can be exercised without a real integration. - fn router_with_request_filters( + /// Builds an `AppState` whose registry contains the given request + /// filters (and no routes), so dispatch-level request-filter behavior can + /// be exercised without a real integration. + fn state_with_request_filters( filters: Vec>, - ) -> RouterService { + ) -> Arc { let settings = test_settings(); - let orchestrator = trusted_server_core::auction::build_orchestrator(&settings) - .expect("should build orchestrator"); + let plan = Arc::new( + trusted_server_core::auction::compile_auction_plan(&settings) + .expect("should compile auction plan"), + ); + let orchestrator = + trusted_server_core::auction::build_orchestrator_with_plan(plan, &settings) + .expect("should build orchestrator"); let registry = IntegrationRegistry::from_request_filters(filters); let default_kv_store = Arc::new(crate::platform::UnavailableKvStore) as Arc; - let state = Arc::new(super::AppState { + Arc::new(super::AppState { auction_telemetry_sink: Arc::new( trusted_server_core::auction::NoopAuctionTelemetrySink, ), @@ -1532,8 +1869,15 @@ mod tests { orchestrator: Arc::new(orchestrator), registry: Arc::new(registry), default_kv_store, - }); - TrustedServerApp::routes_for_state(&state) + }) + } + + /// Builds a router on top of [`state_with_request_filters`] so + /// dispatch-level request-filter behavior can be exercised end-to-end. + fn router_with_request_filters( + filters: Vec>, + ) -> RouterService { + TrustedServerApp::routes_for_state(&state_with_request_filters(filters)) } /// Continues routing while mutating the request and emitting a response @@ -1606,6 +1950,34 @@ mod tests { } } + #[test] + fn startup_registers_aps_renderer_route() { + let mut settings = test_settings(); + settings.auction.providers.clear(); + settings.auction.providers.insert( + "aps-main".parse().expect("should parse APS provider ID"), + trusted_server_core::auction::ProviderConfig { + protocol: "openrtb-2.6".to_string(), + profile: "aps".to_string(), + endpoint: "https://aps.example/e/pb/bid".to_string(), + timeout_ms: None, + routing: trusted_server_core::auction::RoutingMode::AllEligible, + notifications: trusted_server_core::auction::NotificationConfig::default(), + profile_config: serde_json::json!({"account_id":"example-account"}), + }, + ); + + let state = build_state_from_settings(settings) + .expect("Fastly startup should register APS renderer"); + assert!( + state.registry.has_route( + &edgezero_core::http::Method::GET, + "/integrations/aps/renderer" + ), + "Fastly startup registry should expose the APS renderer" + ); + } + #[test] fn startup_error_router_handles_head_and_options() { let report = Report::new(TrustedServerError::BadRequest { @@ -2157,6 +2529,111 @@ mod tests { ); } + /// `Authorization: Basic` header value for `test_settings()`'s + /// `^/_ts/admin` handler (`admin` / `admin-pass`). + fn admin_basic_auth_header() -> edgezero_core::http::HeaderValue { + let credentials = base64::engine::general_purpose::STANDARD.encode("admin:admin-pass"); + format!("Basic {credentials}") + .parse() + .expect("should parse basic-auth header value") + } + + #[test] + fn named_route_attaches_the_table_pattern_verbatim_even_with_a_real_id_in_the_path() { + // A named-route response must carry the route-TABLE pattern + // (`{id}` left as a placeholder), never the caller's actual matched + // path segment — this is what keeps a real EC identifier out of + // access telemetry, independent of anything the row-serialization + // layer does. + let router = test_router(); + let ec_id = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.test01"; + let mut req = empty_request(Method::GET, &format!("/_ts/admin/ec/{ec_id}")); + req.headers_mut() + .insert(header::AUTHORIZATION, admin_basic_auth_header()); + let response = route(&router, req); + + let metadata = response + .extensions() + .get::() + .expect("named-route responses should carry RouteMetadata"); + assert_eq!(metadata.route_class, RouteClass::Ec); + assert_eq!(metadata.route_template, "/_ts/admin/ec/{id}"); + assert!( + !metadata.route_template.contains(ec_id), + "the attached template must never contain the matched id" + ); + } + + #[test] + fn named_route_attaches_metadata_even_on_a_read_only_diagnostic_early_return() { + // AdminEidsLookup is handled by an early-return arm inside + // execute_named, before the normal EC lifecycle runs (see the + // "read-only diagnostics" comment there). named_route_handler wraps + // the whole future, so the attachment must still happen here too. + let router = test_router(); + let mut req = empty_request(Method::GET, "/_ts/admin/eids"); + req.headers_mut() + .insert(header::AUTHORIZATION, admin_basic_auth_header()); + let response = route(&router, req); + + let metadata = response + .extensions() + .get::() + .expect("even a read-only diagnostic early-return response should carry RouteMetadata"); + assert_eq!(metadata.route_class, RouteClass::Ec); + assert_eq!(metadata.route_template, "/_ts/admin/eids"); + } + + #[test] + fn tsjs_fallback_attaches_tsjs_route_metadata() { + let router = test_router(); + let response = route( + &router, + empty_request(Method::GET, "/static/tsjs=tsjs-unified.min.js"), + ); + + let metadata = response + .extensions() + .get::() + .expect("tsjs fallback responses should carry RouteMetadata"); + assert_eq!(metadata.route_class, RouteClass::Tsjs); + assert_eq!(metadata.route_template, TSJS_ROUTE_TEMPLATE); + } + + #[test] + fn integration_proxy_fallback_attaches_integration_proxy_route_metadata() { + // test_settings() enables the prebid integration, which registers a + // proxy route at /integrations/prebid/bundle.js. + let router = test_router(); + let response = route( + &router, + empty_request(Method::GET, "/integrations/prebid/bundle.js"), + ); + + let metadata = response + .extensions() + .get::() + .expect("integration-proxy fallback responses should carry RouteMetadata"); + assert_eq!(metadata.route_class, RouteClass::IntegrationProxy); + assert_eq!( + metadata.route_template, + publisher_route_template("/integrations/prebid/bundle.js") + ); + } + + #[test] + fn publisher_fallback_attaches_publisher_html_route_metadata() { + let router = test_router(); + let response = route(&router, empty_request(Method::GET, "/news/some-article")); + + let metadata = response + .extensions() + .get::() + .expect("publisher fallback responses should carry RouteMetadata"); + assert_eq!(metadata.route_class, RouteClass::PublisherHtml); + assert_eq!(metadata.route_template, "/news/*"); + } + #[test] fn browser_device_signals_from_extension_reach_ec_finalize_state() { // Regression guard for the EdgeZero JA4/H2 signal loss: `edgezero_main` @@ -2577,9 +3054,68 @@ mod tests { ); } + #[test] + fn asset_fallback_carries_geo_state_without_ec_finalize_state() { + // The asset-route fallback is the one exit path that skips + // EcFinalizeState but must still carry GeoLookupState, since + // build_ec_request_state (and its geo lookup) already ran before the + // asset route was matched. Without this, the finalize step would + // silently repeat the lookup for every asset request. + let settings = Settings::from_toml( + r#" + [[handlers]] + path = "^/_ts/admin" + username = "admin" + password = "admin-pass" + + [publisher] + domain = "test-publisher.com" + cookie_domain = ".test-publisher.com" + origin_url = "https://origin.test-publisher.com" + proxy_secret = "unit-test-proxy-secret" + + [ec] + passphrase = "test-secret-key-32-bytes-minimum" + + [request_signing] + enabled = false + config_store_id = "test-config-store-id" + secret_store_id = "test-secret-store-id" + + [proxy] + + [[proxy.asset_routes]] + prefix = "/.image/" + origin_url = "https://assets.example.com" + "#, + ) + .expect("should parse asset-route settings"); + let state = build_state_from_settings(settings).expect("should build state"); + let router = TrustedServerApp::routes_for_state(&state); + + let response = route(&router, empty_request(Method::GET, "/.image/banner.png")); + + assert!( + response.extensions().get::().is_some(), + "asset-route responses should still carry GeoLookupState even though \ + EC finalization is skipped" + ); + assert!( + response + .extensions() + .get::() + .is_none(), + "asset-route responses must skip EC finalization (no EcFinalizeState)" + ); + } + struct FixedBackend; impl PlatformBackend for FixedBackend { + fn naming_policy(&self) -> trusted_server_core::platform::BackendNamingPolicy { + trusted_server_core::platform::BackendNamingPolicy::Fastly + } + fn predict_name( &self, spec: &PlatformBackendSpec, @@ -2804,7 +3340,6 @@ mod tests { [auction] enabled = true - providers = [] [creative_opportunities] gam_network_id = "99999" @@ -2831,11 +3366,16 @@ mod tests { .geo(Arc::new(crate::platform::FastlyPlatformGeo)) .client_info(ClientInfo::default()) .build(); + let plan = Arc::new( + trusted_server_core::auction::compile_auction_plan(&settings) + .expect("should compile auction plan"), + ); let registry = Arc::new( - IntegrationRegistry::new(&settings).expect("should build integration registry"), + IntegrationRegistry::with_plan(&settings, Arc::clone(&plan)) + .expect("should build integration registry"), ); let orchestrator = Arc::new( - trusted_server_core::auction::build_orchestrator(&settings) + trusted_server_core::auction::build_orchestrator_with_plan(plan, &settings) .expect("should build auction orchestrator"), ); @@ -2987,6 +3527,7 @@ mod tests { req, asset_route, &effects, + trusted_server_core::geo::GeoLookupState::NotAttempted, )); assert_eq!( @@ -3030,6 +3571,259 @@ mod tests { ); } + /// A [`PlatformGeo`] stub that counts every `lookup` call and always + /// returns the same canned result, used to prove the request-phase geo + /// lookup is never repeated during finalize. + struct CountingGeo { + calls: Arc, + result: Option, + } + + impl PlatformGeo for CountingGeo { + fn lookup(&self, _: Option) -> Result, Report> { + self.calls.fetch_add(1, Ordering::SeqCst); + Ok(self.result.clone()) + } + } + + fn sample_geo_info() -> GeoInfo { + GeoInfo { + city: "Testville".to_string(), + country: "US".to_string(), + continent: "NorthAmerica".to_string(), + latitude: 0.0, + longitude: 0.0, + metro_code: 0, + region: None, + asn: None, + } + } + + fn runtime_services_with_geo(geo: Arc) -> RuntimeServices { + RuntimeServices::builder() + .config_store(Arc::new(crate::platform::FastlyPlatformConfigStore)) + .secret_store(Arc::new(crate::platform::FastlyPlatformSecretStore)) + .kv_store(Arc::new(NoopKvStore) as Arc) + .backend(Arc::new(FixedBackend)) + .http_client(Arc::new(StreamingHttpClient)) + .geo(geo) + .client_info(ClientInfo::default()) + .build() + } + + #[test] + fn finalize_reuses_request_phase_geo_without_second_lookup() { + // Dispatching a publisher route runs build_ec_request_state, which + // attempts the geo lookup once and carries the result via + // GeoLookupState. The finalize step (resolve_geo_for_response) must + // reuse that carried value instead of calling the geo backend again. + let calls = Arc::new(AtomicUsize::new(0)); + let geo = Arc::new(CountingGeo { + calls: Arc::clone(&calls), + result: Some(sample_geo_info()), + }); + let state = app_state_for_settings(test_settings()); + let services = runtime_services_with_geo(geo); + let req = empty_request(Method::GET, "/some-page"); + + let response = block_on(super::dispatch_fallback(&state, &services, req)); + + let carried = response + .extensions() + .get::() + .cloned() + .expect("dispatch should attach GeoLookupState"); + assert!( + matches!(carried, GeoLookupState::Resolved(_)), + "a successful lookup should carry Resolved" + ); + + let geo_info = + crate::middleware::resolve_geo_for_response(&response, &carried, None, |_| { + panic!("finalize must not repeat a resolved geo lookup"); + }); + + assert_eq!( + calls.load(Ordering::SeqCst), + 1, + "only the request-phase lookup should have run" + ); + + let mut response = response; + geo_info + .expect("geo info should have resolved") + .set_response_headers(&mut response); + assert!( + response.headers().get(HEADER_X_GEO_COUNTRY).is_some(), + "x-geo-country should still be set on the response after reusing the carried geo" + ); + } + + #[test] + fn failed_lookup_is_not_retried() { + // When the request-phase lookup fails (returns None), dispatch must + // carry GeoLookupState::Attempted rather than NotAttempted, and + // finalize must not retry it. + let calls = Arc::new(AtomicUsize::new(0)); + let geo = Arc::new(CountingGeo { + calls: Arc::clone(&calls), + result: None, + }); + let state = app_state_for_settings(test_settings()); + let services = runtime_services_with_geo(geo); + let req = empty_request(Method::GET, "/some-page"); + + let response = block_on(super::dispatch_fallback(&state, &services, req)); + + let carried = response + .extensions() + .get::() + .cloned() + .expect("dispatch should attach GeoLookupState even for a failed lookup"); + assert!( + matches!(carried, GeoLookupState::Attempted), + "a failed lookup should carry Attempted, not Resolved or NotAttempted" + ); + + let geo_info = + crate::middleware::resolve_geo_for_response(&response, &carried, None, |_| { + panic!("finalize must not retry a failed geo lookup"); + }); + + assert_eq!( + calls.load(Ordering::SeqCst), + 1, + "only the request-phase lookup should have run" + ); + assert!( + geo_info.is_none(), + "no geo info should be available after a failed lookup" + ); + } + + #[test] + fn filter_span_recorded_when_request_filter_runs() { + // The Filter phase span should only be recorded when the registry + // actually has a request filter registered, so unconfigured + // deployments omit ts-filter from the Server-Timing header entirely. + let state = state_with_request_filters(vec![Arc::new(RecordingRequestFilter)]); + let services = RuntimeServices::builder() + .config_store(Arc::new(crate::platform::FastlyPlatformConfigStore)) + .secret_store(Arc::new(crate::platform::FastlyPlatformSecretStore)) + .kv_store(Arc::new(NoopKvStore) as Arc) + .backend(Arc::new(FixedBackend)) + .http_client(Arc::new(StreamingHttpClient)) + .geo(Arc::new(crate::platform::FastlyPlatformGeo)) + .client_info(ClientInfo::default()) + .build(); + let mut req = empty_request(Method::GET, "/some-page"); + let timings = RequestTimings::new(); + req.extensions_mut().insert(timings.clone()); + + let _ = block_on(super::run_pre_route_filters( + &state, &services, &mut req, None, + )); + + assert!( + timings.snapshot().filter_ms.is_some(), + "should record the Filter phase span when a request filter is registered and runs" + ); + } + + #[test] + fn filter_span_not_recorded_when_no_request_filters_registered() { + // Mirror test: an empty registry must never record the Filter span, + // even though run_pre_route_filters still runs (as a no-op loop). + let state = state_with_request_filters(Vec::new()); + let services = RuntimeServices::builder() + .config_store(Arc::new(crate::platform::FastlyPlatformConfigStore)) + .secret_store(Arc::new(crate::platform::FastlyPlatformSecretStore)) + .kv_store(Arc::new(NoopKvStore) as Arc) + .backend(Arc::new(FixedBackend)) + .http_client(Arc::new(StreamingHttpClient)) + .geo(Arc::new(crate::platform::FastlyPlatformGeo)) + .client_info(ClientInfo::default()) + .build(); + let mut req = empty_request(Method::GET, "/some-page"); + let timings = RequestTimings::new(); + req.extensions_mut().insert(timings.clone()); + + let _ = block_on(super::run_pre_route_filters( + &state, &services, &mut req, None, + )); + + assert!( + timings.snapshot().filter_ms.is_none(), + "should omit the Filter phase span when no request filters are registered" + ); + } + + fn settings_with_consent_and_ec_store() -> Settings { + Settings::from_toml( + r#" + [[handlers]] + path = "^/_ts/admin" + username = "admin" + password = "admin-pass" + + [publisher] + domain = "test-publisher.com" + cookie_domain = ".test-publisher.com" + origin_url = "https://origin.test-publisher.com" + proxy_secret = "unit-test-proxy-secret" + + [ec] + passphrase = "test-secret-key-32-bytes-minimum" + ec_store = "ec_identity_store" + + [consent] + consent_store = "consent_store" + + [request_signing] + enabled = false + config_store_id = "test-config-store-id" + secret_store_id = "test-secret-store-id" + "#, + ) + .expect("should parse settings with consent and EC KV stores configured") + } + + #[test] + fn consent_store_reads_are_timed_and_pull_sync_is_not() { + // Consent-store access threaded through RuntimeServices uses the same + // TimedKvStore decorator as request-path KvIdentityGraph + // construction, so a read through it records Phase::EcKv. + let settings = settings_with_consent_and_ec_store(); + let services = streaming_runtime_services(); + let timings = RequestTimings::new(); + + let consent_services = + super::runtime_services_for_consent_route(&settings, &services, &timings) + .expect("should open the configured consent store"); + let _ = block_on(consent_services.kv_store().get_bytes("consent-read-key")); + + timings.mark_headers_ready(); + assert!( + timings.snapshot().kv_ms.is_some(), + "a consent-store read through the decorated RuntimeServices store should record Phase::EcKv" + ); + + // Pull-sync's identity graph is built by `require_identity_graph`, + // which takes no `timings` parameter at all — the untimed store it + // constructs cannot record into any handle, including a fresh one. + let graph = crate::require_identity_graph(&settings) + .expect("should construct the pull-sync identity graph"); + let ec_id = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.test01"; + let _ = graph.get(ec_id); + + let pull_sync_timings = RequestTimings::new(); + pull_sync_timings.mark_headers_ready(); + assert!( + pull_sync_timings.snapshot().kv_ms.is_none(), + "pull-sync's untimed graph construction has no timings handle to record into" + ); + } + #[test] fn dispatch_runs_request_filter_and_threads_response_effects() { // Regression guard for the EdgeZero request-filter bypass: the publisher @@ -3088,4 +3882,184 @@ mod tests { "the filter's response-header effect must be threaded out" ); } + + fn recovery_eligible_of(response: &Response) -> bool { + response + .extensions() + .get::() + .expect("response should carry EcFinalizeState") + .ec_context + .recovery_eligible() + } + + fn browser_navigation_request(path: &str) -> edgezero_core::http::Request { + let uri = format!("https://test-publisher.com{path}"); + let mut req = request_builder() + .method(Method::GET) + .uri(uri) + .header("sec-fetch-dest", "document") + .body(Body::empty()) + .expect("should build request"); + req.extensions_mut().insert(DeviceSignals::derive( + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 \ + (KHTML, like Gecko) Chrome/146.0.0.0 Safari/537.36", + Some("t13d1516h2_8daaf6152771_b186095e22b6"), + Some("1:65536;2:0;4:6291456;6:262144"), + )); + req + } + + #[test] + fn named_route_response_is_not_recovery_eligible() { + // Orphan recovery must never be authorized on a named route: it is not a + // publisher-page navigation, so a missing KV row must not rotate the + // identity there. + let router = test_router(); + let response = route( + &router, + empty_request(Method::GET, "/.well-known/trusted-server.json"), + ); + + assert!( + !recovery_eligible_of(&response), + "named-route responses must not authorize orphan recovery" + ); + } + + #[test] + fn filter_short_circuit_response_is_not_recovery_eligible() { + // A request-filter short circuit (e.g. a DataDome challenge/block) must + // not authorize orphan recovery even for a would-be publisher + // navigation: no publisher page was served. + let router = router_with_request_filters(vec![Arc::new(ChallengeRequestFilter)]); + let response = route(&router, browser_navigation_request("/some-page")); + + assert_eq!( + response.status(), + StatusCode::FORBIDDEN, + "the challenge filter should short-circuit routing" + ); + assert!( + !recovery_eligible_of(&response), + "a short-circuit filter response must not authorize orphan recovery" + ); + } + + /// Joins every instance of a response header into one comma-separated + /// string (mirroring how a client sees repeated header fields), or + /// `None` if the header is absent. + fn response_header(response: &Response, name: &str) -> Option { + let values: Vec<&str> = response + .headers() + .get_all(name) + .iter() + .filter_map(|value| value.to_str().ok()) + .collect(); + if values.is_empty() { + None + } else { + Some(values.join(", ")) + } + } + + #[test] + fn server_timing_emitted_on_private_response_when_enabled() { + let mut response = response_builder() + .header("cache-control", "private, no-store") + .body(Body::empty()) + .expect("should build a private response fixture"); + let timings = RequestTimings::new(); + + crate::apply_server_timing_header(&mut response, &timings, true); + + let header = response_header(&response, "server-timing").expect("should emit header"); + assert!( + header.contains("ts-total;dur="), + "should carry the stored total: {header}" + ); + assert_eq!( + header.matches("ts-total").count(), + 1, + "should emit exactly one TS-owned metric set" + ); + } + + #[test] + fn publisher_navigation_origin_start_failure_is_not_recovery_eligible() { + // Recovery is authorized only after a successful origin start. With no + // live backend the publisher origin fails, so even a real-browser + // document navigation must leave recovery unauthorized. + let router = test_router(); + let response = route(&router, browser_navigation_request("/some-page")); + + assert!( + !recovery_eligible_of(&response), + "an origin-start failure must not authorize orphan recovery" + ); + } + + #[test] + fn server_timing_absent_when_flag_off() { + let mut response = response_builder() + .header("cache-control", "private, no-store") + .body(Body::empty()) + .expect("should build a private response fixture"); + let timings = RequestTimings::new(); + + crate::apply_server_timing_header(&mut response, &timings, false); + + assert!( + response_header(&response, "server-timing").is_none(), + "should not emit server-timing when the flag is off" + ); + } + + #[test] + fn server_timing_absent_on_cacheable_responses() { + // tsjs route policy: public, long max-age, immutable. + let mut tsjs_response = response_builder() + .header("cache-control", "public, max-age=31536000, immutable") + .body(Body::empty()) + .expect("should build a tsjs-style response fixture"); + // A bare shared-cacheable response with no private/no-store directive. + let mut public_response = response_builder() + .header("cache-control", "max-age=60") + .body(Body::empty()) + .expect("should build a bare max-age response fixture"); + + crate::apply_server_timing_header(&mut tsjs_response, &RequestTimings::new(), true); + crate::apply_server_timing_header(&mut public_response, &RequestTimings::new(), true); + + assert!( + response_header(&tsjs_response, "server-timing").is_none(), + "should not emit on the public immutable tsjs cache policy" + ); + assert!( + response_header(&public_response, "server-timing").is_none(), + "should not emit on a bare shared-cacheable max-age response" + ); + } + + #[test] + fn preexisting_server_timing_values_survive() { + let mut response = response_builder() + .header("cache-control", "private, no-store") + .header("server-timing", "upstream;dur=1") + .body(Body::empty()) + .expect("should build a private response fixture carrying an upstream Server-Timing"); + let timings = RequestTimings::new(); + + crate::apply_server_timing_header(&mut response, &timings, true); + + let header = + response_header(&response, "server-timing").expect("should still carry a header"); + assert!( + header.contains("upstream;dur=1"), + "should preserve the pre-existing entry: {header}" + ); + assert!( + header.contains("ts-total"), + "should append the TS-owned set: {header}" + ); + } } diff --git a/crates/trusted-server-adapter-fastly/src/backend.rs b/crates/trusted-server-adapter-fastly/src/backend.rs index f2ff5d9e5..0151cbb16 100644 --- a/crates/trusted-server-adapter-fastly/src/backend.rs +++ b/crates/trusted-server-adapter-fastly/src/backend.rs @@ -1,13 +1,16 @@ -use core::fmt::Write as _; use std::time::Duration; use error_stack::{Report, ResultExt as _}; use fastly::backend::Backend; -use sha2::{Digest as _, Sha256}; use url::Url; use trusted_server_core::error::TrustedServerError; -use trusted_server_core::host_header::validate_host_header_override_value; +use trusted_server_core::platform::{BackendNamingPolicy, PlatformBackendSpec, PredictedBackend}; + +#[cfg(test)] +const MAX_BACKEND_NAME_LEN: usize = 255; +#[cfg(test)] +const SPEC_DIGEST_HEX_LEN: usize = 32; /// Returns the default port for the given scheme (443 for HTTPS, 80 for HTTP). #[inline] @@ -19,6 +22,44 @@ fn default_port_for_scheme(scheme: &str) -> u16 { } } +#[derive(Debug, Clone, Eq, PartialEq)] +struct NormalizedBackendHost { + identity: String, + authority: String, + is_ip_literal: bool, +} + +/// Normalize URL-derived and direct hosts for transport and TLS use. +/// +/// `url::Url::host_str()` preserves brackets around IPv6 literals. Fastly's +/// backend target and HTTP authority require those brackets, while certificate +/// identity matching requires the bare address and SNI must not be sent for IP +/// literals. +#[inline] +fn normalize_backend_host(host: &str) -> NormalizedBackendHost { + let unbracketed = host + .strip_prefix('[') + .and_then(|value| value.strip_suffix(']')) + .unwrap_or(host); + match unbracketed.parse::() { + Ok(std::net::IpAddr::V6(_)) => NormalizedBackendHost { + identity: unbracketed.to_owned(), + authority: format!("[{unbracketed}]"), + is_ip_literal: true, + }, + Ok(std::net::IpAddr::V4(_)) => NormalizedBackendHost { + identity: unbracketed.to_owned(), + authority: unbracketed.to_owned(), + is_ip_literal: true, + }, + Err(_) => NormalizedBackendHost { + identity: host.to_owned(), + authority: host.to_owned(), + is_ip_literal: false, + }, + } +} + /// Compute the Host header value for a backend request. /// /// For standard ports (443 for HTTPS, 80 for HTTP), returns just the hostname. @@ -29,54 +70,14 @@ fn default_port_for_scheme(scheme: &str) -> u16 { /// would generate URLs without the port when the Host header didn't include it. #[inline] fn compute_host_header(scheme: &str, host: &str, port: u16) -> String { + let host = normalize_backend_host(host).authority; if port == default_port_for_scheme(scheme) { - host.to_owned() + host } else { format!("{host}:{port}") } } -fn sanitize_backend_name_component(value: &str) -> String { - value - .chars() - .map(|ch| { - if ch.is_ascii_alphanumeric() || matches!(ch, '-' | '_') { - ch - } else { - '_' - } - }) - .collect() -} - -/// Fastly's documented maximum length for a dynamic backend name. -const MAX_BACKEND_NAME_LEN: usize = 255; -/// Maximum length of the human-readable prefix folded into a backend name. -/// -/// Bounds the name so that `backend__` can never exceed -/// [`MAX_BACKEND_NAME_LEN`]: 8 (`backend_`) + 200 + 1 (`_`) + -/// [`SPEC_DIGEST_HEX_LEN`] = 241 ≤ 255. -const MAX_READABLE_PREFIX_LEN: usize = 200; -/// Width of the hex digest suffix — the first 128 bits of a SHA-256 over the -/// full backend spec, which is collision-resistant at the handful-of-hundreds -/// scale of a service's dynamic backends. -const SPEC_DIGEST_HEX_LEN: usize = 32; - -/// Hex-encode the first 128 bits of a SHA-256 digest of `canonical`. -/// -/// Used to make a backend name a collision-resistant function of the complete -/// backend spec (see [`BackendConfig::canonical_spec_string`]). -fn spec_digest_hex(canonical: &str) -> String { - let mut hasher = Sha256::new(); - hasher.update(canonical.as_bytes()); - let digest = hasher.finalize(); - let mut hex = String::with_capacity(SPEC_DIGEST_HEX_LEN); - for byte in digest.iter().take(SPEC_DIGEST_HEX_LEN / 2) { - write!(hex, "{byte:02x}").expect("should write hex digit to string"); - } - hex -} - /// Default first-byte timeout for backends (15 seconds). pub(crate) const DEFAULT_FIRST_BYTE_TIMEOUT: Duration = Duration::from_secs(15); /// Default timeout between response body bytes for backends (10 seconds). @@ -173,149 +174,32 @@ impl<'a> BackendConfig<'a> { self } - /// Build an unambiguous, length-prefixed encoding of the complete backend - /// spec for digesting. - /// - /// Every field is prefixed with its byte length so that no two distinct - /// specs can encode to the same string (a lossy substitution like - /// `sanitize_backend_name_component` cannot guarantee this). `Option` fields - /// are presence-tagged so a `None` never aliases a `Some("")`. The result is - /// fed to [`spec_digest_hex`]; it is never parsed, only hashed. - fn canonical_spec_string(&self, target_port: u16) -> String { - fn push_field(buf: &mut String, field: &str) { - buf.push_str(&field.len().to_string()); - buf.push(':'); - buf.push_str(field); - } - - let mut buf = String::new(); - push_field(&mut buf, self.scheme); - push_field(&mut buf, self.host); - push_field(&mut buf, &target_port.to_string()); - push_field(&mut buf, if self.certificate_check { "1" } else { "0" }); - match self.host_header_override { - Some(value) => { - buf.push('s'); - push_field(&mut buf, value); - } - None => buf.push('n'), + fn platform_spec(&self) -> PlatformBackendSpec { + PlatformBackendSpec { + scheme: self.scheme.to_owned(), + host: normalize_backend_host(self.host).identity, + port: self.port, + host_header_override: self.host_header_override.map(str::to_owned), + certificate_check: self.certificate_check, + first_byte_timeout: self.first_byte_timeout, + between_bytes_timeout: self.between_bytes_timeout, + discriminator: self.discriminator.map(str::to_owned), } - match self.discriminator { - Some(value) => { - buf.push('s'); - push_field(&mut buf, value); - } - None => buf.push('n'), - } - push_field(&mut buf, &self.first_byte_timeout.as_millis().to_string()); - push_field( - &mut buf, - &self.between_bytes_timeout.as_millis().to_string(), - ); - buf } /// Compute the deterministic backend name and resolved port without /// registering anything. - /// - /// The name is `backend__`, where `` is a - /// collision-resistant SHA-256 over an unambiguous encoding of the - /// *complete* backend spec — scheme, host, port, certificate setting, Host - /// override, provider discriminator, and the first-byte/between-bytes - /// timeouts (see [`canonical_spec_string`](Self::canonical_spec_string)). - /// Because distinct specs yield distinct digests, name equality implies spec - /// equality: that is what makes reusing a `NameInUse` backend provably safe, - /// and it prevents "first-registration-wins" poisoning where a later request - /// with a tighter timeout would inherit an earlier registration's value. The - /// `` half is a lossy, bounded slug carried only for logs — any - /// collision there is harmless because uniqueness comes from the digest. The - /// whole name is bounded to [`MAX_BACKEND_NAME_LEN`] so a long host or - /// discriminator can never produce a name Fastly rejects at registration. - fn compute_name(&self) -> Result<(String, u16), Report> { - if self.host.is_empty() { - return Err(Report::new(TrustedServerError::Proxy { - message: "missing host".to_owned(), - })); - } - if self.host.chars().any(char::is_control) { - return Err(Report::new(TrustedServerError::Proxy { - message: "host contains control characters".to_owned(), - })); - } - if self.scheme.chars().any(char::is_control) { - return Err(Report::new(TrustedServerError::Proxy { - message: "scheme contains control characters".to_owned(), - })); - } - if let Some(host_header_override) = self.host_header_override { - validate_host_header_override_value(host_header_override).map_err(|reason| { - Report::new(TrustedServerError::Proxy { - message: format!("host header override {reason}"), - }) - })?; - } - - let target_port = self - .port - .unwrap_or_else(|| default_port_for_scheme(self.scheme)); - - let name_base = format!("{}_{}_{}", self.scheme, self.host, target_port); - let host_override_suffix = self - .host_header_override - .map(|host| format!("_oh_{}", sanitize_backend_name_component(host))) - .unwrap_or_default(); - let cert_suffix = if self.certificate_check { - "" - } else { - "_nocert" - }; - let discriminator_suffix = self - .discriminator - .map(|d| format!("_p_{}", sanitize_backend_name_component(d))) - .unwrap_or_default(); - let first_byte_timeout_ms = self.first_byte_timeout.as_millis(); - let between_bytes_timeout_ms = self.between_bytes_timeout.as_millis(); - - // Lossy, human-readable slug for logs. Correctness does not depend on - // it — uniqueness comes from the digest below — so it is bounded to a - // fixed length. Sanitization only emits ASCII, so a char-boundary take - // is byte-exact. - let readable_full = format!( - "{}{}{}{}_fb{}_bb{}", - sanitize_backend_name_component(&name_base), - host_override_suffix, - cert_suffix, - discriminator_suffix, - first_byte_timeout_ms, - between_bytes_timeout_ms - ); - let readable: String = readable_full - .chars() - .take(MAX_READABLE_PREFIX_LEN) - .collect(); - - // Collision-resistant over the *complete* spec, so name equality implies - // spec equality and `NameInUse` reuse is safe. - let digest = spec_digest_hex(&self.canonical_spec_string(target_port)); - let backend_name = format!("backend_{readable}_{digest}"); - - // Bounded by construction; assert it so any future format change fails - // attributably during prediction rather than at Fastly registration. - if backend_name.len() > MAX_BACKEND_NAME_LEN { - return Err(Report::new(TrustedServerError::Proxy { - message: format!( - "backend name exceeds {MAX_BACKEND_NAME_LEN}-char limit ({} chars)", - backend_name.len() - ), - })); - } - - Ok((backend_name, target_port)) + fn predict_backend(&self) -> Result> { + BackendNamingPolicy::Fastly + .predict(&self.platform_spec()) + .change_context(TrustedServerError::Proxy { + message: "backend name prediction failed".to_owned(), + }) } /// Return the deterministic backend name without registering anything. /// - /// Convenience wrapper over `Self::compute_name` that discards the + /// Convenience wrapper over `Self::predict_backend` that discards the /// resolved port, used by [`crate::platform::PlatformBackend`] /// implementations that only need the name for correlation. /// @@ -323,13 +207,13 @@ impl<'a> BackendConfig<'a> { /// /// Returns an error if the host is empty. pub fn predict_name(self) -> Result> { - self.compute_name().map(|(name, _)| name) + self.predict_backend().map(|prediction| prediction.name) } /// Ensure a dynamic backend exists for this configuration and return its name. /// /// The name is a collision-resistant function of the complete backend spec - /// (see `Self::compute_name`), so different specs — for example, different + /// (see `Self::predict_backend`), so different specs — for example, different /// timeout values — always produce different backend registrations and a /// tight deadline cannot be silently widened by an earlier registration. /// @@ -338,12 +222,15 @@ impl<'a> BackendConfig<'a> { /// Returns an error if the host is empty or if backend creation fails /// (except for `NameInUse` which reuses the existing backend). pub fn ensure(self) -> Result> { - let (backend_name, target_port) = self.compute_name()?; + let prediction = self.predict_backend()?; + let backend_name = prediction.name; + let target_port = prediction.port; + let host = normalize_backend_host(self.host); - let host_with_port = format!("{}:{}", self.host, target_port); + let host_with_port = format!("{}:{target_port}", host.authority); let host_header = self.host_header_override.map_or_else( - || compute_host_header(self.scheme, self.host, target_port), + || compute_host_header(self.scheme, &host.identity, target_port), str::to_owned, ); @@ -354,9 +241,12 @@ impl<'a> BackendConfig<'a> { .first_byte_timeout(self.first_byte_timeout) .between_bytes_timeout(self.between_bytes_timeout); if self.scheme.eq_ignore_ascii_case("https") { - builder = builder.enable_ssl().sni_hostname(self.host); + builder = builder.enable_ssl(); + if !host.is_ip_literal { + builder = builder.sni_hostname(&host.identity); + } if self.certificate_check { - builder = builder.check_certificate(self.host); + builder = builder.check_certificate(&host.identity); } else { log::warn!("INSECURE: certificate check disabled for backend: {backend_name}"); } @@ -474,7 +364,12 @@ impl<'a> BackendConfig<'a> { #[cfg(test)] mod tests { - use super::{BackendConfig, MAX_BACKEND_NAME_LEN, SPEC_DIGEST_HEX_LEN, compute_host_header}; + use trusted_server_core::platform::BackendNamingError; + + use super::{ + BackendConfig, MAX_BACKEND_NAME_LEN, SPEC_DIGEST_HEX_LEN, compute_host_header, + normalize_backend_host, + }; /// Assert a computed name is `backend__` and stays within /// Fastly's length limit. The digest is what makes the name injective, so @@ -503,6 +398,63 @@ mod tests { } // Tests for compute_host_header - the fix for port preservation in Host header + #[test] + fn ipv6_hosts_are_bracketed_only_for_authority_values() { + let bare = normalize_backend_host("2001:db8::1"); + let bracketed = normalize_backend_host("[2001:db8::1]"); + assert_eq!(bare, bracketed); + assert_eq!(bare.identity, "2001:db8::1"); + assert_eq!(bare.authority, "[2001:db8::1]"); + assert!(bare.is_ip_literal, "IPv6 must not be sent as TLS SNI"); + assert_eq!( + normalize_backend_host("cdn.example.com"), + super::NormalizedBackendHost { + identity: "cdn.example.com".to_string(), + authority: "cdn.example.com".to_string(), + is_ip_literal: false, + } + ); + assert_eq!( + compute_host_header("https", "[2001:db8::1]", 443), + "[2001:db8::1]" + ); + assert_eq!( + compute_host_header("https", "[2001:db8::1]", 8443), + "[2001:db8::1]:8443" + ); + } + + #[test] + fn url_derived_ipv6_host_uses_bare_tls_identity_without_sni() { + let (scheme, url_host, port) = + BackendConfig::parse_origin("https://[2001:db8::7]:8443/openrtb") + .expect("should parse IPv6 provider URL"); + assert_eq!(scheme, "https"); + assert_eq!(url_host, "[2001:db8::7]"); + assert_eq!(port, Some(8443)); + + let normalized = normalize_backend_host(&url_host); + assert_eq!(normalized.identity, "2001:db8::7"); + assert_eq!(normalized.authority, "[2001:db8::7]"); + assert!( + normalized.is_ip_literal, + "IP literals must omit TLS SNI while retaining a bare certificate identity" + ); + + let from_url_name = BackendConfig::new(&scheme, &url_host) + .port(port) + .predict_name() + .expect("should predict URL-derived IPv6 backend name"); + let from_bare_name = BackendConfig::new(&scheme, "2001:db8::7") + .port(port) + .predict_name() + .expect("should predict bare IPv6 backend name"); + assert_eq!( + from_url_name, from_bare_name, + "URL and direct IPv6 paths must preserve backend naming parity" + ); + } + #[test] fn host_header_includes_port_for_non_standard_https() { assert_eq!( @@ -584,8 +536,8 @@ mod tests { .predict_name() .expect_err("should reject host containing newline"); assert!( - err.to_string().contains("control characters"), - "should report control characters in error message" + err.contains::(), + "should preserve the backend naming error report context" ); } @@ -594,10 +546,9 @@ mod tests { let err = BackendConfig::new("https", "") .ensure() .expect_err("should reject empty host"); - let msg = err.to_string(); assert!( - msg.contains("missing host"), - "should report missing host in error message" + err.contains::(), + "should preserve the original backend naming error report context" ); } @@ -617,13 +568,13 @@ mod tests { #[test] fn host_header_overrides_produce_different_names() { - let (name_a, _) = BackendConfig::new("https", "origin.example.com") + let name_a = BackendConfig::new("https", "origin.example.com") .host_header_override(Some("www.example.com")) - .compute_name() + .predict_name() .expect("should compute name with host header override"); - let (name_b, _) = BackendConfig::new("https", "origin.example.com") + let name_b = BackendConfig::new("https", "origin.example.com") .host_header_override(Some("m.example.com")) - .compute_name() + .predict_name() .expect("should compute name with different host header override"); assert_ne!( @@ -648,8 +599,8 @@ mod tests { .expect_err("should reject host header override containing newline"); assert!( - err.to_string().contains("control characters"), - "should report control characters in error message" + err.contains::(), + "should preserve the backend naming error report context" ); } @@ -668,8 +619,8 @@ mod tests { .expect_err("should reject invalid host header override"); assert!( - err.to_string().contains("host header override"), - "should report host header override error for {host_header_override:?}" + err.contains::(), + "should preserve the backend naming error report context for {host_header_override:?}" ); } } @@ -678,13 +629,13 @@ mod tests { fn different_timeouts_produce_different_names() { use std::time::Duration; - let (name_a, _) = BackendConfig::new("https", "origin.example.com") + let name_a = BackendConfig::new("https", "origin.example.com") .first_byte_timeout(Duration::from_secs(2)) - .compute_name() + .predict_name() .expect("should compute name with 2000ms timeout"); - let (name_b, _) = BackendConfig::new("https", "origin.example.com") + let name_b = BackendConfig::new("https", "origin.example.com") .first_byte_timeout(Duration::from_millis(500)) - .compute_name() + .predict_name() .expect("should compute name with 500ms timeout"); assert_ne!( name_a, name_b, @@ -704,13 +655,13 @@ mod tests { fn different_between_bytes_timeouts_produce_different_names() { use std::time::Duration; - let (name_a, _) = BackendConfig::new("https", "origin.example.com") + let name_a = BackendConfig::new("https", "origin.example.com") .between_bytes_timeout(Duration::from_secs(2)) - .compute_name() + .predict_name() .expect("should compute name with 2000ms between-bytes timeout"); - let (name_b, _) = BackendConfig::new("https", "origin.example.com") + let name_b = BackendConfig::new("https", "origin.example.com") .between_bytes_timeout(Duration::from_millis(500)) - .compute_name() + .predict_name() .expect("should compute name with 500ms between-bytes timeout"); assert_ne!( diff --git a/crates/trusted-server-adapter-fastly/src/main.rs b/crates/trusted-server-adapter-fastly/src/main.rs index 8a73a80db..15ca6fcf9 100644 --- a/crates/trusted-server-adapter-fastly/src/main.rs +++ b/crates/trusted-server-adapter-fastly/src/main.rs @@ -1,12 +1,14 @@ use std::sync::Arc; +use rand::Rng as _; +use std::time::{Instant, SystemTime, UNIX_EPOCH}; + use edgezero_adapter_fastly::config_store::FastlyConfigStore as EdgeZeroFastlyConfigStore; use edgezero_adapter_fastly::request::into_core_request; use edgezero_adapter_fastly::runtime_env_config; use edgezero_core::app::Hooks as _; use edgezero_core::body::Body as EdgeBody; use edgezero_core::config_store::ConfigStoreHandle; -use edgezero_core::env_config::EnvConfig; use edgezero_core::error::EdgeError; use edgezero_core::http::{Request as HttpRequest, Response as HttpResponse}; use edgezero_core::response::IntoResponse; @@ -14,7 +16,13 @@ use error_stack::Report; use fastly::http::Method as FastlyMethod; use fastly::{Request as FastlyRequest, Response as FastlyResponse}; +use trusted_server_core::access_telemetry::{ + AccessTelemetrySnapshot, RouteClass, RouteMetadata, access_event_row, +}; use trusted_server_core::cache_policy::EdgeCacheHeader; +use trusted_server_core::constants::{ + ENV_FASTLY_IS_STAGING, ENV_FASTLY_POP, ENV_FASTLY_SERVICE_ID, ENV_FASTLY_SERVICE_VERSION, +}; use trusted_server_core::ec::device::DeviceSignals; use trusted_server_core::ec::finalize::ec_finalize_response; use trusted_server_core::ec::kv::KvIdentityGraph; @@ -23,13 +31,15 @@ use trusted_server_core::ec::pull_sync::{ }; use trusted_server_core::ec::registry::PartnerRegistry; use trusted_server_core::error::TrustedServerError; +use trusted_server_core::geo::GeoLookupState; use trusted_server_core::integrations::RequestFilterEffects; use trusted_server_core::platform::PlatformGeo as _; -use trusted_server_core::platform::RuntimeServices; +use trusted_server_core::platform::{RuntimeServices, TimedKvStore}; use trusted_server_core::proxy::{AssetProxyCachePolicy, stream_asset_body}; +use trusted_server_core::publisher::TemplateCacheResponseState; +use trusted_server_core::request_timing::{Phase, RequestTimings, append_server_timing_if_private}; use trusted_server_core::response_privacy::TerminalPrivateResponse; use trusted_server_core::settings::Settings; -use trusted_server_core::settings_data::config_store_name; mod app; mod backend; @@ -44,7 +54,9 @@ mod rate_limiter; mod template_cache; mod tinybird; -use crate::app::{EcFinalizeState, TrustedServerApp, load_settings_from_config_store}; +use crate::app::{ + EcFinalizeState, RuntimeStoreConfig, TrustedServerApp, load_settings_from_config_store, +}; use crate::ec_kv::FastlyEcKvStore; use crate::middleware::{HEADER_X_TS_FINALIZED, apply_finalize_headers, resolve_geo_for_response}; use crate::platform::{FastlyPlatformGeo, client_info_from_request}; @@ -55,9 +67,8 @@ use crate::rate_limiter::{FastlyRateLimiter, RATE_COUNTER_NAME}; /// # Errors /// /// Returns [`fastly::Error`] if the config store cannot be opened. -fn open_trusted_server_config_store(env: &EnvConfig) -> Result { - let store_name = config_store_name(env); - let store = EdgeZeroFastlyConfigStore::try_open(store_name.as_ref()).map_err(|e| { +fn open_trusted_server_config_store(store_name: &str) -> Result { + let store = EdgeZeroFastlyConfigStore::try_open(store_name).map_err(|e| { fastly::Error::msg(format!("failed to open config store `{store_name}`: {e}")) })?; Ok(ConfigStoreHandle::new(Arc::new(store))) @@ -87,16 +98,17 @@ fn main() { logging::init_logger(); let env = runtime_env_config(TrustedServerApp::stores()); - edgezero_main(req, &env); + let runtime_stores = RuntimeStoreConfig::from_env(&env); + edgezero_main(req, &runtime_stores); } /// Handles a request through the `EdgeZero` router path. -fn edgezero_main(mut req: FastlyRequest, env: &EnvConfig) { +fn edgezero_main(mut req: FastlyRequest, runtime_stores: &RuntimeStoreConfig) { // Short-circuit the JA4 debug probe before app construction. Must run here // because TLS/JA4 accessors are only available on FastlyRequest before // conversion to edgezero types. if req.get_method() == FastlyMethod::GET && req.get_path() == "/_ts/debug/ja4" { - match load_settings_from_config_store(env) { + match load_settings_from_config_store(runtime_stores) { Ok(settings) if settings.debug.ja4_endpoint_enabled => { build_ja4_debug_response(&req).send_to_client(); } @@ -113,19 +125,43 @@ fn edgezero_main(mut req: FastlyRequest, env: &EnvConfig) { return; } - let config_store = match open_trusted_server_config_store(env) { - Ok(cs) => cs, - Err(e) => { - log::error!("failed to open config store: {e}"); - FastlyResponse::from_status(fastly::http::StatusCode::INTERNAL_SERVER_ERROR) - .with_body_text_plain("Internal Server Error") - .send_to_client(); - return; - } - }; + let timings = RequestTimings::new(); - let (app, app_state) = TrustedServerApp::build_app_with_state(env); + let (config_store, app, app_state) = { + let _appbuild = timings.span(Phase::AppBuild); + let config_store = + match open_trusted_server_config_store(runtime_stores.config_store_name.as_ref()) { + Ok(cs) => cs, + Err(e) => { + log::error!("failed to open config store: {e}"); + FastlyResponse::from_status(fastly::http::StatusCode::INTERNAL_SERVER_ERROR) + .with_body_text_plain("Internal Server Error") + .send_to_client(); + return; + } + }; + let (app, app_state) = TrustedServerApp::build_app_with_state(runtime_stores); + (config_store, app, app_state) + }; let settings_snapshot = app_state.as_ref().map(|state| Arc::clone(&state.settings)); + let server_timing_enabled = settings_snapshot + .as_deref() + .is_some_and(|settings| settings.observability.server_timing_enabled); + // Both read once here rather than at each `send_edgezero_response` call + // site: if `app_state` failed to build, there is no settings snapshot to + // read them from at all, so every call site would need the same + // degraded-mode fallback. `access_sample_rate` defaults to `0.0` (never + // sampled in) and `publisher_domain` to `"unknown"` in that case. + let access_sample_rate = settings_snapshot + .as_deref() + .map_or(0.0, |settings| settings.tinybird.access_sample_rate); + let access_telemetry_enabled = settings_snapshot + .as_deref() + .is_some_and(|settings| settings.tinybird.enabled && settings.tinybird.access_enabled); + let publisher_domain = settings_snapshot.as_deref().map_or_else( + || "unknown".to_owned(), + |settings| settings.publisher.domain.clone(), + ); let trusted_client_ip = settings_snapshot .as_deref() .and_then(|settings| settings.trusted_client_ip.as_ref()); @@ -145,6 +181,10 @@ fn edgezero_main(mut req: FastlyRequest, env: &EnvConfig) { req.set_header("fastly-ssl", "1"); } + // Capture the method before dispatch consumes the request. The resolved + // client IP is retained below in `ClientInfo`. + let request_method = req.get_method_str().to_owned(); + // Strip any client-supplied x-ts-tls-* headers before injecting the trusted // values from the Fastly SDK. Must run after sanitize_fastly_forwarded_headers. req.remove_header("x-ts-tls-protocol"); @@ -176,6 +216,7 @@ fn edgezero_main(mut req: FastlyRequest, env: &EnvConfig) { core_req.extensions_mut().insert(config_store); core_req.extensions_mut().insert(device_signals); core_req.extensions_mut().insert(client_info); + core_req.extensions_mut().insert(timings.clone()); match futures::executor::block_on(app.router().oneshot(core_req)) { Ok(response) => response, Err(error) => edge_error_response(error), @@ -194,14 +235,34 @@ fn edgezero_main(mut req: FastlyRequest, env: &EnvConfig) { let ec_state = response.extensions_mut().remove::(); let asset_cache_policy = response.extensions_mut().remove::(); let request_filter_effects = response.extensions_mut().remove::(); + // Read rather than pop: the access-telemetry snapshot built later in + // `send_edgezero_response` reads this same extension, so it must still + // be attached to `response` at that point. + let geo_lookup_state = response + .extensions() + .get::() + .cloned() + .unwrap_or(GeoLookupState::NotAttempted); if !take_finalize_sentinel(&mut response) { if let Some(settings) = settings_snapshot.as_deref() { - apply_entry_point_finalize_headers(settings, &mut response, client_ip); + apply_entry_point_finalize_headers( + settings, + &mut response, + client_ip, + &geo_lookup_state, + &timings, + ); } else { - match load_settings_from_config_store(env) { + match load_settings_from_config_store(runtime_stores) { Ok(settings) => { - apply_entry_point_finalize_headers(&settings, &mut response, client_ip); + apply_entry_point_finalize_headers( + &settings, + &mut response, + client_ip, + &geo_lookup_state, + &timings, + ); } Err(e) => { log::warn!("entry-point finalize skipped: failed to reload settings: {e:?}"); @@ -214,12 +275,24 @@ fn edgezero_main(mut req: FastlyRequest, env: &EnvConfig) { policy.apply_after_route_finalization(&mut response, EdgeCacheHeader::SurrogateControl); } - if let Some(ec_state) = ec_state { + if let Some(mut ec_state) = ec_state { if let Some(settings) = settings_snapshot.as_deref() { - match apply_edgezero_ec_finalize(settings, &ec_state, &mut response) { + match apply_edgezero_ec_finalize(settings, &mut ec_state, &mut response, &timings) { Ok(partner_registry) => { - send_edgezero_response(response, request_filter_effects.as_ref()); + let outcome = send_edgezero_response( + response, + request_filter_effects.as_ref(), + &SendContext { + timings: timings.clone(), + server_timing_enabled, + method: request_method.clone(), + publisher_domain: publisher_domain.clone(), + access_sample_rate, + access_telemetry_enabled, + }, + ); run_edgezero_pull_sync_after_send(settings, &partner_registry, &ec_state); + emit_access_telemetry_after_send(settings, &outcome, &timings); return; } Err(e) => { @@ -229,16 +302,33 @@ fn edgezero_main(mut req: FastlyRequest, env: &EnvConfig) { } } } else { - match load_settings_from_config_store(env) { + match load_settings_from_config_store(runtime_stores) { Ok(settings) => { - match apply_edgezero_ec_finalize(&settings, &ec_state, &mut response) { + match apply_edgezero_ec_finalize( + &settings, + &mut ec_state, + &mut response, + &timings, + ) { Ok(partner_registry) => { - send_edgezero_response(response, request_filter_effects.as_ref()); + let outcome = send_edgezero_response( + response, + request_filter_effects.as_ref(), + &SendContext { + timings: timings.clone(), + server_timing_enabled, + method: request_method.clone(), + publisher_domain: publisher_domain.clone(), + access_sample_rate, + access_telemetry_enabled, + }, + ); run_edgezero_pull_sync_after_send( &settings, &partner_registry, &ec_state, ); + emit_access_telemetry_after_send(&settings, &outcome, &timings); return; } Err(e) => { @@ -255,7 +345,27 @@ fn edgezero_main(mut req: FastlyRequest, env: &EnvConfig) { } } - send_edgezero_response(response, request_filter_effects.as_ref()); + let outcome = send_edgezero_response( + response, + request_filter_effects.as_ref(), + &SendContext { + timings: timings.clone(), + server_timing_enabled, + method: request_method, + publisher_domain, + access_sample_rate, + access_telemetry_enabled, + }, + ); + // The asset/admin/error fallback path: no `EcFinalizeState` (or the ec + // finalize branch above failed), so there is no pull-sync dispatch here + // at all — telemetry is the only post-send step. When `app_state` never + // built there is nothing to emit either: `access_telemetry_enabled` was + // necessarily false without a settings snapshot, so the outcome carries + // no access snapshot, and reloading settings here could not change that. + if let Some(settings) = settings_snapshot.as_deref() { + emit_access_telemetry_after_send(settings, &outcome, &timings); + } } fn edge_error_response(error: EdgeError) -> HttpResponse { @@ -283,30 +393,46 @@ fn apply_entry_point_finalize_headers( settings: &Settings, response: &mut HttpResponse, client_ip: Option, + geo_state: &GeoLookupState, + timings: &RequestTimings, ) { - let geo_info = resolve_geo_for_response(response, client_ip, |client_ip| { + let geo_info = resolve_geo_for_response(response, geo_state, client_ip, |client_ip| { + let _span = timings.span(Phase::Geo); FastlyPlatformGeo.lookup(client_ip).unwrap_or_else(|e| { log::warn!("entry-point geo lookup failed: {e}"); None }) }); apply_finalize_headers(settings, geo_info.as_ref(), response); + + // This path runs only when the middleware chain was bypassed (e.g. a + // router-level 404/405 for an unregistered method), so `geo_state` may + // still be `NotAttempted` even after a fresh lookup just ran above. + // Write the resolved outcome back so the access-telemetry snapshot built + // later in `send_edgezero_response` sees what was actually looked up, + // not the stale carried-in state. + let resolved_state = match &geo_info { + Some(info) => GeoLookupState::Resolved(info.clone()), + None => GeoLookupState::Attempted, + }; + response.extensions_mut().insert(resolved_state); } fn apply_edgezero_ec_finalize( settings: &Settings, - ec_state: &EcFinalizeState, + ec_state: &mut EcFinalizeState, response: &mut HttpResponse, + timings: &RequestTimings, ) -> Result> { let partner_registry = PartnerRegistry::from_config(&settings.ec.partners)?; let finalize_kv_graph = if ec_state.use_finalize_kv { - maybe_identity_graph(settings) + identity_graph_with_timing(settings, timings) } else { None }; ec_finalize_response( settings, - &ec_state.ec_context, + &mut ec_state.ec_context, finalize_kv_graph.as_ref(), &partner_registry, ec_state.eids_cookie.as_deref(), @@ -328,6 +454,219 @@ fn run_edgezero_pull_sync_after_send( } } +/// Builds and emits the access-telemetry row for one delivered response, +/// when access telemetry is enabled and this request is sampled in. +/// +/// Called last at every `send_edgezero_response` call site in +/// [`edgezero_main`] — after `run_edgezero_pull_sync_after_send` on the two +/// EC-finalized paths, and directly after send on the asset/admin/error +/// fallback path, which never builds an [`EcFinalizeState`] or route-scoped +/// `RuntimeServices` at all. The Tinybird transport context is therefore +/// constructed fresh from `settings` here rather than threaded through +/// either of those per-route types, so every response class can emit. +/// +/// Sampled-out requests return silently — that is the expected, high-volume +/// case and not worth a log line. The sampling roll uses the rate stored on +/// the snapshot itself, so the emission probability always matches the +/// row's `sample_rate` column by construction. Every other drop (row +/// build, token load, send, or non-2xx status — all folded into +/// `emit_access_event`'s `Result`) logs exactly one warning naming the +/// reason. +fn emit_access_telemetry_after_send( + settings: &Settings, + outcome: &DeliveryOutcome, + timings: &RequestTimings, +) { + if !settings.tinybird.enabled || !settings.tinybird.access_enabled { + return; + } + + // No snapshot means access telemetry was disabled when the response + // was sent (the flag is read once, before dispatch); nothing to emit. + let Some(snapshot) = &outcome.snapshot else { + return; + }; + + let since_epoch = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default(); + let epoch_ms = u64::try_from(since_epoch.as_millis()).unwrap_or(u64::MAX); + // Sample with the rate stored on the snapshot itself — the same value + // serialized into the row's `sample_rate` column — so the emission + // probability and the row's claimed rate cannot diverge, which the + // documented `sum(1.0 / sample_rate)` volume estimator depends on. + let roll = rand::thread_rng().r#gen::(); + if !tinybird::sampled_in(snapshot.sample_rate, roll) { + return; + } + + let row = access_event_row(snapshot, &timings.snapshot(), epoch_ms); + let target = tinybird::TinybirdEventsTarget::from_access_config(settings.tinybird.clone()); + let result = futures::executor::block_on(tinybird::emit_access_event( + &platform::FastlyPlatformHttpClient, + &target, + row, + )); + if let Err(error) = result { + log::warn!("access telemetry emission dropped: {error:?}"); + } +} + +/// Per-response context threaded into [`send_edgezero_response`] so the +/// function stays at or under seven parameters. +struct SendContext { + /// The request's phase-timing collector. + timings: RequestTimings, + /// Whether `observability.server_timing_enabled` is set. + server_timing_enabled: bool, + /// The request's HTTP method, captured before the request was consumed + /// by dispatch. + method: String, + /// The configured publisher domain. + publisher_domain: String, + /// The configured access-telemetry sample rate. + access_sample_rate: f64, + /// Whether `tinybird.enabled` and `tinybird.access_enabled` were both + /// set when settings were first read. Gates building the + /// [`AccessTelemetrySnapshot`] at all: the snapshot costs env reads and + /// `String` allocations on the pre-send path, which a disabled + /// deployment (the default) should not pay. + access_telemetry_enabled: bool, +} + +/// Outcome of handing a finalized response to the client. +pub(crate) struct DeliveryOutcome { + /// Response body size in bytes. + #[allow(dead_code)] + pub bytes: u64, + /// Whether delivery completed or failed partway. Collected as + /// groundwork; not yet emitted on any surface. + #[allow(dead_code)] + pub result: DeliveryResult, + /// Access-telemetry dimensions captured for this response at the + /// freeze point. `None` when access telemetry was disabled at snapshot + /// time; the emitter treats that as nothing to send. + pub snapshot: Option, +} + +/// Whether [`send_edgezero_response`] completed delivery or failed partway. +#[derive(Debug, PartialEq, Eq)] +pub(crate) enum DeliveryResult { + /// The response was handed to the client in full. + Complete, + /// Delivery started but did not finish cleanly: some bytes reached the + /// client's transport before a stream error, or the transport could not + /// be closed cleanly after every byte was written. + Partial, + /// Delivery failed before any bytes reached the client. + Error, +} + +/// Thin Fastly-adapter wrapper around +/// [`append_server_timing_if_private`], the freeze point shared with the +/// Axum adapter's terminal timing layer. See that function's doc for the +/// emission rules (always stamps `mark_headers_ready`; appends rather than +/// overwrites; never promotes a response to shared-cacheable). +pub(crate) fn apply_server_timing_header( + response: &mut HttpResponse, + timings: &RequestTimings, + server_timing_enabled: bool, +) { + append_server_timing_if_private(response, timings, server_timing_enabled); +} + +/// A [`Write`](std::io::Write) wrapper that tallies bytes successfully written +/// to the inner writer. +/// +/// Wraps the client transport during a streaming drive so a truncated or +/// failed drive still reports how many bytes actually reached it, instead of +/// the placeholder `0` a failed/aborted drive would otherwise report. +struct CountingWriter { + inner: W, + bytes: u64, +} + +impl CountingWriter { + fn new(inner: W) -> Self { + Self { inner, bytes: 0 } + } + + /// Bytes successfully written to the inner writer so far. + fn bytes(&self) -> u64 { + self.bytes + } + + fn into_inner(self) -> W { + self.inner + } +} + +impl std::io::Write for CountingWriter { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + let written = self.inner.write(buf)?; + self.bytes = self.bytes.saturating_add(written as u64); + Ok(written) + } + + fn flush(&mut self) -> std::io::Result<()> { + self.inner.flush() + } +} + +/// Drives a streaming `EdgeZero` body through `output`, tallying bytes written +/// and timing the drive into `timings`. +/// +/// Stamps `resp_bytes` and `request_elapsed` immediately once the drive +/// returns — before the caller does anything transport-specific (finishing +/// the streaming body, logging) — so `request_elapsed` never includes that +/// work. Returns the counting writer (so the caller can recover both the +/// tallied byte count and the wrapped transport) alongside the drive's +/// result. +fn drive_streaming_body( + body: EdgeBody, + output: W, + timings: &RequestTimings, +) -> (CountingWriter, Result<(), Report>) { + let mut counting = CountingWriter::new(output); + let drive_started = Instant::now(); + let result = futures::executor::block_on(stream_asset_body(body, &mut counting)); + timings.record(Phase::Stream, drive_started.elapsed()); + timings.set_resp_bytes(counting.bytes()); + timings.mark_request_elapsed(); + (counting, result) +} + +/// Classifies a completed streaming drive into a [`DeliveryResult`]. +/// +/// A drive that failed after writing at least one byte delivered a truncated +/// response rather than nothing at all, so it is [`DeliveryResult::Partial`], +/// not [`DeliveryResult::Error`]. +/// +/// The `Ok(())` arm exists for the classifier's totality, not for the +/// production caller: `send_edgezero_response` consumes this value only in +/// its `Err` branch and re-derives the success outcome from +/// `streaming_body.finish()`. +fn classify_stream_delivery( + drive_result: &Result<(), Report>, + bytes: u64, +) -> DeliveryResult { + match drive_result { + Ok(()) => DeliveryResult::Complete, + Err(_) if bytes > 0 => DeliveryResult::Partial, + Err(_) => DeliveryResult::Error, + } +} + +/// Stamps `resp_bytes`/`request_elapsed` for an already-materialized body, +/// immediately before it is handed to the Fastly client transport, and +/// returns its byte length. +fn record_buffered_delivery(body: &EdgeBody, timings: &RequestTimings) -> u64 { + let bytes = u64::try_from(body.as_bytes().map(<[u8]>::len).unwrap_or(0)).unwrap_or(u64::MAX); + timings.set_resp_bytes(bytes); + timings.mark_request_elapsed(); + bytes +} + /// Sends a finalized `EdgeZero` response to the client. /// /// Streaming `EdgeZero` bodies commit headers first, then pipe chunks to Fastly's @@ -336,8 +675,24 @@ fn run_edgezero_pull_sync_after_send( fn send_edgezero_response( mut response: HttpResponse, request_filter_effects: Option<&RequestFilterEffects>, -) { + context: &SendContext, +) -> DeliveryOutcome { apply_terminal_response_effects(&mut response, request_filter_effects); + apply_server_timing_header( + &mut response, + &context.timings, + context.server_timing_enabled, + ); + + // Built right after the freeze point and before `into_parts()` + // consumes `response`: nothing else survives to post-send on every + // path (the request was consumed by dispatch, and `EcFinalizeState` + // is absent on asset, admin, and error paths). Skipped entirely when + // access telemetry is disabled, so the default configuration pays no + // env reads or allocations here. + let snapshot = context + .access_telemetry_enabled + .then(|| build_access_telemetry_snapshot(&response, context)); let (parts, body) = response.into_parts(); @@ -347,25 +702,133 @@ fn send_edgezero_response( parts, EdgeBody::empty(), )); - let mut streaming_body = skeleton.stream_to_client(); - match futures::executor::block_on(stream_asset_body(body, &mut streaming_body)) { - Ok(()) => { - if let Err(e) = streaming_body.finish() { + let (counting, drive_result) = + drive_streaming_body(body, skeleton.stream_to_client(), &context.timings); + let bytes = counting.bytes(); + let streaming_body = counting.into_inner(); + // Computed before `drive_result` is matched by value below, since + // the `Err` arm there moves its `Report` out. + let result = classify_stream_delivery(&drive_result, bytes); + match drive_result { + Ok(()) => match streaming_body.finish() { + Ok(()) => DeliveryOutcome { + bytes, + result: DeliveryResult::Complete, + snapshot, + }, + Err(e) => { + // Every byte was handed to the transport (the drive + // above returned Ok), but the transport itself could + // not close cleanly — the client may still see a + // truncated response. log::error!("failed to finish EdgeZero streaming body: {e}"); + DeliveryOutcome { + bytes, + result: DeliveryResult::Partial, + snapshot, + } } - } + }, Err(e) => { log::error!("EdgeZero streaming failed: {e:?}"); drop(streaming_body); + DeliveryOutcome { + bytes, + result, + snapshot, + } } } } once => { + let bytes = record_buffered_delivery(&once, &context.timings); compat::to_fastly_response(HttpResponse::from_parts(parts, once)).send_to_client(); + DeliveryOutcome { + bytes, + result: DeliveryResult::Complete, + snapshot, + } } } } +/// Builds the [`AccessTelemetrySnapshot`] for `response` at the +/// `Server-Timing` freeze point. +/// +/// Reads route identity, geo country, and template-cache state from typed +/// response extensions rather than the headers those extensions back — +/// operator-configured response headers can override a managed header, so +/// reading a header here could silently drift from what actually happened. +/// Falls back to `"unknown"`/[`RouteClass::Other`] sentinels when an +/// extension was never attached (router-generated, asset, and other +/// responses that never passed through a `RouteMetadata`-attaching +/// wrapper). +fn build_access_telemetry_snapshot( + response: &HttpResponse, + context: &SendContext, +) -> AccessTelemetrySnapshot { + let (route_class, route_template) = match response.extensions().get::() { + Some(metadata) => (metadata.route_class, metadata.route_template.clone()), + None => (RouteClass::Other, "unknown".to_owned()), + }; + + let country = match response.extensions().get::() { + Some(GeoLookupState::Resolved(info)) => info.country.clone(), + Some(GeoLookupState::Attempted | GeoLookupState::NotAttempted) | None => { + "unknown".to_owned() + } + }; + + let template_cache_state = response + .extensions() + .get::() + .map_or_else(|| "unknown".to_owned(), |state| state.as_str().to_owned()); + + let body_mode = if matches!(response.body(), EdgeBody::Stream(_)) { + "streamed" + } else { + "buffered" + }; + + AccessTelemetrySnapshot { + method: context.method.clone(), + status: response.status().as_u16(), + route_class, + route_template, + publisher_domain: context.publisher_domain.clone(), + env: resolve_env_dimension(), + service_id: env_var_or_unknown(ENV_FASTLY_SERVICE_ID), + pop: env_var_or_unknown(ENV_FASTLY_POP), + ts_version: env_var_or_unknown(ENV_FASTLY_SERVICE_VERSION), + country, + template_cache_state, + body_mode, + sample_rate: context.access_sample_rate, + } +} + +/// Derives the `env` access-telemetry dimension from the same +/// `FASTLY_IS_STAGING` input that drives the `x-ts-env` response header +/// (see [`apply_finalize_headers`]), never from [`Settings`] — `Settings` +/// has no environment field and does not gain one for this. +/// +/// `"unknown"` covers contexts where the variable is entirely absent (for +/// example native unit tests run outside Fastly Compute); on the Fastly +/// platform the variable is always present, as either `"1"` or not. +fn resolve_env_dimension() -> String { + match std::env::var(ENV_FASTLY_IS_STAGING) { + Ok(value) if value == "1" => "staging".to_owned(), + Ok(_) => "production".to_owned(), + Err(_) => "unknown".to_owned(), + } +} + +/// Reads a Fastly-provided environment variable, defaulting to `"unknown"` +/// when unset. +fn env_var_or_unknown(name: &str) -> String { + std::env::var(name).unwrap_or_else(|_| "unknown".to_owned()) +} + /// Apply every late response mutation, then restore privacy invariants before headers commit. fn apply_terminal_response_effects( response: &mut HttpResponse, @@ -437,12 +900,21 @@ fn build_ja4_debug_response(req: &FastlyRequest) -> FastlyResponse { .with_body(body) } -pub(crate) fn maybe_identity_graph(settings: &Settings) -> Option { - settings - .ec - .ec_store - .as_ref() - .map(|store_name| KvIdentityGraph::new(FastlyEcKvStore::new(store_name))) +/// Constructs a `KvIdentityGraph` wrapped in the [`Phase::EcKv`] timing +/// decorator, for request-path callers with a `RequestTimings` handle. +/// +/// Returns `None` when `ec.ec_store` is not configured, matching +/// [`require_identity_graph_with_timing`]'s contract on every other axis. +pub(crate) fn identity_graph_with_timing( + settings: &Settings, + timings: &RequestTimings, +) -> Option { + settings.ec.ec_store.as_ref().map(|store_name| { + KvIdentityGraph::new(TimedKvStore::new( + FastlyEcKvStore::new(store_name), + timings.clone(), + )) + }) } fn run_pull_sync_after_send( @@ -465,6 +937,12 @@ fn run_pull_sync_after_send( /// Constructs a `KvIdentityGraph` from settings, or returns an error if the /// `ec_store` config is not set. +/// +/// Deliberately untimed: pull-sync (this function's only caller) runs after +/// `send_edgezero_response`'s Server-Timing freeze point, so a decorated +/// store here would record into a handle nothing ever renders. +/// Request-path callers with a `RequestTimings` handle use +/// [`require_identity_graph_with_timing`] instead. pub(crate) fn require_identity_graph( settings: &Settings, ) -> Result> { @@ -477,6 +955,27 @@ pub(crate) fn require_identity_graph( Ok(KvIdentityGraph::new(FastlyEcKvStore::new(store_name))) } +/// Constructs a `KvIdentityGraph` wrapped in the [`Phase::EcKv`] timing +/// decorator, or returns an error if the `ec_store` config is not set. +/// +/// Request-path sibling of [`require_identity_graph`], which pull-sync uses +/// unwrapped because pull-sync runs after the Server-Timing freeze point. +pub(crate) fn require_identity_graph_with_timing( + settings: &Settings, + timings: &RequestTimings, +) -> Result> { + let store_name = settings.ec.ec_store.as_deref().ok_or_else(|| { + Report::new(TrustedServerError::KvStore { + store_name: "ec.ec_store".to_owned(), + message: "ec.ec_store is not configured".to_owned(), + }) + })?; + Ok(KvIdentityGraph::new(TimedKvStore::new( + FastlyEcKvStore::new(store_name), + timings.clone(), + ))) +} + /// Extracts a named cookie value from the request's `Cookie` header. pub(crate) fn extract_cookie_value(req: &HttpRequest, name: &str) -> Option { let cookie_header = req.headers().get("cookie").and_then(|v| v.to_str().ok())?; @@ -506,11 +1005,14 @@ pub(crate) fn derive_device_signals(req: &FastlyRequest) -> DeviceSignals { #[cfg(test)] mod tests { use super::*; + use base64::Engine as _; use edgezero_core::body::Body as EdgeBody; use edgezero_core::http::HeaderValue; use edgezero_core::http::response_builder; use fastly::mime; + use std::time::Duration; use trusted_server_core::integrations::HeaderMutation; + use trusted_server_core::request_timing::AuctionWaitPlacement; fn test_settings() -> Settings { Settings::from_toml( @@ -538,6 +1040,26 @@ mod tests { .expect("should parse test settings") } + /// A minimal [`AccessTelemetrySnapshot`] fixture for tests that only + /// need a `DeliveryOutcome` to exist, not its telemetry content. + fn sample_access_snapshot() -> AccessTelemetrySnapshot { + AccessTelemetrySnapshot { + method: "GET".to_owned(), + status: 200, + route_class: RouteClass::Other, + route_template: "/other/*".to_owned(), + publisher_domain: "unknown".to_owned(), + env: "unknown".to_owned(), + service_id: "unknown".to_owned(), + pop: "unknown".to_owned(), + ts_version: "unknown".to_owned(), + country: "unknown".to_owned(), + template_cache_state: "unknown".to_owned(), + body_mode: "buffered", + sample_rate: 0.0, + } + } + #[test] fn health_response_short_circuits_get_health() { let req = FastlyRequest::get("https://example.com/health"); @@ -773,9 +1295,10 @@ mod tests { .body(EdgeBody::empty()) .expect("should build response"); - let geo_info = resolve_geo_for_response(&response, None, |_| { - panic!("should skip entry-point geo lookup for 401 responses"); - }); + let geo_info = + resolve_geo_for_response(&response, &GeoLookupState::NotAttempted, None, |_| { + panic!("should skip entry-point geo lookup for 401 responses"); + }); apply_finalize_headers(&settings, geo_info.as_ref(), &mut response); assert_eq!( @@ -841,4 +1364,408 @@ mod tests { "should include sec-ch-ua-platform fallback" ); } + + fn ec_finalize_settings() -> Settings { + Settings::from_toml( + r#" + [[handlers]] + path = "^/_ts/admin" + username = "admin" + password = "admin-pass" + + [publisher] + domain = "test-publisher.com" + cookie_domain = ".test-publisher.com" + origin_url = "https://origin.test-publisher.com" + proxy_secret = "unit-test-proxy-secret" + + [ec] + passphrase = "test-secret-key-32-bytes-minimum" + ec_store = "ec_identity_store" + + [[ec.partners]] + name = "Example Partner" + source_domain = "example.com" + api_token = "test-vendor-token-32-bytes-minimum" + + [request_signing] + enabled = false + config_store_id = "test-config-store-id" + secret_store_id = "test-secret-store-id" + "#, + ) + .expect("should parse EC finalize test settings") + } + + /// Minimal `RuntimeServices` for `EcFinalizeState.services`. Real + /// `FastlyPlatform*` handles are used as inert placeholders: EC + /// finalization never calls through them, it only satisfies the field. + fn inert_runtime_services() -> RuntimeServices { + RuntimeServices::builder() + .config_store(Arc::new(crate::platform::FastlyPlatformConfigStore)) + .secret_store(Arc::new(crate::platform::FastlyPlatformSecretStore)) + .kv_store(Arc::new(edgezero_core::key_value_store::NoopKvStore) + as Arc) + .backend(Arc::new(crate::platform::FastlyPlatformBackend)) + .http_client(Arc::new(crate::platform::FastlyPlatformHttpClient)) + .geo(Arc::new(crate::platform::FastlyPlatformGeo)) + .client_info(trusted_server_core::platform::ClientInfo::default()) + .build() + } + + #[test] + fn ec_finalize_kv_lands_before_freeze() { + // A pre-seeded EC entry (see fastly.toml's ec_identity_store fixture) + // for a returning user carrying an eids cookie that matches the + // configured partner. This drives ec_finalize_response into + // ingest_eid_cookies, which reads and writes the KV identity graph. + let settings = ec_finalize_settings(); + let ec_id = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.test01"; + let eids = serde_json::json!([{ + "source": "example.com", + "uids": [{ "id": "example-uid", "atype": 1 }] + }]); + let eids_cookie = base64::engine::general_purpose::STANDARD.encode(eids.to_string()); + let request = edgezero_core::http::request_builder() + .method(fastly::http::Method::GET) + .uri("https://test-publisher.com/article") + .header("cookie", format!("ts-ec={ec_id}; ts-eids={eids_cookie}")) + .body(EdgeBody::empty()) + .expect("should build EC finalize test request"); + + let services = inert_runtime_services(); + let geo_info = trusted_server_core::platform::GeoInfo { + city: String::new(), + country: "US".to_owned(), + continent: "NorthAmerica".to_owned(), + latitude: 0.0, + longitude: 0.0, + metro_code: 0, + region: None, + asn: None, + }; + let ec_context = trusted_server_core::ec::EcContext::read_from_request_with_geo( + &settings, + &request, + &services, + Some(&geo_info), + ) + .expect("should read EC context from a non-regulated request"); + assert!( + ec_context.ec_was_present(), + "the pre-seeded ts-ec cookie should be recognized" + ); + + let mut ec_state = EcFinalizeState { + ec_context, + use_finalize_kv: true, + eids_cookie: Some(eids_cookie), + sharedid_cookie: None, + is_real_browser: true, + services, + }; + let mut response = response_builder() + .header("cache-control", "private, no-store") + .body(EdgeBody::empty()) + .expect("should build EC finalize response fixture"); + let timings = RequestTimings::new(); + + // Mirrors edgezero_main's ordering: EC finalize runs, then the freeze + // point (apply_server_timing_header, called just before + // response.into_parts() inside send_edgezero_response) renders the + // header. Calling both directly exercises exactly this order without + // requiring a live Fastly client connection. + apply_edgezero_ec_finalize(&settings, &mut ec_state, &mut response, &timings) + .expect("should finalize EC response"); + apply_server_timing_header(&mut response, &timings, true); + + let header = response + .headers() + .get("server-timing") + .and_then(|v| v.to_str().ok()) + .expect("should emit a Server-Timing header"); + assert!( + header.contains("ts-kv"), + "the freeze point must run after EC finalization recorded KV time: {header}" + ); + } + + #[test] + fn delivery_outcome_reports_bytes_and_request_elapsed_set() { + let timings = RequestTimings::new(); + let body = EdgeBody::stream(futures::stream::iter(vec![ + bytes::Bytes::from_static(b"hello "), + bytes::Bytes::from_static(b"world"), + ])); + + let (counting, drive_result) = drive_streaming_body(body, Vec::new(), &timings); + drive_result.expect("streaming a well-formed body should not fail"); + let bytes = counting.bytes(); + let outcome = DeliveryOutcome { + bytes, + result: DeliveryResult::Complete, + snapshot: Some(sample_access_snapshot()), + }; + + assert_eq!( + counting.into_inner(), + b"hello world", + "should write every byte to the underlying transport" + ); + assert_eq!( + outcome.bytes, + "hello world".len() as u64, + "DeliveryOutcome.bytes should equal the streamed body length" + ); + + let snapshot = timings.snapshot(); + assert_eq!( + snapshot.resp_bytes, + Some("hello world".len() as u64), + "should stamp resp_bytes to the tallied byte count" + ); + assert!( + snapshot.request_elapsed_ms.is_some(), + "should stamp request_elapsed once the drive returns" + ); + } + + #[test] + fn buffered_delivery_stamps_bytes_and_request_elapsed() { + let timings = RequestTimings::new(); + let body = EdgeBody::from(b"a buffered body".to_vec()); + + let bytes = record_buffered_delivery(&body, &timings); + + assert_eq!( + bytes, + "a buffered body".len() as u64, + "should report the buffered body length" + ); + let snapshot = timings.snapshot(); + assert_eq!( + snapshot.resp_bytes, + Some("a buffered body".len() as u64), + "should stamp resp_bytes for the buffered path too" + ); + assert!( + snapshot.request_elapsed_ms.is_some(), + "should stamp request_elapsed for the buffered path too" + ); + } + + fn send_context_fixture() -> SendContext { + SendContext { + timings: RequestTimings::new(), + server_timing_enabled: false, + method: "GET".to_owned(), + publisher_domain: "test-publisher.com".to_owned(), + access_sample_rate: 0.25, + access_telemetry_enabled: true, + } + } + + #[test] + fn access_snapshot_defaults_when_no_extensions_are_attached() { + // Router-generated 404/405 responses and other paths that never pass + // through a RouteMetadata-attaching wrapper must still produce a + // usable snapshot: RouteClass::Other and "unknown" sentinels, never + // a missing/panicking build. + let response = response_builder() + .status(404) + .body(EdgeBody::empty()) + .expect("should build response"); + let context = send_context_fixture(); + + let snapshot = build_access_telemetry_snapshot(&response, &context); + + assert_eq!(snapshot.status, 404); + assert_eq!(snapshot.method, "GET"); + assert!(matches!(snapshot.route_class, RouteClass::Other)); + assert_eq!(snapshot.route_template, "unknown"); + assert_eq!(snapshot.country, "unknown"); + assert_eq!(snapshot.template_cache_state, "unknown"); + assert_eq!(snapshot.body_mode, "buffered"); + assert_eq!(snapshot.publisher_domain, "test-publisher.com"); + assert_eq!(snapshot.sample_rate, 0.25); + } + + #[test] + fn access_snapshot_reads_route_geo_and_template_cache_extensions() { + let mut response = response_builder() + .status(200) + .body(EdgeBody::empty()) + .expect("should build response"); + response.extensions_mut().insert(RouteMetadata { + route_class: RouteClass::AuctionApi, + route_template: "/auction".to_owned(), + }); + response.extensions_mut().insert(GeoLookupState::Resolved( + trusted_server_core::platform::GeoInfo { + city: String::new(), + country: "US".to_owned(), + continent: "NorthAmerica".to_owned(), + latitude: 0.0, + longitude: 0.0, + metro_code: 0, + region: None, + asn: None, + }, + )); + response + .extensions_mut() + .insert(TemplateCacheResponseState::Hit); + let context = send_context_fixture(); + + let snapshot = build_access_telemetry_snapshot(&response, &context); + + assert!(matches!(snapshot.route_class, RouteClass::AuctionApi)); + assert_eq!(snapshot.route_template, "/auction"); + assert_eq!(snapshot.country, "US"); + assert_eq!(snapshot.template_cache_state, "hit"); + } + + #[test] + fn access_snapshot_treats_attempted_geo_lookup_as_unknown_country() { + let mut response = response_builder() + .status(200) + .body(EdgeBody::empty()) + .expect("should build response"); + response.extensions_mut().insert(GeoLookupState::Attempted); + let context = send_context_fixture(); + + let snapshot = build_access_telemetry_snapshot(&response, &context); + + assert_eq!( + snapshot.country, "unknown", + "an attempted-but-unresolved lookup must not surface a stale country" + ); + } + + #[test] + fn access_snapshot_body_mode_reflects_the_response_body_variant() { + let streamed = response_builder() + .status(200) + .body(EdgeBody::stream(futures::stream::empty())) + .expect("should build streaming response"); + let buffered = response_builder() + .status(200) + .body(EdgeBody::from(b"hi".to_vec())) + .expect("should build buffered response"); + let context = send_context_fixture(); + + assert_eq!( + build_access_telemetry_snapshot(&streamed, &context).body_mode, + "streamed" + ); + assert_eq!( + build_access_telemetry_snapshot(&buffered, &context).body_mode, + "buffered" + ); + } + + #[test] + fn stream_drive_records_stream_ms_covering_the_in_stream_auction_wait() { + // A streaming seam wait (Task 6, publisher.rs) records into the same + // `RequestTimings` handle the adapter drives with. `Phase::Stream` + // wraps the entire drive, so it must cover — and therefore be at + // least as large as — any `AuctionWait` recorded while the body was + // being polled. + let timings = RequestTimings::new(); + let wait_timings = timings.clone(); + let stream = futures::stream::once(async move { + let waited = Duration::from_millis(5); + std::thread::sleep(waited); + wait_timings.record_auction_wait(AuctionWaitPlacement::InStream, waited); + bytes::Bytes::from_static(b"") + }); + let body = EdgeBody::stream(stream); + + let (_counting, drive_result) = drive_streaming_body(body, Vec::new(), &timings); + drive_result.expect("streaming a well-formed body should not fail"); + + let snapshot = timings.snapshot(); + assert_eq!( + snapshot.auction_wait_placement, + Some(AuctionWaitPlacement::InStream), + "should preserve the placement recorded from inside the polled body" + ); + let auction_wait_ms = snapshot + .auction_wait_ms + .expect("should record the auction wait"); + let stream_ms = snapshot.stream_ms.expect("should record the stream drive"); + assert!( + stream_ms >= auction_wait_ms, + "the drive's Phase::Stream span must cover the in-stream auction wait: \ + stream_ms={stream_ms} auction_wait_ms={auction_wait_ms}" + ); + } + + #[test] + fn classify_stream_delivery_treats_bytes_written_before_an_error_as_partial() { + let err = Report::new(TrustedServerError::Proxy { + message: "boom".to_string(), + }); + assert_eq!( + classify_stream_delivery(&Err(err), 42), + DeliveryResult::Partial, + "bytes already on the wire before a stream error is a truncated delivery" + ); + } + + #[test] + fn classify_stream_delivery_treats_an_error_with_no_bytes_as_error() { + let err = Report::new(TrustedServerError::Proxy { + message: "boom".to_string(), + }); + assert_eq!( + classify_stream_delivery(&Err(err), 0), + DeliveryResult::Error, + "a failure before any byte reached the client is a clean failure, not a truncation" + ); + } + + #[test] + fn classify_stream_delivery_treats_ok_as_complete() { + assert_eq!( + classify_stream_delivery(&Ok(()), 123), + DeliveryResult::Complete + ); + } + + #[test] + fn request_elapsed_is_stamped_when_send_returns() { + // `edgezero_main`'s post-send ordering (pull-sync before telemetry) + // is a source-order invariant with no injectable seam, so this test + // deliberately proves only the leg that has one: by the time + // `send_edgezero_response` returns, `request_elapsed` is already + // stamped, so everything `edgezero_main` runs afterwards (pull-sync, + // telemetry emission) is excluded from `request_elapsed_ms`. + let timings = RequestTimings::new(); + let response = response_builder() + .body(EdgeBody::from("ok")) + .expect("should build response"); + + let outcome = send_edgezero_response( + response, + None, + &SendContext { + timings: timings.clone(), + server_timing_enabled: false, + method: "GET".to_owned(), + publisher_domain: "test-publisher.com".to_owned(), + access_sample_rate: 1.0, + access_telemetry_enabled: true, + }, + ); + + assert!( + timings.snapshot().request_elapsed_ms.is_some(), + "request_elapsed should be stamped by the time send returns" + ); + assert!( + outcome.snapshot.is_some(), + "the access snapshot should exist for the enabled context" + ); + } } diff --git a/crates/trusted-server-adapter-fastly/src/middleware.rs b/crates/trusted-server-adapter-fastly/src/middleware.rs index 283f16255..79ded8362 100644 --- a/crates/trusted-server-adapter-fastly/src/middleware.rs +++ b/crates/trusted-server-adapter-fastly/src/middleware.rs @@ -24,8 +24,9 @@ use trusted_server_core::constants::{ ENV_FASTLY_IS_STAGING, ENV_FASTLY_SERVICE_VERSION, HEADER_X_GEO_INFO_AVAILABLE, HEADER_X_TS_ENV, HEADER_X_TS_VERSION, }; -use trusted_server_core::geo::GeoInfo; +use trusted_server_core::geo::{GeoInfo, GeoLookupState}; use trusted_server_core::platform::{ClientInfo, PlatformGeo}; +use trusted_server_core::request_timing::{Phase, RequestTimings}; use trusted_server_core::settings::Settings; pub(crate) const HEADER_X_TS_FINALIZED: &str = "x-ts-finalized"; @@ -71,6 +72,12 @@ impl Middleware for FinalizeResponseMiddleware { || FastlyRequestContext::get(ctx.request()).and_then(|c| c.client_ip), |info| info.client_ip, ); + let timings = ctx + .request() + .extensions() + .get::() + .cloned() + .unwrap_or_default(); let mut response = match next.run(ctx).await { Ok(r) => r, @@ -80,13 +87,31 @@ impl Middleware for FinalizeResponseMiddleware { } }; - let geo_info = resolve_geo_for_response(&response, client_ip, |ip| { + let carried = response + .extensions() + .get::() + .cloned() + .unwrap_or(GeoLookupState::NotAttempted); + let geo_info = resolve_geo_for_response(&response, &carried, client_ip, |ip| { + let _span = timings.span(Phase::Geo); self.geo.lookup(ip).unwrap_or_else(|e| { log::warn!("geo lookup failed: {e}"); None }) }); + // Write the resolved outcome back so a downstream access-telemetry + // snapshot (built from response extensions after finalize) sees + // what was actually looked up here rather than the stale carried-in + // state — mirrors the entry-point finalize site in `main.rs` + // (`apply_entry_point_finalize_headers`), which writes back for the + // same reason. + let resolved_state = match &geo_info { + Some(geo) => GeoLookupState::Resolved(geo.clone()), + None => GeoLookupState::Attempted, + }; + response.extensions_mut().insert(resolved_state); + apply_finalize_headers(&self.settings, geo_info.as_ref(), &mut response); response .headers_mut() @@ -145,14 +170,20 @@ impl Middleware for AuthMiddleware { // Shared geo resolution helper // --------------------------------------------------------------------------- -/// Resolves geo for a response, skipping the lookup for 401 responses. +/// Resolves geo for a response, skipping the lookup for 401 responses and +/// reusing a request-phase lookup when one was already carried. /// -/// Returns `None` for authentication rejections (401) without calling `lookup_geo` -/// to avoid unnecessary work and exposing geo data to unauthenticated callers. -/// All other responses call `lookup_geo` and return its result. +/// Returns `None` for authentication rejections (401) without consulting +/// `carried` or calling `lookup_geo`, to avoid unnecessary work and exposing +/// geo data to unauthenticated callers. Otherwise dispatches on `carried`: +/// a [`GeoLookupState::Resolved`] value is reused as-is, a +/// [`GeoLookupState::Attempted`] value is treated as no geo info without +/// retrying the lookup, and [`GeoLookupState::NotAttempted`] falls back to +/// calling `lookup_geo`. /// /// Used by both [`FinalizeResponseMiddleware`] and the entry-point finalization -/// in `main.rs` so the 401-skip rule is defined in one place. +/// in `main.rs` so the 401-skip rule and the dedupe rule are each defined in +/// one place. /// /// # Parity note /// @@ -164,6 +195,7 @@ impl Middleware for AuthMiddleware { /// server or the upstream origin. pub(crate) fn resolve_geo_for_response( response: &Response, + carried: &GeoLookupState, client_ip: Option, lookup_geo: F, ) -> Option @@ -171,9 +203,12 @@ where F: FnOnce(Option) -> Option, { if response.status() == StatusCode::UNAUTHORIZED { - None - } else { - lookup_geo(client_ip) + return None; + } + match carried { + GeoLookupState::Resolved(geo) => Some(geo.clone()), + GeoLookupState::Attempted => None, + GeoLookupState::NotAttempted => lookup_geo(client_ip), } } @@ -280,6 +315,19 @@ mod tests { RequestContext::new(req, PathParams::new(HashMap::new())) } + fn sample_geo_info() -> GeoInfo { + GeoInfo { + city: "Testville".to_string(), + country: "US".to_string(), + continent: "NorthAmerica".to_string(), + latitude: 0.0, + longitude: 0.0, + metro_code: 0, + region: None, + asn: None, + } + } + struct FixedGeo(Option); impl PlatformGeo for FixedGeo { @@ -682,6 +730,67 @@ mod tests { ); } + #[test] + fn finalize_handle_writes_back_resolved_geo_state_after_fallback_lookup() { + // The request phase never attempted a geo lookup (no GeoLookupState + // extension on the handler's response), so the middleware resolves + // one via the fallback closure. That resolved outcome must be + // written back into response extensions -- mirroring + // apply_entry_point_finalize_headers in main.rs -- so a downstream + // access-telemetry snapshot sees the freshly resolved country + // instead of a stale/missing GeoLookupState. + let settings = settings_with_response_headers(vec![]); + let middleware = FinalizeResponseMiddleware::new( + Arc::new(settings), + Arc::new(FixedGeo(Some(sample_geo_info()))), + ); + let handler = + Arc::new( + |_ctx: RequestContext| async move { Ok::(empty_response()) }, + ); + + let response = block_on(middleware.handle(empty_ctx(), Next::new(&[], &*handler))) + .expect("should succeed"); + + match response.extensions().get::() { + Some(GeoLookupState::Resolved(info)) => { + assert_eq!( + info.country, "US", + "should carry the fallback-resolved geo info" + ); + } + other => { + panic!("expected GeoLookupState::Resolved after a fallback lookup, got {other:?}") + } + } + } + + #[test] + fn finalize_handle_writes_back_attempted_geo_state_when_fallback_finds_nothing() { + // The fallback lookup ran but resolved no geo info. The middleware + // must still record that the lookup was attempted, so a later + // consumer of the extension does not mistake this for + // GeoLookupState::NotAttempted and retry the lookup. + let settings = settings_with_response_headers(vec![]); + let middleware = + FinalizeResponseMiddleware::new(Arc::new(settings), Arc::new(FixedGeo(None))); + let handler = + Arc::new( + |_ctx: RequestContext| async move { Ok::(empty_response()) }, + ); + + let response = block_on(middleware.handle(empty_ctx(), Next::new(&[], &*handler))) + .expect("should succeed"); + + assert!( + matches!( + response.extensions().get::(), + Some(GeoLookupState::Attempted) + ), + "should write back Attempted when the fallback lookup finds no geo info" + ); + } + #[test] fn finalize_handle_marks_response_as_finalized() { let settings = settings_with_response_headers(vec![]); @@ -763,6 +872,30 @@ mod tests { ); } + #[test] + #[allow(clippy::panic)] + fn geo_lookup_skipped_for_unauthorized_responses() { + // The 401 short-circuit in resolve_geo_for_response must win + // regardless of what state the request phase carried in, and must + // never invoke the fallback lookup closure. + let mut response = empty_response(); + *response.status_mut() = StatusCode::UNAUTHORIZED; + + for carried in [ + GeoLookupState::NotAttempted, + GeoLookupState::Attempted, + GeoLookupState::Resolved(sample_geo_info()), + ] { + let geo_info = resolve_geo_for_response(&response, &carried, None, |_| { + panic!("401 responses must never trigger a geo lookup"); + }); + assert!( + geo_info.is_none(), + "401 responses should never resolve geo info, regardless of carried state" + ); + } + } + // --------------------------------------------------------------------------- // AuthMiddleware::handle tests // --------------------------------------------------------------------------- diff --git a/crates/trusted-server-adapter-fastly/src/platform.rs b/crates/trusted-server-adapter-fastly/src/platform.rs index 638aed82b..7434bb8f2 100644 --- a/crates/trusted-server-adapter-fastly/src/platform.rs +++ b/crates/trusted-server-adapter-fastly/src/platform.rs @@ -15,11 +15,12 @@ use fastly::{ConfigStore, Request, SecretStore}; use crate::backend::BackendConfig; pub(crate) use trusted_server_core::platform::UnavailableKvStore; use trusted_server_core::platform::{ - ClientInfo, GeoInfo, PlatformBackend, PlatformBackendSpec, PlatformConfigStore, PlatformError, - PlatformGeo, PlatformHttpClient, PlatformHttpRequest, PlatformImageOptimizerCrop, - PlatformImageOptimizerCropMode, PlatformImageOptimizerOptions, PlatformImageOptimizerParams, - PlatformImageOptimizerRegion, PlatformKvStore, PlatformPendingRequest, PlatformResponse, - PlatformSecretStore, PlatformSelectResult, StoreId, StoreName, + BackendNamingPolicy, ClientInfo, GeoInfo, PlatformBackend, PlatformBackendSpec, + PlatformConfigStore, PlatformError, PlatformGeo, PlatformHttpClient, PlatformHttpRequest, + PlatformImageOptimizerCrop, PlatformImageOptimizerCropMode, PlatformImageOptimizerOptions, + PlatformImageOptimizerParams, PlatformImageOptimizerRegion, PlatformKvStore, + PlatformPendingRequest, PlatformResponse, PlatformSecretStore, PlatformSelectResult, StoreId, + StoreName, }; use trusted_server_core::settings::TrustedClientIpConfig; @@ -150,6 +151,11 @@ impl PlatformSecretStore for FastlyPlatformSecretStore { /// timeout → unique name). pub struct FastlyPlatformBackend; +#[cfg(test)] +const TRANSPORT_TIMEOUT_QUANTUM_MS: u32 = 250; +#[cfg(test)] +const SUB_QUANTUM_LADDER_MS: [u32; 4] = [200, 150, 100, 50]; + fn backend_config_from_spec(spec: &PlatformBackendSpec) -> BackendConfig<'_> { BackendConfig::new(&spec.scheme, &spec.host) .port(spec.port) @@ -160,83 +166,15 @@ fn backend_config_from_spec(spec: &PlatformBackendSpec) -> BackendConfig<'_> { .discriminator(spec.discriminator.as_deref()) } -/// Transport-timeout quantum for auction backends (see -/// [`FastlyPlatformBackend::canonicalize_transport_timeout_ms`]). -const TRANSPORT_TIMEOUT_QUANTUM_MS: u32 = 250; - -/// Upper bound of the fine-grained quantum range. -/// -/// Budget-bound values below this ceiling are floored to a -/// [`TRANSPORT_TIMEOUT_QUANTUM_MS`] multiple (the issue #847 behavior for the -/// default 2000 ms auction). At or above it, values snap to the coarse -/// [`TRANSPORT_TIMEOUT_COARSE_LADDER_MS`] instead so the total number of -/// distinct budget-derived buckets stays globally bounded regardless of how -/// large the configured ceiling is. -const TRANSPORT_TIMEOUT_QUANTUM_CEILING_MS: u32 = 2000; - -/// Coarse rungs for budget-bound transport timeouts below one quantum, -/// ordered high to low. -/// -/// Below one quantum, passing the exact wall-clock remainder through would mint -/// a distinct backend name for every millisecond in `1..250`, so the -/// near-exhausted tail alone could exceed Fastly's per-service dynamic backend -/// limit. Snapping to this finite ladder instead bounds the number of -/// budget-derived names an origin can produce. Budgets below the smallest rung -/// round to zero, which callers treat as "budget exhausted — skip the launch". -const SUB_QUANTUM_LADDER_MS: [u32; 4] = [200, 150, 100, 50]; - -/// Coarse rungs for budget-bound transport timeouts at or above the quantum -/// ceiling, ascending. Every rung is a [`TRANSPORT_TIMEOUT_QUANTUM_MS`] -/// multiple. -/// -/// Above [`TRANSPORT_TIMEOUT_QUANTUM_CEILING_MS`], flooring to a 250 ms multiple -/// would let a large configured ceiling (e.g. 60,000 ms) mint hundreds of -/// distinct backend names — recreating the per-service dynamic backend -/// exhaustion this quantization exists to prevent. This fixed, globally finite -/// ladder caps the number of high-budget buckets instead: values are floored to -/// the greatest rung no larger than the remaining budget, and anything above -/// the top rung clamps to it. Rounding down never extends a transport cap past -/// the remaining budget. -/// -/// The rung spacing trades transport window for cardinality: just below a rung -/// the haircut approaches the gap to the rung beneath (worst case ~50%, e.g. a -/// remaining budget of 9,999 ms snaps to 5,000 ms). This is accepted — on the -/// mediator path this value is the effective bound, but a denser ladder would -/// buy back at most half a bucket of transport time at the cost of -/// proportionally more backend names. -const TRANSPORT_TIMEOUT_COARSE_LADDER_MS: [u32; 8] = - [2000, 3000, 5000, 10000, 20000, 30000, 45000, 60000]; - -/// Round a budget-bound transport timeout down to a stable, globally bounded -/// bucket. -/// -/// - At or above [`TRANSPORT_TIMEOUT_QUANTUM_CEILING_MS`], floors to the -/// greatest [`TRANSPORT_TIMEOUT_COARSE_LADDER_MS`] rung no larger than -/// `remaining_ms` (clamping to the top rung above it). -/// - Within the quantum range, floors to a [`TRANSPORT_TIMEOUT_QUANTUM_MS`] -/// multiple. -/// - Below one quantum, snaps down to the greatest [`SUB_QUANTUM_LADDER_MS`] -/// rung no larger than `remaining_ms` (or zero). -fn quantize_transport_timeout_ms(remaining_ms: u32) -> u32 { - if remaining_ms >= TRANSPORT_TIMEOUT_QUANTUM_CEILING_MS { - return TRANSPORT_TIMEOUT_COARSE_LADDER_MS - .into_iter() - .rev() - .find(|&rung| rung <= remaining_ms) - .unwrap_or(TRANSPORT_TIMEOUT_QUANTUM_CEILING_MS); - } - let floored = (remaining_ms / TRANSPORT_TIMEOUT_QUANTUM_MS) * TRANSPORT_TIMEOUT_QUANTUM_MS; - if floored > 0 { - return floored; - } - SUB_QUANTUM_LADDER_MS - .into_iter() - .find(|&rung| rung <= remaining_ms) - .unwrap_or(0) -} - impl PlatformBackend for FastlyPlatformBackend { + fn naming_policy(&self) -> BackendNamingPolicy { + BackendNamingPolicy::Fastly + } + fn predict_name(&self, spec: &PlatformBackendSpec) -> Result> { + // Use the same host normalization as registration. In particular, + // URL-derived IPv6 hosts arrive bracketed, but both forms must predict + // the backend that `ensure` actually registers. backend_config_from_spec(spec) .predict_name() .change_context(PlatformError::Backend) @@ -247,28 +185,6 @@ impl PlatformBackend for FastlyPlatformBackend { .ensure() .change_context(PlatformError::Backend) } - - /// Quantize the transport timeout so budget-derived values do not mint a - /// new dynamic backend name on every request. - /// - /// Fastly embeds the first-byte and between-bytes timeouts in the dynamic - /// backend name (see [`BackendConfig`]) and pools connections per backend - /// name. A per-request wall-clock budget would otherwise defeat that - /// pooling and accumulate registrations toward the per-service dynamic - /// backend limit. - /// - /// A provider's own configured timeout is a constant, so when it is the - /// binding constraint it is returned verbatim — including sub-quantum - /// configured values, which must not be rounded away or the provider could - /// never launch. Only the budget-bound value is snapped to a stable bucket - /// via [`quantize_transport_timeout_ms`]. Rounding down never extends a - /// transport cap past the remaining budget. - fn canonicalize_transport_timeout_ms(&self, remaining_ms: u32, configured_ms: u32) -> u32 { - if remaining_ms >= configured_ms { - return configured_ms; - } - quantize_transport_timeout_ms(remaining_ms) - } } // --------------------------------------------------------------------------- @@ -544,6 +460,14 @@ fn apply_fastly_cache_bypass(request: &mut fastly::Request, bypass_cache: bool) /// - [`select`](PlatformHttpClient::select) downcasts each /// [`PlatformPendingRequest`] back to `fastly::PendingRequest` and calls /// `fastly::http::request::select()`. +/// +/// Fastly's Compute HTTP API sends one request to the named backend and returns +/// the origin response; it has no client-side redirect-follow mode. Consequently +/// each trait call below performs exactly one underlying `.send()` or +/// `.send_async()`, and an original 3xx remains visible to core. The host test +/// environment cannot register a real Fastly backend, so the common +/// `StubHttpClient` driver test records the one-send 3xx behavior while adapter +/// tests cover request conversion and the single-send boundary. pub struct FastlyPlatformHttpClient; #[async_trait::async_trait(?Send)] @@ -552,6 +476,10 @@ impl PlatformHttpClient for FastlyPlatformHttpClient { true } + fn supports_pending_streaming_responses(&self) -> bool { + true + } + async fn send( &self, request: PlatformHttpRequest, @@ -581,17 +509,17 @@ impl PlatformHttpClient for FastlyPlatformHttpClient { return Err(Report::new(PlatformError::HttpClient) .attach("Image Optimizer is not supported with Fastly send_async")); } - if request.stream_response { - return Err(Report::new(PlatformError::HttpClient) - .attach("streaming responses are not supported with Fastly send_async")); - } + let stream_response = request.stream_response; + let request_method = request.request.method().clone(); let bypass_cache = request.bypass_cache; let mut fastly_req = edge_request_to_fastly(request.request)?; apply_fastly_cache_bypass(&mut fastly_req, bypass_cache); let pending = fastly_req .send_async(&backend_name) .change_context(PlatformError::HttpClient)?; - Ok(PlatformPendingRequest::new(pending).with_backend_name(backend_name)) + Ok(PlatformPendingRequest::new(pending) + .with_backend_name(backend_name) + .with_response_handling(stream_response, request_method)) } async fn select( @@ -605,6 +533,14 @@ impl PlatformHttpClient for FastlyPlatformHttpClient { .attach("select called with an empty pending_requests list")); } + if pending_requests + .iter() + .any(PlatformPendingRequest::stream_response) + { + return Err(Report::new(PlatformError::HttpClient) + .attach("stream-marked pending request requires direct wait")); + } + let mut fastly_pending: Vec = Vec::with_capacity(pending_requests.len()); for platform_req in pending_requests { @@ -661,6 +597,33 @@ impl PlatformHttpClient for FastlyPlatformHttpClient { failed_backend_name, }) } + + async fn wait( + &self, + pending: PlatformPendingRequest, + ) -> Result> { + use fastly::http::request::PendingRequest; + + let backend_hint = pending.backend_name().map(str::to_owned); + let stream_response = pending.stream_response(); + let request_is_head = pending.request_method() == Some(&edgezero_core::http::Method::HEAD); + let pending = pending.downcast::().map_err(|pending| { + let backend_name = pending.backend_name().unwrap_or(""); + Report::new(PlatformError::HttpClient).attach(format!( + "PlatformPendingRequest inner type is not fastly::PendingRequest for backend '{backend_name}'" + )) + })?; + let response = pending.wait().change_context(PlatformError::HttpClient)?; + let backend_name = response + .get_backend_name() + .map(str::to_owned) + .or(backend_hint) + .ok_or_else(|| { + Report::new(PlatformError::HttpClient) + .attach("wait: response has no backend name; correlation impossible") + })?; + fastly_response_to_platform(response, backend_name, stream_response, request_is_head) + } } // --------------------------------------------------------------------------- @@ -1136,8 +1099,72 @@ mod tests { ); } + #[test] + fn bracketed_ipv6_predict_name_matches_bare_and_ensured_backend_name() { + let backend = FastlyPlatformBackend; + let bracketed = PlatformBackendSpec { + scheme: "https".to_string(), + host: "[2001:db8::9]".to_string(), + port: Some(8443), + host_header_override: None, + certificate_check: true, + first_byte_timeout: Duration::from_millis(750), + between_bytes_timeout: Duration::from_millis(750), + discriminator: Some("ipv6-provider".to_string()), + }; + let mut bare = bracketed.clone(); + bare.host = "2001:db8::9".to_string(); + + let predicted = backend + .predict_name(&bracketed) + .expect("should predict bracketed IPv6 backend name"); + let bare_predicted = backend + .predict_name(&bare) + .expect("should predict bare IPv6 backend name"); + let ensured = backend + .ensure(&bracketed) + .expect("should register bracketed IPv6 backend"); + + assert_eq!(predicted, bare_predicted); + assert_eq!(predicted, ensured); + } + // --- FastlyPlatformHttpClient ------------------------------------------- + #[test] + fn auction_http_capabilities_are_explicit() { + let client = FastlyPlatformHttpClient; + let capabilities = trusted_server_core::platform::AuctionTargetId::Fastly + .descriptor() + .capabilities(); + assert!(client.supports_concurrent_fanout()); + assert!(capabilities.supports_concurrent_provider_fanout()); + assert!(!client.has_enforceable_total_request_deadline()); + assert!( + !capabilities.has_enforceable_total_request_deadline(), + "first-byte and between-byte timers are not a hard total request deadline" + ); + } + + #[test] + fn response_conversion_preserves_original_redirect_at_single_send_boundary() { + let mut response = fastly::Response::from_status(fastly::http::StatusCode::FOUND); + response.set_header("location", "https://redirect.example/next"); + + let platform = fastly_response_to_platform(response, "origin", false, false) + .expect("should convert redirect response"); + + assert_eq!(platform.response.status().as_u16(), 302); + assert_eq!( + platform + .response + .headers() + .get("location") + .and_then(|value| value.to_str().ok()), + Some("https://redirect.example/next") + ); + } + #[test] fn apply_fastly_cache_bypass_sets_pass_when_enabled() { let mut request = fastly::Request::get("https://example.com/"); @@ -1306,6 +1333,21 @@ mod tests { ); } + #[test] + fn fastly_platform_http_client_rejects_stream_marked_pending_from_select() { + let client = FastlyPlatformHttpClient; + let pending = PlatformPendingRequest::new(42_u32) + .with_backend_name("origin-a") + .with_response_handling(true, edgezero_core::http::Method::GET); + let err = futures::executor::block_on(client.select(vec![pending])) + .expect_err("should reject stream-marked pending handles from select"); + + assert!( + format!("{err:?}").contains("stream-marked pending request"), + "should explain that streaming pendings require direct wait: {err:?}" + ); + } + #[test] fn fastly_platform_http_client_send_returns_error_for_streaming_body() { let client = FastlyPlatformHttpClient; @@ -1357,8 +1399,13 @@ mod tests { } #[test] - fn fastly_platform_http_client_send_async_rejects_stream_response() { + fn fastly_platform_http_client_supports_pending_streaming_responses() { let client = FastlyPlatformHttpClient; + assert!( + client.supports_pending_streaming_responses(), + "should advertise direct pending-response streaming" + ); + let request = request_builder() .method("GET") .uri("https://example.com/image.jpg") @@ -1368,11 +1415,11 @@ mod tests { PlatformHttpRequest::new(request, "nonexistent-backend").with_stream_response(); let err = futures::executor::block_on(client.send_async(platform_request)) - .expect_err("should reject async streaming-response requests"); + .expect_err("should fail only because the backend is unregistered"); assert!( - format!("{err:?}").contains("streaming responses"), - "should explain unsupported async streaming-response path: {err:?}" + !format!("{err:?}").contains("streaming responses are not supported"), + "should accept streaming on the async path before backend dispatch: {err:?}" ); } diff --git a/crates/trusted-server-adapter-fastly/src/tinybird.rs b/crates/trusted-server-adapter-fastly/src/tinybird.rs index f2df61744..85c77e53b 100644 --- a/crates/trusted-server-adapter-fastly/src/tinybird.rs +++ b/crates/trusted-server-adapter-fastly/src/tinybird.rs @@ -11,10 +11,14 @@ use trusted_server_core::auction::telemetry::{ }; use trusted_server_core::error::TrustedServerError; use trusted_server_core::platform::{ - PlatformBackendSpec, PlatformHttpRequest, RuntimeServices, StoreName, + PlatformBackend as _, PlatformBackendSpec, PlatformHttpClient, PlatformHttpRequest, + RuntimeServices, }; +use trusted_server_core::redacted::Redacted; use trusted_server_core::settings::{Settings, TinybirdSettings}; +use crate::platform::FastlyPlatformBackend; + const TINYBIRD_EVENTS_PATH: &str = "/v0/events"; const TINYBIRD_NDJSON_CONTENT_TYPE: &str = "application/x-ndjson"; const TINYBIRD_FIRST_BYTE_TIMEOUT: Duration = Duration::from_secs(2); @@ -22,9 +26,14 @@ const TINYBIRD_BETWEEN_BYTES_TIMEOUT: Duration = Duration::from_secs(2); const TINYBIRD_MAX_ROWS_PER_AUCTION_BATCH: usize = 512; /// Build the configured auction telemetry sink. +/// +/// Auction emission requires both the Tinybird master toggle +/// (`tinybird.enabled`) and the auction-specific toggle +/// (`tinybird.auction_enabled`), so access-log telemetry can be enabled +/// independently without also emitting auction events. #[must_use] pub(crate) fn auction_sink_from_settings(settings: &Settings) -> Arc { - if settings.tinybird.enabled { + if settings.tinybird.enabled && settings.tinybird.auction_enabled { Arc::new(FastlyTinybirdAuctionTelemetrySink::new( settings.tinybird.clone(), )) @@ -40,11 +49,10 @@ struct FastlyTinybirdAuctionTelemetrySink { } #[derive(Debug, Clone)] -struct TinybirdEventsTarget { +pub(crate) struct TinybirdEventsTarget { api_host: String, dataset: String, - secret_store: StoreName, - token_secret: String, + append_token: Redacted, uri: String, backend_spec: PlatformBackendSpec, max_body_bytes: usize, @@ -57,8 +65,31 @@ impl TinybirdEventsTarget { Self { api_host: config.api_host, dataset: config.auction_dataset, - secret_store: StoreName::from(config.secret_store), - token_secret: config.auction_token_secret, + append_token: config + .auction_token_secret + .expect("should contain a resolved Tinybird auction token when enabled"), + uri, + backend_spec, + max_body_bytes: config.max_body_bytes, + } + } + + /// Builds the Events API target for the access-log datasource. + /// + /// Shares [`from_config`](Self::from_config)'s host, resolved-token, and + /// body-size-limit derivation, but points at `access_dataset` and + /// `access_token_secret` instead of the auction pair, so access-log + /// emission never shares a datasource or token with auction telemetry + /// even though both configs come from the same [`TinybirdSettings`]. + pub(crate) fn from_access_config(config: TinybirdSettings) -> Self { + let uri = tinybird_events_uri(&config.api_host, &config.access_dataset); + let backend_spec = tinybird_backend_spec(&config.api_host); + Self { + api_host: config.api_host, + dataset: config.access_dataset, + append_token: config + .access_token_secret + .expect("should contain a resolved Tinybird access token when enabled"), uri, backend_spec, max_body_bytes: config.max_body_bytes, @@ -95,25 +126,6 @@ impl FastlyTinybirdAuctionTelemetrySink { batch.to_ndjson(self.target.max_body_bytes) } - fn load_append_token( - &self, - services: &RuntimeServices, - ) -> Result> { - let token = services - .secret_store() - .get_string(&self.target.secret_store, &self.target.token_secret) - .change_context(TrustedServerError::Proxy { - message: "Tinybird auction append token unavailable".to_owned(), - })?; - let token = token.trim().to_owned(); - if token.is_empty() { - return Err(Report::new(TrustedServerError::Proxy { - message: "Tinybird auction append token is empty".to_owned(), - })); - } - Ok(token) - } - fn ensure_backend( &self, services: &RuntimeServices, @@ -185,8 +197,7 @@ impl AuctionTelemetrySink for FastlyTinybirdAuctionTelemetrySink { Self::validate_batch(&batch)?; let body = self.serialize_batch(&batch)?; let body_len = body.len(); - let token = self.load_append_token(services)?; - let auth_header = Self::authorization_header(&token)?; + let auth_header = Self::authorization_header(self.target.append_token.expose())?; let backend_name = self.ensure_backend(services)?; let request = self.build_events_request(body, auth_header)?; @@ -203,6 +214,124 @@ impl AuctionTelemetrySink for FastlyTinybirdAuctionTelemetrySink { } } +// --------------------------------------------------------------------------- +// Access telemetry: confirmed-delivery emitter +// --------------------------------------------------------------------------- + +/// Decides whether one request's access-telemetry row should be emitted. +/// +/// `roll` is a uniform draw from `[0, 1)`; callers pass +/// `rand::thread_rng().r#gen::()`, which the wasm32-wasip1 guest backs with +/// real WASI randomness (the EC generation path already relies on this and +/// the CI wasm release build verifies it). Comparing the draw directly +/// against `rate` keeps the sampling probability exactly `rate` for every +/// positive value: there is no bucket quantization, so rates below one in a +/// million sample proportionally instead of never, and emitted rows' +/// `sample_rate` matches the probability they were sampled at, which the +/// `sum(1.0 / sample_rate)` volume estimator depends on. +/// +/// `rate <= 0.0` never samples and `rate >= 1.0` always samples, for any +/// `roll` in `[0, 1)`. `0.0` cannot actually occur while `access_enabled` +/// is `true` (`Settings` validation requires `access_sample_rate > 0.0` in +/// that case), but this function stays total rather than leaning on that +/// invariant. +#[must_use] +pub(crate) fn sampled_in(rate: f64, roll: f64) -> bool { + roll < rate +} + +/// Builds the Events API POST request for one access-log row. +fn build_access_events_request( + target: &TinybirdEventsTarget, + body: String, + auth_header: HeaderValue, +) -> Result> { + request_builder() + .method(Method::POST) + .uri(target.uri.as_str()) + .header(header::AUTHORIZATION, auth_header) + .header(header::CONTENT_TYPE, TINYBIRD_NDJSON_CONTENT_TYPE) + .body(Body::from(body)) + .change_context(TrustedServerError::Proxy { + message: "failed to build Tinybird Events API request".to_owned(), + }) +} + +/// Sends one confirmed access-log row to the Tinybird Events API and waits +/// for the response. +/// +/// Unlike [`FastlyTinybirdAuctionTelemetrySink::emit_auction_events`] (fire- +/// and-forget, dispatched mid-request so it never adds latency to the +/// response), this runs post-delivery: the response has already reached the +/// client, so there is no latency budget left to protect, and the send can +/// afford to wait for — and validate — the reply. `client` is the adapter's +/// stateless platform HTTP client in production +/// ([`crate::platform::FastlyPlatformHttpClient`]); accepting it as `&dyn +/// PlatformHttpClient` here (rather than that concrete type) is what lets +/// tests substitute a recording double instead of performing a real network +/// send, matching how [`RuntimeServices::http_client`] is consumed +/// elsewhere. `target` is derived from settings once at the post-send call +/// site rather than threaded through any per-route state. +/// +/// A non-2xx status is reported as `Err` naming the status; there is no +/// retry — the caller logs exactly one warning and moves on. +/// +/// # Errors +/// +/// Returns `Err` when the row exceeds the configured request-body limit, the +/// resolved access-log APPEND token is invalid, the backend cannot be registered, +/// the request cannot be built or sent, or the Tinybird Events API responds +/// with a non-2xx status. +pub(crate) async fn emit_access_event( + client: &dyn PlatformHttpClient, + target: &TinybirdEventsTarget, + row: String, +) -> Result<(), Report> { + let body_len = row.len(); + if body_len > target.max_body_bytes { + return Err(Report::new(TrustedServerError::Proxy { + message: format!( + "Tinybird access telemetry request body has {body_len} bytes, exceeding {} byte limit", + target.max_body_bytes + ), + })); + } + + let auth_header = + FastlyTinybirdAuctionTelemetrySink::authorization_header(target.append_token.expose())?; + let backend_name = FastlyPlatformBackend + .ensure(&target.backend_spec) + .change_context(TrustedServerError::Proxy { + message: "Tinybird backend registration failed".to_owned(), + })?; + let request = build_access_events_request(target, row, auth_header)?; + + log::info!( + "sending access telemetry to Tinybird dataset={} host={} backend={}", + target.dataset, + target.api_host, + backend_name + ); + + let response = client + .send(PlatformHttpRequest::new(request, backend_name)) + .await + .change_context(TrustedServerError::Proxy { + message: "failed to send Tinybird access telemetry request".to_owned(), + })?; + + if response.response.status().is_success() { + Ok(()) + } else { + Err(Report::new(TrustedServerError::Proxy { + message: format!( + "Tinybird access telemetry request failed with status {}", + response.response.status() + ), + })) + } +} + fn tinybird_backend_spec(api_host: &str) -> PlatformBackendSpec { PlatformBackendSpec { scheme: "https".to_owned(), @@ -233,11 +362,14 @@ mod tests { use trusted_server_core::platform::{ ClientInfo, PlatformBackend, PlatformConfigStore, PlatformError, PlatformGeo, PlatformHttpClient, PlatformPendingRequest, PlatformResponse, PlatformSecretStore, - PlatformSelectResult, RuntimeServices, StoreId, + PlatformSelectResult, RuntimeServices, StoreId, StoreName, }; use super::*; + const TEST_USER_AGENT: &str = + "FictionalBrowser/123.4 (FictionalOS 10.2; FictionalDevice) ExampleRenderer/567.8"; + struct NoopConfigStore; impl PlatformConfigStore for NoopConfigStore { @@ -297,6 +429,10 @@ mod tests { } impl PlatformBackend for RecordingBackend { + fn naming_policy(&self) -> trusted_server_core::platform::BackendNamingPolicy { + trusted_server_core::platform::BackendNamingPolicy::Fastly + } + fn predict_name( &self, _spec: &PlatformBackendSpec, @@ -322,25 +458,28 @@ mod tests { body: Vec, } + /// Records outbound requests and, for [`PlatformHttpClient::send`] (the + /// blocking variant `emit_access_event` uses), returns a synthetic + /// response carrying `respond_status` instead of performing a real + /// network send. #[derive(Default)] struct RecordingHttpClient { requests: Mutex>, select_calls: Mutex, + respond_status: Mutex, } - #[async_trait::async_trait(?Send)] - impl PlatformHttpClient for RecordingHttpClient { - async fn send( - &self, - _request: PlatformHttpRequest, - ) -> Result> { - Err(Report::new(PlatformError::Unsupported)) + impl RecordingHttpClient { + /// Status [`PlatformHttpClient::send`] should reply with. Irrelevant + /// to auction-sink tests, which only exercise `send_async`. + fn respond_with(status: u16) -> Self { + Self { + respond_status: Mutex::new(status), + ..Self::default() + } } - async fn send_async( - &self, - request: PlatformHttpRequest, - ) -> Result> { + fn record(&self, request: PlatformHttpRequest) { let backend_name = request.backend_name; let (parts, body) = request.request.into_parts(); let headers = parts @@ -364,6 +503,35 @@ mod tests { .lock() .expect("should lock recorded requests") .push(recorded); + } + } + + #[async_trait::async_trait(?Send)] + impl PlatformHttpClient for RecordingHttpClient { + async fn send( + &self, + request: PlatformHttpRequest, + ) -> Result> { + self.record(request); + let status = *self + .respond_status + .lock() + .expect("should lock configured response status"); + let response = edgezero_core::http::response_builder() + .status( + edgezero_core::http::StatusCode::from_u16(status) + .expect("should build a valid test status code"), + ) + .body(edgezero_core::body::Body::empty()) + .expect("should build test response"); + Ok(PlatformResponse::new(response)) + } + + async fn send_async( + &self, + request: PlatformHttpRequest, + ) -> Result> { + self.record(request); Ok(PlatformPendingRequest::new(()).with_backend_name("tinybird-backend")) } @@ -399,6 +567,7 @@ mod tests { region: None, is_mobile: 0, is_known_browser: 1, + user_agent: Some(TEST_USER_AGENT.to_owned()), gdpr_applies: 0, consent_present: 0, terminal_status: Some("completed".to_owned()), @@ -443,18 +612,52 @@ mod tests { fn enabled_config() -> TinybirdSettings { TinybirdSettings { enabled: true, + auction_enabled: true, api_host: "api.us-east.aws.tinybird.co".to_owned(), - secret_store: "ts_secrets".to_owned(), + secret_store: None, auction_dataset: "auction_events_raw".to_owned(), - auction_token_secret: "tinybird_auction_append_token".to_owned(), + auction_token_secret: Some(Redacted::new("append-token".to_owned())), access_enabled: false, access_dataset: "access_logs_raw".to_owned(), - access_token_secret: "tinybird_access_append_token".to_owned(), + access_token_secret: Some(Redacted::new("access-append-token".to_owned())), access_sample_rate: 0.0, max_body_bytes: 1024 * 1024, } } + #[test] + fn sink_from_settings_disables_when_auction_enabled_is_false() { + let settings = Settings { + tinybird: TinybirdSettings { + auction_enabled: false, + ..enabled_config() + }, + ..Settings::default() + }; + + let sink = auction_sink_from_settings(&settings); + + assert!( + !sink.is_enabled(), + "auction telemetry should stay off when auction_enabled is false, even if tinybird.enabled is true" + ); + } + + #[test] + fn sink_from_settings_enables_when_both_toggles_are_true() { + let settings = Settings { + tinybird: enabled_config(), + ..Settings::default() + }; + + let sink = auction_sink_from_settings(&settings); + + assert!( + sink.is_enabled(), + "auction telemetry should be on when both tinybird.enabled and tinybird.auction_enabled are true" + ); + } + #[test] fn events_uri_targets_dataset_on_region_host() { assert_eq!( @@ -481,16 +684,13 @@ mod tests { } #[test] - fn sink_posts_ndjson_with_secret_token_and_does_not_wait() { + fn sink_posts_ndjson_with_resolved_token_and_does_not_wait() { let backend = Arc::new(RecordingBackend::default()); let http_client = Arc::new(RecordingHttpClient::default()); let services = services( Arc::clone(&backend), Arc::clone(&http_client), - HashMap::from([( - "tinybird_auction_append_token".to_owned(), - b" append-token\n".to_vec(), - )]), + HashMap::new(), ); let sink = FastlyTinybirdAuctionTelemetrySink::new(enabled_config()); @@ -523,11 +723,11 @@ mod tests { header_value(&requests[0].headers, header::AUTHORIZATION.as_str()), Some("Bearer append-token") ); + let body = std::str::from_utf8(&requests[0].body).expect("should record utf8 ndjson body"); + assert!(body.ends_with('\n'), "should send newline-delimited JSON"); assert!( - std::str::from_utf8(&requests[0].body) - .expect("should record utf8 ndjson body") - .ends_with('\n'), - "should send newline-delimited JSON" + body.contains(TEST_USER_AGENT), + "should send the complete user agent to Tinybird" ); assert_eq!( *http_client @@ -602,14 +802,22 @@ mod tests { } #[test] - fn sink_drops_missing_secret_as_setup_error() { + fn sink_drops_row_count_oversize_before_sending() { let backend = Arc::new(RecordingBackend::default()); let http_client = Arc::new(RecordingHttpClient::default()); - let services = services(backend, Arc::clone(&http_client), HashMap::new()); + let services = services( + backend, + Arc::clone(&http_client), + HashMap::from([( + "tinybird_auction_append_token".to_owned(), + b"append-token".to_vec(), + )]), + ); let sink = FastlyTinybirdAuctionTelemetrySink::new(enabled_config()); + let rows = vec![test_row(); TINYBIRD_MAX_ROWS_PER_AUCTION_BATCH + 1]; let result = futures::executor::block_on( - sink.emit_auction_events(&services, AuctionEventBatch::new(vec![test_row()])), + sink.emit_auction_events(&services, AuctionEventBatch::new(rows)), ); assert!( @@ -622,32 +830,24 @@ mod tests { .lock() .expect("should lock recorded requests") .is_empty(), - "should not send without a token" + "should not send oversized row batches" ); } #[test] - fn sink_drops_row_count_oversize_before_sending() { - let backend = Arc::new(RecordingBackend::default()); - let http_client = Arc::new(RecordingHttpClient::default()); - let services = services( - backend, - Arc::clone(&http_client), - HashMap::from([( - "tinybird_auction_append_token".to_owned(), - b"append-token".to_vec(), - )]), - ); - let sink = FastlyTinybirdAuctionTelemetrySink::new(enabled_config()); - let rows = vec![test_row(); TINYBIRD_MAX_ROWS_PER_AUCTION_BATCH + 1]; + fn access_emitter_rejects_oversized_row_before_sending() { + let mut config = enabled_config(); + config.max_body_bytes = 1024; + let target = TinybirdEventsTarget::from_access_config(config); + let http_client = RecordingHttpClient::respond_with(202); + let row = "x".repeat(1025); - let result = futures::executor::block_on( - sink.emit_auction_events(&services, AuctionEventBatch::new(rows)), - ); + let result = futures::executor::block_on(emit_access_event(&http_client, &target, row)); + let error = result.expect_err("should reject a row above the configured body limit"); assert!( - result.is_err(), - "best-effort caller will suppress this error" + error.to_string().contains("1024"), + "error should name the configured body limit: {error}" ); assert!( http_client @@ -655,7 +855,110 @@ mod tests { .lock() .expect("should lock recorded requests") .is_empty(), - "should not send oversized row batches" + "should not send an oversized access row" + ); + } + + #[test] + fn access_emitter_posts_ndjson_and_validates_2xx() { + // Runtime settings carry the access APPEND token after startup secret + // resolution, so post-delivery emission does not reopen a secret store. + let target = TinybirdEventsTarget::from_access_config(enabled_config()); + let http_client = RecordingHttpClient::respond_with(202); + let row = r#"{"status":200}"#.to_owned(); + + futures::executor::block_on(emit_access_event(&http_client, &target, row.clone())) + .expect("should accept a 202 response"); + + let requests = http_client + .requests + .lock() + .expect("should lock recorded requests"); + assert_eq!(requests.len(), 1, "should send exactly one request"); + assert_eq!( + requests[0].uri, + "https://api.us-east.aws.tinybird.co/v0/events?name=access_logs_raw" + ); + assert_eq!(requests[0].method, Method::POST.to_string()); + assert_eq!( + header_value(&requests[0].headers, header::AUTHORIZATION.as_str()), + Some("Bearer access-append-token") + ); + assert_eq!( + std::str::from_utf8(&requests[0].body).expect("should record utf8 body"), + row, + "should send the row verbatim as the request body" + ); + } + + #[test] + fn access_emitter_warns_and_drops_on_non_2xx() { + let target = TinybirdEventsTarget::from_access_config(enabled_config()); + let http_client = RecordingHttpClient::respond_with(422); + + let result = futures::executor::block_on(emit_access_event( + &http_client, + &target, + r#"{"status":422}"#.to_owned(), + )); + + let error = result.expect_err("a 422 response should be reported as an error"); + assert!( + error.to_string().contains("422"), + "error should name the failing status: {error}" + ); + assert_eq!( + http_client + .requests + .lock() + .expect("should lock recorded requests") + .len(), + 1, + "should not retry after a non-2xx response" + ); + } + + #[test] + fn sampled_in_boundary_rates_are_unconditional() { + assert!( + sampled_in(1.0, 0.0), + "a 1.0 sample rate should always sample in" + ); + assert!( + sampled_in(1.0, 0.999_999), + "a 1.0 sample rate should sample in for the largest roll" + ); + assert!( + !sampled_in(0.0, 0.0), + "a 0.0 sample rate should never sample in, even on a zero roll" + ); + assert!( + !sampled_in(-1.0, 0.0), + "a negative rate should never sample in" + ); + } + + #[test] + fn sampled_in_keeps_exact_probability_for_tiny_rates() { + // The previous bucket-quantized sampler truncated rates below one + // in a million to a zero threshold, silently emitting nothing. + // Direct comparison keeps every positive rate proportional. + let rate = 0.000_000_1; + assert!( + sampled_in(rate, rate / 2.0), + "a roll below a tiny positive rate should sample in" + ); + assert!( + !sampled_in(rate, rate * 2.0), + "a roll above a tiny positive rate should sample out" + ); + assert!( + !sampled_in(0.000_001_9, 0.000_001_95), + "no downward quantization: the boundary sits exactly at the rate" + ); + assert!( + sampled_in(0.000_001_9, 0.000_001_85), + "rolls just under the rate should sample in" ); } diff --git a/crates/trusted-server-adapter-spin/spin.toml b/crates/trusted-server-adapter-spin/spin.toml index 9bc3634d8..684c171b0 100644 --- a/crates/trusted-server-adapter-spin/spin.toml +++ b/crates/trusted-server-adapter-spin/spin.toml @@ -25,6 +25,15 @@ version = "0.1.0" [variables] v_current_x2dkid = { default = "" } v_active_x2dkids = { default = "" } +# These declared variables match the example config's secret key names. Regenerate +# or extend them for deployment-specific keys, including handler key names such as +# `admin_password` or `api_handler_password`. Replace the empty defaults with values +# supplied by the deployment's secret provider; never commit values here. +v_trusted_x5fserver_x5fsecrets_v_publisher_x5fproxy_x5fsecret = { default = "", secret = true } +v_trusted_x5fserver_x5fsecrets_v_ec_x5fpassphrase = { default = "", secret = true } +v_trusted_x5fserver_x5fsecrets_v_partner_x5fapi_x5ftoken = { default = "", secret = true } +v_trusted_x5fserver_x5fsecrets_v_partner_x5fts_x5fpull_x5ftoken = { default = "", secret = true } +v_trusted_x5fserver_x5fsecrets_v_handler_x5fpassword = { default = "", secret = true } [[trigger.http]] route = "/..." @@ -43,6 +52,11 @@ key_value_stores = ["default"] [component.trusted-server.variables] v_current_x2dkid = "{{ v_current_x2dkid }}" v_active_x2dkids = "{{ v_active_x2dkids }}" +v_trusted_x5fserver_x5fsecrets_v_publisher_x5fproxy_x5fsecret = "{{ v_trusted_x5fserver_x5fsecrets_v_publisher_x5fproxy_x5fsecret }}" +v_trusted_x5fserver_x5fsecrets_v_ec_x5fpassphrase = "{{ v_trusted_x5fserver_x5fsecrets_v_ec_x5fpassphrase }}" +v_trusted_x5fserver_x5fsecrets_v_partner_x5fapi_x5ftoken = "{{ v_trusted_x5fserver_x5fsecrets_v_partner_x5fapi_x5ftoken }}" +v_trusted_x5fserver_x5fsecrets_v_partner_x5fts_x5fpull_x5ftoken = "{{ v_trusted_x5fserver_x5fsecrets_v_partner_x5fts_x5fpull_x5ftoken }}" +v_trusted_x5fserver_x5fsecrets_v_handler_x5fpassword = "{{ v_trusted_x5fserver_x5fsecrets_v_handler_x5fpassword }}" [component.trusted-server.build] command = "cargo build --target wasm32-wasip1 --release -p trusted-server-adapter-spin --features spin" diff --git a/crates/trusted-server-adapter-spin/src/app.rs b/crates/trusted-server-adapter-spin/src/app.rs index f24b5b717..290de5aec 100644 --- a/crates/trusted-server-adapter-spin/src/app.rs +++ b/crates/trusted-server-adapter-spin/src/app.rs @@ -1,16 +1,24 @@ use std::net::{IpAddr, SocketAddr}; use std::sync::Arc; +#[cfg(all(feature = "spin", target_arch = "wasm32"))] +use edgezero_adapter_spin::config_store::SpinConfigStore; use edgezero_adapter_spin::context::SpinRequestContext; use edgezero_core::app::Hooks; +#[cfg(all(feature = "spin", target_arch = "wasm32"))] +use edgezero_core::config_store::ConfigStoreHandle; use edgezero_core::context::RequestContext; use edgezero_core::error::EdgeError; use edgezero_core::http::{HeaderValue, Method, Request, Response, StatusCode, header}; use edgezero_core::router::RouterService; use error_stack::Report; use trusted_server_core::auction::endpoints::handle_auction; -use trusted_server_core::auction::{AuctionOrchestrator, build_orchestrator}; +use trusted_server_core::auction::{ + AuctionOrchestrator, build_orchestrator_with_plan, compile_auction_plan, +}; use trusted_server_core::cache_policy::EdgeCacheHeader; +#[cfg(all(feature = "spin", target_arch = "wasm32"))] +use trusted_server_core::config_payload::settings_from_config_blob; use trusted_server_core::ec::EcContext; use trusted_server_core::ec::admin::{ admin_ec_lookup_not_supported as core_admin_ec_lookup_not_supported, @@ -21,6 +29,8 @@ use trusted_server_core::error::{IntoHttpResponse as _, TrustedServerError}; use trusted_server_core::http_util::sanitize_forwarded_headers; use trusted_server_core::integrations::{IntegrationRegistry, ProxyDispatchInput}; use trusted_server_core::platform::RuntimeServices; +#[cfg(all(feature = "spin", target_arch = "wasm32"))] +use trusted_server_core::platform::{PlatformConfigStore, StoreName}; use trusted_server_core::proxy::{ handle_first_party_click, handle_first_party_proxy, handle_first_party_proxy_rebuild, handle_first_party_proxy_sign, @@ -34,16 +44,24 @@ use trusted_server_core::request_signing::{ handle_trusted_server_discovery, handle_verify_signature, }; use trusted_server_core::settings::Settings; +#[cfg(all(feature = "spin", target_arch = "wasm32"))] +use trusted_server_core::settings_data::{default_config_key, default_secret_store_name}; use crate::middleware::{ AuthMiddleware, FinalizeResponseMiddleware, NormalizeMiddleware, SanitizeRequestMiddleware, }; use crate::platform::build_runtime_services; +#[cfg(all(feature = "spin", target_arch = "wasm32"))] +use crate::platform::{ConfigStoreHandleAdapter, SpinSecretStoreAdapter}; // --------------------------------------------------------------------------- // AppState // --------------------------------------------------------------------------- +/// Spin auto-provides this key-value store label without runtime configuration. +#[cfg(all(feature = "spin", target_arch = "wasm32"))] +const SPIN_DEFAULT_CONFIG_STORE: &str = "default"; + /// Application state built once at startup and shared across all requests. pub struct AppState { settings: Arc, @@ -58,10 +76,44 @@ pub struct AppState { /// Returns an error when settings, the auction orchestrator, or the integration /// registry fail to initialise. fn build_state() -> Result, Report> { - let settings = Settings::from_toml(include_str!("../../../trusted-server.example.toml"))?; + let settings = load_startup_settings()?; build_state_with_settings(settings) } +#[cfg(all(feature = "spin", target_arch = "wasm32"))] +fn load_startup_settings() -> Result> { + let config_store_name = StoreName::from(SPIN_DEFAULT_CONFIG_STORE); + let config_key = default_config_key(); + let config_store = + futures::executor::block_on(SpinConfigStore::open(config_store_name.as_ref().to_owned())) + .map_err(|error| { + Report::new(TrustedServerError::Configuration { + message: "failed to open Spin Trusted Server config store".to_string(), + }) + .attach(error.to_string()) + })?; + let config_handle = ConfigStoreHandle::new(Arc::new(config_store)); + let config_adapter = ConfigStoreHandleAdapter(config_handle); + let raw_envelope = config_adapter + .get(&config_store_name, &config_key) + .map_err(|error| { + Report::new(TrustedServerError::Configuration { + message: "failed to read Spin Trusted Server app-config blob".to_string(), + }) + .attach(error.to_string()) + })?; + let secret_store = SpinSecretStoreAdapter; + settings_from_config_blob(&raw_envelope, &secret_store, &default_secret_store_name()) +} + +#[cfg(not(all(feature = "spin", target_arch = "wasm32")))] +fn load_startup_settings() -> Result> { + Err(Report::new(TrustedServerError::Configuration { + message: "Spin startup settings require the production config store".to_string(), + }) + .attach("use TrustedServerApp::routes_with_settings for host tests")) +} + /// Build the application state from explicit settings. /// /// # Errors @@ -71,8 +123,10 @@ fn build_state() -> Result, Report> { fn build_state_with_settings( settings: Settings, ) -> Result, Report> { - let orchestrator = build_orchestrator(&settings)?; - let registry = IntegrationRegistry::new(&settings)?; + let plan = Arc::new(compile_auction_plan(&settings)?); + plan.validate_for_target(trusted_server_core::platform::AuctionTargetId::Spin)?; + let orchestrator = build_orchestrator_with_plan(Arc::clone(&plan), &settings)?; + let registry = IntegrationRegistry::with_plan(&settings, plan)?; Ok(Arc::new(AppState { settings: Arc::new(settings), @@ -568,13 +622,13 @@ fn build_router(state: &Arc) -> RouterService { // Build the geo-aware EC context so the auction consent gate sees // the caller's jurisdiction — `EcContext::default()` fails it // closed for consented users. - let ec_context = build_ec_context(&s.settings, &services, &req); + let mut ec_context = build_ec_context(&s.settings, &services, &req); Ok(handle_auction( &s.settings, &s.orchestrator, None, None, - &ec_context, + &mut ec_context, &services, req, ) @@ -598,14 +652,14 @@ fn build_router(state: &Arc) -> RouterService { { return Ok(http_error(&error)); } - let ec_context = build_ec_context(&s.settings, &services, &req); + let mut ec_context = build_ec_context(&s.settings, &services, &req); let auction = AuctionDispatch { orchestrator: &s.orchestrator, slots: s.settings.creative_opportunity_slots(), registry: None, }; Ok( - handle_page_bids(&s.settings, &services, None, auction, &ec_context, req) + handle_page_bids(&s.settings, &services, None, auction, &mut ec_context, req) .await .unwrap_or_else(|e| http_error(&e)), ) @@ -856,6 +910,100 @@ fn build_router(state: &Arc) -> RouterService { mod tests { use super::*; + fn multi_provider_settings() -> Settings { + let mut settings = Settings::from_toml( + r#" + [[handlers]] + path = "^/_ts/admin" + username = "admin" + password = "admin-password" + + [publisher] + domain = "publisher.example" + cookie_domain = ".publisher.example" + origin_url = "https://origin.publisher.example" + proxy_secret = "fictional-proxy-secret" + + [ec] + passphrase = "fictional-secret-key-32-bytes-minimum" + "#, + ) + .expect("should parse startup test settings"); + settings.auction.enabled = true; + settings.auction.providers = + std::iter::IntoIterator::into_iter(["provider-a", "provider-b"]) + .map(|id| { + ( + id.parse().expect("should parse provider ID"), + trusted_server_core::auction::ProviderConfig { + protocol: "openrtb-2.6".to_string(), + profile: "standard".to_string(), + endpoint: format!("https://{id}.example/openrtb"), + timeout_ms: None, + routing: trusted_server_core::auction::RoutingMode::AllEligible, + notifications: + trusted_server_core::auction::NotificationConfig::default(), + profile_config: "{}" + .parse() + .expect("should parse empty profile config object"), + }, + ) + }) + .collect(); + settings + } + + #[test] + fn startup_registers_aps_renderer_route() { + let mut settings = multi_provider_settings(); + settings.auction.providers.clear(); + settings.auction.providers.insert( + "aps-main".parse().expect("should parse APS provider ID"), + trusted_server_core::auction::ProviderConfig { + protocol: "openrtb-2.6".to_string(), + profile: "aps".to_string(), + endpoint: "https://aps.example/e/pb/bid".to_string(), + timeout_ms: None, + routing: trusted_server_core::auction::RoutingMode::AllEligible, + notifications: trusted_server_core::auction::NotificationConfig::default(), + profile_config: "{\"account_id\":\"example-account\"}" + .parse() + .expect("should parse APS profile config"), + }, + ); + + let state = + build_state_with_settings(settings).expect("Spin startup should register APS renderer"); + assert!( + state.registry.has_route( + &edgezero_core::http::Method::GET, + "/integrations/aps/renderer" + ), + "Spin startup registry should expose the APS renderer" + ); + } + + #[test] + fn disabled_startup_accepts_dormant_multi_provider_auction_plan() { + let mut settings = multi_provider_settings(); + settings.auction.enabled = false; + + build_state_with_settings(settings) + .expect("disabled Spin auction should accept dormant fanout"); + } + + #[test] + fn startup_rejects_multi_provider_auction_plan() { + let error = match build_state_with_settings(multi_provider_settings()) { + Ok(_) => panic!("Spin startup should reject multi-provider fanout"), + Err(error) => error, + }; + assert!( + format!("{error:?}").contains("concurrent provider fanout"), + "should identify unsupported fanout: {error:?}" + ); + } + #[test] fn scheme_host_from_spin_url_extracts_localhost_with_port() { assert_eq!( diff --git a/crates/trusted-server-adapter-spin/src/platform.rs b/crates/trusted-server-adapter-spin/src/platform.rs index 492f1a518..8e7cb4bf3 100644 --- a/crates/trusted-server-adapter-spin/src/platform.rs +++ b/crates/trusted-server-adapter-spin/src/platform.rs @@ -5,20 +5,18 @@ use std::time::Duration; use bytes::Bytes; use edgezero_core::config_store::ConfigStoreHandle; use edgezero_core::key_value_store::{KvHandle, KvPage, KvStore}; -use error_stack::Report; +use error_stack::{Report, ResultExt as _}; #[cfg(all(feature = "spin", target_arch = "wasm32"))] use http_body_util::BodyExt as _; use trusted_server_core::platform::{ - ClientInfo, GeoInfo, KvError, PlatformBackend, PlatformBackendSpec, PlatformConfigStore, - PlatformError, PlatformGeo, PlatformHttpClient, PlatformKvStore, PlatformSecretStore, - RuntimeServices, StoreId, StoreName, UnavailableKvStore, + BackendNamingPolicy, ClientInfo, GeoInfo, KvError, PlatformBackend, PlatformBackendSpec, + PlatformConfigStore, PlatformError, PlatformGeo, PlatformHttpClient, PlatformKvStore, + PlatformSecretStore, RuntimeServices, StoreId, StoreName, UnavailableKvStore, }; #[cfg(not(all(feature = "spin", target_arch = "wasm32")))] use trusted_server_core::platform::UnavailableHttpClient; -#[cfg(all(feature = "spin", target_arch = "wasm32"))] -use error_stack::ResultExt as _; #[cfg(any(test, all(feature = "spin", target_arch = "wasm32")))] use std::io::Read as _; #[cfg(any(test, all(feature = "spin", target_arch = "wasm32")))] @@ -39,6 +37,7 @@ type HeaderPairs = Vec<(String, Vec)>; #[cfg(any(test, all(feature = "spin", target_arch = "wasm32")))] type BufferedResponseParts = (HeaderPairs, Vec); +#[cfg(any(test, all(feature = "spin", target_arch = "wasm32")))] const SPIN_VARIABLE_HEX: &[u8; 16] = b"0123456789abcdef"; // --------------------------------------------------------------------------- @@ -82,27 +81,15 @@ impl PlatformSecretStore for NoopSecretStore { struct NoopBackend; impl PlatformBackend for NoopBackend { + fn naming_policy(&self) -> BackendNamingPolicy { + BackendNamingPolicy::Spin + } + fn predict_name(&self, spec: &PlatformBackendSpec) -> Result> { - let port = spec - .port - .unwrap_or(if spec.scheme == "https" { 443 } else { 80 }); - let timeout_ms = spec.first_byte_timeout.as_millis(); - let cert_suffix = if spec.certificate_check { - "" - } else { - "_nocert" - }; - // Keep two providers that share an origin on distinct names so auction - // response correlation cannot cross providers. - let discriminator = spec - .discriminator - .as_deref() - .map(|d| format!("_p_{d}")) - .unwrap_or_default(); - Ok(format!( - "{}_{}_{}_{timeout_ms}ms{cert_suffix}{discriminator}", - spec.scheme, spec.host, port - )) + self.naming_policy() + .predict(spec) + .map(|prediction| prediction.name) + .change_context(PlatformError::Backend) } fn ensure(&self, spec: &PlatformBackendSpec) -> Result> { @@ -116,25 +103,22 @@ impl PlatformBackend for NoopBackend { /// Bridges edgezero's [`ConfigStoreHandle`] to [`PlatformConfigStore`]. /// -/// Reads delegate through the handle after mapping Trusted Server keys to Spin -/// variable names. Writes are unsupported on current Spin runtime config and -/// return typed errors. -struct ConfigStoreHandleAdapter(ConfigStoreHandle); +/// Spin config stores are KV-backed, so reads preserve the requested key +/// verbatim. Writes are unsupported on current Spin runtime config and return +/// typed errors. +pub(crate) struct ConfigStoreHandleAdapter(pub(crate) ConfigStoreHandle); impl PlatformConfigStore for ConfigStoreHandleAdapter { fn get(&self, _store_name: &StoreName, key: &str) -> Result> { - let variable_name = spin_variable_name(key, PlatformError::ConfigStore)?; - futures::executor::block_on(self.0.get(&variable_name)) - .map_err(|e| { - Report::new(PlatformError::ConfigStore) - .attach(format!( - "config store lookup failed for key `{key}` as Spin variable `{variable_name}`: {e}" - )) - })? - .ok_or_else(|| { + futures::executor::block_on(self.0.get(key)) + .map_err(|error| { Report::new(PlatformError::ConfigStore).attach(format!( - "key `{key}` not found as Spin variable `{variable_name}`" + "config store lookup failed for key `{key}`: {error}" )) + })? + .ok_or_else(|| { + Report::new(PlatformError::ConfigStore) + .attach(format!("key `{key}` not found in Spin config store")) }) } @@ -149,6 +133,7 @@ impl PlatformConfigStore for ConfigStoreHandleAdapter { } } +#[cfg(any(test, all(feature = "spin", target_arch = "wasm32")))] fn spin_variable_name( key: &str, error_context: PlatformError, @@ -187,6 +172,7 @@ fn spin_variable_name( Ok(out) } +#[cfg(any(test, all(feature = "spin", target_arch = "wasm32")))] fn push_spin_variable_escape(out: &mut String, byte: u8) { out.push('_'); out.push('x'); @@ -462,6 +448,13 @@ struct SpinPendingResponse { /// request launches. `select` keeps a defense-in-depth rejection for more /// than one pending request, matching the Cloudflare adapter behavior. /// +/// Spin's WASI HTTP API sends one request and returns the original response; it +/// exposes no redirect-follow policy. Each trait call therefore reaches exactly +/// one `spin_sdk::http::send` boundary and returns an original 3xx to core. Host +/// tests cannot instantiate Spin's WASI transport, so the common +/// `StubHttpClient` driver records the one-send 3xx behavior while adapter tests +/// cover request/response policy around that single boundary. +/// /// # Known MVP limits /// /// **No configurable outbound timeout.** `spin_sdk::http::send` does not @@ -676,7 +669,7 @@ fn into_spin_method(method: &edgezero_core::http::Method) -> spin_sdk::http::Met /// with a real secret-provider source (e.g. Vault, Azure Key Vault) to avoid /// storing signing keys in plaintext on disk. #[cfg(all(feature = "spin", target_arch = "wasm32"))] -struct SpinSecretStoreAdapter; +pub(crate) struct SpinSecretStoreAdapter; #[cfg(all(feature = "spin", target_arch = "wasm32"))] impl PlatformSecretStore for SpinSecretStoreAdapter { @@ -794,12 +787,33 @@ mod tests { use super::*; use edgezero_core::body::Body; + use edgezero_core::config_store::{ConfigStore, ConfigStoreError}; use edgezero_core::context::RequestContext; use edgezero_core::http::request_builder; use edgezero_core::params::PathParams; use flate2::Compression; use flate2::write::GzEncoder; use std::io::Write as _; + use trusted_server_core::platform::AuctionTargetId; + + #[test] + fn auction_http_capabilities_are_explicit() { + let capabilities = AuctionTargetId::Spin.descriptor().capabilities(); + assert!(!capabilities.supports_concurrent_provider_fanout()); + assert!( + !capabilities.has_enforceable_total_request_deadline(), + "Spin outbound HTTP does not expose an enforceable hard total request deadline" + ); + } + + struct InMemoryConfigStore(std::collections::BTreeMap); + + #[async_trait::async_trait(?Send)] + impl ConfigStore for InMemoryConfigStore { + async fn get(&self, key: &str) -> Result, ConfigStoreError> { + Ok(self.0.get(key).cloned()) + } + } fn make_ctx_without_spin_context() -> RequestContext { let req = request_builder() @@ -845,6 +859,29 @@ mod tests { apply_spin_response_policy(&edgezero_core::http::Method::GET, 200, headers, body) } + #[test] + fn response_policy_preserves_original_redirect_at_single_send_boundary() { + let (headers, body) = apply_spin_response_policy( + &edgezero_core::http::Method::GET, + 302, + vec![( + "location".to_string(), + b"https://redirect.example/next".to_vec(), + )], + Vec::new(), + ) + .expect("should preserve redirect response"); + + assert_eq!( + headers, + vec![( + "location".to_string(), + b"https://redirect.example/next".to_vec(), + )] + ); + assert!(body.is_empty()); + } + #[test] fn extract_client_ip_reads_spin_request_context() { let mut req = request_builder() @@ -894,6 +931,29 @@ mod tests { ); } + #[test] + fn config_store_handle_adapter_reads_verbatim_kv_key() { + let handle = ConfigStoreHandle::new(Arc::new(InMemoryConfigStore( + std::collections::BTreeMap::from([( + "trusted_server_config".to_owned(), + "blob-envelope".to_owned(), + )]), + ))); + let adapter = ConfigStoreHandleAdapter(handle); + + let value = adapter + .get( + &StoreName::from("trusted_server_config"), + "trusted_server_config", + ) + .expect("should read the verbatim config-store key"); + + assert_eq!( + value, "blob-envelope", + "should not translate a KV-backed config key into a Spin variable name" + ); + } + #[test] fn spin_variable_name_encodes_trusted_server_keys() { assert_eq!( diff --git a/crates/trusted-server-cli/Cargo.toml b/crates/trusted-server-cli/Cargo.toml index fe9c3664b..247d179a7 100644 --- a/crates/trusted-server-cli/Cargo.toml +++ b/crates/trusted-server-cli/Cargo.toml @@ -17,17 +17,24 @@ workspace = true [target.'cfg(not(target_arch = "wasm32"))'.dependencies] chromiumoxide = { workspace = true } clap = { workspace = true } +derive_more = { workspace = true } edgezero-cli = { workspace = true } +edgezero-core = { workspace = true } futures = { workspace = true } +glob = { workspace = true } +http = { workspace = true } log = { workspace = true } +rand = { workspace = true } regex = { workspace = true } scraper = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } +similar = { workspace = true } tempfile = { workspace = true } tokio = { workspace = true } toml = { workspace = true } toml_edit = { workspace = true } +tracing = { workspace = true } trusted-server-core = { workspace = true } url = { workspace = true } which = { workspace = true } @@ -41,7 +48,6 @@ which = { workspace = true } [target.'cfg(target_os = "macos")'.dependencies] base64 = { workspace = true } bytes = { workspace = true } -derive_more = { workspace = true } directories = { workspace = true } error-stack = { workspace = true } http-body-util = { workspace = true } @@ -62,4 +68,5 @@ tokio = { workspace = true, features = ["test-util"] } x509-parser = { workspace = true } [target.'cfg(not(target_arch = "wasm32"))'.dev-dependencies] +temp-env = { workspace = true } tempfile = { workspace = true } diff --git a/crates/trusted-server-cli/src/ad_templates/compare.rs b/crates/trusted-server-cli/src/ad_templates/compare.rs new file mode 100644 index 000000000..48ab71f3e --- /dev/null +++ b/crates/trusted-server-cli/src/ad_templates/compare.rs @@ -0,0 +1,830 @@ +//! Pure comparison of configured expected slots against browser ad evidence. +//! +//! This module is collector-independent and Chrome-free: it takes decoded +//! [`BrowserAdEvidence`] plus the [`ExpectedSlot`] set and produces a +//! [`PageVerificationResult`] with per-slot statuses, warnings, and unmatched +//! extra evidence, mirroring spec §5.3–§5.6. +//! +use serde::Deserialize; + +use trusted_server_core::auction::types::MediaType; +use trusted_server_core::creative_opportunities::RuntimeAdStackExpected; + +use crate::ad_templates::expected::ExpectedSlot; +use crate::ad_templates::output::Warning; + +/// The phase in which a piece of evidence was observed. +#[derive(Debug, Clone, Copy, Eq, PartialEq, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum EvidencePhase { + /// Observed during the initial load and settle. + InitialLoad, + /// Observed only after the deterministic scroll pass. + Scroll, +} + +/// A DOM element ID observed on the page. +#[derive(Debug, Clone, Deserialize)] +pub struct DomEvidence { + /// The element ID. + pub dom_id: String, + /// The phase it was first observed in. + pub phase: EvidencePhase, +} + +/// A GPT slot observed on the page. +#[derive(Debug, Clone, Deserialize)] +pub struct GptSlotEvidence { + /// The observed GAM ad unit path. + pub gam_unit_path: String, + /// The observed GPT slot element ID. + pub div_id: String, + /// Observed numeric sizes as `(width, height)` pairs (non-numeric dropped upstream). + pub sizes: Vec<(u32, u32)>, + /// The phase it was first observed in. + pub phase: EvidencePhase, +} + +/// An `apstag.fetchBids` call the page made, if any were recorded. +/// +/// The collector no longer hooks `apstag`: server-side APS configuration is +/// metadata rather than a client assertion, so a missing client call is not a +/// finding. The field and this shape stay for the evidence payload's schema, and +/// the list arrives empty. +#[derive(Debug, Clone, Deserialize)] +#[allow( + dead_code, + reason = "decoded for schema stability; the collector records no APS calls" +)] +pub struct ApsFetchBidsEvidence { + /// The APS slot ID requested. + pub slot_id: String, + /// Sizes requested for the slot. + pub sizes: Vec<(u32, u32)>, + /// The phase it was observed in. + pub phase: EvidencePhase, +} + +/// A `/__ts/page-bids` observation for SPA routes (spec §5.2). +/// +/// DEFERRED in Phase 1: kept as forward scaffolding so the decoded evidence shape +/// stays forward-compatible. Not populated by the collector or surfaced in JSON. +#[derive(Debug, Clone, Deserialize)] +#[allow( + dead_code, + reason = "reserved decoded shape for the optional bids phase" +)] +pub struct PageBidsEvidence { + /// The slot ID present in the page-bids response. + pub slot_id: String, + /// The phase it was observed in. + pub phase: EvidencePhase, +} + +/// All read-only ad evidence decoded from a single browser page. +#[derive(Debug, Clone, Deserialize)] +pub struct BrowserAdEvidence { + /// DOM element IDs matching configured prefixes. + pub dom_ids: Vec, + /// GPT slots observed via `defineSlot` and `getSlots()`. + pub gpt_slots: Vec, + /// `apstag.fetchBids` calls observed. + pub aps_calls: Vec, + /// `/__ts/page-bids` observations (deferred; default empty). + #[serde(default)] + #[allow(dead_code, reason = "reserved for the optional bids phase")] + pub page_bids: Vec, + /// Collector-level warnings (no page HTML/cookies/storage). + #[serde(default)] + pub warnings: Vec, +} + +/// Summary of the runtime ad-stack gate for a page. +#[derive(Debug, Clone, Copy)] +pub struct RuntimeGateSummary { + /// The three-state ad-stack expectation. + pub expected: RuntimeAdStackExpected, +} + +impl RuntimeGateSummary { + /// Builds a summary from a computed runtime expectation. + #[must_use] + pub fn from_expected(expected: RuntimeAdStackExpected) -> Self { + Self { expected } + } + + #[cfg(test)] + fn unknown_allowed() -> Self { + Self::from_expected(RuntimeAdStackExpected::Unknown) + } + + #[cfg(test)] + fn auction_disabled() -> Self { + Self::from_expected(RuntimeAdStackExpected::No) + } +} + +/// Confirmation status for a single configured slot (compare-side mirror of the +/// output `SlotStatus`). +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +pub enum SlotStatus { + /// GPT evidence matches GAM path, div, and a compatible size. + Confirmed, + /// Some evidence, but not enough to confirm. + Partial, + /// No DOM or GPT evidence confirms the slot. + Missing, + /// The checker cannot confirm this slot type; this is not page drift. + Unconfirmable, +} + +/// The verification result for one audited page. +#[derive(Debug, Clone)] +pub struct PageVerificationResult { + /// Whether the runtime ad stack was expected to run for this page. + pub runtime_ad_stack_expected: RuntimeAdStackExpected, + /// Per-slot results, in expected-slot order. + pub slots: Vec, + /// Live evidence that matched no configured slot. + pub extra_evidence: Vec, +} + +impl PageVerificationResult { + /// Whether `--strict` should fail for this page. + /// + /// False when the runtime ad stack is not expected to run (a known gate + /// suppressed it); otherwise true if any slot is missing or partial. Provider + /// warnings and extra evidence alone never fail strict. + #[must_use] + pub fn strict_failed(&self) -> bool { + if self.runtime_ad_stack_expected == RuntimeAdStackExpected::No { + return false; + } + self.slots + .iter() + .any(|slot| matches!(slot.status, SlotStatus::Missing | SlotStatus::Partial)) + } +} + +/// Per-slot verification result. +#[derive(Debug, Clone)] +pub struct SlotResult { + /// The configured slot id. + pub id: String, + /// The confirmation status. + pub status: SlotStatus, + /// The phase the confirming evidence was observed in. + pub phase: Option, + /// The live evidence observed for this slot. + pub evidence: SlotEvidence, + /// Slot-level warnings (size, provider, etc.). + pub warnings: Vec, +} + +/// Live evidence observed for a configured slot. +#[derive(Debug, Clone)] +pub struct SlotEvidence { + /// The resolved DOM element ID, if any. + pub dom_id: Option, + /// The matched GPT slot, if any. + pub gpt: Option, +} + +/// Live ad-slot evidence with no matching configured slot. +#[derive(Debug, Clone)] +pub struct ExtraEvidence { + /// Evidence kind. Only `gpt` is produced today; the field is a string so a + /// later evidence source can be added without changing the JSON schema. + pub kind: String, + /// The phase it was observed in. + pub phase: EvidencePhase, + /// The DOM element ID, if any. + pub dom_id: Option, + /// The GAM unit path, if any. + pub gam_unit_path: Option, + /// Observed numeric sizes. + pub sizes: Vec<(u32, u32)>, + /// Why this evidence is reported as extra. + pub reason: String, +} + +fn warning(code: &str, message: String) -> Warning { + Warning { + code: code.to_string(), + message, + } +} + +/// Resolves the slot root DOM element per spec §5.3. +/// +/// Exact `div_id` match first, then the first element whose ID starts with +/// `div_id`, ignoring `-container` wrappers. +fn resolve_dom<'a>(dom_ids: &'a [DomEvidence], div_id: &str) -> Option<&'a DomEvidence> { + if let Some(exact) = dom_ids.iter().find(|dom| dom.dom_id == div_id) { + return Some(exact); + } + dom_ids + .iter() + .find(|dom| dom.dom_id.starts_with(div_id) && !dom.dom_id.ends_with("-container")) +} + +/// Returns true when a GPT slot's element ID matches the resolved DOM id (or its +/// `-container`), per spec §5.4. +fn gpt_div_matches(gpt_div: &str, expected: &ExpectedSlot, resolved_dom_id: Option<&str>) -> bool { + match resolved_dom_id { + Some(dom_id) => gpt_div == dom_id || gpt_div == format!("{dom_id}-container"), + None => { + gpt_div == expected.div_id + || (gpt_div.starts_with(&expected.div_id) && !gpt_div.ends_with("-container")) + } + } +} + +fn banner_sizes(expected: &ExpectedSlot) -> Vec<(u32, u32)> { + expected + .formats + .iter() + .filter(|format| format.media_type == MediaType::Banner) + .map(|format| (format.width, format.height)) + .collect() +} + +/// Compares configured expected slots against decoded browser evidence. +#[must_use] +pub fn compare_page_evidence( + expected: &[ExpectedSlot], + evidence: &BrowserAdEvidence, + gate: RuntimeGateSummary, +) -> PageVerificationResult { + let mut consumed_gpt = vec![false; evidence.gpt_slots.len()]; + let mut slots = Vec::with_capacity(expected.len()); + + for slot in expected { + let resolved = resolve_dom(&evidence.dom_ids, &slot.div_id); + let resolved_id = resolved.map(|dom| dom.dom_id.clone()); + // An unrenderable (`None`) configured path can never match live GPT + // evidence; matching on anything else would confirm the wrong unit. + let gpt_idx = slot.gam_unit_path.as_deref().and_then(|unit_path| { + evidence.gpt_slots.iter().position(|gpt| { + gpt.gam_unit_path == unit_path + && gpt_div_matches(&gpt.div_id, slot, resolved_id.as_deref()) + }) + }); + + let banner = banner_sizes(slot); + let mut warnings = Vec::new(); + // `expected_slots_for_path` drops a slot whose template does not render, + // so on the verify path this arm is unreachable; it exists for callers + // that build expected slots directly, and as a guard if that filter ever + // changes. + if slot.gam_unit_path.is_none() { + warnings.push(warning( + "gam_unit_path_unrenderable", + format!( + "slot `{}` gam_unit_path template renders past GAM's unit-path byte limit \ + for this page's section; the runtime omits this slot on this path", + slot.id + ), + )); + } + + let (status, dom_for_evidence, gpt_for_evidence, phase) = if let Some(idx) = gpt_idx { + consumed_gpt[idx] = true; + let gpt = &evidence.gpt_slots[idx]; + let dom_id = resolved_id.clone().or_else(|| Some(gpt.div_id.clone())); + if banner.is_empty() { + warnings.push(warning( + "unsupported_format", + format!( + "slot `{}` has only non-banner formats; not confirmable in Phase 1", + slot.id + ), + )); + ( + SlotStatus::Unconfirmable, + dom_id, + Some(gpt.clone()), + Some(gpt.phase), + ) + } else if gpt.sizes.is_empty() { + warnings.push(warning( + "out_of_page_slot", + format!( + "slot `{}` matched an out-of-page GPT slot with no sizes", + slot.id + ), + )); + ( + SlotStatus::Partial, + dom_id, + Some(gpt.clone()), + Some(gpt.phase), + ) + } else if banner.iter().any(|size| gpt.sizes.contains(size)) { + let extra: Vec<(u32, u32)> = gpt + .sizes + .iter() + .copied() + .filter(|size| !banner.contains(size)) + .collect(); + if !extra.is_empty() { + warnings.push(warning( + "extra_observed_size", + format!("slot `{}` observed extra GPT sizes {extra:?}", slot.id), + )); + } + let missing: Vec<(u32, u32)> = banner + .iter() + .copied() + .filter(|size| !gpt.sizes.contains(size)) + .collect(); + if !missing.is_empty() { + warnings.push(warning( + "configured_size_not_observed", + format!( + "slot `{}` configured sizes {missing:?} were not observed", + slot.id + ), + )); + } + ( + SlotStatus::Confirmed, + dom_id, + Some(gpt.clone()), + Some(gpt.phase), + ) + } else { + warnings.push(warning( + "incompatible_sizes", + format!( + "slot `{}` GPT path and div matched but no configured size overlapped", + slot.id + ), + )); + ( + SlotStatus::Partial, + dom_id, + Some(gpt.clone()), + Some(gpt.phase), + ) + } + } else if let Some(dom) = resolved { + warnings.push(warning( + "dom_without_gpt", + "DOM element matched, but no GPT slot evidence was observed".to_string(), + )); + ( + SlotStatus::Partial, + Some(dom.dom_id.clone()), + None, + Some(dom.phase), + ) + } else { + (SlotStatus::Missing, None, None, None) + }; + + slots.push(SlotResult { + id: slot.id.clone(), + status, + phase, + evidence: SlotEvidence { + dom_id: dom_for_evidence, + gpt: gpt_for_evidence, + }, + warnings, + }); + } + + let extra_evidence = evidence + .gpt_slots + .iter() + .enumerate() + .filter(|(idx, _)| !consumed_gpt[*idx]) + .map(|(_, gpt)| ExtraEvidence { + kind: "gpt".to_string(), + phase: gpt.phase, + dom_id: Some(gpt.div_id.clone()), + gam_unit_path: Some(gpt.gam_unit_path.clone()), + sizes: gpt.sizes.clone(), + reason: "no_configured_slot_matched".to_string(), + }) + .collect(); + + PageVerificationResult { + runtime_ad_stack_expected: gate.expected, + slots, + extra_evidence, + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::ad_templates::expected::ExpectedFormat; + + fn dom(id: &str) -> DomEvidence { + DomEvidence { + dom_id: id.to_string(), + phase: EvidencePhase::InitialLoad, + } + } + + fn gpt_slot(gam_unit_path: &str, div_id: &str, sizes: &[(u32, u32)]) -> GptSlotEvidence { + GptSlotEvidence { + gam_unit_path: gam_unit_path.to_string(), + div_id: div_id.to_string(), + sizes: sizes.to_vec(), + phase: EvidencePhase::InitialLoad, + } + } + + fn aps(slot_id: &str, sizes: &[(u32, u32)]) -> ApsFetchBidsEvidence { + ApsFetchBidsEvidence { + slot_id: slot_id.to_string(), + sizes: sizes.to_vec(), + phase: EvidencePhase::InitialLoad, + } + } + + fn evidence( + doms: Vec, + gpts: Vec, + aps: Vec, + ) -> BrowserAdEvidence { + BrowserAdEvidence { + dom_ids: doms, + gpt_slots: gpts, + aps_calls: aps, + page_bids: Vec::new(), + warnings: Vec::new(), + } + } + + fn expected_slot( + id: &str, + div_id: &str, + gam_unit_path: &str, + sizes: &[(u32, u32)], + providers: &[&str], + ) -> ExpectedSlot { + ExpectedSlot { + id: id.to_string(), + div_id: div_id.to_string(), + gam_unit_path: Some(gam_unit_path.to_string()), + formats: sizes + .iter() + .map(|&(width, height)| ExpectedFormat { + width, + height, + media_type: MediaType::Banner, + }) + .collect(), + providers: providers.iter().copied().map(String::from).collect(), + page_patterns: Vec::new(), + } + } + + fn expected_slot_video(id: &str, div_id: &str, gam_unit_path: &str) -> ExpectedSlot { + ExpectedSlot { + id: id.to_string(), + div_id: div_id.to_string(), + gam_unit_path: Some(gam_unit_path.to_string()), + formats: vec![ExpectedFormat { + width: 0, + height: 0, + media_type: MediaType::Video, + }], + providers: Vec::new(), + page_patterns: Vec::new(), + } + } + + #[test] + fn gpt_path_div_and_size_overlap_confirms_slot() { + let expected = expected_slot("atf", "ad-atf-", "/123/news/atf", &[(300, 250)], &[]); + let evidence = evidence( + vec![dom("ad-atf-0")], + vec![gpt_slot("/123/news/atf", "ad-atf-0", &[(300, 250)])], + Vec::new(), + ); + + let result = compare_page_evidence( + &[expected], + &evidence, + RuntimeGateSummary::unknown_allowed(), + ); + + assert_eq!(result.slots[0].status, SlotStatus::Confirmed); + assert!( + result.slots[0].warnings.is_empty(), + "confirmed slot should carry no warnings" + ); + } + + #[test] + fn unrenderable_gam_unit_path_never_confirms() { + let mut expected = expected_slot("atf", "ad-atf-", "/123/news/atf", &[(300, 250)], &[]); + expected.gam_unit_path = None; + let evidence = evidence( + vec![dom("ad-atf-0")], + vec![gpt_slot("/123/news/atf", "ad-atf-0", &[(300, 250)])], + Vec::new(), + ); + + let result = compare_page_evidence( + &[expected], + &evidence, + RuntimeGateSummary::unknown_allowed(), + ); + + assert_eq!( + result.slots[0].status, + SlotStatus::Partial, + "an unrenderable configured path must not confirm against GPT evidence" + ); + assert!( + result.slots[0] + .warnings + .iter() + .any(|w| w.code == "gam_unit_path_unrenderable"), + "should explain why the slot cannot be confirmed" + ); + } + + #[test] + fn dom_only_is_partial() { + let expected = expected_slot("atf", "ad-atf-", "/123/news/atf", &[(300, 250)], &[]); + let evidence = evidence(vec![dom("ad-atf-0")], Vec::new(), Vec::new()); + + let result = compare_page_evidence( + &[expected], + &evidence, + RuntimeGateSummary::unknown_allowed(), + ); + + assert_eq!(result.slots[0].status, SlotStatus::Partial); + assert!( + result.slots[0] + .warnings + .iter() + .any(|w| w.code == "dom_without_gpt") + ); + } + + #[test] + fn no_dom_or_gpt_is_missing() { + let expected = expected_slot("atf", "ad-atf-", "/123/news/atf", &[(300, 250)], &[]); + let evidence = evidence(Vec::new(), Vec::new(), Vec::new()); + + let result = compare_page_evidence( + &[expected], + &evidence, + RuntimeGateSummary::unknown_allowed(), + ); + + assert_eq!(result.slots[0].status, SlotStatus::Missing); + } + + #[test] + fn prefix_dom_resolution_ignores_container_suffix() { + let expected = expected_slot( + "header", + "ad-header-0-", + "/123/homepage/header", + &[(728, 90)], + &[], + ); + let evidence = evidence( + vec![dom("ad-header-0--container"), dom("ad-header-0-_R_abc123")], + Vec::new(), + Vec::new(), + ); + + let result = compare_page_evidence( + &[expected], + &evidence, + RuntimeGateSummary::unknown_allowed(), + ); + + assert_eq!( + result.slots[0].evidence.dom_id.as_deref(), + Some("ad-header-0-_R_abc123"), + "prefix match should skip -container" + ); + assert_eq!(result.slots[0].status, SlotStatus::Partial); + } + + #[test] + fn unmatched_gpt_slot_becomes_extra_evidence() { + let expected = expected_slot("atf", "ad-atf-", "/123/news/atf", &[(300, 250)], &[]); + let evidence = evidence( + vec![dom("ad-atf-0")], + vec![ + gpt_slot("/123/news/atf", "ad-atf-0", &[(300, 250)]), + gpt_slot( + "/123/publisher/right-rail", + "ad-right-rail-0", + &[(300, 250)], + ), + ], + Vec::new(), + ); + + let result = compare_page_evidence( + &[expected], + &evidence, + RuntimeGateSummary::unknown_allowed(), + ); + + assert_eq!(result.slots[0].status, SlotStatus::Confirmed); + assert_eq!(result.extra_evidence.len(), 1); + assert_eq!(result.extra_evidence[0].kind, "gpt"); + assert!( + !result.strict_failed(), + "extra evidence alone must not fail strict" + ); + } + + #[test] + fn auction_disabled_skips_strict_missing_failure() { + let expected = expected_slot("atf", "ad-atf-", "/123/news/atf", &[(300, 250)], &[]); + let evidence = evidence(Vec::new(), Vec::new(), Vec::new()); + + let result = compare_page_evidence( + &[expected], + &evidence, + RuntimeGateSummary::auction_disabled(), + ); + + assert_eq!(result.runtime_ad_stack_expected, RuntimeAdStackExpected::No); + assert_eq!(result.slots[0].status, SlotStatus::Missing); + assert!( + !result.strict_failed(), + "missing slot must not fail strict when ad stack is No" + ); + } + + #[test] + fn gpt_incompatible_sizes_is_partial() { + let expected = expected_slot("atf", "ad-atf-", "/123/news/atf", &[(300, 250)], &[]); + let evidence = evidence( + vec![dom("ad-atf-0")], + vec![gpt_slot("/123/news/atf", "ad-atf-0", &[(728, 90)])], + Vec::new(), + ); + + let result = compare_page_evidence( + &[expected], + &evidence, + RuntimeGateSummary::unknown_allowed(), + ); + + assert_eq!(result.slots[0].status, SlotStatus::Partial); + assert!( + result.slots[0] + .warnings + .iter() + .any(|w| w.code == "incompatible_sizes") + ); + } + + #[test] + fn non_banner_only_slot_is_unconfirmable_and_does_not_fail_strict() { + let expected = expected_slot_video("video", "ad-video-", "/123/news/video"); + let evidence = evidence( + vec![dom("ad-video-0")], + vec![gpt_slot("/123/news/video", "ad-video-0", &[(640, 480)])], + Vec::new(), + ); + + let result = compare_page_evidence( + &[expected], + &evidence, + RuntimeGateSummary::unknown_allowed(), + ); + + assert_eq!(result.slots[0].status, SlotStatus::Unconfirmable); + assert!( + result.slots[0] + .warnings + .iter() + .any(|w| w.code == "unsupported_format") + ); + assert!( + !result.strict_failed(), + "checker limitations should not fail strict" + ); + } + + #[test] + fn gpt_container_element_id_confirms() { + let expected = expected_slot("atf", "ad-atf-0", "/123/news/atf", &[(300, 250)], &[]); + let evidence = evidence( + vec![dom("ad-atf-0"), dom("ad-atf-0-container")], + vec![gpt_slot( + "/123/news/atf", + "ad-atf-0-container", + &[(300, 250)], + )], + Vec::new(), + ); + + let result = compare_page_evidence( + &[expected], + &evidence, + RuntimeGateSummary::unknown_allowed(), + ); + + assert_eq!( + result.slots[0].status, + SlotStatus::Confirmed, + "container element id is a valid GPT div match" + ); + } + + #[test] + fn sizeless_live_slot_is_partial_when_config_declares_banner_sizes() { + let expected = expected_slot( + "interstitial", + "ad-oop-", + "/123/news/oop", + &[(300, 250)], + &[], + ); + let evidence = evidence( + vec![dom("ad-oop-0")], + vec![gpt_slot("/123/news/oop", "ad-oop-0", &[])], + Vec::new(), + ); + + let result = compare_page_evidence( + &[expected], + &evidence, + RuntimeGateSummary::unknown_allowed(), + ); + + assert_eq!(result.slots[0].status, SlotStatus::Partial); + assert!( + result.slots[0] + .warnings + .iter() + .any(|w| w.code == "out_of_page_slot") + ); + assert!( + result.strict_failed(), + "a live sizeless slot drifting from configured banner sizes must fail strict" + ); + } + + #[test] + fn aps_match_adds_no_warning() { + let expected = expected_slot("atf", "ad-atf-", "/123/news/atf", &[(300, 250)], &["aps"]); + let evidence = evidence( + vec![dom("ad-atf-0")], + vec![gpt_slot("/123/news/atf", "ad-atf-0", &[(300, 250)])], + vec![aps("atf", &[(300, 250)])], + ); + + let result = compare_page_evidence( + &[expected], + &evidence, + RuntimeGateSummary::unknown_allowed(), + ); + + assert_eq!(result.slots[0].status, SlotStatus::Confirmed); + assert!( + !result.slots[0] + .warnings + .iter() + .any(|w| w.code.starts_with("aps_")), + "matching APS should not warn" + ); + } + + #[test] + fn server_side_aps_config_does_not_require_client_fetch_bids_evidence() { + let expected = expected_slot("atf", "ad-atf-", "/123/news/atf", &[(300, 250)], &["aps"]); + let evidence = evidence( + vec![dom("ad-atf-0")], + vec![gpt_slot("/123/news/atf", "ad-atf-0", &[(300, 250)])], + Vec::new(), + ); + + let result = compare_page_evidence( + &[expected], + &evidence, + RuntimeGateSummary::unknown_allowed(), + ); + + assert_eq!( + result.slots[0].status, + SlotStatus::Confirmed, + "missing APS does not flip status" + ); + assert!(result.slots[0].warnings.is_empty()); + assert!( + !result.strict_failed(), + "provider warning alone must not fail strict" + ); + } +} diff --git a/crates/trusted-server-cli/src/ad_templates/expected.rs b/crates/trusted-server-cli/src/ad_templates/expected.rs new file mode 100644 index 000000000..9392963ff --- /dev/null +++ b/crates/trusted-server-cli/src/ad_templates/expected.rs @@ -0,0 +1,336 @@ +//! Pure expected-slot projection from the runtime creative-opportunity matcher. +//! +//! This module owns path/URL normalization and converts the slots matched by +//! [`match_slots`] into stable, owned [`ExpectedSlot`] records for output and +//! browser-evidence comparison. It must not duplicate glob-matching semantics. + +use trusted_server_core::auction::types::MediaType; +use trusted_server_core::creative_opportunities::{CreativeOpportunitiesConfig, match_slots}; +use url::Url; + +/// The expected slots for a single page path, in configured slot order. +#[derive(Debug, Clone, PartialEq)] +pub struct ExpectedSlots { + /// The page path the slots were matched against. + pub path: String, + /// Matched slots projected into stable records, in configured order. + pub slots: Vec, +} + +/// A single configured slot expected to appear for a page path. +#[derive(Debug, Clone, PartialEq)] +pub struct ExpectedSlot { + /// The slot identifier. + pub id: String, + /// Resolved HTML `div` element ID (override or the slot id). + pub div_id: String, + /// Resolved GAM unit path: the rendered `gam_unit_path` template (or + /// `//` when the slot has none). + /// + /// `None` only for manually constructed comparison fixtures. Projection + /// omits a slot when the runtime cannot render it for this path. + pub gam_unit_path: Option, + /// Configured ad formats. + pub formats: Vec, + /// Configured provider names, in `aps`, `prebid` order. + pub providers: Vec, + /// Glob patterns configured for this slot. + pub page_patterns: Vec, +} + +/// A configured ad format as a stable width/height/media-type record. +#[derive(Debug, Clone, PartialEq)] +pub struct ExpectedFormat { + /// Creative width in pixels. + pub width: u32, + /// Creative height in pixels. + pub height: u32, + /// Configured media type. + pub media_type: MediaType, +} + +/// Projects the slots matching `path` into stable expected-slot records. +/// +/// Uses [`match_slots`] so glob semantics stay identical to the runtime, and +/// preserves configured slot order. `path` is assumed already normalized via +/// [`normalize_path_or_url`]. +/// +/// `gam_unit_path` templates are rendered against the section the runtime would +/// derive from `path` (per the config's `section_root`/`section_segment` +/// policy), so `{section}`-bearing configs project the same unit path the live +/// page requests. +// Shared projection used by the audit verifier; the static commands match slots +// directly against the runtime matcher. +#[must_use] +pub fn expected_slots_for_path(path: &str, config: &CreativeOpportunitiesConfig) -> ExpectedSlots { + let section = config.section_for_path(path); + let slots = match_slots(&config.slot, path) + .into_iter() + .filter_map(|slot| { + let gam_unit_path = slot.render_gam_unit_path(&config.gam_network_id, §ion)?; + Some(ExpectedSlot { + id: slot.id.clone(), + div_id: slot.resolved_div_id().to_string(), + gam_unit_path: Some(gam_unit_path), + formats: slot + .formats + .iter() + .map(|format| ExpectedFormat { + width: format.width, + height: format.height, + media_type: format.media_type.clone(), + }) + .collect(), + providers: provider_names(slot), + page_patterns: slot.page_patterns.clone(), + }) + }) + .collect(); + + ExpectedSlots { + path: path.to_string(), + slots, + } +} + +fn provider_names( + slot: &trusted_server_core::creative_opportunities::CreativeOpportunitySlot, +) -> Vec { + let mut providers = Vec::new(); + if slot.providers.aps.is_some() { + providers.push("aps".to_string()); + } + if slot.providers.prebid.is_some() { + providers.push("prebid".to_string()); + } + providers +} + +/// Normalizes a page path or full URL into a request path. +/// +/// Full `scheme://` inputs are parsed and reduced to their path; bare inputs have +/// query and fragment stripped and a leading `/` ensured. Empty paths become `/`. +/// +/// # Errors +/// +/// Returns a user-facing string when a `scheme://` input cannot be parsed as a URL. +pub fn normalize_path_or_url(input: &str) -> Result { + let path_input = input.split(['?', '#']).next().unwrap_or(input); + let scheme_prefix = path_input.split_once("://").map(|(scheme, _)| scheme); + let has_url_scheme = scheme_prefix.is_some_and(|scheme| { + let mut chars = scheme.chars(); + chars.next().is_some_and(|ch| ch.is_ascii_alphabetic()) + && chars.all(|ch| ch.is_ascii_alphanumeric() || matches!(ch, '+' | '-' | '.')) + }); + if has_url_scheme { + let url = Url::parse(input).map_err(|err| format!("invalid URL `{input}`: {err}"))?; + let path = url.path(); + return Ok(if path.is_empty() { + "/".to_string() + } else { + path.to_string() + }); + } + + let base = Url::parse("https://path-normalizer.example/") + .expect("should parse static path normalization base"); + let relative = input.trim_start_matches('/'); + let normalized = base + .join(&format!("./{relative}")) + .map_err(|error| format!("invalid path `{input}`: {error}"))?; + Ok(normalized.path().to_string()) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn creative_config_with_slots(patterns: &[&str]) -> CreativeOpportunitiesConfig { + let page_patterns = patterns + .iter() + .map(|pattern| format!("\"{pattern}\"")) + .collect::>() + .join(", "); + let toml = format!( + "gam_network_id = \"123\"\n\ + \n\ + [[slot]]\n\ + id = \"atf\"\n\ + gam_unit_path = \"/123/news/atf\"\n\ + div_id = \"ad-atf-\"\n\ + page_patterns = [{page_patterns}]\n\ + formats = [{{ width = 300, height = 250 }}]\n\ + \n\ + [slot.providers.prebid]\n\ + bidders = {{}}\n" + ); + let mut config = toml::from_str::(&toml) + .expect("should deserialize creative opportunities config"); + config.compile_slots(); + config + } + + #[test] + fn expected_slots_use_runtime_matcher_and_config_order() { + let config = creative_config_with_slots(&["/news/*", "/"]); + let expected = expected_slots_for_path("/news/story", &config); + + assert_eq!(expected.path, "/news/story"); + assert_eq!( + expected + .slots + .iter() + .map(|slot| slot.id.as_str()) + .collect::>(), + ["atf"] + ); + assert_eq!(expected.slots[0].div_id, "ad-atf-"); + assert_eq!( + expected.slots[0].gam_unit_path.as_deref(), + Some("/123/news/atf") + ); + assert_eq!(expected.slots[0].providers, ["prebid"]); + assert_eq!( + expected.slots[0].formats, + vec![ExpectedFormat { + width: 300, + height: 250, + media_type: MediaType::Banner, + }] + ); + } + + #[test] + fn expected_slots_default_resolution_without_overrides() { + let toml = "gam_network_id = \"42\"\n\ + \n\ + [[slot]]\n\ + id = \"footer\"\n\ + page_patterns = [\"/\"]\n\ + formats = [{ width = 728, height = 90 }]\n"; + let mut config = + toml::from_str::(toml).expect("should deserialize"); + config.compile_slots(); + + let expected = expected_slots_for_path("/", &config); + assert_eq!(expected.slots[0].div_id, "footer"); + assert_eq!( + expected.slots[0].gam_unit_path.as_deref(), + Some("/42/footer") + ); + assert!(expected.slots[0].providers.is_empty()); + } + + #[test] + fn expected_slots_render_section_templates_per_path() { + let toml = "gam_network_id = \"99999\"\n\ + section_root = \"homepage\"\n\ + \n\ + [[slot]]\n\ + id = \"ad-header-0\"\n\ + gam_unit_path = \"/{network_id}/example/{section}\"\n\ + page_patterns = [\"/\", \"/news\", \"/news/*\"]\n\ + formats = [{ width = 728, height = 90 }]\n"; + let mut config = + toml::from_str::(toml).expect("should deserialize"); + config.compile_slots(); + + // A path with a section segment renders that segment. + assert_eq!( + expected_slots_for_path("/news/story", &config).slots[0] + .gam_unit_path + .as_deref(), + Some("/99999/example/news"), + "a section template should render the path's section" + ); + // The site root falls back to the configured section_root. + assert_eq!( + expected_slots_for_path("/", &config).slots[0] + .gam_unit_path + .as_deref(), + Some("/99999/example/homepage"), + "the root path should render section_root" + ); + } + + #[test] + fn expected_slots_omit_dynamic_template_the_runtime_cannot_render() { + // A `{section}` template that renders past GAM's 100-byte unit-path + // limit. The runtime omits this slot for the request path, so diagnostics + // must not match it against a truncated or otherwise different path. + let toml = "gam_network_id = \"99999\"\n\ + section_root = \"homepage\"\n\ + \n\ + [[slot]]\n\ + id = \"ad-header-0\"\n\ + gam_unit_path = \"/{section}/{section}\"\n\ + page_patterns = [\"/*\"]\n\ + formats = [{ width = 728, height = 90 }]\n"; + let mut config = + toml::from_str::(toml).expect("should deserialize"); + config.compile_slots(); + + let long_path = format!("/{}", "a".repeat(60)); + let expected = expected_slots_for_path(&long_path, &config); + + assert!( + expected.slots.is_empty(), + "the runtime omits an over-limit dynamic slot on this path" + ); + } + + #[test] + fn normalize_path_or_url_strips_query_and_fragment() { + assert_eq!( + normalize_path_or_url("https://www.example.com/news/story?x=1#top") + .expect("should normalize"), + "/news/story" + ); + assert_eq!( + normalize_path_or_url("news/story?x=1").expect("should normalize"), + "/news/story" + ); + } + + #[test] + fn normalize_path_or_url_roots_empty_input() { + assert_eq!( + normalize_path_or_url("https://www.example.com").expect("should normalize"), + "/" + ); + assert_eq!(normalize_path_or_url("").expect("should normalize"), "/"); + } + + #[test] + fn normalize_path_or_url_uses_identical_url_rules_for_bare_paths() { + assert_eq!( + normalize_path_or_url("/a/../b").expect("should normalize bare dot segment"), + "/b" + ); + assert_eq!( + normalize_path_or_url("https://example.com/a/../b") + .expect("should normalize URL dot segment"), + "/b" + ); + assert_eq!( + normalize_path_or_url("/a b").expect("should encode bare path"), + "/a%20b" + ); + assert_eq!( + normalize_path_or_url("/r?to=https://example.com") + .expect("query URL should not change input classification"), + "/r" + ); + assert_eq!( + normalize_path_or_url("/news:latest").expect("colon should stay in bare path"), + "/news:latest", + "a colon in the first segment must not be parsed as a URL scheme" + ); + assert_eq!( + normalize_path_or_url("https://example.com/news:latest") + .expect("colon should stay in URL path"), + "/news:latest", + "bare and absolute forms should normalize identically" + ); + } +} diff --git a/crates/trusted-server-cli/src/ad_templates/mod.rs b/crates/trusted-server-cli/src/ad_templates/mod.rs new file mode 100644 index 000000000..3c26bf121 --- /dev/null +++ b/crates/trusted-server-cli/src/ad_templates/mod.rs @@ -0,0 +1,7 @@ +//! Pure, host-only ad-template CLI logic shared by the static `ts config +//! ad-templates ...` commands and the browser-backed `ts audit ad-templates +//! verify` command. + +pub mod compare; +pub mod expected; +pub mod output; diff --git a/crates/trusted-server-cli/src/ad_templates/output.rs b/crates/trusted-server-cli/src/ad_templates/output.rs new file mode 100644 index 000000000..afcf78ed0 --- /dev/null +++ b/crates/trusted-server-cli/src/ad_templates/output.rs @@ -0,0 +1,483 @@ +//! Stable, serializable output model for ad-template diagnostics. +//! +//! These types mirror the `--json` contract in +//! `docs/superpowers/specs/2026-06-26-server-side-ad-template-cli-design.md` §8. +//! Field names and declaration order are load-bearing: `serde` serializes struct +//! fields in declaration order, so the order here must match the spec examples. +//! +//! The model is consumed by the `ts audit ad-templates verify` orchestrator, +//! which assembles these wire types from the URL/gate context and comparison result. + +use std::borrow::Cow; + +use serde::{Deserialize, Serialize}; + +use trusted_server_core::creative_opportunities::RuntimeAdStackExpected; + +/// Escapes control characters in page-controlled text bound for a terminal. +/// +/// Page titles and collector warning messages are attacker-controlled: an +/// audited page can put ANSI/OSC escape sequences in `document.title` and drive +/// the operator's terminal (cursor movement, clipboard writes, forged output) +/// when the value is printed verbatim. Every C0 control (including ESC), DEL, +/// and the C1 range are rendered as `\u{XXXX}` so the text stays inert. JSON +/// output is unaffected — `serde_json` escapes these already. +/// +/// Returns a borrowed `Cow` when the input needs no escaping. +#[must_use] +pub fn escape_terminal_text(value: &str) -> Cow<'_, str> { + if !value.chars().any(is_terminal_control) { + return Cow::Borrowed(value); + } + let mut escaped = String::with_capacity(value.len()); + for ch in value.chars() { + if is_terminal_control(ch) { + escaped.push_str(&format!("\\u{{{:04X}}}", ch as u32)); + } else { + escaped.push(ch); + } + } + Cow::Owned(escaped) +} + +/// Whether `ch` can act as a terminal control code (C0, DEL, or C1). +fn is_terminal_control(ch: char) -> bool { + let code = ch as u32; + code < 0x20 + || (0x7f..=0x9f).contains(&code) + || (0x202a..=0x202e).contains(&code) + || (0x2066..=0x2069).contains(&code) +} + +/// Confirmation status for a single configured slot. +#[derive(Debug, Clone, Copy, Eq, PartialEq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum SlotStatus { + /// GPT evidence matches GAM path, div, and a compatible size. + Confirmed, + /// Some evidence, but not enough to confirm. + Partial, + /// No DOM or GPT evidence confirms the slot. + Missing, + /// The checker does not support confirming this slot type. + Unconfirmable, +} + +/// JSON rendering of the runtime ad-stack expectation. +#[derive(Debug, Clone, Copy, Eq, PartialEq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum RuntimeAdStackExpectedJson { + /// The server-side ad stack is expected to run. + Yes, + /// A known gate blocks the server-side ad stack. + No, + /// Consent or another gate is unprovable. + Unknown, +} + +impl From for RuntimeAdStackExpectedJson { + fn from(value: RuntimeAdStackExpected) -> Self { + match value { + RuntimeAdStackExpected::Yes => Self::Yes, + RuntimeAdStackExpected::No => Self::No, + RuntimeAdStackExpected::Unknown => Self::Unknown, + } + } +} + +/// State of a single runtime gate. +#[derive(Debug, Clone, Copy, Eq, PartialEq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum GateState { + /// The gate passed. + Pass, + /// The gate blocked the ad stack. + Fail, + /// The gate state could not be proven. + Unknown, +} + +/// Evidence-collection phase, rendered for JSON output. +#[derive(Debug, Clone, Copy, Eq, PartialEq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum EvidencePhaseJson { + /// Observed during the initial page load and settle. + InitialLoad, + /// Observed only after the deterministic scroll pass. + Scroll, +} + +/// A structured warning with a stable machine code and human message. +/// +/// `Serialize` for output; `Deserialize` because the browser collector payload +/// carries warning objects decoded into the comparison input. +#[derive(Debug, Clone, Eq, PartialEq, Serialize, Deserialize)] +pub struct Warning { + /// Stable machine-readable code (e.g. `dom_without_gpt`). + pub code: String, + /// Human-readable message; JSON consumers must not parse this. + pub message: String, +} + +/// Top-level `--json` document for `ts audit ad-templates verify`. +#[derive(Debug, Clone, PartialEq, Serialize)] +pub struct VerificationReport { + /// True when no strict failure and no page-level error occurred. + pub ok: bool, + /// Whether `--strict` was set. + pub strict: bool, + /// One entry per requested URL, in input order. + pub pages: Vec, + /// Run-level warnings not attributable to a single page. + /// + /// Always empty today — every warning the verifier raises belongs to a page + /// or a slot. Kept because the JSON schema declares it, so a consumer can + /// read it unconditionally. + pub warnings: Vec, +} + +/// A single audited page result. +/// +/// `error` is declared immediately after `path` so the serialized key order +/// matches the spec §8 `navigation_failed` shape; on normal pages it is `None` +/// and skipped, leaving the runtime/gates fields in §8 order. +#[derive(Debug, Clone, PartialEq, Serialize)] +pub struct PageJson { + /// The requested URL. + pub url: String, + /// The final URL after redirects, or `null` on navigation failure. + pub final_url: Option, + /// The requested URL's path. + pub requested_path: String, + /// The final path used for matching, or `null` on navigation failure. + pub path: Option, + /// Present only on a page-level collection failure. + #[serde(skip_serializing_if = "Option::is_none")] + pub error: Option, + /// Three-state runtime ad-stack expectation; absent on error pages. + #[serde(skip_serializing_if = "Option::is_none")] + pub runtime_ad_stack_expected: Option, + /// Per-gate evidence; absent on error pages. + #[serde(skip_serializing_if = "Option::is_none")] + pub gates: Option, + /// Number of configured slots matched for the final path; absent on error pages. + #[serde(skip_serializing_if = "Option::is_none")] + pub matched_slot_count: Option, + /// Per-slot verification results. + pub slots: Vec, + /// Live ad-slot evidence with no matching configured slot. + pub extra_evidence: Vec, + /// Page-level warnings. + pub warnings: Vec, +} + +/// Runtime gate states for a page, one field per spec §5.2 gate. +#[derive(Debug, Clone, PartialEq, Serialize)] +pub struct Gates { + /// Request method is `GET`. + pub method_get: GateState, + /// Request is a top-level navigation. + pub navigation: GateState, + /// Request is not a prefetch. + pub not_prefetch: GateState, + /// Request is not from a known bot. + pub not_bot: GateState, + /// At least one configured slot matched the final path. + pub matched_slots: GateState, + /// The `[auction].enabled` kill switch is on. + pub auction_enabled: GateState, + /// The `[creative_opportunities].enabled` template switch is on. + pub ad_templates_enabled: GateState, + /// Consent allows the auction (often `unknown` for live requests). + pub consent_allows_auction: GateState, +} + +/// A single configured slot's verification result. +#[derive(Debug, Clone, PartialEq, Serialize)] +pub struct SlotJson { + /// The configured slot id. + pub id: String, + /// The slot's confirmation status. + pub status: SlotStatus, + /// The phase the confirming evidence was observed in. + #[serde(skip_serializing_if = "Option::is_none")] + pub phase: Option, + /// The configured shape of the slot (no `id`/`page_patterns` per §8). + pub configured: ConfiguredJson, + /// The live evidence observed for this slot. + pub evidence: SlotEvidenceJson, + /// Slot-level warnings (e.g. provider or size warnings). + pub warnings: Vec, +} + +/// The configured shape of a slot, as rendered in §8 `configured`. +#[derive(Debug, Clone, PartialEq, Serialize)] +pub struct ConfiguredJson { + /// Resolved div element ID. + pub div_id: String, + /// Resolved GAM unit path, or `null` when a dynamic template renders past + /// GAM's unit-path byte limit for this page's section. + pub gam_unit_path: Option, + /// Configured formats. + pub formats: Vec, + /// Configured provider names. + pub providers: Vec, +} + +/// A configured format, as rendered in §8. +#[derive(Debug, Clone, PartialEq, Serialize)] +pub struct FormatJson { + /// Creative width in pixels. + pub width: u32, + /// Creative height in pixels. + pub height: u32, + /// Media type string (`banner`, `video`, `native`). + pub media_type: String, +} + +/// Live evidence observed for a configured slot. +#[derive(Debug, Clone, PartialEq, Serialize)] +pub struct SlotEvidenceJson { + /// The resolved DOM element ID observed, if any. + pub dom_id: Option, + /// GPT slot evidence, if any (no `phase` key per §8). + pub gpt: Option, +} + +/// GPT slot evidence, as rendered in §8 `evidence.gpt`. +#[derive(Debug, Clone, PartialEq, Serialize)] +pub struct GptEvidenceJson { + /// The observed GAM ad unit path. + pub gam_unit_path: String, + /// The observed GPT slot element ID. + pub div_id: String, + /// Observed numeric sizes as `[width, height]` pairs. + pub sizes: Vec<[u32; 2]>, +} + +/// Live ad-slot evidence with no matching configured slot. +#[derive(Debug, Clone, PartialEq, Serialize)] +pub struct ExtraEvidenceJson { + /// Evidence kind: `dom`, `gpt`, or `aps`. + pub kind: String, + /// The phase the evidence was observed in. + pub phase: EvidencePhaseJson, + /// The DOM element ID, if any. + pub dom_id: Option, + /// The GAM unit path, if any. + pub gam_unit_path: Option, + /// Observed numeric sizes as `[width, height]` pairs. + pub sizes: Vec<[u32; 2]>, + /// Why this evidence is reported as extra. + pub reason: String, +} + +#[cfg(test)] +impl VerificationReport { + fn example_confirmed_with_extra_evidence() -> Self { + VerificationReport { + ok: true, + strict: false, + pages: vec![PageJson { + url: "https://www.example.com/news/story".to_string(), + final_url: Some("https://www.example.com/news/story".to_string()), + requested_path: "/news/story".to_string(), + path: Some("/news/story".to_string()), + error: None, + runtime_ad_stack_expected: Some(RuntimeAdStackExpectedJson::Unknown), + gates: Some(Gates { + method_get: GateState::Pass, + navigation: GateState::Pass, + not_prefetch: GateState::Pass, + not_bot: GateState::Pass, + matched_slots: GateState::Pass, + auction_enabled: GateState::Pass, + ad_templates_enabled: GateState::Pass, + consent_allows_auction: GateState::Unknown, + }), + matched_slot_count: Some(1), + slots: vec![SlotJson { + id: "atf".to_string(), + status: SlotStatus::Confirmed, + phase: Some(EvidencePhaseJson::InitialLoad), + configured: ConfiguredJson { + div_id: "ad-atf-".to_string(), + gam_unit_path: Some("/123/news/atf".to_string()), + formats: vec![FormatJson { + width: 300, + height: 250, + media_type: "banner".to_string(), + }], + providers: vec!["aps".to_string()], + }, + evidence: SlotEvidenceJson { + dom_id: Some("ad-atf-0".to_string()), + gpt: Some(GptEvidenceJson { + gam_unit_path: "/123/news/atf".to_string(), + div_id: "ad-atf-0".to_string(), + sizes: vec![[300, 250]], + }), + }, + warnings: Vec::new(), + }], + extra_evidence: vec![ExtraEvidenceJson { + kind: "gpt".to_string(), + phase: EvidencePhaseJson::InitialLoad, + dom_id: Some("ad-right-rail-0".to_string()), + gam_unit_path: Some("/123/publisher/right-rail".to_string()), + sizes: vec![[300, 250]], + reason: "no_configured_slot_matched".to_string(), + }], + warnings: vec![Warning { + code: "redirected".to_string(), + message: "navigation redirected to the final path".to_string(), + }], + }], + warnings: Vec::new(), + } + } + + fn example_navigation_failed() -> Self { + VerificationReport { + ok: false, + strict: false, + pages: vec![PageJson { + url: "https://www.example.com/broken".to_string(), + final_url: None, + requested_path: "/broken".to_string(), + path: None, + error: Some(Warning { + code: "navigation_failed".to_string(), + message: "failed to read main document navigation response".to_string(), + }), + runtime_ad_stack_expected: None, + gates: None, + matched_slot_count: None, + slots: Vec::new(), + extra_evidence: Vec::new(), + warnings: Vec::new(), + }], + warnings: Vec::new(), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn escape_terminal_text_passes_through_ordinary_titles() { + assert!( + matches!( + escape_terminal_text("Example News — Story"), + Cow::Borrowed(_) + ), + "text with no control characters should not allocate" + ); + assert_eq!( + escape_terminal_text("Example News — Story"), + "Example News — Story" + ); + } + + #[test] + fn escape_terminal_text_neutralizes_control_sequences() { + // ESC-based CSI/OSC sequences and a raw newline are the terminal-driving + // primitives a hostile page would put in `document.title`. + assert_eq!( + escape_terminal_text("a\u{1b}]0;pwned\u{7}b"), + "a\\u{001B}]0;pwned\\u{0007}b", + "ESC and BEL should be rendered inert" + ); + assert_eq!( + escape_terminal_text("line\nforged: ok"), + "line\\u{000A}forged: ok", + "a newline should not let a title forge an output line" + ); + assert_eq!( + escape_terminal_text("del\u{7f}c1\u{9b}"), + "del\\u{007F}c1\\u{009B}", + "DEL and the C1 range should be escaped too" + ); + assert_eq!( + escape_terminal_text("safe\u{202E}forged\u{2066}tail"), + "safe\\u{202E}forged\\u{2066}tail", + "Unicode bidi controls should be rendered inert" + ); + } + + #[test] + fn verification_json_contains_gate_state_and_extra_evidence() { + let result = VerificationReport::example_confirmed_with_extra_evidence(); + let value = serde_json::to_value(&result).expect("should serialize"); + + assert_eq!(value["ok"], true); + assert_eq!(value["pages"][0]["requested_path"], "/news/story"); + assert_eq!(value["pages"][0]["runtime_ad_stack_expected"], "unknown"); + assert_eq!( + value["pages"][0]["gates"]["consent_allows_auction"], + "unknown" + ); + assert_eq!(value["pages"][0]["slots"][0]["status"], "confirmed"); + assert_eq!( + value["pages"][0]["slots"][0]["evidence"]["gpt"]["sizes"][0][0], + 300 + ); + assert_eq!(value["pages"][0]["extra_evidence"][0]["kind"], "gpt"); + assert_eq!(value["pages"][0]["warnings"][0]["code"], "redirected"); + // `configured` excludes id/page_patterns per §8. + assert!(value["pages"][0]["slots"][0]["configured"]["id"].is_null()); + assert!(value["pages"][0]["slots"][0]["configured"]["page_patterns"].is_null()); + // `evidence.gpt` has no `phase` key per §8. + assert!(value["pages"][0]["slots"][0]["evidence"]["gpt"]["phase"].is_null()); + } + + #[test] + fn page_error_json_matches_navigation_failed_shape() { + let result = VerificationReport::example_navigation_failed(); + let value = serde_json::to_value(&result).expect("should serialize"); + let page = &value["pages"][0]; + + assert_eq!(page["error"]["code"], "navigation_failed"); + assert!(page["final_url"].is_null(), "final_url should be null"); + assert!(page["path"].is_null(), "path should be null"); + assert!( + page.get("runtime_ad_stack_expected").is_none(), + "runtime field absent on error page" + ); + assert!(page.get("gates").is_none(), "gates absent on error page"); + assert!( + page.get("matched_slot_count").is_none(), + "matched_slot_count absent on error page" + ); + assert_eq!(value["ok"], false); + } + + #[test] + fn missing_slot_json_omits_evidence_phase() { + let slot = SlotJson { + id: "missing".to_string(), + status: SlotStatus::Missing, + phase: None, + configured: ConfiguredJson { + div_id: "ad-missing-".to_string(), + gam_unit_path: Some("/123/publisher/missing".to_string()), + formats: Vec::new(), + providers: Vec::new(), + }, + evidence: SlotEvidenceJson { + dom_id: None, + gpt: None, + }, + warnings: Vec::new(), + }; + + let value = serde_json::to_value(slot).expect("should serialize missing slot"); + + assert!( + value.get("phase").is_none(), + "missing evidence should not claim an initial-load phase" + ); + } +} diff --git a/crates/trusted-server-cli/src/app_config.rs b/crates/trusted-server-cli/src/app_config.rs new file mode 100644 index 000000000..bee536146 --- /dev/null +++ b/crates/trusted-server-cli/src/app_config.rs @@ -0,0 +1,171 @@ +//! Shared effective Trusted Server app-config loading for the `ts` CLI. +//! +//! Both the static `ts config ad-templates ...` commands and the browser-backed +//! `ts audit ad-templates verify` command load the same effective app config +//! through [`load_settings`], so config-path resolution and the `EdgeZero` +//! environment overlay stay consistent across command families. + +use std::path::{Path, PathBuf}; + +use clap::Args; +use edgezero_core::app_config::{self, AppConfigLoadOptions}; +use edgezero_core::manifest::ManifestLoader; +use trusted_server_core::config::TrustedServerAppConfig; +use trusted_server_core::settings::Settings; + +/// Shared local app-config flags accepted by every config/audit ad-template command. +#[derive(Clone, Debug, Args)] +pub struct AppConfigArgs { + /// Path to `trusted-server.toml`. Defaults to `.toml` beside `edgezero.toml`. + #[arg(long)] + pub app_config: Option, + /// Path to `edgezero.toml`. + #[arg(long, default_value = "edgezero.toml")] + pub manifest: PathBuf, + /// Skip app-config environment overlay. + #[arg(long)] + pub no_env: bool, +} + +/// Effective settings plus the resolved app-config path they were loaded from. +#[derive(Debug)] +pub struct LoadedSettings { + /// The `trusted-server.toml` path the settings were loaded from. + pub app_config_path: PathBuf, + /// The deserialized effective settings. + pub settings: Settings, +} + +/// Loads the effective Trusted Server settings described by `args`. +/// +/// Resolves the app-config path from `args` (or the manifest's `.toml` +/// default), applies the `EdgeZero` environment overlay unless `no_env` is set, and +/// returns the deserialized [`Settings`]. +/// +/// # Errors +/// +/// Returns a user-facing string when the manifest cannot be loaded, has no +/// `[app].name`, or the resolved app-config file cannot be read or parsed. When an +/// explicit `--app-config` path is given and is missing, the error names that +/// exact path rather than silently falling back. +pub fn load_settings(args: &AppConfigArgs) -> Result { + load_settings_with_env_overlay(args, !args.no_env) +} + +/// Loads Trusted Server settings from the resolved app-config file without +/// applying environment overlays. +/// +/// Mutating commands use this path so environment-only values are never +/// persisted into the operator-owned TOML file. +/// +/// # Errors +/// +/// Returns the same path-resolution, read, and parse errors as +/// [`load_settings`]. +#[cfg(test)] +pub(crate) fn load_file_settings(args: &AppConfigArgs) -> Result { + load_settings_with_env_overlay(args, false) +} + +/// Resolves the operator-owned app-config path without deserializing settings. +/// +/// Mutating recovery commands use this when the existing config may already be +/// invalid but still needs a narrowly scoped structural repair. +/// +/// # Errors +/// +/// Returns a user-facing string when the manifest cannot be loaded or has no +/// `[app].name` and no explicit config path was supplied. +pub fn resolve_app_config_file(args: &AppConfigArgs) -> Result { + if let Some(path) = &args.app_config { + return Ok(path.clone()); + } + let manifest_loader = ManifestLoader::from_path(&args.manifest) + .map_err(|err| format!("failed to load {}: {err}", args.manifest.display()))?; + let app_name = manifest_loader.manifest().app.name.clone().ok_or_else(|| { + format!( + "{} has no [app].name; cannot resolve trusted-server.toml", + args.manifest.display() + ) + })?; + Ok(resolve_app_config_path(None, &args.manifest, &app_name)) +} + +fn load_settings_with_env_overlay( + args: &AppConfigArgs, + env_overlay: bool, +) -> Result { + let manifest_loader = ManifestLoader::from_path(&args.manifest) + .map_err(|err| format!("failed to load {}: {err}", args.manifest.display()))?; + let app_name = manifest_loader.manifest().app.name.clone().ok_or_else(|| { + format!( + "{} has no [app].name; cannot resolve trusted-server.toml", + args.manifest.display() + ) + })?; + let app_config_path = + resolve_app_config_path(args.app_config.as_deref(), &args.manifest, &app_name); + + let mut opts = AppConfigLoadOptions::default(); + opts.env_overlay = env_overlay; + let app_config = app_config::deserialize_app_config_with_options::( + &app_config_path, + &app_name, + &opts, + ) + .map_err(|err| format!("failed to load {}: {err}", app_config_path.display()))?; + + Ok(LoadedSettings { + app_config_path, + settings: app_config.into_settings(), + }) +} + +fn resolve_app_config_path( + explicit: Option<&Path>, + manifest_path: &Path, + app_name: &str, +) -> PathBuf { + if let Some(path) = explicit { + return path.to_path_buf(); + } + let file_name = format!("{app_name}.toml"); + if let Some(parent) = manifest_path + .parent() + .filter(|parent| !parent.as_os_str().is_empty()) + { + parent.join(file_name) + } else { + PathBuf::from(file_name) + } +} + +#[cfg(test)] +mod tests { + use std::fs; + + use tempfile::TempDir; + + use super::*; + + #[test] + fn explicit_missing_app_config_does_not_fall_back() { + let temp = TempDir::new().expect("should create temp dir"); + let manifest_path = temp.path().join("edgezero.toml"); + fs::write(&manifest_path, "[app]\nname = \"trusted-server\"\n") + .expect("should write manifest"); + let missing_path = temp.path().join("missing.toml"); + + let args = AppConfigArgs { + app_config: Some(missing_path.clone()), + manifest: manifest_path, + no_env: true, + }; + + let err = load_settings(&args).expect_err("should reject missing explicit config"); + assert!( + err.contains(missing_path.to_string_lossy().as_ref()), + "error should mention the explicit missing path" + ); + } +} diff --git a/crates/trusted-server-cli/src/commands/audit/ad_template_collector.js b/crates/trusted-server-cli/src/commands/audit/ad_template_collector.js new file mode 100644 index 000000000..6938808f5 --- /dev/null +++ b/crates/trusted-server-cli/src/commands/audit/ad_template_collector.js @@ -0,0 +1,241 @@ +// Bounded ad-template evidence collector, injected before publisher scripts run. +// +// This body runs inside an IIFE that defines `__TS_CONFIG` (the configured div +// prefixes). It records evidence into `window.__tsAdTemplateEvidence` +// and never captures page HTML, cookies, storage, request bodies, or arbitrary DOM. +// It always calls original page functions with unchanged arguments and never +// spoofs the browser automation flag. + +const __ts_config = typeof __TS_CONFIG === "object" && __TS_CONFIG ? __TS_CONFIG : {} +const __ts_prefixes = Array.isArray(__ts_config.div_prefixes) ? __ts_config.div_prefixes : [] + +const __ts_ev = (window.__tsAdTemplateEvidence = window.__tsAdTemplateEvidence || { + dom_ids: [], + gpt_slots: [], + aps_calls: [], + warnings: [] +}) + +const __ts_phase = () => (window.__tsScrollPhase ? "scroll" : "initial_load") + +// Hard cap per evidence list so a hostile page cannot grow the store without +// bound; the page controls how many slots/elements/warnings it produces. +const __ts_max_entries = 128 +const __ts_max_string_length = 512 +const __ts_wrapped_googletags = new WeakSet() + +function __ts_text(value) { + return String(value).slice(0, __ts_max_string_length) +} + +// Truncation has to be visible: surplus configured slots classify Missing, and +// `--strict` counts that, so a silent drop is indistinguishable from real drift. +let __ts_truncated = false +function __ts_push(list, entry) { + if (list.length < __ts_max_entries) { + list.push(entry) + return + } + if (__ts_truncated) return + __ts_truncated = true + if (__ts_ev.warnings.length < __ts_max_entries) { + __ts_ev.warnings.push({ + code: "evidence_truncated", + message: "an evidence list hit the " + __ts_max_entries + "-entry cap; results are incomplete" + }) + } +} + +function __ts_warn(code, error) { + __ts_push(__ts_ev.warnings, { code, message: __ts_text(error) }) +} + +// GPT sizes reach Rust as u32 pairs, so anything non-integral (fluid slots, +// NaN, negative or fractional dimensions) must be dropped here — a single bad +// pair would fail deserialization of the whole evidence payload and discard +// every other slot's otherwise valid evidence. +function __ts_size_pair(width, height) { + if (!Number.isInteger(width) || !Number.isInteger(height)) return null + if (width < 0 || height < 0 || width > 4294967295 || height > 4294967295) return null + return [width, height] +} + +function __ts_warn_ignored_size(width, height) { + const numeric = Number.isInteger(width) && Number.isInteger(height) + const outOfRange = + numeric && (width < 0 || height < 0 || width > 4294967295 || height > 4294967295) + __ts_push(__ts_ev.warnings, { + code: outOfRange ? "size_out_of_range" : "fluid_size_ignored", + message: outOfRange ? "GPT size outside u32 range ignored" : "non-integer GPT size ignored" + }) +} + +function __ts_normalize_sizes(sizes) { + const out = [] + if (!Array.isArray(sizes)) return out + // Accept [w, h] or [[w, h], ...]; treat numeric-leading arrays as a single pair. + const pairs = typeof sizes[0] === "number" ? [sizes] : sizes + for (const size of pairs) { + if (out.length >= __ts_max_entries) break + const pair = Array.isArray(size) ? __ts_size_pair(size[0], size[1]) : null + if (pair) { + out.push(pair) + } else { + __ts_warn_ignored_size( + Array.isArray(size) ? size[0] : undefined, + Array.isArray(size) ? size[1] : undefined + ) + } + } + return out +} + +function __ts_record_define_slot(adUnitPath, sizes, divId) { + __ts_push(__ts_ev.gpt_slots, { + gam_unit_path: __ts_text(adUnitPath), + div_id: __ts_text(divId), + sizes: __ts_normalize_sizes(sizes), + phase: __ts_phase() + }) +} + +function __ts_wrap_googletag(googletag) { + if (!googletag || (typeof googletag !== "object" && typeof googletag !== "function")) { + return googletag + } + if (__ts_wrapped_googletags.has(googletag)) return googletag + __ts_wrapped_googletags.add(googletag) + // Wrap defineSlot so both direct calls and calls dispatched from the cmd queue + // are recorded (queued callbacks call this same wrapped function). + const originalDefineSlot = googletag.defineSlot + if (typeof originalDefineSlot === "function") { + try { + const descriptor = Object.getOwnPropertyDescriptor(googletag, "defineSlot") + Object.defineProperty(googletag, "defineSlot", { + configurable: true, + enumerable: descriptor ? descriptor.enumerable : true, + writable: true, + value: function (adUnitPath, sizes, divId) { + const slot = originalDefineSlot.apply(this, arguments) + try { + __ts_record_define_slot(adUnitPath, sizes, divId) + } catch (error) { + __ts_warn("define_slot_capture_failed", error) + } + return slot + } + }) + } catch (error) { + __ts_warn("define_slot_wrap_failed", error) + } + } + return googletag +} + +// Wrap an existing global or intercept a later assignment of it. +function __ts_install(name, wrap) { + if (window[name]) { + try { + wrap(window[name]) + } catch (error) { + __ts_warn(name + "_wrap_failed", error) + } + return + } + let internal + Object.defineProperty(window, name, { + configurable: true, + // A real `window.googletag` is an ordinary enumerable global; matching that + // keeps `Object.keys(window)` identical with and without the collector. + enumerable: true, + get() { + return internal + }, + set(value) { + internal = value + try { + internal = wrap(value) + } catch (error) { + __ts_warn(name + "_wrap_failed", error) + } + } + }) +} + +__ts_install("googletag", __ts_wrap_googletag) + +// On-demand DOM + getSlots scrape, invoked by the collector after settle/scroll. +window.__tsCollectAdTemplateEvidence = function () { + try { + const seen = new Set(__ts_ev.dom_ids.map((entry) => entry.dom_id)) + for (const element of document.querySelectorAll("[id]")) { + const id = __ts_text(element.id) + if (id.endsWith("-container")) continue + if (__ts_prefixes.some((prefix) => id.startsWith(prefix)) && !seen.has(id)) { + __ts_push(__ts_ev.dom_ids, { dom_id: id, phase: __ts_phase() }) + seen.add(id) + } + } + const googletag = window.googletag + if (googletag && typeof googletag.pubads === "function") { + const pubads = googletag.pubads() + const slots = typeof pubads.getSlots === "function" ? pubads.getSlots() : [] + for (const slot of slots) { + try { + const path = typeof slot.getAdUnitPath === "function" ? slot.getAdUnitPath() : "" + const divId = typeof slot.getSlotElementId === "function" ? slot.getSlotElementId() : "" + const rawSizes = typeof slot.getSizes === "function" ? slot.getSizes() : [] + const sizes = [] + for (const size of rawSizes) { + if (sizes.length >= __ts_max_entries) break + let pair = null + if ( + size && + typeof size.getWidth === "function" && + typeof size.getHeight === "function" + ) { + // A fluid GPT size answers getWidth()/getHeight() with a + // non-numeric value rather than throwing. + pair = __ts_size_pair(size.getWidth(), size.getHeight()) + } else if (Array.isArray(size)) { + pair = __ts_size_pair(size[0], size[1]) + } + if (pair) { + sizes.push(pair) + } else { + const width = + size && typeof size.getWidth === "function" + ? size.getWidth() + : Array.isArray(size) + ? size[0] + : undefined + const height = + size && typeof size.getHeight === "function" + ? size.getHeight() + : Array.isArray(size) + ? size[1] + : undefined + __ts_warn_ignored_size(width, height) + } + } + const exists = __ts_ev.gpt_slots.some( + (entry) => entry.gam_unit_path === __ts_text(path) && entry.div_id === __ts_text(divId) + ) + if (!exists) { + __ts_push(__ts_ev.gpt_slots, { + gam_unit_path: __ts_text(path), + div_id: __ts_text(divId), + sizes, + phase: __ts_phase() + }) + } + } catch (error) { + __ts_warn("gpt_scrape_failed", error) + } + } + } + } catch (error) { + __ts_warn("collect_failed", error) + } + return __ts_ev +} diff --git a/crates/trusted-server-cli/src/commands/audit/ad_templates.rs b/crates/trusted-server-cli/src/commands/audit/ad_templates.rs new file mode 100644 index 000000000..0e2b51371 --- /dev/null +++ b/crates/trusted-server-cli/src/commands/audit/ad_templates.rs @@ -0,0 +1,1014 @@ +//! Browser-backed `ts audit ad-templates verify` orchestration. +//! +//! For each URL: collect live evidence through an [`AuditCollector`], match +//! configured slots against the **final** (post-redirect) path, evaluate the +//! runtime gate, compare evidence, and assemble the stable §8 wire result. The +//! orchestration is collector-agnostic so it is fully tested with an in-memory +//! fake collector, with no Chrome dependency. + +use std::io::{self, Write}; + +use trusted_server_core::auction::types::MediaType; +use trusted_server_core::creative_opportunities::{ + AdStackGateInput, CreativeOpportunitiesConfig, evaluate_ad_stack_gate, +}; + +use crate::ad_templates::compare::{ + BrowserAdEvidence, EvidencePhase, ExtraEvidence, RuntimeGateSummary, SlotEvidence, SlotResult, + SlotStatus as CompareStatus, compare_page_evidence, +}; +use crate::ad_templates::expected::{ExpectedSlot, expected_slots_for_path, normalize_path_or_url}; +use crate::ad_templates::output::{ + ConfiguredJson, EvidencePhaseJson, ExtraEvidenceJson, FormatJson, GateState, Gates, + GptEvidenceJson, PageJson, RuntimeAdStackExpectedJson, SlotEvidenceJson, SlotJson, SlotStatus, + VerificationReport, Warning, escape_terminal_text, +}; +use crate::commands::audit::AuditAdTemplatesVerifyArgs; +use crate::commands::audit::collector::{ + AdTemplateCollectorConfig, AuditCollector, BrowserCollectRequest, build_ad_template_init_script, +}; +use crate::run::RunOutcome; + +/// Verifies configured ad-template slots against live page evidence. +/// +/// # Errors +/// +/// Returns a user-facing string when config loading fails, or when verification +/// surfaces a page-level error or a `--strict` failure (after writing output). +pub(crate) fn run_verify(args: &AuditAdTemplatesVerifyArgs) -> Result { + args.browser.validate()?; + validate_cookie_scope(&args.urls, &args.cookies)?; + let loaded = crate::app_config::load_settings(&args.config)?; + let collector = crate::commands::audit::browser::BrowserCollector::from_opts(&args.browser); + let report = build_report( + &collector, + loaded.settings.creative_opportunities.as_ref(), + loaded.settings.auction.enabled, + &args.urls, + VerifyOptions { + strict: args.strict, + scroll: args.scroll, + allow_cross_origin_redirect: args.allow_cross_origin_redirect, + }, + &args.cookies, + )?; + + let stdout = io::stdout(); + let mut out = stdout.lock(); + if args.json { + write_json(&mut out, &report)?; + } else { + write_human(&mut out, &report)?; + } + + if report.pages.iter().any(|page| page.error.is_some()) { + Err("ad-template verification reported problems".to_string()) + } else if report.ok { + Ok(RunOutcome::Success) + } else { + Ok(RunOutcome::AssertionFailed) + } +} + +fn validate_cookie_scope(urls: &[url::Url], cookies: &[(String, String)]) -> Result<(), String> { + if cookies.is_empty() { + return Ok(()); + } + let origins: std::collections::BTreeSet = urls + .iter() + .map(|url| url.origin().ascii_serialization()) + .collect(); + if origins.len() > 1 { + return Err( + "--cookie may be used only when every verification URL has one origin; split this run so credentials are never copied to another origin" + .to_string(), + ); + } + Ok(()) +} + +/// Run-level verification switches. +#[derive(Debug, Clone, Copy)] +struct VerifyOptions { + /// Exit non-zero when a matched slot is missing or only partially confirmed. + strict: bool, + /// Perform a deterministic scroll pass after the initial settle. + scroll: bool, + /// Accept evidence from a page that redirected to a different origin. + allow_cross_origin_redirect: bool, +} + +/// Builds the verification report for `urls` using `collector`. +/// +/// `creative` is the effective `[creative_opportunities]` config (if any) and +/// `auction_enabled` is the `[auction].enabled` kill switch. +fn build_report( + collector: &dyn AuditCollector, + creative: Option<&CreativeOpportunitiesConfig>, + auction_enabled: bool, + urls: &[url::Url], + options: VerifyOptions, + cookies: &[(String, String)], +) -> Result { + let init_script = build_init_script(creative)?; + + let requests: Vec<_> = urls + .iter() + .map(|url| BrowserCollectRequest { + url: url.clone(), + init_scripts: vec![init_script.clone()], + scroll: options.scroll, + collect_ad_evidence: true, + cookies: cookies.to_vec(), + }) + .collect(); + let collected_pages = collector.collect_pages(&requests); + + let mut pages = Vec::with_capacity(urls.len()); + let mut any_error = false; + let mut any_strict_fail = false; + + for (url, collected) in urls.iter().zip(collected_pages) { + match collected { + Err(message) => { + any_error = true; + pages.push(error_page(url, &message)); + } + // Slots are matched on the *final* path, so a redirect to a + // different origin would let an unrelated site's evidence satisfy + // `--strict` — and the path-equality redirect warning would not even + // fire when the paths happen to agree. Reject unless opted in. + Ok(collected) + if !options.allow_cross_origin_redirect + && origin_changed(url, &collected.final_url) => + { + any_error = true; + pages.push(cross_origin_page(url, &collected.final_url)); + } + Ok(collected) => { + let (page, strict_failed) = build_page(url, &collected, creative, auction_enabled); + if options.strict && strict_failed { + any_strict_fail = true; + } + pages.push(page); + } + } + } + + let ok = !(any_error || (options.strict && any_strict_fail)); + Ok(VerificationReport { + ok, + strict: options.strict, + pages, + warnings: Vec::new(), + }) +} + +/// The URL without its fragment, for comparisons the server can observe. +pub(super) fn without_fragment(url: &url::Url) -> url::Url { + let mut url = url.clone(); + url.set_fragment(None); + url +} + +/// Whether navigation left the requested URL's origin (scheme, host, or port). +/// +/// A same-host default-port `http:80` to `https:443` redirect is *not* a change: +/// the host is the cookie boundary, and that upgrade is the ordinary canonical +/// redirect. Host changes, port changes, and HTTPS downgrades all are. +pub(super) fn origin_changed(requested: &url::Url, final_url: &url::Url) -> bool { + if requested.host_str() != final_url.host_str() { + return true; + } + + match (requested.scheme(), final_url.scheme()) { + ("http", "https") => { + requested.port_or_known_default() != Some(80) + || final_url.port_or_known_default() != Some(443) + } + (requested_scheme @ ("http" | "https"), final_scheme) + if requested_scheme == final_scheme => + { + requested.port_or_known_default() != final_url.port_or_known_default() + } + // Refuse HTTPS downgrades and any unexpected scheme transition. + _ => true, + } +} + +/// Builds the read-only collector init script from the configured slots. +fn build_init_script(creative: Option<&CreativeOpportunitiesConfig>) -> Result { + let config = AdTemplateCollectorConfig { + div_prefixes: creative + .map(|creative| { + creative + .slot + .iter() + .map(|slot| slot.resolved_div_id().to_string()) + .collect() + }) + .unwrap_or_default(), + }; + build_ad_template_init_script(&config) +} + +/// Assembles a successful page result, returning the wire `PageJson` and whether +/// the page would fail `--strict`. +fn build_page( + requested: &url::Url, + collected: &crate::commands::audit::collector::CollectedPage, + creative: Option<&CreativeOpportunitiesConfig>, + auction_enabled: bool, +) -> (PageJson, bool) { + let requested_path = normalize_path_or_url(requested.as_str()).unwrap_or_else(|_| "/".into()); + let final_url = &collected.final_url; + let final_path = normalize_path_or_url(final_url.as_str()).unwrap_or_else(|_| "/".into()); + + let expected = creative + .map(|creative| expected_slots_for_path(&final_path, creative).slots) + .unwrap_or_default(); + let matched = !expected.is_empty(); + + let gate = evaluate_ad_stack_gate(AdStackGateInput { + method_get: true, + navigation: true, + prefetch: false, + bot: false, + matched_slots: matched, + consent_allows_auction: None, + auction_enabled, + // Absent creative opportunities block here as they do at runtime. + ad_templates_enabled: creative.is_some_and(|creative| creative.enabled), + }); + + let evidence = collected.ad_evidence.clone().unwrap_or_else(empty_evidence); + let result = compare_page_evidence( + &expected, + &evidence, + RuntimeGateSummary::from_expected(gate.expected), + ); + let strict_failed = result.strict_failed(); + + let mut warnings: Vec = collected.warnings.to_vec(); + warnings.extend(evidence.warnings.iter().map(|warning| Warning { + code: format!("page_{}", warning.code), + message: warning.message.clone(), + })); + // Fragments never reach the server, so a fragment-only difference is not a + // redirect and slots match on the path either way. + if without_fragment(requested) != without_fragment(final_url) { + warnings.push(Warning { + code: "redirected".to_string(), + message: format!("navigation redirected from {requested} to {final_url}"), + }); + } + + let slots = expected + .iter() + .zip(result.slots.iter()) + .map(|(expected_slot, slot_result)| to_slot_json(expected_slot, slot_result)) + .collect(); + let extra_evidence = result.extra_evidence.iter().map(to_extra_json).collect(); + + let page = PageJson { + url: requested.to_string(), + final_url: Some(final_url.to_string()), + requested_path, + path: Some(final_path), + error: None, + runtime_ad_stack_expected: Some(RuntimeAdStackExpectedJson::from( + result.runtime_ad_stack_expected, + )), + gates: Some(to_gates( + matched, + auction_enabled, + creative.is_some_and(|creative| creative.enabled), + )), + matched_slot_count: Some(expected.len()), + slots, + extra_evidence, + warnings, + }; + (page, strict_failed) +} + +/// Builds a page-level navigation-failure result (spec §8 `navigation_failed`). +fn error_page(requested: &url::Url, message: &str) -> PageJson { + let requested_path = normalize_path_or_url(requested.as_str()).unwrap_or_else(|_| "/".into()); + PageJson { + url: requested.to_string(), + final_url: None, + requested_path, + path: None, + error: Some(Warning { + code: "navigation_failed".to_string(), + message: message.to_string(), + }), + runtime_ad_stack_expected: None, + gates: None, + matched_slot_count: None, + slots: Vec::new(), + extra_evidence: Vec::new(), + warnings: Vec::new(), + } +} + +/// Builds a page-level cross-origin-redirect refusal. +/// +/// The final URL is reported so the operator can re-run against it explicitly +/// (or pass `--allow-cross-origin-redirect`) once they have confirmed it is +/// their own property. +fn cross_origin_page(requested: &url::Url, final_url: &url::Url) -> PageJson { + let requested_path = normalize_path_or_url(requested.as_str()).unwrap_or_else(|_| "/".into()); + PageJson { + url: requested.to_string(), + final_url: Some(final_url.to_string()), + requested_path, + path: None, + error: Some(Warning { + code: "cross_origin_redirect".to_string(), + message: format!( + "navigation left the requested origin ({} -> {}); \ + evidence from another origin is not accepted as verification. \ + Re-run against the final URL, or pass --allow-cross-origin-redirect", + requested.origin().ascii_serialization(), + final_url.origin().ascii_serialization(), + ), + }), + runtime_ad_stack_expected: None, + gates: None, + matched_slot_count: None, + slots: Vec::new(), + extra_evidence: Vec::new(), + warnings: Vec::new(), + } +} + +fn empty_evidence() -> BrowserAdEvidence { + BrowserAdEvidence { + dom_ids: Vec::new(), + gpt_slots: Vec::new(), + aps_calls: Vec::new(), + page_bids: Vec::new(), + warnings: Vec::new(), + } +} + +fn to_gates(matched: bool, auction_enabled: bool, ad_templates_enabled: bool) -> Gates { + let pass_if = |cond: bool| { + if cond { + GateState::Pass + } else { + GateState::Fail + } + }; + Gates { + method_get: GateState::Pass, + navigation: GateState::Pass, + not_prefetch: GateState::Pass, + not_bot: GateState::Pass, + matched_slots: pass_if(matched), + auction_enabled: pass_if(auction_enabled), + ad_templates_enabled: pass_if(ad_templates_enabled), + // Live consent is not provable from a browser navigation in Phase 1. + consent_allows_auction: GateState::Unknown, + } +} + +fn to_slot_json(expected: &ExpectedSlot, result: &SlotResult) -> SlotJson { + SlotJson { + id: result.id.clone(), + status: to_status(result.status), + phase: result.phase.map(to_phase), + configured: ConfiguredJson { + div_id: expected.div_id.clone(), + gam_unit_path: expected.gam_unit_path.clone(), + formats: expected + .formats + .iter() + .map(|format| FormatJson { + width: format.width, + height: format.height, + media_type: media_type_label(&format.media_type).to_string(), + }) + .collect(), + providers: expected.providers.clone(), + }, + evidence: to_slot_evidence(&result.evidence), + warnings: result.warnings.clone(), + } +} + +fn to_slot_evidence(evidence: &SlotEvidence) -> SlotEvidenceJson { + SlotEvidenceJson { + dom_id: evidence.dom_id.clone(), + gpt: evidence.gpt.as_ref().map(|gpt| GptEvidenceJson { + gam_unit_path: gpt.gam_unit_path.clone(), + div_id: gpt.div_id.clone(), + sizes: gpt.sizes.iter().map(|&(w, h)| [w, h]).collect(), + }), + } +} + +fn to_extra_json(extra: &ExtraEvidence) -> ExtraEvidenceJson { + ExtraEvidenceJson { + kind: extra.kind.clone(), + phase: to_phase(extra.phase), + dom_id: extra.dom_id.clone(), + gam_unit_path: extra.gam_unit_path.clone(), + sizes: extra.sizes.iter().map(|&(w, h)| [w, h]).collect(), + reason: extra.reason.clone(), + } +} + +fn to_status(status: CompareStatus) -> SlotStatus { + match status { + CompareStatus::Confirmed => SlotStatus::Confirmed, + CompareStatus::Partial => SlotStatus::Partial, + CompareStatus::Missing => SlotStatus::Missing, + CompareStatus::Unconfirmable => SlotStatus::Unconfirmable, + } +} + +fn to_phase(phase: EvidencePhase) -> EvidencePhaseJson { + match phase { + EvidencePhase::InitialLoad => EvidencePhaseJson::InitialLoad, + EvidencePhase::Scroll => EvidencePhaseJson::Scroll, + } +} + +fn media_type_label(media_type: &MediaType) -> &'static str { + match media_type { + MediaType::Banner => "banner", + MediaType::Video => "video", + MediaType::Native => "native", + } +} + +fn write_json(out: &mut dyn Write, report: &VerificationReport) -> Result<(), String> { + let json = serde_json::to_string_pretty(report) + .map_err(|error| format!("failed to serialize verification report: {error}"))?; + writeln!(out, "{json}").map_err(write_err) +} + +fn write_human(out: &mut dyn Write, report: &VerificationReport) -> Result<(), String> { + // Warning codes and messages can originate in the audited page (the + // collector forwards `String(error)` from page scripts), so escape control + // characters before writing them to the operator's terminal. + let write_warning = |out: &mut dyn Write, indent: &str, warning: &Warning| { + writeln!( + out, + "{indent}warning [{}]: {}", + escape_terminal_text(&warning.code), + escape_terminal_text(&warning.message) + ) + .map_err(write_err) + }; + + for warning in &report.warnings { + write_warning(out, "", warning)?; + } + for page in &report.pages { + writeln!(out, "url: {}", escape_terminal_text(&page.url)).map_err(write_err)?; + if let Some(error) = &page.error { + writeln!( + out, + " error [{}]: {}", + escape_terminal_text(&error.code), + escape_terminal_text(&error.message) + ) + .map_err(write_err)?; + continue; + } + if let Some(path) = &page.path { + writeln!(out, " path: {}", escape_terminal_text(path)).map_err(write_err)?; + } + if let Some(expected) = page.runtime_ad_stack_expected { + writeln!(out, " runtime ad stack: {}", runtime_label(expected)).map_err(write_err)?; + } + if let Some(count) = page.matched_slot_count { + writeln!(out, " matched slots: {count}").map_err(write_err)?; + } + if let Some(gates) = &page.gates { + writeln!(out, " gates: {}", gates_label(gates)).map_err(write_err)?; + } + for slot in &page.slots { + writeln!( + out, + " slot {}: {}", + escape_terminal_text(&slot.id), + status_label(slot.status) + ) + .map_err(write_err)?; + for warning in &slot.warnings { + write_warning(out, " ", warning)?; + } + } + for extra in &page.extra_evidence { + writeln!( + out, + " extra {} evidence: div={} gam={} sizes={:?} ({})", + escape_terminal_text(&extra.kind), + escape_terminal_text(extra.dom_id.as_deref().unwrap_or("-")), + escape_terminal_text(extra.gam_unit_path.as_deref().unwrap_or("-")), + extra.sizes, + escape_terminal_text(&extra.reason), + ) + .map_err(write_err)?; + } + for warning in &page.warnings { + write_warning(out, " ", warning)?; + } + } + writeln!(out, "ok: {}", report.ok).map_err(write_err) +} + +fn status_label(status: SlotStatus) -> &'static str { + match status { + SlotStatus::Confirmed => "confirmed", + SlotStatus::Partial => "partial", + SlotStatus::Missing => "missing", + SlotStatus::Unconfirmable => "unconfirmable", + } +} + +fn runtime_label(expected: RuntimeAdStackExpectedJson) -> &'static str { + match expected { + RuntimeAdStackExpectedJson::Yes => "yes", + RuntimeAdStackExpectedJson::No => "no", + RuntimeAdStackExpectedJson::Unknown => "unknown", + } +} + +fn gate_label(gate: GateState) -> &'static str { + match gate { + GateState::Pass => "pass", + GateState::Fail => "fail", + GateState::Unknown => "unknown", + } +} + +fn gates_label(gates: &Gates) -> String { + format!( + "method_get={} navigation={} not_prefetch={} not_bot={} matched_slots={} auction_enabled={} consent={}", + gate_label(gates.method_get), + gate_label(gates.navigation), + gate_label(gates.not_prefetch), + gate_label(gates.not_bot), + gate_label(gates.matched_slots), + gate_label(gates.auction_enabled), + gate_label(gates.consent_allows_auction), + ) +} + +#[allow( + clippy::needless_pass_by_value, + reason = "used as a map_err fn that receives io::Error by value" +)] +fn write_err(error: io::Error) -> String { + format!("failed to write command output: {error}") +} + +#[cfg(test)] +mod tests { + use std::cell::Cell; + use std::collections::HashMap; + + use super::*; + use crate::ad_templates::compare::{DomEvidence, GptSlotEvidence}; + use crate::commands::audit::collector::CollectedPage; + + struct FakeCollector { + pages: HashMap>, + batch_calls: Cell, + } + + impl FakeCollector { + fn page(requested: &str, final_url: &str, evidence: BrowserAdEvidence) -> Self { + let mut pages = HashMap::new(); + pages.insert( + requested.to_string(), + Ok(CollectedPage { + final_url: url::Url::parse(final_url).expect("should parse final URL"), + title: String::new(), + script_count: 0, + resource_count: 0, + warnings: Vec::new(), + ad_evidence: Some(evidence), + }), + ); + Self { + pages, + batch_calls: Cell::new(0), + } + } + + fn with_error(mut self, requested: &str, message: &str) -> Self { + self.pages + .insert(requested.to_string(), Err(message.to_string())); + self + } + } + + impl AuditCollector for FakeCollector { + fn collect_page(&self, request: BrowserCollectRequest) -> Result { + self.pages + .get(request.url.as_str()) + .cloned() + .unwrap_or_else(|| Err(format!("no fake page for {}", request.url))) + } + + fn collect_pages( + &self, + requests: &[BrowserCollectRequest], + ) -> Vec> { + self.batch_calls.set(self.batch_calls.get() + 1); + requests + .iter() + .cloned() + .map(|request| self.collect_page(request)) + .collect() + } + } + + fn news_config() -> CreativeOpportunitiesConfig { + let toml = "gam_network_id = \"123\"\n\ + \n\ + [[slot]]\n\ + id = \"atf\"\n\ + gam_unit_path = \"/123/news/atf\"\n\ + div_id = \"ad-atf-\"\n\ + page_patterns = [\"/news/*\"]\n\ + formats = [{ width = 300, height = 250 }]\n"; + let mut config = + toml::from_str::(toml).expect("should deserialize"); + config.compile_slots(); + config + } + + fn confirmed_news_evidence() -> BrowserAdEvidence { + BrowserAdEvidence { + dom_ids: vec![DomEvidence { + dom_id: "ad-atf-0".to_string(), + phase: EvidencePhase::InitialLoad, + }], + gpt_slots: vec![GptSlotEvidence { + gam_unit_path: "/123/news/atf".to_string(), + div_id: "ad-atf-0".to_string(), + sizes: vec![(300, 250)], + phase: EvidencePhase::InitialLoad, + }], + aps_calls: Vec::new(), + page_bids: Vec::new(), + warnings: Vec::new(), + } + } + + fn report_for( + collector: &dyn AuditCollector, + auction_enabled: bool, + strict: bool, + urls: &[&str], + ) -> VerificationReport { + report_for_with_options( + collector, + auction_enabled, + urls, + VerifyOptions { + strict, + scroll: false, + allow_cross_origin_redirect: false, + }, + ) + } + + fn report_for_with_options( + collector: &dyn AuditCollector, + auction_enabled: bool, + urls: &[&str], + options: VerifyOptions, + ) -> VerificationReport { + let config = news_config(); + let parsed: Vec = urls + .iter() + .map(|url| url::Url::parse(url).expect("should parse URL")) + .collect(); + build_report( + collector, + Some(&config), + auction_enabled, + &parsed, + options, + &[], + ) + .expect("typed collector configuration should serialize") + } + + #[test] + fn verify_uses_final_url_for_matching_after_redirect() { + let collector = FakeCollector::page( + "https://www.example.com/", + "https://www.example.com/news/story", + confirmed_news_evidence(), + ); + let report = report_for(&collector, true, false, &["https://www.example.com/"]); + let json = serde_json::to_value(&report).expect("should serialize"); + + assert_eq!(json["pages"][0]["path"], "/news/story"); + assert_eq!(json["pages"][0]["slots"][0]["status"], "confirmed"); + let warnings = json["pages"][0]["warnings"] + .as_array() + .expect("should have warnings array"); + assert!( + warnings.iter().any(|w| w["code"] == "redirected"), + "redirect should emit a `redirected` warning" + ); + } + + #[test] + fn cross_origin_redirect_is_rejected_even_when_paths_match() { + // Same path on a different origin: the redirect warning would not fire, + // so without the origin check this unrelated page's evidence would + // satisfy --strict. + let collector = FakeCollector::page( + "https://www.example.com/news/story", + "https://impostor.example.net/news/story", + confirmed_news_evidence(), + ); + let report = report_for( + &collector, + true, + true, + &["https://www.example.com/news/story"], + ); + + assert!(!report.ok, "a cross-origin redirect must not report ok"); + let json = serde_json::to_value(&report).expect("should serialize"); + assert_eq!(json["pages"][0]["error"]["code"], "cross_origin_redirect"); + assert!( + json["pages"][0]["slots"] + .as_array() + .expect("should have slots array") + .is_empty(), + "off-origin evidence must not be reported as slot verification" + ); + } + + #[test] + fn cross_origin_redirect_is_accepted_with_explicit_opt_in() { + let collector = FakeCollector::page( + "https://example.com/news/story", + "https://www.example.com/news/story", + confirmed_news_evidence(), + ); + let report = report_for_with_options( + &collector, + true, + &["https://example.com/news/story"], + VerifyOptions { + strict: true, + scroll: false, + allow_cross_origin_redirect: true, + }, + ); + + assert!( + report.ok, + "an opted-in apex -> www redirect should verify normally" + ); + assert_eq!(report.pages[0].matched_slot_count, Some(1)); + } + + #[test] + fn same_origin_path_redirect_still_verifies() { + let collector = FakeCollector::page( + "https://www.example.com/", + "https://www.example.com/news/story", + confirmed_news_evidence(), + ); + let report = report_for(&collector, true, true, &["https://www.example.com/"]); + + assert!( + report.ok, + "a same-origin redirect should still be verified, not refused" + ); + } + + #[test] + fn same_host_http_to_https_upgrade_is_accepted() { + let collector = FakeCollector::page( + "http://www.example.com/news/story", + "https://www.example.com/news/story", + confirmed_news_evidence(), + ); + let report = report_for( + &collector, + true, + true, + &["http://www.example.com/news/story"], + ); + + assert!(report.ok, "a default-port HTTPS upgrade should be accepted"); + } + + #[test] + fn downgrade_and_port_changes_are_rejected() { + for (requested, final_url) in [ + ( + "https://www.example.com/news/story", + "http://www.example.com/news/story", + ), + ( + "https://www.example.com:8443/news/story", + "https://www.example.com:9443/news/story", + ), + ( + "http://www.example.com:8080/news/story", + "https://www.example.com:8443/news/story", + ), + ] { + let collector = FakeCollector::page(requested, final_url, confirmed_news_evidence()); + let report = report_for(&collector, true, true, &[requested]); + assert!(!report.ok, "redirect {requested} -> {final_url} must fail"); + } + } + + #[test] + fn confirmed_page_is_ok_in_default_mode() { + let collector = FakeCollector::page( + "https://www.example.com/news/story", + "https://www.example.com/news/story", + confirmed_news_evidence(), + ); + let report = report_for( + &collector, + true, + false, + &["https://www.example.com/news/story"], + ); + + assert!(report.ok, "confirmed page should be ok"); + assert_eq!(report.pages[0].matched_slot_count, Some(1)); + } + + #[test] + fn verifier_surfaces_injected_collector_warnings() { + let mut evidence = confirmed_news_evidence(); + evidence.warnings.push(Warning { + code: "fluid_size_ignored".to_string(), + message: "a fluid size could not be compared".to_string(), + }); + let collector = FakeCollector::page( + "https://www.example.com/news/story", + "https://www.example.com/news/story", + evidence, + ); + + let report = report_for( + &collector, + true, + false, + &["https://www.example.com/news/story"], + ); + + assert!( + report.pages[0] + .warnings + .iter() + .any(|warning| warning.code == "page_fluid_size_ignored"), + "collector warning should be visible in the page report" + ); + } + + #[test] + fn human_output_includes_runtime_and_extra_evidence_diagnostics() { + let mut evidence = confirmed_news_evidence(); + evidence.gpt_slots.push(GptSlotEvidence { + gam_unit_path: "/123/publisher/extra".to_string(), + div_id: "ad-extra-0".to_string(), + sizes: vec![(728, 90)], + phase: EvidencePhase::InitialLoad, + }); + let collector = FakeCollector::page( + "https://www.example.com/news/story", + "https://www.example.com/news/story", + evidence, + ); + let report = report_for( + &collector, + true, + false, + &["https://www.example.com/news/story"], + ); + let mut output = Vec::new(); + + write_human(&mut output, &report).expect("should write human report"); + let output = String::from_utf8(output).expect("should be UTF-8 output"); + + assert!(output.contains("runtime ad stack: unknown")); + assert!(output.contains("matched slots: 1")); + assert!(output.contains("gates: method_get=pass")); + assert!(output.contains("extra gpt evidence")); + } + + #[test] + fn strict_missing_slot_fails() { + let collector = FakeCollector::page( + "https://www.example.com/news/story", + "https://www.example.com/news/story", + empty_evidence(), + ); + let report = report_for( + &collector, + true, + true, + &["https://www.example.com/news/story"], + ); + + assert!( + !report.ok, + "strict mode with a missing slot should not be ok" + ); + } + + #[test] + fn auction_disabled_skips_strict_missing_failure() { + let collector = FakeCollector::page( + "https://www.example.com/news/story", + "https://www.example.com/news/story", + empty_evidence(), + ); + // auction disabled -> runtime expected No -> strict does not fail on missing. + let report = report_for( + &collector, + false, + true, + &["https://www.example.com/news/story"], + ); + + assert!( + report.ok, + "missing slot must not fail strict when auction is disabled" + ); + assert_eq!( + report.pages[0].runtime_ad_stack_expected, + Some(RuntimeAdStackExpectedJson::No) + ); + } + + #[test] + fn multi_url_page_error_sets_ok_false() { + let collector = FakeCollector::page( + "https://www.example.com/news/story", + "https://www.example.com/news/story", + confirmed_news_evidence(), + ) + .with_error("https://www.example.com/broken", "navigation failed"); + let report = report_for( + &collector, + true, + false, + &[ + "https://www.example.com/news/story", + "https://www.example.com/broken", + ], + ); + + assert!(!report.ok, "a page-level error sets ok=false"); + assert_eq!( + collector.batch_calls.get(), + 1, + "all verifier URLs should use one collector batch" + ); + let json = serde_json::to_value(&report).expect("should serialize"); + assert_eq!(json["pages"][1]["error"]["code"], "navigation_failed"); + assert!(json["pages"][1]["final_url"].is_null()); + } + + #[test] + fn supplied_cookies_are_rejected_for_multiple_origins() { + let urls = [ + url::Url::parse("https://a.example/x").expect("should parse first URL"), + url::Url::parse("https://b.example/y").expect("should parse second URL"), + ]; + + let error = validate_cookie_scope(&urls, &[("session".to_string(), "secret".to_string())]) + .expect_err("should not replicate one cookie across origins"); + + assert!( + error.contains("one origin"), + "the refusal should explain cookie scope, got {error}" + ); + } + + #[test] + fn supplied_cookies_are_allowed_for_same_origin_urls() { + let urls = [ + url::Url::parse("https://a.example/x").expect("should parse first URL"), + url::Url::parse("https://a.example/y").expect("should parse second URL"), + ]; + + validate_cookie_scope(&urls, &[("session".to_string(), "secret".to_string())]) + .expect("same-origin URLs share the intended cookie scope"); + } +} diff --git a/crates/trusted-server-cli/src/commands/audit/browser.rs b/crates/trusted-server-cli/src/commands/audit/browser.rs new file mode 100644 index 000000000..fd79155f0 --- /dev/null +++ b/crates/trusted-server-cli/src/commands/audit/browser.rs @@ -0,0 +1,1194 @@ +//! Chrome/Chromium-backed implementation of [`AuditCollector`] using +//! `chromiumoxide` (CDP). +//! +//! The collector installs optional pre-navigation init scripts, sets any +//! operator-supplied cookies, navigates, waits for the page to settle, optionally +//! scrolls, and reads back a bounded set of evidence. It never *captures* page +//! HTML, cookies, or storage; supplied cookies are only *sent* to carry an +//! existing session past origin gates. + +use std::time::Duration; + +use chromiumoxide::browser::{Browser, BrowserConfig}; +use chromiumoxide::cdp::browser_protocol::network::CookieParam; +use chromiumoxide::handler::viewport::Viewport; +use chromiumoxide::page::Page; +use futures::StreamExt as _; + +use crate::ad_templates::compare::BrowserAdEvidence; +use crate::ad_templates::output::Warning; +use crate::commands::audit::browser_scroll::{self, CDP_OPERATION_TIMEOUT}; +use crate::commands::audit::collector::{ + AuditCollector, BrowserCollectRequest, BrowserOpts, BrowserProfile, CollectedPage, + PAGE_SETTLE_MAX_MS, PAGE_SETTLE_QUIET_MS, +}; + +/// Candidate Chrome/Chromium executable names searched on `PATH`. +pub(crate) const CHROME_NAMES: &[&str] = &[ + "google-chrome", + "google-chrome-stable", + "chromium", + "chromium-browser", + "chrome", + "Google Chrome", + "Google Chrome for Testing", +]; + +/// Poll interval while waiting for the page network to settle, in milliseconds. +const SETTLE_POLL_MS: u64 = 250; +/// Hard cap on page navigation so a stalled load cannot hang the audit. +const NAVIGATION_TIMEOUT: Duration = Duration::from_secs(30); +/// Hard cap per decoded evidence list, so a hostile page cannot inflate CLI +/// memory. +/// +/// Must equal `__ts_max_entries` in `ad_template_collector.js`. The collector +/// already caps each list, but the evidence object lives on `window`, so a page +/// that appends to it directly is bounded here instead. Anything the collector +/// itself dropped is reported as an `evidence_truncated` warning. +const MAX_EVIDENCE_ENTRIES: usize = 128; +/// Hard cap on the UTF-8 JSON payload before CDP transfers it back to Rust. +const MAX_EVIDENCE_PAYLOAD_BYTES: usize = 1024 * 1024; +/// Hard cap on browser teardown so a wedged Chrome cannot hang the audit. +const BROWSER_CLOSE_TIMEOUT: Duration = Duration::from_secs(5); + +/// Page-settle timing thresholds. +#[derive(Debug, Clone, Copy)] +struct SettleConfig { + /// Quiet window with no new resources marking the page settled. + quiet: Duration, + /// Hard cap on total settle time. + max: Duration, +} + +/// Immutable browser/session settings shared by every URL in one audit batch. +struct BrowserSessionOptions<'a> { + chrome: &'a std::path::Path, + profile_dir: &'a std::path::Path, + settle: SettleConfig, + accept_invalid_certs: bool, + headful: bool, + assume_consent: bool, + proxy: Option<&'a str>, + profile: BrowserProfile, +} + +/// A `chromiumoxide`-backed page collector launching a local Chrome/Chromium. +#[derive(Debug, Clone)] +pub struct BrowserCollector { + /// Explicit Chrome/Chromium executable override (else `$CHROME`, else auto-detect). + chrome: Option, + /// Quiet window marking the page settled. + settle_quiet: Duration, + /// Hard cap on settling. + settle_max: Duration, + /// Navigate to origins with invalid TLS certificates (dangerous opt-in). + accept_invalid_certs: bool, + /// Run visible Chrome rather than new headless Chrome. + headful: bool, + /// Install the standard consent API stub before publisher scripts. + assume_consent: bool, + /// Optional browser proxy endpoint. + proxy: Option, + /// Device viewport/user-agent profile. + profile: BrowserProfile, +} + +impl Default for BrowserCollector { + fn default() -> Self { + Self::new() + } +} + +impl BrowserCollector { + /// Creates a collector with default tuning and auto-detected Chrome. + #[must_use] + pub fn new() -> Self { + Self { + chrome: None, + settle_quiet: Duration::from_millis(PAGE_SETTLE_QUIET_MS), + settle_max: Duration::from_millis(PAGE_SETTLE_MAX_MS), + accept_invalid_certs: false, + headful: false, + assume_consent: true, + proxy: None, + profile: BrowserProfile::Desktop, + } + } + + /// Creates a collector from operator-supplied browser options. + #[must_use] + pub fn from_opts(opts: &BrowserOpts) -> Self { + Self { + chrome: opts.chrome.clone(), + settle_quiet: Duration::from_millis(opts.settle_quiet_ms), + settle_max: Duration::from_millis(opts.settle_max_ms), + accept_invalid_certs: opts.danger_accept_invalid_certs, + headful: opts.headful, + assume_consent: !opts.no_assume_consent, + proxy: opts.browser_proxy.clone(), + profile: opts.profile, + } + } +} + +/// Pre-document consent behavior shared with the generation crawler. +pub(crate) const CONSENT_STUB_SCRIPT: &str = include_str!("consent_stub.js"); + +/// Shared browser launch inputs used by both audit collectors. +pub(crate) struct BrowserLaunchOptions<'a> { + pub(crate) chrome: &'a std::path::Path, + pub(crate) profile_dir: &'a std::path::Path, + pub(crate) headful: bool, + pub(crate) proxy: Option<&'a str>, + pub(crate) accept_invalid_certs: bool, + pub(crate) viewport: Viewport, + pub(crate) user_agent: Option<&'a str>, +} + +/// Builds the common Chrome configuration for all browser-backed audits. +pub(crate) fn build_browser_config( + options: BrowserLaunchOptions<'_>, +) -> Result { + let mut builder = BrowserConfig::builder() + .chrome_executable(options.chrome) + .user_data_dir(options.profile_dir); + if !options.accept_invalid_certs { + builder = builder.respect_https_errors(); + } + if let Some(proxy) = options.proxy { + let endpoint = if proxy.contains("://") { + proxy.to_string() + } else { + format!("http://{proxy}") + }; + builder = builder + .arg(("proxy-server", endpoint.as_str())) + .arg(("proxy-bypass-list", "<-loopback>")); + } + builder = if options.headful { + builder.with_head() + } else { + builder.new_headless_mode() + }; + builder = builder + .window_size(options.viewport.width, options.viewport.height) + .viewport(options.viewport); + if let Some(user_agent) = options.user_agent { + builder = builder.arg(("user-agent", user_agent)); + } + builder + .build() + .map_err(|error| format!("failed to build browser config: {error}")) +} + +fn browser_profile(profile: BrowserProfile) -> (Viewport, Option<&'static str>) { + match profile { + BrowserProfile::Desktop => ( + Viewport { + width: 1280, + height: 800, + device_scale_factor: Some(1.0), + emulating_mobile: false, + is_landscape: true, + has_touch: false, + }, + None, + ), + BrowserProfile::Mobile => ( + Viewport { + width: 390, + height: 844, + device_scale_factor: Some(3.0), + emulating_mobile: true, + is_landscape: false, + has_touch: true, + }, + Some( + "Mozilla/5.0 (iPhone; CPU iPhone OS 17_0 like Mac OS X) \ + AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Mobile/15E148 Safari/604.1", + ), + ), + } +} + +/// Resolves the Chrome/Chromium executable to launch. +/// +/// Precedence: explicit `--chrome` override, then the `CHROME` environment +/// variable, then auto-detection on `PATH` and standard install locations. +pub(crate) fn resolve_chrome( + override_path: Option<&std::path::Path>, +) -> Result { + if let Some(path) = override_path { + return if path.is_file() { + Ok(path.to_path_buf()) + } else { + Err(format!( + "--chrome path does not point to a file: {}", + path.display() + )) + }; + } + if let Ok(env_path) = std::env::var("CHROME") { + let path = std::path::PathBuf::from(&env_path); + return if path.is_file() { + Ok(path) + } else { + Err(format!("CHROME={env_path} does not point to a file")) + }; + } + find_chrome() +} + +/// Builds a host-only cookie that applies to every path on `url`'s host. +/// +/// Scoped by origin rather than by the full URL: only the origin is load-bearing +/// for a host-only cookie, and a full URL would carry the path, query, and any +/// `user:password@` into CDP and into this function's error message. +pub(crate) fn host_cookie(name: &str, value: &str, url: &url::Url) -> Result { + let origin = url.origin(); + if !origin.is_tuple() { + return Err(format!( + "cannot scope cookie `{name}` because the audited URL has no host" + )); + } + let mut cookie = CookieParam::new(name.to_string(), value.to_string()); + cookie.url = Some(origin.ascii_serialization()); + cookie.path = Some("/".to_string()); + cookie.secure = Some(url.scheme() == "https"); + Ok(cookie) +} + +fn format_cookie_install_error(name: &str, _error: impl std::fmt::Display) -> String { + // Do not forward the CDP error: a browser implementation may include the + // rejected cookie value in its diagnostic. + format!("failed to set cookie `{name}`") +} + +/// Installs host-only, root-scoped cookies before a page has an origin. +pub(crate) async fn set_browser_cookies( + browser: &Browser, + cookies: &[(String, String)], + url: &url::Url, +) -> Result<(), String> { + for (name, value) in cookies { + let cookie = host_cookie(name, value, url)?; + browser + .set_cookies(vec![cookie]) + .await + .map_err(|error| format_cookie_install_error(name, error))?; + } + Ok(()) +} + +/// Auto-detects a Chrome/Chromium executable. +/// +/// Searches `PATH` by common names first, then well-known per-OS install +/// locations (e.g. the macOS `.app` bundle, which is not on `PATH`). +fn find_chrome() -> Result { + if let Some(path) = CHROME_NAMES.iter().find_map(|name| which::which(name).ok()) { + return Ok(path); + } + if let Some(path) = well_known_chrome_paths() + .into_iter() + .find(|path| path.is_file()) + { + return Ok(path); + } + Err(format!( + "could not find Chrome/Chromium on PATH or in standard install locations (looked for: {})", + CHROME_NAMES.join(", ") + )) +} + +/// Well-known absolute Chrome/Chromium install locations for the host OS. +fn well_known_chrome_paths() -> Vec { + let mut paths = Vec::new(); + + #[cfg(target_os = "macos")] + { + const APPS: &[&str] = &[ + "Google Chrome.app/Contents/MacOS/Google Chrome", + "Google Chrome Canary.app/Contents/MacOS/Google Chrome Canary", + "Chromium.app/Contents/MacOS/Chromium", + ]; + for app in APPS { + paths.push(std::path::PathBuf::from(format!("/Applications/{app}"))); + if let Ok(home) = std::env::var("HOME") { + paths.push(std::path::PathBuf::from(format!( + "{home}/Applications/{app}" + ))); + } + } + } + + #[cfg(target_os = "linux")] + { + for path in [ + "/usr/bin/google-chrome", + "/usr/bin/google-chrome-stable", + "/usr/bin/chromium", + "/usr/bin/chromium-browser", + "/snap/bin/chromium", + ] { + paths.push(std::path::PathBuf::from(path)); + } + } + + #[cfg(target_os = "windows")] + { + for path in [ + r"C:\Program Files\Google\Chrome\Application\chrome.exe", + r"C:\Program Files (x86)\Google\Chrome\Application\chrome.exe", + ] { + paths.push(std::path::PathBuf::from(path)); + } + } + + paths +} + +impl AuditCollector for BrowserCollector { + fn collect_page(&self, request: BrowserCollectRequest) -> Result { + self.collect_pages(std::slice::from_ref(&request)) + .into_iter() + .next() + .expect("should return one result for one browser request") + } + + fn collect_pages( + &self, + requests: &[BrowserCollectRequest], + ) -> Vec> { + if requests.is_empty() { + return Vec::new(); + } + // HTTP(S) scheme is enforced by the CLI value parser before we get here. + let chrome = match resolve_chrome(self.chrome.as_deref()) { + Ok(chrome) => chrome, + Err(error) => return vec![Err(error); requests.len()], + }; + let profile = match tempfile::tempdir() { + Ok(profile) => profile, + Err(error) => { + let error = format!("failed to create browser profile dir: {error}"); + return vec![Err(error); requests.len()]; + } + }; + + let runtime = match tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + { + Ok(runtime) => runtime, + Err(error) => { + let error = format!("failed to build browser runtime: {error}"); + return vec![Err(error); requests.len()]; + } + }; + + let settle = SettleConfig { + quiet: self.settle_quiet, + max: self.settle_max, + }; + + let accept_invalid_certs = self.accept_invalid_certs; + let headful = self.headful; + let assume_consent = self.assume_consent; + let proxy = self.proxy.clone(); + let browser_profile = self.profile; + let request_count = requests.len(); + let requests = requests.to_vec(); + let result = runtime.block_on(async move { + let options = BrowserSessionOptions { + chrome: &chrome, + profile_dir: profile.path(), + settle, + accept_invalid_certs, + headful, + assume_consent, + proxy: proxy.as_deref(), + profile: browser_profile, + }; + collect(requests, &options).await + }); + match result { + Ok(results) => results, + Err(error) => vec![Err(error); request_count], + } + } +} + +/// Drives a single page collection on the current-thread runtime. +async fn collect( + requests: Vec, + options: &BrowserSessionOptions<'_>, +) -> Result>, String> { + // chromiumoxide defaults to ignoring TLS errors. The audit sends + // operator-supplied session cookies and treats what it reads back as + // verification evidence, so a certificate-invalid impersonator could both + // harvest the session and fabricate the evidence. Validate certificates + // unless the operator explicitly opts out. + let (viewport, user_agent) = browser_profile(options.profile); + let config = build_browser_config(BrowserLaunchOptions { + chrome: options.chrome, + profile_dir: options.profile_dir, + headful: options.headful, + proxy: options.proxy, + accept_invalid_certs: options.accept_invalid_certs, + viewport, + user_agent, + })?; + + let (mut browser, mut handler) = Browser::launch(config) + .await + .map_err(|error| format!("failed to launch browser: {error}"))?; + + // Drive the CDP event loop for the duration of the session. + let handler_task = tokio::spawn(async move { while handler.next().await.is_some() {} }); + + let mut results = Vec::with_capacity(requests.len()); + for request in requests { + results.push( + collect_with_browser(&browser, request, options.settle, options.assume_consent).await, + ); + } + + // Best-effort teardown; ignore errors since we already have a result, but + // bound it so a Chrome that ignores `close` cannot hang the command. + let _ = tokio::time::timeout(BROWSER_CLOSE_TIMEOUT, browser.close()).await; + let _ = tokio::time::timeout(BROWSER_CLOSE_TIMEOUT, browser.wait()).await; + handler_task.abort(); + + Ok(results) +} + +async fn collect_with_browser( + browser: &Browser, + request: BrowserCollectRequest, + settle_config: SettleConfig, + assume_consent: bool, +) -> Result { + set_browser_cookies(browser, &request.cookies, &request.url).await?; + + // Open a blank page first so init scripts are installed before the real + // document loads (evaluate-on-new-document applies to subsequent navigations). + let page = browser + .new_page("about:blank") + .await + .map_err(|error| format!("failed to open browser page: {error}"))?; + + let result = collect_open_page(&page, &request, settle_config, assume_consent).await; + let close_result = tokio::time::timeout(BROWSER_CLOSE_TIMEOUT, page.close()).await; + + match (result, close_result) { + (Err(error), _) => Err(error), + (Ok(mut collected), Err(_)) => { + collected.warnings.push(Warning { + code: "page_close_timeout".to_string(), + message: "timed out closing the browser tab after collection".to_string(), + }); + Ok(collected) + } + (Ok(mut collected), Ok(Err(error))) => { + collected.warnings.push(Warning { + code: "page_close_failed".to_string(), + message: format!("failed to close the browser tab after collection: {error}"), + }); + Ok(collected) + } + (Ok(collected), Ok(Ok(_))) => Ok(collected), + } +} + +/// Collects from an open tab. The caller owns tab teardown so every return path, +/// including an error from this function, closes the page before continuing. +async fn collect_open_page( + page: &Page, + request: &BrowserCollectRequest, + settle_config: SettleConfig, + assume_consent: bool, +) -> Result { + let mut warnings = Vec::new(); + + if assume_consent { + page.evaluate_on_new_document(CONSENT_STUB_SCRIPT) + .await + .map_err(|error| format!("failed to install consent init script: {error}"))?; + warnings.push(Warning { + code: "consent_stub_active".to_string(), + message: "audit consent APIs were stubbed; re-run with --no-assume-consent to observe the publisher CMP without substitution".to_string(), + }); + } + page.evaluate_on_new_document("performance.setResourceTimingBufferSize(100000)") + .await + .map_err(|error| format!("failed to increase resource timing buffer: {error}"))?; + + for script in &request.init_scripts { + page.evaluate_on_new_document(script.clone()) + .await + .map_err(|error| format!("failed to install init script: {error}"))?; + } + + tokio::time::timeout(NAVIGATION_TIMEOUT, page.goto(request.url.as_str())) + .await + .map_err(|_| format!("navigation to {} timed out", request.url))? + .map_err(|error| format!("failed to navigate to {}: {error}", request.url))?; + match tokio::time::timeout(NAVIGATION_TIMEOUT, page.wait_for_navigation()).await { + Ok(Ok(_)) => {} + Ok(Err(error)) => warnings.push(Warning { + code: "navigation_wait_failed".to_string(), + message: format!( + "navigation load event could not be read ({error}); continuing with settled page evidence" + ), + }), + Err(_) => warnings.push(Warning { + code: "navigation_wait_timeout".to_string(), + message: format!( + "navigation did not fire its load event within {} seconds; continuing with settled page evidence", + NAVIGATION_TIMEOUT.as_secs() + ), + }), + } + + settle(page, settle_config, &mut warnings).await; + + if request.scroll { + if request.collect_ad_evidence { + // Snapshot evidence before scrolling so entries already present at + // initial load keep phase "load"; the store dedups first-seen, so + // the post-scroll scrape only adds genuinely scroll-phase entries. + if tokio::time::timeout( + CDP_OPERATION_TIMEOUT, + page.evaluate( + "(typeof window.__tsCollectAdTemplateEvidence === 'function' \ + && window.__tsCollectAdTemplateEvidence(), null)", + ), + ) + .await + .is_err() + { + warnings.push(Warning { + code: "ad_evidence_snapshot_timeout".to_string(), + message: "timed out snapshotting ad evidence before scroll".to_string(), + }); + } + } + // Mark subsequent observations as scroll-phase for the verifier's + // injected evidence collector before shared scrolling begins. + eval_discard(page, "window.__tsScrollPhase = true", &mut warnings).await; + warnings.extend( + browser_scroll::scroll_page(page) + .await + .into_iter() + .map(|failure| Warning { + code: failure.code().to_string(), + message: failure.to_string(), + }), + ); + settle(page, settle_config, &mut warnings).await; + } + + let final_url_text = tokio::time::timeout(CDP_OPERATION_TIMEOUT, page.url()) + .await + .map_err(|_| "timed out reading final page URL".to_string())? + .map_err(|error| format!("failed to read final page URL: {error}"))? + .ok_or_else(|| "browser page URL was empty after navigation".to_string())?; + let final_url = url::Url::parse(&final_url_text).map_err(|error| { + format!("browser returned invalid final URL `{final_url_text}`: {error}") + })?; + let title = match tokio::time::timeout(CDP_OPERATION_TIMEOUT, page.get_title()).await { + Ok(Ok(title)) => title.unwrap_or_default(), + Ok(Err(error)) => { + warnings.push(Warning { + code: "page_title_failed".to_string(), + message: format!("failed to read page title: {error}"), + }); + String::new() + } + Err(_) => { + warnings.push(Warning { + code: "page_title_timeout".to_string(), + message: "timed out reading page title".to_string(), + }); + String::new() + } + }; + let script_count = eval_usize(page, "document.querySelectorAll('script').length") + .await + .unwrap_or_else(|message| { + warnings.push(Warning { + code: "script_count_failed".to_string(), + message, + }); + 0 + }); + let resource_count = resource_count(page).await.unwrap_or_else(|message| { + warnings.push(Warning { + code: "resource_count_failed".to_string(), + message, + }); + 0 + }); + + if resource_count >= 250 { + warnings.push(Warning { + code: "resource_timing_heavy".to_string(), + message: format!("page recorded {resource_count} network resources"), + }); + } + + if let Ok(Ok(frames)) = tokio::time::timeout(CDP_OPERATION_TIMEOUT, page.frames()).await + && frames.len() > 1 + { + warnings.push(Warning { + code: "child_frames_not_inspected".to_string(), + message: format!( + "ad-template evidence inspected only the main frame; {} child frame(s) were present", + frames.len() - 1 + ), + }); + } + + let ad_evidence = if request.collect_ad_evidence { + extract_ad_evidence(page, &mut warnings).await + } else { + None + }; + + Ok(CollectedPage { + final_url, + title, + script_count, + resource_count, + warnings, + ad_evidence, + }) +} + +/// Waits for the page network to go quiet after navigation or scroll. +/// +/// Polls the resource-entry count and returns once it stays unchanged for a +/// quiet window, or when the hard cap elapses — so ad-heavy pages finish loading +/// before evidence is read, without hanging on pages that never go idle. +async fn settle(page: &Page, config: SettleConfig, warnings: &mut Vec) { + let start = std::time::Instant::now(); + let mut last = None; + let mut quiet_since = None; + + loop { + if start.elapsed() >= config.max { + warnings.push(Warning { + code: "settle_timeout".to_string(), + message: "page did not settle before the configured maximum wait".to_string(), + }); + return; + } + + let ready_state = match eval_string(page, "document.readyState").await { + Ok(state) => state, + Err(message) => { + warnings.push(Warning { + code: "settle_read_failed".to_string(), + message, + }); + return; + } + }; + let current = match resource_count(page).await { + Ok(count) => count, + Err(message) => { + warnings.push(Warning { + code: "settle_read_failed".to_string(), + message, + }); + return; + } + }; + let ready = matches!(ready_state.as_str(), "interactive" | "complete"); + if ready && last == Some(current) { + let quiet_start = quiet_since.get_or_insert_with(std::time::Instant::now); + if quiet_start.elapsed() >= config.quiet { + return; + } + } else { + quiet_since = None; + } + last = Some(current); + + let remaining_max = config.max.saturating_sub(start.elapsed()); + let remaining_quiet = quiet_since + .map(|quiet_start| config.quiet.saturating_sub(quiet_start.elapsed())) + .unwrap_or(config.quiet); + let sleep_for = Duration::from_millis(SETTLE_POLL_MS) + .min(remaining_max) + .min(remaining_quiet.max(Duration::from_millis(1))); + tokio::time::sleep(sleep_for).await; + } +} + +/// Reads the number of resource timing entries observed so far. +async fn resource_count(page: &Page) -> Result { + eval_usize(page, "performance.getEntriesByType('resource').length").await +} + +async fn eval_discard(page: &Page, expression: impl Into, warnings: &mut Vec) { + if let Err(failure) = browser_scroll::evaluate(page, expression).await { + warnings.push(Warning { + code: failure.code().to_string(), + message: failure.to_string(), + }); + } +} + +async fn eval_usize(page: &Page, expression: &str) -> Result { + tokio::time::timeout(CDP_OPERATION_TIMEOUT, page.evaluate(expression)) + .await + .map_err(|_| format!("timed out evaluating `{expression}`"))? + .map_err(|error| format!("failed to evaluate `{expression}`: {error}"))? + .into_value::() + .map_err(|error| format!("failed to decode `{expression}`: {error}")) +} + +async fn eval_string(page: &Page, expression: &str) -> Result { + tokio::time::timeout(CDP_OPERATION_TIMEOUT, page.evaluate(expression)) + .await + .map_err(|_| format!("timed out evaluating `{expression}`"))? + .map_err(|error| format!("failed to evaluate `{expression}`: {error}"))? + .into_value::() + .map_err(|error| format!("failed to decode `{expression}`: {error}")) +} + +/// Reads and decodes `window.__tsAdTemplateEvidence`, warning (not failing) on a +/// decode error. +async fn extract_ad_evidence( + page: &Page, + warnings: &mut Vec, +) -> Option { + // Serialize and size-check in the page so a hostile publisher-controlled + // evidence object cannot force an unbounded CDP response and Rust decode. + let evaluation = tokio::time::timeout( + CDP_OPERATION_TIMEOUT, + page.evaluate(format!( + r#"(() => {{ + const evidence = typeof window.__tsCollectAdTemplateEvidence === 'function' + ? window.__tsCollectAdTemplateEvidence() + : (window.__tsAdTemplateEvidence || null) + if (evidence === null) return {{ kind: 'absent' }} + try {{ + const json = JSON.stringify(evidence) + const bytes = new TextEncoder().encode(json).byteLength + if (bytes > {MAX_EVIDENCE_PAYLOAD_BYTES}) return {{ kind: 'too_large' }} + return {{ kind: 'evidence', json }} + }} catch (error) {{ + return {{ + kind: 'serialization_failed', + message: String(error).slice(0, 512), + }} + }} + }})()"# + )), + ) + .await; + + let envelope = match evaluation { + Ok(Ok(result)) => match result.into_value::() { + Ok(envelope) => Some(envelope), + Err(error) => { + warnings.push(Warning { + code: "ad_evidence_decode_failed".to_string(), + message: format!("failed to decode ad-template evidence envelope: {error}"), + }); + return None; + } + }, + Ok(Err(error)) => { + warnings.push(Warning { + code: "ad_evidence_read_failed".to_string(), + message: format!("failed to read ad-template evidence: {error}"), + }); + return None; + } + Err(_) => { + warnings.push(Warning { + code: "ad_evidence_read_timeout".to_string(), + message: "timed out reading ad-template evidence".to_string(), + }); + return None; + } + }; + + match envelope { + Some(envelope) => decode_ad_evidence_envelope(envelope, warnings), + None => { + warnings.push(Warning { + code: "ad_evidence_absent".to_string(), + message: "no ad-template evidence was collected from the page".to_string(), + }); + None + } + } +} + +#[derive(Debug, serde::Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case")] +enum EvidenceEnvelope { + Absent, + TooLarge, + Evidence { json: String }, + SerializationFailed { message: String }, +} + +fn decode_ad_evidence_envelope( + envelope: EvidenceEnvelope, + warnings: &mut Vec, +) -> Option { + match envelope { + EvidenceEnvelope::Absent => { + warnings.push(Warning { + code: "ad_evidence_absent".to_string(), + message: "no ad-template evidence was collected from the page".to_string(), + }); + None + } + EvidenceEnvelope::TooLarge => { + warnings.push(Warning { + code: "ad_evidence_too_large".to_string(), + message: format!( + "ad-template evidence exceeded the {MAX_EVIDENCE_PAYLOAD_BYTES}-byte limit" + ), + }); + None + } + EvidenceEnvelope::SerializationFailed { message } => { + warnings.push(Warning { + code: "ad_evidence_encode_failed".to_string(), + message: format!("failed to serialize ad-template evidence in the page: {message}"), + }); + None + } + EvidenceEnvelope::Evidence { json } => { + match serde_json::from_str::(&json) { + Ok(mut evidence) => { + // Defense in depth: the injected script caps these lists, but the + // page owns that store, so re-cap after decode. + evidence.dom_ids.truncate(MAX_EVIDENCE_ENTRIES); + evidence.gpt_slots.truncate(MAX_EVIDENCE_ENTRIES); + evidence.aps_calls.truncate(MAX_EVIDENCE_ENTRIES); + evidence.warnings.truncate(MAX_EVIDENCE_ENTRIES); + Some(evidence) + } + Err(error) => { + warnings.push(Warning { + code: "ad_evidence_decode_failed".to_string(), + message: format!("failed to decode ad-template evidence: {error}"), + }); + None + } + } + } + } +} + +/// Whether a Chrome/Chromium fixture is available for browser-backed tests. +/// +/// Skips optional local runs, but makes the scripted/CI contract fail loudly. +/// Shared with the generation collector's tests so the contract has one +/// definition. +#[cfg(test)] +pub(crate) fn browser_fixture_available() -> bool { + if resolve_chrome(None).is_ok() { + return true; + } + assert!( + std::env::var_os("TS_AUDIT_BROWSER_TESTS").is_none(), + "TS_AUDIT_BROWSER_TESTS requires Chrome/Chromium; set CHROME to its executable" + ); + false +} + +#[cfg(test)] +mod tests { + use std::io::{Read as _, Write as _}; + use std::net::TcpListener; + use std::sync::mpsc; + + use super::*; + use crate::commands::audit::collector::{ + AdTemplateCollectorConfig, build_ad_template_init_script, + }; + + const AD_TEMPLATE_COLLECTOR_JS: &str = include_str!("ad_template_collector.js"); + + #[test] + fn rust_and_javascript_evidence_entry_caps_match() { + // Parse the declared value rather than matching the whole line, so JS + // punctuation or spacing cannot false-alarm on a still-correct cap. + let declared = AD_TEMPLATE_COLLECTOR_JS + .lines() + .find_map(|line| line.trim().strip_prefix("const __ts_max_entries =")) + .and_then(|value| value.trim().trim_end_matches(';').parse::().ok()) + .expect("should declare __ts_max_entries in the collector script"); + + assert_eq!( + declared, MAX_EVIDENCE_ENTRIES, + "should keep the JS cap equal to MAX_EVIDENCE_ENTRIES" + ); + } + + #[test] + fn well_known_chrome_paths_are_known_for_this_os() { + // macOS/Linux/Windows each have candidate paths; guards the cfg branches. + assert!( + !well_known_chrome_paths().is_empty(), + "supported OSes should list candidate Chrome install paths" + ); + } + + #[test] + fn oversized_ad_evidence_is_an_explicit_warning() { + let mut warnings = Vec::new(); + let evidence = decode_ad_evidence_envelope(EvidenceEnvelope::TooLarge, &mut warnings); + + assert!(evidence.is_none()); + assert_eq!(warnings.len(), 1); + assert_eq!(warnings[0].code, "ad_evidence_too_large"); + } + + #[test] + fn supplied_cookie_is_host_only_and_root_scoped() { + let url = + url::Url::parse("https://publisher.example/news/story").expect("should parse test URL"); + let cookie = host_cookie("clearance", "token", &url).expect("should build cookie"); + + assert!(cookie.domain.is_none(), "host-only cookies omit Domain"); + assert_eq!(cookie.path.as_deref(), Some("/")); + assert_eq!( + cookie.url.as_deref(), + Some("https://publisher.example"), + "the origin scopes a host-only cookie before first navigation" + ); + assert_eq!(cookie.secure, Some(true), "HTTPS cookies must be Secure"); + } + + #[test] + fn cookie_install_error_identifies_name_without_a_value() { + let error = format_cookie_install_error( + "datadome", + "invalid cookie value operator-secret-cookie-value", + ); + + assert_eq!(error, "failed to set cookie `datadome`"); + assert!(!error.contains("operator-secret-cookie-value")); + } + + #[test] + #[ignore = "requires local Chrome/Chromium; run through scripts/test-cli.sh"] + fn supplied_cookie_reaches_first_navigation() { + if !browser_fixture_available() { + return; + } + + let listener = TcpListener::bind("127.0.0.1:0").expect("should bind fixture server"); + let address = listener.local_addr().expect("should read fixture address"); + let (request_tx, request_rx) = mpsc::channel(); + std::thread::spawn(move || { + let (mut stream, _) = listener.accept().expect("should accept browser request"); + stream + .set_read_timeout(Some(Duration::from_secs(10))) + .expect("should set fixture read timeout"); + let mut request = Vec::new(); + while !request.ends_with(b"\r\n\r\n") { + let mut chunk = [0_u8; 1024]; + let chunk_len = stream.read(&mut chunk).expect("should read HTTP request"); + assert!(chunk_len > 0, "request should contain complete headers"); + request.extend_from_slice(&chunk[..chunk_len]); + assert!( + request.len() <= 16 * 1024, + "request headers should be bounded" + ); + } + request_tx + .send(String::from_utf8_lossy(&request).into_owned()) + .expect("should send captured request"); + + let body = b"cookie fixture"; + write!( + stream, + "HTTP/1.1 200 OK\r\nContent-Type: text/html\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", + body.len() + ) + .expect("should write fixture headers"); + stream.write_all(body).expect("should write fixture body"); + }); + + let collector = BrowserCollector { + settle_quiet: Duration::from_millis(100), + settle_max: Duration::from_secs(1), + ..BrowserCollector::new() + }; + collector + .collect_page(BrowserCollectRequest { + url: url::Url::parse(&format!("http://{address}/")) + .expect("should parse fixture URL"), + init_scripts: Vec::new(), + scroll: false, + collect_ad_evidence: false, + cookies: vec![("clearance".to_string(), "token".to_string())], + }) + .expect("cookie should be installed before first navigation"); + + let request = request_rx + .recv_timeout(Duration::from_secs(5)) + .expect("fixture should receive the first navigation"); + assert!( + request.lines().any(|line| { + line.split_once(':').is_some_and(|(name, value)| { + name.eq_ignore_ascii_case("cookie") + && value + .trim() + .split(';') + .any(|cookie| cookie.trim() == "clearance=token") + }) + }), + "first navigation should carry the supplied cookie; request was {request:?}" + ); + } + + /// A self-contained page that stubs just enough of GPT (no network) for the + /// collector to observe a defined slot via the wrapped `defineSlot` and the + /// `getSlots()` scrape. + const GPT_FIXTURE: &str = r#" + + + +
+ + + +"#; + + #[test] + #[ignore = "requires local Chrome/Chromium; run through scripts/test-cli.sh"] + fn collects_gpt_slot_from_local_fixture() { + if !browser_fixture_available() { + // Browser fixture test requires a local Chrome/Chromium; skipping. + return; + } + let mut fixture = tempfile::Builder::new() + .suffix(".html") + .tempfile() + .expect("should create fixture file"); + fixture + .write_all(GPT_FIXTURE.as_bytes()) + .expect("should write fixture"); + let url = url::Url::from_file_path(fixture.path()).expect("should build file url"); + + let script = build_ad_template_init_script(&AdTemplateCollectorConfig { + div_prefixes: vec!["ad-atf-".to_string()], + }) + .expect("should build init script"); + + let collector = BrowserCollector::new(); + let page = collector + .collect_page(BrowserCollectRequest { + url, + init_scripts: vec![script], + scroll: false, + collect_ad_evidence: true, + cookies: Vec::new(), + }) + .expect("should collect fixture page"); + + let evidence = page.ad_evidence.expect("fixture should yield ad evidence"); + assert!( + evidence + .gpt_slots + .iter() + .any(|slot| slot.gam_unit_path == "/123/news/atf"), + "should capture the defined GPT slot" + ); + assert!( + evidence.dom_ids.iter().any(|dom| dom.dom_id == "ad-atf-0"), + "should capture the configured-prefix DOM id" + ); + } + + #[test] + #[ignore = "requires local Chrome/Chromium; run through scripts/test-cli.sh"] + fn scroll_pass_keeps_initial_load_phase_for_load_time_evidence() { + if !browser_fixture_available() { + // Browser fixture test requires a local Chrome/Chromium; skipping. + return; + } + let mut fixture = tempfile::Builder::new() + .suffix(".html") + .tempfile() + .expect("should create fixture file"); + fixture + .write_all(GPT_FIXTURE.as_bytes()) + .expect("should write fixture"); + let url = url::Url::from_file_path(fixture.path()).expect("should build file url"); + + let script = build_ad_template_init_script(&AdTemplateCollectorConfig { + div_prefixes: vec!["ad-atf-".to_string()], + }) + .expect("should build init script"); + + let collector = BrowserCollector::new(); + let page = collector + .collect_page(BrowserCollectRequest { + url, + init_scripts: vec![script], + scroll: true, + collect_ad_evidence: true, + cookies: Vec::new(), + }) + .expect("should collect fixture page"); + + // The slot and DOM id exist at load time, so the pre-scroll snapshot + // must record them as initial-load even though a scroll pass ran. + let evidence = page.ad_evidence.expect("fixture should yield ad evidence"); + assert!( + evidence.dom_ids.iter().any(|dom| dom.dom_id == "ad-atf-0" + && dom.phase == crate::ad_templates::compare::EvidencePhase::InitialLoad), + "load-time DOM id should keep phase initial_load under --scroll" + ); + assert!( + evidence.gpt_slots.iter().any(|slot| { + slot.gam_unit_path == "/123/news/atf" + && slot.phase == crate::ad_templates::compare::EvidencePhase::InitialLoad + }), + "load-time GPT slot should keep phase initial_load under --scroll" + ); + } +} diff --git a/crates/trusted-server-cli/src/commands/audit/browser_collector.rs b/crates/trusted-server-cli/src/commands/audit/browser_collector.rs deleted file mode 100644 index 87a2ccc2c..000000000 --- a/crates/trusted-server-cli/src/commands/audit/browser_collector.rs +++ /dev/null @@ -1,435 +0,0 @@ -use std::path::{Path, PathBuf}; -use std::time::Duration; - -use chromiumoxide::ArcHttpRequest; -use chromiumoxide::browser::{Browser, BrowserConfig}; -use futures::StreamExt as _; -use serde::Deserialize; -use tempfile::TempDir; -use tokio::runtime::Builder; -use tokio::time::{sleep, timeout}; -use url::Url; -use which::which; - -use crate::commands::audit::collector::{ - AuditCollector, CollectedPage, CollectedRequest, CollectedScriptTag, -}; -use crate::error::{CliResult, report_error}; - -const SETTLE_QUIET_PERIOD: Duration = Duration::from_millis(750); -const SETTLE_POLL_INTERVAL: Duration = Duration::from_millis(250); -const SETTLE_MAX_WAIT: Duration = Duration::from_secs(6); -const NAVIGATION_TIMEOUT: Duration = Duration::from_secs(30); -const BROWSER_CLOSE_TIMEOUT: Duration = Duration::from_secs(5); -const RESOURCE_TIMING_BUFFER_WARNING_THRESHOLD: usize = 250; -const RESOURCE_TIMING_BUFFER_WARNING: &str = - "browser resource timing buffer reached its default size; some network assets may be missing"; - -#[derive(Default)] -pub(crate) struct BrowserAuditCollector; - -impl AuditCollector for BrowserAuditCollector { - fn collect_page(&self, target_url: &Url) -> CliResult { - let runtime = Builder::new_current_thread() - .enable_all() - .build() - .map_err(|error| { - report_error(format!( - "failed to build Tokio runtime for browser audit: {error}" - )) - })?; - - runtime.block_on(collect_page_via_browser_async(target_url)) - } -} - -async fn collect_page_via_browser_async(target_url: &Url) -> CliResult { - let chrome_executable = find_browser_executable()?; - let user_data_dir = TempDir::new().map_err(|error| { - report_error(format!( - "failed to create temporary browser profile for audit: {error}" - )) - })?; - let config = BrowserConfig::builder() - .chrome_executable(chrome_executable) - .user_data_dir(user_data_dir.path()) - .new_headless_mode() - .build() - .map_err(|error| { - report_error(format!( - "failed to build Chromium configuration for audit: {error}" - )) - })?; - - let (mut browser, mut handler) = Browser::launch(config).await.map_err(|error| { - report_error(format!( - "failed to launch Chrome/Chromium for audit: {error}" - )) - })?; - - let handler_task = tokio::spawn(async move { - while let Some(event) = handler.next().await { - if event.is_err() { - break; - } - } - }); - - let result = collect_page_from_browser(&mut browser, target_url).await; - - let close_result = timeout(BROWSER_CLOSE_TIMEOUT, browser.close()) - .await - .map_err(|_| report_error("timed out closing browser after audit")) - .and_then(|result| { - result.map_err(|error| { - report_error(format!("failed to close browser after audit: {error}")) - }) - }); - if close_result.is_err() { - handler_task.abort(); - } - let _ = handler_task.await; - - match (result, close_result) { - (Ok(collected), Ok(_)) => Ok(collected), - (Ok(_), Err(error)) | (Err(error), _) => Err(error), - } -} - -async fn collect_page_from_browser( - browser: &mut Browser, - target_url: &Url, -) -> CliResult { - let page = browser.new_page("about:blank").await.map_err(|error| { - report_error(format!("failed to create browser page for audit: {error}")) - })?; - - timeout(NAVIGATION_TIMEOUT, page.goto(target_url.as_str())) - .await - .map_err(|_| report_error(format!("timed out navigating to `{target_url}`")))? - .map_err(|error| report_error(format!("failed to navigate to `{target_url}`: {error}")))?; - - let navigation_response = timeout(NAVIGATION_TIMEOUT, page.wait_for_navigation_response()) - .await - .map_err(|_| { - report_error(format!( - "timed out waiting for main document navigation response from `{target_url}`" - )) - })? - .map_err(|error| { - report_error(format!( - "failed to read main document navigation response: {error}" - )) - })?; - - let mut warnings = Vec::new(); - if let Some(warning) = validate_navigation_response(navigation_response)? { - warnings.push(warning); - } - if !wait_for_page_settle(&page).await? { - warnings.push( - "browser audit timed out while waiting for the page to settle; results may be partial" - .to_string(), - ); - } - - let final_url = page - .url() - .await - .map_err(|error| report_error(format!("failed to read final page URL: {error}")))? - .ok_or_else(|| report_error("browser page URL was empty after navigation"))?; - let page_title = page - .get_title() - .await - .map_err(|error| report_error(format!("failed to read page title: {error}")))?; - let html = page - .content() - .await - .map_err(|error| report_error(format!("failed to read rendered page HTML: {error}")))?; - - let script_tags: Vec = page - .evaluate( - r#"() => Array.from(document.scripts).map((script) => ({ - src: script.src || null, - inline_text: script.src ? null : (script.textContent || null), - }))"#, - ) - .await - .map_err(|error| report_error(format!("failed to read rendered script tags: {error}")))? - .into_value() - .map_err(|error| { - report_error(format!( - "failed to decode rendered script tag data: {error}" - )) - })?; - - let network_requests: Vec = page - .evaluate( - r#"() => performance.getEntriesByType('resource').map((entry) => ({ - url: entry.name, - initiator_type: entry.initiatorType || null, - }))"#, - ) - .await - .map_err(|error| { - report_error(format!( - "failed to read browser performance resource entries: {error}" - )) - })? - .into_value() - .map_err(|error| { - report_error(format!( - "failed to decode browser performance resource data: {error}" - )) - })?; - - if let Some(warning) = resource_timing_buffer_warning(network_requests.len()) { - warnings.push(warning.to_string()); - } - - Ok(CollectedPage { - requested_url: target_url.to_string(), - final_url, - page_title: page_title.filter(|title| !title.trim().is_empty()), - html, - script_tags: script_tags - .into_iter() - .map(|script| CollectedScriptTag { - src: script.src, - inline_text: script.inline_text.filter(|text| !text.trim().is_empty()), - }) - .collect(), - network_requests: network_requests - .into_iter() - .map(|entry| CollectedRequest { - url: entry.url, - resource_type: entry.initiator_type, - }) - .collect(), - warnings, - }) -} - -async fn wait_for_page_settle(page: &chromiumoxide::Page) -> CliResult { - let mut elapsed = Duration::ZERO; - let mut previous_count = None; - let mut stable_for = Duration::ZERO; - - while elapsed < SETTLE_MAX_WAIT { - let ready_state: String = page - .evaluate("document.readyState") - .await - .map_err(|error| report_error(format!("failed to read document ready state: {error}")))? - .into_value() - .map_err(|error| { - report_error(format!("failed to decode document ready state: {error}")) - })?; - let resource_count: usize = page - .evaluate("performance.getEntriesByType('resource').length") - .await - .map_err(|error| report_error(format!("failed to read resource count: {error}")))? - .into_value() - .map_err(|error| report_error(format!("failed to decode resource count: {error}")))?; - - if ready_state == "complete" { - if previous_count == Some(resource_count) { - stable_for += SETTLE_POLL_INTERVAL; - } else { - stable_for = Duration::ZERO; - } - - if stable_for >= SETTLE_QUIET_PERIOD { - return Ok(true); - } - } - - previous_count = Some(resource_count); - sleep(SETTLE_POLL_INTERVAL).await; - elapsed += SETTLE_POLL_INTERVAL; - } - - Ok(false) -} - -fn validate_navigation_response(navigation_response: ArcHttpRequest) -> CliResult> { - let request = navigation_response - .ok_or_else(|| report_error("browser audit did not capture the main document response"))?; - - if let Some(failure_text) = &request.failure_text { - return Err(report_error(format!( - "main document request failed: {failure_text}" - ))); - } - - let response = request.response.as_ref().ok_or_else(|| { - report_error("browser audit did not capture the main document HTTP response") - })?; - - if is_successful_navigation_status(response.status) { - return Ok(None); - } - - Ok(Some(format!( - "audit request returned HTTP {} {} for `{}`; results may be partial", - response.status, response.status_text, response.url - ))) -} - -fn is_successful_navigation_status(status: i64) -> bool { - (200..400).contains(&status) -} - -fn resource_timing_buffer_warning(resource_count: usize) -> Option<&'static str> { - (resource_count >= RESOURCE_TIMING_BUFFER_WARNING_THRESHOLD) - .then_some(RESOURCE_TIMING_BUFFER_WARNING) -} - -fn find_browser_executable() -> CliResult { - for candidate in browser_executable_path_candidates() { - if let Ok(path) = which(candidate) { - return Ok(path); - } - } - - for candidate in browser_executable_fallbacks() { - let candidate_path = Path::new(candidate); - if candidate_path.is_file() { - return Ok(candidate_path.to_path_buf()); - } - } - - Err(report_error( - "Chrome/Chromium was not found on PATH or in the standard local install locations checked by `ts audit`. Install a local Chrome or Chromium binary before running `ts audit`.", - )) -} - -fn browser_executable_path_candidates() -> &'static [&'static str] { - &[ - "google-chrome", - "google-chrome-stable", - "chromium", - "chromium-browser", - "chrome", - "Google Chrome", - "Google Chrome for Testing", - ] -} - -fn browser_executable_fallbacks() -> &'static [&'static str] { - #[cfg(target_os = "macos")] - { - &[ - "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome", - "/Applications/Chromium.app/Contents/MacOS/Chromium", - "/Applications/Google Chrome for Testing.app/Contents/MacOS/Google Chrome for Testing", - ] - } - - #[cfg(target_os = "linux")] - { - &[ - "/usr/bin/google-chrome", - "/usr/bin/google-chrome-stable", - "/usr/bin/chromium", - "/usr/bin/chromium-browser", - "/snap/bin/chromium", - ] - } - - #[cfg(not(any(target_os = "macos", target_os = "linux")))] - { - &[] - } -} - -#[derive(Debug, Deserialize)] -struct BrowserScriptTag { - src: Option, - inline_text: Option, -} - -#[derive(Debug, Deserialize)] -struct BrowserPerformanceEntry { - url: String, - initiator_type: Option, -} - -#[cfg(test)] -mod tests { - use std::sync::Arc; - - use chromiumoxide::cdp::browser_protocol::network::{Headers, RequestId, Response}; - use chromiumoxide::cdp::browser_protocol::security::SecurityState; - use chromiumoxide::handler::http::HttpRequest; - - use super::*; - - #[test] - fn successful_navigation_status_allows_redirects_but_rejects_errors() { - assert!(is_successful_navigation_status(200)); - assert!(is_successful_navigation_status(302)); - assert!(is_successful_navigation_status(399)); - assert!(!is_successful_navigation_status(199)); - assert!(!is_successful_navigation_status(400)); - assert!(!is_successful_navigation_status(500)); - } - - #[test] - fn navigation_response_returns_warning_for_http_error_status() { - let warning = - validate_navigation_response(navigation_response_with_status(403, "Forbidden")) - .expect("should validate navigation response") - .expect("should return warning for HTTP error status"); - - assert_eq!( - warning, - "audit request returned HTTP 403 Forbidden for `https://example.com/`; results may be partial", - "should warn and continue when the main document returns an HTTP error" - ); - } - - #[test] - fn resource_timing_buffer_warning_starts_at_threshold() { - assert_eq!( - resource_timing_buffer_warning(RESOURCE_TIMING_BUFFER_WARNING_THRESHOLD - 1), - None, - "should not warn before the resource timing buffer threshold" - ); - assert_eq!( - resource_timing_buffer_warning(RESOURCE_TIMING_BUFFER_WARNING_THRESHOLD), - Some(RESOURCE_TIMING_BUFFER_WARNING), - "should warn when the resource timing buffer reaches the threshold" - ); - } - - #[test] - fn browser_path_candidates_include_common_names() { - let candidates = browser_executable_path_candidates(); - - assert!(candidates.contains(&"google-chrome")); - assert!(candidates.contains(&"chromium")); - assert!(candidates.contains(&"Google Chrome for Testing")); - } - - fn navigation_response_with_status(status: i64, status_text: &str) -> ArcHttpRequest { - let mut request = - HttpRequest::new(RequestId::new("request-1"), None, None, false, Vec::new()); - request.response = Some( - Response::builder() - .url("https://example.com/") - .status(status) - .status_text(status_text) - .headers(Headers::default()) - .mime_type("text/html") - .charset("utf-8") - .connection_reused(false) - .connection_id(1.0) - .encoded_data_length(0.0) - .security_state(SecurityState::Secure) - .build() - .expect("should build navigation response"), - ); - - Some(Arc::new(request)) - } -} diff --git a/crates/trusted-server-cli/src/commands/audit/browser_scroll.rs b/crates/trusted-server-cli/src/commands/audit/browser_scroll.rs new file mode 100644 index 000000000..17b05a491 --- /dev/null +++ b/crates/trusted-server-cli/src/commands/audit/browser_scroll.rs @@ -0,0 +1,83 @@ +//! Shared deterministic browser scrolling for audit commands. + +use std::time::Duration; + +use chromiumoxide::Page; + +const SCROLL_STEP_DELAY: Duration = Duration::from_millis(250); +/// Bound for each CDP operation after navigation. +pub(crate) const CDP_OPERATION_TIMEOUT: Duration = Duration::from_secs(5); + +/// A best-effort browser scroll operation that could not be completed. +#[derive(Debug, derive_more::Display)] +pub(crate) enum ScrollFailure { + /// Chrome rejected the page evaluation. + #[display("browser page evaluation failed: {_0}")] + Evaluation(String), + /// Chrome did not complete the page evaluation within the operation bound. + #[display("browser page evaluation timed out")] + Timeout, +} + +impl core::error::Error for ScrollFailure {} + +impl ScrollFailure { + /// Stable warning code used by structured audit output. + pub(crate) const fn code(&self) -> &'static str { + match self { + Self::Evaluation(_) => "page_evaluation_failed", + Self::Timeout => "page_evaluation_timeout", + } + } +} + +/// Scrolls a page through deterministic fractions to trigger lazy content. +pub(crate) async fn scroll_page(page: &chromiumoxide::Page) -> Vec { + let mut failures = Vec::new(); + for fraction in ["0.33", "0.66", "1"] { + let script = format!( + "window.scrollTo(0, Math.floor(Math.max(document.body.scrollHeight, \ + document.documentElement.scrollHeight) * {fraction}))" + ); + if let Err(failure) = evaluate(page, script).await { + failures.push(failure); + } + tokio::time::sleep(SCROLL_STEP_DELAY).await; + } + if let Err(failure) = evaluate(page, "window.scrollTo(0, 0)").await { + failures.push(failure); + } + failures +} + +/// Evaluates a browser expression with the shared operation bound and errors. +pub(crate) async fn evaluate( + page: &Page, + expression: impl Into, +) -> Result<(), ScrollFailure> { + tokio::time::timeout(CDP_OPERATION_TIMEOUT, page.evaluate(expression.into())) + .await + .map_err(|_| ScrollFailure::Timeout)? + .map(|_| ()) + .map_err(|error| ScrollFailure::Evaluation(error.to_string())) +} + +#[cfg(test)] +mod tests { + use super::ScrollFailure; + + #[test] + fn scroll_failures_have_stable_messages() { + assert_eq!( + ScrollFailure::Evaluation("execution context was destroyed".to_string()).to_string(), + "browser page evaluation failed: execution context was destroyed" + ); + assert_eq!( + ScrollFailure::Timeout.to_string(), + "browser page evaluation timed out" + ); + + fn assert_error() {} + assert_error::(); + } +} diff --git a/crates/trusted-server-cli/src/commands/audit/collector.rs b/crates/trusted-server-cli/src/commands/audit/collector.rs index 314ae54fc..25aa9236f 100644 --- a/crates/trusted-server-cli/src/commands/audit/collector.rs +++ b/crates/trusted-server-cli/src/commands/audit/collector.rs @@ -1,41 +1,350 @@ -use serde::{Deserialize, Serialize}; -use url::Url; +//! Collector abstraction shared by the generic page audit and the ad-template +//! verifier. +//! +//! Decoupling collection behind [`AuditCollector`] lets the verifier orchestration +//! (Task 9) be tested with an in-memory fake collector, with no Chrome dependency. -use crate::error::CliResult; +use std::path::PathBuf; -pub(crate) trait AuditCollector { - fn collect_page(&self, target_url: &Url) -> CliResult; +use clap::{Args, ValueEnum}; + +use crate::ad_templates::compare::BrowserAdEvidence; + +/// Default quiet window for generation's browser collector. +pub(crate) const GENERATE_SETTLE_QUIET_MS: u64 = 750; +/// Default maximum settle wait for generation's browser collector. +pub(crate) const GENERATE_SETTLE_MAX_MS: u64 = 12_000; +/// Default quiet window for `ts audit page` and `ts audit ad-templates verify`. +/// +/// [`BrowserOpts`] and `BrowserCollector::new` must agree, or a collector built +/// in code drifts from the parsed flags without anything failing. +pub(crate) const PAGE_SETTLE_QUIET_MS: u64 = 750; +/// Default maximum settle wait for `ts audit page` and +/// `ts audit ad-templates verify`. +/// +/// See [`PAGE_SETTLE_QUIET_MS`] for why this is shared rather than duplicated. +pub(crate) const PAGE_SETTLE_MAX_MS: u64 = 10_000; + +/// Operator-tunable browser options shared by `ts audit page` and +/// `ts audit ad-templates verify`. +/// +/// These are audit-tool knobs, not publisher runtime config, so they live on the +/// CLI (flags / `CHROME` env) rather than in `trusted-server.toml`. +#[derive(Debug, Clone, Args)] +pub struct BrowserOpts { + /// Path to the Chrome/Chromium executable. Falls back to `$CHROME`, then + /// auto-detection on `PATH` and standard install locations. + #[arg(long)] + pub chrome: Option, + /// Browser device profile used for viewport and user-agent emulation. + #[arg(long = "browser-profile", value_enum, default_value_t = BrowserProfile::Desktop)] + pub profile: BrowserProfile, + /// Run a visible browser instead of Chrome's new headless mode. + #[arg(long)] + pub headful: bool, + /// Do not answer the standard IAB consent APIs for the fresh audit profile. + #[arg(long)] + pub no_assume_consent: bool, + /// Route the browser through this proxy, as `host:port` or a full URL. + #[arg(long, value_name = "HOST:PORT")] + pub browser_proxy: Option, + /// Quiet window in milliseconds (no new network resources) that marks the + /// page settled. + #[arg(long, default_value_t = PAGE_SETTLE_QUIET_MS)] + pub settle_quiet_ms: u64, + /// Hard cap in milliseconds on waiting for the page to settle. + #[arg(long, default_value_t = PAGE_SETTLE_MAX_MS)] + pub settle_max_ms: u64, + /// Navigate to origins whose TLS certificate does not validate. + /// + /// DANGEROUS: the audit sends any `--cookie` session to the origin and + /// treats what it reads back as verification evidence, so an invalid + /// certificate could mean an impersonator is harvesting the session and + /// fabricating the evidence. Use only against a host you control with a + /// known self-signed certificate. + #[arg(long)] + pub danger_accept_invalid_certs: bool, +} + +/// Browser options for generation, whose device selection is controlled by +/// `--profiles` rather than the verifier's singular `--browser-profile`. +#[derive(Debug, Clone, Args)] +pub struct GenerateBrowserOpts { + /// Path to the Chrome/Chromium executable. Falls back to `$CHROME`, then auto-detection. + #[arg(long)] + pub chrome: Option, + /// Run a visible browser instead of Chrome's new headless mode. + #[arg(long)] + pub headful: bool, + /// Do not answer the standard IAB consent APIs for the fresh audit profile. + #[arg(long)] + pub no_assume_consent: bool, + /// Route the browser through this proxy, as `host:port` or a full URL. + #[arg(long, value_name = "HOST:PORT")] + pub browser_proxy: Option, + /// Quiet window in milliseconds that marks the page settled. + #[arg(long, default_value_t = GENERATE_SETTLE_QUIET_MS)] + pub settle_quiet_ms: u64, + /// Hard cap in milliseconds on waiting for the page to settle. + #[arg(long, default_value_t = GENERATE_SETTLE_MAX_MS)] + pub settle_max_ms: u64, + /// Navigate to origins whose TLS certificate does not validate. + /// + /// DANGEROUS: the audit sends any `--cookie` session to the origin and + /// treats what it reads back as the evidence it writes config from, so an + /// invalid certificate could mean an impersonator is harvesting the session + /// and fabricating the evidence. Use only against a host you control with a + /// known self-signed certificate. + #[arg(long)] + pub danger_accept_invalid_certs: bool, +} + +/// Defaults mirroring the `#[arg(default_value_t)]` values above, so a path that +/// builds these options in code (the legacy `ts audit ` form) behaves like +/// the parsed command. +impl Default for GenerateBrowserOpts { + fn default() -> Self { + Self { + chrome: None, + headful: false, + no_assume_consent: false, + browser_proxy: None, + settle_quiet_ms: GENERATE_SETTLE_QUIET_MS, + settle_max_ms: GENERATE_SETTLE_MAX_MS, + danger_accept_invalid_certs: false, + } + } +} + +impl GenerateBrowserOpts { + /// Validates relationships between independently parsed browser flags. + pub fn validate(&self) -> Result<(), String> { + validate_settle_window(self.settle_quiet_ms, self.settle_max_ms) + } +} + +/// Browser device profile shared by page audits and ad-template verification. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, ValueEnum)] +pub enum BrowserProfile { + /// Desktop Chrome at 1280×800. + #[default] + Desktop, + /// Mobile-sized viewport with a mobile user agent. + Mobile, +} + +impl BrowserOpts { + /// Validates relationships between independently parsed browser flags. + pub fn validate(&self) -> Result<(), String> { + validate_settle_window(self.settle_quiet_ms, self.settle_max_ms) + } } -#[derive(Debug, Clone, Deserialize, Serialize, PartialEq, Eq)] -pub(crate) struct CollectedPage { - pub(crate) requested_url: String, - pub(crate) final_url: String, - pub(crate) page_title: Option, - pub(crate) html: String, - pub(crate) script_tags: Vec, - pub(crate) network_requests: Vec, - pub(crate) warnings: Vec, +fn validate_settle_window(quiet_ms: u64, max_ms: u64) -> Result<(), String> { + if quiet_ms > max_ms { + return Err(format!( + "--settle-quiet-ms ({quiet_ms}) cannot exceed --settle-max-ms ({max_ms})" + )); + } + Ok(()) +} + +/// A request to collect a single page. +#[derive(Debug, Clone)] +pub struct BrowserCollectRequest { + /// The URL to navigate to. + pub url: url::Url, + /// Pre-navigation init scripts (evaluate-on-new-document). Empty for a plain + /// page audit; the ad-template verifier supplies the read-only collector here. + pub init_scripts: Vec, + /// Whether to perform the deterministic scroll pass after settle. + pub scroll: bool, + /// Whether to extract `window.__tsAdTemplateEvidence` after settle/scroll. + pub collect_ad_evidence: bool, + /// Operator-supplied `(name, value)` cookies set on the browser context + /// before navigation, scoped to the request URL. Used to carry an existing + /// authenticated session (e.g. a valid bot-protection clearance cookie) so + /// the origin serves the real page instead of a challenge. The collector + /// only sends these; it never reads cookies back. + pub cookies: Vec<(String, String)>, +} + +/// The result of collecting a single page. +#[derive(Debug, Clone)] +pub struct CollectedPage { + /// The final URL after redirects. + pub final_url: url::Url, + /// The page title. + pub title: String, + /// Number of ` + +"#; + + const DELAYED_GPT_FIXTURE: &str = r#" + + +
+
+ + +"#; + + fn gpt_fixture_url(html: &'static str) -> Url { + let listener = TcpListener::bind("127.0.0.1:0").expect("should bind fixture server"); + let address = listener.local_addr().expect("should read fixture address"); + std::thread::spawn(move || { + let (mut stream, _) = listener.accept().expect("should accept browser request"); + stream + .set_read_timeout(Some(Duration::from_secs(10))) + .expect("should set fixture read timeout"); + let mut request = Vec::new(); + while !request.ends_with(b"\r\n\r\n") { + let mut chunk = [0_u8; 1024]; + let chunk_len = stream.read(&mut chunk).expect("should read HTTP request"); + assert!(chunk_len > 0, "request should contain complete headers"); + request.extend_from_slice(&chunk[..chunk_len]); + assert!( + request.len() <= 16 * 1024, + "request headers should be bounded" + ); + } + write!( + stream, + "HTTP/1.1 200 OK\r\nContent-Type: text/html\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}", + html.len(), + html, + ) + .expect("should write fixture response"); + }); + Url::parse(&format!("http://{address}/")).expect("should parse fixture URL") + } + + #[test] + fn successful_navigation_status_allows_redirects_but_rejects_errors() { + assert!(is_successful_navigation_status(200)); + assert!(is_successful_navigation_status(302)); + assert!(is_successful_navigation_status(399)); + assert!(!is_successful_navigation_status(199)); + assert!(!is_successful_navigation_status(400)); + assert!(!is_successful_navigation_status(500)); + } + + #[test] + fn navigation_response_returns_warning_for_http_error_status() { + let warning = + validate_navigation_response(navigation_response_with_status(403, "Forbidden")) + .expect("should validate navigation response") + .expect("should return warning for HTTP error status"); + + assert_eq!( + warning, + "audit request returned HTTP 403 Forbidden for `https://example.com/`; results may be partial", + "should warn and continue when the main document returns an HTTP error" + ); + } + + #[test] + fn navigation_response_reports_chromium_request_failure() { + let mut request = + HttpRequest::new(RequestId::new("request-1"), None, None, false, Vec::new()); + request.failure_text = Some("net::ERR_BLOCKED_BY_ORB".to_string()); + + let error = validate_navigation_response(Some(Arc::new(request))) + .expect_err("should reject Chromium request failures"); + + assert_eq!( + error, "main document request failed: net::ERR_BLOCKED_BY_ORB", + "the crawl should retain the browser failure for its final skipped-page note" + ); + } + + #[test] + fn resource_timing_buffer_warning_starts_at_threshold() { + assert_eq!( + resource_timing_buffer_warning(RESOURCE_TIMING_BUFFER_SIZE - 1), + None, + "should not warn before the resource timing buffer threshold" + ); + assert_eq!( + resource_timing_buffer_warning(RESOURCE_TIMING_BUFFER_SIZE), + Some(RESOURCE_TIMING_BUFFER_WARNING), + "should warn when the resource timing buffer reaches the threshold" + ); + } + + #[test] + fn browser_path_candidates_include_common_names() { + let candidates = crate::commands::audit::browser::CHROME_NAMES; + + assert!(candidates.contains(&"google-chrome")); + assert!(candidates.contains(&"chromium")); + assert!(candidates.contains(&"Google Chrome for Testing")); + } + + #[test] + fn browser_run_reports_close_error_before_wait_error() { + let result = combine_browser_run_results( + Ok(()), + Ok(()), + Err("close failed".to_string()), + Err("wait failed".to_string()), + ); + + assert_eq!( + result.expect_err("should preserve teardown error"), + "close failed", + "the close failure is the first teardown failure" + ); + } + + #[test] + fn browser_run_reports_wait_error_when_close_succeeds() { + let result = + combine_browser_run_results(Ok(()), Ok(()), Ok(()), Err("wait failed".to_string())); + + assert_eq!( + result.expect_err("should preserve wait error"), + "wait failed", + "a wait failure must not be mislabeled as a close failure" + ); + } + + #[test] + fn browser_run_preserves_collection_error_over_later_failures() { + let result = combine_browser_run_results( + Err("collection failed".to_string()), + Err("finalization progress failed".to_string()), + Err("close failed".to_string()), + Err("wait failed".to_string()), + ); + + assert_eq!( + result.expect_err("should preserve first browser run error"), + "collection failed" + ); + } + + #[test] + fn browser_run_reports_finalization_progress_before_teardown_errors() { + let result = combine_browser_run_results( + Ok(()), + Err("finalization progress failed".to_string()), + Err("close failed".to_string()), + Err("wait failed".to_string()), + ); + + assert_eq!( + result.expect_err("should preserve finalization progress error"), + "finalization progress failed" + ); + } + + #[test] + #[ignore = "requires local Chrome/Chromium; run through scripts/test-cli.sh"] + fn progress_failure_still_finalizes_browser_session() { + if !browser_fixture_available() { + return; + } + + let collector = BrowserAuditCollector::default(); + let target = Url::parse("http://127.0.0.1:9/").expect("should parse fixture URL"); + let mut phases = Vec::new(); + let error = collector + .collect_pages( + &[target], + &[], + &mut |progress| match progress { + CollectionProgress::Launching => { + phases.push("launching"); + Ok(()) + } + CollectionProgress::Loading { .. } => { + phases.push("loading"); + Err(report_error("simulated progress failure")) + } + CollectionProgress::Planning => { + phases.push("planning"); + Ok(()) + } + CollectionProgress::Finalizing => { + phases.push("finalizing"); + Ok(()) + } + }, + &mut |_, _| panic!("page sink should not run after progress failure"), + ) + .expect_err("should return progress failure after browser teardown"); + + let rendered_error = format!("{error:?}"); + assert!( + rendered_error.contains("simulated progress failure"), + "should preserve progress failure, got {rendered_error}" + ); + assert_eq!(phases, ["launching", "loading", "finalizing"]); + } + + #[test] + #[ignore = "requires local Chrome/Chromium; run through scripts/test-cli.sh"] + fn collects_lazy_gpt_slot_only_when_scroll_is_enabled() { + if !browser_fixture_available() { + return; + } + + let without_scroll = BrowserAuditCollector::default() + .collect_page(&gpt_fixture_url(LAZY_GPT_FIXTURE), &[]) + .expect("should collect without scrolling"); + let with_scroll = BrowserAuditCollector::default() + .with_scroll(true) + .collect_page(&gpt_fixture_url(LAZY_GPT_FIXTURE), &[]) + .expect("should collect with scrolling"); + + assert!( + without_scroll.gpt_slots.is_empty(), + "lazy GPT slot should not exist before scrolling" + ); + assert!( + with_scroll + .gpt_slots + .iter() + .any(|slot| { slot.gam_unit_path == "/123/lazy" && slot.div_id == "ad-lazy-0" }), + "scrolling should trigger and collect the lazy GPT slot" + ); + } + + #[test] + #[ignore = "requires local Chrome/Chromium; run through scripts/test-cli.sh"] + fn waits_for_delayed_gpt_registry_to_stabilize_in_definition_order() { + if !browser_fixture_available() { + return; + } + + let collected = BrowserAuditCollector::default() + .collect_page(&gpt_fixture_url(DELAYED_GPT_FIXTURE), &[]) + .expect("should collect delayed GPT registry"); + + assert_eq!( + collected.gpt_slots, + vec![ + CollectedGptSlot { + gam_unit_path: "/123/z-delayed".to_string(), + div_id: "ad-z-delayed-0".to_string(), + sizes: vec![(300, 250)], + }, + CollectedGptSlot { + gam_unit_path: "/123/a-delayed".to_string(), + div_id: "ad-a-delayed-0".to_string(), + sizes: vec![(728, 90)], + }, + ], + "collector should wait for stable registration without reordering slots" + ); + } + + fn navigation_response_with_status(status: i64, status_text: &str) -> ArcHttpRequest { + let mut request = + HttpRequest::new(RequestId::new("request-1"), None, None, false, Vec::new()); + request.response = Some( + Response::builder() + .url("https://example.com/") + .status(status) + .status_text(status_text) + .headers(Headers::default()) + .mime_type("text/html") + .charset("utf-8") + .connection_reused(false) + .connection_id(1.0) + .encoded_data_length(0.0) + .security_state(SecurityState::Secure) + .build() + .expect("should build navigation response"), + ); + + Some(Arc::new(request)) + } +} diff --git a/crates/trusted-server-cli/src/commands/audit/generate/collector.rs b/crates/trusted-server-cli/src/commands/audit/generate/collector.rs new file mode 100644 index 000000000..dc23af09c --- /dev/null +++ b/crates/trusted-server-cli/src/commands/audit/generate/collector.rs @@ -0,0 +1,366 @@ +use serde::{Deserialize, Serialize}; +use url::Url; + +use crate::error::CliResult; + +/// Warning recorded on a page collected with the audit consent stub installed. +/// +/// A whole-run fact rather than a property of one page, so consumers report it +/// once and unscoped instead of once per page and per profile. +pub(crate) const CONSENT_STUB_WARNING: &str = "consent_stub_active: audit consent APIs were stubbed; re-run with --no-assume-consent to observe the publisher CMP without substitution"; + +/// A user-visible phase reached while collecting browser audit evidence. +#[derive(Debug, Clone, Copy)] +pub(crate) enum CollectionProgress<'a> { + /// The browser process is about to launch. + Launching, + /// A page navigation is about to begin. + Loading { + /// One-based position of this attempted page in the crawl. + current: usize, + /// Total pages when planning has completed, or `None` for the root. + total: Option, + /// Target page; renderers must omit credentials, query, and fragment. + url: &'a Url, + }, + /// Follow-up pages are being selected from the collected root page. + Planning, + /// The browser session is being closed and its process reaped. + Finalizing, +} + +/// Sink invoked synchronously when browser collection reaches a visible phase. +/// +/// Returning an error stops new collection work. An already-launched browser +/// must still be finalized, closed, and waited on before that error is returned. +pub(crate) type ProgressSink<'a> = + &'a mut dyn for<'event> FnMut(CollectionProgress<'event>) -> CliResult<()>; + +/// Sink invoked once per collected page during a batch crawl. +/// +/// Receives the per-page outcome so a failed page can be folded into the run as +/// a warning rather than aborting it; returning `Err` stops the crawl. +pub(crate) type PageSink<'a> = + &'a mut dyn FnMut(&Url, CliResult) -> CliResult; + +/// Plans follow-up URLs from the successfully collected root page. +pub(crate) type RootPlanner<'a> = &'a mut dyn FnMut(&Url, &CollectedPage) -> CliResult>; + +/// Whether a batch crawl should keep going after a page. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum ControlFlow { + /// Collect the next target. + Continue, + /// Stop the crawl without an error (budget reached, challenge rate exceeded). + /// + /// What this can prevent depends on the collector: a sequential one loads no + /// further pages, while the browser collector has already finished + /// navigating by the time it folds, so there it only stops the fold. + Stop, +} + +pub(crate) trait AuditCollector { + /// Collects a live page. `cookies` are `(name, value)` pairs set on the + /// browser context before navigation (scoped to `target_url`) so an existing + /// session — e.g. a valid bot-protection clearance cookie — can carry the + /// audit past an origin challenge. + fn collect_page( + &self, + target_url: &Url, + cookies: &[(String, String)], + ) -> CliResult; + + /// Collects several pages in one session, handing each result to `on_page`. + /// + /// The default implementation loops over [`collect_page`](Self::collect_page), + /// which keeps every existing implementor working unchanged. The browser + /// collector overrides it to reuse one Chrome instance and profile across the + /// crawl — a fresh launch per page dominates the cost of a multi-page run, + /// and a shared profile carries bot-protection clearance cookies site-wide. + /// + /// Collectors may buffer results until the browser session closes so CPU-heavy + /// HTML analysis cannot starve a single-threaded CDP event pump. The sink API + /// keeps that buffering policy private and lets simple collectors stream. + /// + /// # Errors + /// + /// Returns an error when `on_page` does, or when the session itself cannot + /// be established. Individual page failures are delivered to `on_page`. + fn collect_pages( + &self, + targets: &[Url], + cookies: &[(String, String)], + on_progress: ProgressSink<'_>, + on_page: PageSink<'_>, + ) -> CliResult<()> { + for (index, target) in targets.iter().enumerate() { + on_progress(CollectionProgress::Loading { + current: index + 1, + total: Some(targets.len()), + url: target, + })?; + let collected = self.collect_page(target, cookies); + if on_page(target, collected)? == ControlFlow::Stop { + break; + } + } + Ok(()) + } + + /// Collects a root and follow-up URLs planned from it in one logical crawl. + /// + /// The browser implementation overrides this so planning happens while the + /// root's browser/profile remains open. Simple collectors retain equivalent + /// behavior through the default implementation. + fn collect_site( + &self, + root: &Url, + cookies: &[(String, String)], + on_progress: ProgressSink<'_>, + planner: RootPlanner<'_>, + on_page: PageSink<'_>, + ) -> CliResult<()> { + on_progress(CollectionProgress::Loading { + current: 1, + total: None, + url: root, + })?; + // A root failure is reported through `on_page` rather than returned, so + // the caller sees the reason as a per-page note exactly as it does from + // the browser collector. With no root page there is nothing to plan + // from, so the crawl ends here. + let root_page = match self.collect_page(root, cookies) { + Ok(page) => page, + Err(error) => { + on_page(root, Err(error))?; + return Ok(()); + } + }; + on_progress(CollectionProgress::Planning)?; + let targets = planner(root, &root_page)?; + if on_page(root, Ok(root_page))? == ControlFlow::Stop { + return Ok(()); + } + let total = targets.len() + 1; + for (index, target) in targets.iter().enumerate() { + on_progress(CollectionProgress::Loading { + current: index + 2, + total: Some(total), + url: target, + })?; + let collected = self.collect_page(target, cookies); + if on_page(target, collected)? == ControlFlow::Stop { + break; + } + } + Ok(()) + } +} + +#[derive(Debug, Clone, Deserialize, Serialize, PartialEq, Eq)] +pub(crate) struct CollectedPage { + pub(crate) requested_url: String, + pub(crate) final_url: String, + pub(crate) page_title: Option, + pub(crate) html: String, + pub(crate) script_tags: Vec, + pub(crate) network_requests: Vec, + /// Slots read from the live GPT registry (`googletag.pubads().getSlots()`). + /// + /// Populated at `defineSlot` time, so this captures configured slots even + /// when the ad request never fires (consent-gated or iframe-issued). + #[serde(default)] + pub(crate) gpt_slots: Vec, + /// Same-origin `a[href]` targets read from the hydrated DOM, absolutized. + /// + /// Read from the live DOM rather than the served HTML on purpose: an + /// app-router page keeps its link graph in the framework payload, so parsing + /// the raw markup finds only a fraction of the site's sections. + #[serde(default)] + pub(crate) links: Vec, + /// Sitemap `` entries discovered from `robots.txt`, when fetched. + /// + /// Empty unless sitemap discovery ran (root page only). + #[serde(default)] + pub(crate) sitemap_locs: Vec, + pub(crate) warnings: Vec, +} + +/// A same-origin link observed in the hydrated DOM. +#[derive(Debug, Clone, Deserialize, Serialize, PartialEq, Eq)] +pub(crate) struct CollectedLink { + /// Absolute URL of the link target. + pub(crate) url: String, + /// Whether the anchor sits inside site navigation (`nav`, `header`, + /// `[role="navigation"]`). Nav links are the publisher's own declaration of + /// its taxonomy, so they rank above body links when choosing sections. + pub(crate) in_nav: bool, +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::error::{cli_error, report_error}; + + struct ProgressCollector; + + impl AuditCollector for ProgressCollector { + fn collect_page( + &self, + target_url: &Url, + _cookies: &[(String, String)], + ) -> CliResult { + if target_url.path() == "/broken" { + return cli_error("simulated page failure"); + } + Ok(CollectedPage { + requested_url: target_url.to_string(), + final_url: target_url.to_string(), + page_title: None, + html: String::new(), + script_tags: Vec::new(), + network_requests: Vec::new(), + gpt_slots: Vec::new(), + links: Vec::new(), + sitemap_locs: Vec::new(), + warnings: Vec::new(), + }) + } + } + + fn record_progress(event: CollectionProgress<'_>) -> String { + match event { + CollectionProgress::Launching => "launching".to_string(), + CollectionProgress::Loading { + current, + total, + url, + } => format!( + "loading:{current}/{}:{}", + total.map_or_else(|| "?".to_string(), |total| total.to_string()), + url.path() + ), + CollectionProgress::Planning => "planning".to_string(), + CollectionProgress::Finalizing => "finalizing".to_string(), + } + } + + #[test] + fn default_collect_site_reports_root_planning_and_offset_followups() { + let collector = ProgressCollector; + let root = Url::parse("https://publisher.example/").expect("should parse root URL"); + let news = Url::parse("https://publisher.example/news").expect("should parse news URL"); + let broken = + Url::parse("https://publisher.example/broken").expect("should parse broken URL"); + let mut events = Vec::new(); + let mut outcomes = Vec::new(); + + collector + .collect_site( + &root, + &[], + &mut |event| { + events.push(record_progress(event)); + Ok(()) + }, + &mut |_, _| Ok(vec![news.clone(), broken.clone()]), + &mut |url, result| { + outcomes.push((url.path().to_string(), result.is_ok())); + Ok(ControlFlow::Continue) + }, + ) + .expect("should collect site despite one page outcome failing"); + + assert_eq!( + events, + [ + "loading:1/?:/", + "planning", + "loading:2/3:/news", + "loading:3/3:/broken", + ] + ); + assert_eq!( + outcomes, + [ + ("/".to_string(), true), + ("/news".to_string(), true), + ("/broken".to_string(), false) + ] + ); + } + + #[test] + fn default_collect_pages_reports_a_fixed_total() { + let collector = ProgressCollector; + let targets = [ + Url::parse("https://publisher.example/").expect("should parse root URL"), + Url::parse("https://publisher.example/broken").expect("should parse broken URL"), + ]; + let mut events = Vec::new(); + + collector + .collect_pages( + &targets, + &[], + &mut |event| { + events.push(record_progress(event)); + Ok(()) + }, + &mut |_, _| Ok(ControlFlow::Continue), + ) + .expect("should deliver failed page as an outcome"); + + assert_eq!(events, ["loading:1/2:/", "loading:2/2:/broken"]); + } + + #[test] + fn default_collection_stops_when_progress_fails() { + let collector = ProgressCollector; + let targets = [Url::parse("https://publisher.example/").expect("should parse root URL")]; + + let error = collector + .collect_pages( + &targets, + &[], + &mut |_| Err(report_error("simulated progress failure")), + &mut |_, _| panic!("page sink should not run after progress failure"), + ) + .expect_err("should return progress failure"); + + assert!(format!("{error:?}").contains("simulated progress failure")); + } +} + +/// A single slot read from the page's live GPT registry. +#[derive(Debug, Clone, Deserialize, Serialize, PartialEq, Eq)] +pub(crate) struct CollectedGptSlot { + /// The GAM ad-unit path (`slot.getAdUnitPath()`). + pub(crate) gam_unit_path: String, + /// The slot's div element id (`slot.getSlotElementId()`). + pub(crate) div_id: String, + /// Numeric `[width, height]` sizes (`slot.getSizes()`, fluid entries dropped). + pub(crate) sizes: Vec<(u32, u32)>, +} + +#[derive(Debug, Clone, Deserialize, Serialize, PartialEq, Eq)] +pub(crate) struct CollectedScriptTag { + pub(crate) src: Option, + pub(crate) inline_text: Option, +} + +#[derive(Debug, Clone, Deserialize, Serialize, PartialEq, Eq)] +pub(crate) struct CollectedRequest { + pub(crate) url: String, + pub(crate) resource_type: Option, +} + +impl CollectedPage { + pub(crate) fn requested_url(&self) -> Result { + Url::parse(&self.requested_url) + } + + pub(crate) fn final_url(&self) -> Result { + Url::parse(&self.final_url) + } +} diff --git a/crates/trusted-server-cli/src/commands/audit/generate/crawl_plan.rs b/crates/trusted-server-cli/src/commands/audit/generate/crawl_plan.rs new file mode 100644 index 000000000..76d57cf9e --- /dev/null +++ b/crates/trusted-server-cli/src/commands/audit/generate/crawl_plan.rs @@ -0,0 +1,864 @@ +//! Pure crawl planning: turn discovered links and sitemap entries into the +//! bounded set of pages worth loading in a browser. +//! +//! The goal is deliberately *not* site coverage. Ad slots repeat per site +//! section, and the generated config needs one glob pair per section +//! (`/news` and `/news/*`), so one representative page per section is enough. +//! That keeps the crawl proportional to the publisher's taxonomy (a dozen +//! sections) rather than its catalog (tens of thousands of articles). +//! +//! Two sources feed the plan and each supplies a half the other cannot: +//! +//! - **Navigation links** give section *landing* paths (`/news`), which +//! sitemaps routinely omit, and are the publisher's own taxonomy declaration. +//! - **Sitemap entries** give a real *article* per section (`/news/story-abc`), +//! which is where in-content slots live, and reveal sections hidden behind a +//! navigation overflow menu. + +use std::collections::BTreeMap; + +use url::Url; + +use super::collector::CollectedLink; + +/// Path segments that are never a content section worth sampling. +/// +/// These carry either no ad stack at all or an unrepresentative one, and +/// crawling them spends budget that a real section needs. +const NOISE_SEGMENTS: &[&str] = &[ + "about", + "about-us", + "account", + "author", + "cart", + "contact", + "editorial-policy", + "login", + "logout", + "newsletter", + "page", + "press", + "privacy", + "register", + "search", + "sitemap", + "subscribe", + "terms", +]; + +/// File extensions that are assets rather than pages. +const NON_PAGE_EXTENSIONS: &[&str] = &[ + ".jpg", ".jpeg", ".png", ".gif", ".webp", ".avif", ".svg", ".ico", ".css", ".js", ".json", + ".xml", ".pdf", ".zip", ".mp4", ".mp3", ".rss", +]; + +/// ISO 639-1 alpha-2 language codes, sorted for binary search. +/// +/// Country codes are deliberately absent: `/us` and `/tv` are section roots on +/// plenty of publishers, and only the language form appears as a URL locale +/// prefix on its own. +const ISO_639_1_CODES: &[&str] = &[ + "aa", "ab", "ae", "af", "ak", "am", "an", "ar", "as", "av", "ay", "az", "ba", "be", "bg", "bh", + "bi", "bm", "bn", "bo", "br", "bs", "ca", "ce", "ch", "co", "cr", "cs", "cu", "cv", "cy", "da", + "de", "dv", "dz", "ee", "el", "en", "eo", "es", "et", "eu", "fa", "ff", "fi", "fj", "fo", "fr", + "fy", "ga", "gd", "gl", "gn", "gu", "gv", "ha", "he", "hi", "ho", "hr", "ht", "hu", "hy", "hz", + "ia", "id", "ie", "ig", "ii", "ik", "io", "is", "it", "iu", "ja", "jv", "ka", "kg", "ki", "kj", + "kk", "kl", "km", "kn", "ko", "kr", "ks", "ku", "kv", "kw", "ky", "la", "lb", "lg", "li", "ln", + "lo", "lt", "lu", "lv", "mg", "mh", "mi", "mk", "ml", "mn", "mr", "ms", "mt", "my", "na", "nb", + "nd", "ne", "ng", "nl", "nn", "no", "nr", "nv", "ny", "oc", "oj", "om", "or", "os", "pa", "pi", + "pl", "ps", "pt", "qu", "rm", "rn", "ro", "ru", "rw", "sa", "sc", "sd", "se", "sg", "si", "sk", + "sl", "sm", "sn", "so", "sq", "sr", "ss", "st", "su", "sv", "sw", "ta", "te", "tg", "th", "ti", + "tk", "tl", "tn", "to", "tr", "ts", "tt", "tw", "ty", "ug", "uk", "ur", "uz", "ve", "vi", "vo", + "wa", "wo", "xh", "yi", "yo", "za", "zh", "zu", +]; + +/// Filenames that name a directory's index document rather than a page of their +/// own, so a link to one is treated as a link to the parent directory. +const DIRECTORY_INDEX_NAMES: &[&str] = &[ + "index.html", + "index.htm", + "index.php", + "default.html", + "default.htm", + "default.php", + "home.html", + "home.htm", + "home.php", +]; + +/// Bounds on how much of a site a single run will load. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) struct CrawlBudget { + /// Maximum number of sections to sample. + pub(crate) max_sections: usize, + /// Maximum number of pages to load in total, including the root. + pub(crate) max_pages: usize, +} + +impl Default for CrawlBudget { + fn default() -> Self { + Self { + max_sections: 8, + max_pages: 17, + } + } +} + +/// One section selected for sampling. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(super) struct PlannedSection { + /// The path segment at [`CrawlPlan::section_segment`] identifying the section. + pub(super) segment: String, + /// The section landing page, when one was observed. + pub(super) landing: Option, + /// A representative content page inside the section, when one was observed. + pub(super) article: Option, +} + +impl PlannedSection { + /// The pages to load for this section, landing first. + fn targets(&self) -> impl Iterator { + self.landing.iter().chain(self.article.iter()) + } +} + +/// The bounded outcome of planning. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(super) struct CrawlPlan { + /// Sections selected for sampling, highest confidence first. + pub(super) sections: Vec, + /// Sections found but dropped because the budget was already spent. + pub(super) dropped_sections: Vec, + /// Human-readable notes about how the plan was reached. + pub(super) notes: Vec, + /// Path segment used to distinguish sections in this crawl. + pub(super) section_segment: usize, +} + +impl CrawlPlan { + /// Page URLs to load, in crawl order. The root is *not* included — the + /// caller has already collected it in order to plan at all. + pub(super) fn targets(&self) -> Vec { + self.sections + .iter() + .flat_map(PlannedSection::targets) + .cloned() + .collect() + } +} + +/// Evidence gathered about one candidate section before ranking. +#[derive(Debug, Default)] +struct SectionCandidate { + landing: Option, + article: Option, + in_nav: bool, + in_sitemap: bool, + link_count: usize, +} + +impl SectionCandidate { + /// Confidence ordering: corroborated by both sources beats either alone, + /// and navigation beats a sitemap-only hit because navigation is the + /// publisher's own statement of what its sections are. + fn rank(&self) -> u8 { + match (self.in_nav, self.in_sitemap) { + (true, true) => 3, + (true, false) => 2, + (false, true) => 1, + (false, false) => 0, + } + } +} + +/// Plans the crawl from the root page's links and any sitemap entries. +/// +/// `root` bounds the crawl: every candidate must share its origin, which also +/// stops a hostile or misconfigured `robots.txt` from redirecting the crawl (and +/// the operator's cookies) at an unrelated host. +pub(super) fn plan_crawl( + root: &Url, + links: &[CollectedLink], + sitemap_locs: &[String], + budget: CrawlBudget, +) -> CrawlPlan { + let section_segment = usize::from(root_is_locale_prefix(root)); + let mut candidates: BTreeMap = BTreeMap::new(); + let mut notes = Vec::new(); + + for link in links { + let Some(url) = same_origin_page_url(root, &link.url, section_segment) else { + continue; + }; + let Some(segment) = section_at(&url, section_segment) else { + continue; + }; + let entry = candidates.entry(segment).or_default(); + entry.in_nav |= link.in_nav; + entry.link_count += 1; + record_url(entry, &url, section_segment); + } + + let mut sitemap_pages = 0_usize; + for loc in sitemap_locs { + let Some(url) = same_origin_page_url(root, loc, section_segment) else { + continue; + }; + let Some(segment) = section_at(&url, section_segment) else { + continue; + }; + sitemap_pages += 1; + let entry = candidates.entry(segment).or_default(); + entry.in_sitemap = true; + record_url(entry, &url, section_segment); + } + + if !sitemap_locs.is_empty() { + notes.push(format!( + "sitemap contributed {sitemap_pages} same-origin page(s) across {} section(s)", + candidates.values().filter(|c| c.in_sitemap).count() + )); + } + if links.iter().all(|link| !link.in_nav) && !links.is_empty() { + notes.push( + "no navigation links were found; sections were inferred from body links only" + .to_string(), + ); + } + + // Rank before truncating: confidence first, then how heavily the section is + // linked, then the segment name so runs are reproducible. + let mut ranked: Vec<(String, SectionCandidate)> = candidates.into_iter().collect(); + ranked.sort_by(|(left_segment, left), (right_segment, right)| { + right + .rank() + .cmp(&left.rank()) + .then(right.link_count.cmp(&left.link_count)) + .then(left_segment.cmp(right_segment)) + }); + + let mut sections = Vec::new(); + let mut dropped_sections = Vec::new(); + // The root page is already collected and counts against the page budget. + let mut pages_used = 1_usize; + for (segment, candidate) in ranked { + let planned = PlannedSection { + segment: segment.clone(), + landing: candidate.landing, + article: candidate.article, + }; + let cost = planned.targets().count(); + if cost == 0 { + continue; + } + if sections.len() >= budget.max_sections || pages_used + cost > budget.max_pages { + dropped_sections.push(segment); + continue; + } + pages_used += cost; + sections.push(planned); + } + + if !dropped_sections.is_empty() { + let shown = dropped_sections + .iter() + .take(10) + .cloned() + .collect::>() + .join(", "); + let remainder = dropped_sections.len().saturating_sub(10); + let suffix = if remainder == 0 { + String::new() + } else { + format!(", and {remainder} more") + }; + notes.push(format!( + "budget reached: {} section(s) not sampled ({shown}{suffix}); raise --max-sections/--max-pages to include them", + dropped_sections.len(), + )); + } + + CrawlPlan { + sections, + dropped_sections, + notes, + section_segment, + } +} + +/// Files a URL as the section's landing page or its representative article. +/// +/// The first candidate of each kind wins, so a run is stable given stable input. +fn record_url(entry: &mut SectionCandidate, url: &Url, section_segment: usize) { + if segment_count(url) == section_segment + 1 { + if entry.landing.is_none() { + entry.landing = Some(url.clone()); + } + } else if entry.article.is_none() { + entry.article = Some(url.clone()); + } +} + +/// Parses `raw` against `root` and keeps it only if it is a same-origin page. +/// +/// Rejects other origins, non-HTTP schemes, asset extensions, and paginated or +/// utility paths. Query and fragment are dropped so `/news?page=2` and +/// `/news#top` collapse onto `/news`. +fn same_origin_page_url(root: &Url, raw: &str, section_segment: usize) -> Option { + let mut url = root.join(raw).ok()?; + if !matches!(url.scheme(), "http" | "https") || url.origin() != root.origin() { + return None; + } + url.set_query(None); + url.set_fragment(None); + + let path = percent_decode_for_filtering(url.path()).to_ascii_lowercase(); + if NON_PAGE_EXTENSIONS + .iter() + .any(|extension| path.ends_with(extension)) + { + return None; + } + // A section reachable only through its index document is still that section: + // `/news/index.html` is `/news`. Rejecting the URL outright loses the + // section; dropping the filename keeps it. + if path + .split('/') + .rfind(|part| !part.is_empty()) + .is_some_and(|last| DIRECTORY_INDEX_NAMES.contains(&last)) + { + url.path_segments_mut().ok()?.pop(); + } + let path = percent_decode_for_filtering(url.path()).to_ascii_lowercase(); + let segments: Vec<&str> = path.split('/').filter(|part| !part.is_empty()).collect(); + if segments.is_empty() { + return None; + } + if NOISE_SEGMENTS.contains(&segments.get(section_segment).copied().unwrap_or_default()) { + return None; + } + if section_segment > 0 { + let root_path = percent_decode_for_filtering(root.path()).to_ascii_lowercase(); + let root_segments: Vec<&str> = root_path + .split('/') + .filter(|part| !part.is_empty()) + .collect(); + if !segments.starts_with(&root_segments) { + return None; + } + } + // `/news/page/2` is the same inventory as `/news`, so it is not a second + // sample worth spending a page load on. + if segments.contains(&"page") { + return None; + } + Some(url) +} + +/// The non-empty path segment at `index`, percent-decoded and lowercased. +fn section_at(url: &Url, index: usize) -> Option { + percent_decode_for_filtering(url.path()) + .split('/') + .filter(|part| !part.is_empty()) + .nth(index) + .map(str::to_ascii_lowercase) +} + +/// Whether the requested root is nothing but a locale prefix, which puts +/// sections one segment deeper than usual. +fn root_is_locale_prefix(root: &Url) -> bool { + let segments: Vec<&str> = root + .path() + .split('/') + .filter(|part| !part.is_empty()) + .collect(); + matches!(segments.as_slice(), [locale] if is_locale_segment(locale)) +} + +/// Whether a root's single path segment is a locale prefix (`/en`, `/en-gb`) +/// rather than a content section. +/// +/// The language half must be a real ISO 639-1 code. Accepting any two letters +/// read ordinary section roots — `/tv`, `/ai`, `/us` — as locales, which shifts +/// `section_segment` by one: article slugs then become "sections" and the +/// containment check below discards the root's real siblings. +fn is_locale_segment(segment: &str) -> bool { + let segment = segment.to_ascii_lowercase(); + match segment.as_bytes() { + [_, _] => is_language_code(&segment), + [_, _, b'-', c, d] => { + is_language_code(&segment[..2]) && c.is_ascii_alphabetic() && d.is_ascii_alphabetic() + } + _ => false, + } +} + +/// Whether `segment` is an ISO 639-1 alpha-2 language code. +fn is_language_code(segment: &str) -> bool { + ISO_639_1_CODES.binary_search(&segment).is_ok() +} + +/// Decodes percent escapes solely for normalized path classification. +fn percent_decode_for_filtering(path: &str) -> String { + let bytes = path.as_bytes(); + let mut decoded = Vec::with_capacity(bytes.len()); + let mut index = 0; + while index < bytes.len() { + if bytes[index] == b'%' + && index + 2 < bytes.len() + && let (Some(high), Some(low)) = + (hex_value(bytes[index + 1]), hex_value(bytes[index + 2])) + { + decoded.push((high << 4) | low); + index += 3; + } else { + decoded.push(bytes[index]); + index += 1; + } + } + String::from_utf8_lossy(&decoded).into_owned() +} + +/// Converts one ASCII hexadecimal digit to its numeric value. +fn hex_value(byte: u8) -> Option { + match byte { + b'0'..=b'9' => Some(byte - b'0'), + b'a'..=b'f' => Some(byte - b'a' + 10), + b'A'..=b'F' => Some(byte - b'A' + 10), + _ => None, + } +} + +/// Count of non-empty path segments. +fn segment_count(url: &Url) -> usize { + url.path() + .split('/') + .filter(|part| !part.is_empty()) + .count() +} + +#[cfg(test)] +mod tests { + use super::*; + + fn root() -> Url { + Url::parse("https://publisher.example/").expect("valid root") + } + + fn nav(path: &str) -> CollectedLink { + CollectedLink { + url: format!("https://publisher.example{path}"), + in_nav: true, + } + } + + fn body(path: &str) -> CollectedLink { + CollectedLink { + url: format!("https://publisher.example{path}"), + in_nav: false, + } + } + + fn segments(plan: &CrawlPlan) -> Vec<&str> { + plan.sections + .iter() + .map(|section| section.segment.as_str()) + .collect() + } + + #[test] + fn pairs_a_landing_page_with_an_article_from_the_sitemap() { + let plan = plan_crawl( + &root(), + &[nav("/news")], + &["https://publisher.example/news/story-abc".to_string()], + CrawlBudget::default(), + ); + + assert_eq!( + segments(&plan), + ["news"], + "only the witnessed section should be planned" + ); + let section = &plan.sections[0]; + assert_eq!( + section.landing.as_ref().map(Url::as_str), + Some("https://publisher.example/news") + ); + assert_eq!( + section.article.as_ref().map(Url::as_str), + Some("https://publisher.example/news/story-abc") + ); + assert_eq!(plan.targets().len(), 2, "should load landing then article"); + } + + #[test] + fn cross_origin_candidates_are_dropped() { + // Guards both the sitemap (a `Sitemap:` directive can point anywhere) + // and links: the crawl carries operator cookies, so it must not leave + // the requested origin. + let plan = plan_crawl( + &root(), + &[CollectedLink { + url: "https://tracker.example/news".to_string(), + in_nav: true, + }], + &["https://other.example/deals/x".to_string()], + CrawlBudget::default(), + ); + + assert!( + plan.sections.is_empty(), + "no off-origin section should survive, got {:?}", + segments(&plan) + ); + } + + #[test] + fn utility_paths_and_assets_are_filtered() { + let plan = plan_crawl( + &root(), + &[ + nav("/about-us"), + nav("/search"), + nav("/editorial-policy"), + nav("/logo.png"), + nav("/feed.xml"), + nav("/news/page/2"), + nav("/news"), + ], + &[], + CrawlBudget::default(), + ); + + assert_eq!( + segments(&plan), + ["news"], + "only the real content section should remain" + ); + } + + #[test] + fn query_and_fragment_collapse_onto_one_landing_page() { + let plan = plan_crawl( + &root(), + &[nav("/news?utm_source=x"), nav("/news#top"), nav("/news")], + &[], + CrawlBudget::default(), + ); + + assert_eq!( + segments(&plan), + ["news"], + "only the witnessed section should be planned" + ); + assert_eq!( + plan.sections[0].landing.as_ref().map(Url::as_str), + Some("https://publisher.example/news"), + "tracking query and fragment should be stripped" + ); + } + + #[test] + fn nav_and_sitemap_corroboration_outranks_either_alone() { + let plan = plan_crawl( + &root(), + &[nav("/features"), body("/reviews")], + &[ + "https://publisher.example/features/story".to_string(), + "https://publisher.example/deals/x".to_string(), + ], + CrawlBudget::default(), + ); + + assert_eq!( + segments(&plan)[0], + "features", + "nav + sitemap should rank first, got {:?}", + segments(&plan) + ); + } + + #[test] + fn budget_truncates_and_reports_what_was_dropped() { + let links: Vec = ["a", "b", "c", "d"] + .iter() + .map(|segment| nav(&format!("/{segment}"))) + .collect(); + + let plan = plan_crawl( + &root(), + &links, + &[], + CrawlBudget { + max_sections: 2, + max_pages: 17, + }, + ); + + assert_eq!(plan.sections.len(), 2, "section cap should be honoured"); + assert_eq!( + plan.dropped_sections.len(), + 2, + "sections past the budget should be reported as dropped" + ); + assert!( + plan.notes + .iter() + .any(|note| note.contains("budget reached")), + "dropping sections must be reported, not silent: {:?}", + plan.notes + ); + } + + #[test] + fn page_budget_counts_the_already_collected_root() { + // max_pages = 3 leaves room for exactly one landing+article pair on top + // of the root page the caller already loaded. + let plan = plan_crawl( + &root(), + &[nav("/news"), nav("/deals")], + &[ + "https://publisher.example/news/a".to_string(), + "https://publisher.example/deals/b".to_string(), + ], + CrawlBudget { + max_sections: 8, + max_pages: 3, + }, + ); + + assert_eq!( + plan.targets().len(), + 2, + "root + 2 pages fills max_pages = 3" + ); + assert_eq!( + plan.dropped_sections.len(), + 1, + "the section past the budget should be reported as dropped" + ); + } + + #[test] + fn body_only_links_still_yield_sections_with_a_note() { + let plan = plan_crawl( + &root(), + &[body("/news"), body("/deals")], + &[], + CrawlBudget::default(), + ); + + assert_eq!(segments(&plan), ["deals", "news"]); + assert!( + plan.notes + .iter() + .any(|note| note.contains("no navigation links")), + "a nav-less page should say so: {:?}", + plan.notes + ); + } + + #[test] + fn empty_input_plans_nothing_rather_than_panicking() { + let plan = plan_crawl(&root(), &[], &[], CrawlBudget::default()); + + assert!( + plan.sections.is_empty(), + "no input means no sections to sample" + ); + assert!( + plan.targets().is_empty(), + "no sections means nothing to load" + ); + } + + #[test] + fn locale_root_plans_sections_from_the_second_segment() { + let locale_root = Url::parse("https://publisher.example/en").expect("should parse root"); + let plan = plan_crawl( + &locale_root, + &[nav("/en/news"), nav("/en/deals")], + &[ + "https://publisher.example/en/news/story".to_string(), + "https://publisher.example/en/deals/item".to_string(), + ], + CrawlBudget::default(), + ); + + assert_eq!( + plan.section_segment, 1, + "a locale root puts sections one segment deeper" + ); + assert_eq!(segments(&plan), ["deals", "news"]); + assert_eq!( + plan.targets().len(), + 4, + "each section contributes a landing page and an article" + ); + } + + #[test] + fn encoded_noise_and_page_extensions_are_filtered() { + let plan = plan_crawl( + &root(), + &[ + nav("/%70rivacy"), + nav("/index.html"), + nav("/archive.htm"), + nav("/story.php"), + nav("/news"), + ], + &[], + CrawlBudget::default(), + ); + + assert_eq!( + segments(&plan), + ["archive.htm", "news", "story.php"], + "only directory-index documents should be excluded by extension" + ); + } + + #[test] + fn a_two_letter_section_root_is_not_read_as_a_locale() { + // `/tv`, `/ai` and `/us` are section roots, not locales. Reading them as + // locales moves the section segment to 1, so article slugs become + // "sections" and the root's real siblings are discarded. + for root_path in ["/tv", "/ai", "/us"] { + let section_root = Url::parse(&format!("https://publisher.example{root_path}")) + .expect("should parse root"); + let plan = plan_crawl( + §ion_root, + &[ + nav(&format!("{root_path}/story-one")), + nav(&format!("{root_path}/story-two")), + ], + &[], + CrawlBudget::default(), + ); + + assert_eq!( + plan.section_segment, 0, + "`{root_path}` should be a section root, not a locale prefix" + ); + assert_eq!( + segments(&plan), + [root_path.trim_start_matches('/')], + "articles below `{root_path}` should stay one section" + ); + } + } + + #[test] + fn a_real_language_prefix_is_still_read_as_a_locale() { + for root_path in ["/en", "/fr", "/pt-br"] { + let locale_root = Url::parse(&format!("https://publisher.example{root_path}")) + .expect("should parse root"); + let plan = plan_crawl( + &locale_root, + &[nav(&format!("{root_path}/news"))], + &[], + CrawlBudget::default(), + ); + + assert_eq!( + plan.section_segment, 1, + "`{root_path}` is a locale prefix, so sections start one segment in" + ); + assert_eq!(segments(&plan), ["news"]); + } + } + + #[test] + fn a_section_reachable_only_by_its_index_document_collapses_to_the_parent() { + let plan = plan_crawl( + &root(), + &[ + nav("/news/index.html"), + nav("/deals/index.php"), + nav("/sport/home.htm"), + ], + &[], + CrawlBudget::default(), + ); + + assert_eq!( + segments(&plan), + ["deals", "news", "sport"], + "an index document names its section rather than disqualifying it" + ); + let targets: Vec = plan + .targets() + .iter() + .map(|url| url.path().to_string()) + .collect(); + assert_eq!( + targets, + ["/deals", "/news", "/sport"], + "the parent directory is what gets loaded" + ); + } + + #[test] + fn locale_root_rejects_candidates_outside_its_path_prefix() { + let locale_root = Url::parse("https://publisher.example/en").expect("should parse root"); + let plan = plan_crawl( + &locale_root, + &[nav("/en/news"), nav("/fr/deals")], + &[], + CrawlBudget::default(), + ); + + assert_eq!( + segments(&plan), + ["news"], + "a locale-root crawl must not mix another locale on the same origin" + ); + } + + #[test] + fn a_section_root_does_not_treat_article_slugs_as_sections() { + let section_root = Url::parse("https://publisher.example/news").expect("should parse root"); + let plan = plan_crawl( + §ion_root, + &[nav("/news/story-one"), nav("/news/story-two")], + &[], + CrawlBudget::default(), + ); + + assert_eq!( + plan.section_segment, 0, + "a generic one-segment root is a section, not necessarily a locale" + ); + assert_eq!( + segments(&plan), + ["news"], + "articles below a section root should remain one section" + ); + } + + #[test] + fn dropped_section_note_is_capped() { + let links: Vec<_> = (0..15) + .map(|index| nav(&format!("/section-{index:02}"))) + .collect(); + let plan = plan_crawl( + &root(), + &links, + &[], + CrawlBudget { + max_sections: 0, + max_pages: 1, + }, + ); + let note = plan + .notes + .iter() + .find(|note| note.contains("budget reached")) + .expect("should report dropped sections"); + + assert!(note.contains("and 5 more")); + assert!(!note.contains("section-14")); + } +} diff --git a/crates/trusted-server-cli/src/commands/audit/generate/evidence.rs b/crates/trusted-server-cli/src/commands/audit/generate/evidence.rs new file mode 100644 index 000000000..5c50e34e5 --- /dev/null +++ b/crates/trusted-server-cli/src/commands/audit/generate/evidence.rs @@ -0,0 +1,794 @@ +//! Cross-page slot evidence: what each slot looked like on every page it was +//! observed on. +//! +//! A single page cannot distinguish a literal ad-unit path from a templated one, +//! so inference needs the *set* of observations per slot rather than one +//! snapshot. This module accumulates that set and is deliberately the only place +//! that reconciles a slot seen more than once: +//! +//! - **Formats union.** A size that appears only on article pages (a 300x600 +//! rail, say) must survive alongside the homepage's sizes. Taking the first +//! page's formats would silently narrow the slot. +//! - **Unit paths are kept, not collapsed.** Divergence across pages is the +//! signal inference reads; discarding it is what makes templating impossible. +//! - **Network ids must agree.** Two different GAM networks in one crawl means +//! the pages are not one property, and writing either one would be a guess. +//! +//! Slots are keyed on the *normalized div stem* produced by +//! [`discover_gpt_slots`](super::gpt_slots::discover_gpt_slots), because raw GPT +//! div ids carry per-render framework hashes and would otherwise look like a new +//! slot on every page. + +use std::collections::{BTreeMap, BTreeSet}; + +use super::gpt_slots::DiscoveredSlots; +use crate::error::{CliResult, cli_error}; + +/// One observation of a slot on one page. +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] +pub(super) struct EvidenceRow { + /// The page path the slot was observed on, normalized (leading `/`, no + /// query or fragment). + pub(super) path: String, + /// The literal GAM ad-unit path the live page used for this slot. + pub(super) unit_path: String, +} + +/// Everything observed about one slot across the crawl. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(super) struct SlotEvidence { + /// Config slot id derived from the div stem. + pub(super) id: String, + /// Normalized div stem, used as the runtime `div_id` prefix. + pub(super) div_id: String, + /// Union of every pixel size observed for this slot, smallest first. + pub(super) formats: BTreeSet<(u32, u32)>, + /// Whether any page carrying this slot showed header-bidding signals. + pub(super) has_prebid: bool, + /// Distinct `(path, unit_path)` observations, in a stable order. + pub(super) rows: BTreeSet, +} + +impl SlotEvidence { + /// The distinct literal unit paths observed for this slot. + pub(super) fn unit_paths(&self) -> BTreeSet<&str> { + self.rows.iter().map(|row| row.unit_path.as_str()).collect() + } + + /// The distinct page paths this slot was observed on. + pub(super) fn paths(&self) -> BTreeSet<&str> { + self.rows.iter().map(|row| row.path.as_str()).collect() + } +} + +/// Slots grouped by the shape that would make them one placement: an identical +/// ad-unit path and an identical format set. +type SlotsByShape<'a> = BTreeMap<(String, Vec<(u32, u32)>), Vec<&'a SlotEvidence>>; + +/// Several observed slots that are really one placement under volatile div ids. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(super) struct FragmentGroup { + /// The volatile div ids observed, in evidence order. + pub(super) div_ids: Vec, + /// The ad-unit path every fragment shared. + pub(super) unit_path: String, + /// The stable prefix the ids share, when they share a useful one. + /// + /// Offered to the operator as a starting point only. It is deliberately not + /// written as a `div_id`: the shared prefix reaches only as far as the + /// *observed* tokens happen to agree, so it would keep matching this crawl's + /// ids and stop matching the next render's. + pub(super) suggested_prefix: Option, +} + +/// Whether no two slots were ever seen on the same page. +fn pages_are_disjoint(slots: &[&SlotEvidence]) -> bool { + for (index, slot) in slots.iter().enumerate() { + let pages = slot.paths(); + if slots[index + 1..] + .iter() + .any(|other| other.paths().intersection(&pages).next().is_some()) + { + return false; + } + } + true +} + +/// The longest prefix the div ids share, trimmed back to a separator. +/// +/// Trimming matters: the raw common prefix usually ends mid-token (the leading +/// digits of a timestamp two fragments happen to share), which is worse than +/// useless as a suggestion. Cutting at the last `-` or `_` yields the part a +/// human would recognise as the placement's name. +fn shared_div_prefix(slots: &[&SlotEvidence]) -> Option { + let mut prefix: &str = slots.first()?.div_id.as_str(); + for slot in &slots[1..] { + let mut shared_end = 0; + for ((byte_index, left), right) in prefix.char_indices().zip(slot.div_id.chars()) { + if left != right { + break; + } + shared_end = byte_index + left.len_utf8(); + } + prefix = &prefix[..shared_end]; + } + let trimmed = prefix.trim_end_matches(|ch: char| ch != '-' && ch != '_'); + let candidate = trimmed.trim_end_matches(['-', '_']); + (!candidate.is_empty()).then(|| candidate.to_string()) +} + +/// Slot evidence accumulated across every collected page. +#[derive(Debug, Clone, Default)] +pub(super) struct EvidenceTable { + slots: BTreeMap, + /// Div stems in first-seen order, so generated config keeps crawl order + /// rather than alphabetical order. + order: Vec, + network_ids: BTreeSet, + /// Every page path folded in, including those that yielded no slots. + pages: BTreeSet, + /// Page paths that produced no slot evidence at all. + empty_pages: BTreeSet, + /// Page paths that produced slot evidence on at least one selected profile. + non_empty_pages: BTreeSet, + /// Div stems any page refused as ambiguous, unioned across the crawl. + /// + /// The verdict has to outlive the page that reached it. Article pages carry + /// several in-content units and refuse the shared prefix; a landing page + /// carries one and would otherwise contribute it as a usable slot, so the + /// written config would depend on which pages the crawl happened to sample. + ambiguous_stems: BTreeSet, + /// Normalized div IDs refused from generation but observed live. + refused_div_ids: BTreeSet, +} + +impl EvidenceTable { + /// Folds one page's discovered slots into the table. + /// + /// `path` is the page's normalized request path; it is what page patterns + /// and `{section}` derivation are computed from later, so it must be the + /// post-redirect path actually audited. + pub(super) fn fold_page(&mut self, path: &str, discovered: &DiscoveredSlots) { + self.pages.insert(path.to_string()); + if let Some(network_id) = &discovered.gam_network_id { + self.network_ids.insert(network_id.clone()); + } + if !discovered.had_slot_evidence { + if !self.non_empty_pages.contains(path) { + self.empty_pages.insert(path.to_string()); + } + return; + } + self.non_empty_pages.insert(path.to_string()); + self.empty_pages.remove(path); + self.ambiguous_stems + .extend(discovered.ambiguous_stems.iter().cloned()); + self.refused_div_ids + .extend(discovered.refused_div_ids.iter().cloned()); + + for slot in &discovered.slots { + let entry = self.slots.entry(slot.div_id.clone()).or_insert_with(|| { + self.order.push(slot.div_id.clone()); + SlotEvidence { + id: slot.id.clone(), + div_id: slot.div_id.clone(), + formats: BTreeSet::new(), + has_prebid: false, + rows: BTreeSet::new(), + } + }); + // Union rather than replace: a size seen only on one page type is + // still a size this slot serves. + entry.formats.extend(slot.formats.iter().copied()); + entry.has_prebid |= slot.has_prebid; + entry.rows.insert(EvidenceRow { + path: path.to_string(), + unit_path: slot.gam_unit_path.clone(), + }); + } + } + + /// Slots in first-seen order, excluding stems any page refused as ambiguous. + pub(super) fn slots(&self) -> impl Iterator { + self.order + .iter() + .filter(|div_id| !self.ambiguous_stems.contains(*div_id)) + .filter_map(|div_id| self.slots.get(div_id)) + } + + /// Every normalized div ID observed, including all refused evidence. + pub(super) fn observed_div_ids(&self) -> impl Iterator { + self.order + .iter() + .map(String::as_str) + .chain(self.ambiguous_stems.iter().map(String::as_str)) + .chain(self.refused_div_ids.iter().map(String::as_str)) + } + + /// Normalized div IDs observed as concrete live elements. + /// + /// Unlike [`EvidenceTable::observed_div_ids`], this excludes identifiers + /// that exist only as refused ambiguity or volatility evidence. Prefix + /// routing must not treat those inferred stems as literal DOM elements. + pub(super) fn observed_literals(&self) -> impl Iterator { + self.order + .iter() + .filter(|div_id| !self.ambiguous_stems.contains(*div_id)) + .filter(|div_id| !self.refused_div_ids.contains(*div_id)) + .map(String::as_str) + } + + /// Number of usable distinct slots observed. + pub(super) fn slot_count(&self) -> usize { + self.slots().count() + } + + /// Every page path folded in, whether or not it yielded slots. + pub(super) fn pages(&self) -> &BTreeSet { + &self.pages + } + + /// Page paths that produced no slot evidence. + /// + /// A high proportion of these is the signature of a bot challenge serving + /// interstitials instead of the real site, which is worth refusing to write + /// from rather than persisting a half-empty config. + pub(super) fn empty_pages(&self) -> &BTreeSet { + &self.empty_pages + } + + /// Whether any slot was observed at all, ambiguous ones included. + /// + /// Deliberately not `slot_count() == 0`: a crawl that saw only ambiguous + /// placements did observe an ad stack, and the caller distinguishes "this + /// page has no slots" from "every slot found was refused". + pub(super) fn is_empty(&self) -> bool { + self.slots.is_empty() + } + + /// Groups of slots that are one slot wearing a different div id per page. + /// + /// Some ad stacks build div ids from a per-render token — a timestamp, a + /// framework id — so the same placement arrives under a new key on every + /// page. Written verbatim those ids never match at runtime, and the + /// fragmentation also starves template inference, which needs to see one + /// slot more than once. + /// + /// Detection is by evidence rather than by guessing at token shapes, because + /// each stack invents its own. Candidates share an identical ad-unit path and + /// identical formats; what separates a fragmented slot from two legitimate + /// siblings on the same unit is **co-occurrence**. Real siblings appear + /// together on a page; fragments of one slot never do, because each page + /// produces exactly one of them. + pub(super) fn fragmented_slots(&self) -> Vec { + let mut by_shape: SlotsByShape<'_> = BTreeMap::new(); + for slot in self.slots() { + // Only slots pinned to exactly one unit path can be compared this + // way; a slot whose unit varies is inference's problem, not this one. + let units = slot.unit_paths(); + if units.len() != 1 { + continue; + } + let unit = (*units.iter().next().expect("should have one unit path")).to_string(); + let formats: Vec<(u32, u32)> = slot.formats.iter().copied().collect(); + by_shape.entry((unit, formats)).or_default().push(slot); + } + + by_shape + .into_iter() + .filter(|(_, slots)| slots.len() > 1) + .filter(|(_, slots)| pages_are_disjoint(slots)) + .filter_map(|((unit_path, _), slots)| { + let suggested_prefix = shared_div_prefix(&slots); + (suggested_prefix.is_some() || slots.len() >= 3).then(|| FragmentGroup { + div_ids: slots.iter().map(|slot| slot.div_id.clone()).collect(), + unit_path, + suggested_prefix, + }) + }) + .collect() + } + + /// The single GAM network id observed across the crawl. + /// + /// # Errors + /// + /// Returns an error when pages disagreed. Two networks in one crawl means + /// the pages are not one property (a syndicated subdomain, a child network, + /// an off-origin redirect that slipped through), and picking either would be + /// a guess that silently bids against the wrong inventory. + pub(super) fn network_id(&self) -> CliResult> { + let mut found = self.network_ids.iter(); + let Some(first) = found.next() else { + return Ok(None); + }; + if self.network_ids.len() > 1 { + let all: Vec<&str> = self.network_ids.iter().map(String::as_str).collect(); + return cli_error(format!( + "the crawled pages reported more than one GAM network id ({}); \ + they do not appear to be one property, so no network id can be \ + chosen safely. Audit a single property, or pass explicit URLs", + all.join(", ") + )); + } + Ok(Some(first.clone())) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::commands::audit::generate::collector::CollectedGptSlot; + use crate::commands::audit::generate::gpt_slots::discover_gpt_slots; + + /// One live slot as `(unit path, div id, sizes)`. + type SlotFixture<'a> = (&'a str, &'a str, &'a [(u32, u32)]); + + fn page(slots: &[SlotFixture<'_>], has_prebid: bool) -> DiscoveredSlots { + let registry: Vec = slots + .iter() + .map(|(unit_path, div_id, sizes)| CollectedGptSlot { + gam_unit_path: (*unit_path).to_string(), + div_id: (*div_id).to_string(), + sizes: sizes.to_vec(), + }) + .collect(); + discover_gpt_slots(®istry, &[], has_prebid) + } + + #[test] + fn formats_union_across_pages_instead_of_first_seen_winning() { + // The 300x600 rail only ever renders on article pages. Keeping the + // homepage's format list alone would silently narrow the slot. + let mut table = EvidenceTable::default(); + table.fold_page( + "/", + &page(&[("/123/site/home", "ad-rail", &[(300, 250)])], false), + ); + table.fold_page( + "/news/story", + &page(&[("/123/site/news", "ad-rail", &[(300, 600)])], false), + ); + + let slot = table.slots().next().expect("should have one slot"); + assert_eq!( + slot.formats.iter().copied().collect::>(), + [(300, 250), (300, 600)], + "both pages' sizes should survive" + ); + assert_eq!(table.slot_count(), 1, "one div stem is one slot"); + } + + #[test] + fn divergent_unit_paths_are_preserved_as_separate_rows() { + // This divergence is the entire signal template inference reads. + let mut table = EvidenceTable::default(); + table.fold_page( + "/", + &page(&[("/123/site/home", "ad-header", &[(728, 90)])], false), + ); + table.fold_page( + "/news/story", + &page(&[("/123/site/news", "ad-header", &[(728, 90)])], false), + ); + + let slot = table.slots().next().expect("should have one slot"); + assert_eq!( + slot.unit_paths().into_iter().collect::>(), + ["/123/site/home", "/123/site/news"], + "both observed unit paths must be retained" + ); + assert_eq!( + slot.paths().into_iter().collect::>(), + ["/", "/news/story"] + ); + } + + #[test] + fn repeated_identical_observations_collapse() { + let mut table = EvidenceTable::default(); + let observed = page(&[("/123/site/home", "ad-header", &[(728, 90)])], false); + table.fold_page("/", &observed); + table.fold_page("/", &observed); + + let slot = table.slots().next().expect("should have one slot"); + assert_eq!(slot.rows.len(), 1, "the same page twice is one observation"); + } + + #[test] + fn prebid_is_sticky_once_any_page_shows_it() { + let mut table = EvidenceTable::default(); + table.fold_page( + "/", + &page(&[("/123/site/home", "ad-header", &[(728, 90)])], false), + ); + table.fold_page( + "/news/story", + &page(&[("/123/site/news", "ad-header", &[(728, 90)])], true), + ); + + let slot = table.slots().next().expect("should have one slot"); + assert!( + slot.has_prebid, + "a slot proven to run prebid on any page runs prebid" + ); + } + + #[test] + fn slots_keep_first_seen_order_not_alphabetical_order() { + let mut table = EvidenceTable::default(); + table.fold_page( + "/", + &page( + &[ + ("/123/site/home", "zeta-slot", &[(728, 90)]), + ("/123/site/home", "alpha-slot", &[(300, 250)]), + ], + false, + ), + ); + + let ids: Vec<&str> = table.slots().map(|slot| slot.div_id.as_str()).collect(); + assert_eq!( + ids, + ["zeta-slot", "alpha-slot"], + "generated config should follow crawl order" + ); + } + + #[test] + fn refused_only_div_ids_are_observed_but_not_literals() { + let mut discovered = page(&[("/123/site/home", "ad-x-stable", &[(300, 250)])], false); + discovered.refused_div_ids.insert("ad-x".to_string()); + let mut table = EvidenceTable::default(); + table.fold_page("/", &discovered); + + assert_eq!( + table.observed_div_ids().collect::>(), + ["ad-x-stable", "ad-x"], + "the staleness view should retain refused evidence" + ); + assert_eq!( + table.observed_literals().collect::>(), + ["ad-x-stable"], + "prefix routing should use only concrete live-element evidence" + ); + } + + #[test] + fn later_refusal_removes_a_previously_accepted_literal() { + let mut table = EvidenceTable::default(); + table.fold_page( + "/", + &page(&[("/123/site/home", "ad-x", &[(300, 250)])], false), + ); + let mut refused = DiscoveredSlots { + had_slot_evidence: true, + ..DiscoveredSlots::default() + }; + refused.refused_div_ids.insert("ad-x".to_string()); + table.fold_page("/news", &refused); + + assert!( + table.observed_literals().next().is_none(), + "a site-wide refusal should remove an earlier literal-routing candidate" + ); + assert_eq!( + table.observed_div_ids().collect::>(), + BTreeSet::from(["ad-x"]), + "the refused stem should remain available to staleness accounting" + ); + } + + #[test] + fn one_placement_under_per_render_div_ids_is_detected() { + // Each page yields a new key for the same placement: same unit, same + // formats, never co-occurring. The tokens here deliberately do *not* + // match the digit-led shape `discover_gpt_slots` refuses on sight, so + // this exercises the evidence-based detector that catches the stacks + // whose token shape cannot be recognized from one observation. + let mut table = EvidenceTable::default(); + for (path, div) in [ + ("/features/a", "ex_slot_ce6Bj0uc8sL0aa_overlay_1"), + ("/news/b", "ex_slot_aoYmv4RQyN3nbb_overlay_1"), + ("/deals/c", "ex_slot_mYPDB3tz8cpBcc_overlay_1"), + ] { + table.fold_page( + path, + &page(&[("/99/site_Overlay", div, &[(300, 250)])], false), + ); + } + + let groups = table.fragmented_slots(); + + assert_eq!(groups.len(), 1, "the three fragments should form one group"); + assert_eq!(groups[0].div_ids.len(), 3); + assert_eq!(groups[0].unit_path, "/99/site_Overlay"); + assert_eq!( + groups[0].suggested_prefix.as_deref(), + Some("ex_slot"), + "the suggestion should be trimmed back off the volatile token" + ); + } + + #[test] + fn an_ambiguous_stem_stays_refused_on_every_page() { + // The article page carries two in-content units and refuses the shared + // prefix; the landing page carries one. Folding the landing page must + // not resurrect a prefix that cannot resolve to one element site-wide. + let mut table = EvidenceTable::default(); + table.fold_page( + "/news/story", + &page( + &[ + ( + "/123/site/news", + "ad-in_content-de669245b2ea4b05826dc96f07a36272-in_content-0", + &[(300, 250)], + ), + ( + "/123/site/news", + "ad-in_content-8aec8129a83d4e5abc197423120cb19e-in_content-1", + &[(300, 250)], + ), + ], + false, + ), + ); + table.fold_page( + "/", + &page( + &[( + "/123/site/home", + "ad-in_content-1c0de08e5a2f4d6f9b3a7e5c8d1f2a4b-in_content-0", + &[(300, 250)], + )], + false, + ), + ); + + assert_eq!( + table.slots().count(), + 0, + "a stem refused on one page must stay refused, got {:?}", + table.slots().map(|slot| &slot.div_id).collect::>() + ); + assert_eq!( + table.slot_count(), + 0, + "the count should match what is written" + ); + assert!( + !table.is_empty(), + "the crawl did observe an ad stack, so this is not an empty result" + ); + assert!( + table.observed_literals().next().is_none(), + "a globally ambiguous stem must not remain a literal-routing candidate" + ); + } + + #[test] + fn genuine_siblings_on_one_unit_are_not_treated_as_fragments() { + // Two real in-content positions can share a unit path and formats. What + // distinguishes them from fragments is that they appear *together* on a + // page, so refusing to write them would lose real inventory. + let mut table = EvidenceTable::default(); + table.fold_page( + "/news/story", + &page( + &[ + ("/99/site/news", "ad-in_content-1", &[(300, 250)]), + ("/99/site/news", "ad-in_content-2", &[(300, 250)]), + ], + false, + ), + ); + + assert!( + table.fragmented_slots().is_empty(), + "co-occurring slots are siblings, not fragments" + ); + } + + #[test] + fn slots_differing_in_formats_are_not_fragments() { + let mut table = EvidenceTable::default(); + table.fold_page( + "/a", + &page(&[("/99/site/x", "slot-aaaa", &[(300, 250)])], false), + ); + table.fold_page( + "/b", + &page(&[("/99/site/x", "slot-bbbb", &[(728, 90)])], false), + ); + + assert!( + table.fragmented_slots().is_empty(), + "a differing format set means these are different placements" + ); + } + + #[test] + fn a_slot_seen_alone_is_never_a_fragment() { + let mut table = EvidenceTable::default(); + table.fold_page( + "/a", + &page(&[("/99/site/x", "only-slot", &[(300, 250)])], false), + ); + + assert!(table.fragmented_slots().is_empty()); + } + + #[test] + fn fragments_with_no_shared_prefix_report_none() { + let mut table = EvidenceTable::default(); + table.fold_page( + "/a", + &page(&[("/99/site/x", "alpha-1111", &[(300, 250)])], false), + ); + table.fold_page( + "/b", + &page(&[("/99/site/x", "beta-2222", &[(300, 250)])], false), + ); + + let groups = table.fragmented_slots(); + + assert!( + groups.is_empty(), + "two unrelated placements are too ambiguous to classify as fragments" + ); + } + + #[test] + fn three_disjoint_same_shape_ids_are_fragment_evidence_without_a_prefix() { + let mut table = EvidenceTable::default(); + for (path, div_id) in [("/a", "alpha"), ("/b", "bravo"), ("/c", "charlie")] { + table.fold_page(path, &page(&[("/99/site/x", div_id, &[(300, 250)])], false)); + } + + let groups = table.fragmented_slots(); + assert_eq!(groups.len(), 1); + assert_eq!(groups[0].suggested_prefix, None); + } + + #[test] + fn unicode_shared_prefix_uses_a_utf8_boundary() { + let mut table = EvidenceTable::default(); + table.fold_page( + "/a", + &page(&[("/99/site/x", "ünicode-ad-a", &[(300, 250)])], false), + ); + table.fold_page( + "/b", + &page(&[("/99/site/x", "ünicode-ad-b", &[(300, 250)])], false), + ); + + let groups = table.fragmented_slots(); + assert_eq!(groups.len(), 1); + assert_eq!(groups[0].suggested_prefix.as_deref(), Some("ünicode-ad")); + } + + #[test] + fn a_later_non_empty_profile_clears_the_empty_page_marker() { + let mut table = EvidenceTable::default(); + table.fold_page("/news", &page(&[], false)); + table.fold_page( + "/news", + &page(&[("/99/site/news", "ad-atf", &[(300, 250)])], false), + ); + + assert!(table.empty_pages().is_empty()); + } + + #[test] + fn a_later_empty_profile_does_not_re_mark_a_non_empty_page() { + let mut table = EvidenceTable::default(); + table.fold_page( + "/news", + &page(&[("/99/site/news", "ad-atf", &[(300, 250)])], false), + ); + table.fold_page("/news", &page(&[], false)); + + assert!( + table.empty_pages().is_empty(), + "emptiness is a page-level fact across all selected profiles" + ); + } + + #[test] + fn conflicting_network_ids_are_a_hard_error() { + let mut table = EvidenceTable::default(); + table.fold_page( + "/", + &page(&[("/111/site/home", "ad-header", &[(728, 90)])], false), + ); + table.fold_page( + "/news/story", + &page(&[("/222/site/news", "ad-header", &[(728, 90)])], false), + ); + + let error = table + .network_id() + .expect_err("two networks in one crawl should not resolve"); + + let rendered = format!("{error:?}"); + assert!( + rendered.contains("111") && rendered.contains("222"), + "the error should name both observed ids, got {rendered}" + ); + } + + #[test] + fn agreeing_network_ids_resolve_to_one_value() { + let mut table = EvidenceTable::default(); + table.fold_page( + "/", + &page(&[("/123/site/home", "ad-header", &[(728, 90)])], false), + ); + table.fold_page( + "/news/story", + &page(&[("/123/site/news", "ad-header", &[(728, 90)])], false), + ); + + assert_eq!( + table.network_id().expect("agreeing ids should resolve"), + Some("123".to_string()) + ); + } + + #[test] + fn pages_without_slots_are_recorded_for_challenge_detection() { + let mut table = EvidenceTable::default(); + table.fold_page( + "/", + &page(&[("/123/site/home", "ad-header", &[(728, 90)])], false), + ); + table.fold_page("/blocked", &page(&[], false)); + + assert_eq!( + table + .empty_pages() + .iter() + .map(String::as_str) + .collect::>(), + ["/blocked"], + "a slot-less page must be visible to the caller, not silently dropped" + ); + assert_eq!( + table.pages().len(), + 2, + "every folded page should be counted" + ); + } + + #[test] + fn collision_only_page_is_not_classified_as_empty() { + let discovered = page( + &[ + ("/123/site/home", "ad-x-aaaaaaaaaaaaaaaa-0", &[(300, 250)]), + ("/123/site/home", "ad-x-bbbbbbbbbbbbbbbb-1", &[(300, 250)]), + ], + false, + ); + let mut table = EvidenceTable::default(); + + table.fold_page("/collision-only", &discovered); + + assert!(discovered.had_slot_evidence); + assert!(discovered.slots.is_empty()); + assert!( + table.empty_pages().is_empty(), + "intentionally omitted GPT evidence must not look like a bot challenge" + ); + } + + #[test] + fn empty_table_resolves_no_network_id_rather_than_erroring() { + let table = EvidenceTable::default(); + + assert!(table.is_empty()); + assert_eq!(table.network_id().expect("empty is not a conflict"), None); + } +} diff --git a/crates/trusted-server-cli/src/commands/audit/generate/gpt_slots.rs b/crates/trusted-server-cli/src/commands/audit/generate/gpt_slots.rs new file mode 100644 index 000000000..24d17d3c2 --- /dev/null +++ b/crates/trusted-server-cli/src/commands/audit/generate/gpt_slots.rs @@ -0,0 +1,1546 @@ +//! Reconstructs `[creative_opportunities]` slots from a live page's GPT state. +//! +//! Two complementary sources feed the reconstruction: +//! +//! 1. The **live GPT registry** (`googletag.pubads().getSlots()`) is the primary +//! source. It exposes each defined slot's ad-unit path, div id, and sizes +//! directly, and is populated at `defineSlot` time — so it captures slots even +//! when the ad request never fires (consent-gated stacks, iframe-issued +//! requests). It carries no per-slot header-bidding signal, so Prebid is +//! inferred from page-level detection. +//! 2. Captured **`gampad/ads` requests** are a fallback for any div the registry +//! did not report. Each request URL encodes the ad-unit path (`iu_parts`), div +//! id (`dids`), sizes (`prev_iu_szs`), and targeting (`prev_scp`, which does +//! carry a per-slot Prebid signal). +//! +//! Neither source executes the page's ad-stack logic ourselves; both read state +//! the page's own GPT/Prebid setup produced. + +use std::collections::{BTreeMap, BTreeSet}; +use std::sync::LazyLock; + +use regex::Regex; +use trusted_server_core::creative_opportunities::validate_slot_id; +use url::Url; + +use crate::commands::audit::generate::collector::{CollectedGptSlot, CollectedRequest}; + +/// A hyphen-delimited hex hash *segment* (16+ hex chars bounded by `-` or end), +/// e.g. the UUID GPT embeds in `ad-in_content--in_content-0`. Marks the +/// start of ephemeral div-id noise, like the React `_R_` hash. The trailing +/// boundary avoids truncating a legit token that merely starts with hex-like +/// characters (only `start()` of the match is used). +static HEX_HASH_SEGMENT: LazyLock = + LazyLock::new(|| Regex::new(r"-[0-9a-f]{16,}(?:-|$)").expect("should compile hex hash regex")); +static UUID_SEGMENT: LazyLock = LazyLock::new(|| { + Regex::new(r"-[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}(?:-|$)") + .expect("should compile UUID regex") +}); + +/// Matches a React `useId` token, which changes on every render. +/// +/// React emits these in both cases — `_R_3f_` from a server render and `_r_0_` +/// from a client one — so matching only the uppercase form leaves the lowercase +/// variant in the stem. That is not merely untidy: the suffix differs per +/// render, so one logical slot fragments into a new key on every page, which +/// both breaks runtime div matching and starves template inference of the +/// repeated observations it needs. +/// +/// The uppercase form is distinctive enough to match bare, and its hash is +/// included so the match spans the whole ephemeral token — [`normalize_div_stem`] +/// only reads the match *start*, but [`ephemeral_marker_residue`] excises the +/// match, and a residue that still carried the hash would make two renders of one +/// element look like two elements. The lowercase form is anchored (`_r_`, a short +/// alphanumeric run, `_`) so an ordinary id that merely contains `_r_` keeps its +/// full stem. +static REACT_USE_ID: LazyLock = LazyLock::new(|| { + Regex::new(r"_R_[0-9a-z]*_?|_r_[0-9a-z]{1,8}_").expect("should compile react id regex") +}); + +/// Hosts that serve GPT `gampad/ads` requests. +const GAMPAD_HOSTS: &[&str] = &["securepubads.g.doubleclick.net", "pubads.g.doubleclick.net"]; + +/// Common GPT div-id prefix stripped when deriving a slot id. +const GPT_DIV_PREFIX: &str = "div-gpt-ad-"; + +/// Minimum width/height for a format to be treated as a real creative size. +/// +/// GPT encodes fluid/native aspect-ratio markers (e.g. `4x1`, `8x1`) alongside +/// pixel sizes in `prev_iu_szs`; those are not banner dimensions, so they are +/// dropped from the drafted `formats`. +const MIN_FORMAT_DIMENSION: u32 = 50; + +/// A slot reconstructed from a single GPT ad request. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct DiscoveredSlot { + /// Slot id derived from the div id (GPT prefix stripped). + pub(crate) id: String, + /// The HTML div id that holds the creative. + pub(crate) div_id: String, + /// The full GAM ad-unit path (e.g. `/123/desktop/homepage/leaderboard`). + pub(crate) gam_unit_path: String, + /// Candidate creative sizes as `(width, height)` pixel pairs. + pub(crate) formats: Vec<(u32, u32)>, + /// Whether the slot's targeting shows Prebid/header-bidding signals. + pub(crate) has_prebid: bool, +} + +/// The result of scanning captured requests for GPT slots. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub(crate) struct DiscoveredSlots { + /// GAM network id shared by the discovered slots, if any were found. + pub(crate) gam_network_id: Option, + /// Whether the page exposed any otherwise usable slot evidence, including + /// ambiguous placements that were intentionally omitted from `slots`. + pub(crate) had_slot_evidence: bool, + /// The reconstructed slots, deduplicated by div id in first-seen order. + pub(crate) slots: Vec, + /// Div stems refused because several live elements normalized onto them. + /// + /// Carried separately from `slots` because the verdict is a property of the + /// *site*, not of this page: another page that happens to render only one + /// member of the group must not resurrect the ambiguous prefix. + pub(crate) ambiguous_stems: BTreeSet, + /// Normalized div IDs refused from generation but still observed live. + pub(crate) refused_div_ids: BTreeSet, + /// Diagnostics for placements whose normalized stable stems collided. + pub(crate) warnings: Vec, +} + +/// Reconstructs GPT slots from the page's live registry and ad requests. +/// +/// The live registry (`googletag.pubads().getSlots()`) is the primary source: it +/// carries the authoritative path/div/size for every defined slot and is present +/// even when the ad request never fires. Captured `gampad/ads` requests are a +/// fallback for any div the registry did not report, and also supply per-slot +/// Prebid signals. Slots are deduplicated by div id in first-seen order. +/// +/// `page_has_prebid` marks registry slots as Prebid-enabled when the page as a +/// whole was detected running Prebid (the registry alone carries no such signal). +pub(crate) fn discover_gpt_slots( + registry: &[CollectedGptSlot], + requests: &[CollectedRequest], + page_has_prebid: bool, +) -> DiscoveredSlots { + let mut slots = Vec::new(); + let mut warnings = Vec::new(); + let mut ambiguous_stems = BTreeSet::new(); + let mut refused_div_ids = BTreeSet::new(); + let mut gam_network_id = None; + let mut had_slot_evidence = false; + let mut registry_residues: BTreeMap> = BTreeMap::new(); + // Stems refused outright, so the request fallback cannot re-add them. Kept + // apart from `registry_residues` so a later registry entry cannot read a + // refused stem as a one-member collision group. + let mut refused_stems: BTreeSet = BTreeSet::new(); + + for entry in registry { + let Some(slot) = slot_from_registry(entry, page_has_prebid) else { + continue; + }; + had_slot_evidence = true; + if gam_network_id.is_none() { + gam_network_id = network_id_from_unit_path(&entry.gam_unit_path); + } + if let Some(prefix) = volatile_prefix_before_placement(&entry.div_id) { + refused_stems.insert(slot.div_id.clone()); + refused_div_ids.insert(slot.div_id); + push_unique_warning(&mut warnings, volatile_prefix_warning(&prefix)); + continue; + } + if let Some(prefix) = + push_slot_refusing_collisions(&mut slots, &mut registry_residues, slot, &entry.div_id) + { + warnings.push(ambiguous_collision_warning(&prefix)); + ambiguous_stems.insert(prefix); + } + } + + let registry_stems: BTreeSet = registry_residues + .keys() + .cloned() + .chain(refused_stems) + .collect(); + let mut request_residues: BTreeMap> = BTreeMap::new(); + for request in requests { + let Some((network_id, slot, raw_div)) = parse_gampad_request(&request.url) else { + continue; + }; + had_slot_evidence = true; + if gam_network_id.is_none() { + gam_network_id = Some(network_id); + } + if registry_stems.contains(&slot.div_id) { + continue; + } + if let Some(prefix) = volatile_prefix_before_placement(&raw_div) { + refused_div_ids.insert(slot.div_id); + push_unique_warning(&mut warnings, volatile_prefix_warning(&prefix)); + continue; + } + if let Some(prefix) = + push_slot_refusing_collisions(&mut slots, &mut request_residues, slot, &raw_div) + { + warnings.push(ambiguous_collision_warning(&prefix)); + ambiguous_stems.insert(prefix); + } + } + make_slot_ids_unique(&mut slots); + + DiscoveredSlots { + gam_network_id, + had_slot_evidence, + slots, + ambiguous_stems, + refused_div_ids, + warnings, + } +} + +/// Adds one source-local slot unless two distinct *elements* share its stem. +/// +/// Sharing a stem is not by itself ambiguity: one element re-rendered under a +/// fresh framework token is exactly what normalization exists to absorb, and it +/// produces two raw ids that collapse onto one stem. Ambiguity is two elements, +/// which [`ephemeral_marker_residue`] separates from two renders of one. +/// +/// The first distinct residue removes the tentatively accepted slot and returns +/// its stem for one diagnostic. Repeats and later collision members stay +/// suppressed and return `None`. +fn push_slot_refusing_collisions( + slots: &mut Vec, + seen_residues: &mut BTreeMap>, + slot: DiscoveredSlot, + raw_div: &str, +) -> Option { + let normalized = slot.div_id.clone(); + let residue = ephemeral_marker_residue(raw_div); + match seen_residues.get_mut(&normalized) { + None => { + seen_residues.insert(normalized, BTreeSet::from([residue])); + slots.push(slot); + None + } + Some(residues) if residues.contains(&residue) => None, + Some(residues) => { + let became_ambiguous = residues.len() == 1; + residues.insert(residue); + if became_ambiguous { + slots.retain(|entry| entry.div_id != normalized); + Some(normalized) + } else { + None + } + } + } +} + +/// Operator-facing text for a stem several live elements normalized onto. +fn ambiguous_collision_warning(prefix: &str) -> String { + format!( + "skipped ambiguous div-id prefix `{prefix}`: multiple active elements normalized to it, \ + but the runtime can resolve a prefix to only one active element and exact div ids change \ + across renders; expose distinct stable div ids in publisher markup before configuring \ + these placements" + ) +} + +/// The stable prefix of a div id whose per-render token precedes more of the id. +/// +/// Some ad stacks build ids as `__` — a +/// millisecond timestamp plus a random suffix sitting *before* the part that +/// distinguishes one placement from the next. Such an id can be written neither +/// literally (the token changes on the next render) nor as a prefix: the only +/// stable prefix stops at the token, and that prefix reaches every placement in +/// the family, while the runtime resolves a prefix to a single element. So the +/// slot is refused from a single observation, without waiting for a second +/// placement to prove the collision. +/// +/// The shape decides, not the vendor: any segment that is a long digit run +/// followed by more alphanumerics counts, so a new stack with the same layout +/// needs no code change. A token in *trailing* position is deliberately not this +/// case — everything before it still identifies the element — and is left to +/// normalization and the same-page collision check. +fn volatile_prefix_before_placement(div_id: &str) -> Option { + let div_id = div_id.strip_suffix("-container").unwrap_or(div_id); + let mut start = 0_usize; + for (index, character) in div_id.char_indices() { + if character != '_' && character != '-' { + continue; + } + if is_per_render_token(&div_id[start..index]) { + let prefix = div_id[..start].trim_end_matches(['_', '-']); + // A delimiter is one byte, so the remainder starts just past it. + return (!prefix.is_empty() && !div_id[index + 1..].is_empty()) + .then(|| prefix.to_string()); + } + start = index + character.len_utf8(); + } + None +} + +/// Whether one div-id segment is a per-render token: a long leading digit run +/// followed by alphanumerics, or a shorter counter paired with a long random +/// suffix. +/// +/// Both halves are required. Eight-digit values need at least eight suffix +/// characters with a random-looking shape; this avoids treating calendar labels +/// followed by stable words as generated ids while still catching single-case +/// hashes and mixed alphanumeric tokens. A bare digit run is how publishers +/// write stable placement indices, and a token with a non-alphanumeric character +/// is some other structure than a generated id. +fn is_per_render_token(segment: &str) -> bool { + let leading_digits = segment.bytes().take_while(u8::is_ascii_digit).count(); + let suffix_length = segment.len().saturating_sub(leading_digits); + let suffix = &segment[leading_digits..]; + ((leading_digits >= 10 && suffix_length >= 1) + || (leading_digits >= 8 && suffix_length >= 8 && looks_random_suffix(suffix))) + && segment.bytes().all(|byte| byte.is_ascii_alphanumeric()) +} + +/// Whether a long suffix has structural signals of generated randomness. +fn looks_random_suffix(value: &str) -> bool { + let digit_count = value.bytes().filter(u8::is_ascii_digit).count(); + let letter_count = value.bytes().filter(u8::is_ascii_alphabetic).count(); + if digit_count >= 4 && letter_count >= 4 { + return true; + } + + let distinct = distinct_ascii_bytes(value); + if value.bytes().all(|byte| byte.is_ascii_hexdigit()) && distinct >= 4 { + return true; + } + if value.bytes().all(|byte| byte.is_ascii_uppercase()) && distinct >= 4 { + return true; + } + + has_random_case_alternation(value) && !has_wordlike_camel_segments(value) +} + +/// Number of distinct ASCII bytes in a candidate token. +fn distinct_ascii_bytes(value: &str) -> usize { + let mut seen = [false; 256]; + for byte in value.bytes() { + seen[usize::from(byte)] = true; + } + seen.into_iter().filter(|present| *present).count() +} + +/// Whether every CamelCase component contains a vowel-like letter. +/// +/// This distinguishes short word sequences such as `TopUsNewsAd` and +/// `MyAdUnitXy` from dense random alternation such as `AbCdEfGh`. +fn has_wordlike_camel_segments(value: &str) -> bool { + let mut segment_has_vowel = false; + for (index, byte) in value.bytes().enumerate() { + if index > 0 && byte.is_ascii_uppercase() { + if !segment_has_vowel { + return false; + } + segment_has_vowel = is_ascii_vowel(byte); + } else { + segment_has_vowel |= is_ascii_vowel(byte); + } + } + segment_has_vowel +} + +/// Whether an ASCII letter is a vowel, treating `y` as vowel-like for labels. +const fn is_ascii_vowel(byte: u8) -> bool { + matches!( + byte.to_ascii_lowercase(), + b'a' | b'e' | b'i' | b'o' | b'u' | b'y' + ) +} + +/// Whether letter case alternates densely enough to resemble a random token. +fn has_random_case_alternation(value: &str) -> bool { + let mut previous = None; + let mut comparisons = 0_usize; + let mut transitions = 0_usize; + for uppercase in value.bytes().filter_map(|byte| { + byte.is_ascii_lowercase() + .then_some(false) + .or_else(|| byte.is_ascii_uppercase().then_some(true)) + }) { + if let Some(previous) = previous { + comparisons += 1; + transitions += usize::from(previous != uppercase); + } + previous = Some(uppercase); + } + transitions >= 3 && transitions.saturating_mul(3) >= comparisons.saturating_mul(2) +} + +/// Operator-facing text for a div-id family carrying a per-render token. +fn volatile_prefix_warning(prefix: &str) -> String { + format!( + "skipped volatile div-id family `{prefix}`: a per-render token sits before the placement \ + suffix, so exact div ids change across renders and no distinct stable element prefix is \ + available; expose distinct stable div ids in publisher markup before configuring these \ + placements" + ) +} + +/// Records `warning` unless the same text was already recorded for this page. +fn push_unique_warning(warnings: &mut Vec, warning: String) { + if !warnings.contains(&warning) { + warnings.push(warning); + } +} + +/// Converts a live-registry slot into a [`DiscoveredSlot`]. +/// +/// Returns `None` when the slot has no usable pixel size or its div id is a +/// multi-slot (SRA) concatenation rather than a single element. +fn slot_from_registry(entry: &CollectedGptSlot, page_has_prebid: bool) -> Option { + if is_multi_slot_div(&entry.div_id) { + return None; + } + if !is_usable_unit_path(&entry.gam_unit_path) { + return None; + } + let formats: Vec<(u32, u32)> = entry + .sizes + .iter() + .copied() + .filter(|(width, height)| *width >= MIN_FORMAT_DIMENSION && *height >= MIN_FORMAT_DIMENSION) + .collect(); + if formats.is_empty() { + return None; + } + let div_stem = normalize_div_stem(&entry.div_id); + // Normalization truncates at the first ephemeral marker, so a div id that is + // *entirely* ephemeral (`_R_9sl…`, or exactly `-container`) reduces to the + // empty string. An empty `div_id` override fails config load outright, and + // an empty prefix would bind the slot to the first id-bearing element on the + // page, so such a slot is unusable rather than merely imprecise. + if div_stem.is_empty() { + return None; + } + Some(DiscoveredSlot { + id: slot_id_from_div(&div_stem), + div_id: div_stem, + gam_unit_path: entry.gam_unit_path.clone(), + formats, + has_prebid: page_has_prebid, + }) +} + +/// Whether a div id is a GPT single-request (SRA) concatenation of multiple +/// slots (joined with `~`) rather than one element. +fn is_multi_slot_div(div_id: &str) -> bool { + div_id.contains('~') +} + +/// Whether a scraped GAM ad-unit path can be represented in config. +/// +/// `gam_unit_path` is a template: `{` and `}` delimit placeholders and +/// [`parse_unit_template`](trusted_server_core::creative_opportunities) offers no +/// escape syntax. A live path containing a brace would either fail config load +/// or, worse, be silently reinterpreted as a placeholder-bearing template. A +/// blank path is rejected for the same reason config load rejects it. +fn is_usable_unit_path(path: &str) -> bool { + !path.trim().is_empty() && !path.contains(['{', '}']) +} + +/// Strips ephemeral GPT div-id noise so the stored id is stable across renders. +/// +/// Removes a trailing `-container` wrapper, then truncates at the first ephemeral +/// marker — a React SSR hash (`_R_`) or a hex-UUID segment — since both +/// change on every page load. Truncating (rather than excising) keeps the result +/// a valid **prefix** of the live div id, which is how verify matches slots. +/// +/// `div-gpt-ad-leaderboard-1` (stable) is unchanged; `ad-header-0-_R_9sl…-container` +/// and `ad-header-0-_r_8_` → `ad-header-0`; `ad-in_content-de66…f272-in_content-0` +/// → `ad-in_content`. +fn normalize_div_stem(div_id: &str) -> String { + let stem = div_id.strip_suffix("-container").unwrap_or(div_id); + let cut = ephemeral_marker_ranges(stem) + .first() + .map_or(stem.len(), |range| range.start); + stem[..cut].trim_end_matches('-').to_string() +} + +/// Byte ranges of every ephemeral per-render marker in `stem`, in order and +/// without overlaps. +/// +/// A hex-hash candidate must contain at least one `a`-`f`; a run of 16+ digits +/// is how publishers write stable ids, not a hash. +fn ephemeral_marker_ranges(stem: &str) -> Vec> { + let mut ranges: Vec> = REACT_USE_ID + .find_iter(stem) + .chain(UUID_SEGMENT.find_iter(stem)) + .chain(HEX_HASH_SEGMENT.find_iter(stem).filter(|matched| { + matched + .as_str() + .bytes() + .any(|byte| matches!(byte, b'a'..=b'f')) + })) + .map(|matched| matched.range()) + .collect(); + ranges.sort_by_key(|range| range.start); + let mut merged: Vec> = Vec::with_capacity(ranges.len()); + for range in ranges { + match merged.last_mut() { + Some(last) if range.start < last.end => last.end = last.end.max(range.end), + _ => merged.push(range), + } + } + merged +} + +/// The parts of a raw div id that no ephemeral marker covered, NUL-joined. +/// +/// [`normalize_div_stem`] truncates at the first marker, so two ids differing +/// only *inside* a marker collapse onto one stem — the signature of one element +/// re-rendered. What the markers did not cover separates that from two elements: +/// `ad-header-0-_R_3f_` and `ad-header-0-_r_0_` leave the same residue (one +/// element, two renders), while `…-in_content-0` and `…-in_content-1` do not +/// (two siblings). A live div id cannot contain NUL, so joining on it cannot +/// make two different residues compare equal. +fn ephemeral_marker_residue(div_id: &str) -> String { + let stem = div_id.strip_suffix("-container").unwrap_or(div_id); + let mut residue = String::with_capacity(stem.len()); + let mut previous = 0_usize; + for range in ephemeral_marker_ranges(stem) { + residue.push_str(&stem[previous..range.start]); + residue.push('\0'); + previous = range.end; + } + residue.push_str(&stem[previous..]); + residue +} + +/// Extracts the leading network id from a GAM ad-unit path (`//...`). +fn network_id_from_unit_path(path: &str) -> Option { + let segment = path.trim_start_matches('/').split('/').next()?; + (!segment.is_empty() && segment.bytes().all(|byte| byte.is_ascii_digit())) + .then(|| segment.to_string()) +} + +/// Parses a single `gampad/ads` request URL into `(network_id, slot)`. +/// +/// Returns `None` when the URL is not a GPT ad request or is missing the fields +/// needed to describe a slot (ad-unit path, div id, and at least one size). +fn parse_gampad_request(raw_url: &str) -> Option<(String, DiscoveredSlot, String)> { + let url = Url::parse(raw_url).ok()?; + let host = url.host_str()?; + if !GAMPAD_HOSTS.contains(&host) || !url.path().ends_with("/gampad/ads") { + return None; + } + + let mut iu_parts = None; + let mut dids = None; + let mut sizes_raw = None; + let mut fallback_sizes_raw = None; + let mut scp = None; + for (key, value) in url.query_pairs() { + match key.as_ref() { + "iu_parts" => iu_parts = Some(value.into_owned()), + "dids" => dids = Some(value.into_owned()), + "prev_iu_szs" => sizes_raw = Some(value.into_owned()), + "pb_szs" => fallback_sizes_raw = Some(value.into_owned()), + "prev_scp" => scp = Some(value.into_owned()), + _ => {} + } + } + + let iu_parts = iu_parts?; + let mut parts = iu_parts.split(',').filter(|part| !part.is_empty()); + // Mirror the registry path's validation: a GAM network id is digits only. + // The percent-decoded query value is page-controlled and gets spliced into + // generated TOML, so reject anything else. + let network_id = parts + .next() + .filter(|segment| segment.bytes().all(|byte| byte.is_ascii_digit()))? + .to_string(); + let gam_unit_path = format!("/{}", iu_parts.replace(',', "/")); + if !is_usable_unit_path(&gam_unit_path) { + return None; + } + // A usable unit path needs the network id plus at least one path segment. + parts.next()?; + + let raw_div = dids?; + if raw_div.contains(',') { + return None; + } + let raw_div = raw_div.trim().to_string(); + if raw_div.is_empty() { + return None; + } + if is_multi_slot_div(&raw_div) { + return None; + } + let div_id = normalize_div_stem(&raw_div); + // See `slot_from_registry`: a fully ephemeral div id normalizes to nothing, + // which is neither a valid config value nor a usable runtime prefix. + if div_id.is_empty() { + return None; + } + + let formats = parse_sizes(sizes_raw.as_deref().or(fallback_sizes_raw.as_deref())?); + if formats.is_empty() { + return None; + } + + let id = slot_id_from_div(&div_id); + let has_prebid = scp.as_deref().is_some_and(scp_shows_prebid); + + Some(( + network_id, + DiscoveredSlot { + id, + div_id, + gam_unit_path, + formats, + has_prebid, + }, + raw_div, + )) +} + +/// Parses a GPT size list (e.g. `970x250|4x1|620x366`) into pixel pairs. +/// +/// Accepts `|` or `,` separators, ignores non-`WxH` tokens, and drops +/// fluid/native ratio markers below [`MIN_FORMAT_DIMENSION`]. +fn parse_sizes(raw: &str) -> Vec<(u32, u32)> { + let mut sizes = Vec::new(); + for token in raw.split(['|', ',']) { + let Some((width, height)) = token.trim().split_once('x') else { + continue; + }; + let (Ok(width), Ok(height)) = (width.parse::(), height.parse::()) else { + continue; + }; + if width < MIN_FORMAT_DIMENSION || height < MIN_FORMAT_DIMENSION { + continue; + } + if !sizes.contains(&(width, height)) { + sizes.push((width, height)); + } + } + sizes +} + +/// Derives a runtime-safe slot id from a div id. +/// +/// The common GPT prefix is stripped, invalid character runs become one +/// hyphen, and an all-invalid value falls back to `slot`. +fn slot_id_from_div(div_id: &str) -> String { + let candidate = div_id.strip_prefix(GPT_DIV_PREFIX).unwrap_or(div_id); + let mut id = String::with_capacity(candidate.len()); + let mut previous_was_hyphen = false; + for character in candidate.chars() { + if character.is_ascii_alphanumeric() || character == '_' { + id.push(character); + previous_was_hyphen = false; + } else if !id.is_empty() && !previous_was_hyphen { + id.push('-'); + previous_was_hyphen = true; + } + } + while id.ends_with('-') { + id.pop(); + } + if id.is_empty() { + id.push_str("slot"); + } + + if validate_slot_id(&id).is_ok() { + id + } else { + "slot".to_string() + } +} + +/// Adds deterministic numeric suffixes when sanitization produces duplicate ids. +fn make_slot_ids_unique(slots: &mut [DiscoveredSlot]) { + let mut used = BTreeSet::new(); + for slot in slots { + if used.insert(slot.id.clone()) { + continue; + } + + let base = slot.id.clone(); + let mut suffix = 2_usize; + loop { + let candidate = format!("{base}-{suffix}"); + if used.insert(candidate.clone()) { + slot.id = candidate; + break; + } + suffix += 1; + } + } +} + +/// Detects Prebid/header-bidding signals in a slot's `prev_scp` targeting. +fn scp_shows_prebid(scp: &str) -> bool { + url::form_urlencoded::parse(scp.as_bytes()).any(|(key, value)| { + let key = key.to_ascii_lowercase(); + let value = value.to_ascii_lowercase(); + (key == "test" && value == "prebid") + || (key == "tude" && value == "true") + || key.starts_with("prebid") + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// A sample GPT leaderboard ad request (truncated to the fields the + /// parser reads; values are otherwise unmodified live output). + const SAMPLE_LEADERBOARD: &str = "https://securepubads.g.doubleclick.net/gampad/ads?\ + gdfp_req=1&iu_parts=123456789%2Cdesktop%2Chomepage%2Cleaderboard1\ + &prev_iu_szs=970x250%7C4x1%7C8x1%7C620x366%7C325x508%7C325x204\ + &dids=div-gpt-ad-leaderboard-1\ + &prev_scp=ad-loc%3Dleaderboard-1%26baseDivId%3Ddiv-gpt-ad-leaderboard-1%26test%3Dprebid%26tude%3Dtrue\ + &pb_szs=970x250%7C620x366"; + const SHORT_VOLATILE_DIV: &str = "vendor-tag_12345678AbCdEfGhIjKl_slot_overlay_1"; + + fn request(url: &str) -> CollectedRequest { + CollectedRequest { + url: url.to_string(), + resource_type: Some("fetch".to_string()), + } + } + + /// Discovers slots from ad requests only (no live registry). + fn from_requests(requests: &[CollectedRequest]) -> DiscoveredSlots { + discover_gpt_slots(&[], requests, false) + } + + #[test] + fn parses_leaderboard_slot() { + let discovered = from_requests(&[request(SAMPLE_LEADERBOARD)]); + + assert_eq!(discovered.gam_network_id.as_deref(), Some("123456789")); + assert_eq!(discovered.slots.len(), 1, "should find one slot"); + let slot = &discovered.slots[0]; + assert_eq!(slot.id, "leaderboard-1", "should strip the GPT div prefix"); + assert_eq!(slot.div_id, "div-gpt-ad-leaderboard-1"); + assert_eq!( + slot.gam_unit_path, + "/123456789/desktop/homepage/leaderboard1" + ); + assert_eq!( + slot.formats, + vec![(970, 250), (620, 366), (325, 508), (325, 204)], + "should keep pixel sizes and drop 4x1/8x1 fluid markers" + ); + assert!(slot.has_prebid, "prev_scp test=prebid should flag prebid"); + } + + #[test] + fn prebid_detection_requires_a_targeting_key_not_a_substring() { + assert!(scp_shows_prebid("test=prebid")); + assert!(!scp_shows_prebid("noprebid=true")); + } + + #[test] + fn deduplicates_refreshed_slot_requests() { + // GPT refreshes the same slot; a second identical request must not + // produce a duplicate slot. + let discovered = from_requests(&[request(SAMPLE_LEADERBOARD), request(SAMPLE_LEADERBOARD)]); + + assert_eq!( + discovered.slots.len(), + 1, + "repeat requests for the same div should collapse" + ); + } + + #[test] + fn ignores_non_gampad_requests() { + let discovered = from_requests(&[ + request("https://securepubads.g.doubleclick.net/tag/js/gpt.js"), + request("https://cdn.example.com/app.js"), + request("https://analytics.example.com/collect?iu_parts=1%2Cfoo&dids=x"), + ]); + + assert!( + discovered.slots.is_empty(), + "only doubleclick gampad/ads requests should yield slots" + ); + assert_eq!(discovered.gam_network_id, None); + } + + #[test] + fn skips_requests_missing_sizes() { + let discovered = from_requests(&[request( + "https://securepubads.g.doubleclick.net/gampad/ads?iu_parts=123%2Cslot&dids=div-gpt-ad-x", + )]); + + assert!( + discovered.slots.is_empty(), + "a slot with no usable size should be skipped" + ); + } + + #[test] + fn skips_requests_with_only_network_id() { + // iu_parts with just the network id yields no unit path segment. + let discovered = from_requests(&[request( + "https://securepubads.g.doubleclick.net/gampad/ads?iu_parts=123&dids=div-gpt-ad-x&prev_iu_szs=300x250", + )]); + + assert!( + discovered.slots.is_empty(), + "a bare network id is not a usable ad-unit path" + ); + } + + #[test] + fn skips_requests_with_non_numeric_network_id() { + // A page-controlled iu_parts value must not smuggle a non-numeric + // network id (it gets spliced into generated TOML). + let discovered = from_requests(&[request( + "https://securepubads.g.doubleclick.net/gampad/ads?iu_parts=123%22evil%2Cslot&dids=div-gpt-ad-x&prev_iu_szs=300x250", + )]); + + assert!( + discovered.slots.is_empty(), + "a non-numeric network id should be rejected" + ); + assert_eq!(discovered.gam_network_id, None); + } + + #[test] + fn falls_back_to_pb_szs_when_prev_iu_szs_absent() { + let discovered = from_requests(&[request( + "https://securepubads.g.doubleclick.net/gampad/ads?iu_parts=123%2Cslot&dids=div-gpt-ad-x&pb_szs=300x250%7C728x90", + )]); + + assert_eq!(discovered.slots.len(), 1); + assert_eq!(discovered.slots[0].formats, vec![(300, 250), (728, 90)]); + } + + fn registry_slot(path: &str, div: &str, sizes: &[(u32, u32)]) -> CollectedGptSlot { + CollectedGptSlot { + gam_unit_path: path.to_string(), + div_id: div.to_string(), + sizes: sizes.to_vec(), + } + } + + #[test] + fn lowercase_react_use_id_suffixes_collapse_to_one_slot() { + // React emits `_r_0_` client-side and `_R_3f_` server-side, and the + // token changes per render. Leaving it in the stem fragments one slot + // into a new key on every page, which starves template inference. + for volatile in [ + "ad-header-0-_r_0_", + "ad-header-0-_r_8_", + "ad-header-0-_r_a_", + "ad-header-0-_R_3f_", + ] { + let registry = vec![registry_slot("/123/site/news", volatile, &[(728, 90)])]; + let discovered = discover_gpt_slots(®istry, &[], false); + assert_eq!( + discovered.slots[0].div_id, "ad-header-0", + "`{volatile}` should normalize to a stable stem" + ); + } + } + + #[test] + fn an_ordinary_id_containing_r_is_left_alone() { + // The React shape is anchored, so a legitimate id keeps its full stem. + let registry = vec![registry_slot("/123/site/news", "ad_r_rail", &[(300, 250)])]; + + let discovered = discover_gpt_slots(®istry, &[], false); + + assert_eq!(discovered.slots[0].div_id, "ad_r_rail"); + } + + #[test] + fn registry_slot_with_brace_in_unit_path_is_skipped() { + // `gam_unit_path` is a template and there is no escape syntax, so a + // literal brace either fails config load or is silently reinterpreted as + // a placeholder. Neither is acceptable to persist. + let registry = vec![ + registry_slot("/123/home/{section}", "div-gpt-ad-a", &[(300, 250)]), + registry_slot("/123/home/ok", "div-gpt-ad-b", &[(300, 250)]), + ]; + + let discovered = discover_gpt_slots(®istry, &[], false); + + assert_eq!( + discovered.slots.len(), + 1, + "the brace-bearing slot should be dropped, the clean one kept" + ); + assert_eq!(discovered.slots[0].gam_unit_path, "/123/home/ok"); + } + + #[test] + fn registry_slot_whose_div_id_is_entirely_ephemeral_is_skipped() { + // `_R_…` is a React SSR marker; normalizing truncates at it, leaving an + // empty stem. An empty div_id fails config load, and as a runtime prefix + // it would match the first id-bearing element on the page. + let registry = vec![registry_slot( + "/123/home/header", + "_R_9slkta7pd6", + &[(728, 90)], + )]; + + let discovered = discover_gpt_slots(®istry, &[], false); + + assert!( + discovered.slots.is_empty(), + "a slot with no stable div stem should be dropped, got {:?}", + discovered.slots + ); + } + + #[test] + fn volatile_guid_div_id_still_normalizes_to_a_usable_prefix() { + // A GUID between two copies of the placement name must still yield a + // usable stable stem; only an entirely ephemeral id is dropped. + let registry = vec![registry_slot( + "/123456789/publisher/homepage", + "ad-in_content-0949b6c5726343bf8bbec2ac47b494b4-in_content-0", + &[(300, 250)], + )]; + + let discovered = discover_gpt_slots(®istry, &[], false); + + assert_eq!(discovered.slots.len(), 1); + assert_eq!( + discovered.slots[0].div_id, "ad-in_content", + "the GUID and trailing index should be truncated to a stable prefix" + ); + } + + #[test] + fn reads_slots_from_live_registry() { + let registry = vec![registry_slot( + "/123456789/desktop/homepage/leaderboard1", + "div-gpt-ad-leaderboard-1", + &[(970, 250), (1, 1), (620, 366)], + )]; + + let discovered = discover_gpt_slots(®istry, &[], true); + + assert_eq!( + discovered.gam_network_id.as_deref(), + Some("123456789"), + "network id should come from the unit path" + ); + assert_eq!(discovered.slots.len(), 1); + let slot = &discovered.slots[0]; + assert_eq!(slot.id, "leaderboard-1"); + assert_eq!( + slot.formats, + vec![(970, 250), (620, 366)], + "should drop the 1x1 out-of-page marker" + ); + assert!( + slot.has_prebid, + "page-level prebid should mark registry slots" + ); + } + + #[test] + fn registry_wins_and_requests_fill_gaps() { + // The registry reports the leaderboard; a gampad request reports a + // different div that the registry missed. Both should appear once. + let registry = vec![registry_slot( + "/123456789/desktop/homepage/leaderboard1", + "div-gpt-ad-leaderboard-1", + &[(970, 250)], + )]; + let requests = vec![ + // Same div as the registry — must not duplicate. + request(SAMPLE_LEADERBOARD), + // A div the registry did not report — must be added. + request( + "https://securepubads.g.doubleclick.net/gampad/ads?iu_parts=123456789%2Cdesktop%2Chomepage%2Csidebar1&dids=div-gpt-ad-sidebar-1&prev_iu_szs=300x600", + ), + ]; + + let discovered = discover_gpt_slots(®istry, &requests, false); + + let ids: Vec<&str> = discovered + .slots + .iter() + .map(|slot| slot.id.as_str()) + .collect(); + assert_eq!( + ids, + vec!["leaderboard-1", "sidebar-1"], + "registry slot kept, request fills the missing div, no duplicate" + ); + } + + #[test] + fn registry_slot_without_pixel_sizes_is_skipped() { + let registry = vec![registry_slot("/123/fluid", "div-gpt-ad-fluid", &[(1, 1)])]; + + let discovered = discover_gpt_slots(®istry, &[], false); + + assert!( + discovered.slots.is_empty(), + "a registry slot with only fluid markers is not usable" + ); + } + + #[test] + fn normalizes_ephemeral_hash_and_container_and_dedups() { + // A framework-hashed div: the same placement appears as a hashed inner div, + // a `-container` wrapper, and re-rendered with a different hash. All must + // collapse to one stable stem. + let registry = vec![ + registry_slot( + "/987654321/homepage/header-0", + "ad-header-0-_R_9slinpflik6lb_", + &[(728, 90)], + ), + registry_slot( + "/987654321/homepage/header-0", + "ad-header-0-_R_9slinpflik6lb_-container", + &[(728, 90)], + ), + ]; + + let discovered = discover_gpt_slots(®istry, &[], false); + + assert_eq!( + discovered.slots.len(), + 1, + "hash + container variants collapse" + ); + assert_eq!( + discovered.slots[0].div_id, "ad-header-0", + "ephemeral React hash and -container are stripped to a stable stem" + ); + assert_eq!(discovered.slots[0].id, "ad-header-0"); + } + + #[test] + fn drops_sra_multi_slot_concatenations() { + let registry = vec![registry_slot( + "/987654321/homepage/header-0/fixed_bottom-0", + "ad-header-0-_R_9slin~ad-fixed_bottom-0-_R_ainp", + &[(728, 90)], + )]; + + let discovered = discover_gpt_slots(®istry, &[], false); + + assert!( + discovered.slots.is_empty(), + "tilde-joined SRA multi-slot divs are not real single elements" + ); + } + + #[test] + fn leaves_clean_div_ids_unchanged() { + assert_eq!( + normalize_div_stem("div-gpt-ad-leaderboard-1"), + "div-gpt-ad-leaderboard-1" + ); + } + + #[test] + fn sanitizes_page_controlled_div_ids_for_runtime_slot_ids() { + let registry = vec![registry_slot( + "/123456789/homepage/header", + "div-gpt-ad-header.main: 1", + &[(728, 90)], + )]; + + let discovered = discover_gpt_slots(®istry, &[], false); + + assert_eq!(discovered.slots[0].id, "header-main-1"); + assert_eq!( + discovered.slots[0].div_id, "div-gpt-ad-header.main: 1", + "matching should retain the original normalized div stem" + ); + trusted_server_core::creative_opportunities::validate_slot_id(&discovered.slots[0].id) + .expect("generated id should pass runtime validation"); + } + + #[test] + fn uses_fallback_for_div_id_without_safe_slot_id_characters() { + let registry = vec![registry_slot( + "/123456789/homepage/fallback", + "div-gpt-ad-...", + &[(300, 250)], + )]; + + let discovered = discover_gpt_slots(®istry, &[], false); + + assert_eq!(discovered.slots[0].id, "slot"); + } + + #[test] + fn makes_colliding_sanitized_slot_ids_unique() { + let registry = vec![ + registry_slot( + "/123456789/homepage/dotted", + "div-gpt-ad-header.main", + &[(728, 90)], + ), + registry_slot( + "/123456789/homepage/colon", + "div-gpt-ad-header:main", + &[(300, 250)], + ), + ]; + + let discovered = discover_gpt_slots(®istry, &[], false); + let ids = discovered + .slots + .iter() + .map(|slot| slot.id.as_str()) + .collect::>(); + + assert_eq!(ids, ["header-main", "header-main-2"]); + } + + #[test] + fn normalizes_react_and_hex_hashes_to_stable_prefixes() { + assert_eq!( + normalize_div_stem("ad-header-0-_R_9slinpflik6lb_-container"), + "ad-header-0" + ); + let stem = + normalize_div_stem("ad-in_content-de669245b2ea4b05826dc96f07a36272-in_content-0"); + assert_eq!(stem, "ad-in_content"); + assert!( + "ad-in_content-de669245b2ea4b05826dc96f07a36272-in_content-0".starts_with(&stem), + "stem must prefix-match any re-rendered hex variant" + ); + } + + #[test] + fn hex_hash_truncation_requires_a_segment_boundary() { + // Hex UUID bounded by `-` → truncated to the stem. + assert_eq!( + normalize_div_stem("ad-x-de669245b2ea4b05826dc96f07a36272-y"), + "ad-x" + ); + // A token that merely starts with 16 hex chars (no boundary) is left intact. + assert_eq!( + normalize_div_stem("ad-de669245b2ea4b05z"), + "ad-de669245b2ea4b05z" + ); + } + + #[test] + fn long_numeric_segments_are_stable_ids_not_hex_hashes() { + assert_eq!( + normalize_div_stem("ad-slot-1234567890123456-tail"), + "ad-slot-1234567890123456-tail" + ); + } + + #[test] + fn comma_separated_sra_dids_are_ignored() { + let discovered = from_requests(&[request( + "https://securepubads.g.doubleclick.net/gampad/ads?iu_parts=123%2Cnews%2Catf&dids=ad-a%2Cad-b&prev_iu_szs=300x250", + )]); + + assert!( + discovered.slots.is_empty(), + "a comma-joined SRA did list is not one element" + ); + } + + #[test] + fn one_element_under_two_render_tokens_is_not_a_collision() { + // Both ids describe in-content placement 0; only the hash between the + // two copies of the placement name differs, which is what one element + // re-rendered looks like. Refusing here would refuse the very shape + // normalization exists to absorb. + let registry = vec![ + registry_slot( + "/987654321/site/homepage", + "ad-in_content-de669245b2ea4b05826dc96f07a36272-in_content-0", + &[(300, 250)], + ), + registry_slot( + "/987654321/site/homepage", + "ad-in_content-8aec8129a83d4e5abc197423120cb19e-in_content-0", + &[(300, 250)], + ), + ]; + + let discovered = discover_gpt_slots(®istry, &[], false); + + assert_eq!( + discovered.slots.len(), + 1, + "two renders of one element are one slot, got {:?}", + discovered.slots + ); + assert_eq!(discovered.slots[0].div_id, "ad-in_content"); + assert!( + discovered.warnings.is_empty(), + "a re-render is not an ambiguity to report, got {:?}", + discovered.warnings + ); + assert!(discovered.ambiguous_stems.is_empty()); + } + + #[test] + fn sibling_placements_sharing_one_stem_are_refused() { + // Same shape as above, but the trailing placement index differs: these + // are two live elements, and one prefix cannot resolve to both. + let registry = vec![ + registry_slot( + "/987654321/site/homepage", + "ad-in_content-de669245b2ea4b05826dc96f07a36272-in_content-0", + &[(300, 250)], + ), + registry_slot( + "/987654321/site/homepage", + "ad-in_content-8aec8129a83d4e5abc197423120cb19e-in_content-1", + &[(300, 250)], + ), + ]; + + let discovered = discover_gpt_slots(®istry, &[], false); + + assert!( + discovered.had_slot_evidence, + "a refused placement is still evidence of an ad stack" + ); + assert!( + discovered.slots.is_empty(), + "neither a broad prefix nor per-render exact IDs are safe" + ); + assert_ambiguous_collision_warning(&discovered, "ad-in_content"); + assert!( + discovered.ambiguous_stems.contains("ad-in_content"), + "the verdict must travel with the evidence, got {:?}", + discovered.ambiguous_stems + ); + } + + #[test] + fn react_server_and_client_render_tokens_are_one_slot() { + // A hydrating publisher reports the SSR id and the client id for the + // same element. Both must collapse rather than refuse each other. + let registry = vec![ + registry_slot("/123456789/site/news", "ad-header-0-_R_3f_", &[(728, 90)]), + registry_slot("/123456789/site/news", "ad-header-0-_r_0_", &[(728, 90)]), + ]; + + let discovered = discover_gpt_slots(®istry, &[], false); + + assert_eq!( + discovered.slots.len(), + 1, + "SSR and client renders of one element are one slot, got {:?}", + discovered.slots + ); + assert_eq!(discovered.slots[0].div_id, "ad-header-0"); + assert!(discovered.warnings.is_empty()); + } + + #[test] + fn repeated_raw_div_after_a_normalization_collision_is_deduplicated() { + let first = "ad-x-aaaaaaaaaaaaaaaa-0"; + let second = "ad-x-bbbbbbbbbbbbbbbb-1"; + let third = "ad-x-cccccccccccccccc-2"; + let registry = vec![ + registry_slot("/123456789/site/home", first, &[(300, 250)]), + registry_slot("/123456789/site/home", second, &[(300, 250)]), + registry_slot("/123456789/site/home", first, &[(300, 250)]), + registry_slot("/123456789/site/home", second, &[(300, 250)]), + registry_slot("/123456789/site/home", third, &[(300, 250)]), + ]; + + let discovered = discover_gpt_slots(®istry, &[], false); + + assert!( + discovered.had_slot_evidence, + "a refused placement is still evidence of an ad stack" + ); + assert!( + discovered.slots.is_empty(), + "no repeat or later collision member may resurrect the group" + ); + assert_ambiguous_collision_warning(&discovered, "ad-x"); + } + + #[test] + fn request_normalization_collision_is_refused() { + let discovered = from_requests(&[ + request( + "https://securepubads.g.doubleclick.net/gampad/ads?iu_parts=123456789%2Csite%2Chome&dids=ad-x-aaaaaaaaaaaaaaaa-0&prev_iu_szs=300x250", + ), + request( + "https://securepubads.g.doubleclick.net/gampad/ads?iu_parts=123456789%2Csite%2Chome&dids=ad-x-bbbbbbbbbbbbbbbb-1&prev_iu_szs=300x250", + ), + ]); + + assert!( + discovered.had_slot_evidence, + "a refused placement is still evidence of an ad stack" + ); + assert!( + discovered.slots.is_empty(), + "a refused placement must not be written, got {:?}", + discovered.slots + ); + assert_eq!( + discovered.gam_network_id.as_deref(), + Some("123456789"), + "refusing a slot must not discard the network id" + ); + assert_ambiguous_collision_warning(&discovered, "ad-x"); + } + + #[test] + fn single_volatile_family_registry_slot_is_refused() { + let discovered = discover_gpt_slots( + &[registry_slot( + "/123456789/site_in-article_desktop_1", + "vendor-tag_1724112345678AbCdEfGh_slot_inarticle_1", + &[(300, 250)], + )], + &[], + false, + ); + + assert!( + discovered.had_slot_evidence, + "a refused placement is still evidence of an ad stack" + ); + assert!( + discovered.slots.is_empty(), + "one observation of a per-render family must not be written literally" + ); + assert_eq!(discovered.gam_network_id.as_deref(), Some("123456789")); + assert!( + discovered + .refused_div_ids + .contains("vendor-tag_1724112345678AbCdEfGh_slot_inarticle_1"), + "registry refusal should retain its normalized div as observed evidence" + ); + assert_volatile_prefix_warning(&discovered, "vendor-tag"); + } + + #[test] + fn single_volatile_family_request_slot_is_refused() { + let discovered = from_requests(&[request( + "https://securepubads.g.doubleclick.net/gampad/ads?iu_parts=123456789%2Csite_in-article_desktop_1&dids=vendor-tag_1724112345678AbCdEfGh_slot_inarticle_1&prev_iu_szs=300x250", + )]); + + assert!( + discovered.had_slot_evidence, + "a refused placement is still evidence of an ad stack" + ); + assert!( + discovered.slots.is_empty(), + "a refused placement must not be written, got {:?}", + discovered.slots + ); + assert_eq!( + discovered.gam_network_id.as_deref(), + Some("123456789"), + "refusing a slot must not discard the network id" + ); + assert!( + discovered + .refused_div_ids + .contains("vendor-tag_1724112345678AbCdEfGh_slot_inarticle_1"), + "request refusal should retain its normalized div as observed evidence" + ); + assert_volatile_prefix_warning(&discovered, "vendor-tag"); + } + + #[test] + fn shorter_high_entropy_singleton_registry_slot_is_refused() { + let discovered = discover_gpt_slots( + &[registry_slot( + "/123456789/publisher.example_overlay_mobile", + SHORT_VOLATILE_DIV, + &[(300, 250)], + )], + &[], + false, + ); + + assert!(discovered.had_slot_evidence); + assert!( + discovered.slots.is_empty(), + "a singleton per-render ID must not be written literally" + ); + assert_volatile_prefix_warning(&discovered, "vendor-tag"); + } + + #[test] + fn shorter_high_entropy_singleton_request_slot_is_refused() { + let discovered = from_requests(&[request(&format!( + "https://securepubads.g.doubleclick.net/gampad/ads?\ + iu_parts=123456789%2Cpublisher.example_overlay_mobile\ + &dids={SHORT_VOLATILE_DIV}&prev_iu_szs=300x250" + ))]); + + assert!(discovered.had_slot_evidence); + assert!( + discovered.slots.is_empty(), + "request fallback must not write a singleton per-render ID" + ); + assert_volatile_prefix_warning(&discovered, "vendor-tag"); + } + + #[test] + fn volatile_prefix_covers_every_placement_after_the_token() { + // The token's position is what makes the id unusable, so the placement + // that follows it is irrelevant: every one of these leaves `vendor-tag` + // as the only stable prefix, and that prefix reaches all of them. + for volatile in [ + "vendor-tag_1724112345678AbCdEfGh_slot_inarticle_1", + "vendor-tag_12345678AbCdEfGh_slot_inarticle_1", + "vendor-tag_20260820AbCdEfGh_slot_inarticle_1", + "vendor-tag_20260820deadbeef_slot_inarticle_1", + "vendor-tag_20260820ABCDEFGH_slot_inarticle_1", + "vendor-tag_20260820ABCD1234_slot_inarticle_1", + "vendor-tag_20260820A1B2C3D4_slot_inarticle_1", + "vendor-tag_1724112345678AbCdEfGh_slot_overlay_1-container", + "vendor-tag_1724112345678AbCdEfGh_slot_sidebar_1", + "vendor-tag_1724112345678AbCdEfGh_slot_overlay_stable", + "vendor-tag_1724112345678AbCdEfGh_slot_overlay_1_extra", + ] { + assert_eq!( + volatile_prefix_before_placement(volatile).as_deref(), + Some("vendor-tag"), + "`{volatile}` should be refused as a volatile family" + ); + } + } + + #[test] + fn volatile_prefix_does_not_claim_stable_div_ids() { + for stable in [ + // No per-render token at all. + "vendor-tag_stable_slot_inarticle_1", + // A bare digit run is how stable placement indices are written. + "vendor-tag_12345678_slot_inarticle_1", + "ad-slot-1234567890123456-tail", + // Shorter counter/suffix combinations do not carry enough entropy. + "vendor-tag_1234567AbCdEfGh_slot_inarticle_1", + "vendor-tag_12345678AbCdEfG_slot_inarticle_1", + // An eight-digit calendar date plus a stable suffix is not a + // timestamp-like per-render token. + "promo-20260820a-sidebar", + "promo-20260820Football-sidebar", + "promo-20260820football-sidebar", + "promo-20260820TopStories-sidebar", + "promo-20260820TopUsNewsAd-sidebar", + "promo-20260820MyAdUnitXy-sidebar", + "promo-20260820Top10Stories-sidebar", + "ad-19700101Thumbnail-rail", + "ad-00000001AAAAAAAA-rail", + // The token is trailing, so the prefix before it still identifies + // this element and normalization/collision handling own the case. + "vendor-tag_slot_inarticle_1724112345678AbCdEfGh", + "vendor-tag-header", + ] { + assert_eq!( + volatile_prefix_before_placement(stable), + None, + "`{stable}` should stay eligible" + ); + } + } + + #[test] + fn ambiguous_registry_stem_still_suppresses_request_fallback() { + let registry = vec![ + registry_slot( + "/123456789/site/home", + "ad-x-aaaaaaaaaaaaaaaa-0", + &[(300, 250)], + ), + registry_slot( + "/123456789/site/home", + "ad-x-bbbbbbbbbbbbbbbb-1", + &[(300, 250)], + ), + ]; + let requests = vec![request( + "https://securepubads.g.doubleclick.net/gampad/ads?iu_parts=123456789%2Csite%2Chome&dids=ad-x-cccccccccccccccc-2&prev_iu_szs=300x250", + )]; + + let discovered = discover_gpt_slots(®istry, &requests, false); + + assert!( + discovered.had_slot_evidence, + "a refused placement is still evidence of an ad stack" + ); + assert!( + discovered.slots.is_empty(), + "request fallback must not resurrect an ambiguous registry stem" + ); + assert_eq!(discovered.gam_network_id.as_deref(), Some("123456789")); + assert_ambiguous_collision_warning(&discovered, "ad-x"); + } + + #[test] + fn request_rerender_does_not_rewrite_a_stable_registry_slot() { + let registry = vec![registry_slot( + "/123456789/site/home", + "ad-x-aaaaaaaaaaaaaaaa-0", + &[(300, 250)], + )]; + let requests = vec![request( + "https://securepubads.g.doubleclick.net/gampad/ads?iu_parts=123456789%2Csite%2Chome&dids=ad-x-bbbbbbbbbbbbbbbb-1&prev_iu_szs=300x250", + )]; + + let discovered = discover_gpt_slots(®istry, &requests, false); + + assert_eq!(discovered.slots.len(), 1, "registry evidence should win"); + assert_eq!( + discovered.slots[0].div_id, "ad-x", + "request fallback must not destabilize a registry-derived prefix" + ); + } + + fn assert_ambiguous_collision_warning(discovered: &DiscoveredSlots, prefix: &str) { + assert_eq!(discovered.warnings.len(), 1); + let warning = &discovered.warnings[0]; + assert!(warning.contains(prefix), "warning should name the prefix"); + assert!( + warning.contains("one active element"), + "warning should explain why the broad prefix is unsafe" + ); + assert!( + warning.contains("change across renders"), + "warning should explain why raw IDs are unsafe" + ); + assert!( + warning.contains("distinct stable div ids"), + "warning should tell the operator how to make the placements configurable" + ); + } + + fn assert_volatile_prefix_warning(discovered: &DiscoveredSlots, prefix: &str) { + assert_eq!( + discovered.warnings.len(), + 1, + "should report the family once, got {:?}", + discovered.warnings + ); + let warning = &discovered.warnings[0]; + assert!( + warning.contains(prefix), + "warning should name the family prefix, got {warning}" + ); + assert!( + warning.contains("change across renders"), + "warning should explain why the exact ids are unsafe, got {warning}" + ); + assert!( + warning.contains("distinct stable div ids"), + "warning should tell the operator how to make the placements configurable, got {warning}" + ); + } +} diff --git a/crates/trusted-server-cli/src/commands/audit/generate/mod.rs b/crates/trusted-server-cli/src/commands/audit/generate/mod.rs new file mode 100644 index 000000000..67fe45275 --- /dev/null +++ b/crates/trusted-server-cli/src/commands/audit/generate/mod.rs @@ -0,0 +1,4182 @@ +mod analyzer; +pub(crate) mod browser_collector; +pub(crate) mod collector; +mod crawl_plan; +mod evidence; +mod gpt_slots; +mod page_patterns; +mod slot_toml; +mod unit_template; +mod validate; + +use std::collections::BTreeSet; +use std::fmt::Write as _; +use std::fs; +use std::io::Write; +use std::path::{Path, PathBuf}; + +use rand::RngCore as _; +use serde::Serialize; +use trusted_server_core::creative_opportunities::{ + CreativeOpportunitiesConfig, validate_page_pattern, +}; +use url::Url; + +use crate::commands::audit::ad_templates::{origin_changed, without_fragment}; +use crate::commands::audit::collector::GenerateBrowserOpts; +use crate::commands::audit::generate::collector::AuditCollector; +use crate::commands::audit::generate::slot_toml::{ + render_slots, replace_key_in_section, resolve_network_id, splice_creative_slots, toml_string, +}; +use crate::commands::config::init::EXAMPLE_CONFIG; +use crate::error::{CliResult, cli_error, report_error}; + +use analyzer::{analyze_collected_page, extract_gtm_container_id}; + +pub(crate) use browser_collector::DeviceProfile; +pub(crate) use crawl_plan::CrawlBudget; + +/// Writes `contents` to `path` atomically: a same-directory temp file is +/// written and fsynced, then renamed over the target, then the directory entry +/// is fsynced. +/// +/// A plain `fs::write` truncates the destination before writing, so a full disk +/// or an interrupted run would leave an operator's `trusted-server.toml` empty +/// or half-written. `rename` within a directory is atomic, so a reader sees +/// either the old file or the complete new one. +/// +/// The target's existing permissions are carried onto the replacement, since +/// the temp file is created 0600 and the config may intentionally be broader. +/// +/// # Errors +/// +/// Returns the underlying I/O error when the temp file cannot be created, +/// written, synced, or renamed over `path`. +fn write_file_atomically(path: &Path, contents: &str) -> std::io::Result<()> { + let directory = path + .parent() + .filter(|parent| !parent.as_os_str().is_empty()) + .unwrap_or_else(|| Path::new(".")); + + let mut temp = tempfile::Builder::new() + .prefix(".ts-audit-") + .tempfile_in(directory)?; + temp.write_all(contents.as_bytes())?; + temp.as_file().sync_all()?; + if let Ok(metadata) = fs::metadata(path) { + temp.as_file().set_permissions(metadata.permissions())?; + } + temp.persist(path).map_err(|error| error.error)?; + + // Best-effort durability for the rename itself. Opening a directory handle + // is not portable (Windows rejects it), and the content is already safely + // on disk either way, so a failure here is not worth failing the command. + let _ = fs::File::open(directory).and_then(|handle| handle.sync_all()); + Ok(()) +} + +/// Arguments for `ts audit generate ` — bootstraps draft Trusted Server +/// config and JavaScript asset audit files from a live page (issue #800). +#[derive(Debug, clap::Args)] +pub(crate) struct GenerateArgs { + /// Public HTTP(S) URL to audit. + pub(crate) url: String, + /// JavaScript asset audit output path. + #[arg(long)] + pub(crate) js_assets: Option, + /// Draft Trusted Server config output path. + #[arg(long)] + pub(crate) config: Option, + /// Do not write the JavaScript asset audit file. + #[arg(long)] + pub(crate) no_js_assets: bool, + /// Do not write the draft Trusted Server config file. + #[arg(long)] + pub(crate) no_config: bool, + /// Overwrite existing output files. + #[arg(long)] + pub(crate) force: bool, + /// Cookie to send with the page request, as `name=value`. Repeatable. + /// Use to carry an existing session (e.g. a valid bot-protection clearance + /// cookie) so the origin serves the real page instead of a challenge. + #[arg(long = "cookie", value_name = "NAME=VALUE", value_parser = crate::commands::audit::parse_cookie)] + pub(crate) cookies: Vec<(String, String)>, + /// Browser and consent options shared with `ts audit ad-templates generate`. + #[command(flatten)] + pub(crate) browser: GenerateBrowserOpts, +} + +const DEFAULT_JS_ASSETS_PATH: &str = "js-assets.toml"; +const DEFAULT_CONFIG_PATH: &str = "trusted-server.toml"; + +#[derive(Debug, Clone, Serialize, PartialEq, Eq)] +#[serde(rename_all = "kebab-case")] +pub(crate) enum AssetParty { + FirstParty, + ThirdParty, +} + +#[derive(Debug, Clone, Serialize, PartialEq, Eq)] +pub(crate) struct AuditedAsset { + pub(crate) kind: String, + pub(crate) url: String, + pub(crate) host: String, + pub(crate) party: AssetParty, + #[serde(skip_serializing_if = "Option::is_none")] + pub(crate) integration: Option, +} + +#[derive(Debug, Clone, Serialize, PartialEq, Eq)] +pub(crate) struct DetectedIntegration { + pub(crate) id: String, + pub(crate) evidence: String, +} + +#[derive(Debug, Clone, Serialize, PartialEq, Eq)] +pub(crate) struct AuditArtifact { + pub(crate) audited_url: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub(crate) page_title: Option, + pub(crate) js_asset_count: usize, + pub(crate) third_party_asset_count: usize, + pub(crate) detected_integrations: Vec, + pub(crate) assets: Vec, + pub(crate) warnings: Vec, +} + +#[derive(Debug, Clone)] +pub(crate) struct AuditOutputs { + pub(crate) artifact: AuditArtifact, + pub(crate) js_assets_toml: String, + pub(crate) draft_config_toml: String, + pub(crate) ad_slot_count: usize, + pub(crate) js_asset_proxy_candidate_count: usize, +} + +#[derive(Debug, Clone)] +struct DraftConfig { + toml: String, + js_asset_proxy_candidate_count: usize, +} + +#[derive(Debug, Clone)] +struct JsAssetProxySection { + toml: String, + candidate_count: usize, +} + +#[derive(Debug, Default)] +struct JsAssetProxySkipCounts { + first_party: usize, + malformed_url: usize, + non_https: usize, + duplicate_url: usize, + non_script: usize, +} + +#[derive(Debug)] +struct JsAssetProxyCandidate<'a> { + origin_url: String, + integration: Option<&'a str>, +} + +trait OpaqueAssetPathGenerator { + fn next_path(&mut self) -> String; +} + +#[derive(Debug, Default)] +struct RandomOpaqueAssetPathGenerator; + +impl OpaqueAssetPathGenerator for RandomOpaqueAssetPathGenerator { + fn next_path(&mut self) -> String { + let mut bytes = [0_u8; 12]; + rand::rngs::OsRng.fill_bytes(&mut bytes); + format!("/assets/{}.js", lowercase_hex(&bytes)) + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +struct AuditOutputPlan { + js_assets_path: Option, + config_path: Option, +} + +pub(crate) fn run_generate( + args: &GenerateArgs, + collector: &dyn AuditCollector, + out: &mut dyn Write, +) -> CliResult<()> { + let target_url = parse_audit_url(&args.url)?; + let plan = resolve_output_plan(args)?; + let collected = collector.collect_page(&target_url, &args.cookies)?; + let outputs = build_audit_outputs(&collected)?; + let wrote_config = plan.config_path.is_some(); + let written = write_audit_outputs(&outputs, &plan)?; + write_success_summary(&outputs, &written, wrote_config, out) +} + +fn parse_audit_url(value: &str) -> CliResult { + let url = Url::parse(value) + .map_err(|error| report_error(format!("invalid audit URL `{value}`: {error}")))?; + if !matches!(url.scheme(), "http" | "https") { + return cli_error(format!( + "`ts audit` only supports http/https URLs, got `{}`", + url.scheme() + )); + } + Ok(url) +} + +fn resolve_output_plan(args: &GenerateArgs) -> CliResult { + if args.no_js_assets && args.no_config { + return cli_error("nothing to do: both --no-js-assets and --no-config were set"); + } + + let js_assets_path = if args.no_js_assets { + None + } else { + Some(resolve_output_path( + args.js_assets.as_deref(), + DEFAULT_JS_ASSETS_PATH, + )?) + }; + let config_path = if args.no_config { + None + } else { + Some(resolve_output_path( + args.config.as_deref(), + DEFAULT_CONFIG_PATH, + )?) + }; + + if js_assets_path.is_some() && js_assets_path == config_path { + return cli_error("audit output paths must be distinct"); + } + + for path in [&js_assets_path, &config_path].into_iter().flatten() { + if path.exists() && !args.force { + return cli_error(format!( + "refusing to overwrite existing file `{}`; re-run with --force", + path.display() + )); + } + } + + Ok(AuditOutputPlan { + js_assets_path, + config_path, + }) +} + +fn resolve_output_path(path: Option<&Path>, default: &str) -> CliResult { + let candidate = path.unwrap_or_else(|| Path::new(default)); + if candidate.is_absolute() { + Ok(candidate.to_path_buf()) + } else { + Ok(std::env::current_dir() + .map_err(|error| report_error(format!("failed to read current directory: {error}")))? + .join(candidate)) + } +} + +fn build_audit_outputs(collected: &collector::CollectedPage) -> CliResult { + let artifact = analyze_collected_page(collected)?; + let final_url = collected + .final_url() + .map_err(|error| report_error(format!("invalid final URL: {error}")))?; + let js_assets_toml = toml::to_string_pretty(&artifact) + .map_err(|error| report_error(format!("failed to serialize audit artifact: {error}")))?; + let page_has_prebid = artifact + .detected_integrations + .iter() + .any(|integration| integration.id == "prebid"); + let slots = gpt_slots::discover_gpt_slots( + &collected.gpt_slots, + &collected.network_requests, + page_has_prebid, + ); + let ad_slot_count = slots.slots.len(); + let mut path_generator = RandomOpaqueAssetPathGenerator; + let draft_config = + build_draft_config_with_generator(&final_url, &artifact, &slots, &mut path_generator)?; + + Ok(AuditOutputs { + artifact, + js_assets_toml, + draft_config_toml: draft_config.toml, + ad_slot_count, + js_asset_proxy_candidate_count: draft_config.js_asset_proxy_candidate_count, + }) +} + +fn write_audit_outputs(outputs: &AuditOutputs, plan: &AuditOutputPlan) -> CliResult> { + let selected_paths = [&plan.js_assets_path, &plan.config_path] + .into_iter() + .flatten() + .collect::>(); + for path in &selected_paths { + if let Some(parent) = path + .parent() + .filter(|parent| !parent.as_os_str().is_empty()) + { + fs::create_dir_all(parent).map_err(|error| { + report_error(format!( + "failed to create parent directory {}: {error}", + parent.display() + )) + })?; + } + } + + let mut written_paths = Vec::new(); + if let Some(path) = &plan.js_assets_path { + write_file_atomically(path, &outputs.js_assets_toml).map_err(|error| { + report_error(format!( + "failed to write JS asset audit {}: {error}", + path.display() + )) + })?; + written_paths.push(path.display().to_string()); + } + if let Some(path) = &plan.config_path { + write_file_atomically(path, &outputs.draft_config_toml).map_err(|error| { + report_error(format!( + "failed to write draft config {}: {error}", + path.display() + )) + })?; + written_paths.push(path.display().to_string()); + } + + Ok(written_paths) +} + +fn write_success_summary( + outputs: &AuditOutputs, + written: &[String], + wrote_config: bool, + out: &mut dyn Write, +) -> CliResult<()> { + let integrations = outputs + .artifact + .detected_integrations + .iter() + .map(|integration| integration.id.as_str()) + .collect::>(); + let draft_note = if wrote_config { + "\nDraft config: review before validation and push" + } else { + "" + }; + let asset_proxy_note = if wrote_config && outputs.js_asset_proxy_candidate_count > 0 { + format!( + "{} disabled entries written to draft config", + outputs.js_asset_proxy_candidate_count + ) + } else if wrote_config { + "none".to_string() + } else { + "not written (--no-config)".to_string() + }; + writeln!( + out, + "Audited {}\nTitle: {}\nJS assets: {}\nThird-party assets: {}\nAd slots: {}\nDetected integrations: {}\nJS asset proxy candidates: {}\nWrote: {}{}", + outputs.artifact.audited_url, + outputs + .artifact + .page_title + .as_deref() + .unwrap_or(""), + outputs.artifact.js_asset_count, + outputs.artifact.third_party_asset_count, + outputs.ad_slot_count, + if integrations.is_empty() { + "none".to_string() + } else { + integrations.join(", ") + }, + asset_proxy_note, + if written.is_empty() { + "none".to_string() + } else { + written.join(", ") + }, + draft_note + ) + .map_err(|error| report_error(format!("failed to write command output: {error}"))) +} + +#[cfg(test)] +fn build_draft_config( + target_url: &Url, + artifact: &AuditArtifact, + slots: &gpt_slots::DiscoveredSlots, +) -> CliResult { + let mut path_generator = RandomOpaqueAssetPathGenerator; + build_draft_config_with_generator(target_url, artifact, slots, &mut path_generator) + .map(|draft| draft.toml) +} + +fn build_draft_config_with_generator( + target_url: &Url, + artifact: &AuditArtifact, + slots: &gpt_slots::DiscoveredSlots, + path_generator: &mut dyn OpaqueAssetPathGenerator, +) -> CliResult { + let host = target_url + .host_str() + .ok_or_else(|| report_error("audited URL is missing a host"))?; + let origin = target_url.origin().ascii_serialization(); + let mut draft = EXAMPLE_CONFIG.to_string(); + + draft = replace_key_in_section( + &draft, + "publisher", + "domain", + &format!("domain = \"{host}\""), + )?; + draft = replace_key_in_section( + &draft, + "publisher", + "cookie_domain", + &format!("cookie_domain = \".{host}\""), + )?; + draft = replace_key_in_section( + &draft, + "publisher", + "origin_url", + &format!("origin_url = \"{origin}\""), + )?; + + let detected = artifact + .detected_integrations + .iter() + .map(|integration| integration.id.as_str()) + .collect::>(); + + if detected.contains("gpt") { + draft = replace_key_in_section(&draft, "integrations.gpt", "enabled", "enabled = true")?; + } + if detected.contains("didomi") { + draft = replace_key_in_section(&draft, "integrations.didomi", "enabled", "enabled = true")?; + } + if detected.contains("datadome") { + draft = + replace_key_in_section(&draft, "integrations.datadome", "enabled", "enabled = true")?; + } + + let asset_proxy_section = build_js_asset_proxy_section(artifact, path_generator)?; + draft = replace_js_asset_proxy_section(&draft, &asset_proxy_section.toml)?; + + let mut manual_review = Vec::new(); + if detected.contains("google_tag_manager") { + if let Some(gtm_id) = extract_gtm_container_id(artifact) { + draft = replace_key_in_section( + &draft, + "integrations.google_tag_manager", + "enabled", + "enabled = true", + )?; + draft = replace_key_in_section( + &draft, + "integrations.google_tag_manager", + "container_id", + &format!("container_id = \"{gtm_id}\""), + )?; + } else { + manual_review.push("google_tag_manager"); + } + } + + for integration in detected { + if !matches!( + integration, + "gpt" | "didomi" | "datadome" | "google_tag_manager" + ) { + manual_review.push(integration); + } + } + + if !manual_review.is_empty() { + if !draft.ends_with('\n') { + draft.push('\n'); + } + draft.push_str("\n# Audit findings requiring manual review\n"); + for integration in manual_review { + draft.push_str(&format!( + "# - Detected {integration}; review the corresponding [integrations.{integration}] section before enabling it.\n" + )); + } + } + + if !slots.slots.is_empty() { + if let Some(network_id) = &slots.gam_network_id { + draft = replace_key_in_section( + &draft, + "creative_opportunities", + "gam_network_id", + &format!("gam_network_id = {}", toml_string(network_id)), + )?; + } + draft.push_str(&render_discovered_slots(target_url, slots)); + } + + Ok(DraftConfig { + toml: draft, + js_asset_proxy_candidate_count: asset_proxy_section.candidate_count, + }) +} + +fn build_js_asset_proxy_section( + artifact: &AuditArtifact, + path_generator: &mut dyn OpaqueAssetPathGenerator, +) -> CliResult { + let (candidates, skipped) = select_js_asset_proxy_candidates(artifact); + let mut used_paths = BTreeSet::new(); + let mut toml = String::new(); + + toml.push_str("[integrations.js_asset_proxy]\n"); + toml.push_str("enabled = false\n"); + toml.push_str("# Uncomment to override upstream cache headers for every asset below.\n"); + toml.push_str("# cache_ttl_seconds = 3600\n\n"); + toml.push_str("# Generated by `ts audit`; review before enabling.\n"); + toml.push_str( + "# Audit note: some discovered scripts may be runtime-injected and may not appear\n", + ); + toml.push_str( + "# in origin HTML. JS Asset Proxy rewrites only matching script src URLs present in\n", + ); + toml.push_str("# HTML processed by Trusted Server.\n"); + + if candidates.is_empty() { + toml.push_str( + "# No eligible third-party HTTPS script assets were detected by `ts audit`.\n", + ); + } + + for candidate in &candidates { + let generated_path = generate_unique_asset_path(path_generator, &mut used_paths)?; + toml.push('\n'); + toml.push_str("# Generated by `ts audit`; review before enabling.\n"); + if let Some(integration) = candidate.integration { + let integration = sanitized_comment_value(integration); + toml.push_str(&format!("# Detected integration: {integration}\n")); + toml.push_str(&format!( + "# Native integration may be preferable: [integrations.{integration}]\n" + )); + } + toml.push_str("[[integrations.js_asset_proxy.assets]]\n"); + toml.push_str(&format!("path = {}\n", toml_quoted_string(&generated_path))); + toml.push_str(&format!( + "origin_url = {}\n", + toml_quoted_string(&candidate.origin_url) + )); + if Url::parse(&candidate.origin_url).is_ok_and(|url| url.query().is_some()) { + toml.push_str( + "# This URL includes a query string and must remain stable for proxy matching.\n", + ); + } + toml.push_str("proxy = \"disabled\"\n"); + } + + append_js_asset_proxy_skip_comments(&mut toml, &skipped); + toml.push('\n'); + + Ok(JsAssetProxySection { + toml, + candidate_count: candidates.len(), + }) +} + +fn select_js_asset_proxy_candidates( + artifact: &AuditArtifact, +) -> (Vec>, JsAssetProxySkipCounts) { + let mut candidates = Vec::new(); + let mut skipped = JsAssetProxySkipCounts::default(); + let mut seen_origin_urls = BTreeSet::new(); + + for asset in &artifact.assets { + if asset.kind != "script" { + skipped.non_script += 1; + continue; + } + if asset.party != AssetParty::ThirdParty { + skipped.first_party += 1; + continue; + } + + let Ok(url) = Url::parse(&asset.url) else { + skipped.malformed_url += 1; + continue; + }; + if url.host_str().is_none() { + skipped.malformed_url += 1; + continue; + } + if url.scheme() != "https" { + skipped.non_https += 1; + continue; + } + + let origin_url = url.to_string(); + if !seen_origin_urls.insert(origin_url.clone()) { + skipped.duplicate_url += 1; + continue; + } + + candidates.push(JsAssetProxyCandidate { + origin_url, + integration: asset.integration.as_deref(), + }); + } + + (candidates, skipped) +} + +fn generate_unique_asset_path( + path_generator: &mut dyn OpaqueAssetPathGenerator, + used_paths: &mut BTreeSet, +) -> CliResult { + for _ in 0..128 { + let path = path_generator.next_path(); + if !is_valid_generated_asset_path(&path) { + return cli_error(format!( + "generated JS asset proxy path `{path}` is invalid; expected /assets/.js" + )); + } + if used_paths.insert(path.clone()) { + return Ok(path); + } + } + + cli_error("failed to generate a unique JS asset proxy path after 128 attempts") +} + +fn is_valid_generated_asset_path(path: &str) -> bool { + let Some(opaque_id) = path + .strip_prefix("/assets/") + .and_then(|value| value.strip_suffix(".js")) + else { + return false; + }; + + !opaque_id.is_empty() + && opaque_id + .chars() + .all(|ch| ch.is_ascii_hexdigit() && !ch.is_ascii_uppercase()) +} + +fn replace_js_asset_proxy_section(document: &str, replacement: &str) -> CliResult { + let lines = document.lines().collect::>(); + let start = lines + .iter() + .position(|line| line.trim() == "[integrations.js_asset_proxy]") + .ok_or_else(|| { + report_error( + "failed to update starter config because section `[integrations.js_asset_proxy]` was not found", + ) + })?; + let mut end = start + 1; + + while end < lines.len() { + let trimmed = lines[end].trim(); + if trimmed.starts_with('[') + && trimmed.ends_with(']') + && trimmed != "[[integrations.js_asset_proxy.assets]]" + { + break; + } + end += 1; + } + + let mut output_lines = Vec::new(); + output_lines.extend_from_slice(&lines[..start]); + output_lines.extend(replacement.trim_end_matches('\n').lines()); + if end < lines.len() { + output_lines.push(""); + } + output_lines.extend_from_slice(&lines[end..]); + + let mut output = output_lines.join("\n"); + if document.ends_with('\n') { + output.push('\n'); + } + Ok(output) +} + +fn append_js_asset_proxy_skip_comments(toml: &mut String, skipped: &JsAssetProxySkipCounts) { + if skipped.first_party == 0 + && skipped.malformed_url == 0 + && skipped.non_https == 0 + && skipped.duplicate_url == 0 + && skipped.non_script == 0 + { + return; + } + + toml.push('\n'); + toml.push_str("# Skipped JS Asset Proxy audit candidates:\n"); + append_skip_count(toml, skipped.first_party, "first-party script"); + append_skip_count(toml, skipped.malformed_url, "malformed script URL"); + append_skip_count(toml, skipped.non_https, "non-HTTPS third-party script"); + append_skip_count(toml, skipped.duplicate_url, "duplicate script URL"); + append_skip_count(toml, skipped.non_script, "non-script asset"); +} + +fn append_skip_count(toml: &mut String, count: usize, label: &str) { + if count == 0 { + return; + } + + let plural = if count == 1 { "" } else { "s" }; + toml.push_str(&format!("# - {count} {label}{plural}\n")); +} + +fn sanitized_comment_value(value: &str) -> String { + value + .chars() + .map(|ch| if ch.is_control() { ' ' } else { ch }) + .collect() +} + +fn toml_quoted_string(value: &str) -> String { + let mut quoted = String::from("\""); + for ch in value.chars() { + match ch { + '\\' => quoted.push_str("\\\\"), + '"' => quoted.push_str("\\\""), + '\n' => quoted.push_str("\\n"), + '\r' => quoted.push_str("\\r"), + '\t' => quoted.push_str("\\t"), + ch if ch.is_control() => { + write!(&mut quoted, "\\u{:04X}", ch as u32).expect("should write to string"); + } + ch => quoted.push(ch), + } + } + quoted.push('"'); + quoted +} + +fn lowercase_hex(bytes: &[u8]) -> String { + const HEX: &[u8; 16] = b"0123456789abcdef"; + let mut encoded = String::with_capacity(bytes.len() * 2); + for byte in bytes { + encoded.push(HEX[(byte >> 4) as usize] as char); + encoded.push(HEX[(byte & 0x0f) as usize] as char); + } + encoded +} + +/// Renders discovered GPT slots as appended `[[creative_opportunities.slot]]` +/// tables. Page patterns default to the audited path and are flagged for review. +fn render_discovered_slots(target_url: &Url, slots: &gpt_slots::DiscoveredSlots) -> String { + let path = target_url.path(); + let page_pattern = if path.is_empty() { "/" } else { path }; + + let mut out = String::from( + "\n# Slots discovered from live GPT ad requests during the audit.\n\ + # Review page_patterns and formats before validating/pushing.\n", + ); + for slot in &slots.slots { + let formats = slot + .formats + .iter() + .map(|(width, height)| format!("{{ width = {width}, height = {height} }}")) + .collect::>() + .join(", "); + out.push_str(&format!( + "\n[[creative_opportunities.slot]]\n\ + id = {id}\n\ + div_id = {div_id}\n\ + gam_unit_path = {gam_unit_path}\n\ + page_patterns = [{page_pattern}]\n\ + formats = [{formats}]\n", + id = toml_string(&slot.id), + div_id = toml_string(&slot.div_id), + gam_unit_path = toml_string(&slot.gam_unit_path), + page_pattern = toml_string(page_pattern), + )); + if slot.has_prebid { + out.push_str("[creative_opportunities.slot.providers.prebid]\nbidders = {}\n"); + } + } + out +} + +/// Everything one `ts audit ad-templates generate` invocation needs. +pub(crate) struct UpdateSlotsRequest<'a> { + /// Page URL to start from; also bounds the crawl to its origin. + pub(crate) url: &'a str, + /// Operator config to rewrite in place. + pub(crate) config_path: &'a Path, + /// The config's current `[creative_opportunities]`, when it has one. + pub(crate) existing_creative: Option<&'a CreativeOpportunitiesConfig>, + /// Explicit `--page-pattern` values. When non-empty these apply to every + /// slot and pattern inference is skipped entirely. + pub(crate) page_patterns: &'a [String], + /// Replace existing slots rather than merging into them. + pub(crate) replace: bool, + /// Cookies to carry into the crawl. + pub(crate) cookies: &'a [(String, String)], + /// Print the candidate instead of writing it. + pub(crate) dry_run: bool, + /// Whether the crawl used the deterministic scroll pass. + pub(crate) scroll: bool, + /// Crawl bounds. + pub(crate) budget: crawl_plan::CrawlBudget, +} + +/// Share of crawled pages that may yield no slots before the run is refused. +/// +/// A bot-protection challenge serves an interstitial that loads fine and +/// contains no ad stack, so it looks like a page with no slots. Writing a config +/// from a crawl that was mostly challenges would silently narrow the operator's +/// slot set; refusing is the safer failure. +const MAX_EMPTY_PAGE_SHARE: f64 = 0.25; + +/// Runs `ts audit ad-templates generate`: crawl the site's sections, reconcile +/// what each slot looked like across them, infer a `{section}` ad-unit template +/// where the evidence proves one, and rewrite the config's slot array in place. +/// +/// # Errors +/// +/// Returns an error when the config cannot be read, the root page cannot be +/// collected, no slots are discovered, too many pages came back empty, the +/// pages disagree about the GAM network id, or the resulting config would not +/// load. +pub(crate) fn run_update_slots( + request: &UpdateSlotsRequest<'_>, + collectors: &[(&str, &dyn AuditCollector)], + out: &mut dyn Write, + err: &mut dyn Write, +) -> CliResult<()> { + let Some((first_label, first_collector)) = collectors.first() else { + return cli_error("no device profile was selected to audit with"); + }; + let target_url = parse_audit_url(request.url)?; + let existing = fs::read_to_string(request.config_path).map_err(|error| { + report_error(format!( + "failed to read config {}: {error}", + request.config_path.display() + )) + })?; + + let mut table = evidence::EvidenceTable::default(); + let mut notes = Vec::new(); + let mut root_url = target_url.clone(); + let mut planned = None; + let mut fold_error = None; + + { + let mut progress_writer = CollectionProgressWriter { + out: err, + profile_label: first_label, + }; + let mut report_progress = + |progress: collector::CollectionProgress<'_>| progress_writer.write(progress); + first_collector.collect_site( + &target_url, + request.cookies, + &mut report_progress, + &mut |_, root| { + root_url = root.final_url().unwrap_or_else(|_| target_url.clone()); + if origin_changed(&target_url, &root_url) { + // Origins only: the origin is what the refusal is about, and + // a full URL would echo any `user:password@` the operator + // passed into stderr. + return cli_error(format!( + "refusing cross-origin root redirect from {} to {}; the requested origin is the audit and cookie trust boundary", + target_url.origin().ascii_serialization(), + root_url.origin().ascii_serialization() + )); + } + let plan = crawl_plan::plan_crawl( + &root_url, + &root.links, + &root.sitemap_locs, + request.budget, + ); + let targets = plan.targets(); + planned = Some(plan); + Ok(targets) + }, + &mut |url, collected| { + match collected { + Ok(page) => { + let final_url = page.final_url().unwrap_or_else(|_| url.clone()); + // The requested origin is the trust boundary for every + // page, not just the root: a section page that redirects + // away would otherwise contribute foreign slots, formats + // and ad-unit paths to the generated config. + if origin_changed(&target_url, &final_url) { + notes.push(format!( + "skipped `{}` on {first_label}: it left the audited origin for {}", + url.path(), + final_url.origin().ascii_serialization() + )); + return Ok(collector::ControlFlow::Continue); + } + if let Err(error) = + fold_collected( + &mut table, + &final_url, + &page, + first_label, + &mut notes, + ) + { + fold_error = Some(error); + return Ok(collector::ControlFlow::Stop); + } + } + Err(error) => { + // Path only, like the progress lines: a planned target + // still carries the origin and any userinfo. + notes.push(format!( + "skipped `{}` on {first_label}: {error}", + url.path() + )); + } + } + Ok(collector::ControlFlow::Continue) + }, + )?; + } + if let Some(error) = fold_error { + return Err(error); + } + let plan = planned.ok_or_else(|| { + report_error(format!( + "the {first_label} browser session did not produce a root page" + )) + })?; + notes.extend(plan.notes.iter().cloned()); + // Fragments never reach the server, so only a difference the origin acted on + // counts as a redirect worth reporting. + if without_fragment(&root_url) != without_fragment(&target_url) { + notes.push(format!( + "followed a root redirect from `{}{}` to `{}{}`; slots and page patterns are derived from the final URL", + target_url.origin().ascii_serialization(), + target_url.path(), + root_url.origin().ascii_serialization(), + root_url.path() + )); + } + + // Every profile walks the same pages into the same table. When two profiles + // disagree about a slot's ad-unit path, that shows up as two observations of + // one page, which inference already refuses to represent. + for (label, collector) in collectors.iter().skip(1) { + let mut progress_writer = CollectionProgressWriter { + out: err, + profile_label: label, + }; + let successful_pages = crawl_sections( + *collector, + &root_url, + &plan, + request.cookies, + &mut table, + &mut notes, + &mut progress_writer, + )?; + if successful_pages == 0 { + return cli_error(format!( + "the selected {label} device profile did not collect any required page; refusing to generate from incomplete profile coverage" + )); + } + } + if collectors.len() > 1 { + notes.push(format!( + "audited {} device profile(s): {}", + collectors.len(), + collectors + .iter() + .map(|(label, _)| *label) + .collect::>() + .join(", ") + )); + } + + // Emit what the crawl learned before any refusal below can return early. + // The guards exist precisely for runs that went wrong, so that is when the + // per-page reasons matter most. + emit_notes(err, &mut notes)?; + + if table.is_empty() { + return cli_error(format!( + "no ad-template slots were discovered on any of the {} crawled page(s); \ + see the notes above for what each page reported", + table.pages().len() + )); + } + guard_challenge_rate(&table)?; + + let discovered_network_id = table.network_id()?; + let network_id = resolve_network_id( + request.existing_creative, + discovered_network_id.as_deref(), + request.replace, + ); + + // Templating needs a network id to bind `{network_id}` against; without one + // every path stays literal. + let inference = network_id + .as_deref() + .map(|id| unit_template::infer_unit_templates(&table, id)); + if let Some(outcome) = &inference { + notes.extend(outcome.diagnostics.iter().cloned()); + } + let policy = inference + .as_ref() + .and_then(|outcome| outcome.policy.clone()); + validate_merge_policy(request.existing_creative, policy.as_ref(), request.replace)?; + + // Slots that are one placement wearing a per-render div id cannot be + // written: the ids never match at runtime. Report them so the operator can + // add the placement once with a prefix they know is stable. + let fragmented = table.fragmented_slots(); + for group in &fragmented { + let suggestion = group.suggested_prefix.as_deref().map_or_else( + || "no stable prefix was shared".to_string(), + |prefix| format!("they share the prefix `{prefix}`"), + ); + notes.push(format!( + "skipped {} slot(s) that look like one placement under a per-render div id on \ + `{}` ({}); {suggestion}. Add it once by hand with a div_id prefix that is \ + stable across renders", + group.div_ids.len(), + group.unit_path, + group.div_ids.join(", "), + )); + } + + let slots = build_render_slots( + &table, + inference.as_ref(), + policy.as_ref(), + request, + plan.section_segment, + &fragmented, + &mut notes, + )?; + let observed_div_ids = table + .observed_div_ids() + .map(str::to_string) + .collect::>(); + let observed_literals = table + .observed_literals() + .map(str::to_string) + .collect::>(); + let (merged, merge_diagnostics) = slot_toml::merge_render_slots_with_observed_diagnostics( + request.existing_creative, + slots, + &observed_div_ids, + &observed_literals, + request.replace, + ); + notes.extend(merge_diagnostics.notes); + if !merge_diagnostics.unobserved_existing_slot_ids.is_empty() { + let slot_ids = merge_diagnostics.unobserved_existing_slot_ids.join(", "); + let follow_up = if request.scroll { + "Re-run with broader page/profile coverage; `--replace` prunes them but also discards every hand-written field on the slots the run did rediscover." + } else { + "Re-run with broader coverage or --scroll; `--replace` prunes them but also discards every hand-written field on the slots the run did rediscover." + }; + notes.push(format!( + "preserved {} configured slot(s) not observed during this crawl: {slot_ids}. {follow_up}", + merge_diagnostics.unobserved_existing_slot_ids.len(), + )); + } + if merged.is_empty() { + emit_notes(err, &mut notes)?; + return cli_error( + "refusing to write zero generated slots after the crawl discovered slot evidence; review the refused-slot notes and keep the existing configuration", + ); + } + let rendered_slots = render_slots(&merged); + let updated = splice_creative_slots( + &existing, + &slot_toml::CreativeSectionKeys { + network_id: network_id.as_deref(), + section_root: policy.as_ref().map(|policy| policy.section_root.as_str()), + section_segment: policy.as_ref().map(|policy| policy.section_segment), + }, + &rendered_slots, + )?; + + // Everything above is derived from a live, page-controlled ad stack, so the + // candidate has to clear the runtime's own load path before it can replace + // the operator's file. This runs on the dry-run path too — otherwise "the + // preview looked fine" would not be evidence that the config loads. + notes.extend(validate::check_candidate(&updated, &existing)?); + + emit_notes(err, &mut notes)?; + if policy.is_some() { + writeln!( + err, + "note: this config now uses a {{section}} ad-unit template. Deploy a \ + template-aware binary BEFORE pushing it, and do not roll that binary \ + back while this config is live — an older binary rejects the whole \ + config and serves an error on every route." + ) + .map_err(|error| report_error(format!("failed to write command output: {error}")))?; + } + + if request.dry_run { + let old_managed = managed_creative_projection(&existing)?; + let new_managed = managed_creative_projection(&updated)?; + if old_managed == new_managed { + // Stdout is the diff surface, so an English sentence there would + // break a redirected `--dry-run`; an empty diff is the stdout answer. + writeln!(err, "No managed creative-opportunity changes.").map_err(|error| { + report_error(format!("failed to write preview output: {error}")) + })?; + return Ok(()); + } + let diff = similar::TextDiff::from_lines(&old_managed, &new_managed); + writeln!( + out, + "{}", + diff.unified_diff().context_radius(0).header( + "configured creative opportunities", + "generated creative opportunities" + ) + ) + .map_err(|error| report_error(format!("failed to write preview diff: {error}")))?; + return Ok(()); + } + let current = fs::read_to_string(request.config_path).map_err(|error| { + report_error(format!( + "failed to re-read config {} before writing: {error}", + request.config_path.display() + )) + })?; + if current != existing { + return cli_error(format!( + "refusing to overwrite {} because it changed during the browser audit; re-run against the current file", + request.config_path.display() + )); + } + // A writer could still land between this check and the rename below. That + // window is microseconds against a browser crawl's minutes, and the rename + // is atomic, so the loser of the race loses a whole write rather than half + // of one. Closing it properly would need file locking the operator's editor + // does not take part in. + write_file_atomically(request.config_path, &updated).map_err(|error| { + report_error(format!( + "failed to write config {}: {error}", + request.config_path.display() + )) + })?; + writeln!( + out, + "Wrote {} slot(s) to {} ({} slot(s) seen across {} page(s))", + merged.len(), + request.config_path.display(), + table.slot_count(), + table.pages().len(), + ) + .map_err(|error| report_error(format!("failed to write command output: {error}"))) +} + +/// Renders only fields managed by ad-template generation, excluding secrets and +/// unrelated operator configuration from dry-run output. +fn managed_creative_projection(document: &str) -> CliResult { + let value = toml::from_str::(document).map_err(|error| { + report_error(format!("failed to parse config for dry-run diff: {error}")) + })?; + let creative = value + .get("creative_opportunities") + .and_then(toml::Value::as_table); + let mut managed = toml::map::Map::new(); + if let Some(creative) = creative { + for key in ["gam_network_id", "section_root", "section_segment", "slot"] { + if let Some(value) = creative.get(key) { + managed.insert(key.to_string(), value.clone()); + } + } + } + let mut root = toml::map::Map::new(); + root.insert( + "creative_opportunities".to_string(), + toml::Value::Table(managed), + ); + toml::to_string_pretty(&toml::Value::Table(root)) + .map_err(|error| report_error(format!("failed to render dry-run projection: {error}"))) +} + +/// A page carrying fewer scripts than this is not a real publisher page. +/// +/// A production page runs dozens: the ad stack, analytics, consent, and the +/// site's own bundles. A bot-protection interstitial runs its own challenge +/// script and little else. +const INTERSTITIAL_SCRIPT_CEILING: usize = 3; + +/// Whether a page that loaded successfully is nonetheless not the real page. +/// +/// Bot protection commonly answers with **200** and a challenge document rather +/// than a 4xx, so status-code checks pass and the page simply appears to have no +/// ad stack. Left unexplained, that is indistinguishable from a publisher who +/// genuinely runs no ads on that page — and the operator's next move is entirely +/// different in each case. +fn looks_like_an_interstitial(artifact: &AuditArtifact) -> Option { + if artifact.js_asset_count > INTERSTITIAL_SCRIPT_CEILING + || !artifact.detected_integrations.is_empty() + { + return None; + } + Some(format!( + "the page returned successfully but carried only {} script(s) and no recognised \ + integrations, which is the shape of a bot-protection challenge rather than the \ + real page. Supply a current --cookie for the origin", + artifact.js_asset_count + )) +} + +/// Writes and clears the pending notes, so each is reported exactly once. +fn emit_notes(out: &mut dyn Write, notes: &mut Vec) -> CliResult<()> { + for note in notes.drain(..) { + writeln!( + out, + "note: {}", + crate::ad_templates::output::escape_terminal_text(¬e) + ) + .map_err(|error| report_error(format!("failed to write command output: {error}")))?; + } + Ok(()) +} + +/// Writes one immediately visible, profile-aware crawl progress line. +fn write_collection_progress( + out: &mut dyn Write, + profile_label: &str, + progress: collector::CollectionProgress<'_>, +) -> CliResult<()> { + let line = match progress { + collector::CollectionProgress::Launching => { + format!("Auditing {profile_label}: launching browser") + } + collector::CollectionProgress::Loading { + current, + total, + url, + } => { + let path = if url.path().is_empty() { + "/" + } else { + url.path() + }; + let path = crate::ad_templates::output::escape_terminal_text(path); + let total = total.map_or_else(|| "?".to_string(), |total| total.to_string()); + format!("Auditing {profile_label} [{current}/{total}]: {path}") + } + collector::CollectionProgress::Planning => { + format!("Auditing {profile_label}: planning site crawl") + } + collector::CollectionProgress::Finalizing => { + format!("Auditing {profile_label}: finalizing browser session") + } + }; + writeln!(out, "{line}") + .map_err(|error| report_error(format!("failed to write audit progress: {error}")))?; + out.flush() + .map_err(|error| report_error(format!("failed to flush audit progress: {error}"))) +} + +struct CollectionProgressWriter<'a> { + out: &'a mut dyn Write, + profile_label: &'a str, +} + +impl CollectionProgressWriter<'_> { + fn write(&mut self, progress: collector::CollectionProgress<'_>) -> CliResult<()> { + write_collection_progress(self.out, self.profile_label, progress) + } +} + +/// Discovers a collected page's slots and folds them into `table`. +/// +/// Per-page collector warnings are appended to `notes`. They carry the reason a +/// page came back without slots — a non-2xx main document, a navigation that +/// never settled — which is the difference between "this publisher has no ad +/// stack here" and "bot protection served a challenge". Dropping them leaves +/// the operator with a refusal and no way to act on it. +fn fold_collected( + table: &mut evidence::EvidenceTable, + url: &Url, + collected: &collector::CollectedPage, + profile_label: &str, + notes: &mut Vec, +) -> CliResult<()> { + // `analyze_collected_page` already carries the collector's warnings forward, + // so this is the complete set, not a second copy. + let artifact = analyze_collected_page(collected)?; + for warning in &artifact.warnings { + // The consent stub is a property of the run, not of this page. Scoping it + // to a path and repeating it per page and profile buries the per-page + // diagnostics an operator is reading these notes for. + let note = if warning == collector::CONSENT_STUB_WARNING { + warning.clone() + } else { + format!("`{}` on {profile_label}: {warning}", url.path()) + }; + if !notes.contains(¬e) { + notes.push(note); + } + } + if let Some(reason) = looks_like_an_interstitial(&artifact) { + notes.push(format!("`{}` on {profile_label}: {reason}", url.path())); + } + let page_has_prebid = artifact + .detected_integrations + .iter() + .any(|integration| integration.id == "prebid"); + let discovered = gpt_slots::discover_gpt_slots( + &collected.gpt_slots, + &collected.network_requests, + page_has_prebid, + ); + for warning in &discovered.warnings { + if !notes.contains(warning) { + notes.push(warning.clone()); + } + } + table.fold_page(url.path(), &discovered); + Ok(()) +} + +/// Walks the planned section pages, folding each into `table`. +/// +/// A page that fails to collect is recorded as a note rather than aborting: on a +/// multi-section crawl one blocked or slow page should not discard the sections +/// that did work. The empty-page guard afterwards catches the case where enough +/// of them failed that the result is untrustworthy. +fn crawl_sections( + collector: &dyn AuditCollector, + root_url: &Url, + plan: &crawl_plan::CrawlPlan, + cookies: &[(String, String)], + table: &mut evidence::EvidenceTable, + notes: &mut Vec, + progress_writer: &mut CollectionProgressWriter<'_>, +) -> CliResult { + let additional_targets = plan.targets(); + if additional_targets.is_empty() { + notes.push( + "no additional site sections were discovered, so only the requested page was \ + audited; pass explicit --page-pattern values or more URLs to widen coverage" + .to_string(), + ); + } + // The root is deliberately part of every profile's shared batch: browser + // clearance/session state established there then carries into section pages. + let mut targets = Vec::with_capacity(additional_targets.len() + 1); + targets.push(root_url.clone()); + targets.extend(additional_targets); + + let mut fold_error = None; + let mut successful_pages = 0_usize; + { + let profile_label = progress_writer.profile_label; + let mut report_progress = + |progress: collector::CollectionProgress<'_>| progress_writer.write(progress); + collector.collect_pages( + &targets, + cookies, + &mut report_progress, + &mut |url, collected| { + match collected { + Ok(page) => { + let final_url = page.final_url().unwrap_or_else(|_| url.clone()); + // Same boundary as the first profile, and it covers this + // profile's root page too: a cross-origin redirect is not + // a page this run may learn inventory from, so it must + // not count towards profile coverage either. + if origin_changed(root_url, &final_url) { + notes.push(format!( + "skipped `{}` on {profile_label}: it left the audited origin for {}", + url.path(), + final_url.origin().ascii_serialization() + )); + return Ok(collector::ControlFlow::Continue); + } + successful_pages += 1; + if let Err(error) = + fold_collected(table, &final_url, &page, profile_label, notes) + { + fold_error = Some(error); + return Ok(collector::ControlFlow::Stop); + } + } + Err(error) => { + notes.push(format!( + "skipped `{}` on {profile_label}: {error}", + url.path() + )); + } + } + Ok(collector::ControlFlow::Continue) + }, + )?; + } + match fold_error { + Some(error) => Err(error), + None => Ok(successful_pages), + } +} + +/// Refuses a crawl where too many pages produced no slots. +fn guard_challenge_rate(table: &evidence::EvidenceTable) -> CliResult<()> { + let total = table.pages().len(); + let empty = table.empty_pages().len(); + if total == 0 || (empty as f64) <= (total as f64) * MAX_EMPTY_PAGE_SHARE { + return Ok(()); + } + let blocked: Vec<&str> = table.empty_pages().iter().map(String::as_str).collect(); + cli_error(format!( + "{empty} of {total} crawled page(s) produced no ad slots ({}), which usually means \ + bot protection served a challenge instead of the real page. Refusing to write a \ + config from partial evidence; re-run with a valid --cookie for the origin", + blocked.join(", ") + )) +} + +/// Refuses a merge that would reinterpret templated slots the config already has. +/// +/// # Errors +/// +/// Returns an error when preserved `{section}` slots were written against a +/// different section policy than this run inferred, since the merge would leave +/// them pointing at ad units nobody configured. +fn validate_merge_policy( + existing: Option<&CreativeOpportunitiesConfig>, + inferred: Option<&unit_template::SectionPolicy>, + replace: bool, +) -> CliResult<()> { + if replace { + return Ok(()); + } + let Some(existing) = existing else { + return Ok(()); + }; + let preserves_template = existing.slot.iter().any(|slot| { + slot.gam_unit_path + .as_deref() + .is_some_and(|path| path.contains("{section}")) + }); + let Some(inferred) = inferred.filter(|_| preserves_template) else { + return Ok(()); + }; + if let Some(configured_segment) = existing.section_segment + && configured_segment != inferred.section_segment + { + return cli_error(format!( + "refusing to change the section_segment used by preserved templated slots during merge: configured section_segment={configured_segment}; inferred section_segment={}. Re-run with --replace only for an intentional migration", + inferred.section_segment + )); + } + // A `{section}` slot with no `section_root` cannot load at all — + // `validate_runtime` requires one — so there is no root value to preserve. + // Adopting the inferred root makes such a config loadable, provided the + // independently configured section segment above still agrees. + let Some(configured_root) = existing + .section_root + .as_deref() + .filter(|root| !root.is_empty()) + else { + return Ok(()); + }; + let configured_segment = existing.section_segment.unwrap_or(0); + if configured_root != inferred.section_root || configured_segment != inferred.section_segment { + return cli_error(format!( + "refusing to change the section policy used by preserved templated slots during merge: configured section_root={configured_root:?}, section_segment={configured_segment}; inferred section_root={:?}, section_segment={}. Re-run with --replace only for an intentional migration", + inferred.section_root, inferred.section_segment + )); + } + Ok(()) +} + +/// Turns the evidence table into slots ready to render. +fn build_render_slots( + table: &evidence::EvidenceTable, + inference: Option<&unit_template::InferenceOutcome>, + policy: Option<&unit_template::SectionPolicy>, + request: &UpdateSlotsRequest<'_>, + fallback_section_segment: usize, + fragmented: &[evidence::FragmentGroup], + notes: &mut Vec, +) -> CliResult> { + let skip: std::collections::BTreeSet<&str> = fragmented + .iter() + .flat_map(|group| group.div_ids.iter().map(String::as_str)) + .collect(); + // Explicit `--page-pattern` values are an operator override: they apply to + // every slot and disable inference from observed paths entirely. + let explicit = !request.page_patterns.is_empty(); + if explicit { + validate_page_patterns(request.page_patterns)?; + // Not filtered against `skip`: a borrowed root implies the slot's + // ad-unit path varied across pages, and `fragmented_slots` only groups + // slots pinned to exactly one unit path, so the two sets are disjoint. + if let Some(outcome) = inference + && !outcome.borrowed_section_root.is_empty() + { + let affected = outcome + .borrowed_section_root + .iter() + .map(|stem| format!("`{stem}`")) + .collect::>() + .join(", "); + return cli_error(format!( + "cannot apply --page-pattern to slot(s) with div id(s) {affected} because their \ + {{section}} templates borrow section_root; remove --page-pattern so patterns \ + can be derived from the paths where each slot was observed" + )); + } + } + let section_segment = policy.map_or(fallback_section_segment, |policy| policy.section_segment); + + let mut slots = Vec::with_capacity(table.slot_count()); + for slot in table.slots() { + if skip.contains(slot.div_id.as_str()) { + continue; + } + let patterns = if explicit { + request.page_patterns.to_vec() + } else { + let derived = page_patterns::patterns_for_paths(slot.paths(), section_segment); + validate_page_patterns(&derived)?; + derived + }; + let unit_path = match inference.and_then(|outcome| outcome.decision(&slot.div_id)) { + Some(unit_template::SlotDecision::Template(template)) => Some(template.clone()), + Some(unit_template::SlotDecision::Literal(path)) => Some(path.clone()), + Some(unit_template::SlotDecision::Refuse { reasons }) => { + notes.push(format!( + "skipped refused slot `{}` (`{}`): {}", + slot.id, + slot.div_id, + reasons.join("; ") + )); + continue; + } + None => None, + }; + slots.push(slot_toml::RenderSlot::from_evidence( + &slot.id, + &slot.div_id, + unit_path, + slot.formats.iter().copied(), + patterns, + slot.has_prebid, + )); + } + Ok(slots) +} +/// Rejects any page pattern the runtime's glob compiler would not accept. +/// +/// Uses [`validate_page_pattern`] so the accepted set is exactly what +/// `CreativeOpportunitySlot::compile_patterns` accepts at startup, including the +/// `**`→`*` normalisation. All patterns are reported at once so an operator +/// passing several `--page-pattern` values fixes them in one pass. +/// +/// # Errors +/// +/// Returns a user-facing error listing every pattern that does not compile. +fn validate_page_patterns(patterns: &[String]) -> CliResult<()> { + let invalid: Vec = patterns + .iter() + .filter_map(|pattern| validate_page_pattern(pattern).err()) + .collect(); + if invalid.is_empty() { + return Ok(()); + } + cli_error(format!( + "refusing to write invalid page pattern(s): {}", + invalid.join("; ") + )) +} + +#[cfg(test)] +mod tests { + use std::cell::{Cell, RefCell}; + use std::collections::VecDeque; + use std::io; + use std::rc::Rc; + + use tempfile::TempDir; + + use super::*; + use crate::app_config::AppConfigArgs; + use crate::commands::audit::generate::collector::{ + CollectedPage, CollectedRequest, CollectedScriptTag, + }; + use crate::commands::config::init::EXAMPLE_CONFIG; + + struct FakeCollector { + collected: CollectedPage, + calls: Cell, + } + + struct FixedPathGenerator { + paths: VecDeque, + } + + impl FixedPathGenerator { + fn new(paths: &[&str]) -> Self { + Self { + paths: paths.iter().map(|path| (*path).to_string()).collect(), + } + } + } + + impl OpaqueAssetPathGenerator for FixedPathGenerator { + fn next_path(&mut self) -> String { + self.paths + .pop_front() + .expect("should have a fixed generated asset path") + } + } + + struct MutatingCollector { + collected: CollectedPage, + config_path: std::path::PathBuf, + replacement: String, + } + + impl AuditCollector for MutatingCollector { + fn collect_page( + &self, + _target_url: &Url, + _cookies: &[(String, String)], + ) -> CliResult { + fs::write(&self.config_path, &self.replacement) + .map_err(|error| report_error(format!("failed to mutate test config: {error}")))?; + Ok(self.collected.clone()) + } + } + + impl FakeCollector { + fn new(collected: CollectedPage) -> Self { + Self { + collected, + calls: Cell::new(0), + } + } + } + + impl AuditCollector for FakeCollector { + fn collect_page( + &self, + _target_url: &Url, + _cookies: &[(String, String)], + ) -> CliResult { + self.calls.set(self.calls.get() + 1); + Ok(self.collected.clone()) + } + } + + /// A collector serving a distinct page per URL, recording the crawl order. + struct SiteCollector { + pages: std::collections::HashMap, + visited: std::cell::RefCell>, + } + + struct FailingCollector; + + #[derive(Clone, Default)] + struct SharedProgressState { + bytes: Rc>>, + flushes: Rc>, + } + + struct SharedProgressWriter { + state: SharedProgressState, + } + + impl Write for SharedProgressWriter { + fn write(&mut self, buffer: &[u8]) -> io::Result { + self.state.bytes.borrow_mut().extend_from_slice(buffer); + Ok(buffer.len()) + } + + fn flush(&mut self) -> io::Result<()> { + self.state.flushes.set(self.state.flushes.get() + 1); + Ok(()) + } + } + + struct ObservingProgressCollector { + collected: CollectedPage, + state: SharedProgressState, + saw_flushed_progress: Cell, + } + + impl AuditCollector for ObservingProgressCollector { + fn collect_page( + &self, + _target_url: &Url, + _cookies: &[(String, String)], + ) -> CliResult { + Ok(self.collected.clone()) + } + + fn collect_site( + &self, + root: &Url, + _cookies: &[(String, String)], + on_progress: collector::ProgressSink<'_>, + planner: collector::RootPlanner<'_>, + on_page: collector::PageSink<'_>, + ) -> CliResult<()> { + on_progress(collector::CollectionProgress::Loading { + current: 1, + total: None, + url: root, + })?; + self.saw_flushed_progress + .set(!self.state.bytes.borrow().is_empty() && self.state.flushes.get() > 0); + on_progress(collector::CollectionProgress::Planning)?; + let _ = planner(root, &self.collected)?; + let _ = on_page(root, Ok(self.collected.clone()))?; + Ok(()) + } + } + + #[derive(Default)] + struct ProgressWriter { + bytes: Vec, + flushes: usize, + fail_write: bool, + fail_flush: bool, + } + + impl Write for ProgressWriter { + fn write(&mut self, buffer: &[u8]) -> io::Result { + if self.fail_write { + return Err(io::Error::other("simulated progress write failure")); + } + self.bytes.extend_from_slice(buffer); + Ok(buffer.len()) + } + + fn flush(&mut self) -> io::Result<()> { + self.flushes += 1; + if self.fail_flush { + return Err(io::Error::other("simulated progress flush failure")); + } + Ok(()) + } + } + + #[test] + fn progress_lines_are_profile_aware_and_flush_immediately() { + let url = + Url::parse("https://user:pass@publisher.example/news\u{1b}[31m?token=secret#fragment") + .expect("should parse progress URL"); + let mut writer = ProgressWriter::default(); + + for progress in [ + collector::CollectionProgress::Launching, + collector::CollectionProgress::Loading { + current: 1, + total: None, + url: &url, + }, + collector::CollectionProgress::Planning, + collector::CollectionProgress::Loading { + current: 2, + total: Some(17), + url: &url, + }, + collector::CollectionProgress::Finalizing, + ] { + write_collection_progress(&mut writer, "desktop", progress) + .expect("should write progress"); + } + + let rendered = String::from_utf8(writer.bytes).expect("should render UTF-8 progress"); + assert_eq!( + rendered, + "Auditing desktop: launching browser\n\ + Auditing desktop [1/?]: /news%1B[31m\n\ + Auditing desktop: planning site crawl\n\ + Auditing desktop [2/17]: /news%1B[31m\n\ + Auditing desktop: finalizing browser session\n" + ); + assert_eq!(writer.flushes, 5, "should flush every progress line"); + assert!(!rendered.contains("user"), "should omit URL userinfo"); + assert!(!rendered.contains("secret"), "should omit URL query values"); + assert!(!rendered.contains("fragment"), "should omit URL fragments"); + assert!( + !rendered.contains('\u{1b}'), + "should not emit terminal escapes" + ); + } + + #[test] + fn progress_write_and_flush_failures_are_reported() { + let mut write_failure = ProgressWriter { + fail_write: true, + ..ProgressWriter::default() + }; + let write_error = write_collection_progress( + &mut write_failure, + "desktop", + collector::CollectionProgress::Launching, + ) + .expect_err("should report progress write failure"); + assert!(format!("{write_error:?}").contains("failed to write audit progress")); + + let mut flush_failure = ProgressWriter { + fail_flush: true, + ..ProgressWriter::default() + }; + let flush_error = write_collection_progress( + &mut flush_failure, + "desktop", + collector::CollectionProgress::Finalizing, + ) + .expect_err("should report progress flush failure"); + assert!(format!("{flush_error:?}").contains("failed to flush audit progress")); + } + + #[test] + fn update_slots_flushes_progress_before_collection_returns() { + let temp = TempDir::new().expect("should create temp dir"); + let config_path = temp.path().join("trusted-server.toml"); + fs::write( + &config_path, + "[creative_opportunities]\ngam_network_id = \"123456789\"\n", + ) + .expect("should write config"); + let state = SharedProgressState::default(); + let collector = ObservingProgressCollector { + collected: collected_page_with_header_slot(), + state: state.clone(), + saw_flushed_progress: Cell::new(false), + }; + let mut progress_writer = SharedProgressWriter { state }; + let mut out = Vec::new(); + + run_update_slots( + &UpdateSlotsRequest { + url: "https://publisher.example/", + config_path: &config_path, + existing_creative: None, + page_patterns: &[], + replace: false, + cookies: &[], + dry_run: false, + scroll: false, + budget: CrawlBudget::default(), + }, + &[("desktop", &collector)], + &mut out, + &mut progress_writer, + ) + .expect("should generate slots"); + + assert!( + collector.saw_flushed_progress.get(), + "collector should observe flushed progress before returning" + ); + assert!( + !String::from_utf8(out) + .expect("should write UTF-8 output") + .contains("Auditing "), + "stdout should not contain progress" + ); + } + + impl AuditCollector for FailingCollector { + fn collect_page( + &self, + target_url: &Url, + _cookies: &[(String, String)], + ) -> CliResult { + cli_error(format!("simulated navigation failure for {target_url}")) + } + } + + impl SiteCollector { + fn new(pages: Vec<(&str, CollectedPage)>) -> Self { + Self { + pages: pages + .into_iter() + .map(|(url, page)| (url.to_string(), page)) + .collect(), + visited: std::cell::RefCell::new(Vec::new()), + } + } + } + + impl AuditCollector for SiteCollector { + fn collect_page( + &self, + target_url: &Url, + _cookies: &[(String, String)], + ) -> CliResult { + self.visited.borrow_mut().push(target_url.to_string()); + self.pages + .get(target_url.as_str()) + .cloned() + .ok_or_else(|| report_error(format!("no fake page for {target_url}"))) + } + } + + /// Builds a page carrying one GPT slot plus same-origin nav links. + fn site_page(url: &str, unit_path: &str, nav_paths: &[&str]) -> CollectedPage { + let mut page = collected_page(); + page.requested_url = url.to_string(); + page.final_url = url.to_string(); + page.gpt_slots = vec![collector::CollectedGptSlot { + gam_unit_path: unit_path.to_string(), + div_id: "ad-header-0".to_string(), + sizes: vec![(728, 90)], + }]; + page.links = nav_paths + .iter() + .map(|path| collector::CollectedLink { + url: format!("https://publisher.example{path}"), + in_nav: true, + }) + .collect(); + page + } + + fn collected_page() -> CollectedPage { + CollectedPage { + requested_url: "https://publisher.example/page".to_string(), + final_url: "https://publisher.example/page".to_string(), + page_title: Some("Example Publisher".to_string()), + html: r#"Example Publisher"#.to_string(), + script_tags: vec![ + CollectedScriptTag { + src: Some("https://www.googletagmanager.com/gtm.js?id=GTM-ABC123".to_string()), + inline_text: None, + }, + CollectedScriptTag { + src: Some("https://securepubads.g.doubleclick.net/tag/js/gpt.js".to_string()), + inline_text: None, + }, + ], + network_requests: vec![CollectedRequest { + url: "https://cdn.publisher.example/app.js".to_string(), + resource_type: Some("script".to_string()), + }], + gpt_slots: Vec::new(), + links: Vec::new(), + sitemap_locs: Vec::new(), + warnings: Vec::new(), + } + } + + /// A collected page carrying one discoverable GPT slot, for `run_update_slots`. + fn collected_page_with_header_slot() -> CollectedPage { + let mut collected = collected_page(); + collected.requested_url = "https://publisher.example/".to_string(); + collected.final_url = "https://publisher.example/".to_string(); + collected.gpt_slots = vec![collector::CollectedGptSlot { + gam_unit_path: "/222/homepage/header".to_string(), + div_id: "div-gpt-ad-header".to_string(), + sizes: vec![(728, 90)], + }]; + collected + } + + fn collected_page_with_ambiguous_slots(url: &str) -> CollectedPage { + let mut collected = collected_page(); + collected.requested_url = url.to_string(); + collected.final_url = url.to_string(); + collected.gpt_slots = vec![ + collector::CollectedGptSlot { + gam_unit_path: "/222/homepage/in-content".to_string(), + div_id: "ad-x-aaaaaaaaaaaaaaaa-0".to_string(), + sizes: vec![(300, 250)], + }, + collector::CollectedGptSlot { + gam_unit_path: "/222/homepage/in-content".to_string(), + div_id: "ad-x-bbbbbbbbbbbbbbbb-1".to_string(), + sizes: vec![(300, 250)], + }, + ]; + collected + } + + fn audited_asset(url: &str, party: AssetParty, integration: Option<&str>) -> AuditedAsset { + AuditedAsset { + kind: "script".to_string(), + url: url.to_string(), + host: Url::parse(url) + .ok() + .and_then(|parsed| parsed.host_str().map(str::to_string)) + .unwrap_or_default(), + party, + integration: integration.map(str::to_string), + } + } + + fn audit_args(url: &str) -> GenerateArgs { + GenerateArgs { + url: url.to_string(), + js_assets: None, + config: None, + no_js_assets: false, + no_config: false, + force: false, + cookies: Vec::new(), + browser: GenerateBrowserOpts::default(), + } + } + + #[test] + fn parse_audit_url_accepts_http_and_https() { + assert!(parse_audit_url("http://publisher.example").is_ok()); + assert!(parse_audit_url("https://publisher.example").is_ok()); + } + + #[test] + fn parse_audit_url_rejects_non_http_schemes() { + for url in [ + "file:///etc/passwd", + "data:text/html,hello", + "chrome://version", + ] { + let error = parse_audit_url(url).expect_err("should reject non-http URL"); + assert!( + format!("{error:?}").contains("only supports http/https"), + "should explain scheme restriction" + ); + } + } + + #[test] + fn repeated_ambiguous_collision_note_is_emitted_once() { + let mut table = evidence::EvidenceTable::default(); + let mut notes = Vec::new(); + for url in [ + "https://publisher.example/", + "https://publisher.example/news", + ] { + fold_collected( + &mut table, + &Url::parse(url).expect("should parse fixture URL"), + &collected_page_with_ambiguous_slots(url), + "desktop", + &mut notes, + ) + .expect("should fold ambiguous page evidence"); + } + + assert_eq!( + notes.len(), + 1, + "the same site-wide collision guidance should not repeat per page" + ); + } + + #[test] + fn merge_refuses_to_change_policy_used_by_preserved_templates() { + let existing: CreativeOpportunitiesConfig = toml::from_str( + "gam_network_id = \"123\"\nsection_root = \"home\"\nsection_segment = 0\n\ + [[slot]]\nid = \"header\"\ndiv_id = \"ad-header\"\n\ + gam_unit_path = \"/{network_id}/site/{section}\"\npage_patterns = [\"/\"]\n\ + formats = [{ width = 728, height = 90 }]\n", + ) + .expect("should parse creative config"); + let inferred = unit_template::SectionPolicy { + section_root: "homepage".to_string(), + section_segment: 1, + }; + + let error = validate_merge_policy(Some(&existing), Some(&inferred), false) + .expect_err("merge must preserve the existing template policy"); + + assert!(format!("{error:?}").contains("--replace")); + validate_merge_policy(Some(&existing), Some(&inferred), true) + .expect("replace is an explicit policy migration"); + } + + #[test] + fn the_consent_stub_note_is_reported_once_and_unscoped() { + let mut table = evidence::EvidenceTable::default(); + let mut notes = Vec::new(); + for url in [ + "https://publisher.example/", + "https://publisher.example/news", + ] { + let mut page = collected_page(); + page.requested_url = url.to_string(); + page.final_url = url.to_string(); + page.warnings + .push(collector::CONSENT_STUB_WARNING.to_string()); + fold_collected( + &mut table, + &Url::parse(url).expect("should parse fixture URL"), + &page, + "desktop", + &mut notes, + ) + .expect("should fold page evidence"); + } + + assert_eq!( + notes, + [collector::CONSENT_STUB_WARNING.to_string()], + "a run-wide fact should appear once, without a page path" + ); + } + + #[test] + fn page_warnings_remain_distinct_across_profiles() { + let mut table = evidence::EvidenceTable::default(); + let mut notes = Vec::new(); + let mut page = collected_page(); + page.requested_url = "https://publisher.example/news".to_string(); + page.final_url = page.requested_url.clone(); + page.warnings.push("navigation did not settle".to_string()); + let url = Url::parse(&page.final_url).expect("should parse fixture URL"); + + fold_collected(&mut table, &url, &page, "desktop", &mut notes) + .expect("should fold desktop evidence"); + fold_collected(&mut table, &url, &page, "mobile", &mut notes) + .expect("should fold mobile evidence"); + + assert_eq!( + notes.len(), + 2, + "profile-specific warnings must not collapse" + ); + assert!(notes.iter().any(|note| note.contains("on desktop"))); + assert!(notes.iter().any(|note| note.contains("on mobile"))); + } + + #[test] + fn merge_adopts_the_inferred_policy_when_none_is_configured() { + // A hand-written `{section}` slot with no `section_root` describes a + // config the runtime refuses to load, so the first merge should repair it + // rather than demand `--replace` (which would discard the hand-tuned + // slots it is preserving). + let existing: CreativeOpportunitiesConfig = toml::from_str( + "gam_network_id = \"123\"\n\ + [[slot]]\nid = \"header\"\ndiv_id = \"ad-header\"\n\ + gam_unit_path = \"/{network_id}/site/{section}\"\npage_patterns = [\"/\"]\n\ + formats = [{ width = 728, height = 90 }]\n", + ) + .expect("should parse creative config"); + let inferred = unit_template::SectionPolicy { + section_root: "homepage".to_string(), + section_segment: 1, + }; + + validate_merge_policy(Some(&existing), Some(&inferred), false) + .expect("should have no policy to preserve when section_root is unset"); + } + + #[test] + fn merge_preserves_an_explicit_segment_when_section_root_is_unset() { + let existing: CreativeOpportunitiesConfig = toml::from_str( + "gam_network_id = \"123\"\nsection_segment = 1\n\ + [[slot]]\nid = \"header\"\ndiv_id = \"ad-header\"\n\ + gam_unit_path = \"/{network_id}/site/{section}\"\npage_patterns = [\"/\"]\n\ + formats = [{ width = 728, height = 90 }]\n", + ) + .expect("should parse creative config"); + let mismatched = unit_template::SectionPolicy { + section_root: "homepage".to_string(), + section_segment: 0, + }; + + let error = validate_merge_policy(Some(&existing), Some(&mismatched), false) + .expect_err("should preserve an explicitly configured segment"); + + assert!(format!("{error:?}").contains("section_segment=1")); + + let matching = unit_template::SectionPolicy { + section_root: "homepage".to_string(), + section_segment: 1, + }; + validate_merge_policy(Some(&existing), Some(&matching), false) + .expect("should adopt a root without changing the configured segment"); + } + + #[test] + fn resolve_output_plan_rejects_no_outputs() { + let mut args = audit_args("https://publisher.example"); + args.no_js_assets = true; + args.no_config = true; + + let error = resolve_output_plan(&args).expect_err("should reject empty output set"); + + assert!( + format!("{error:?}").contains("nothing to do"), + "should explain no-output error" + ); + } + + #[test] + fn resolve_output_plan_rejects_existing_files_without_force() { + let temp = TempDir::new().expect("should create temp dir"); + let path = temp.path().join("js-assets.toml"); + fs::write(&path, "existing").expect("should write existing file"); + let mut args = audit_args("https://publisher.example"); + args.js_assets = Some(path); + args.no_config = true; + + let error = resolve_output_plan(&args).expect_err("should reject overwrite"); + + assert!( + format!("{error:?}").contains("refusing to overwrite"), + "should explain overwrite refusal" + ); + } + + #[test] + fn resolve_output_plan_allows_existing_files_with_force() { + let temp = TempDir::new().expect("should create temp dir"); + let path = temp.path().join("js-assets.toml"); + fs::write(&path, "existing").expect("should write existing file"); + let mut args = audit_args("https://publisher.example"); + args.js_assets = Some(path.clone()); + args.no_config = true; + args.force = true; + + let plan = resolve_output_plan(&args).expect("should allow forced overwrite"); + + assert_eq!(plan.js_assets_path.as_deref(), Some(path.as_path())); + } + + #[test] + fn run_generate_writes_selected_outputs_and_summary() { + let temp = TempDir::new().expect("should create temp dir"); + let js_assets = temp.path().join("audit/js-assets.toml"); + let config = temp.path().join("audit/trusted-server.toml"); + let args = GenerateArgs { + url: "https://publisher.example/page".to_string(), + js_assets: Some(js_assets.clone()), + config: Some(config.clone()), + no_js_assets: false, + no_config: false, + force: false, + cookies: Vec::new(), + browser: GenerateBrowserOpts::default(), + }; + let collector = FakeCollector::new(collected_page()); + let mut out = Vec::new(); + + run_generate(&args, &collector, &mut out).expect("should run audit"); + + assert_eq!(collector.calls.get(), 1, "should collect page once"); + assert!(js_assets.exists(), "should write JS assets"); + assert!(config.exists(), "should write draft config"); + let summary = String::from_utf8(out).expect("summary should be UTF-8"); + assert!(summary.contains("Audited https://publisher.example/page")); + assert!(summary.contains("Detected integrations: google_tag_manager, gpt")); + assert!(summary.contains("Draft config: review before validation and push")); + } + + #[test] + fn run_generate_respects_no_config() { + let temp = TempDir::new().expect("should create temp dir"); + let js_assets = temp.path().join("js-assets.toml"); + let mut args = audit_args("https://publisher.example/page"); + args.js_assets = Some(js_assets.clone()); + args.no_config = true; + let collector = FakeCollector::new(collected_page()); + + run_generate(&args, &collector, &mut Vec::new()).expect("should run audit"); + + assert!(js_assets.exists(), "should write assets"); + assert!( + !temp.path().join("trusted-server.toml").exists(), + "should not write config" + ); + } + + #[test] + fn run_generate_respects_no_js_assets() { + let temp = TempDir::new().expect("should create temp dir"); + let config = temp.path().join("trusted-server.toml"); + let mut args = audit_args("https://publisher.example/page"); + args.config = Some(config.clone()); + args.no_js_assets = true; + let collector = FakeCollector::new(collected_page()); + let mut out = Vec::new(); + + run_generate(&args, &collector, &mut out).expect("should run audit"); + + assert!(config.exists(), "should write config"); + assert!( + !temp.path().join("js-assets.toml").exists(), + "should not write JS assets" + ); + let summary = String::from_utf8(out).expect("summary should be UTF-8"); + assert!(summary.contains("Draft config: review before validation and push")); + } + + #[test] + fn run_generate_writes_collector_warnings_to_asset_artifact() { + let temp = TempDir::new().expect("should create temp dir"); + let js_assets = temp.path().join("js-assets.toml"); + let mut args = audit_args("https://publisher.example/page"); + args.js_assets = Some(js_assets.clone()); + args.no_config = true; + let mut collected = collected_page(); + collected.warnings.push( + "browser audit timed out while waiting for the page to settle; results may be partial" + .to_string(), + ); + let collector = FakeCollector::new(collected); + + run_generate(&args, &collector, &mut Vec::new()).expect("should run audit"); + + let artifact = fs::read_to_string(js_assets).expect("should read artifact"); + assert!( + artifact.contains("results may be partial"), + "should persist collector warning" + ); + } + + #[test] + fn run_generate_conflict_prevents_collection() { + let temp = TempDir::new().expect("should create temp dir"); + let js_assets = temp.path().join("js-assets.toml"); + fs::write(&js_assets, "existing").expect("should write existing file"); + let mut args = audit_args("https://publisher.example/page"); + args.js_assets = Some(js_assets); + args.no_config = true; + let collector = FakeCollector::new(collected_page()); + + let error = run_generate(&args, &collector, &mut Vec::new()) + .expect_err("should reject existing output"); + + assert_eq!(collector.calls.get(), 0, "should not collect page"); + assert!( + format!("{error:?}").contains("refusing to overwrite"), + "should report overwrite conflict" + ); + } + + #[test] + fn build_draft_config_writes_disabled_js_asset_proxy_candidates() { + let url = Url::parse("https://publisher.example/page").expect("should parse URL"); + let artifact = AuditArtifact { + audited_url: url.to_string(), + page_title: Some("Example".to_string()), + js_asset_count: 2, + third_party_asset_count: 2, + detected_integrations: vec![DetectedIntegration { + id: "gpt".to_string(), + evidence: "https://securepubads.g.doubleclick.net/tag/js/gpt.js".to_string(), + }], + assets: vec![ + audited_asset( + "https://cdn.vendor.example/sdk.js", + AssetParty::ThirdParty, + None, + ), + audited_asset( + "https://securepubads.g.doubleclick.net/tag/js/gpt.js", + AssetParty::ThirdParty, + Some("gpt"), + ), + ], + warnings: Vec::new(), + }; + let mut generator = FixedPathGenerator::new(&[ + "/assets/aaaaaaaaaaaaaaaaaaaaaaaa.js", + "/assets/bbbbbbbbbbbbbbbbbbbbbbbb.js", + ]); + + let draft = build_draft_config_with_generator( + &url, + &artifact, + &gpt_slots::DiscoveredSlots::default(), + &mut generator, + ) + .expect("should build draft config"); + + assert_eq!(draft.js_asset_proxy_candidate_count, 2); + assert!( + draft + .toml + .contains("[integrations.js_asset_proxy]\nenabled = false") + ); + assert!(draft.toml.contains("/assets/aaaaaaaaaaaaaaaaaaaaaaaa.js")); + assert!(draft.toml.contains("/assets/bbbbbbbbbbbbbbbbbbbbbbbb.js")); + assert!( + draft + .toml + .contains("origin_url = \"https://cdn.vendor.example/sdk.js\"") + ); + assert!(draft.toml.contains("proxy = \"disabled\"")); + assert!(draft.toml.contains("Detected integration: gpt")); + assert!( + !draft.toml.contains("example-vendor-loader"), + "should remove the starter placeholder asset" + ); + toml::from_str::(&draft.toml).expect("draft should parse as TOML"); + } + + #[test] + fn asset_proxy_generation_deduplicates_and_summarizes_skips() { + let artifact = AuditArtifact { + audited_url: "https://publisher.example/page".to_string(), + page_title: None, + js_asset_count: 4, + third_party_asset_count: 3, + detected_integrations: Vec::new(), + assets: vec![ + audited_asset( + "https://cdn.vendor.example/sdk.js", + AssetParty::ThirdParty, + None, + ), + audited_asset( + "https://cdn.vendor.example/sdk.js", + AssetParty::ThirdParty, + None, + ), + audited_asset( + "https://publisher.example/app.js", + AssetParty::FirstParty, + None, + ), + audited_asset( + "http://cdn.vendor.example/insecure.js", + AssetParty::ThirdParty, + None, + ), + ], + warnings: Vec::new(), + }; + let mut generator = FixedPathGenerator::new(&["/assets/111111111111111111111111.js"]); + + let section = + build_js_asset_proxy_section(&artifact, &mut generator).expect("should build section"); + + assert_eq!(section.candidate_count, 1); + assert_eq!( + section + .toml + .matches("[[integrations.js_asset_proxy.assets]]") + .count(), + 1 + ); + assert!(section.toml.contains("# - 1 first-party script")); + assert!(section.toml.contains("# - 1 non-HTTPS third-party script")); + assert!(section.toml.contains("# - 1 duplicate script URL")); + } + + #[test] + fn asset_proxy_generation_with_no_candidates_removes_placeholder_asset() { + let url = Url::parse("https://publisher.example/page").expect("should parse URL"); + let artifact = AuditArtifact { + audited_url: url.to_string(), + page_title: None, + js_asset_count: 1, + third_party_asset_count: 0, + detected_integrations: Vec::new(), + assets: vec![audited_asset( + "https://publisher.example/app.js", + AssetParty::FirstParty, + None, + )], + warnings: Vec::new(), + }; + let mut generator = FixedPathGenerator::new(&[]); + + let draft = build_draft_config_with_generator( + &url, + &artifact, + &gpt_slots::DiscoveredSlots::default(), + &mut generator, + ) + .expect("should build draft config"); + + assert_eq!(draft.js_asset_proxy_candidate_count, 0); + assert!( + draft + .toml + .contains("No eligible third-party HTTPS script assets") + ); + assert!( + !draft + .toml + .contains("[[integrations.js_asset_proxy.assets]]") + ); + assert!(!draft.toml.contains("example-vendor-loader")); + } + + #[test] + fn asset_proxy_generation_warns_about_query_string_candidates() { + let artifact = AuditArtifact { + audited_url: "https://publisher.example/page".to_string(), + page_title: None, + js_asset_count: 1, + third_party_asset_count: 1, + detected_integrations: Vec::new(), + assets: vec![audited_asset( + "https://cdn.vendor.example/sdk.js?v=one", + AssetParty::ThirdParty, + None, + )], + warnings: Vec::new(), + }; + let mut generator = FixedPathGenerator::new(&["/assets/aaaaaaaaaaaaaaaaaaaaaaaa.js"]); + + let section = + build_js_asset_proxy_section(&artifact, &mut generator).expect("should build section"); + + assert!( + section + .toml + .contains("query string and must remain stable for proxy matching") + ); + } + + #[test] + fn run_generate_summary_reports_written_asset_proxy_candidates() { + let temp = TempDir::new().expect("should create temp dir"); + let config = temp.path().join("trusted-server.toml"); + let mut args = audit_args("https://publisher.example/page"); + args.config = Some(config); + args.no_js_assets = true; + let collector = FakeCollector::new(collected_page()); + let mut out = Vec::new(); + + run_generate(&args, &collector, &mut out).expect("should run audit"); + + let summary = String::from_utf8(out).expect("summary should be UTF-8"); + assert!(summary.contains("JS asset proxy candidates:")); + assert!(summary.contains("disabled entries written to draft config")); + } + + #[test] + fn build_draft_config_uses_final_url_and_detected_integrations() { + let url = Url::parse("https://www.publisher.example:8443/path").expect("should parse URL"); + let artifact = AuditArtifact { + audited_url: url.to_string(), + page_title: Some("Example".to_string()), + js_asset_count: 2, + third_party_asset_count: 2, + detected_integrations: vec![ + DetectedIntegration { + id: "google_tag_manager".to_string(), + evidence: "GTM-ABC123".to_string(), + }, + DetectedIntegration { + id: "gpt".to_string(), + evidence: "https://securepubads.g.doubleclick.net/tag/js/gpt.js".to_string(), + }, + DetectedIntegration { + id: "prebid".to_string(), + evidence: "inline script matched `prebid`".to_string(), + }, + ], + assets: Vec::new(), + warnings: Vec::new(), + }; + + let draft = build_draft_config(&url, &artifact, &gpt_slots::DiscoveredSlots::default()) + .expect("should build draft config"); + + assert!(draft.contains("domain = \"www.publisher.example\"")); + assert!(draft.contains("cookie_domain = \".www.publisher.example\"")); + assert!(draft.contains("origin_url = \"https://www.publisher.example:8443\"")); + assert!(draft.contains("[integrations.gpt]\nenabled = true")); + assert!(draft.contains("[integrations.google_tag_manager]\nenabled = true")); + assert!(draft.contains("container_id = \"GTM-ABC123\"")); + assert!(draft.contains("Detected prebid")); + toml::from_str::(&draft).expect("draft should parse as TOML"); + } + + #[test] + fn build_draft_config_does_not_enable_gtm_without_container_id() { + let url = Url::parse("https://publisher.example/path").expect("should parse URL"); + let artifact = AuditArtifact { + audited_url: url.to_string(), + page_title: None, + js_asset_count: 1, + third_party_asset_count: 1, + detected_integrations: vec![DetectedIntegration { + id: "google_tag_manager".to_string(), + evidence: "https://www.googletagmanager.com/gtm.js".to_string(), + }], + assets: Vec::new(), + warnings: Vec::new(), + }; + + let draft = build_draft_config(&url, &artifact, &gpt_slots::DiscoveredSlots::default()) + .expect("should build draft config"); + + assert!(draft.contains("[integrations.google_tag_manager]\nenabled = false")); + assert!(draft.contains("Detected google_tag_manager")); + } + + #[test] + fn build_audit_outputs_reconstructs_creative_opportunity_slots() { + let collected = CollectedPage { + requested_url: "https://example.com/".to_string(), + final_url: "https://example.com/".to_string(), + page_title: Some("Example Publisher".to_string()), + html: "".to_string(), + script_tags: Vec::new(), + network_requests: vec![CollectedRequest { + url: "https://securepubads.g.doubleclick.net/gampad/ads?\ + iu_parts=123456789%2Cdesktop%2Chomepage%2Cleaderboard1\ + &prev_iu_szs=970x250%7C4x1%7C620x366\ + &dids=div-gpt-ad-leaderboard-1\ + &prev_scp=baseDivId%3Ddiv-gpt-ad-leaderboard-1%26test%3Dprebid" + .to_string(), + resource_type: Some("fetch".to_string()), + }], + gpt_slots: Vec::new(), + links: Vec::new(), + sitemap_locs: Vec::new(), + warnings: Vec::new(), + }; + + let outputs = build_audit_outputs(&collected).expect("should build outputs"); + assert_eq!(outputs.ad_slot_count, 1, "should discover one slot"); + + // The drafted config must be valid TOML with the reconstructed slot. + let value = toml::from_str::(&outputs.draft_config_toml) + .expect("should parse draft config"); + let creative = &value["creative_opportunities"]; + assert_eq!(creative["gam_network_id"].as_str(), Some("123456789")); + let slot = &creative["slot"][0]; + assert_eq!(slot["id"].as_str(), Some("leaderboard-1")); + assert_eq!( + slot["gam_unit_path"].as_str(), + Some("/123456789/desktop/homepage/leaderboard1") + ); + assert_eq!( + slot["formats"][0]["width"].as_integer(), + Some(970), + "should keep the 970x250 pixel size" + ); + assert!( + slot["providers"]["prebid"].is_table(), + "prev_scp test=prebid should emit a prebid provider" + ); + } + + #[test] + fn render_discovered_slots_escapes_page_controlled_strings() { + // Slot fields scraped from the live page must be escaped so a quote + // cannot inject TOML into the drafted config. + let registry = vec![collector::CollectedGptSlot { + gam_unit_path: "/222/homepage/head\"er".to_string(), + div_id: "div-gpt-ad-head\"er".to_string(), + sizes: vec![(728, 90)], + }]; + let slots = gpt_slots::discover_gpt_slots(®istry, &[], false); + let url = Url::parse("https://publisher.example/").expect("should parse URL"); + + let rendered = render_discovered_slots(&url, &slots); + + let value = toml::from_str::(&rendered) + .expect("should render valid TOML despite embedded quotes"); + let slot = &value["creative_opportunities"]["slot"][0]; + assert_eq!( + slot["div_id"].as_str(), + Some("div-gpt-ad-head\"er"), + "should keep the quote as data, not TOML syntax" + ); + } + + #[test] + fn update_slots_defaults_pattern_to_final_url_after_redirect() { + let temp = TempDir::new().expect("should create temp dir"); + let config_path = temp.path().join("trusted-server.toml"); + fs::write( + &config_path, + "[creative_opportunities]\ngam_network_id = \"111\"\n", + ) + .expect("should write config"); + // The requested URL redirects; slots are scraped from the final page. + let mut collected = collected_page(); + collected.requested_url = "https://publisher.example/".to_string(); + collected.final_url = "https://publisher.example/news/story".to_string(); + collected.gpt_slots = vec![collector::CollectedGptSlot { + gam_unit_path: "/222/homepage/header".to_string(), + div_id: "div-gpt-ad-header".to_string(), + sizes: vec![(728, 90)], + }]; + let collector = FakeCollector::new(collected); + let mut out = Vec::new(); + + run_update_slots( + &UpdateSlotsRequest { + url: "https://publisher.example/", + config_path: &config_path, + existing_creative: None, + page_patterns: &[], + replace: false, + cookies: &[], + dry_run: false, + scroll: false, + budget: CrawlBudget::default(), + }, + &[("desktop", &collector)], + &mut out, + &mut std::io::sink(), + ) + .expect("should update slots"); + + let written = fs::read_to_string(&config_path).expect("should read config"); + let value = toml::from_str::(&written).expect("should parse valid TOML"); + let patterns: Vec<&str> = value["creative_opportunities"]["slot"][0]["page_patterns"] + .as_array() + .expect("should have page_patterns array") + .iter() + .map(|entry| entry.as_str().expect("should have pattern string")) + .collect(); + // Patterns come from the post-redirect path: had the requested `/` been + // used, this would be `["/"]`. They now cover the whole section rather + // than only the one article that happened to be scraped. + assert_eq!( + patterns, + ["/news", "/news/*"], + "should derive section patterns from the post-redirect path" + ); + } + + #[test] + fn update_slots_reports_preserved_unobserved_slots_contextually() { + for (scroll, expected_follow_up, unexpected_follow_up) in [ + (false, "or --scroll", "page/profile coverage"), + (true, "page/profile coverage", "or --scroll"), + ] { + let temp = TempDir::new().expect("should create temp dir"); + let config_path = temp.path().join("trusted-server.toml"); + let mut original = loadable_config() + .replace("gam_network_id = \"123456789\"", "gam_network_id = \"222\""); + original.push_str( + "\n[[creative_opportunities.slot]]\n\ + id = \"header\"\n\ + div_id = \"div-gpt-ad-header\"\n\ + gam_unit_path = \"/222/homepage/header\"\n\ + page_patterns = [\"/\"]\n\ + formats = [{ width = 728, height = 90 }]\n\n\ + [[creative_opportunities.slot]]\n\ + id = \"sidebar\"\n\ + div_id = \"ad-sidebar\"\n\ + gam_unit_path = \"/222/sidebar\"\n\ + page_patterns = [\"/news/*\"]\n\ + formats = [{ width = 300, height = 250 }]\n", + ); + fs::write(&config_path, &original).expect("should write config"); + let existing = crate::commands::audit::creative_config(&original, &config_path) + .expect("should parse config") + .expect("should have creative opportunities"); + let collector = FakeCollector::new(collected_page_with_header_slot()); + let mut out = Vec::new(); + let mut notes = Vec::new(); + + run_update_slots( + &UpdateSlotsRequest { + url: "https://publisher.example/", + config_path: &config_path, + existing_creative: Some(&existing), + page_patterns: &[], + replace: false, + cookies: &[], + dry_run: true, + scroll, + budget: CrawlBudget::default(), + }, + &[("desktop", &collector)], + &mut out, + &mut notes, + ) + .expect("should preserve unobserved slot"); + + let notes = String::from_utf8(notes).expect("notes should be UTF-8"); + assert!( + notes.contains( + "preserved 1 configured slot(s) not observed during this crawl: sidebar" + ), + "should name the preserved slot, got {notes:?}" + ); + assert!( + notes.contains(expected_follow_up), + "should suggest the follow-up matching the scroll setting, got {notes:?}" + ); + assert!( + !notes.contains(unexpected_follow_up), + "should omit the follow-up that does not apply, got {notes:?}" + ); + assert!( + notes.contains("discards every hand-written field"), + "should explain the full cost of --replace, got {notes:?}" + ); + assert!(out.is_empty(), "unchanged dry-run stdout should stay empty"); + assert_eq!( + fs::read_to_string(&config_path).expect("should read config"), + original, + "dry-run should preserve the original config" + ); + } + } + + #[test] + fn observed_but_refused_slot_is_not_reported_as_unobserved() { + let temp = TempDir::new().expect("should create temp dir"); + let config_path = temp.path().join("trusted-server.toml"); + let mut original = loadable_config(); + original.push_str( + "\n[[creative_opportunities.slot]]\n\ + id = \"stable\"\n\ + div_id = \"ad-stable\"\n\ + gam_unit_path = \"/123456789/site/header\"\n\ + page_patterns = [\"/\"]\n\ + formats = [{ width = 728, height = 90 }]\n\n\ + [[creative_opportunities.slot]]\n\ + id = \"ad-refused\"\n\ + gam_unit_path = \"/123456789/desktop/homepage\"\n\ + page_patterns = [\"/\"]\n\ + formats = [{ width = 300, height = 250 }]\n", + ); + fs::write(&config_path, &original).expect("should write config"); + let existing = crate::commands::audit::creative_config(&original, &config_path) + .expect("should parse config") + .expect("should have creative opportunities"); + + let page = |profile: &str| { + let mut page = collected_page(); + page.requested_url = "https://publisher.example/".to_string(); + page.final_url = page.requested_url.clone(); + page.gpt_slots = vec![ + collector::CollectedGptSlot { + gam_unit_path: "/123456789/site/header".to_string(), + div_id: "ad-stable".to_string(), + sizes: vec![(728, 90)], + }, + collector::CollectedGptSlot { + gam_unit_path: format!("/123456789/{profile}/homepage"), + div_id: "ad-refused".to_string(), + sizes: vec![(300, 250)], + }, + ]; + page + }; + let desktop = FakeCollector::new(page("desktop")); + let mobile = FakeCollector::new(page("mobile")); + let mut out = Vec::new(); + let mut notes = Vec::new(); + + run_update_slots( + &UpdateSlotsRequest { + url: "https://publisher.example/", + config_path: &config_path, + existing_creative: Some(&existing), + page_patterns: &[], + replace: false, + cookies: &[], + dry_run: true, + scroll: false, + budget: CrawlBudget::default(), + }, + &[("desktop", &desktop), ("mobile", &mobile)], + &mut out, + &mut notes, + ) + .expect("the accepted slot should let generation complete"); + + let notes = String::from_utf8(notes).expect("notes should be UTF-8"); + assert!( + notes.contains("skipped refused slot `ad-refused` (`ad-refused`)"), + "should retain the refusal diagnostic, got {notes:?}" + ); + assert!( + !notes.contains("not observed during this crawl: ad-refused"), + "a crawl-observed refused slot must not be labeled unobserved, got {notes:?}" + ); + } + + #[test] + fn ambiguous_configured_stem_is_not_reported_as_unobserved() { + let temp = TempDir::new().expect("should create temp dir"); + let config_path = temp.path().join("trusted-server.toml"); + let mut original = + loadable_config().replace("gam_network_id = \"123456789\"", "gam_network_id = \"222\""); + original.push_str( + "\n[[creative_opportunities.slot]]\n\ + id = \"in-content\"\n\ + div_id = \"ad-x\"\n\ + gam_unit_path = \"/222/homepage/in-content\"\n\ + page_patterns = [\"/\"]\n\ + formats = [{ width = 300, height = 250 }]\n", + ); + fs::write(&config_path, &original).expect("should write config"); + let existing = crate::commands::audit::creative_config(&original, &config_path) + .expect("should parse config") + .expect("should have creative opportunities"); + let mut page = collected_page_with_ambiguous_slots("https://publisher.example/"); + page.gpt_slots.push(collector::CollectedGptSlot { + gam_unit_path: "/222/site/header".to_string(), + div_id: "ad-stable".to_string(), + sizes: vec![(728, 90)], + }); + let collector = FakeCollector::new(page); + let mut notes = Vec::new(); + + run_update_slots( + &UpdateSlotsRequest { + url: "https://publisher.example/", + config_path: &config_path, + existing_creative: Some(&existing), + page_patterns: &[], + replace: false, + cookies: &[], + dry_run: true, + scroll: true, + budget: CrawlBudget::default(), + }, + &[("desktop", &collector)], + &mut std::io::sink(), + &mut notes, + ) + .expect("should preserve the configured ambiguous placement"); + + let notes = String::from_utf8(notes).expect("notes should be UTF-8"); + assert!( + notes.contains("skipped ambiguous div-id prefix `ad-x`"), + "should retain the ambiguity diagnostic, got {notes:?}" + ); + assert!( + !notes.contains("not observed during this crawl: in-content"), + "an ambiguity-refused placement must not be labeled unobserved, got {notes:?}" + ); + } + + #[test] + fn volatile_refusals_from_registry_and_requests_keep_prefix_observed() { + for source in ["registry", "request"] { + let temp = TempDir::new().expect("should create temp dir"); + let config_path = temp.path().join("trusted-server.toml"); + let mut original = loadable_config(); + original.push_str( + "\n[[creative_opportunities.slot]]\n\ + id = \"stable\"\n\ + div_id = \"ad-stable\"\n\ + gam_unit_path = \"/123456789/site/header\"\n\ + page_patterns = [\"/\"]\n\ + formats = [{ width = 728, height = 90 }]\n\n\ + [[creative_opportunities.slot]]\n\ + id = \"volatile-family\"\n\ + div_id = \"vendor-tag\"\n\ + gam_unit_path = \"/123456789/site/overlay\"\n\ + page_patterns = [\"/\"]\n\ + formats = [{ width = 300, height = 250 }]\n", + ); + fs::write(&config_path, &original).expect("should write config"); + let existing = crate::commands::audit::creative_config(&original, &config_path) + .expect("should parse config") + .expect("should have creative opportunities"); + let mut page = collected_page(); + page.requested_url = "https://publisher.example/".to_string(); + page.final_url = page.requested_url.clone(); + page.gpt_slots.push(collector::CollectedGptSlot { + gam_unit_path: "/123456789/site/header".to_string(), + div_id: "ad-stable".to_string(), + sizes: vec![(728, 90)], + }); + let volatile_div = "vendor-tag_1724112345678AbCdEfGh_slot_overlay_1"; + if source == "registry" { + page.gpt_slots.push(collector::CollectedGptSlot { + gam_unit_path: "/123456789/site/overlay".to_string(), + div_id: volatile_div.to_string(), + sizes: vec![(300, 250)], + }); + } else { + page.network_requests.push(CollectedRequest { + url: format!( + "https://securepubads.g.doubleclick.net/gampad/ads?\ + iu_parts=123456789%2Csite%2Coverlay&dids={volatile_div}\ + &prev_iu_szs=300x250" + ), + resource_type: Some("fetch".to_string()), + }); + } + let collector = FakeCollector::new(page); + let mut notes = Vec::new(); + + run_update_slots( + &UpdateSlotsRequest { + url: "https://publisher.example/", + config_path: &config_path, + existing_creative: Some(&existing), + page_patterns: &[], + replace: false, + cookies: &[], + dry_run: true, + scroll: true, + budget: CrawlBudget::default(), + }, + &[("desktop", &collector)], + &mut std::io::sink(), + &mut notes, + ) + .expect("the stable slot should let generation complete"); + + let notes = String::from_utf8(notes).expect("notes should be UTF-8"); + assert!( + notes.contains("skipped volatile div-id family `vendor-tag`"), + "should retain the {source} volatile refusal, got {notes:?}" + ); + assert!( + !notes.contains("not observed during this crawl: volatile-family"), + "a live configured prefix refused from {source} evidence must stay observed, got {notes:?}" + ); + } + } + + #[test] + fn static_locale_root_slot_uses_the_planned_section_depth_for_patterns() { + let temp = TempDir::new().expect("should create temp dir"); + let config_path = temp.path().join("trusted-server.toml"); + fs::write( + &config_path, + "[creative_opportunities]\ngam_network_id = \"123456789\"\n", + ) + .expect("should write config"); + let nav = ["/en/news"]; + let mut root_page = site_page("https://publisher.example/en", "/123456789/site/root", &nav); + root_page.gpt_slots[0].div_id = "ad-root-only".to_string(); + let collector = SiteCollector::new(vec![ + ("https://publisher.example/en", root_page), + ( + "https://publisher.example/en/news", + site_page( + "https://publisher.example/en/news", + "/123456789/site/static", + &nav, + ), + ), + ]); + + run_update_slots( + &UpdateSlotsRequest { + url: "https://publisher.example/en", + config_path: &config_path, + existing_creative: None, + page_patterns: &[], + replace: false, + cookies: &[], + dry_run: false, + scroll: false, + budget: CrawlBudget::default(), + }, + &[("desktop", &collector)], + &mut std::io::sink(), + &mut std::io::sink(), + ) + .expect("should write static locale-root slot"); + + let written = fs::read_to_string(&config_path).expect("should read config"); + let value = toml::from_str::(&written).expect("should parse config"); + let slots = value["creative_opportunities"]["slot"] + .as_array() + .expect("should have slots"); + let target = slots + .iter() + .find(|slot| slot["div_id"].as_str() == Some("ad-header-0")) + .expect("should have the section slot"); + let patterns = target["page_patterns"] + .as_array() + .expect("should have patterns") + .iter() + .map(|pattern| pattern.as_str().expect("should be string")) + .collect::>(); + assert_eq!(patterns, ["/en/news", "/en/news/*"]); + } + + #[test] + fn update_slots_rejects_a_cross_origin_root_redirect() { + let temp = TempDir::new().expect("should create temp dir"); + let config_path = temp.path().join("trusted-server.toml"); + let original = "[creative_opportunities]\ngam_network_id = \"111\"\n"; + fs::write(&config_path, original).expect("should write config"); + let mut collected = collected_page_with_header_slot(); + collected.final_url = "https://foreign.example/news".to_string(); + let collector = FakeCollector::new(collected); + + let error = run_update_slots( + &UpdateSlotsRequest { + url: "https://publisher.example/", + config_path: &config_path, + existing_creative: None, + page_patterns: &[], + replace: false, + cookies: &[("session".to_string(), "secret".to_string())], + dry_run: false, + scroll: false, + budget: CrawlBudget::default(), + }, + &[("desktop", &collector)], + &mut std::io::sink(), + &mut std::io::sink(), + ) + .expect_err("cross-origin redirect must leave the requested trust boundary"); + + assert!(format!("{error:?}").contains("cross-origin")); + assert_eq!( + fs::read_to_string(&config_path).expect("should read config"), + original, + "foreign evidence must not rewrite the config" + ); + } + + #[test] + fn update_slots_skips_a_section_page_that_redirects_off_origin() { + // Only the root navigation was origin-checked before planning. A section + // page that redirects away must not contribute its slots, unit paths or + // page patterns to the generated config either. + let temp = TempDir::new().expect("should create temp dir"); + let config_path = temp.path().join("trusted-server.toml"); + fs::write( + &config_path, + "[creative_opportunities]\ngam_network_id = \"123456789\"\n", + ) + .expect("should write config"); + let nav = ["/news"]; + let mut root_page = site_page( + "https://publisher.example/", + "/123456789/site/homepage", + &nav, + ); + root_page.gpt_slots[0].div_id = "ad-root".to_string(); + let mut redirected = site_page( + "https://publisher.example/news", + "/999888777/foreign/news", + &nav, + ); + redirected.final_url = "https://foreign.example/news".to_string(); + redirected.gpt_slots[0].div_id = "ad-foreign".to_string(); + let collector = SiteCollector::new(vec![ + ("https://publisher.example/", root_page), + ("https://publisher.example/news", redirected), + ]); + let mut err = Vec::new(); + + run_update_slots( + &UpdateSlotsRequest { + url: "https://publisher.example/", + config_path: &config_path, + existing_creative: None, + page_patterns: &[], + replace: false, + cookies: &[], + dry_run: false, + budget: CrawlBudget::default(), + scroll: false, + }, + &[("desktop", &collector)], + &mut std::io::sink(), + &mut err, + ) + .expect("should generate from the same-origin evidence alone"); + + let written = fs::read_to_string(&config_path).expect("should read config"); + assert!( + written.contains("ad-root"), + "same-origin evidence should still be written, got:\n{written}" + ); + assert!( + !written.contains("ad-foreign") && !written.contains("999888777"), + "the redirect destination must not reach the config, got:\n{written}" + ); + let progress = String::from_utf8_lossy(&err); + assert!( + progress.contains( + "skipped `/news` on desktop: it left the audited origin for https://foreign.example" + ), + "the skipped section page should be reported, got:\n{progress}" + ); + } + + #[test] + fn update_slots_skips_a_later_profile_root_that_redirects_off_origin() { + // The later profiles re-walk the plan without a fresh root origin check. + // A mobile root that redirects away carries a foreign ad unit for the + // same div the desktop profile saw; folding it would both write foreign + // inventory and fake a device disagreement on the real slot. + let temp = TempDir::new().expect("should create temp dir"); + let config_path = temp.path().join("trusted-server.toml"); + fs::write( + &config_path, + "[creative_opportunities]\ngam_network_id = \"123456789\"\n", + ) + .expect("should write config"); + let nav = ["/news"]; + let section_page = |unit_path: &str| { + let mut page = site_page("https://publisher.example/news", unit_path, &nav); + page.gpt_slots[0].div_id = "ad-news".to_string(); + page + }; + let mut desktop_root = site_page( + "https://publisher.example/", + "/123456789/site/homepage", + &nav, + ); + desktop_root.gpt_slots[0].div_id = "ad-root".to_string(); + let mut mobile_root = site_page( + "https://publisher.example/", + "/999888777/foreign/homepage", + &nav, + ); + mobile_root.gpt_slots[0].div_id = "ad-root".to_string(); + mobile_root.final_url = "https://foreign.example/".to_string(); + let desktop = SiteCollector::new(vec![ + ("https://publisher.example/", desktop_root), + ( + "https://publisher.example/news", + section_page("/123456789/site/news"), + ), + ]); + let mobile = SiteCollector::new(vec![ + ("https://publisher.example/", mobile_root), + ( + "https://publisher.example/news", + section_page("/123456789/site/news"), + ), + ]); + let mut err = Vec::new(); + + run_update_slots( + &UpdateSlotsRequest { + url: "https://publisher.example/", + config_path: &config_path, + existing_creative: None, + page_patterns: &[], + replace: false, + cookies: &[], + dry_run: false, + budget: CrawlBudget::default(), + scroll: false, + }, + &[("desktop", &desktop), ("mobile", &mobile)], + &mut std::io::sink(), + &mut err, + ) + .expect("the same-origin pages of both profiles agree"); + + let written = fs::read_to_string(&config_path).expect("should read config"); + assert!( + written.contains("/123456789/site/homepage"), + "the same-origin root unit path should be written, got:\n{written}" + ); + assert!( + !written.contains("999888777"), + "the redirect destination must not reach the config, got:\n{written}" + ); + let progress = String::from_utf8_lossy(&err); + assert!( + progress.contains( + "skipped `/` on mobile: it left the audited origin for https://foreign.example" + ), + "the skipped profile root should be reported, got:\n{progress}" + ); + } + + #[test] + fn update_slots_accepts_a_same_host_https_upgrade() { + // The ordinary canonical redirect: an operator types the bare http URL + // and the site upgrades it. The host is unchanged, so the cookie and + // audit trust boundary is unchanged, and generation must not stall on it. + let temp = TempDir::new().expect("should create temp dir"); + let config_path = temp.path().join("trusted-server.toml"); + fs::write( + &config_path, + "[creative_opportunities]\ngam_network_id = \"111\"\n", + ) + .expect("should write config"); + let mut collected = collected_page_with_header_slot(); + collected.requested_url = "http://publisher.example/".to_string(); + collected.final_url = "https://publisher.example/".to_string(); + let collector = FakeCollector::new(collected); + let mut notes = Vec::new(); + + run_update_slots( + &UpdateSlotsRequest { + url: "http://publisher.example/", + config_path: &config_path, + existing_creative: None, + page_patterns: &[], + replace: false, + cookies: &[("session".to_string(), "secret".to_string())], + dry_run: false, + scroll: false, + budget: CrawlBudget::default(), + }, + &[("desktop", &collector)], + &mut std::io::sink(), + &mut notes, + ) + .expect("a same-host HTTPS upgrade should not be treated as cross-origin"); + + let written = fs::read_to_string(&config_path).expect("should read config"); + let value = toml::from_str::(&written).expect("should parse config"); + assert_eq!( + value["creative_opportunities"]["slot"][0]["div_id"].as_str(), + Some("div-gpt-ad-header"), + "evidence from the upgraded root should be written" + ); + let notes = String::from_utf8(notes).expect("notes should be UTF-8"); + assert!( + notes.contains( + "followed a root redirect from `http://publisher.example/` to \ + `https://publisher.example/`" + ), + "an accepted redirect should say the run switched URLs, got {notes:?}" + ); + } + + #[test] + fn update_slots_rejects_an_https_downgrade_root_redirect() { + // The mirror image of the accepted upgrade: same host, but dropping TLS + // leaves the requested trust boundary and must still be refused. + let temp = TempDir::new().expect("should create temp dir"); + let config_path = temp.path().join("trusted-server.toml"); + let original = "[creative_opportunities]\ngam_network_id = \"111\"\n"; + fs::write(&config_path, original).expect("should write config"); + let mut collected = collected_page_with_header_slot(); + collected.final_url = "http://publisher.example/".to_string(); + let collector = FakeCollector::new(collected); + + let error = run_update_slots( + &UpdateSlotsRequest { + url: "https://publisher.example/", + config_path: &config_path, + existing_creative: None, + page_patterns: &[], + replace: false, + cookies: &[("session".to_string(), "secret".to_string())], + dry_run: false, + scroll: false, + budget: CrawlBudget::default(), + }, + &[("desktop", &collector)], + &mut std::io::sink(), + &mut std::io::sink(), + ) + .expect_err("an HTTPS downgrade must leave the requested trust boundary"); + + assert!(format!("{error:?}").contains("cross-origin")); + assert_eq!( + fs::read_to_string(&config_path).expect("should read config"), + original, + "downgraded evidence must not rewrite the config" + ); + } + + #[test] + fn update_slots_requires_evidence_from_every_selected_profile() { + let temp = TempDir::new().expect("should create temp dir"); + let config_path = temp.path().join("trusted-server.toml"); + let original = loadable_config(); + fs::write(&config_path, &original).expect("should write config"); + let desktop = FakeCollector::new(collected_page_with_header_slot()); + + let error = run_update_slots( + &UpdateSlotsRequest { + url: "https://publisher.example/", + config_path: &config_path, + existing_creative: None, + page_patterns: &[], + replace: false, + cookies: &[], + dry_run: false, + scroll: false, + budget: CrawlBudget::default(), + }, + &[("desktop", &desktop), ("mobile", &FailingCollector)], + &mut std::io::sink(), + &mut std::io::sink(), + ) + .expect_err("a selected profile with no usable page must refuse generation"); + + assert!(format!("{error:?}").contains("mobile")); + assert_eq!( + fs::read_to_string(&config_path).expect("should read config"), + original, + "incomplete profile coverage must not rewrite the config" + ); + } + + #[test] + fn update_slots_rejects_invalid_page_pattern_without_touching_config() { + let temp = TempDir::new().expect("should create temp dir"); + let config_path = temp.path().join("trusted-server.toml"); + let original = "[creative_opportunities]\ngam_network_id = \"111\"\n"; + fs::write(&config_path, original).expect("should write config"); + let collector = FakeCollector::new(collected_page_with_header_slot()); + let mut out = Vec::new(); + + let error = run_update_slots( + &UpdateSlotsRequest { + url: "https://publisher.example/", + config_path: &config_path, + existing_creative: None, + page_patterns: &["[".to_string()], + replace: false, + cookies: &[], + dry_run: false, + scroll: false, + budget: CrawlBudget::default(), + }, + &[("desktop", &collector)], + &mut out, + &mut std::io::sink(), + ) + .expect_err("should reject an invalid glob"); + + assert!( + format!("{error:?}").contains("page pattern '['"), + "error should name the offending pattern, got {error:?}" + ); + assert_eq!( + fs::read_to_string(&config_path).expect("should read config"), + original, + "a rejected pattern must leave the operator config untouched" + ); + } + + #[test] + fn explicit_page_patterns_refuse_a_template_that_borrows_section_root() { + let temp = TempDir::new().expect("should create temp dir"); + let config_path = temp.path().join("trusted-server.toml"); + let original = loadable_config(); + fs::write(&config_path, &original).expect("should write config"); + + let nav = ["/news", "/deals"]; + let root = site_page( + "https://publisher.example/", + "/123456789/site/homepage", + &nav, + ); + let mut news = site_page( + "https://publisher.example/news", + "/123456789/site/news", + &nav, + ); + news.gpt_slots.push(collector::CollectedGptSlot { + gam_unit_path: "/123456789/site/news".to_string(), + div_id: "ad-sidebar".to_string(), + sizes: vec![(300, 250)], + }); + let mut deals = site_page( + "https://publisher.example/deals", + "/123456789/site/deals", + &nav, + ); + deals.gpt_slots.push(collector::CollectedGptSlot { + gam_unit_path: "/123456789/site/deals".to_string(), + div_id: "ad-sidebar".to_string(), + sizes: vec![(300, 250)], + }); + let collector = SiteCollector::new(vec![ + ("https://publisher.example/", root), + ("https://publisher.example/news", news), + ("https://publisher.example/deals", deals), + ]); + + let error = run_update_slots( + &UpdateSlotsRequest { + url: "https://publisher.example/", + config_path: &config_path, + existing_creative: None, + page_patterns: &["/".to_string(), "/*".to_string()], + replace: false, + cookies: &[], + dry_run: false, + scroll: false, + budget: CrawlBudget::default(), + }, + &[("desktop", &collector)], + &mut std::io::sink(), + &mut std::io::sink(), + ) + .expect_err("explicit patterns cannot preserve borrowed-root safety"); + + let message = format!("{error:?}"); + assert!(message.contains("--page-pattern"), "got {message}"); + assert!(message.contains("ad-sidebar"), "got {message}"); + assert_eq!( + fs::read_to_string(&config_path).expect("should read config"), + original, + "a refused override must leave the config unchanged" + ); + } + + #[test] + fn update_slots_accepts_double_star_pattern_like_the_runtime() { + // `/20**` does not compile directly but the runtime normalises it to + // `/20*`; validation must accept exactly what the runtime accepts. + let temp = TempDir::new().expect("should create temp dir"); + let config_path = temp.path().join("trusted-server.toml"); + fs::write( + &config_path, + "[creative_opportunities]\ngam_network_id = \"111\"\n", + ) + .expect("should write config"); + let collector = FakeCollector::new(collected_page_with_header_slot()); + let mut out = Vec::new(); + + run_update_slots( + &UpdateSlotsRequest { + url: "https://publisher.example/", + config_path: &config_path, + existing_creative: None, + page_patterns: &["/20**".to_string()], + replace: false, + cookies: &[], + dry_run: false, + scroll: false, + budget: CrawlBudget::default(), + }, + &[("desktop", &collector)], + &mut out, + &mut std::io::sink(), + ) + .expect("should accept a runtime-normalisable pattern"); + + let written = fs::read_to_string(&config_path).expect("should read config"); + let value = toml::from_str::(&written).expect("valid TOML"); + assert_eq!( + value["creative_opportunities"]["slot"][0]["page_patterns"][0].as_str(), + Some("/20**") + ); + } + + #[test] + fn update_slots_write_replaces_the_config_without_leaving_temp_files() { + let temp = TempDir::new().expect("should create temp dir"); + let config_path = temp.path().join("trusted-server.toml"); + fs::write( + &config_path, + "[creative_opportunities]\ngam_network_id = \"111\"\n", + ) + .expect("should write config"); + let collector = FakeCollector::new(collected_page_with_header_slot()); + let mut out = Vec::new(); + + run_update_slots( + &UpdateSlotsRequest { + url: "https://publisher.example/", + config_path: &config_path, + existing_creative: None, + page_patterns: &[], + replace: false, + cookies: &[], + dry_run: false, + scroll: false, + budget: CrawlBudget::default(), + }, + &[("desktop", &collector)], + &mut out, + &mut std::io::sink(), + ) + .expect("should update slots"); + + let entries: Vec = fs::read_dir(temp.path()) + .expect("should read temp dir") + .map(|entry| { + entry + .expect("should read entry") + .file_name() + .to_string_lossy() + .into_owned() + }) + .collect(); + assert_eq!( + entries, + ["trusted-server.toml"], + "the atomic write should leave no stray temp file behind" + ); + let written = fs::read_to_string(&config_path).expect("should read config"); + toml::from_str::(&written).expect("rewritten config is valid TOML"); + } + + /// A full, loadable config with real secrets substituted, so the write-side + /// validation gate is live rather than downgraded by a broken baseline. + fn loadable_config() -> String { + EXAMPLE_CONFIG + .replace( + "password = \"handler_password\"", + "password = \"test-admin-password-32-bytes-minimum\"", + ) + .replace( + "passphrase = \"ec_passphrase\"", + "passphrase = \"test-ec-passphrase-32-bytes-minimum\"", + ) + .replace( + "proxy_secret = \"publisher_proxy_secret\"", + "proxy_secret = \"test-proxy-secret-32-bytes-minimum\"", + ) + } + + #[test] + fn a_crawl_writes_a_section_template_and_per_section_patterns() { + // The end-to-end payoff: crawl sections, reconcile the slot across them, + // infer `{section}`, and write a config the runtime loads. + let temp = TempDir::new().expect("should create temp dir"); + let config_path = temp.path().join("trusted-server.toml"); + let original = loadable_config(); + fs::write(&config_path, &original).expect("should write config"); + + let nav = ["/news", "/deals"]; + let collector = SiteCollector::new(vec![ + ( + "https://publisher.example/", + site_page( + "https://publisher.example/", + "/123456789/site/homepage", + &nav, + ), + ), + ( + "https://publisher.example/news", + site_page( + "https://publisher.example/news", + "/123456789/site/news", + &nav, + ), + ), + ( + "https://publisher.example/deals", + site_page( + "https://publisher.example/deals", + "/123456789/site/deals", + &nav, + ), + ), + ]); + let mut out = Vec::new(); + let mut err = Vec::new(); + + run_update_slots( + &UpdateSlotsRequest { + url: "https://publisher.example/", + config_path: &config_path, + existing_creative: None, + page_patterns: &[], + replace: false, + cookies: &[], + dry_run: false, + scroll: false, + budget: CrawlBudget::default(), + }, + &[("desktop", &collector)], + &mut out, + &mut err, + ) + .expect("should crawl and update slots"); + + let written = fs::read_to_string(&config_path).expect("should read config"); + let value = toml::from_str::(&written).expect("valid TOML"); + let creative = &value["creative_opportunities"]; + + assert_eq!( + creative["section_root"].as_str(), + Some("homepage"), + "the unvisited-section fallback should come from the root page" + ); + assert_eq!(creative["section_segment"].as_integer(), Some(0)); + let slot = &creative["slot"][0]; + assert_eq!( + slot["gam_unit_path"].as_str(), + Some("/{network_id}/site/{section}"), + "the varying segment should become a template" + ); + let patterns: Vec<&str> = slot["page_patterns"] + .as_array() + .expect("patterns array") + .iter() + .map(|entry| entry.as_str().expect("pattern")) + .collect(); + assert_eq!( + patterns, + ["/", "/deals", "/deals/*", "/news", "/news/*"], + "each witnessed section should contribute both halves of its pair" + ); + + // The whole point of the gate: what was written must actually load. + trusted_server_core::settings::Settings::from_toml(&written) + .expect("generated config must load through the runtime path"); + + let report = String::from_utf8(err).expect("should produce UTF-8 output"); + assert!( + report.contains("Deploy a template-aware binary BEFORE pushing"), + "a templated config must warn about the rollback contract, got:\n{report}" + ); + } + + #[test] + fn disagreeing_device_profiles_refuse_to_write_a_unit_path() { + // Two profiles serving different ad units for the same page is exactly + // the failure a single-profile crawl cannot see. Writing either path + // would be correct for one device and silently wrong for the other. + let temp = TempDir::new().expect("should create temp dir"); + let config_path = temp.path().join("trusted-server.toml"); + let original = loadable_config(); + fs::write(&config_path, &original).expect("should write config"); + + let nav = ["/news"]; + let desktop = SiteCollector::new(vec![ + ( + "https://publisher.example/", + site_page( + "https://publisher.example/", + "/123456789/desktop/homepage", + &nav, + ), + ), + ( + "https://publisher.example/news", + site_page( + "https://publisher.example/news", + "/123456789/desktop/news", + &nav, + ), + ), + ]); + let mobile = SiteCollector::new(vec![ + ( + "https://publisher.example/", + site_page( + "https://publisher.example/", + "/123456789/mobile/homepage", + &nav, + ), + ), + ( + "https://publisher.example/news", + site_page( + "https://publisher.example/news", + "/123456789/mobile/news", + &nav, + ), + ), + ]); + let mut out = Vec::new(); + let mut err = Vec::new(); + + let error = run_update_slots( + &UpdateSlotsRequest { + url: "https://publisher.example/", + config_path: &config_path, + existing_creative: None, + page_patterns: &[], + replace: false, + cookies: &[], + dry_run: false, + scroll: false, + budget: CrawlBudget::default(), + }, + &[("desktop", &desktop), ("mobile", &mobile)], + &mut out, + &mut err, + ) + .expect_err("an all-refused crawl must not write an empty slot array"); + + assert!(format!("{error:?}").contains("zero generated slots")); + let progress = String::from_utf8_lossy(&err); + for expected in [ + "Auditing desktop [1/?]: /", + "Auditing desktop: planning site crawl", + "Auditing desktop [2/2]: /news", + "Auditing mobile [1/2]: /", + "Auditing mobile [2/2]: /news", + ] { + assert!( + progress.contains(expected), + "should report `{expected}` while crawling, got:\n{progress}" + ); + } + assert!( + !String::from_utf8_lossy(&out).contains("Auditing "), + "progress must remain on stderr" + ); + assert!( + progress.contains("skipped refused slot"), + "the refusal reason should be reported" + ); + assert_eq!( + fs::read_to_string(&config_path).expect("read config"), + original, + "a refused crawl must preserve the operator config" + ); + } + + #[test] + fn a_root_only_site_is_still_collected_on_every_device_profile() { + // A site whose root offers no crawl targets is audited on the root page + // alone. If the later profiles never load it, a device split there is + // invisible and the first profile's literal path gets written as if + // every device agreed with it. + let temp = TempDir::new().expect("should create temp dir"); + let config_path = temp.path().join("trusted-server.toml"); + let original = loadable_config(); + fs::write(&config_path, &original).expect("should write config"); + + let desktop = SiteCollector::new(vec![( + "https://publisher.example/", + site_page( + "https://publisher.example/", + "/123456789/desktop/homepage", + &[], + ), + )]); + let mobile = SiteCollector::new(vec![( + "https://publisher.example/", + site_page( + "https://publisher.example/", + "/123456789/mobile/homepage", + &[], + ), + )]); + let mut out = Vec::new(); + + let error = run_update_slots( + &UpdateSlotsRequest { + url: "https://publisher.example/", + config_path: &config_path, + existing_creative: None, + page_patterns: &[], + replace: false, + cookies: &[], + dry_run: false, + scroll: false, + budget: CrawlBudget::default(), + }, + &[("desktop", &desktop), ("mobile", &mobile)], + &mut out, + &mut std::io::sink(), + ) + .expect_err("an all-refused crawl must not write an empty slot array"); + + assert_eq!( + mobile.visited.borrow().as_slice(), + ["https://publisher.example/"], + "the mobile profile must load the root even when there is nothing else to crawl" + ); + assert!(format!("{error:?}").contains("zero generated slots")); + assert_eq!( + fs::read_to_string(&config_path).expect("read config"), + original, + "a root-only refusal must preserve the operator config" + ); + } + + #[test] + fn a_crawl_refuses_when_most_pages_are_challenged() { + // Bot protection serves an interstitial that loads fine and has no ad + // stack, so it looks like a page with no slots. Writing from that would + // silently narrow the operator's slot set. + let temp = TempDir::new().expect("should create temp dir"); + let config_path = temp.path().join("trusted-server.toml"); + let original = loadable_config(); + fs::write(&config_path, &original).expect("should write config"); + + let nav = ["/news", "/deals"]; + let mut blocked_news = site_page("https://publisher.example/news", "/123456789/x", &nav); + blocked_news.gpt_slots.clear(); + let mut blocked_deals = site_page("https://publisher.example/deals", "/123456789/x", &nav); + blocked_deals.gpt_slots.clear(); + let collector = SiteCollector::new(vec![ + ( + "https://publisher.example/", + site_page( + "https://publisher.example/", + "/123456789/site/homepage", + &nav, + ), + ), + ("https://publisher.example/news", blocked_news), + ("https://publisher.example/deals", blocked_deals), + ]); + let mut out = Vec::new(); + + let error = run_update_slots( + &UpdateSlotsRequest { + url: "https://publisher.example/", + config_path: &config_path, + existing_creative: None, + page_patterns: &[], + replace: false, + cookies: &[], + dry_run: false, + scroll: false, + budget: CrawlBudget::default(), + }, + &[("desktop", &collector)], + &mut out, + &mut std::io::sink(), + ) + .expect_err("a mostly-challenged crawl should refuse"); + + assert!( + format!("{error:?}").contains("bot protection"), + "the error should name the likely cause, got {error:?}" + ); + assert_eq!( + fs::read_to_string(&config_path).expect("read config"), + original, + "a refused run must leave the config untouched" + ); + } + + #[test] + fn max_pages_one_restores_single_page_behavior() { + let temp = TempDir::new().expect("should create temp dir"); + let config_path = temp.path().join("trusted-server.toml"); + fs::write(&config_path, loadable_config()).expect("should write config"); + + let nav = ["/news", "/deals"]; + let collector = SiteCollector::new(vec![( + "https://publisher.example/", + site_page( + "https://publisher.example/", + "/123456789/site/homepage", + &nav, + ), + )]); + let mut out = Vec::new(); + + run_update_slots( + &UpdateSlotsRequest { + url: "https://publisher.example/", + config_path: &config_path, + existing_creative: None, + page_patterns: &[], + replace: false, + cookies: &[], + dry_run: false, + scroll: false, + budget: CrawlBudget { + max_sections: 8, + max_pages: 1, + }, + }, + &[("desktop", &collector)], + &mut out, + &mut std::io::sink(), + ) + .expect("should update from the single page"); + + assert_eq!( + collector.visited.borrow().len(), + 1, + "max_pages = 1 must not crawl beyond the requested page" + ); + let written = fs::read_to_string(&config_path).expect("read config"); + let value = toml::from_str::(&written).expect("valid TOML"); + assert!( + value["creative_opportunities"] + .get("section_root") + .is_none(), + "one page cannot witness a section, so no rollback-fatal key may be written" + ); + assert_eq!( + value["creative_opportunities"]["slot"][0]["gam_unit_path"].as_str(), + Some("/123456789/site/homepage"), + "a single page keeps the literal path" + ); + } + + #[test] + fn generated_config_loads_through_the_runtime_settings_path() { + // The end-to-end contract: whatever `generate` writes must survive the + // same load path the adapter runs at startup. An unloadable config is a + // full-site outage once pushed, not a degraded ad stack. + let temp = TempDir::new().expect("should create temp dir"); + let config_path = temp.path().join("trusted-server.toml"); + let baseline = loadable_config(); + trusted_server_core::settings::Settings::from_toml(&baseline) + .expect("test baseline must itself be loadable or the gate is not exercised"); + fs::write(&config_path, &baseline).expect("should write config"); + let collector = FakeCollector::new(collected_page_with_header_slot()); + let mut out = Vec::new(); + + run_update_slots( + &UpdateSlotsRequest { + url: "https://publisher.example/", + config_path: &config_path, + existing_creative: None, + page_patterns: &[], + replace: false, + cookies: &[], + dry_run: false, + scroll: false, + budget: CrawlBudget::default(), + }, + &[("desktop", &collector)], + &mut out, + &mut std::io::sink(), + ) + .expect("should update slots"); + + let written = fs::read_to_string(&config_path).expect("should read config"); + let settings = trusted_server_core::settings::Settings::from_toml(&written) + .expect("generated config must load through the runtime path"); + let creative = settings + .creative_opportunities + .expect("generated config should carry creative opportunities"); + assert_eq!( + creative.slot.len(), + 1, + "the discovered slot should be present after a real load" + ); + assert_eq!( + creative.slot[0].div_id.as_deref(), + Some("div-gpt-ad-header") + ); + } + + #[test] + fn update_slots_dry_run_does_not_persist_environment_overlay_config() { + let temp = TempDir::new().expect("should create temp dir"); + let manifest_path = temp.path().join("edgezero.toml"); + let config_path = temp.path().join("trusted-server.toml"); + fs::write(&manifest_path, "[app]\nname = \"trusted-server\"\n") + .expect("should write manifest"); + let config = EXAMPLE_CONFIG + .replace( + "password = \"handler_password\"", + "password = \"test-admin-password-32-bytes-minimum\"", + ) + .replace( + "passphrase = \"ec_passphrase\"", + "passphrase = \"test-ec-passphrase-32-bytes-minimum\"", + ) + .replace( + "proxy_secret = \"publisher_proxy_secret\"", + "proxy_secret = \"test-proxy-secret-32-bytes-minimum\"", + ); + let config = format!( + "{config}\n\ + [[creative_opportunities.slot]]\n\ + id = \"file-only\"\n\ + div_id = \"div-gpt-ad-file\"\n\ + gam_unit_path = \"/123456789/homepage/file\"\n\ + page_patterns = [\"/\"]\n\ + formats = [{{ width = 728, height = 90 }}]\n" + ); + fs::write(&config_path, &config).expect("should write config"); + let args = AppConfigArgs { + app_config: Some(config_path.clone()), + manifest: manifest_path, + no_env: false, + }; + + temp_env::with_var( + "TRUSTED_SERVER__CREATIVE_OPPORTUNITIES__GAM_NETWORK_ID", + Some("987654321"), + || { + let effective = crate::app_config::load_settings(&args) + .expect("should load effective settings"); + assert_eq!( + effective + .settings + .creative_opportunities + .as_ref() + .expect("should have creative config") + .gam_network_id, + "987654321", + "test environment should override the network id" + ); + let loaded = crate::app_config::load_file_settings(&args) + .expect("should load file-only settings"); + let mut collected = collected_page(); + collected.gpt_slots = vec![collector::CollectedGptSlot { + gam_unit_path: "/123456789/homepage/file".to_string(), + div_id: "div-gpt-ad-file".to_string(), + sizes: vec![(728, 90)], + }]; + let collector = FakeCollector::new(collected); + let mut out = Vec::new(); + + run_update_slots( + &UpdateSlotsRequest { + url: "https://publisher.example/", + config_path: &loaded.app_config_path, + existing_creative: loaded.settings.creative_opportunities.as_ref(), + page_patterns: &[], + replace: false, + cookies: &[], + dry_run: true, + scroll: false, + budget: CrawlBudget::default(), + }, + &[("desktop", &collector)], + &mut out, + &mut std::io::sink(), + ) + .expect("should render dry-run update"); + + let output = String::from_utf8(out).expect("output should be UTF-8"); + assert!(output.starts_with("--- configured creative opportunities\n")); + assert!(output.contains("+++ generated creative opportunities\n")); + assert!( + !output.contains("test-admin-password-32-bytes-minimum"), + "dry run must not expose unrelated secrets" + ); + assert!( + !output.contains("987654321"), + "dry run must not persist environment-only config" + ); + assert_eq!( + fs::read_to_string(&config_path).expect("should re-read config"), + config, + "dry run must not modify the config file" + ); + }, + ); + } + + #[test] + fn update_slots_refuses_to_overwrite_a_config_changed_during_collection() { + let temp = TempDir::new().expect("should create temp dir"); + let config_path = temp.path().join("trusted-server.toml"); + let original = loadable_config(); + let replacement = format!("{original}\n# edited while the browser was running\n"); + fs::write(&config_path, &original).expect("should write config"); + let collector = MutatingCollector { + collected: collected_page_with_header_slot(), + config_path: config_path.clone(), + replacement: replacement.clone(), + }; + + let error = run_update_slots( + &UpdateSlotsRequest { + url: "https://publisher.example/", + config_path: &config_path, + existing_creative: None, + page_patterns: &[], + replace: false, + cookies: &[], + dry_run: false, + scroll: false, + budget: CrawlBudget::default(), + }, + &[("desktop", &collector)], + &mut std::io::sink(), + &mut std::io::sink(), + ) + .expect_err("a stale update should be refused"); + + assert!(format!("{error:?}").contains("changed during the browser audit")); + assert_eq!( + fs::read_to_string(&config_path).expect("should re-read config"), + replacement, + "the concurrent edit must not be overwritten" + ); + } +} diff --git a/crates/trusted-server-cli/src/commands/audit/generate/page_patterns.rs b/crates/trusted-server-cli/src/commands/audit/generate/page_patterns.rs new file mode 100644 index 000000000..740acc86f --- /dev/null +++ b/crates/trusted-server-cli/src/commands/audit/generate/page_patterns.rs @@ -0,0 +1,150 @@ +//! Derives `page_patterns` globs from the paths a slot was actually observed on. +//! +//! A slot seen on `/news/story-abc` should serve every article in that section, +//! not just that one URL — but nothing here extrapolates beyond a *witnessed* +//! section. Each observed path contributes the section prefix it belongs to and +//! nothing else, so a crawl that never visited `/reviews` never claims it. +//! +//! Each section yields a pair, because one glob cannot cover both halves: +//! `*` crosses `/` in this glob dialect, so `/news/*` matches `/news/a/b` but +//! **not** the bare `/news` landing page. Emitting only the star form silently +//! drops the landing page from the slot. + +use std::collections::BTreeSet; + +/// The root pattern, matching only the site root. +const ROOT_PATTERN: &str = "/"; + +/// Expands observed page paths into the glob set a slot should carry. +/// +/// `section_segment` is the index the section is taken from, matching the +/// config key of the same name: a path is reduced to its first +/// `section_segment + 1` segments, which is the prefix every page of that +/// section shares. A shorter observed landing path is emitted literally; only +/// the actual site root contributes `/`. +/// +/// Results are deduplicated and ordered with `/` first, then alphabetically, so +/// re-running against unchanged evidence produces an unchanged file. +pub(super) fn patterns_for_paths<'a>( + paths: impl IntoIterator, + section_segment: usize, +) -> Vec { + let mut patterns: BTreeSet = BTreeSet::new(); + let mut has_root = false; + + for path in paths { + let segments: Vec<&str> = path.split('/').filter(|part| !part.is_empty()).collect(); + if segments.len() <= section_segment { + if segments.is_empty() { + has_root = true; + } else { + patterns.insert(glob::Pattern::escape(path)); + } + continue; + } + let prefix = glob::Pattern::escape(&format!("/{}", segments[..=section_segment].join("/"))); + // The landing page and everything beneath it. + patterns.insert(prefix.clone()); + patterns.insert(format!("{prefix}/*")); + } + + let mut out = Vec::with_capacity(patterns.len() + usize::from(has_root)); + if has_root { + out.push(ROOT_PATTERN.to_string()); + } + out.extend(patterns); + out +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_section_article_yields_both_halves_of_the_pair() { + // `/news/*` alone would not match the bare `/news` landing page, because + // `*` crosses `/` but does not match the empty remainder. + let patterns = patterns_for_paths(["/news/story-abc"], 0); + + assert_eq!(patterns, ["/news", "/news/*"]); + } + + #[test] + fn the_root_path_contributes_the_root_pattern_first() { + let patterns = patterns_for_paths(["/deals/x", "/", "/news/y"], 0); + + assert_eq!( + patterns, + ["/", "/deals", "/deals/*", "/news", "/news/*"], + "root first, then sections alphabetically" + ); + } + + #[test] + fn a_landing_page_and_its_article_collapse_to_one_pair() { + let patterns = patterns_for_paths(["/news", "/news/story-abc"], 0); + + assert_eq!(patterns, ["/news", "/news/*"], "no duplicate entries"); + } + + #[test] + fn a_locale_prefixed_site_keeps_the_locale_in_the_prefix() { + // section_segment = 1 means the section is the second segment, so the + // shared prefix every page of that section carries includes the locale. + let patterns = patterns_for_paths(["/en/news/story", "/en/deals/x", "/en"], 1); + + assert_eq!( + patterns, + ["/en", "/en/deals", "/en/deals/*", "/en/news", "/en/news/*"] + ); + } + + #[test] + fn literal_glob_metacharacters_are_escaped_and_match_the_source() { + let source = "/news[local]/story"; + let patterns = patterns_for_paths([source], 0); + + assert_eq!(patterns, ["/news[[]local[]]", "/news[[]local[]]/*"]); + assert!(patterns.iter().any(|pattern| { + glob::Pattern::new(pattern) + .expect("should compile emitted glob") + .matches(source) + })); + } + + #[test] + fn unwitnessed_sections_are_never_invented() { + let patterns = patterns_for_paths(["/news/story"], 0); + + assert_eq!( + patterns, + ["/news", "/news/*"], + "only the crawled section may appear" + ); + } + + #[test] + fn output_is_stable_regardless_of_input_order() { + let one = patterns_for_paths(["/news/a", "/deals/b", "/"], 0); + let two = patterns_for_paths(["/", "/deals/b", "/news/a"], 0); + + assert_eq!(one, two, "re-running should not reorder the written file"); + } + + #[test] + fn every_emitted_pattern_compiles_as_a_runtime_glob() { + let patterns = patterns_for_paths(["/", "/news/story", "/site-news/x"], 0); + + for pattern in &patterns { + trusted_server_core::creative_opportunities::validate_page_pattern(pattern) + .unwrap_or_else(|error| { + panic!("emitted pattern `{pattern}` must compile: {error}") + }); + } + } + + #[test] + fn no_paths_yield_no_patterns() { + assert!(patterns_for_paths([], 0).is_empty()); + } +} diff --git a/crates/trusted-server-cli/src/commands/audit/generate/slot_toml.rs b/crates/trusted-server-cli/src/commands/audit/generate/slot_toml.rs new file mode 100644 index 000000000..a7b4f1c58 --- /dev/null +++ b/crates/trusted-server-cli/src/commands/audit/generate/slot_toml.rs @@ -0,0 +1,2341 @@ +//! TOML-side slot config: the [`RenderSlot`] model, run merging, rendering, +//! and in-place `[creative_opportunities]` splicing for `ts audit ad-templates +//! generate`. + +use std::collections::{BTreeMap, BTreeSet}; + +use toml_edit::{DocumentMut, Item, Table}; +use trusted_server_core::auction::types::MediaType; +use trusted_server_core::creative_opportunities::{ + CreativeOpportunitiesConfig, CreativeOpportunitySlot, +}; + +#[cfg(test)] +use crate::commands::audit::generate::gpt_slots; +use crate::error::{CliResult, cli_error, report_error}; + +/// A slot ready to render — the union of discovered and existing fields, without +/// the core type's `pub(crate)` compiled-pattern cache. +#[derive(Debug, Clone)] +pub(super) struct RenderSlot { + id: String, + div_id: Option, + gam_unit_path: Option, + page_patterns: Vec, + /// `(width, height, non-banner media type)`. + formats: Vec<(u32, u32, Option<&'static str>)>, + floor_price: Option, + targeting: BTreeMap, + aps_slot_id: Option, + /// `Some` when the slot runs Prebid; the map is per-bidder params (often empty). + prebid_bidders: Option>, +} + +impl RenderSlot { + /// The stable exact identity fallback used when no configured div prefix + /// matches a discovered slot. + fn key(&self) -> String { + self.div_id + .as_deref() + .unwrap_or(&self.id) + .trim_end_matches('-') + .to_string() + } + + /// Whether this configured slot carries fields that discovery cannot infer. + fn has_tuned_fields(&self) -> bool { + self.floor_price.is_some() + || !self.targeting.is_empty() + || self.aps_slot_id.is_some() + || self.prebid_bidders.is_some() + } + + /// Builds a slot from one page's discovery. + /// + /// Superseded in production by [`RenderSlot::from_evidence`], which reads + /// cross-page evidence; retained as test scaffolding for the merge cases. + #[cfg(test)] + fn from_discovered(slot: &gpt_slots::DiscoveredSlot, patterns: &[String]) -> Self { + Self { + id: slot.id.clone(), + div_id: Some(slot.div_id.clone()), + gam_unit_path: Some(slot.gam_unit_path.clone()), + page_patterns: patterns.to_vec(), + formats: slot + .formats + .iter() + .map(|&(width, height)| (width, height, None)) + .collect(), + floor_price: None, + targeting: BTreeMap::new(), + aps_slot_id: None, + prebid_bidders: slot.has_prebid.then(BTreeMap::new), + } + } + + /// Builds a slot from cross-page evidence and the inferred unit path. + /// + /// Refused inference decisions are filtered before this constructor. A + /// `None` path therefore means inference was unavailable and deliberately + /// leaves the runtime's configured default-path behavior in effect. + pub(super) fn from_evidence( + id: &str, + div_id: &str, + gam_unit_path: Option, + formats: impl IntoIterator, + page_patterns: Vec, + has_prebid: bool, + ) -> Self { + Self { + id: id.to_string(), + div_id: Some(div_id.to_string()), + gam_unit_path, + page_patterns, + formats: formats + .into_iter() + .map(|(width, height)| (width, height, None)) + .collect(), + floor_price: None, + targeting: BTreeMap::new(), + aps_slot_id: None, + prebid_bidders: has_prebid.then(BTreeMap::new), + } + } + + fn from_existing(slot: &CreativeOpportunitySlot) -> Self { + Self { + id: slot.id.clone(), + div_id: slot.div_id.clone(), + gam_unit_path: slot.gam_unit_path.clone(), + page_patterns: slot.page_patterns.clone(), + formats: slot + .formats + .iter() + .map(|format| { + ( + format.width, + format.height, + media_type_label(&format.media_type), + ) + }) + .collect(), + floor_price: slot.floor_price, + targeting: slot + .targeting + .iter() + .map(|(key, value)| (key.clone(), value.clone())) + .collect(), + aps_slot_id: slot.providers.aps.as_ref().map(|aps| aps.slot_id.clone()), + prebid_bidders: slot.providers.prebid.as_ref().map(|prebid| { + prebid + .bidders + .iter() + .map(|(name, params)| (name.clone(), params.clone())) + .collect() + }), + } + } +} + +/// The non-default (non-banner) media-type label to emit, or `None` for banner. +fn media_type_label(media_type: &MediaType) -> Option<&'static str> { + match media_type { + MediaType::Banner => None, + MediaType::Video => Some("video"), + MediaType::Native => Some("native"), + } +} + +/// Merges discovered slots into the existing slot set, keyed by [`RenderSlot::key`]. +/// +/// - `--replace` (or no existing slots): the result is exactly the discovered set. +/// - Otherwise existing slots are preserved (covering other pages / hand-tuned +/// fields); a slot re-seen this run has its page patterns and formats unioned; +/// slots seen only this run are appended. +/// - Format identity includes media type, so equal dimensions observed for two +/// media types remain two intentional entries. +#[cfg(test)] +pub(super) fn merge_slots( + existing: Option<&CreativeOpportunitiesConfig>, + discovered: &gpt_slots::DiscoveredSlots, + run_patterns: &[String], + replace: bool, +) -> Vec { + let discovered_slots: Vec = discovered + .slots + .iter() + .map(|slot| RenderSlot::from_discovered(slot, run_patterns)) + .collect(); + merge_render_slots(existing, discovered_slots, replace) +} + +/// Merges already-built slots into the existing set. +/// +/// Same reconciliation as the single-page test helper, but the caller supplies the slots — +/// the crawl path builds them from cross-page evidence rather than from one +/// page's discoveries. A slot re-seen this run keeps its configured fields and +/// gains this run's patterns; a genuinely new slot is appended with a +/// non-colliding id. +#[cfg(test)] +pub(super) fn merge_render_slots( + existing: Option<&CreativeOpportunitiesConfig>, + discovered_slots: Vec, + replace: bool, +) -> Vec { + merge_render_slots_with_diagnostics(existing, discovered_slots, replace).0 +} + +/// Diagnostics produced while merging discovered and configured slots. +#[derive(Debug, Default, PartialEq, Eq)] +pub(super) struct MergeDiagnostics { + /// Operator-facing reconciliation notes. + pub(super) notes: Vec, + /// Configured slots preserved without matching any normalized evidence div. + pub(super) unobserved_existing_slot_ids: Vec, +} + +/// Merges slots and reports prefix collisions and unobserved preserved slots. +#[cfg(test)] +pub(super) fn merge_render_slots_with_diagnostics( + existing: Option<&CreativeOpportunitiesConfig>, + discovered_slots: Vec, + replace: bool, +) -> (Vec, MergeDiagnostics) { + let observed_div_ids = discovered_slots + .iter() + .filter_map(|slot| slot.div_id.clone()) + .collect::>(); + merge_render_slots_with_observed_diagnostics( + existing, + discovered_slots, + &observed_div_ids, + &observed_div_ids, + replace, + ) +} + +/// Merges renderable slots using normalized evidence div IDs for observation. +/// +/// `observed_div_ids` must be the full normalized evidence set, including divs +/// refused by template inference, skipped as fragments, or refused as +/// ambiguous. `observed_literal_div_ids` contains only concrete live elements; +/// it controls whether a configured div ID remains eligible as a runtime prefix. +/// Passing only the rendered subset for observation can falsely report a live +/// configured slot as unobserved, while treating refused stems as literals can +/// incorrectly disqualify a configured prefix from merge routing. +pub(super) fn merge_render_slots_with_observed_diagnostics( + existing: Option<&CreativeOpportunitiesConfig>, + discovered_slots: Vec, + observed_div_ids: &[String], + observed_literal_div_ids: &[String], + replace: bool, +) -> (Vec, MergeDiagnostics) { + let existing_slots = existing.map(|config| config.slot.as_slice()).unwrap_or(&[]); + if replace || existing_slots.is_empty() { + return (discovered_slots, MergeDiagnostics::default()); + } + + let observed_literals = observed_literal_div_ids + .iter() + .map(String::as_str) + .collect::>(); + + let mut merged: Vec = existing_slots + .iter() + .map(RenderSlot::from_existing) + .collect(); + let existing_count = merged.len(); + let mut prefix_claims: BTreeMap> = BTreeMap::new(); + let mut split_warnings = BTreeSet::new(); + let mut observed_existing = observed_div_ids + .iter() + .flat_map(|div_id| matching_observed_div_indexes(&merged, div_id, &observed_literals)) + .collect::>(); + for mut slot in discovered_slots { + // Prefix reconciliation is a property of the operator's config, so only + // the slots that were already configured may claim a discovered div. + // Slots this run appended match by exact identity instead, otherwise + // discovery order decides whether `ad-top` swallows a later + // `ad-top-sidebar` and discards its unit path and provider state. + let matched = matching_slot_index(&merged[..existing_count], &slot, &observed_literals) + .or_else(|| { + let key = slot.key(); + merged[existing_count..] + .iter() + .position(|added| added.key() == key) + .map(|offset| offset + existing_count) + }); + if let Some(index) = matched { + if index < existing_count { + observed_existing.insert(index); + } + if index < existing_count + && let (Some(prefix), Some(discovered_div)) = + (merged[index].div_id.as_deref(), slot.div_id.as_deref()) + && discovered_div.starts_with(prefix) + { + prefix_claims + .entry(index) + .or_default() + .insert(discovered_div.to_string()); + } + let present = &mut merged[index]; + for pattern in &slot.page_patterns { + if !present.page_patterns.contains(pattern) { + present.page_patterns.push(pattern.clone()); + } + } + for format in &slot.formats { + if !present.formats.contains(format) { + present.formats.push(*format); + } + } + } else { + if let Some(discovered_div) = slot.div_id.as_deref() { + for parent in merged[..existing_count].iter().filter(|configured| { + configured.div_id.as_deref().is_some_and(|prefix| { + !prefix.is_empty() + && observed_literals.contains(prefix) + && discovered_div != prefix + && discovered_div.starts_with(prefix) + }) && configured.has_tuned_fields() + }) { + split_warnings.insert(format!( + "discovered div `{discovered_div}` was split from configured div_id prefix \ + `{}`; the new slot does not inherit that configured slot's floor price, \ + targeting, or provider settings", + parent.div_id.as_deref().unwrap_or_default(), + )); + } + } + slot.id = unique_slot_id(&slot.id, &merged); + merged.push(slot); + } + } + let notes = split_warnings + .into_iter() + .chain( + prefix_claims + .into_iter() + .filter(|(_, divs)| divs.len() > 1) + .map(|(index, divs)| { + let slot = &merged[index]; + let sample = divs.iter().take(5).cloned().collect::>().join(", "); + let remainder = divs.len().saturating_sub(5); + let suffix = if remainder == 0 { + String::new() + } else { + format!(", and {remainder} more") + }; + format!( + "configured slot `{}` with div_id prefix `{}` matched {} discovered divs \ + ({sample}{suffix}); runtime can resolve this configured slot to at most one \ + active element, so review whether they are distinct placements", + slot.id, + slot.div_id.as_deref().unwrap_or_default(), + divs.len(), + ) + }), + ) + .collect(); + let unobserved_existing_slot_ids = existing_slots + .iter() + .enumerate() + .filter(|(index, _)| !observed_existing.contains(index)) + .map(|(_, slot)| slot.id.clone()) + .collect(); + ( + merged, + MergeDiagnostics { + notes, + unobserved_existing_slot_ids, + }, + ) +} + +fn unique_slot_id(candidate: &str, existing: &[RenderSlot]) -> String { + if existing.iter().all(|slot| slot.id != candidate) { + return candidate.to_string(); + } + + let mut suffix = 2_usize; + loop { + let unique = format!("{candidate}-{suffix}"); + if existing.iter().all(|slot| slot.id != unique) { + return unique; + } + suffix += 1; + } +} + +/// Finds the configured slot matching a discovered normalized slot. +/// +/// Stable-key equality wins first. Otherwise, configured `div_id` values are +/// eligible runtime prefixes unless that value was itself observed as a +/// distinct literal. Equal-length prefix ties retain configuration order. +fn matching_slot_index( + existing: &[RenderSlot], + discovered: &RenderSlot, + observed_literals: &BTreeSet<&str>, +) -> Option { + let key = discovered.key(); + if let Some(index) = existing.iter().position(|slot| slot.key() == key) { + return Some(index); + } + + discovered + .div_id + .as_deref() + .and_then(|div_id| matching_div_id_index(existing, div_id, observed_literals)) +} + +fn matching_div_id_index( + existing: &[RenderSlot], + discovered_div: &str, + observed_literals: &BTreeSet<&str>, +) -> Option { + let mut best = None; + let mut best_length = 0; + for (index, slot) in existing.iter().enumerate() { + let Some(prefix) = slot.div_id.as_deref().filter(|prefix| !prefix.is_empty()) else { + continue; + }; + if observed_literals.contains(prefix) { + continue; + } + if discovered_div.starts_with(prefix) && prefix.len() > best_length { + best = Some(index); + best_length = prefix.len(); + } + } + best +} + +/// Finds every configured slot that can resolve to one normalized evidence div. +/// +/// Merge routing remains exact-then-longest-prefix through +/// [`matching_slot_index`], but observation is deliberately multi-match: an +/// exact configured slot and every eligible broad prefix are all live when the +/// element exists. +fn matching_observed_div_indexes<'a>( + existing: &'a [RenderSlot], + discovered_div: &'a str, + observed_literals: &'a BTreeSet<&'a str>, +) -> impl Iterator + 'a { + let discovered_key = discovered_div.trim_end_matches('-'); + existing + .iter() + .enumerate() + .filter_map(move |(index, slot)| { + if slot.key() == discovered_key { + return Some(index); + } + let prefix = slot.div_id.as_deref().filter(|prefix| !prefix.is_empty())?; + (!observed_literals.contains(prefix) && discovered_div.starts_with(prefix)) + .then_some(index) + }) +} + +/// Header comment emitted above the structurally replaced managed slot array. +const MANAGED_SLOTS_COMMENT: &str = "# Slots managed by `ts audit ad-templates generate`."; +/// Second line of the managed-slot header comment. +const MANAGED_SLOTS_REVIEW_COMMENT: &str = + "# Review page_patterns and formats before validating/pushing."; + +/// Renders merged slots as compact `[[creative_opportunities.slot]]` TOML blocks. +pub(super) fn render_slots(slots: &[RenderSlot]) -> String { + let mut out = format!("\n{MANAGED_SLOTS_COMMENT}\n{MANAGED_SLOTS_REVIEW_COMMENT}\n"); + for slot in slots { + out.push_str("\n[[creative_opportunities.slot]]\n"); + out.push_str(&format!("id = {}\n", toml_string(&slot.id))); + if let Some(div_id) = &slot.div_id { + out.push_str(&format!("div_id = {}\n", toml_string(div_id))); + } + if let Some(path) = &slot.gam_unit_path { + out.push_str(&format!("gam_unit_path = {}\n", toml_string(path))); + } + out.push_str("page_patterns = [\n"); + for pattern in &slot.page_patterns { + out.push_str(&format!(" {},\n", toml_string(pattern))); + } + out.push_str("]\n"); + out.push_str("formats = [\n"); + for (width, height, media_type) in &slot.formats { + let rendered = match media_type { + Some(kind) => { + format!("{{ width = {width}, height = {height}, media_type = \"{kind}\" }}") + } + None => format!("{{ width = {width}, height = {height} }}"), + }; + out.push_str(&format!(" {rendered},\n")); + } + out.push_str("]\n"); + if let Some(floor) = slot.floor_price { + // `f64` Display prints `NaN`, which is not valid TOML (`nan` is); + // normalize non-finite values so the spliced config stays parseable. + if floor.is_finite() { + out.push_str(&format!("floor_price = {floor}\n")); + } else if floor.is_nan() { + out.push_str("floor_price = nan\n"); + } else if floor.is_sign_positive() { + out.push_str("floor_price = inf\n"); + } else { + out.push_str("floor_price = -inf\n"); + } + } + if !slot.targeting.is_empty() { + let pairs = slot + .targeting + .iter() + .map(|(key, value)| format!("{} = {}", toml_key(key), toml_string(value))) + .collect::>() + .join(", "); + out.push_str(&format!("targeting = {{ {pairs} }}\n")); + } + if let Some(slot_id) = &slot.aps_slot_id { + out.push_str("[creative_opportunities.slot.providers.aps]\n"); + out.push_str(&format!("slot_id = {}\n", toml_string(slot_id))); + } + if let Some(bidders) = &slot.prebid_bidders { + out.push_str("[creative_opportunities.slot.providers.prebid]\n"); + let rendered = bidders + .iter() + .map(|(name, params)| format!("{} = {}", toml_key(name), toml_inline_value(params))) + .collect::>() + .join(", "); + if rendered.is_empty() { + out.push_str("bidders = {}\n"); + } else { + out.push_str(&format!("bidders = {{ {rendered} }}\n")); + } + } + } + out +} + +/// Quotes and escapes a string as a TOML basic string, including control chars. +pub(super) fn toml_string(value: &str) -> String { + let mut out = String::with_capacity(value.len() + 2); + out.push('"'); + for ch in value.chars() { + match ch { + '"' => out.push_str("\\\""), + '\\' => out.push_str("\\\\"), + '\n' => out.push_str("\\n"), + '\r' => out.push_str("\\r"), + '\t' => out.push_str("\\t"), + // TOML basic strings reject U+0000..U+001F and DEL (U+007F). + control if (control as u32) < 0x20 || control == '\u{7f}' => { + out.push_str(&format!("\\u{:04X}", control as u32)); + } + other => out.push(other), + } + } + out.push('"'); + out +} + +/// Renders a TOML table key: bare when it is a valid bare key, else a quoted key. +fn toml_key(key: &str) -> String { + let is_bare = !key.is_empty() + && key + .chars() + .all(|ch| ch.is_ascii_alphanumeric() || ch == '_' || ch == '-'); + if is_bare { + key.to_string() + } else { + toml_string(key) + } +} + +/// Renders a JSON value as a compact inline TOML value (for prebid bidder params). +fn toml_inline_value(value: &serde_json::Value) -> String { + match value { + serde_json::Value::Null => "{}".to_string(), + serde_json::Value::Bool(bool) => bool.to_string(), + serde_json::Value::Number(number) => number.to_string(), + serde_json::Value::String(string) => toml_string(string), + serde_json::Value::Array(items) => { + let rendered = items + .iter() + .map(toml_inline_value) + .collect::>() + .join(", "); + format!("[{rendered}]") + } + serde_json::Value::Object(map) => { + let rendered = map + .iter() + .map(|(key, value)| format!("{} = {}", toml_key(key), toml_inline_value(value))) + .collect::>() + .join(", "); + format!("{{ {rendered} }}") + } + } +} + +/// The config-level values a splice writes alongside the slot array. +#[derive(Debug, Clone, Default)] +pub(super) struct CreativeSectionKeys<'a> { + /// GAM network id, when one was resolved. + pub(super) network_id: Option<&'a str>, + /// `section_root`, written only when a slot uses a `{section}` template. + pub(super) section_root: Option<&'a str>, + /// `section_segment`, written only alongside `section_root`. + pub(super) section_segment: Option, +} + +fn max_table_position(table: &Table) -> Option { + table.iter().fold(table.position(), |maximum, (_, item)| { + let child_maximum = match item { + Item::Table(child) => max_table_position(child), + Item::ArrayOfTables(array) => array.iter().filter_map(max_table_position).max(), + Item::None | Item::Value(_) => None, + }; + maximum.max(child_maximum) + }) +} + +fn set_table_position_recursive(table: &mut Table, position: isize) { + table.set_position(position); + for (_, item) in table.iter_mut() { + match item { + Item::Table(child) => set_table_position_recursive(child, position), + Item::ArrayOfTables(array) => { + for child in array.iter_mut() { + set_table_position_recursive(child, position); + } + } + Item::None | Item::Value(_) => {} + } + } +} + +/// Structurally replaces the generator-managed creative-opportunities fields. +/// +/// All unrelated TOML items and their decorations remain in the parsed +/// document. Missing inferred scalar values preserve their existing values; a +/// fresh section is created only when a network id is available. +pub(super) fn splice_creative_slots( + existing: &str, + keys: &CreativeSectionKeys<'_>, + rendered_slots: &str, +) -> CliResult { + let mut document = existing.parse::().map_err(|error| { + report_error(format!( + "failed to parse target config before updating slots: {error}" + )) + })?; + let had_section = document.get("creative_opportunities").is_some(); + let existing_section_position = document + .get("creative_opportunities") + .and_then(Item::as_table) + .and_then(Table::position); + let section_position = existing_section_position + .unwrap_or_else(|| max_table_position(document.as_table()).unwrap_or(0) + 1); + if !had_section && keys.network_id.is_none() { + return cli_error( + "refusing to create a `[creative_opportunities]` section without a \ + GAM network id: none could be determined from the audited page, and \ + the key is required. Add `[creative_opportunities]` with a \ + `gam_network_id` to the config and re-run", + ); + } + + let generated = format!( + "[creative_opportunities]\n{}\n", + rendered_slots.trim_matches('\n') + ); + let mut generated = generated + .parse::() + .map_err(|error| report_error(format!("failed to parse generated slot tables: {error}")))?; + let mut generated_slots = generated["creative_opportunities"] + .as_table_mut() + .and_then(|table| table.remove("slot")) + .unwrap_or_else(|| Item::ArrayOfTables(toml_edit::ArrayOfTables::new())); + if let Item::ArrayOfTables(array) = &mut generated_slots { + for table in array.iter_mut() { + set_table_position_recursive(table, section_position); + } + } + + if !had_section { + document["creative_opportunities"] = Item::Table(toml_edit::Table::new()); + } + let creative = document["creative_opportunities"] + .as_table_mut() + .ok_or_else(|| { + report_error( + "target config's `creative_opportunities` value is not an editable table; \ + rewrite it as a `[creative_opportunities]` table and re-run", + ) + })?; + // `toml_edit` stably sorts tables by document position. Imported tables + // retain positions from their source document, so anchor the whole subtree + // here to keep the parent, slots, and provider tables together. + creative.set_position(section_position); + if let Some(network_id) = keys.network_id { + creative["gam_network_id"] = toml_edit::value(network_id); + } + if let Some(section_root) = keys.section_root { + creative["section_root"] = toml_edit::value(section_root); + if let Some(section_segment) = keys.section_segment { + creative["section_segment"] = toml_edit::value(section_segment as i64); + } + } + creative.insert("slot", generated_slots); + + let mut result = document.to_string(); + if uses_crlf(existing) { + result = convert_document_lf_to_crlf(&result); + } + ensure_only_managed_fields_changed(existing, &result)?; + Ok(result) +} + +/// Verifies that the structural update changed only generator-managed fields. +fn ensure_only_managed_fields_changed(before: &str, after: &str) -> CliResult<()> { + fn unmanaged(document: &str) -> CliResult { + let mut value = toml::from_str::(document) + .map_err(|error| report_error(format!("failed to validate updated config: {error}")))?; + if let Some(root) = value.as_table_mut() { + let remove_empty = if let Some(creative) = root + .get_mut("creative_opportunities") + .and_then(toml::Value::as_table_mut) + { + for key in ["slot", "gam_network_id", "section_root", "section_segment"] { + creative.remove(key); + } + creative.is_empty() + } else { + false + }; + if remove_empty { + root.remove("creative_opportunities"); + } + } + Ok(value) + } + + if unmanaged(before)? != unmanaged(after)? { + return cli_error( + "refusing to update config because fields outside the managed \ + creative-opportunities keys would change", + ); + } + Ok(()) +} + +/// Byte offsets of the `\n` bytes that terminate a document line. +/// +/// Only newlines outside comments and string values delimit lines, so the scan +/// skips a `#` comment to end of line, skips single-line basic and literal +/// strings, and tracks multiline `"""` / `'''` bodies. Without the comment and +/// single-line-string cases a stray triple quote desynchronizes the scan and the +/// document's line endings are flipped or left mixed — a rewrite +/// [`ensure_only_managed_fields_changed`] cannot catch, because it compares +/// parsed values. +fn document_newlines(document: &str) -> Vec { + let bytes = document.as_bytes(); + let mut newlines = Vec::new(); + let mut index = 0_usize; + while index < bytes.len() { + match bytes[index] { + b'#' => { + while index < bytes.len() && bytes[index] != b'\n' { + index += 1; + } + } + b'\n' => { + newlines.push(index); + index += 1; + } + quote @ (b'"' | b'\'') => { + if bytes[index..].starts_with(&[quote, quote, quote]) { + index += 3; + while index < bytes.len() && !bytes[index..].starts_with(&[quote, quote, quote]) + { + index += 1; + } + index = index.saturating_add(3).min(bytes.len()); + } else { + index += 1; + while index < bytes.len() && bytes[index] != quote && bytes[index] != b'\n' { + index += if quote == b'"' && bytes[index] == b'\\' { + 2 + } else { + 1 + }; + } + if index < bytes.len() && bytes[index] == quote { + index += 1; + } + } + } + _ => index += 1, + } + } + newlines +} + +/// Whether `document` uses CRLF line endings (so edits preserve them). +fn uses_crlf(document: &str) -> bool { + let bytes = document.as_bytes(); + document_newlines(document) + .first() + .is_some_and(|&index| index > 0 && bytes[index - 1] == b'\r') +} + +/// Converts document line terminators while leaving string content intact. +fn convert_document_lf_to_crlf(document: &str) -> String { + let bytes = document.as_bytes(); + let mut output = String::with_capacity(document.len()); + let mut previous = 0_usize; + for index in document_newlines(document) { + output.push_str(&document[previous..index]); + if index == 0 || bytes[index - 1] != b'\r' { + output.push('\r'); + } + output.push('\n'); + previous = index + 1; + } + output.push_str(&document[previous..]); + output +} + +/// Strips a trailing inline `# comment` from a candidate table-header line. +/// +/// Only valid on header candidates: header lines cannot contain `#` before the +/// closing bracket unless it is inside a quoted key, which the configs this +/// updater manages never use. +fn strip_inline_comment(line: &str) -> &str { + match line.find('#') { + Some(position) => line[..position].trim_end(), + None => line, + } +} + +pub(super) fn replace_key_in_section( + document: &str, + section: &str, + key: &str, + replacement_line: &str, +) -> CliResult { + let section_header = format!("[{section}]"); + let mut in_section = false; + let mut replaced = false; + let mut saw_section = false; + let mut lines = Vec::new(); + + for line in document.lines() { + let trimmed = line.trim(); + let header_candidate = strip_inline_comment(trimmed); + if header_candidate.starts_with('[') && header_candidate.ends_with(']') { + in_section = header_candidate == section_header; + saw_section |= in_section; + } + + if in_section && !replaced && is_key_line(trimmed, key) { + lines.push(replacement_line.to_string()); + replaced = true; + } else { + lines.push(line.to_string()); + } + } + + if !saw_section { + return cli_error(format!( + "failed to update starter config because section `{section_header}` was not found" + )); + } + if !replaced { + return cli_error(format!( + "failed to update starter config because key `{key}` was not found in `{section_header}`" + )); + } + + let mut output = lines.join("\n"); + if document.ends_with('\n') { + output.push('\n'); + } + if uses_crlf(document) { + // `lines()` stripped the `\r`s; restore the document's CRLF endings. + output = output.replace("\r\n", "\n").replace('\n', "\r\n"); + } + Ok(output) +} + +fn is_key_line(trimmed_line: &str, key: &str) -> bool { + trimmed_line + .strip_prefix(key) + .and_then(|remaining| remaining.trim_start().strip_prefix('=')) + .is_some() +} + +/// Chooses the `gam_network_id` to write. +/// +/// The existing id is kept only when a real merge preserves existing slots. +/// On `--replace`, or when the config had no slots (e.g. a placeholder +/// `[creative_opportunities]` section), the discovered id wins — mirroring +/// the slot merge, which returns discovered-only in those cases. +pub(super) fn resolve_network_id( + existing: Option<&CreativeOpportunitiesConfig>, + discovered_network_id: Option<&str>, + replace: bool, +) -> Option { + let existing_network_id = existing.map(|config| config.gam_network_id.clone()); + let preserving_existing = !replace && existing.is_some_and(|config| !config.slot.is_empty()); + if preserving_existing { + existing_network_id.or_else(|| discovered_network_id.map(str::to_string)) + } else { + discovered_network_id + .map(str::to_string) + .or(existing_network_id) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::commands::audit::generate::collector; + + fn discovered_header_slot() -> gpt_slots::DiscoveredSlots { + let registry = vec![collector::CollectedGptSlot { + gam_unit_path: "/222/homepage/header".to_string(), + div_id: "div-gpt-ad-header".to_string(), + sizes: vec![(728, 90)], + }]; + gpt_slots::discover_gpt_slots(®istry, &[], false) + } + + /// Rendered slot text for the discovered header slot, patterns = `/`. + fn header_rendered() -> String { + let merged = merge_slots(None, &discovered_header_slot(), &["/".to_string()], true); + render_slots(&merged) + } + + fn two_provider_slots_rendered() -> &'static str { + r#" +# Slots managed by `ts audit ad-templates generate`. +# Review page_patterns and formats before validating/pushing. + +[[creative_opportunities.slot]] +id = "header" +div_id = "header" +gam_unit_path = "/222/{section}/header" +page_patterns = ["/"] +formats = [{ width = 728, height = 90 }] +[creative_opportunities.slot.providers.prebid] +bidders = {} + +[[creative_opportunities.slot]] +id = "sidebar" +div_id = "sidebar" +gam_unit_path = "/222/{section}/sidebar" +page_patterns = ["/"] +formats = [{ width = 300, height = 250 }] +[creative_opportunities.slot.providers.aps] +slot_id = "sidebar" +"# + } + + fn table_headers(document: &str) -> Vec<&str> { + document + .lines() + .map(str::trim) + .filter(|line| line.starts_with('[')) + .collect() + } + + /// Section keys carrying only a network id, the common test case. + fn network_keys(network_id: &str) -> CreativeSectionKeys<'_> { + CreativeSectionKeys { + network_id: Some(network_id), + ..CreativeSectionKeys::default() + } + } + + fn existing_config(toml_str: &str) -> CreativeOpportunitiesConfig { + toml::from_str::(toml_str).expect("valid creative config") + } + + #[test] + fn splice_replaces_slots_and_preserves_other_sections() { + let existing = "[publisher]\ndomain = \"x\"\n\n\ + [creative_opportunities]\ngam_network_id = \"111\"\nprice_granularity = \"dense\"\n\n\ + [[creative_opportunities.slot]]\nid = \"old\"\ndiv_id = \"old\"\n\ + gam_unit_path = \"/111/old\"\npage_patterns = [\"/\"]\n\ + formats = [{ width = 300, height = 250 }]\n\n\ + [auction]\nenabled = true\n"; + + let out = splice_creative_slots(existing, &network_keys("222"), &header_rendered()) + .expect("should splice"); + + assert!( + out.contains("gam_network_id = \"222\""), + "network id updated" + ); + assert!(!out.contains("id = \"old\""), "old slot removed"); + assert!( + out.contains("gam_unit_path = \"/222/homepage/header\""), + "new slot written" + ); + assert!( + out.contains("[publisher]") && out.contains("domain = \"x\""), + "publisher section preserved" + ); + assert!( + out.contains("[auction]") && out.contains("enabled = true"), + "trailing auction section preserved" + ); + toml::from_str::(&out).expect("spliced config is valid TOML"); + } + + #[test] + fn splice_updates_a_quoted_section_header_structurally() { + let existing = "[\"creative_opportunities\"]\ngam_network_id = \"111\"\n"; + + let updated = splice_creative_slots(existing, &network_keys("222"), &header_rendered()) + .expect("should update quoted table structurally"); + + assert_eq!(updated.matches("creative_opportunities").count(), 2); + assert!(updated.contains("gam_network_id = \"222\"")); + toml::from_str::(&updated).expect("should remain valid TOML"); + } + + #[test] + fn splice_preserves_multiline_values_comments_and_noncontiguous_tables() { + let existing = "title = \"publisher\" # keep this comment\n\ + description = \"\"\"a line that looks like [creative_opportunities]\n\ + and another [[creative_opportunities.slot]] line\"\"\"\n\ + dimensions = [\n 300,\n 250,\n]\n\n\ + [creative_opportunities] # managed section\n\ + gam_network_id = \"111\" # old network\n\n\ + [[creative_opportunities.slot]]\nid = \"old-a\"\ndiv_id = \"old-a\"\n\ + gam_unit_path = \"/111/a\"\npage_patterns = [\"/\"]\n\ + formats = [{ width = 300, height = 250 }]\n\n\ + [auction]\nenabled = true # keep auction comment\n\n\ + [[creative_opportunities.slot]]\nid = \"old-b\"\ndiv_id = \"old-b\"\n\ + gam_unit_path = \"/111/b\"\npage_patterns = [\"/b\"]\n\ + formats = [{ width = 320, height = 50 }]\n"; + + let updated = splice_creative_slots(existing, &network_keys("222"), &header_rendered()) + .expect("should update structurally"); + + assert!(updated.contains("looks like [creative_opportunities]")); + assert!(updated.contains("dimensions = [\n 300,\n 250,\n]")); + assert!(updated.contains("enabled = true # keep auction comment")); + assert!(!updated.contains("id = \"old-a\"")); + assert!(!updated.contains("id = \"old-b\"")); + let value = toml::from_str::(&updated).expect("should remain valid TOML"); + assert_eq!( + value["creative_opportunities"]["slot"] + .as_array() + .map(Vec::len), + Some(1) + ); + } + + #[test] + fn splice_keeps_generated_slots_and_providers_contiguous() { + let existing = "[publisher]\ndomain = \"example.com\"\n\n\ + [tester_cookie]\nenabled = true\n\n\ + [creative_opportunities]\ngam_network_id = \"111\"\n\n\ + [debug]\nauction_html_comment = true\n"; + + let updated = splice_creative_slots( + existing, + &network_keys("222"), + two_provider_slots_rendered(), + ) + .expect("should splice slots"); + + assert_eq!( + table_headers(&updated), + vec![ + "[publisher]", + "[tester_cookie]", + "[creative_opportunities]", + "[[creative_opportunities.slot]]", + "[creative_opportunities.slot.providers.prebid]", + "[[creative_opportunities.slot]]", + "[creative_opportunities.slot.providers.aps]", + "[debug]", + ] + ); + } + + #[test] + fn splice_groups_a_new_creative_section_with_its_slots() { + let existing = "[publisher]\ndomain = \"example.com\"\n\n\ + [debug]\nauction_html_comment = true\n\n\ + [auction]\nenabled = true\n"; + + let updated = splice_creative_slots( + existing, + &network_keys("222"), + two_provider_slots_rendered(), + ) + .expect("should create creative section and splice slots"); + + assert_eq!( + table_headers(&updated), + vec![ + "[publisher]", + "[debug]", + "[auction]", + "[creative_opportunities]", + "[[creative_opportunities.slot]]", + "[creative_opportunities.slot.providers.prebid]", + "[[creative_opportunities.slot]]", + "[creative_opportunities.slot.providers.aps]", + ] + ); + } + + #[test] + fn splice_rejects_top_level_inline_creative_opportunities_table() { + let existing = "creative_opportunities = { gam_network_id = \"111\" }\n"; + + let error = splice_creative_slots(existing, &network_keys("222"), &header_rendered()) + .expect_err("should refuse a top-level inline table"); + + assert!( + format!("{error:?}").contains("rewrite it as"), + "error should tell the operator to rewrite the section, got {error:?}" + ); + } + + /// Section keys for a templated run: network id plus the section policy. + fn template_keys<'a>( + network_id: &'a str, + root: &'a str, + segment: usize, + ) -> CreativeSectionKeys<'a> { + CreativeSectionKeys { + network_id: Some(network_id), + section_root: Some(root), + section_segment: Some(segment), + } + } + + #[test] + fn splice_inserts_section_policy_keys_a_config_does_not_have_yet() { + // The whole point of `upsert`: every config predating templating lacks + // these keys, so a replace-only writer could never add them. + let existing = "[creative_opportunities]\ngam_network_id = \"111\"\n\n\ + [auction]\nenabled = true\n"; + + let out = splice_creative_slots( + existing, + &template_keys("222", "homepage", 0), + &header_rendered(), + ) + .expect("should splice"); + + let value = toml::from_str::(&out).expect("spliced config is valid TOML"); + let creative = &value["creative_opportunities"]; + assert_eq!(creative["gam_network_id"].as_str(), Some("222")); + assert_eq!(creative["section_root"].as_str(), Some("homepage")); + assert_eq!(creative["section_segment"].as_integer(), Some(0)); + assert_eq!( + value["auction"]["enabled"].as_bool(), + Some(true), + "inserting must not disturb later sections" + ); + } + + #[test] + fn splice_replaces_section_policy_keys_that_are_already_present() { + let existing = "[creative_opportunities]\ngam_network_id = \"111\"\n\ + section_root = \"old\"\nsection_segment = 2\n"; + + let out = splice_creative_slots( + existing, + &template_keys("111", "homepage", 1), + &header_rendered(), + ) + .expect("should splice"); + + let value = toml::from_str::(&out).expect("valid TOML"); + let creative = &value["creative_opportunities"]; + assert_eq!(creative["section_root"].as_str(), Some("homepage")); + assert_eq!(creative["section_segment"].as_integer(), Some(1)); + assert_eq!( + out.matches("section_root").count(), + 1, + "the key must be replaced, not duplicated" + ); + } + + #[test] + fn splice_omits_section_policy_when_no_slot_needs_it() { + // `section_root`/`section_segment` are `deny_unknown_fields` additions: + // writing them into a config that does not need them would make it + // unloadable by an older binary for no benefit. + let existing = "[creative_opportunities]\ngam_network_id = \"111\"\n"; + + let out = splice_creative_slots(existing, &network_keys("222"), &header_rendered()) + .expect("should splice"); + + assert!( + !out.contains("section_root") && !out.contains("section_segment"), + "an untemplated run must not add rollback-fatal keys, got:\n{out}" + ); + } + + #[test] + fn splice_writes_section_policy_into_a_freshly_created_section() { + let existing = "[publisher]\ndomain = \"x\"\n"; + + let out = splice_creative_slots( + existing, + &template_keys("222", "homepage", 0), + &header_rendered(), + ) + .expect("should append a fresh section"); + + let value = toml::from_str::(&out).expect("valid TOML"); + let creative = &value["creative_opportunities"]; + assert_eq!(creative["gam_network_id"].as_str(), Some("222")); + assert_eq!(creative["section_root"].as_str(), Some("homepage")); + assert_eq!(creative["section_segment"].as_integer(), Some(0)); + } + + #[test] + fn splice_refuses_fresh_section_without_a_network_id() { + // Reachable whenever the scraped unit path has no all-digit leading + // segment (MCM/child-network paths). Writing the section anyway produces + // a config missing a required field, which fails load and takes every + // route to the startup error router once pushed. + let existing = "[publisher]\ndomain = \"x\"\n"; + + let error = splice_creative_slots( + existing, + &CreativeSectionKeys::default(), + &header_rendered(), + ) + .expect_err("should refuse to create a section with no network id"); + + assert!( + format!("{error:?}").contains("without a GAM network id"), + "error should name the missing network id, got {error:?}" + ); + } + + #[test] + fn splice_appends_section_when_config_has_none() { + let existing = "[publisher]\ndomain = \"x\"\n"; + + let out = splice_creative_slots(existing, &network_keys("222"), &header_rendered()) + .expect("should append a fresh section"); + + let value = toml::from_str::(&out).expect("appended config is valid TOML"); + assert_eq!( + value["creative_opportunities"]["gam_network_id"].as_str(), + Some("222") + ); + } + + #[test] + fn splice_preserves_section_scalars_and_provider_subtables() { + // Mirrors the templated operator shape: section policy scalars in the + // head block and a per-slot prebid provider subtable. + let existing = "[creative_opportunities]\n\ + gam_network_id = \"111\"\n\ + auction_timeout_ms = 2000\n\ + section_root = \"homepage\"\n\n\ + [[creative_opportunities.slot]]\n\ + id = \"ad-header-0\"\n\ + div_id = \"ad-header-0\"\n\ + gam_unit_path = \"/{network_id}/example/{section}\"\n\ + page_patterns = [\"/\"]\n\ + formats = [{ width = 728, height = 90 }]\n\ + [creative_opportunities.slot.providers.prebid]\n\ + bidders = {}\n\n\ + [auction]\nenabled = true\n"; + let existing_config = existing_config( + &existing + .replace("[creative_opportunities]\n", "") + .replace("[[creative_opportunities.slot]]", "[[slot]]") + .replace("[creative_opportunities.slot.", "[slot.") + .replace("\n[auction]\nenabled = true\n", ""), + ); + let discovered = discovered_header_slot(); + let merged = merge_slots( + Some(&existing_config), + &discovered, + &["/news/*".to_string()], + false, + ); + + let out = splice_creative_slots(existing, &network_keys("111"), &render_slots(&merged)) + .expect("should splice"); + + let value = toml::from_str::(&out).expect("spliced config is valid TOML"); + let creative = &value["creative_opportunities"]; + assert_eq!( + creative["section_root"].as_str(), + Some("homepage"), + "section policy scalars must survive the splice" + ); + assert_eq!(creative["auction_timeout_ms"].as_integer(), Some(2000)); + assert_eq!( + creative["slot"][0]["gam_unit_path"].as_str(), + Some("/{network_id}/example/{section}"), + "an existing templated unit path must not be rewritten to a literal" + ); + assert!( + creative["slot"][0]["providers"]["prebid"]["bidders"].is_table(), + "the prebid provider subtable must be re-emitted" + ); + assert_eq!( + value["auction"]["enabled"].as_bool(), + Some(true), + "trailing sections must be preserved" + ); + } + + #[test] + fn splice_preserves_crlf_line_endings() { + let existing = "[creative_opportunities]\r\ngam_network_id = \"111\"\r\n\r\n\ + [auction]\r\nenabled = true\r\n"; + + let out = splice_creative_slots(existing, &network_keys("222"), &header_rendered()) + .expect("should splice"); + + assert!( + !out.replace("\r\n", "").contains('\n'), + "every line ending should stay CRLF" + ); + let value = toml::from_str::(&out).expect("spliced CRLF config is valid TOML"); + assert_eq!( + value["creative_opportunities"]["gam_network_id"].as_str(), + Some("222"), + "network id updated in CRLF config" + ); + } + + #[test] + fn splice_does_not_infer_document_endings_from_multiline_string_content() { + let existing = "[publisher]\nother = \"\"\"a\r\nb\"\"\"\n\n\ + [creative_opportunities]\ngam_network_id = \"111\"\n"; + + let out = splice_creative_slots(existing, &network_keys("222"), &header_rendered()) + .expect("should splice LF document"); + + assert!( + out.contains("[publisher]\nother"), + "an embedded CRLF must not convert document line endings" + ); + assert!( + out.contains("a\r\nb"), + "an unrelated multiline string value must remain byte-identical" + ); + } + + #[test] + fn a_triple_quote_in_a_comment_does_not_desynchronize_the_line_scan() { + // A `"""` inside a comment is not a multiline string. Treating it as one + // makes the rest of the document read as string content, so a CRLF file + // is detected as LF and gets rewritten wholesale. + let existing = "# see \"\"\" docs\r\n[creative_opportunities]\r\n\ + gam_network_id = \"111\"\r\n"; + + let out = splice_creative_slots(existing, &network_keys("222"), &header_rendered()) + .expect("should splice CRLF document"); + + assert!( + !out.replace("\r\n", "").contains('\n'), + "the document's CRLF endings must survive a triple quote in a comment, got {out:?}" + ); + } + + #[test] + fn a_triple_quote_in_a_single_line_string_does_not_desynchronize_the_line_scan() { + let existing = "[publisher]\r\nlabel = 'a \"\"\" b'\r\n\r\n\ + [creative_opportunities]\r\ngam_network_id = \"111\"\r\n"; + + let out = splice_creative_slots(existing, &network_keys("222"), &header_rendered()) + .expect("should splice CRLF document"); + + assert!( + !out.replace("\r\n", "").contains('\n'), + "the document's CRLF endings must survive a triple quote in a value, got {out:?}" + ); + } + + #[test] + fn splice_does_not_rewrite_bare_lf_inside_crlf_multiline_string() { + let existing = "[publisher]\r\nother = \"\"\"a\nb\"\"\"\r\n\r\n\ + [creative_opportunities]\r\ngam_network_id = \"111\"\r\n"; + + let out = splice_creative_slots(existing, &network_keys("222"), &header_rendered()) + .expect("should splice CRLF document"); + + assert!( + out.contains("a\nb"), + "a bare LF inside an unrelated multiline value must remain unchanged" + ); + } + + #[test] + fn render_slots_writes_non_finite_floor_price_as_valid_toml() { + let slot = RenderSlot { + id: "header".to_string(), + div_id: Some("div-gpt-ad-header".to_string()), + gam_unit_path: Some("/222/homepage/header".to_string()), + page_patterns: vec!["/".to_string()], + formats: vec![(728, 90, None)], + floor_price: Some(f64::NAN), + targeting: BTreeMap::new(), + aps_slot_id: None, + prebid_bidders: None, + }; + + let rendered = render_slots(&[slot]); + + assert!( + rendered.contains("floor_price = nan"), + "NaN should render as TOML `nan`, not Rust `NaN`" + ); + toml::from_str::(&rendered).expect("rendered slots are valid TOML"); + } + + #[test] + fn render_slots_formats_long_arrays_across_indented_lines() { + let slot = RenderSlot { + id: "header".to_string(), + div_id: Some("div-gpt-ad-header".to_string()), + gam_unit_path: Some("/222/homepage/header".to_string()), + page_patterns: vec!["/".to_string(), "/news".to_string(), "/news/*".to_string()], + formats: vec![(728, 90, None), (970, 250, None), (300, 250, None)], + floor_price: None, + targeting: BTreeMap::new(), + aps_slot_id: None, + prebid_bidders: None, + }; + + let rendered = render_slots(&[slot]); + + assert!( + rendered.contains("page_patterns = [\n \"/\",\n \"/news\",\n \"/news/*\",\n]\n"), + "page patterns should be readable one-per-line" + ); + assert!( + rendered.contains( + "formats = [\n { width = 728, height = 90 },\n \ + { width = 970, height = 250 },\n \ + { width = 300, height = 250 },\n]\n" + ), + "formats should be readable one-per-line" + ); + toml::from_str::(&rendered).expect("formatted slots are valid TOML"); + } + + #[test] + fn splice_creates_section_when_absent() { + // Config with no [creative_opportunities] at all — generate should append it. + let existing = "[publisher]\ndomain = \"x\"\n\n[auction]\nenabled = true\n"; + + let out = splice_creative_slots(existing, &network_keys("222"), &header_rendered()) + .expect("should splice"); + + let value = toml::from_str::(&out).expect("valid TOML"); + assert_eq!( + value["creative_opportunities"]["gam_network_id"].as_str(), + Some("222"), + "appended section carries the discovered network id" + ); + assert_eq!( + value["creative_opportunities"]["slot"][0]["id"].as_str(), + Some("header") + ); + assert!( + value["publisher"]["domain"].as_str() == Some("x") + && value["auction"]["enabled"].as_bool() == Some(true), + "existing sections preserved when appending" + ); + } + + #[test] + fn resplice_does_not_accumulate_managed_comment() { + // A re-run splices into a config that already carries the managed + // header comment; it must keep exactly one copy, not append another. + let first = splice_creative_slots( + "[publisher]\ndomain = \"x\"\n\n[auction]\nenabled = true\n", + &network_keys("222"), + &header_rendered(), + ) + .expect("first splice"); + let second = splice_creative_slots(&first, &network_keys("222"), &header_rendered()) + .expect("second splice"); + let third = splice_creative_slots(&second, &network_keys("222"), &header_rendered()) + .expect("third splice"); + + assert_eq!( + third + .lines() + .filter(|line| line.trim() == MANAGED_SLOTS_COMMENT) + .count(), + 1, + "managed header comment must not accumulate across re-splices" + ); + toml::from_str::(&third).expect("re-spliced config stays valid TOML"); + } + + #[test] + fn splice_recognizes_inline_commented_section_header() { + // `[creative_opportunities] # comment` is valid TOML; the splice must + // update it in place instead of appending a duplicate section. + let existing = "[creative_opportunities] # ad templates\ngam_network_id = \"111\"\n\n\ + [auction] # flags\nenabled = true\n"; + + let out = splice_creative_slots(existing, &network_keys("222"), &header_rendered()) + .expect("should splice"); + + assert_eq!( + out.lines() + .filter(|line| { strip_inline_comment(line.trim()) == "[creative_opportunities]" }) + .count(), + 1, + "commented header must not be duplicated" + ); + let value = toml::from_str::(&out).expect("spliced config is valid TOML"); + assert_eq!( + value["creative_opportunities"]["gam_network_id"].as_str(), + Some("222"), + "network id updated under a commented header" + ); + assert_eq!( + value["creative_opportunities"]["slot"][0]["id"].as_str(), + Some("header") + ); + assert_eq!( + value["auction"]["enabled"].as_bool(), + Some(true), + "commented trailing section preserved" + ); + } + + #[test] + fn splice_inserts_when_no_existing_slots() { + let existing = + "[creative_opportunities]\ngam_network_id = \"111\"\n\n[auction]\nenabled = true\n"; + + let out = splice_creative_slots(existing, &network_keys("222"), &header_rendered()) + .expect("should splice"); + + let value = toml::from_str::(&out).expect("valid TOML"); + assert_eq!( + value["creative_opportunities"]["slot"][0]["id"].as_str(), + Some("header"), + "inserted slot id strips the div-gpt-ad- prefix" + ); + assert_eq!( + value["creative_opportunities"]["slot"][0]["div_id"].as_str(), + Some("div-gpt-ad-header"), + "div_id keeps the stable stem" + ); + assert!( + value["auction"]["enabled"].as_bool() == Some(true), + "auction section preserved after inserted slots" + ); + } + + #[test] + fn splice_replaces_inline_slot_array() { + let existing = "[creative_opportunities]\n\ + gam_network_id = \"111\"\n\ + slot = [{ id = \"old\", div_id = \"old\", gam_unit_path = \"/111/old\", page_patterns = [\"/\"], formats = [{ width = 300, height = 250 }] }]\n\n\ + [auction]\nenabled = true\n"; + + let out = splice_creative_slots(existing, &network_keys("222"), &header_rendered()) + .expect("should replace inline slot array"); + + let value = toml::from_str::(&out).expect("spliced config should be valid"); + let slots = value["creative_opportunities"]["slot"] + .as_array() + .expect("slots should be an array"); + assert_eq!(slots.len(), 1, "old inline slot should be removed"); + assert_eq!(slots[0]["id"].as_str(), Some("header")); + assert_eq!( + value["auction"]["enabled"].as_bool(), + Some(true), + "unrelated tables should be preserved" + ); + } + + #[test] + fn splice_replaces_inline_slot_map() { + let existing = "[creative_opportunities]\n\ + gam_network_id = \"111\"\n\ + slot = { \"0\" = { id = \"old\", div_id = \"old\", gam_unit_path = \"/111/old\", page_patterns = [\"/\"], formats = [{ width = 300, height = 250 }] } }\n"; + + let out = splice_creative_slots(existing, &network_keys("222"), &header_rendered()) + .expect("should replace inline slot map"); + + let value = toml::from_str::(&out).expect("spliced config should be valid"); + let slots = value["creative_opportunities"]["slot"] + .as_array() + .expect("slots should be an array"); + assert_eq!(slots.len(), 1, "old inline slot should be removed"); + assert_eq!(slots[0]["id"].as_str(), Some("header")); + } + + #[test] + fn merge_second_run_unions_page_patterns() { + // Existing slot on "/"; re-discovered this run with "/news/*". + let existing = existing_config( + "gam_network_id = \"222\"\n\n\ + [[slot]]\nid = \"header\"\ndiv_id = \"div-gpt-ad-header\"\n\ + gam_unit_path = \"/222/homepage/header\"\npage_patterns = [\"/\"]\n\ + formats = [{ width = 728, height = 90 }]\n", + ); + + let merged = merge_slots( + Some(&existing), + &discovered_header_slot(), + &["/news/*".to_string()], + false, + ); + + assert_eq!(merged.len(), 1, "same slot is not duplicated"); + assert_eq!( + merged[0].page_patterns, + vec!["/".to_string(), "/news/*".to_string()], + "this run's pattern is unioned into the existing slot" + ); + } + + #[test] + fn merge_second_run_unions_formats() { + let existing = existing_config( + "gam_network_id = \"222\"\n\n\ + [[slot]]\nid = \"header\"\ndiv_id = \"div-gpt-ad-header\"\n\ + gam_unit_path = \"/222/homepage/header\"\npage_patterns = [\"/\"]\n\ + formats = [{ width = 728, height = 90 }]\n", + ); + let registry = vec![collector::CollectedGptSlot { + gam_unit_path: "/222/homepage/header".to_string(), + div_id: "div-gpt-ad-header".to_string(), + sizes: vec![(728, 90), (970, 250)], + }]; + let discovered = gpt_slots::discover_gpt_slots(®istry, &[], false); + + let merged = merge_slots(Some(&existing), &discovered, &["/".to_string()], false); + + assert_eq!( + merged[0].formats, + [(728, 90, None), (970, 250, None)], + "a later audit must retain newly observed formats" + ); + } + + #[test] + fn merge_uses_longest_existing_div_prefix() { + let existing = existing_config( + "gam_network_id = \"222\"\n\n\ + [[slot]]\nid = \"broad\"\ndiv_id = \"ad-\"\n\ + gam_unit_path = \"/222/broad\"\npage_patterns = [\"/broad/*\"]\n\ + formats = [{ width = 300, height = 250 }]\n\n\ + [[slot]]\nid = \"atf\"\ndiv_id = \"ad-atf-\"\n\ + gam_unit_path = \"/222/atf\"\npage_patterns = [\"/\"]\n\ + formats = [{ width = 728, height = 90 }]\n", + ); + let registry = vec![collector::CollectedGptSlot { + gam_unit_path: "/222/atf".to_string(), + div_id: "ad-atf-0".to_string(), + sizes: vec![(728, 90)], + }]; + let discovered = gpt_slots::discover_gpt_slots(®istry, &[], false); + + let merged = merge_slots( + Some(&existing), + &discovered, + &["/news/*".to_string()], + false, + ); + + assert_eq!( + merged.len(), + 2, + "prefix match should not append a duplicate" + ); + let broad = merged + .iter() + .find(|slot| slot.id == "broad") + .expect("should keep broad slot"); + assert_eq!( + broad.page_patterns, + ["/broad/*"], + "shorter prefix should not claim the discovered div" + ); + let atf = merged + .iter() + .find(|slot| slot.id == "atf") + .expect("should keep specific slot"); + assert_eq!( + atf.page_patterns, + ["/", "/news/*"], + "longest matching prefix should receive this run's pattern" + ); + } + + #[test] + fn observed_literal_does_not_claim_numeric_siblings() { + let existing = existing_config( + "gam_network_id = \"222\"\n\n\ + [[slot]]\nid = \"ad-sidebar-1\"\ndiv_id = \"ad-sidebar-1\"\n\ + gam_unit_path = \"/222/sidebar\"\npage_patterns = [\"/\"]\n\ + formats = [{ width = 300, height = 250 }]\n", + ); + let discovered = ["ad-sidebar-1", "ad-sidebar-10", "ad-sidebar-11"] + .into_iter() + .map(|div_id| { + RenderSlot::from_evidence( + div_id, + div_id, + Some("/222/sidebar".to_string()), + [(300, 250)], + vec!["/news/*".to_string()], + false, + ) + }) + .collect(); + + let (merged, diagnostics) = + merge_render_slots_with_diagnostics(Some(&existing), discovered, false); + + assert_eq!(merged.len(), 3); + assert!(merged.iter().any(|slot| slot.id == "ad-sidebar-10")); + assert!(merged.iter().any(|slot| slot.id == "ad-sidebar-11")); + assert!(diagnostics.notes.is_empty()); + assert!(diagnostics.unobserved_existing_slot_ids.is_empty()); + } + + #[test] + fn split_sibling_warns_when_tuned_parent_fields_are_not_inherited() { + let existing = existing_config( + "gam_network_id = \"222\"\n\n\ + [[slot]]\nid = \"ad-sidebar-1\"\ndiv_id = \"ad-sidebar-1\"\n\ + gam_unit_path = \"/222/sidebar\"\npage_patterns = [\"/\"]\n\ + formats = [{ width = 300, height = 250 }]\nfloor_price = 1.5\n", + ); + let discovered = ["ad-sidebar-1", "ad-sidebar-10"] + .into_iter() + .map(|div_id| { + RenderSlot::from_evidence( + div_id, + div_id, + Some("/222/sidebar".to_string()), + [(300, 250)], + vec!["/news/*".to_string()], + false, + ) + }) + .collect(); + + let (merged, diagnostics) = + merge_render_slots_with_diagnostics(Some(&existing), discovered, false); + + let sibling = merged + .iter() + .find(|slot| slot.id == "ad-sidebar-10") + .expect("should append the distinct sibling"); + assert_eq!( + sibling.floor_price, None, + "a distinct placement must not inherit the configured parent's floor" + ); + assert_eq!(diagnostics.notes.len(), 1, "should emit one split warning"); + assert!( + diagnostics.notes[0].contains("discovered div `ad-sidebar-10`"), + "should name the split sibling, got {:?}", + diagnostics.notes + ); + assert!( + diagnostics.notes[0].contains("configured div_id prefix `ad-sidebar-1`"), + "should name the disqualified parent prefix, got {:?}", + diagnostics.notes + ); + assert!( + diagnostics.notes[0].contains("does not inherit"), + "should explain the tuned-field consequence, got {:?}", + diagnostics.notes + ); + } + + #[test] + fn refused_stem_observes_prefix_without_disqualifying_prefix_routing() { + let existing = existing_config( + "gam_network_id = \"222\"\n\n\ + [[slot]]\nid = \"ad-x\"\ndiv_id = \"ad-x\"\n\ + gam_unit_path = \"/222/ad-x\"\npage_patterns = [\"/\"]\n\ + formats = [{ width = 300, height = 250 }]\nfloor_price = 1.5\n", + ); + let discovered = vec![RenderSlot::from_evidence( + "ad-x-stable", + "ad-x-stable", + Some("/222/ad-x".to_string()), + [(300, 250)], + vec!["/news/*".to_string()], + false, + )]; + + let (merged, diagnostics) = merge_render_slots_with_observed_diagnostics( + Some(&existing), + discovered, + &["ad-x".to_string(), "ad-x-stable".to_string()], + &["ad-x-stable".to_string()], + false, + ); + + assert_eq!( + merged.len(), + 1, + "the configured prefix should absorb its sibling" + ); + assert_eq!(merged[0].page_patterns, ["/", "/news/*"]); + assert!( + diagnostics.notes.is_empty(), + "a refused stem is not a literal split boundary" + ); + assert!( + diagnostics.unobserved_existing_slot_ids.is_empty(), + "the refused stem should still prove the configured prefix was observed" + ); + } + + #[test] + fn split_sibling_warns_for_every_tuned_parent_prefix() { + let existing = existing_config( + "gam_network_id = \"222\"\n\n\ + [[slot]]\nid = \"broad\"\ndiv_id = \"ad\"\n\ + gam_unit_path = \"/222/broad\"\npage_patterns = [\"/\"]\n\ + formats = [{ width = 300, height = 250 }]\nfloor_price = 1.0\n\n\ + [[slot]]\nid = \"side\"\ndiv_id = \"ad-side\"\n\ + gam_unit_path = \"/222/side\"\npage_patterns = [\"/\"]\n\ + formats = [{ width = 300, height = 250 }]\nfloor_price = 2.0\n", + ); + let discovered = ["ad", "ad-side", "ad-sidebar"] + .into_iter() + .map(|div_id| { + RenderSlot::from_evidence( + div_id, + div_id, + Some("/222/new".to_string()), + [(300, 250)], + vec!["/news/*".to_string()], + false, + ) + }) + .collect(); + + let (_, diagnostics) = + merge_render_slots_with_diagnostics(Some(&existing), discovered, false); + + assert_eq!( + diagnostics.notes.len(), + 2, + "both tuned ancestors should be named" + ); + assert!( + diagnostics + .notes + .iter() + .any(|note| note.contains("prefix `ad`")) + ); + assert!( + diagnostics + .notes + .iter() + .any(|note| note.contains("prefix `ad-side`")) + ); + } + + #[test] + fn newly_appended_literal_does_not_claim_numeric_sibling() { + let existing = existing_config( + "gam_network_id = \"222\"\n\n\ + [[slot]]\nid = \"legacy\"\ndiv_id = \"legacy-slot\"\n\ + gam_unit_path = \"/222/legacy\"\npage_patterns = [\"/\"]\n\ + formats = [{ width = 300, height = 250 }]\n", + ); + let discovered = ["ad-sidebar-1", "ad-sidebar-10"] + .into_iter() + .map(|div_id| { + RenderSlot::from_evidence( + div_id, + div_id, + Some("/222/sidebar".to_string()), + [(300, 250)], + vec!["/news/*".to_string()], + false, + ) + }) + .collect(); + + let (merged, diagnostics) = + merge_render_slots_with_diagnostics(Some(&existing), discovered, false); + + assert_eq!(merged.len(), 3); + assert!(merged.iter().any(|slot| slot.id == "ad-sidebar-1")); + assert!(merged.iter().any(|slot| slot.id == "ad-sidebar-10")); + assert!(diagnostics.notes.is_empty()); + } + + #[test] + fn normalized_stem_is_the_literal_merge_boundary() { + let existing = existing_config( + "gam_network_id = \"222\"\n\n\ + [[slot]]\nid = \"ad-header-0\"\ndiv_id = \"ad-header-0\"\n\ + gam_unit_path = \"/222/header\"\npage_patterns = [\"/\"]\n\ + formats = [{ width = 728, height = 90 }]\n", + ); + let registry = vec![ + collector::CollectedGptSlot { + gam_unit_path: "/222/header".to_string(), + div_id: "ad-header-0-_R_3f_".to_string(), + sizes: vec![(728, 90)], + }, + collector::CollectedGptSlot { + gam_unit_path: "/222/header".to_string(), + div_id: "ad-header-01".to_string(), + sizes: vec![(728, 90)], + }, + ]; + let discovered = gpt_slots::discover_gpt_slots(®istry, &[], false); + + let merged = merge_slots(Some(&existing), &discovered, &["/".to_string()], false); + + assert_eq!(merged.len(), 2); + assert!(merged.iter().any(|slot| slot.id == "ad-header-0")); + assert!(merged.iter().any(|slot| slot.id == "ad-header-01")); + } + + #[test] + fn merge_reports_when_a_broad_prefix_claims_multiple_discovered_divs() { + let existing = existing_config( + "gam_network_id = \"222\"\n\n\ + [[slot]]\nid = \"broad\"\ndiv_id = \"ad-\"\n\ + gam_unit_path = \"/222/broad\"\npage_patterns = [\"/\"]\n\ + formats = [{ width = 300, height = 250 }]\n", + ); + let discovered = vec![ + RenderSlot::from_evidence( + "header", + "ad-header", + Some("/222/header".to_string()), + [(728, 90)], + vec!["/".to_string()], + false, + ), + RenderSlot::from_evidence( + "footer", + "ad-footer", + Some("/222/footer".to_string()), + [(300, 250)], + vec!["/".to_string()], + false, + ), + ]; + + let (merged, diagnostics) = + merge_render_slots_with_diagnostics(Some(&existing), discovered, false); + + assert_eq!( + merged.len(), + 1, + "the configured prefix still controls merging" + ); + assert_eq!(diagnostics.notes.len(), 1); + assert!(diagnostics.notes[0].contains("matched 2 discovered divs")); + assert!(diagnostics.notes[0].contains("ad-footer")); + assert!( + diagnostics.notes[0] + .contains("runtime can resolve this configured slot to at most one"), + "diagnostic should explain the runtime consequence" + ); + assert!(diagnostics.notes[0].contains("ad-header")); + } + + #[test] + fn a_slot_appended_this_run_never_absorbs_a_later_discovery() { + // Prefix reconciliation belongs to the operator's config. If a slot + // appended during this run could act as a prefix, `ad-top` would swallow + // `ad-top-sidebar` whenever discovery happened to see it first, dropping + // the absorbed slot's unit path and provider state, and no broad-prefix + // diagnostic would report it. + let existing = existing_config( + "gam_network_id = \"222\"\n\n\ + [[slot]]\nid = \"sidebar\"\ndiv_id = \"sidebar-ad\"\n\ + gam_unit_path = \"/222/sidebar\"\npage_patterns = [\"/\"]\n\ + formats = [{ width = 300, height = 600 }]\n", + ); + let candidates = [ + RenderSlot::from_evidence( + "ad-top", + "ad-top", + Some("/222/top".to_string()), + [(728, 90)], + vec!["/".to_string()], + false, + ), + RenderSlot::from_evidence( + "ad-top-sidebar", + "ad-top-sidebar", + Some("/222/top-sidebar".to_string()), + [(300, 250)], + vec!["/news/*".to_string()], + true, + ), + ]; + + for order in [[0_usize, 1], [1, 0]] { + let discovered: Vec = order + .iter() + .map(|index| candidates[*index].clone()) + .collect(); + + let (merged, diagnostics) = + merge_render_slots_with_diagnostics(Some(&existing), discovered, false); + + assert!( + diagnostics.notes.is_empty(), + "no configured prefix claimed a discovered div in order {order:?}, got {diagnostics:?}" + ); + assert_eq!( + merged.len(), + 3, + "both discovered slots must survive in order {order:?}" + ); + let sidebar_ad = merged + .iter() + .find(|slot| slot.div_id.as_deref() == Some("ad-top-sidebar")) + .unwrap_or_else(|| { + panic!("the longer div must stay its own slot in order {order:?}") + }); + assert_eq!( + sidebar_ad.gam_unit_path.as_deref(), + Some("/222/top-sidebar"), + "the absorbed slot's unit path must survive in order {order:?}" + ); + assert_eq!( + sidebar_ad.page_patterns, + ["/news/*"], + "patterns must not be pooled in order {order:?}" + ); + assert!( + sidebar_ad.prebid_bidders.is_some(), + "provider state must survive in order {order:?}" + ); + let top = merged + .iter() + .find(|slot| slot.div_id.as_deref() == Some("ad-top")) + .unwrap_or_else(|| panic!("the shorter div must stay in order {order:?}")); + assert_eq!( + top.page_patterns, + ["/"], + "the longer slot's pattern must not leak into the shorter one in order {order:?}" + ); + } + } + + #[test] + fn merge_renames_new_slot_id_that_collides_with_existing_config() { + let existing = existing_config( + "gam_network_id = \"222\"\n\n\ + [[slot]]\nid = \"header-main\"\ndiv_id = \"legacy-header\"\n\ + gam_unit_path = \"/222/legacy\"\npage_patterns = [\"/\"]\n\ + formats = [{ width = 300, height = 250 }]\n", + ); + let registry = vec![collector::CollectedGptSlot { + gam_unit_path: "/222/header".to_string(), + div_id: "div-gpt-ad-header.main".to_string(), + sizes: vec![(728, 90)], + }]; + let discovered = gpt_slots::discover_gpt_slots(®istry, &[], false); + + let merged = merge_slots( + Some(&existing), + &discovered, + &["/news/*".to_string()], + false, + ); + let ids = merged + .iter() + .map(|slot| slot.id.as_str()) + .collect::>(); + + assert_eq!(ids, ["header-main", "header-main-2"]); + } + + #[test] + fn merge_keeps_existing_only_slots() { + // Existing has header + sidebar; this run re-sees only header. + let existing = existing_config( + "gam_network_id = \"222\"\n\n\ + [[slot]]\nid = \"header\"\ndiv_id = \"div-gpt-ad-header\"\n\ + gam_unit_path = \"/222/homepage/header\"\npage_patterns = [\"/\"]\n\ + formats = [{ width = 728, height = 90 }]\n\n\ + [[slot]]\nid = \"sidebar\"\ndiv_id = \"ad-sidebar\"\n\ + gam_unit_path = \"/222/sidebar\"\npage_patterns = [\"/news/*\"]\n\ + formats = [{ width = 300, height = 250 }]\nfloor_price = 0.5\n", + ); + + let merged = merge_slots( + Some(&existing), + &discovered_header_slot(), + &["/".to_string()], + false, + ); + + let ids: Vec<&str> = merged.iter().map(|slot| slot.id.as_str()).collect(); + assert_eq!(ids, vec!["header", "sidebar"], "sidebar preserved"); + let sidebar = merged + .iter() + .find(|slot| slot.id == "sidebar") + .expect("sidebar"); + assert_eq!( + sidebar.floor_price, + Some(0.5), + "hand-tuned fields preserved" + ); + } + + #[test] + fn merge_reports_preserved_unobserved_slots_in_config_order() { + let existing = existing_config( + "gam_network_id = \"222\"\n\n\ + [[slot]]\nid = \"header\"\ndiv_id = \"div-gpt-ad-header\"\n\ + gam_unit_path = \"/222/header\"\npage_patterns = [\"/\"]\n\ + formats = [{ width = 728, height = 90 }]\n\n\ + [[slot]]\nid = \"sidebar\"\ndiv_id = \"ad-sidebar\"\n\ + gam_unit_path = \"/222/sidebar\"\npage_patterns = [\"/news/*\"]\n\ + formats = [{ width = 300, height = 250 }]\n\n\ + [[slot]]\nid = \"footer\"\ndiv_id = \"ad-footer\"\n\ + gam_unit_path = \"/222/footer\"\npage_patterns = [\"/\"]\n\ + formats = [{ width = 728, height = 90 }]\n", + ); + let discovered = vec![RenderSlot::from_evidence( + "header", + "div-gpt-ad-header", + Some("/222/header".to_string()), + [(728, 90)], + vec!["/".to_string()], + false, + )]; + + let (_, diagnostics) = + merge_render_slots_with_diagnostics(Some(&existing), discovered, false); + + assert_eq!( + diagnostics.unobserved_existing_slot_ids, + ["sidebar", "footer"], + "unobserved slots should retain configuration order" + ); + + let all_discovered = vec![ + RenderSlot::from_evidence( + "header", + "div-gpt-ad-header", + Some("/222/header".to_string()), + [(728, 90)], + vec!["/".to_string()], + false, + ), + RenderSlot::from_evidence( + "sidebar", + "ad-sidebar", + Some("/222/sidebar".to_string()), + [(300, 250)], + vec!["/news/*".to_string()], + false, + ), + RenderSlot::from_evidence( + "footer", + "ad-footer", + Some("/222/footer".to_string()), + [(728, 90)], + vec!["/".to_string()], + false, + ), + ]; + let (_, fully_observed) = + merge_render_slots_with_diagnostics(Some(&existing), all_discovered.clone(), false); + let (_, replaced) = + merge_render_slots_with_diagnostics(Some(&existing), all_discovered.clone(), true); + let (_, no_existing) = merge_render_slots_with_diagnostics(None, all_discovered, false); + + assert!( + fully_observed.unobserved_existing_slot_ids.is_empty(), + "fully observed slots should not be reported as stale" + ); + assert!( + replaced.unobserved_existing_slot_ids.is_empty(), + "--replace should not report discarded existing slots as stale" + ); + assert!( + no_existing.unobserved_existing_slot_ids.is_empty(), + "a config without existing slots should not report stale slots" + ); + } + + #[test] + fn observed_div_marks_exact_slot_and_live_broad_prefix() { + let existing = existing_config( + "gam_network_id = \"222\"\n\n\ + [[slot]]\nid = \"broad\"\ndiv_id = \"ad-\"\n\ + gam_unit_path = \"/222/broad\"\npage_patterns = [\"/\"]\n\ + formats = [{ width = 300, height = 250 }]\n\n\ + [[slot]]\nid = \"header\"\ndiv_id = \"ad-header\"\n\ + gam_unit_path = \"/222/header\"\npage_patterns = [\"/\"]\n\ + formats = [{ width = 728, height = 90 }]\n", + ); + let discovered = vec![RenderSlot::from_evidence( + "header", + "ad-header", + Some("/222/header".to_string()), + [(728, 90)], + vec!["/news/*".to_string()], + false, + )]; + + let (_, diagnostics) = merge_render_slots_with_observed_diagnostics( + Some(&existing), + discovered, + &["ad-header".to_string()], + &["ad-header".to_string()], + false, + ); + + assert!( + diagnostics.unobserved_existing_slot_ids.is_empty(), + "the exact slot and every live configured prefix should be observed, got {diagnostics:?}" + ); + } + + #[test] + fn merge_replace_wipes_existing() { + let existing = existing_config( + "gam_network_id = \"222\"\n\n\ + [[slot]]\nid = \"sidebar\"\ndiv_id = \"ad-sidebar\"\n\ + gam_unit_path = \"/222/sidebar\"\npage_patterns = [\"/\"]\n\ + formats = [{ width = 300, height = 250 }]\n", + ); + + let merged = merge_slots( + Some(&existing), + &discovered_header_slot(), + &["/".to_string()], + true, + ); + + let ids: Vec<&str> = merged.iter().map(|slot| slot.id.as_str()).collect(); + assert_eq!(ids, vec!["header"], "--replace keeps only discovered slots"); + } + + #[test] + fn resolve_network_id_prefers_discovered_unless_preserving_existing() { + let with_slots = existing_config( + "gam_network_id = \"111\"\n\n[[slot]]\nid = \"s\"\ndiv_id = \"ad-s\"\n\ + gam_unit_path = \"/111/s\"\npage_patterns = [\"/\"]\n\ + formats = [{ width = 300, height = 250 }]\n", + ); + let empty = existing_config("gam_network_id = \"111\"\n"); + + // Real merge → keep existing. + assert_eq!( + resolve_network_id(Some(&with_slots), Some("222"), false).as_deref(), + Some("111") + ); + // Placeholder section with no slots → discovered wins. + assert_eq!( + resolve_network_id(Some(&empty), Some("222"), false).as_deref(), + Some("222") + ); + // --replace → discovered wins. + assert_eq!( + resolve_network_id(Some(&with_slots), Some("222"), true).as_deref(), + Some("222") + ); + // No existing config → discovered. + assert_eq!( + resolve_network_id(None, Some("222"), false).as_deref(), + Some("222") + ); + } + + #[test] + fn toml_key_quotes_only_non_bare_keys() { + assert_eq!(toml_key("zone"), "zone"); + assert_eq!(toml_key("ad-loc"), "ad-loc"); + assert_eq!(toml_key("a.b"), "\"a.b\""); + assert_eq!(toml_key("with space"), "\"with space\""); + assert_eq!(toml_key(""), "\"\""); + } + + #[test] + fn toml_string_escapes_quotes_backslashes_and_controls() { + assert_eq!(toml_string("a\"b\\c"), "\"a\\\"b\\\\c\""); + assert_eq!(toml_string("line\nbreak\t!"), "\"line\\nbreak\\t!\""); + } + + #[test] + fn toml_string_escapes_del_control_char() { + assert_eq!(toml_string("a\u{7f}b"), "\"a\\u007Fb\""); + let doc = format!("value = {}", toml_string("a\u{7f}b")); + let value = toml::from_str::(&doc).expect("DEL escapes to valid TOML"); + assert_eq!( + value["value"].as_str(), + Some("a\u{7f}b"), + "escaped DEL round-trips as data" + ); + } + + #[test] + fn replace_key_handles_inline_commented_headers() { + let document = "[creative_opportunities] # managed\ngam_network_id = \"111\"\n\n\ + [auction] # flags\nenabled = true\n"; + + let updated = replace_key_in_section( + document, + "creative_opportunities", + "gam_network_id", + "gam_network_id = \"222\"", + ) + .expect("should find the commented section header"); + + assert!( + updated.contains("gam_network_id = \"222\""), + "key replaced under a commented header" + ); + assert!( + updated.contains("enabled = true"), + "later commented section left untouched" + ); + } + + #[test] + fn render_quotes_exotic_targeting_keys_to_valid_toml() { + let existing = existing_config( + "gam_network_id = \"1\"\n\n\ + [[slot]]\nid = \"s\"\ndiv_id = \"ad-s\"\ngam_unit_path = \"/1/s\"\n\ + page_patterns = [\"/\"]\nformats = [{ width = 300, height = 250 }]\n\ + targeting = { \"a.b\" = \"x\" }\n", + ); + + let merged = merge_slots( + Some(&existing), + &discovered_header_slot(), + &["/".to_string()], + false, + ); + let doc = format!( + "[creative_opportunities]\ngam_network_id = \"1\"\n{}", + render_slots(&merged) + ); + + toml::from_str::(&doc).expect("exotic targeting key renders as valid TOML"); + } +} diff --git a/crates/trusted-server-cli/src/commands/audit/generate/unit_template.rs b/crates/trusted-server-cli/src/commands/audit/generate/unit_template.rs new file mode 100644 index 000000000..06b7c955d --- /dev/null +++ b/crates/trusted-server-cli/src/commands/audit/generate/unit_template.rs @@ -0,0 +1,1028 @@ +//! Infers a `{network_id}`/`{section}` ad-unit template from observed evidence. +//! +//! The generator otherwise writes the literal path each page happened to +//! request, which pins a slot to the one section it was scraped from. A template +//! generalizes across sections — but a *wrong* template makes the publisher bid +//! against inventory that does not exist, which is worse than a narrow literal. +//! So this module is built to refuse rather than guess. +//! +//! The inference applies three evidence rules: +//! +//! 1. **Positional binding.** `{network_id}` is bound to unit segment 0 and only +//! if that segment is the resolved network id. Substring replacement would +//! corrupt `/123/sports123/home` into `/{network_id}/sports{network_id}/home`. +//! 2. **Exactly one varying segment.** Zero means nothing was proven and the +//! path stays literal; two means the unit varies along a dimension the +//! request path cannot supply (device, geo, experiment), so it is refused. +//! 3. **Cross-page variation.** Two pages must show *different* derived sections +//! and different unit segments. A single-page crawl is +//! indistinguishable from a static path — literal, `{network_id}`-only and +//! `{section}` all reproduce one observation equally well, and round-trip +//! verification cannot tell them apart. Only variation can. +//! +//! Every accepted template is then replayed through the runtime's own +//! [`render_gam_unit_path`](CreativeOpportunitySlot::render_gam_unit_path) and +//! [`derive_section`] against every observation. A template that does not +//! reproduce what the live page actually requested is downgraded, not written. + +use std::collections::{BTreeMap, BTreeSet}; + +use trusted_server_core::creative_opportunities::{CreativeOpportunitySlot, derive_section}; + +use super::evidence::{EvidenceTable, SlotEvidence}; +use super::slot_toml::toml_string; + +/// Candidate `section_segment` values considered, `0..=MAX_SECTION_SEGMENT`. +/// +/// A locale-prefixed site (`/en/news/story`) needs 1. Beyond 2 the "section" is +/// no longer a taxonomy the operator would recognise, and every extra candidate +/// is another chance for two indices to both fit and force a refusal. +const MAX_SECTION_SEGMENT: usize = 2; + +/// The config-level section policy an inferred template depends on. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(super) struct SectionPolicy { + /// Value substituted for `{section}` on paths with no section segment. + pub(super) section_root: String, + /// Index of the path segment `{section}` is taken from. + pub(super) section_segment: usize, +} + +/// What to write for one slot's `gam_unit_path`. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(super) enum SlotDecision { + /// Write this templated path; it reproduced every observation. + Template(String), + /// Write this literal path; nothing generalizable was proven. + Literal(String), + /// Write no path at all — the observations cannot be represented. + Refuse { + /// Operator-facing explanations, one per reason. + reasons: Vec, + }, +} + +/// The outcome of inference across the whole evidence table. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(super) struct InferenceOutcome { + /// Section policy to write, present only when some slot templated. + pub(super) policy: Option, + /// Per-slot decision, keyed by div stem, in evidence order. + pub(super) decisions: Vec<(String, SlotDecision)>, + /// Operator-facing notes about why inference went the way it did. + pub(super) diagnostics: Vec, + /// Div stems whose templates rely on a root witnessed by another slot. + pub(super) borrowed_section_root: Vec, +} + +impl InferenceOutcome { + /// The decision for a slot, by div stem. + pub(super) fn decision(&self, div_id: &str) -> Option<&SlotDecision> { + self.decisions + .iter() + .find(|(key, _)| key == div_id) + .map(|(_, decision)| decision) + } +} + +/// Per-slot analysis under one candidate `section_segment`. +#[derive(Debug, Clone, PartialEq, Eq)] +enum SlotAnalysis { + /// Templatable: unit segment `varying` tracks the derived section, and root + /// pages agreed on `section_root`. + Templatable { + varying: usize, + section_root: String, + }, + /// The unit path never varied, so nothing about `{section}` was proven. + Static, + /// Cannot be represented; carries the operator-facing reason. + Refuse(String), + /// Unit segment `varying` tracks the derived section on every page this slot + /// was seen on, but none of those pages lacked the section segment, so the + /// slot witnessed no `section_root` of its own. + /// + /// Carries `varying` because such a slot is still templatable *when another + /// slot witnessed the config-level `section_root`*: a placement that only + /// exists on section pages (a sidebar, an in-article unit) never renders on a + /// path where `{section}` would fall back to the root. + RootUnwitnessed { varying: usize }, +} + +/// Infers unit-path templates for every slot in `table`. +/// +/// `network_id` is the resolved GAM network id; `{network_id}` is only ever +/// bound to a unit segment that already equals it. +pub(super) fn infer_unit_templates(table: &EvidenceTable, network_id: &str) -> InferenceOutcome { + let slots: Vec<&SlotEvidence> = table.slots().collect(); + let mut diagnostics = Vec::new(); + + // Evaluate every candidate index independently; ambiguity between two that + // both fit is a refusal, not a preference for the smaller one. + let mut qualifying: Vec<(usize, String, BTreeMap)> = Vec::new(); + let mut root_witness_missing = false; + let mut root_unwitnessed_stems = BTreeSet::new(); + for segment in 0..=MAX_SECTION_SEGMENT { + let analyses: BTreeMap = slots + .iter() + .map(|slot| (slot.div_id.clone(), analyse_slot(slot, network_id, segment))) + .collect(); + + let roots: BTreeSet<&str> = analyses + .values() + .filter_map(|analysis| match analysis { + SlotAnalysis::Templatable { section_root, .. } => Some(section_root.as_str()), + _ => None, + }) + .collect(); + // Slots must agree: `section_root` is one config-level value, so two + // slots claiming different roots means this index is not the real one. + let Some(root) = roots.iter().next().copied() else { + // Distinguish "nothing tracks the section" from "everything does but + // no crawled page lacked the section segment": the second is a crawl + // gap the operator can close, and the generic literal-path refusal + // below does not say so. + root_witness_missing |= analyses + .values() + .any(|analysis| matches!(analysis, SlotAnalysis::RootUnwitnessed { .. })); + root_unwitnessed_stems.extend( + analyses + .iter() + .filter(|(_, analysis)| { + matches!(analysis, SlotAnalysis::RootUnwitnessed { .. }) + }) + .map(|(stem, _)| stem.clone()), + ); + continue; + }; + if roots.len() > 1 { + continue; + } + qualifying.push((segment, root.to_string(), analyses)); + } + + let chosen = match qualifying.len() { + 0 => None, + 1 => qualifying.into_iter().next(), + _ => { + let indices: Vec = qualifying + .iter() + .map(|(segment, _, _)| segment.to_string()) + .collect(); + diagnostics.push(format!( + "more than one section_segment ({}) explains the observed ad-unit paths \ + equally well, so no template can be chosen safely; slots without one safe literal path are omitted", + indices.join(", ") + )); + None + } + }; + + let Some((section_segment, section_root, analyses)) = chosen else { + // Only a crawl gap justifies rewriting the per-slot reasons. When + // inference stopped on segment ambiguity instead, that pushed its own + // diagnostic, and blaming the crawl here would send the operator to + // widen it when the remedy is pinning `section_segment`. + let root_gap = diagnostics.is_empty() && root_witness_missing; + if diagnostics.is_empty() { + diagnostics.push(if root_witness_missing { + "the ad-unit paths do track the page section, but no crawled page lacked a \ + section segment, so `section_root` could not be witnessed and no {section} \ + template can be written; include the site root in the crawl (or set \ + section_root by hand) to template these slots" + .to_string() + } else { + "no ad-unit path varied by page section across the crawl, so paths were kept \ + literal; crawl more sections to enable a {section} template" + .to_string() + }); + } + let mut decisions = literal_decisions(&slots); + if root_gap { + for (stem, decision) in &mut decisions { + if root_unwitnessed_stems.contains(stem) + && let SlotDecision::Refuse { reasons } = decision + { + *reasons = vec![ + "the paths tracked the page section, but no crawled page lacked a \ + section segment, so `section_root` could not be witnessed" + .to_string(), + ]; + } + } + } + return InferenceOutcome { + policy: None, + decisions, + diagnostics, + borrowed_section_root: Vec::new(), + }; + }; + + let mut decisions = Vec::with_capacity(slots.len()); + let mut borrowed_section_root = Vec::new(); + let mut templated = 0_usize; + for slot in &slots { + let analysis = analyses + .get(&slot.div_id) + .cloned() + .unwrap_or(SlotAnalysis::Static); + let templatable = match analysis { + SlotAnalysis::Templatable { varying, .. } => Some((varying, true)), + // The config-level `section_root` is witnessed by another slot on the + // same property, and this slot's page patterns are derived from the + // paths it was seen on — all of which carry a section segment — so + // `{section}` never falls back to the root for it. Refusing here cost + // real inventory: a sidebar or in-article unit that simply does not + // exist on the site root was omitted from the config entirely. + SlotAnalysis::RootUnwitnessed { varying } => Some((varying, false)), + SlotAnalysis::Static | SlotAnalysis::Refuse(_) => None, + }; + let decision = match (templatable, analysis) { + (Some((varying, witnessed_root)), _) => { + let template = build_template(slot, varying); + match verify_round_trip(&template, slot, network_id, §ion_root, section_segment) + { + Ok(()) => { + templated += 1; + if !witnessed_root { + borrowed_section_root.push(slot.div_id.clone()); + diagnostics.push(format!( + "slot `{}` was never observed on a page without a section \ + segment, so its `{{section}}` template relies on the \ + config-level section_root `{section_root}` witnessed by other \ + slots; it is only rendered for the paths this slot was seen on", + slot.id + )); + } + SlotDecision::Template(template) + } + Err(reason) => { + diagnostics.push(format!( + "slot `{}` template `{template}` did not reproduce the observed \ + ad-unit paths ({reason}); refusing any unsafe fallback", + slot.id + )); + literal_decision(slot) + } + } + } + (None, SlotAnalysis::Refuse(reason)) => SlotDecision::Refuse { + reasons: vec![reason], + }, + (None, _) => literal_decision(slot), + }; + decisions.push((slot.div_id.clone(), decision)); + } + + if templated == 0 { + return InferenceOutcome { + policy: None, + decisions, + diagnostics, + borrowed_section_root: Vec::new(), + }; + } + + diagnostics.push(format!( + "inferred section_segment = {section_segment} and section_root = \"{section_root}\" \ + from {} page(s); {templated} slot(s) templated", + table.pages().len() + )); + InferenceOutcome { + policy: Some(SectionPolicy { + section_root, + section_segment, + }), + decisions, + diagnostics, + borrowed_section_root, + } +} + +/// Checks the properties of a slot's observations that do not depend on which +/// `section_segment` is being considered. +/// +/// Kept separate because these refusals are final: no candidate index can +/// rescue a slot whose observations are not one template with a single hole in +/// them, and the operator needs the specific reason rather than a generic one. +/// +/// Returns the single varying unit segment, `None` when nothing varied, or the +/// reason the observations cannot be represented at all. +fn structural_check(slot: &SlotEvidence) -> Result, String> { + // One page reporting two different ad-unit paths for the same slot means the + // unit varies along something the request path cannot express — a device or + // geo split, or two profiles disagreeing. Nothing here can represent that. + let mut per_path: BTreeMap<&str, BTreeSet<&str>> = BTreeMap::new(); + for row in &slot.rows { + per_path + .entry(row.path.as_str()) + .or_default() + .insert(row.unit_path.as_str()); + } + if let Some((path, units)) = per_path.iter().find(|(_, units)| units.len() > 1) { + let observed: Vec<&str> = units.iter().copied().collect(); + return Err(format!( + "page `{path}` requested more than one ad-unit path for this slot ({}); \ + the unit varies by something the request path cannot derive", + observed.join(", ") + )); + } + + let split: Vec> = slot + .rows + .iter() + .map(|row| segments(&row.unit_path)) + .collect(); + let Some(first) = split.first() else { + return Ok(None); + }; + // Differing shapes are not one template with a hole in it. + if split.iter().any(|parts| parts.len() != first.len()) { + return Err( + "the observed ad-unit paths have different segment counts, so they are not \ + one template" + .to_string(), + ); + } + + let varying: Vec = (0..first.len()) + .filter(|index| { + split + .iter() + .map(|parts| parts[*index]) + .collect::>() + .len() + > 1 + }) + .collect(); + match varying.len() { + 0 => Ok(None), + 1 if varying[0] == 0 => { + Err("the network-id segment of the ad-unit path varied across pages".to_string()) + } + 1 => Ok(Some(varying[0])), + count => Err(format!( + "{count} ad-unit segments vary across pages, so the path does not track the \ + page section alone" + )), + } +} + +/// Analyses one slot under a candidate `section_segment`. +/// +/// [`structural_check`] has already established that a templatable candidate +/// contains more than one observed unit path. Therefore a successful derived +/// section match here is itself the required variation witness; a second +/// witness predicate would only restate that invariant. +fn analyse_slot(slot: &SlotEvidence, network_id: &str, section_segment: usize) -> SlotAnalysis { + let varying = match structural_check(slot) { + Err(reason) => return SlotAnalysis::Refuse(reason), + Ok(None) => return SlotAnalysis::Static, + Ok(Some(varying)) => varying, + }; + + let split: Vec> = slot + .rows + .iter() + .map(|row| segments(&row.unit_path)) + .collect(); + // `{network_id}` binds positionally and only to the resolved id. Substring + // replacement would rewrite an unrelated segment that merely contains it. + if split.first().and_then(|parts| parts.first()) != Some(&network_id) { + return SlotAnalysis::Static; + } + + // Partition observations into pages that have a section segment and pages + // that do not; the latter are what determine `section_root`. + let mut root_values = BTreeSet::new(); + for (row, parts) in slot.rows.iter().zip(split.iter()) { + let observed = parts[varying]; + if path_segments(&row.path).len() > section_segment { + // The empty root is unused here: the path has this segment. + if derive_section(&row.path, "", section_segment) != observed { + return SlotAnalysis::Static; + } + } else { + root_values.insert(observed); + } + } + + let mut roots = root_values.into_iter(); + let Some(section_root) = roots.next() else { + // Without a root observation, `section_root` would be a guess that + // silently mis-renders every short path. + return SlotAnalysis::RootUnwitnessed { varying }; + }; + if roots.next().is_some() { + return SlotAnalysis::Static; + } + // A root that is not `[A-Za-z0-9_-]+` makes any `{section}` template fail + // config load; catch it here rather than at push time. + if section_root.is_empty() + || !section_root + .chars() + .all(|ch| ch.is_ascii_alphanumeric() || ch == '_' || ch == '-') + { + return SlotAnalysis::Static; + } + + SlotAnalysis::Templatable { + varying, + section_root: section_root.to_string(), + } +} + +/// Builds the template text by substituting the two proven placeholders. +fn build_template(slot: &SlotEvidence, varying: usize) -> String { + let first = slot + .rows + .iter() + .next() + .map(|row| row.unit_path.as_str()) + .unwrap_or_default(); + let rendered: Vec = segments(first) + .into_iter() + .enumerate() + .map(|(index, value)| { + if index == 0 { + "{network_id}".to_string() + } else if index == varying { + "{section}".to_string() + } else { + value.to_string() + } + }) + .collect(); + format!("/{}", rendered.join("/")) +} + +/// Replays `template` through the runtime renderer against every observation. +/// +/// Defense in depth rather than the primary gate: [`analyse_slot`] already +/// refuses to call a slot templatable when the derived section and the observed +/// segment disagree — a publisher whose `/site-news` pages request +/// `.../sitenews`, say — so a mismatch reaching here would mean inference and +/// the runtime renderer disagree. The template is then dropped instead of +/// written, and the diagnostic names the paths that did not reproduce. +fn verify_round_trip( + template: &str, + slot: &SlotEvidence, + network_id: &str, + section_root: &str, + section_segment: usize, +) -> Result<(), String> { + let probe = probe_slot(template)?; + for row in &slot.rows { + let section = derive_section(&row.path, section_root, section_segment); + match probe.render_gam_unit_path(network_id, §ion) { + Some(rendered) if rendered == row.unit_path => {} + Some(rendered) => { + return Err(format!( + "on `{}` it renders `{rendered}` but the page requested `{}`", + row.path, row.unit_path + )); + } + None => { + return Err(format!( + "on `{}` it renders past the GAM ad-unit path byte limit", + row.path + )); + } + } + } + Ok(()) +} + +/// Builds a throwaway slot carrying `template`, for rendering only. +/// +/// Deserializing is how the runtime itself builds slots, so this exercises the +/// same template parsing rather than a parallel implementation. +fn probe_slot(template: &str) -> Result { + let document = format!( + "id = \"probe\"\ngam_unit_path = {}\npage_patterns = [\"/\"]\n\ + formats = [{{ width = 1, height = 1 }}]\n", + toml_string(template) + ); + toml::from_str::(&document) + .map_err(|error| format!("template is not representable in config: {error}")) +} + +/// The decision for a slot no template was proven for. +/// +/// A structural refusal wins over the generic "several paths" message, so the +/// operator sees *why* the slot could not be represented (a device split, an +/// extra varying dimension) rather than only that it could not. +fn literal_decision(slot: &SlotEvidence) -> SlotDecision { + if let Err(reason) = structural_check(slot) { + return SlotDecision::Refuse { + reasons: vec![reason], + }; + } + let units = slot.unit_paths(); + let mut found = units.iter(); + match (found.next(), found.next()) { + (Some(only), None) => SlotDecision::Literal((*only).to_string()), + (Some(_), Some(_)) => SlotDecision::Refuse { + reasons: vec![format!( + "the slot used several ad-unit paths ({}) and none generalized, so no \ + single literal path is correct", + units.into_iter().collect::>().join(", ") + )], + }, + _ => SlotDecision::Refuse { + reasons: vec!["no ad-unit path was observed for this slot".to_string()], + }, + } +} + +fn literal_decisions(slots: &[&SlotEvidence]) -> Vec<(String, SlotDecision)> { + slots + .iter() + .map(|slot| (slot.div_id.clone(), literal_decision(slot))) + .collect() +} + +/// Non-empty path segments of an ad-unit path. +fn segments(unit_path: &str) -> Vec<&str> { + unit_path + .split('/') + .filter(|part| !part.is_empty()) + .collect() +} + +/// Non-empty path segments of a request path. +fn path_segments(path: &str) -> Vec<&str> { + path.split('/').filter(|part| !part.is_empty()).collect() +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::commands::audit::generate::collector::CollectedGptSlot; + use crate::commands::audit::generate::gpt_slots::discover_gpt_slots; + + /// Folds `(path, unit_path)` observations for one div into a table. + fn table_for(div_id: &str, observations: &[(&str, &str)]) -> EvidenceTable { + let mut table = EvidenceTable::default(); + for (path, unit_path) in observations { + let registry = vec![CollectedGptSlot { + gam_unit_path: (*unit_path).to_string(), + div_id: div_id.to_string(), + sizes: vec![(728, 90)], + }]; + table.fold_page(path, &discover_gpt_slots(®istry, &[], false)); + } + table + } + + /// Folds pages carrying different slot sets into one table. + /// + /// Each entry is `(request path, [(div id, ad-unit path)])`. + fn table_for_pages(pages: &[(&str, &[(&str, &str)])]) -> EvidenceTable { + let mut table = EvidenceTable::default(); + for (path, slots) in pages { + let registry: Vec = slots + .iter() + .map(|(div_id, unit_path)| CollectedGptSlot { + gam_unit_path: (*unit_path).to_string(), + div_id: (*div_id).to_string(), + sizes: vec![(728, 90)], + }) + .collect(); + table.fold_page(path, &discover_gpt_slots(®istry, &[], false)); + } + table + } + + fn only_decision(outcome: &InferenceOutcome) -> &SlotDecision { + assert_eq!(outcome.decisions.len(), 1, "fixture should have one slot"); + &outcome.decisions[0].1 + } + + #[test] + fn templates_a_section_varying_unit_path() { + // The shape the operator writes by hand today. + let table = table_for( + "ad-header", + &[ + ("/", "/123456789/publisher/homepage"), + ("/news/story-abc", "/123456789/publisher/news"), + ("/deals/thing", "/123456789/publisher/deals"), + ], + ); + + let outcome = infer_unit_templates(&table, "123456789"); + + assert_eq!( + outcome.policy, + Some(SectionPolicy { + section_root: "homepage".to_string(), + section_segment: 0, + }) + ); + assert_eq!( + only_decision(&outcome), + &SlotDecision::Template("/{network_id}/publisher/{section}".to_string()) + ); + } + + #[test] + fn a_single_page_never_templates() { + // Literal, {network_id}-only and {section} all reproduce one observation, + // so only variation can distinguish them. This is the witness rule. + let table = table_for("ad-header", &[("/news/story", "/123/site/news")]); + + let outcome = infer_unit_templates(&table, "123"); + + assert_eq!(outcome.policy, None); + assert_eq!( + only_decision(&outcome), + &SlotDecision::Literal("/123/site/news".to_string()) + ); + } + + #[test] + fn a_static_unit_path_across_sections_stays_literal() { + let table = table_for( + "ad-header", + &[ + ("/", "/123/site/fixed"), + ("/news/story", "/123/site/fixed"), + ("/deals/x", "/123/site/fixed"), + ], + ); + + let outcome = infer_unit_templates(&table, "123"); + + assert_eq!(outcome.policy, None, "nothing varied, so nothing is proven"); + assert_eq!( + only_decision(&outcome), + &SlotDecision::Literal("/123/site/fixed".to_string()) + ); + } + + #[test] + fn a_device_split_is_refused_rather_than_guessed() { + // Two units for the SAME path: the desktop/mobile cross-check surfaces + // here, and the request path cannot express the difference. + let table = table_for( + "ad-header", + &[ + ("/news/story", "/123/desktop/news"), + ("/news/story", "/123/mobile/news"), + ], + ); + + let outcome = infer_unit_templates(&table, "123"); + + let SlotDecision::Refuse { reasons } = only_decision(&outcome) else { + panic!( + "a device split must refuse, got {:?}", + only_decision(&outcome) + ); + }; + assert!( + reasons[0].contains("more than one ad-unit path"), + "reason should name the conflict, got {reasons:?}" + ); + } + + #[test] + fn two_varying_segments_are_refused() { + let table = table_for( + "ad-header", + &[ + ("/news/story", "/123/desktop/news"), + ("/deals/x", "/123/mobile/deals"), + ], + ); + + let outcome = infer_unit_templates(&table, "123"); + + let SlotDecision::Refuse { reasons } = only_decision(&outcome) else { + panic!("two varying dimensions must refuse"); + }; + assert!( + reasons[0].contains("segments vary"), + "reason should name the extra dimension, got {reasons:?}" + ); + } + + #[test] + fn a_slug_the_path_cannot_reproduce_is_refused() { + // `/site-news` requests `.../sitenews`: the derived section and + // the observed segment differ, so the template would render the wrong + // unit. Candidate analysis rejects the inconsistent section mapping. + let table = table_for( + "ad-header", + &[ + ("/", "/123/site/homepage"), + ("/news/story", "/123/site/news"), + ("/site-news/x", "/123/site/sitenews"), + ], + ); + + let outcome = infer_unit_templates(&table, "123"); + + assert_eq!( + outcome.policy, None, + "a section whose slug is not derivable must not template" + ); + assert!(matches!( + only_decision(&outcome), + SlotDecision::Refuse { .. } + )); + } + + #[test] + fn an_unwitnessed_root_is_refused() { + // Every crawled page had a section, so `section_root` would be a guess + // that silently mis-renders the homepage. + let table = table_for( + "ad-header", + &[ + ("/news/story", "/123/site/news"), + ("/deals/x", "/123/site/deals"), + ], + ); + + let outcome = infer_unit_templates(&table, "123"); + + assert_eq!(outcome.policy, None); + let SlotDecision::Refuse { reasons } = only_decision(&outcome) else { + panic!("two literal paths and no template is not representable as one literal"); + }; + assert!( + reasons + .iter() + .any(|reason| reason.contains("section_root") && reason.contains("witnessed")), + "the per-slot reason should name the crawl gap; got {reasons:?}" + ); + assert!( + outcome + .diagnostics + .iter() + .any(|note| note.contains("section_root` could not be witnessed")), + "the crawl gap, not \"nothing generalized\", is the reason; got {:?}", + outcome.diagnostics + ); + } + + #[test] + fn a_slot_absent_from_the_root_templates_from_the_witnessed_policy() { + // The live shape behind the `ad-atf_sidebar-0` refusal: a header on the + // root and every section witnesses `section_root`, while a sidebar exists + // only on section pages. The sidebar's unit path tracks the section just + // as well, and its page patterns never cover the root, so refusing it + // dropped real inventory from the config. + let mut table = EvidenceTable::default(); + let pages: &[(&str, &[(&str, &str)])] = &[ + ("/", &[("ad-header", "/123/site/homepage")]), + ( + "/news/story", + &[ + ("ad-header", "/123/site/news"), + ("ad-sidebar", "/123/site/news"), + ], + ), + ( + "/deals/x", + &[ + ("ad-header", "/123/site/deals"), + ("ad-sidebar", "/123/site/deals"), + ], + ), + ]; + for (path, slots) in pages { + let registry: Vec = slots + .iter() + .map(|(div_id, unit_path)| CollectedGptSlot { + gam_unit_path: (*unit_path).to_string(), + div_id: (*div_id).to_string(), + sizes: vec![(728, 90)], + }) + .collect(); + table.fold_page(path, &discover_gpt_slots(®istry, &[], false)); + } + + let outcome = infer_unit_templates(&table, "123"); + + assert_eq!( + outcome.policy, + Some(SectionPolicy { + section_root: "homepage".to_string(), + section_segment: 0, + }), + "the header witnesses the config-level policy" + ); + assert_eq!( + outcome.decision("ad-sidebar"), + Some(&SlotDecision::Template( + "/{network_id}/site/{section}".to_string() + )), + "a slot that only exists on section pages is still templatable" + ); + assert_eq!( + outcome.decision("ad-header"), + Some(&SlotDecision::Template( + "/{network_id}/site/{section}".to_string() + )) + ); + assert_eq!( + outcome.borrowed_section_root, + ["ad-sidebar".to_string()], + "the outcome should identify templates whose safety depends on derived patterns" + ); + assert!( + outcome.diagnostics.iter().any(|note| note + .contains("`ad-sidebar` was never observed on a page without a section segment")), + "the borrowed section_root should be stated; got {:?}", + outcome.diagnostics + ); + } + + #[test] + fn segment_ambiguity_does_not_blame_the_crawl_for_an_unwitnessed_root() { + // `ad-header` fits section_segment 0 and `ad-locale` fits 1, so + // inference stops on ambiguity. `ad-deep` is separately + // `RootUnwitnessed` at segment 2. Its refusal must not tell the + // operator to widen the crawl when the remedy is pinning + // `section_segment`. + let table = table_for_pages(&[ + ("/", &[("ad-header", "/99/site/home")]), + ("/news", &[("ad-header", "/99/site/news")]), + ("/en", &[("ad-locale", "/99/site/en-root")]), + ("/en/news", &[("ad-locale", "/99/site/news")]), + ("/a/b/news", &[("ad-deep", "/99/site/news")]), + ("/a/b/deals", &[("ad-deep", "/99/site/deals")]), + ]); + + let outcome = infer_unit_templates(&table, "99"); + + assert!( + outcome + .diagnostics + .iter() + .any(|note| note.contains("more than one section_segment")), + "the fixture should stop on ambiguity; got {:?}", + outcome.diagnostics + ); + assert!( + !outcome + .diagnostics + .iter() + .any(|note| note.contains("include the site root in the crawl")), + "an ambiguous run must not also blame the crawl; got {:?}", + outcome.diagnostics + ); + let Some(SlotDecision::Refuse { reasons }) = outcome.decision("ad-deep") else { + panic!("expected a refusal, got {:?}", outcome.decision("ad-deep")); + }; + assert!( + reasons + .iter() + .all(|reason| !reason.contains("no crawled page lacked a section segment")), + "the crawl-gap reason belongs only to a run that stopped on the crawl gap; got {reasons:?}" + ); + } + + #[test] + fn a_locale_prefixed_site_infers_the_deeper_segment() { + let table = table_for( + "ad-header", + &[ + ("/en", "/123/site/homepage"), + ("/en/news/story", "/123/site/news"), + ("/en/deals/x", "/123/site/deals"), + ], + ); + + let outcome = infer_unit_templates(&table, "123"); + + assert_eq!( + outcome.policy, + Some(SectionPolicy { + section_root: "homepage".to_string(), + section_segment: 1, + }), + "the locale prefix should push the section one segment deeper" + ); + } + + #[test] + fn network_id_is_bound_positionally_not_by_substring() { + // `sports123` merely contains the network id; substring replacement + // would corrupt it into `sports{network_id}`. + let table = table_for( + "ad-header", + &[ + ("/", "/123/sports123/homepage"), + ("/news/story", "/123/sports123/news"), + ("/deals/x", "/123/sports123/deals"), + ], + ); + + let outcome = infer_unit_templates(&table, "123"); + + assert_eq!( + only_decision(&outcome), + &SlotDecision::Template("/{network_id}/sports123/{section}".to_string()), + "only segment 0 may become {{network_id}}" + ); + } + + #[test] + fn a_unit_path_not_starting_with_the_network_id_stays_literal() { + let table = table_for( + "ad-header", + &[ + ("/", "/999/site/homepage"), + ("/news/story", "/999/site/news"), + ], + ); + + let outcome = infer_unit_templates(&table, "123"); + + assert_eq!( + outcome.policy, None, + "segment 0 must equal the resolved network id" + ); + } + + #[test] + fn differing_segment_counts_are_refused() { + let table = table_for( + "ad-header", + &[ + ("/", "/123/site/homepage"), + ("/news/story", "/123/site/news/extra"), + ], + ); + + let outcome = infer_unit_templates(&table, "123"); + + let SlotDecision::Refuse { reasons } = only_decision(&outcome) else { + panic!("differing shapes are not one template"); + }; + assert!( + reasons[0].contains("segment counts"), + "reason should name the shape mismatch, got {reasons:?}" + ); + } + + #[test] + fn a_static_slot_stays_literal_alongside_a_templated_one() { + let mut table = EvidenceTable::default(); + for (path, section_unit) in [ + ("/", "homepage"), + ("/news/story", "news"), + ("/deals/x", "deals"), + ] { + let registry = vec![ + CollectedGptSlot { + gam_unit_path: format!("/123/site/{section_unit}"), + div_id: "ad-header".to_string(), + sizes: vec![(728, 90)], + }, + CollectedGptSlot { + gam_unit_path: "/123/site/sticky".to_string(), + div_id: "ad-sticky".to_string(), + sizes: vec![(300, 250)], + }, + ]; + table.fold_page(path, &discover_gpt_slots(®istry, &[], false)); + } + + let outcome = infer_unit_templates(&table, "123"); + + assert!(outcome.policy.is_some(), "the varying slot should template"); + assert_eq!( + outcome.decision("ad-header"), + Some(&SlotDecision::Template( + "/{network_id}/site/{section}".to_string() + )) + ); + assert_eq!( + outcome.decision("ad-sticky"), + Some(&SlotDecision::Literal("/123/site/sticky".to_string())), + "a genuinely static slot must not be dragged into the template" + ); + } + + #[test] + fn diagnostics_explain_why_nothing_templated() { + let table = table_for("ad-header", &[("/news/story", "/123/site/news")]); + + let outcome = infer_unit_templates(&table, "123"); + + assert!( + outcome + .diagnostics + .iter() + .any(|note| note.contains("crawl more sections")), + "the operator should learn why, got {:?}", + outcome.diagnostics + ); + } +} diff --git a/crates/trusted-server-cli/src/commands/audit/generate/validate.rs b/crates/trusted-server-cli/src/commands/audit/generate/validate.rs new file mode 100644 index 000000000..2178a1d5d --- /dev/null +++ b/crates/trusted-server-cli/src/commands/audit/generate/validate.rs @@ -0,0 +1,119 @@ +//! Write-side validation for generated ad-template config. +//! +//! Everything the generator writes is derived from a live, page-controlled ad +//! stack, so the candidate document has to clear the same bar the runtime +//! applies at startup *before* it replaces the operator's file. A config the +//! runtime rejects is not a degraded ad stack — `build_state` fails and the +//! adapter answers every route from the startup error router, so an unloadable +//! `trusted-server.toml` is a full-site outage once pushed. + +use trusted_server_core::settings::Settings; + +use crate::error::{CliResult, cli_error}; + +/// Validates the candidate config text the generator is about to persist. +/// +/// Runs [`Settings::from_toml`], which drives the identical +/// `finalize_deserialized` chain the runtime uses — serde (`deny_unknown_fields` +/// plus required fields), then `compile_slots` → `compile_unit_templates` → +/// `validate_runtime`, then the validator pass — with no I/O. +/// +/// `baseline` is the config as it was read from disk. When the baseline is +/// *already* unloadable, this run cannot be blamed for it: the candidate is +/// accepted and the pre-existing error is returned as a warning instead. Without +/// that escape hatch a freshly bootstrapped config carrying placeholder secrets +/// could never be updated by `generate`. +/// +/// # Errors +/// +/// Returns a user-facing error when the candidate fails to load and the baseline +/// loaded cleanly — that is, when this run introduced the failure. +pub(super) fn check_candidate(candidate: &str, baseline: &str) -> CliResult> { + let Err(candidate_error) = Settings::from_toml(candidate) else { + return Ok(Vec::new()); + }; + + if let Err(baseline_error) = Settings::from_toml(baseline) { + return Ok(vec![format!( + "target config was already invalid before this run, so the generated \ + result could not be verified: {baseline_error}" + )]); + } + + cli_error(format!( + "refusing to write: the generated config would fail to load, which would \ + take the service down once pushed: {candidate_error}" + )) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// A minimal config that loads cleanly, used as the valid baseline. + fn baseline() -> String { + crate::commands::config::init::EXAMPLE_CONFIG + .replace( + "password = \"handler_password\"", + "password = \"test-admin-password-32-bytes-minimum\"", + ) + .replace( + "passphrase = \"ec_passphrase\"", + "passphrase = \"test-ec-passphrase-32-bytes-minimum\"", + ) + .replace( + "proxy_secret = \"publisher_proxy_secret\"", + "proxy_secret = \"test-proxy-secret-32-bytes-minimum\"", + ) + } + + #[test] + fn valid_candidate_passes_without_warnings() { + let config = baseline(); + + let warnings = check_candidate(&config, &config).expect("should accept valid candidate"); + + assert!( + warnings.is_empty(), + "a clean candidate should not warn, got {warnings:?}" + ); + } + + #[test] + fn candidate_this_run_broke_is_refused() { + let good = baseline(); + // An empty div_id override is exactly what a div id normalized down to + // nothing would produce, and `validate_runtime` rejects it. + let broken = format!( + "{good}\n[[creative_opportunities.slot]]\n\ + id = \"broken\"\ndiv_id = \"\"\n\ + page_patterns = [\"/\"]\n\ + formats = [{{ width = 300, height = 250 }}]\n" + ); + + let error = check_candidate(&broken, &good).expect_err("should refuse a broken candidate"); + + assert!( + format!("{error:?}").contains("refusing to write"), + "error should name the refusal, got {error:?}" + ); + } + + #[test] + fn pre_existing_breakage_downgrades_to_a_warning() { + // The operator's file was already unloadable; `generate` must still be + // able to update it rather than blaming this run for the old error. + let broken_baseline = "[creative_opportunities]\n"; + let broken_candidate = "[creative_opportunities]\n"; + + let warnings = check_candidate(broken_candidate, broken_baseline) + .expect("a pre-existing failure should not block the write"); + + assert_eq!(warnings.len(), 1, "should surface exactly one warning"); + assert!( + warnings[0].contains("already invalid"), + "warning should name the pre-existing failure, got {:?}", + warnings[0] + ); + } +} diff --git a/crates/trusted-server-cli/src/commands/audit/mod.rs b/crates/trusted-server-cli/src/commands/audit/mod.rs index 2f473defd..211060662 100644 --- a/crates/trusted-server-cli/src/commands/audit/mod.rs +++ b/crates/trusted-server-cli/src/commands/audit/mod.rs @@ -1,710 +1,611 @@ -mod analyzer; -pub(crate) mod browser_collector; -pub(crate) mod collector; - -use std::collections::BTreeSet; -use std::fs; -use std::io::Write; -use std::path::{Path, PathBuf}; - -use serde::Serialize; -use url::Url; - -use crate::commands::audit::collector::AuditCollector; -use crate::commands::config::init::EXAMPLE_CONFIG; -use crate::error::{CliResult, cli_error, report_error}; +//! Browser-backed `ts audit` command namespace. +//! +//! `ts audit page ` is the generic page audit; `ts audit ad-templates verify +//! ...` is the ad-template verifier; `ts audit generate ` bootstraps a +//! draft config from a live page (issue #800). `ts audit ` is a hidden +//! compatibility alias for `ts audit generate `. + +pub mod ad_templates; +pub mod browser; +mod browser_scroll; +pub mod collector; +pub mod generate; +pub mod page; + +use clap::{Args, Subcommand}; + +use crate::app_config::AppConfigArgs; +use crate::commands::audit::collector::{BrowserOpts, GenerateBrowserOpts}; +use crate::commands::audit::page::PageAuditArgs; +use crate::error::{CliResult, cli_error}; +use crate::run::RunOutcome; + +/// Parses and validates an `http`/`https` URL, rejecting all other schemes. +/// +/// # Errors +/// +/// Returns a user-facing string when the input is not a valid `http`/`https` URL. +pub(crate) fn parse_http_url(raw: &str) -> Result { + let url = url::Url::parse(raw).map_err(|error| format!("invalid URL `{raw}`: {error}"))?; + match url.scheme() { + "http" | "https" => Ok(url), + other => Err(format!( + "unsupported URL scheme `{other}` (expected http or https)" + )), + } +} -use analyzer::{analyze_collected_page, extract_gtm_container_id}; +/// Parses a `name=value` cookie argument into its `(name, value)` parts. +/// +/// Splits on the first `=` so cookie values may themselves contain `=`. The name +/// must be non-empty; the value may be empty. +/// +/// # Errors +/// +/// Returns a user-facing string when the input has no `=` or an empty name. +pub(crate) fn parse_cookie(raw: &str) -> Result<(String, String), String> { + let (name, value) = raw + .split_once('=') + .ok_or_else(|| format!("invalid cookie `{raw}` (expected NAME=VALUE)"))?; + if name.is_empty() { + return Err(format!("invalid cookie `{raw}` (empty name)")); + } + Ok((name.to_string(), value.to_string())) +} -/// Arguments for the `ts audit` command. -#[derive(Debug, clap::Args)] +/// `ts audit` arguments: an optional subcommand plus a hidden legacy URL positional. +#[derive(Debug, Args)] +#[command(arg_required_else_help = true)] pub(crate) struct AuditArgs { - /// Public HTTP(S) URL to audit. - pub(crate) url: String, + #[command(subcommand)] + pub(crate) command: Option, + /// Hidden compatibility alias: `ts audit ` behaves like `ts audit generate `. + /// + /// The hidden flags below all `requires` this positional, so putting one + /// before a subcommand (`ts audit --chrome X generate `) is rejected + /// rather than silently dropped. `value_name` keeps that rejection from + /// naming the field: an operator told to supply `` cannot find + /// it in `--help`, because the alias is deliberately undocumented. + #[arg(value_parser = parse_http_url, hide = true, value_name = "URL")] + pub(crate) legacy_url: Option, + #[command(flatten)] + pub(crate) legacy_generate: LegacyGenerateArgs, +} + +/// Hidden generation flags retained for the legacy `ts audit ` form. +#[derive(Debug, Default, Args)] +pub(crate) struct LegacyGenerateArgs { /// JavaScript asset audit output path. - #[arg(long)] + #[arg(long, hide = true, requires = "legacy_url")] pub(crate) js_assets: Option, /// Draft Trusted Server config output path. - #[arg(long)] + #[arg(long, hide = true, requires = "legacy_url")] pub(crate) config: Option, /// Do not write the JavaScript asset audit file. - #[arg(long)] + #[arg(long, hide = true, requires = "legacy_url")] pub(crate) no_js_assets: bool, /// Do not write the draft Trusted Server config file. - #[arg(long)] + #[arg(long, hide = true, requires = "legacy_url")] pub(crate) no_config: bool, /// Overwrite existing output files. - #[arg(long)] + #[arg(long, hide = true, requires = "legacy_url")] pub(crate) force: bool, + /// Cookie to send with the page request, as `name=value`. Repeatable. + #[arg( + long = "cookie", + value_name = "NAME=VALUE", + value_parser = parse_cookie, + hide = true, + requires = "legacy_url" + )] + pub(crate) cookies: Vec<(String, String)>, + #[command(flatten)] + pub(crate) browser: LegacyBrowserOpts, } -const DEFAULT_JS_ASSETS_PATH: &str = "js-assets.toml"; -const DEFAULT_CONFIG_PATH: &str = "trusted-server.toml"; - -#[derive(Debug, Clone, Serialize, PartialEq, Eq)] -#[serde(rename_all = "kebab-case")] -pub(crate) enum AssetParty { - FirstParty, - ThirdParty, -} - -#[derive(Debug, Clone, Serialize, PartialEq, Eq)] -pub(crate) struct AuditedAsset { - pub(crate) kind: String, - pub(crate) url: String, - pub(crate) host: String, - pub(crate) party: AssetParty, - #[serde(skip_serializing_if = "Option::is_none")] - pub(crate) integration: Option, +/// Hidden browser flags retained for the legacy `ts audit ` form. +#[derive(Debug, Args)] +pub(crate) struct LegacyBrowserOpts { + /// Path to the Chrome/Chromium executable. + #[arg(long, hide = true, requires = "legacy_url")] + pub(crate) chrome: Option, + /// Run a visible browser instead of Chrome's new headless mode. + #[arg(long, hide = true, requires = "legacy_url")] + pub(crate) headful: bool, + /// Do not answer the standard IAB consent APIs for the fresh audit profile. + #[arg(long, hide = true, requires = "legacy_url")] + pub(crate) no_assume_consent: bool, + /// Route the browser through this proxy. + #[arg(long, value_name = "HOST:PORT", hide = true, requires = "legacy_url")] + pub(crate) browser_proxy: Option, + /// Quiet window in milliseconds that marks the page settled. + #[arg( + long, + default_value_t = crate::commands::audit::collector::GENERATE_SETTLE_QUIET_MS, + hide = true, + requires = "legacy_url" + )] + pub(crate) settle_quiet_ms: u64, + /// Hard cap in milliseconds on waiting for the page to settle. + #[arg( + long, + default_value_t = crate::commands::audit::collector::GENERATE_SETTLE_MAX_MS, + hide = true, + requires = "legacy_url" + )] + pub(crate) settle_max_ms: u64, + /// Navigate to origins whose TLS certificate does not validate. + #[arg(long, hide = true, requires = "legacy_url")] + pub(crate) danger_accept_invalid_certs: bool, } -#[derive(Debug, Clone, Serialize, PartialEq, Eq)] -pub(crate) struct DetectedIntegration { - pub(crate) id: String, - pub(crate) evidence: String, -} - -#[derive(Debug, Clone, Serialize, PartialEq, Eq)] -pub(crate) struct AuditArtifact { - pub(crate) audited_url: String, - #[serde(skip_serializing_if = "Option::is_none")] - pub(crate) page_title: Option, - pub(crate) js_asset_count: usize, - pub(crate) third_party_asset_count: usize, - pub(crate) detected_integrations: Vec, - pub(crate) assets: Vec, - pub(crate) warnings: Vec, -} - -#[derive(Debug, Clone)] -pub(crate) struct AuditOutputs { - pub(crate) artifact: AuditArtifact, - pub(crate) js_assets_toml: String, - pub(crate) draft_config_toml: String, -} - -#[derive(Debug, Clone, PartialEq, Eq)] -struct AuditOutputPlan { - js_assets_path: Option, - config_path: Option, -} - -pub(crate) fn run_audit( - args: &AuditArgs, - collector: &dyn AuditCollector, - out: &mut dyn Write, -) -> CliResult<()> { - let target_url = parse_audit_url(&args.url)?; - let plan = resolve_output_plan(args)?; - let collected = collector.collect_page(&target_url)?; - let outputs = build_audit_outputs(&collected)?; - let wrote_config = plan.config_path.is_some(); - let written = write_audit_outputs(&outputs, &plan)?; - write_success_summary(&outputs, &written, wrote_config, out) -} - -fn parse_audit_url(value: &str) -> CliResult { - let url = Url::parse(value) - .map_err(|error| report_error(format!("invalid audit URL `{value}`: {error}")))?; - if !matches!(url.scheme(), "http" | "https") { - return cli_error(format!( - "`ts audit` only supports http/https URLs, got `{}`", - url.scheme() - )); +impl Default for LegacyBrowserOpts { + fn default() -> Self { + Self { + chrome: None, + headful: false, + no_assume_consent: false, + browser_proxy: None, + settle_quiet_ms: crate::commands::audit::collector::GENERATE_SETTLE_QUIET_MS, + settle_max_ms: crate::commands::audit::collector::GENERATE_SETTLE_MAX_MS, + danger_accept_invalid_certs: false, + } } - Ok(url) } -fn resolve_output_plan(args: &AuditArgs) -> CliResult { - if args.no_js_assets && args.no_config { - return cli_error("nothing to do: both --no-js-assets and --no-config were set"); - } - - let js_assets_path = if args.no_js_assets { - None - } else { - Some(resolve_output_path( - args.js_assets.as_deref(), - DEFAULT_JS_ASSETS_PATH, - )?) - }; - let config_path = if args.no_config { - None - } else { - Some(resolve_output_path( - args.config.as_deref(), - DEFAULT_CONFIG_PATH, - )?) - }; - - if js_assets_path.is_some() && js_assets_path == config_path { - return cli_error("audit output paths must be distinct"); - } - - for path in [&js_assets_path, &config_path].into_iter().flatten() { - if path.exists() && !args.force { - return cli_error(format!( - "refusing to overwrite existing file `{}`; re-run with --force", - path.display() - )); +impl From<&LegacyBrowserOpts> for GenerateBrowserOpts { + fn from(options: &LegacyBrowserOpts) -> Self { + Self { + chrome: options.chrome.clone(), + headful: options.headful, + no_assume_consent: options.no_assume_consent, + browser_proxy: options.browser_proxy.clone(), + settle_quiet_ms: options.settle_quiet_ms, + settle_max_ms: options.settle_max_ms, + danger_accept_invalid_certs: options.danger_accept_invalid_certs, } } +} - Ok(AuditOutputPlan { - js_assets_path, - config_path, - }) +/// `ts audit` subcommands. +#[derive(Debug, Subcommand)] +pub(crate) enum AuditSubcommand { + /// Audit a single page and print a read-only summary. + Page(PageAuditArgs), + /// Verify configured ad-template slots against live page evidence. + #[command(name = "ad-templates", subcommand)] + AdTemplates(AuditAdTemplatesCommand), + /// Bootstrap a draft Trusted Server config + JS asset audit from a live page. + Generate(generate::GenerateArgs), } -fn resolve_output_path(path: Option<&Path>, default: &str) -> CliResult { - let candidate = path.unwrap_or_else(|| Path::new(default)); - if candidate.is_absolute() { - Ok(candidate.to_path_buf()) - } else { - Ok(std::env::current_dir() - .map_err(|error| report_error(format!("failed to read current directory: {error}")))? - .join(candidate)) - } +/// `ts audit ad-templates` subcommands. +#[derive(Debug, Subcommand)] +pub(crate) enum AuditAdTemplatesCommand { + /// Scrape a live page's GPT slots and update the config's + /// `[creative_opportunities]` slots in place. + Generate(AuditAdTemplatesGenerateArgs), + /// Verify ad-template slots for one or more live URLs. + Verify(AuditAdTemplatesVerifyArgs), } -fn build_audit_outputs(collected: &collector::CollectedPage) -> CliResult { - let artifact = analyze_collected_page(collected)?; - let final_url = collected - .final_url() - .map_err(|error| report_error(format!("invalid final URL: {error}")))?; - let js_assets_toml = toml::to_string_pretty(&artifact) - .map_err(|error| report_error(format!("failed to serialize audit artifact: {error}")))?; - let draft_config_toml = build_draft_config(&final_url, &artifact)?; - - Ok(AuditOutputs { - artifact, - js_assets_toml, - draft_config_toml, - }) +/// Arguments for `ts audit ad-templates generate `. +#[derive(Debug, Args)] +pub(crate) struct AuditAdTemplatesGenerateArgs { + #[command(flatten)] + pub config: AppConfigArgs, + /// Page URL to scrape for GPT slots (http or https). + #[arg(value_parser = parse_http_url)] + pub url: url::Url, + /// Glob applied to every slot discovered this run (e.g. `/`, `/news/*`). + /// Repeatable. Defaults to the scraped URL's path. Re-running with a + /// different pattern unions it into slots already in the config. + #[arg(long = "page-pattern", value_name = "GLOB")] + pub page_patterns: Vec, + /// Replace all existing slots instead of merging this run into them. + #[arg(long)] + pub replace: bool, + /// Preview the updated config on stdout instead of writing it. + #[arg(long)] + pub dry_run: bool, + /// Perform a deterministic scroll pass after each page initially settles. + #[arg(long)] + pub scroll: bool, + /// Cookie to send with the page request, as `name=value`. Repeatable. + /// Use to carry an existing session (e.g. a valid bot-protection clearance + /// cookie) so the origin serves the real page instead of a challenge. + #[arg(long = "cookie", value_name = "NAME=VALUE", value_parser = parse_cookie)] + pub cookies: Vec<(String, String)>, + /// Maximum site sections to sample. Each contributes a landing page and an + /// article, so this bounds how much of the publisher's taxonomy is covered. + #[arg(long, default_value_t = 8)] + pub max_sections: usize, + /// Maximum pages to load in total, including the requested page. + /// + /// Set to 1 to restore single-page behavior: no crawl, no section + /// discovery, and the audited path as the only page pattern. + #[arg(long, default_value_t = 17)] + pub max_pages: usize, + /// Device profiles to audit, comma-separated: `desktop`, `mobile`. + /// + /// Defaults to `desktop`. Publishers often serve different GAM ad units per + /// device, which a single-profile crawl cannot see — it would infer a + /// template correct for the profile it used and silently wrong elsewhere. + /// Passing both crawls each page twice and refuses to write an ad-unit path + /// for any slot where the profiles disagree. + #[arg(long, value_delimiter = ',', default_value = "desktop")] + pub profiles: Vec, + /// Pause in milliseconds between page loads during the crawl. + /// + /// A crawl issues a dozen navigations in a row. Firing them back to back is + /// discourteous to the origin, and request pacing is one of the signals bot + /// protection scores, so an unpaced crawl can trigger the challenge that + /// empties the rest of the run. + #[arg(long, default_value_t = 750)] + pub page_delay_ms: u64, + /// Browser and consent options shared with `ts audit generate`. + #[command(flatten)] + pub browser: GenerateBrowserOpts, } -fn write_audit_outputs(outputs: &AuditOutputs, plan: &AuditOutputPlan) -> CliResult> { - let selected_paths = [&plan.js_assets_path, &plan.config_path] - .into_iter() - .flatten() - .collect::>(); - for path in &selected_paths { - if let Some(parent) = path - .parent() - .filter(|parent| !parent.as_os_str().is_empty()) - { - fs::create_dir_all(parent).map_err(|error| { - report_error(format!( - "failed to create parent directory {}: {error}", - parent.display() - )) - })?; +impl AuditAdTemplatesGenerateArgs { + /// The crawl bounds these arguments describe. + pub(crate) fn budget(&self) -> generate::CrawlBudget { + generate::CrawlBudget { + max_sections: self.max_sections, + max_pages: self.max_pages, } } - let mut written_paths = Vec::new(); - if let Some(path) = &plan.js_assets_path { - fs::write(path, &outputs.js_assets_toml).map_err(|error| { - report_error(format!( - "failed to write JS asset audit {}: {error}", - path.display() - )) - })?; - written_paths.push(path.display().to_string()); - } - if let Some(path) = &plan.config_path { - fs::write(path, &outputs.draft_config_toml).map_err(|error| { - report_error(format!( - "failed to write draft config {}: {error}", - path.display() - )) - })?; - written_paths.push(path.display().to_string()); + /// The device profiles to audit, deduplicated in the order given. + /// + /// # Errors + /// + /// Returns an error when a name is not a known profile, or when none were + /// given. + pub(crate) fn profiles(&self) -> Result, String> { + let mut profiles: Vec = Vec::new(); + for raw in &self.profiles { + let profile = generate::DeviceProfile::parse(raw)?; + if !profiles.contains(&profile) { + profiles.push(profile); + } + } + if profiles.is_empty() { + return Err("--profiles needs at least one of: desktop, mobile".to_string()); + } + Ok(profiles) } - - Ok(written_paths) } -fn write_success_summary( - outputs: &AuditOutputs, - written: &[String], - wrote_config: bool, - out: &mut dyn Write, -) -> CliResult<()> { - let integrations = outputs - .artifact - .detected_integrations - .iter() - .map(|integration| integration.id.as_str()) - .collect::>(); - let draft_note = if wrote_config { - "\nDraft config: review before validation and push" - } else { - "" - }; - writeln!( - out, - "Audited {}\nTitle: {}\nJS assets: {}\nThird-party assets: {}\nDetected integrations: {}\nWrote: {}{}", - outputs.artifact.audited_url, - outputs - .artifact - .page_title - .as_deref() - .unwrap_or(""), - outputs.artifact.js_asset_count, - outputs.artifact.third_party_asset_count, - if integrations.is_empty() { - "none".to_string() - } else { - integrations.join(", ") - }, - if written.is_empty() { - "none".to_string() - } else { - written.join(", ") - }, - draft_note - ) - .map_err(|error| report_error(format!("failed to write command output: {error}"))) +/// Arguments for `ts audit ad-templates verify ...`. +#[derive(Debug, Args)] +pub(crate) struct AuditAdTemplatesVerifyArgs { + #[command(flatten)] + pub config: AppConfigArgs, + /// One or more page URLs to verify (http or https). + #[arg(required = true, value_parser = parse_http_url)] + pub urls: Vec, + /// Exit non-zero when a matched slot is missing or only partially confirmed. + #[arg(long)] + pub strict: bool, + /// Emit machine-readable JSON instead of human output. + #[arg(long)] + pub json: bool, + /// Perform a deterministic scroll pass after the initial settle. + #[arg(long)] + pub scroll: bool, + /// Accept evidence from a page that redirected to a different origin. + /// + /// Off by default: slots are matched on the post-redirect path, so an + /// off-origin page could otherwise satisfy `--strict`. Enable only for a + /// known redirect between your own properties (e.g. apex to `www`). + #[arg(long)] + pub allow_cross_origin_redirect: bool, + /// Cookie to send with each page request, as `name=value`. Repeatable. + /// Use to carry an existing session (e.g. a valid bot-protection clearance + /// cookie) so the origin serves the real page instead of a challenge. + #[arg(long = "cookie", value_name = "NAME=VALUE", value_parser = parse_cookie)] + pub cookies: Vec<(String, String)>, + #[command(flatten)] + pub browser: BrowserOpts, } -fn build_draft_config(target_url: &Url, artifact: &AuditArtifact) -> CliResult { - let host = target_url - .host_str() - .ok_or_else(|| report_error("audited URL is missing a host"))?; - let origin = target_url.origin().ascii_serialization(); - let mut draft = EXAMPLE_CONFIG.to_string(); - - draft = replace_key_in_section( - &draft, - "publisher", - "domain", - &format!("domain = \"{host}\""), - )?; - draft = replace_key_in_section( - &draft, - "publisher", - "cookie_domain", - &format!("cookie_domain = \".{host}\""), - )?; - draft = replace_key_in_section( - &draft, - "publisher", - "origin_url", - &format!("origin_url = \"{origin}\""), - )?; - - let detected = artifact - .detected_integrations - .iter() - .map(|integration| integration.id.as_str()) - .collect::>(); - - if detected.contains("gpt") { - draft = replace_key_in_section(&draft, "integrations.gpt", "enabled", "enabled = true")?; - } - if detected.contains("didomi") { - draft = replace_key_in_section(&draft, "integrations.didomi", "enabled", "enabled = true")?; - } - if detected.contains("datadome") { - draft = - replace_key_in_section(&draft, "integrations.datadome", "enabled", "enabled = true")?; - } - - let mut manual_review = Vec::new(); - if detected.contains("google_tag_manager") { - if let Some(gtm_id) = extract_gtm_container_id(artifact) { - draft = replace_key_in_section( - &draft, - "integrations.google_tag_manager", - "enabled", - "enabled = true", - )?; - draft = replace_key_in_section( - &draft, - "integrations.google_tag_manager", - "container_id", - &format!("container_id = \"{gtm_id}\""), - )?; - } else { - manual_review.push("google_tag_manager"); +/// Dispatches a `ts audit` invocation. +/// +/// `legacy_url` (if present) routes to artifact generation, while the `page` +/// subcommand routes to the generic read-only page audit. +/// +/// # Errors +/// +/// Returns a user-facing string when no URL or subcommand is provided, or when +/// the underlying command fails. +pub(crate) fn run_audit(args: &AuditArgs) -> Result { + match &args.command { + Some(AuditSubcommand::Page(page_args)) => { + page::run_page(page_args).map(|()| RunOutcome::Success) } - } - - for integration in detected { - if !matches!( - integration, - "gpt" | "didomi" | "datadome" | "google_tag_manager" - ) { - manual_review.push(integration); + Some(AuditSubcommand::AdTemplates(AuditAdTemplatesCommand::Generate(gen_args))) => { + gen_args.browser.validate()?; + let app_config_path = crate::app_config::resolve_app_config_file(&gen_args.config)?; + let raw_config = std::fs::read_to_string(&app_config_path).map_err(|error| { + format!("failed to read {}: {error}", app_config_path.display()) + })?; + let existing_creative = creative_config(&raw_config, &app_config_path)?; + let profiles = gen_args.profiles()?; + let collectors: Vec = profiles + .iter() + .map(|profile| { + generate::browser_collector::BrowserAuditCollector::with_profile(*profile) + .with_page_delay(std::time::Duration::from_millis(gen_args.page_delay_ms)) + .with_browser_options(&gen_args.browser) + .with_scroll(gen_args.scroll) + }) + .collect(); + let selected: Vec<(&str, &dyn generate::collector::AuditCollector)> = profiles + .iter() + .zip(collectors.iter()) + .map(|(profile, collector)| { + ( + profile.label(), + collector as &dyn generate::collector::AuditCollector, + ) + }) + .collect(); + let stdout = std::io::stdout(); + let mut out = stdout.lock(); + let stderr = std::io::stderr(); + let mut err = stderr.lock(); + generate::run_update_slots( + &generate::UpdateSlotsRequest { + url: gen_args.url.as_str(), + config_path: &app_config_path, + existing_creative: existing_creative.as_ref(), + page_patterns: &gen_args.page_patterns, + replace: gen_args.replace, + cookies: &gen_args.cookies, + dry_run: gen_args.dry_run, + scroll: gen_args.scroll, + budget: gen_args.budget(), + }, + &selected, + &mut out, + &mut err, + ) + .map(|()| RunOutcome::Success) } - } - - if !manual_review.is_empty() { - if !draft.ends_with('\n') { - draft.push('\n'); + Some(AuditSubcommand::AdTemplates(AuditAdTemplatesCommand::Verify(verify_args))) => { + ad_templates::run_verify(verify_args) } - draft.push_str("\n# Audit findings requiring manual review\n"); - for integration in manual_review { - draft.push_str(&format!( - "# - Detected {integration}; review the corresponding [integrations.{integration}] section before enabling it.\n" - )); + Some(AuditSubcommand::Generate(generate_args)) => { + generate_args.browser.validate()?; + let stdout = std::io::stdout(); + let mut out = stdout.lock(); + let collector = generate::browser_collector::BrowserAuditCollector::default() + .with_browser_options(&generate_args.browser); + generate::run_generate(generate_args, &collector, &mut out) + .map(|()| RunOutcome::Success) } + None => match args.legacy_url.as_ref() { + Some(url) => { + let generate_args = legacy_generate_args(args, url); + generate_args.browser.validate()?; + let stdout = std::io::stdout(); + let mut out = stdout.lock(); + let collector = generate::browser_collector::BrowserAuditCollector::default() + .with_browser_options(&generate_args.browser); + generate::run_generate(&generate_args, &collector, &mut out) + .map(|()| RunOutcome::Success) + } + None => Err( + "provide a URL or a subcommand (`generate`, `page`, `ad-templates`)".to_string(), + ), + }, } - - Ok(draft) } -fn replace_key_in_section( +/// Reads the config's `[creative_opportunities]` section, when it has one. +/// +/// An unrelated invalid setting elsewhere in the document must not hide the +/// section — the runtime rejects such a file, but the operator still has to be +/// able to update slots in it — so the document is read as plain TOML rather +/// than through [`Settings`](trusted_server_core::settings::Settings). +/// +/// A section that is present but unreadable is *not* treated as absent. +/// `CreativeOpportunitiesConfig` uses `deny_unknown_fields`, so one mistyped key +/// would otherwise leave the merge with nothing to merge into and replace the +/// operator's entire slot array. +/// +/// # Errors +/// +/// Returns a user-facing error when the document is malformed or the section is +/// present but cannot be deserialized. +fn creative_config( document: &str, - section: &str, - key: &str, - replacement_line: &str, -) -> CliResult { - let section_header = format!("[{section}]"); - let mut in_section = false; - let mut replaced = false; - let mut saw_section = false; - let mut lines = Vec::new(); - - for line in document.lines() { - let trimmed = line.trim(); - if trimmed.starts_with('[') && trimmed.ends_with(']') { - in_section = trimmed == section_header; - saw_section |= in_section; - } - - if in_section && !replaced && is_key_line(trimmed, key) { - lines.push(replacement_line.to_string()); - replaced = true; - } else { - lines.push(line.to_string()); - } - } - - if !saw_section { - return cli_error(format!( - "failed to update starter config because section `{section_header}` was not found" - )); - } - if !replaced { - return cli_error(format!( - "failed to update starter config because key `{key}` was not found in `{section_header}`" - )); - } - - let mut output = lines.join("\n"); - if document.ends_with('\n') { - output.push('\n'); + path: &std::path::Path, +) -> CliResult> { + // Plain `format!`, not `report_error`: the top-level `[ts]` printer already + // logs whatever is returned here, and this message embeds a multi-line + // `toml::de::Error`, so logging it here too would print the whole block + // twice. The guidance leads so the parse error can trail unbroken. + let value = toml::from_str::(document).map_err(|error| { + format!( + "failed to parse {} before generating slots; fix the TOML syntax and re-run:\n{error}", + path.display() + ) + })?; + let Some(section) = value.get("creative_opportunities").cloned() else { + return Ok(None); + }; + match section.try_into() { + Ok(config) => Ok(Some(config)), + Err(error) => cli_error(format!( + "failed to read the existing `[creative_opportunities]` section, so generating \ + slots would discard the configured ones: {error}. Fix the section (or delete it) \ + and re-run" + )), } - Ok(output) } -fn is_key_line(trimmed_line: &str, key: &str) -> bool { - trimmed_line - .strip_prefix(key) - .and_then(|remaining| remaining.trim_start().strip_prefix('=')) - .is_some() +fn legacy_generate_args(args: &AuditArgs, url: &url::Url) -> generate::GenerateArgs { + generate::GenerateArgs { + url: url.to_string(), + js_assets: args.legacy_generate.js_assets.clone(), + config: args.legacy_generate.config.clone(), + no_js_assets: args.legacy_generate.no_js_assets, + no_config: args.legacy_generate.no_config, + force: args.legacy_generate.force, + cookies: args.legacy_generate.cookies.clone(), + browser: GenerateBrowserOpts::from(&args.legacy_generate.browser), + } } #[cfg(test)] mod tests { - use std::cell::Cell; - - use tempfile::TempDir; - use super::*; - use crate::commands::audit::collector::{CollectedPage, CollectedRequest, CollectedScriptTag}; - - struct FakeCollector { - collected: CollectedPage, - calls: Cell, - } - - impl FakeCollector { - fn new(collected: CollectedPage) -> Self { - Self { - collected, - calls: Cell::new(0), - } - } - } - - impl AuditCollector for FakeCollector { - fn collect_page(&self, _target_url: &Url) -> CliResult { - self.calls.set(self.calls.get() + 1); - Ok(self.collected.clone()) - } - } - - fn collected_page() -> CollectedPage { - CollectedPage { - requested_url: "https://publisher.example/page".to_string(), - final_url: "https://publisher.example/page".to_string(), - page_title: Some("Example Publisher".to_string()), - html: r#"Example Publisher"#.to_string(), - script_tags: vec![ - CollectedScriptTag { - src: Some("https://www.googletagmanager.com/gtm.js?id=GTM-ABC123".to_string()), - inline_text: None, - }, - CollectedScriptTag { - src: Some("https://securepubads.g.doubleclick.net/tag/js/gpt.js".to_string()), - inline_text: None, - }, - ], - network_requests: vec![CollectedRequest { - url: "https://cdn.publisher.example/app.js".to_string(), - resource_type: Some("script".to_string()), - }], - warnings: Vec::new(), - } - } - - fn audit_args(url: &str) -> AuditArgs { - AuditArgs { - url: url.to_string(), - js_assets: None, - config: None, - no_js_assets: false, - no_config: false, - force: false, - } - } #[test] - fn parse_audit_url_accepts_http_and_https() { - assert!(parse_audit_url("http://publisher.example").is_ok()); - assert!(parse_audit_url("https://publisher.example").is_ok()); + fn parse_cookie_splits_on_first_equals() { + let (name, value) = parse_cookie("datadome=abc=def~ghi").expect("should parse cookie"); + assert_eq!(name, "datadome", "name should be the pre-`=` portion"); + assert_eq!( + value, "abc=def~ghi", + "value should keep later `=` characters" + ); } #[test] - fn parse_audit_url_rejects_non_http_schemes() { - for url in [ - "file:///etc/passwd", - "data:text/html,hello", - "chrome://version", - ] { - let error = parse_audit_url(url).expect_err("should reject non-http URL"); - assert!( - format!("{error:?}").contains("only supports http/https"), - "should explain scheme restriction" - ); - } + fn parse_cookie_allows_empty_value() { + let (name, value) = parse_cookie("session=").expect("should parse empty value"); + assert_eq!(name, "session"); + assert!(value.is_empty(), "empty value should be allowed"); } #[test] - fn resolve_output_plan_rejects_no_outputs() { - let mut args = audit_args("https://publisher.example"); - args.no_js_assets = true; - args.no_config = true; + fn invalid_setting_outside_the_section_still_yields_creative_config() { + let document = "unknown_runtime_key = true\n\ + [creative_opportunities]\ngam_network_id = \"123\"\n"; - let error = resolve_output_plan(&args).expect_err("should reject empty output set"); + let creative = creative_config(document, std::path::Path::new("trusted-server.toml")) + .expect("an unrelated invalid setting must not hide creative config") + .expect("the section is present"); - assert!( - format!("{error:?}").contains("nothing to do"), - "should explain no-output error" - ); + assert_eq!(creative.gam_network_id, "123"); } #[test] - fn resolve_output_plan_rejects_existing_files_without_force() { - let temp = TempDir::new().expect("should create temp dir"); - let path = temp.path().join("js-assets.toml"); - fs::write(&path, "existing").expect("should write existing file"); - let mut args = audit_args("https://publisher.example"); - args.js_assets = Some(path); - args.no_config = true; - - let error = resolve_output_plan(&args).expect_err("should reject overwrite"); + fn absent_section_reads_as_absent() { + let creative = creative_config( + "[auction]\nenabled = true\n", + std::path::Path::new("trusted-server.toml"), + ) + .expect("should read the document"); assert!( - format!("{error:?}").contains("refusing to overwrite"), - "should explain overwrite refusal" + creative.is_none(), + "a document with no `[creative_opportunities]` has no configured slots" ); } #[test] - fn resolve_output_plan_allows_existing_files_with_force() { - let temp = TempDir::new().expect("should create temp dir"); - let path = temp.path().join("js-assets.toml"); - fs::write(&path, "existing").expect("should write existing file"); - let mut args = audit_args("https://publisher.example"); - args.js_assets = Some(path.clone()); - args.no_config = true; - args.force = true; - - let plan = resolve_output_plan(&args).expect("should allow forced overwrite"); - - assert_eq!(plan.js_assets_path.as_deref(), Some(path.as_path())); - } - - #[test] - fn run_audit_writes_selected_outputs_and_summary() { - let temp = TempDir::new().expect("should create temp dir"); - let js_assets = temp.path().join("audit/js-assets.toml"); - let config = temp.path().join("audit/trusted-server.toml"); - let args = AuditArgs { - url: "https://publisher.example/page".to_string(), - js_assets: Some(js_assets.clone()), - config: Some(config.clone()), - no_js_assets: false, - no_config: false, - force: false, - }; - let collector = FakeCollector::new(collected_page()); - let mut out = Vec::new(); - - run_audit(&args, &collector, &mut out).expect("should run audit"); - - assert_eq!(collector.calls.get(), 1, "should collect page once"); - assert!(js_assets.exists(), "should write JS assets"); - assert!(config.exists(), "should write draft config"); - let summary = String::from_utf8(out).expect("summary should be UTF-8"); - assert!(summary.contains("Audited https://publisher.example/page")); - assert!(summary.contains("Detected integrations: google_tag_manager, gpt")); - assert!(summary.contains("Draft config: review before validation and push")); - } - - #[test] - fn run_audit_respects_no_config() { - let temp = TempDir::new().expect("should create temp dir"); - let js_assets = temp.path().join("js-assets.toml"); - let mut args = audit_args("https://publisher.example/page"); - args.js_assets = Some(js_assets.clone()); - args.no_config = true; - let collector = FakeCollector::new(collected_page()); - - run_audit(&args, &collector, &mut Vec::new()).expect("should run audit"); + fn malformed_document_is_rejected_before_creative_config_extraction() { + let error = creative_config( + "[creative_opportunities\ngam_network_id = \"123\"\n", + std::path::Path::new("/tmp/example/trusted-server.toml"), + ) + .expect_err("should reject malformed TOML"); - assert!(js_assets.exists(), "should write assets"); assert!( - !temp.path().join("trusted-server.toml").exists(), - "should not write config" + error.contains("failed to parse /tmp/example/trusted-server.toml"), + "error should name the config file it could not parse, got {error}" ); - } - - #[test] - fn run_audit_respects_no_js_assets() { - let temp = TempDir::new().expect("should create temp dir"); - let config = temp.path().join("trusted-server.toml"); - let mut args = audit_args("https://publisher.example/page"); - args.config = Some(config.clone()); - args.no_js_assets = true; - let collector = FakeCollector::new(collected_page()); - let mut out = Vec::new(); - - run_audit(&args, &collector, &mut out).expect("should run audit"); - - assert!(config.exists(), "should write config"); assert!( - !temp.path().join("js-assets.toml").exists(), - "should not write JS assets" + error.contains("fix the TOML syntax and re-run:\n"), + "the guidance should lead so the multi-line parse error trails it, got {error}" ); - let summary = String::from_utf8(out).expect("summary should be UTF-8"); - assert!(summary.contains("Draft config: review before validation and push")); } #[test] - fn run_audit_writes_collector_warnings_to_asset_artifact() { - let temp = TempDir::new().expect("should create temp dir"); - let js_assets = temp.path().join("js-assets.toml"); - let mut args = audit_args("https://publisher.example/page"); - args.js_assets = Some(js_assets.clone()); - args.no_config = true; - let mut collected = collected_page(); - collected.warnings.push( - "browser audit timed out while waiting for the page to settle; results may be partial" - .to_string(), - ); - let collector = FakeCollector::new(collected); + fn unreadable_section_is_refused_rather_than_read_as_absent() { + // `deny_unknown_fields` makes one mistyped key inside the section fail + // to deserialize. Reading that as "no slots configured" would let a + // merge replace the operator's entire slot array. + let document = "[creative_opportunities]\n\ + gam_network_id = \"123\"\n\ + gam_netwrok_id = \"123\"\n\ + [[creative_opportunities.slot]]\n\ + id = \"header\"\n\ + div_id = \"ad-header\"\n\ + page_patterns = [\"/\"]\n\ + formats = [{ width = 728, height = 90 }]\n"; + + let error = creative_config(document, std::path::Path::new("trusted-server.toml")) + .expect_err("should refuse an unreadable section"); - run_audit(&args, &collector, &mut Vec::new()).expect("should run audit"); - - let artifact = fs::read_to_string(js_assets).expect("should read artifact"); assert!( - artifact.contains("results may be partial"), - "should persist collector warning" + error.contains("would discard the configured ones"), + "error should say what merging would cost, got {error}" ); } #[test] - fn run_audit_conflict_prevents_collection() { - let temp = TempDir::new().expect("should create temp dir"); - let js_assets = temp.path().join("js-assets.toml"); - fs::write(&js_assets, "existing").expect("should write existing file"); - let mut args = audit_args("https://publisher.example/page"); - args.js_assets = Some(js_assets); - args.no_config = true; - let collector = FakeCollector::new(collected_page()); - - let error = run_audit(&args, &collector, &mut Vec::new()) - .expect_err("should reject existing output"); - - assert_eq!(collector.calls.get(), 0, "should not collect page"); + fn parse_cookie_rejects_missing_equals() { + let err = parse_cookie("datadome").expect_err("should reject missing `=`"); assert!( - format!("{error:?}").contains("refusing to overwrite"), - "should report overwrite conflict" + err.contains("NAME=VALUE"), + "error should show expected form" ); } #[test] - fn build_draft_config_uses_final_url_and_detected_integrations() { - let url = Url::parse("https://www.publisher.example:8443/path").expect("should parse URL"); - let artifact = AuditArtifact { - audited_url: url.to_string(), - page_title: Some("Example".to_string()), - js_asset_count: 2, - third_party_asset_count: 2, - detected_integrations: vec![ - DetectedIntegration { - id: "google_tag_manager".to_string(), - evidence: "GTM-ABC123".to_string(), - }, - DetectedIntegration { - id: "gpt".to_string(), - evidence: "https://securepubads.g.doubleclick.net/tag/js/gpt.js".to_string(), - }, - DetectedIntegration { - id: "prebid".to_string(), - evidence: "inline script matched `prebid`".to_string(), - }, - ], - assets: Vec::new(), - warnings: Vec::new(), - }; - - let draft = build_draft_config(&url, &artifact).expect("should build draft config"); - - assert!(draft.contains("domain = \"www.publisher.example\"")); - assert!(draft.contains("cookie_domain = \".www.publisher.example\"")); - assert!(draft.contains("origin_url = \"https://www.publisher.example:8443\"")); - assert!(draft.contains("[integrations.gpt]\nenabled = true")); - assert!(draft.contains("[integrations.google_tag_manager]\nenabled = true")); - assert!(draft.contains("container_id = \"GTM-ABC123\"")); - assert!(draft.contains("Detected prebid")); - toml::from_str::(&draft).expect("draft should parse as TOML"); + fn parse_cookie_rejects_empty_name() { + let err = parse_cookie("=value").expect_err("should reject empty name"); + assert!(err.contains("empty name"), "error should name the problem"); } #[test] - fn build_draft_config_does_not_enable_gtm_without_container_id() { - let url = Url::parse("https://publisher.example/path").expect("should parse URL"); - let artifact = AuditArtifact { - audited_url: url.to_string(), - page_title: None, - js_asset_count: 1, - third_party_asset_count: 1, - detected_integrations: vec![DetectedIntegration { - id: "google_tag_manager".to_string(), - evidence: "https://www.googletagmanager.com/gtm.js".to_string(), - }], - assets: Vec::new(), - warnings: Vec::new(), + fn legacy_url_builds_artifact_generation_args() { + let args = AuditArgs { + command: None, + legacy_url: Some( + url::Url::parse("https://www.example.com/").expect("should parse URL"), + ), + legacy_generate: LegacyGenerateArgs { + js_assets: Some("audit/assets.toml".into()), + config: Some("audit/config.toml".into()), + no_js_assets: false, + no_config: false, + force: true, + cookies: vec![("session".to_string(), "example".to_string())], + browser: LegacyBrowserOpts { + headful: true, + ..LegacyBrowserOpts::default() + }, + }, }; - let draft = build_draft_config(&url, &artifact).expect("should build draft config"); + let generate = legacy_generate_args( + &args, + args.legacy_url.as_ref().expect("should have legacy URL"), + ); - assert!(draft.contains("[integrations.google_tag_manager]\nenabled = false")); - assert!(draft.contains("Detected google_tag_manager")); + assert_eq!(generate.url, "https://www.example.com/"); + assert_eq!( + generate.js_assets.as_deref(), + Some(std::path::Path::new("audit/assets.toml")) + ); + assert_eq!( + generate.config.as_deref(), + Some(std::path::Path::new("audit/config.toml")) + ); + assert!(generate.force); + assert_eq!( + generate.cookies, + [("session".to_string(), "example".to_string())] + ); + assert!( + generate.browser.headful, + "browser flags passed to the legacy form should reach generation" + ); } } diff --git a/crates/trusted-server-cli/src/commands/audit/page.rs b/crates/trusted-server-cli/src/commands/audit/page.rs new file mode 100644 index 000000000..31cbf4b1b --- /dev/null +++ b/crates/trusted-server-cli/src/commands/audit/page.rs @@ -0,0 +1,157 @@ +//! Generic `ts audit page ` command: a read-only page summary. + +use std::io::{self, Write}; + +use clap::Args; + +use crate::ad_templates::output::escape_terminal_text; +use crate::commands::audit::browser::BrowserCollector; +use crate::commands::audit::collector::{ + AuditCollector, BrowserCollectRequest, BrowserOpts, CollectedPage, +}; + +/// Arguments for `ts audit page `. +#[derive(Debug, Args)] +pub(crate) struct PageAuditArgs { + /// The page URL to audit (http or https). + #[arg(value_parser = crate::commands::audit::parse_http_url)] + pub url: url::Url, + /// Perform a deterministic scroll pass after the initial settle. + #[arg(long)] + pub scroll: bool, + #[command(flatten)] + pub browser: BrowserOpts, +} + +/// Runs the generic page audit for the `page` subcommand. +/// +/// # Errors +/// +/// Returns a user-facing string when the browser cannot collect the page. +pub(crate) fn run_page(args: &PageAuditArgs) -> Result<(), String> { + args.browser.validate()?; + run_with_collector( + &BrowserCollector::from_opts(&args.browser), + &args.url, + args.scroll, + ) +} + +fn run_with_collector( + collector: &dyn AuditCollector, + url: &url::Url, + scroll: bool, +) -> Result<(), String> { + let page = collector.collect_page(BrowserCollectRequest { + url: url.clone(), + init_scripts: Vec::new(), + scroll, + collect_ad_evidence: false, + cookies: Vec::new(), + })?; + + let stdout = io::stdout(); + let mut out = stdout.lock(); + write_summary(&mut out, url, &page) +} + +fn write_summary(out: &mut dyn Write, url: &url::Url, page: &CollectedPage) -> Result<(), String> { + let to_err = |error: io::Error| format!("failed to write command output: {error}"); + writeln!(out, "url: {url}").map_err(to_err)?; + // The final URL, title, and collector warning messages are page-controlled, + // so escape control characters before they reach the operator's terminal. + writeln!( + out, + "final url: {}", + escape_terminal_text(page.final_url.as_str()) + ) + .map_err(to_err)?; + writeln!(out, "title: {}", escape_terminal_text(&page.title)).map_err(to_err)?; + writeln!(out, "scripts: {}", page.script_count).map_err(to_err)?; + writeln!(out, "resources: {}", page.resource_count).map_err(to_err)?; + for warning in &page.warnings { + writeln!( + out, + "warning [{}]: {}", + escape_terminal_text(&warning.code), + escape_terminal_text(&warning.message) + ) + .map_err(to_err)?; + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::ad_templates::output::Warning; + + fn collected(final_url: &str, title: &str, warnings: Vec) -> CollectedPage { + CollectedPage { + final_url: url::Url::parse(final_url).expect("should parse fixture URL"), + title: title.to_string(), + script_count: 3, + resource_count: 42, + warnings, + ad_evidence: None, + } + } + + fn summary(page: &CollectedPage, requested: &str) -> String { + let url = url::Url::parse(requested).expect("should parse requested URL"); + let mut out = Vec::new(); + write_summary(&mut out, &url, page).expect("should write summary"); + String::from_utf8(out).expect("summary should be UTF-8") + } + + #[test] + fn summary_reports_the_requested_and_final_urls_with_counts() { + let page = collected( + "https://publisher.example/news/story", + "Example Publisher", + Vec::new(), + ); + + let out = summary(&page, "https://publisher.example/news"); + + assert!( + out.contains("url: https://publisher.example/news\n"), + "should echo the requested URL, got {out:?}" + ); + assert!( + out.contains("final url: https://publisher.example/news/story\n"), + "should report the post-redirect URL, got {out:?}" + ); + assert!(out.contains("scripts: 3"), "got {out:?}"); + assert!(out.contains("resources: 42"), "got {out:?}"); + } + + #[test] + fn page_controlled_text_is_escaped_before_it_reaches_the_terminal() { + // Title and warning text are page-controlled and can contain raw + // terminal controls. URL percent-encoding is asserted separately. + let page = collected( + "https://publisher.example/a%1B%5B2Jb", + "Example\u{1b}[2J", + vec![Warning { + code: "page_\u{1b}[31m".to_string(), + message: "message\u{1b}[0m".to_string(), + }], + ); + + let out = summary(&page, "https://publisher.example/"); + + assert!( + !out.contains('\u{1b}'), + "no escape sequence may reach the terminal, got {out:?}" + ); + assert!( + out.contains("final url: https://publisher.example/a%1B%5B2Jb\n"), + "the final URL should retain URL's percent encoding, got {out:?}" + ); + assert!( + out.contains("warning [page_"), + "warnings should still be reported, got {out:?}" + ); + } +} diff --git a/crates/trusted-server-cli/src/commands/config/ad_templates.rs b/crates/trusted-server-cli/src/commands/config/ad_templates.rs new file mode 100644 index 000000000..b8d9f538e --- /dev/null +++ b/crates/trusted-server-cli/src/commands/config/ad_templates.rs @@ -0,0 +1,832 @@ +use std::collections::BTreeSet; +use std::io::{self, Write}; + +use crate::ad_templates::expected::normalize_path_or_url; +use crate::ad_templates::output::escape_terminal_text; +use crate::app_config::{AppConfigArgs, load_settings}; +use clap::{ArgGroup, Args, Subcommand}; +use http::Method; +use trusted_server_core::auction::types::MediaType; +use trusted_server_core::creative_opportunities::{ + AdStackGateInput, AdStackGateName, CreativeOpportunityFormat, CreativeOpportunitySlot, + RuntimeAdStackExpected, evaluate_ad_stack_gate, match_slots, validate_page_pattern, +}; + +use crate::run::RunOutcome; + +enum CheckFailure { + Tool(String), + Assertion(String), +} + +#[derive(Debug, Subcommand)] +pub enum AdTemplatesCommand { + /// Validate ad-template config and summarize deploy-time implications. + Lint(AdTemplatesLintArgs), + /// Show creative opportunity slots matching a page path or URL. + Match(AdTemplatesMatchArgs), + /// Assert that a page path or URL matches the expected slot set. + Check(AdTemplatesCheckArgs), + /// Explain why a page path or URL would or would not run the ad stack. + Explain(AdTemplatesExplainArgs), +} + +#[derive(Debug, Args)] +pub struct AdTemplatesLintArgs { + #[command(flatten)] + pub config: AppConfigArgs, +} + +#[derive(Debug, Args)] +pub struct AdTemplatesMatchArgs { + #[command(flatten)] + pub config: AppConfigArgs, + /// Page path or full URL to evaluate. + pub path_or_url: String, + /// Include slot div, GAM path, formats, and providers. + #[arg(long)] + pub details: bool, +} + +#[derive(Debug, Args)] +#[command(group( + ArgGroup::new("expectation") + .required(true) + .args(["expected_slots", "expect_no_slots"]) +))] +pub struct AdTemplatesCheckArgs { + #[command(flatten)] + pub config: AppConfigArgs, + /// Page path or full URL to evaluate. + pub path_or_url: String, + /// Expected slot id. Repeat for multiple slots. + #[arg(long = "expected-slot", value_name = "ID")] + pub expected_slots: Vec, + /// Assert that no slots match the page path or URL. + #[arg(long)] + pub expect_no_slots: bool, + /// Allow additional matched slots beyond --expected-slot values. + #[arg(long, conflicts_with = "expect_no_slots")] + pub allow_extra_slots: bool, +} + +#[derive(Debug, Args)] +pub struct AdTemplatesExplainArgs { + #[command(flatten)] + pub config: AppConfigArgs, + /// Page path or full URL to evaluate. + pub path_or_url: String, + /// HTTP method to model. + #[arg(long, default_value = "GET", value_parser = parse_http_method)] + pub method: Method, + /// Model a non-navigation request. + #[arg(long)] + pub non_navigation: bool, + /// Model a prefetch request. + #[arg(long)] + pub prefetch: bool, + /// Model a known crawler user agent. + #[arg(long)] + pub bot: bool, + /// Model consent denying server-side auction. + #[arg(long)] + pub consent_denied: bool, +} + +fn parse_http_method(raw: &str) -> Result { + let normalized = raw.to_ascii_uppercase(); + Method::from_bytes(normalized.as_bytes()) + .map_err(|error| format!("invalid HTTP method `{raw}`: {error}")) +} + +/// Run an ad-template CLI command. +/// +/// # Errors +/// +/// Returns a user-facing string when config loading, matching, or assertion +/// checks fail. +pub fn run_ad_templates(args: &AdTemplatesCommand) -> Result { + let stdout = io::stdout(); + let mut out = stdout.lock(); + if let AdTemplatesCommand::Check(args) = args { + return match run_check_classified(args, &mut out) { + Ok(()) => Ok(RunOutcome::Success), + Err(CheckFailure::Tool(error)) => Err(error), + Err(CheckFailure::Assertion(message)) => { + let stderr = io::stderr(); + let mut err = stderr.lock(); + writeln!(err, "{message}").map_err(output_error)?; + Ok(RunOutcome::AssertionFailed) + } + }; + } + run_ad_templates_with_writer(args, &mut out).map(|()| RunOutcome::Success) +} + +fn run_ad_templates_with_writer( + args: &AdTemplatesCommand, + out: &mut dyn Write, +) -> Result<(), String> { + match args { + AdTemplatesCommand::Lint(args) => run_lint(args, out), + AdTemplatesCommand::Match(args) => run_match(args, out), + AdTemplatesCommand::Check(args) => run_check(args, out), + AdTemplatesCommand::Explain(args) => run_explain(args, out), + } +} + +fn run_lint(args: &AdTemplatesLintArgs, out: &mut dyn Write) -> Result<(), String> { + let loaded = load_settings(&args.config)?; + writeln!(out, "app config: {}", loaded.app_config_path.display()).map_err(output_error)?; + + let Some(config) = &loaded.settings.creative_opportunities else { + writeln!(out, "server-side ad templates: not configured").map_err(output_error)?; + return Ok(()); + }; + + writeln!( + out, + "server-side ad templates: configured ({} slot{})", + config.slot.len(), + plural(config.slot.len()) + ) + .map_err(output_error)?; + writeln!( + out, + "gam_network_id: {}", + escape_terminal_text(&config.gam_network_id) + ) + .map_err(output_error)?; + writeln!( + out, + "auction_timeout_ms: {}", + config + .auction_timeout_ms + .unwrap_or(loaded.settings.auction.timeout_ms) + ) + .map_err(output_error)?; + writeln!( + out, + "creative_opportunities.enabled: {}", + if config.enabled { "true" } else { "false" } + ) + .map_err(output_error)?; + writeln!( + out, + "auction.enabled: {}", + if loaded.settings.auction.enabled { + "true" + } else { + "false" + } + ) + .map_err(output_error)?; + writeln!( + out, + "auction.providers: {}", + if loaded.settings.auction.providers.is_empty() { + "(none)".to_string() + } else { + let providers = loaded + .settings + .auction + .providers + .keys() + .map(trusted_server_core::auction::ProviderId::as_str) + .collect::>() + .join(", "); + escape_terminal_text(&providers).into_owned() + } + ) + .map_err(output_error)?; + + if config.slot.is_empty() { + writeln!(out, "status: disabled because no slots are configured").map_err(output_error)?; + } else if !config.enabled { + writeln!( + out, + "status: slots are configured, but [creative_opportunities].enabled is false" + ) + .map_err(output_error)?; + } else if !loaded.settings.auction.enabled { + writeln!( + out, + "status: slots are configured, but [auction].enabled is false" + ) + .map_err(output_error)?; + } else if loaded.settings.auction.providers.is_empty() { + writeln!( + out, + "status: slots are configured, but [auction].providers is empty" + ) + .map_err(output_error)?; + } else { + writeln!(out, "status: eligible for legacy-path server-side auctions") + .map_err(output_error)?; + } + + for slot in &config.slot { + for pattern in &slot.page_patterns { + if let Err(error) = validate_page_pattern(pattern) { + writeln!( + out, + "invalid page pattern for slot `{}`: {}", + escape_terminal_text(&slot.id), + escape_terminal_text(&error), + ) + .map_err(output_error)?; + } + } + } + + Ok(()) +} + +fn run_match(args: &AdTemplatesMatchArgs, out: &mut dyn Write) -> Result<(), String> { + let loaded = load_settings(&args.config)?; + let path = normalize_path_or_url(&args.path_or_url)?; + let Some(config) = &loaded.settings.creative_opportunities else { + writeln!( + out, + "{path}: no slots matched (creative_opportunities not configured)" + ) + .map_err(output_error)?; + return Ok(()); + }; + let matched = match_slots(&config.slot, &path); + + write_match_result( + out, + &path, + &matched, + &config.gam_network_id, + &config.section_for_path(&path), + args.details, + ) +} + +fn run_check(args: &AdTemplatesCheckArgs, out: &mut dyn Write) -> Result<(), String> { + run_check_classified(args, out).map_err(|failure| match failure { + CheckFailure::Tool(error) | CheckFailure::Assertion(error) => error, + }) +} + +fn run_check_classified( + args: &AdTemplatesCheckArgs, + out: &mut dyn Write, +) -> Result<(), CheckFailure> { + let loaded = load_settings(&args.config).map_err(CheckFailure::Tool)?; + let path = normalize_path_or_url(&args.path_or_url).map_err(CheckFailure::Tool)?; + let matched = loaded + .settings + .creative_opportunities + .as_ref() + .map(|config| match_slots(&config.slot, &path)) + .unwrap_or_default(); + let actual: BTreeSet<&str> = matched.iter().map(|slot| slot.id.as_str()).collect(); + + if args.expect_no_slots { + if actual.is_empty() { + writeln!(out, "{path}: OK, no slots matched") + .map_err(output_error) + .map_err(CheckFailure::Tool)?; + return Ok(()); + } + return Err(CheckFailure::Assertion(format!( + "{path}: expected no slots, matched {}", + join_set(&actual) + ))); + } + + let expected: BTreeSet<&str> = args.expected_slots.iter().map(String::as_str).collect(); + let missing: BTreeSet<&str> = expected.difference(&actual).copied().collect(); + let extra: BTreeSet<&str> = actual.difference(&expected).copied().collect(); + + if missing.is_empty() && (args.allow_extra_slots || extra.is_empty()) { + writeln!(out, "{path}: OK, matched {}", join_set(&actual)) + .map_err(output_error) + .map_err(CheckFailure::Tool)?; + return Ok(()); + } + + let mut problems = Vec::new(); + if !missing.is_empty() { + problems.push(format!("missing {}", join_set(&missing))); + } + if !args.allow_extra_slots && !extra.is_empty() { + problems.push(format!("unexpected {}", join_set(&extra))); + } + Err(CheckFailure::Assertion(format!( + "{path}: {}", + problems.join("; ") + ))) +} + +fn run_explain(args: &AdTemplatesExplainArgs, out: &mut dyn Write) -> Result<(), String> { + let loaded = load_settings(&args.config)?; + let path = normalize_path_or_url(&args.path_or_url)?; + writeln!(out, "path: {path}").map_err(output_error)?; + + let has_matches = if let Some(config) = &loaded.settings.creative_opportunities { + let matched = match_slots(&config.slot, &path); + write_match_result( + out, + &path, + &matched, + &config.gam_network_id, + &config.section_for_path(&path), + true, + )?; + !matched.is_empty() + } else { + writeln!(out, "creative_opportunities: not configured").map_err(output_error)?; + false + }; + + let method_pass = args.method == Method::GET; + let navigation_pass = !args.non_navigation; + let consent_pass = !args.consent_denied; + let auction_enabled = loaded.settings.auction.enabled; + let ad_templates_enabled = loaded + .settings + .creative_opportunities + .as_ref() + .is_some_and(|config| config.enabled); + let providers_configured = !loaded.settings.auction.providers.is_empty(); + + let gate = evaluate_ad_stack_gate(AdStackGateInput { + method_get: method_pass, + navigation: navigation_pass, + prefetch: args.prefetch, + bot: args.bot, + matched_slots: has_matches, + consent_allows_auction: Some(consent_pass), + auction_enabled, + ad_templates_enabled, + }); + let blocked: Vec = gate.blocking_gates().collect(); + write_gate( + out, + "method GET", + !blocked.contains(&AdStackGateName::MethodGet), + )?; + write_gate( + out, + "navigation", + !blocked.contains(&AdStackGateName::Navigation), + )?; + write_gate( + out, + "not prefetch", + !blocked.contains(&AdStackGateName::NotPrefetch), + )?; + write_gate(out, "not bot", !blocked.contains(&AdStackGateName::NotBot))?; + write_gate( + out, + "consent allows auction", + !blocked.contains(&AdStackGateName::ConsentAllowsAuction), + )?; + write_gate( + out, + "auction.enabled", + !blocked.contains(&AdStackGateName::AuctionEnabled), + )?; + write_gate( + out, + "creative_opportunities.enabled", + !blocked.contains(&AdStackGateName::AdTemplatesEnabled), + )?; + write_gate( + out, + "matched slots", + !blocked.contains(&AdStackGateName::MatchedSlots), + )?; + writeln!( + out, + "advisory auction providers configured: {}", + if providers_configured { "yes" } else { "no" } + ) + .map_err(output_error)?; + writeln!( + out, + "server-side ad stack: {}", + match gate.expected { + RuntimeAdStackExpected::Yes => "yes", + RuntimeAdStackExpected::No => "no", + // `explain` always supplies a consent decision, which is the only + // input that yields `Unknown`; the arm is here for exhaustiveness. + RuntimeAdStackExpected::Unknown => "unknown", + } + ) + .map_err(output_error)?; + + Ok(()) +} + +fn write_match_result( + out: &mut dyn Write, + path: &str, + matched: &[&CreativeOpportunitySlot], + gam_network_id: &str, + section: &str, + details: bool, +) -> Result<(), String> { + if matched.is_empty() { + writeln!(out, "{}: no slots matched", escape_terminal_text(path)).map_err(output_error)?; + return Ok(()); + } + + let ids = matched + .iter() + .map(|slot| escape_terminal_text(&slot.id).into_owned()) + .collect::>() + .join(", "); + writeln!(out, "{}: matched {ids}", escape_terminal_text(path)).map_err(output_error)?; + + if details { + for slot in matched { + writeln!(out, "- {}", format_slot(slot, gam_network_id, section)) + .map_err(output_error)?; + } + } + + Ok(()) +} + +fn write_gate(out: &mut dyn Write, label: &str, pass: bool) -> Result<(), String> { + writeln!(out, "gate {label}: {}", if pass { "pass" } else { "block" }).map_err(output_error) +} + +/// Formats one matched slot for `--details` output. +/// +/// `section` is the value the runtime derives from the evaluated path, so a +/// `{section}` template renders the same unit path the live request would use. +fn format_slot(slot: &CreativeOpportunitySlot, gam_network_id: &str, section: &str) -> String { + let formats = slot + .formats + .iter() + .map(format_format) + .collect::>() + .join(", "); + let providers = format_providers(slot); + // `None` means a dynamic template renders past GAM's unit-path byte limit — + // a config the runtime rejects, so surface it rather than printing a path. + let gam_unit_path = slot + .render_gam_unit_path(gam_network_id, section) + .unwrap_or_else(|| "".to_string()); + format!( + "{} div={} gam={} patterns=[{}] formats=[{}] providers=[{}]", + escape_terminal_text(&slot.id), + escape_terminal_text(slot.resolved_div_id()), + escape_terminal_text(&gam_unit_path), + escape_terminal_text(&slot.page_patterns.join(", ")), + formats, + providers, + ) +} + +fn format_format(format: &CreativeOpportunityFormat) -> String { + let media_type = match format.media_type { + MediaType::Banner => "banner", + MediaType::Video => "video", + MediaType::Native => "native", + }; + format!("{}x{} {media_type}", format.width, format.height) +} + +fn format_providers(slot: &CreativeOpportunitySlot) -> String { + let mut providers = Vec::new(); + if slot.providers.aps.is_some() { + providers.push("aps"); + } + if slot.providers.prebid.is_some() { + providers.push("prebid"); + } + if providers.is_empty() { + return "none".to_string(); + } + providers.join(", ") +} + +/// Renders a set of config-derived slot ids for the terminal. +/// +/// Config can arrive from a pushed blob or the env overlay, not only from a file +/// the operator read, so the ids are escaped before they reach a terminal — the +/// assertion-failure path prints them too. +fn join_set(set: &BTreeSet<&str>) -> String { + if set.is_empty() { + return "(none)".to_string(); + } + set.iter() + .map(|id| escape_terminal_text(id).into_owned()) + .collect::>() + .join(", ") +} + +fn plural(count: usize) -> &'static str { + if count == 1 { "" } else { "s" } +} + +#[allow( + clippy::needless_pass_by_value, + reason = "used as a map_err fn that receives io::Error by value" +)] +fn output_error(err: io::Error) -> String { + format!("failed to write command output: {err}") +} + +#[cfg(test)] +mod tests { + use std::fs; + + use tempfile::TempDir; + + use super::*; + + const EXAMPLE_CONFIG: &str = include_str!(concat!( + env!("CARGO_MANIFEST_DIR"), + "/../../trusted-server.example.toml" + )); + + fn project_with_config(config: &str) -> (TempDir, AppConfigArgs) { + let temp = TempDir::new().expect("should create temp dir"); + let manifest_path = temp.path().join("edgezero.toml"); + let config_path = temp.path().join("trusted-server.toml"); + fs::write(&manifest_path, "[app]\nname = \"trusted-server\"\n") + .expect("should write manifest"); + fs::write(&config_path, config).expect("should write app config"); + ( + temp, + AppConfigArgs { + app_config: Some(config_path), + manifest: manifest_path, + no_env: true, + }, + ) + } + + fn config_with_slots() -> String { + let base_config = EXAMPLE_CONFIG + .replace( + "password = \"handler_password\"", + "password = \"test-admin-password-32-bytes-minimum\"", + ) + .replace( + "passphrase = \"ec_passphrase\"", + "passphrase = \"test-ec-passphrase-32-bytes-minimum\"", + ) + .replace( + "proxy_secret = \"publisher_proxy_secret\"", + "proxy_secret = \"test-proxy-secret-32-bytes-minimum\"", + ); + format!( + "{base_config}\n\ + [[creative_opportunities.slot]]\n\ + id = \"atf\"\n\ + page_patterns = [\"/news/*\", \"/\"]\n\ + formats = [{{ width = 300, height = 250 }}]\n\ + targeting = {{ zone = \"atf\" }}\n\ + [creative_opportunities.slot.providers.prebid]\n\ + bidders = {{}}\n\ + \n\ + [[creative_opportunities.slot]]\n\ + id = \"sports-sidebar\"\n\ + div_id = \"sports-ad\"\n\ + page_patterns = [\"/sports/*\"]\n\ + formats = [{{ width = 300, height = 600 }}]\n" + ) + } + + #[test] + fn match_reports_slots_for_path() { + let (_temp, config) = project_with_config(&config_with_slots()); + let mut out = Vec::new(); + + run_ad_templates_with_writer( + &AdTemplatesCommand::Match(AdTemplatesMatchArgs { + config, + path_or_url: "https://example.com/news/story?utm=1".to_string(), + details: true, + }), + &mut out, + ) + .expect("should match slots"); + + let output = String::from_utf8(out).expect("should be utf8"); + assert!( + output.contains("/news/story: matched atf"), + "should report matched slot" + ); + assert!( + output.contains("formats=[300x250 banner]"), + "should include details" + ); + } + + #[test] + fn check_rejects_unexpected_extra_slots_by_default() { + let (_temp, config) = project_with_config(&config_with_slots()); + + let err = run_ad_templates_with_writer( + &AdTemplatesCommand::Check(AdTemplatesCheckArgs { + config, + path_or_url: "/sports/game".to_string(), + expected_slots: vec!["atf".to_string()], + expect_no_slots: false, + allow_extra_slots: false, + }), + &mut Vec::new(), + ) + .expect_err("should reject mismatch"); + + assert!( + err.contains("missing atf") && err.contains("unexpected sports-sidebar"), + "should describe missing and unexpected slots" + ); + } + + #[test] + fn check_accepts_no_slots() { + let (_temp, config) = project_with_config(&config_with_slots()); + let mut out = Vec::new(); + + run_ad_templates_with_writer( + &AdTemplatesCommand::Check(AdTemplatesCheckArgs { + config, + path_or_url: "/weather/today".to_string(), + expected_slots: Vec::new(), + expect_no_slots: true, + allow_extra_slots: false, + }), + &mut out, + ) + .expect("should accept no slots"); + + let output = String::from_utf8(out).expect("should be utf8"); + assert!( + output.contains("/weather/today: OK, no slots matched"), + "should report no-slot assertion" + ); + } + + #[test] + fn explain_keeps_provider_state_separate_from_runtime_verdict() { + let config_text = config_with_slots().replacen( + "\nenabled = false\n# Rewrite", + "\nenabled = true\n# Rewrite", + 1, + ); + let (_temp, config) = project_with_config(&config_text); + let mut out = Vec::new(); + + run_ad_templates_with_writer( + &AdTemplatesCommand::Explain(AdTemplatesExplainArgs { + config, + path_or_url: "/news/story".to_string(), + method: Method::GET, + non_navigation: false, + prefetch: false, + bot: false, + consent_denied: false, + }), + &mut out, + ) + .expect("should explain path"); + + let output = String::from_utf8(out).expect("should be utf8"); + assert!( + output.contains("server-side ad stack: yes"), + "runtime verdict should not include provider configuration: {output}" + ); + assert!( + output.contains("advisory auction providers configured: yes"), + "provider state should be a separate advisory: {output}" + ); + } + + #[test] + fn lint_reports_configured_slot_count_and_auction_state() { + let (_temp, config) = project_with_config(&config_with_slots()); + let mut out = Vec::new(); + + run_ad_templates_with_writer( + &AdTemplatesCommand::Lint(AdTemplatesLintArgs { config }), + &mut out, + ) + .expect("should lint configured slots"); + + let output = String::from_utf8(out).expect("should be utf8"); + assert!( + output.contains("server-side ad templates: configured (2 slots)"), + "should report the configured slot count" + ); + assert!( + output.contains("auction.enabled:"), + "should report the auction kill-switch state" + ); + assert!(!output.contains("legacy fallback")); + } + + #[test] + fn lint_and_explain_report_the_disabled_template_switch() { + // `[creative_opportunities].enabled = false` is a runtime kill switch: + // the publisher path matches no slots at all while it is off, so the + // diagnostics must not claim the ad stack would run. + let config_text = config_with_slots().replace("enabled = true", "enabled = false"); + let (_temp, config) = project_with_config(&config_text); + let mut out = Vec::new(); + + run_ad_templates_with_writer( + &AdTemplatesCommand::Lint(AdTemplatesLintArgs { + config: config.clone(), + }), + &mut out, + ) + .expect("should lint a disabled template switch"); + let lint_output = String::from_utf8(out).expect("should be utf8"); + + assert!( + lint_output.contains("creative_opportunities.enabled: false"), + "lint should report the template switch state: {lint_output}" + ); + assert!( + lint_output.contains( + "status: slots are configured, but [creative_opportunities].enabled is false" + ), + "lint status should name the template switch: {lint_output}" + ); + + let mut out = Vec::new(); + run_ad_templates_with_writer( + &AdTemplatesCommand::Explain(AdTemplatesExplainArgs { + config, + path_or_url: "/news/story".to_string(), + method: Method::GET, + non_navigation: false, + prefetch: false, + bot: false, + consent_denied: false, + }), + &mut out, + ) + .expect("should explain a disabled template switch"); + let explain_output = String::from_utf8(out).expect("should be utf8"); + + assert!( + explain_output.contains("gate creative_opportunities.enabled: block"), + "explain should fail the template-switch gate: {explain_output}" + ); + assert!( + explain_output.contains("server-side ad stack: no"), + "explain verdict should follow the switch: {explain_output}" + ); + } + + #[test] + fn lint_reports_page_patterns_the_runtime_drops() { + let config_text = config_with_slots().replace( + "page_patterns = [\"/news/*\", \"/\"]", + "page_patterns = [\"/news/*\", \"[\"]", + ); + let (_temp, config) = project_with_config(&config_text); + let mut out = Vec::new(); + + run_ad_templates_with_writer( + &AdTemplatesCommand::Lint(AdTemplatesLintArgs { config }), + &mut out, + ) + .expect("should lint mixed valid and invalid patterns"); + let output = String::from_utf8(out).expect("should be utf8"); + + assert!( + output.contains("invalid page pattern for slot `atf`") + && output.contains("page pattern '[' is not a valid glob"), + "lint should surface the runtime-dropped pattern: {output}" + ); + } + + #[test] + fn public_check_reports_drift_as_assertion_outcome() { + let (_temp, config) = project_with_config(&config_with_slots()); + + let outcome = run_ad_templates(&AdTemplatesCommand::Check(AdTemplatesCheckArgs { + config, + path_or_url: "/sports/game".to_string(), + expected_slots: vec!["atf".to_string()], + expect_no_slots: false, + allow_extra_slots: false, + })) + .expect("assertion drift should not be a tool error"); + + assert_eq!(outcome, RunOutcome::AssertionFailed); + } + + #[test] + fn http_method_parser_normalizes_standard_methods() { + assert_eq!( + parse_http_method("get").expect("should parse lowercase GET"), + Method::GET, + "lowercase GET must evaluate the same runtime gate as uppercase GET" + ); + } +} diff --git a/crates/trusted-server-cli/src/commands/config/mod.rs b/crates/trusted-server-cli/src/commands/config/mod.rs index 43763f10a..77af7de85 100644 --- a/crates/trusted-server-cli/src/commands/config/mod.rs +++ b/crates/trusted-server-cli/src/commands/config/mod.rs @@ -1 +1,2 @@ +pub mod ad_templates; pub mod init; diff --git a/crates/trusted-server-cli/src/lib.rs b/crates/trusted-server-cli/src/lib.rs index 405bc6187..a3352ba93 100644 --- a/crates/trusted-server-cli/src/lib.rs +++ b/crates/trusted-server-cli/src/lib.rs @@ -1,4 +1,8 @@ #[cfg(not(target_arch = "wasm32"))] +mod ad_templates; +#[cfg(not(target_arch = "wasm32"))] +mod app_config; +#[cfg(not(target_arch = "wasm32"))] mod error; #[cfg(not(target_arch = "wasm32"))] mod prebid_bundle; @@ -6,7 +10,7 @@ mod prebid_bundle; mod run; #[cfg(not(target_arch = "wasm32"))] -pub use run::run_from_env; +pub use run::{RunOutcome, run_from_env}; // Every `ts` subcommand's implementation lives under `commands/`. The // `ts dev` group is available on every host target; its only subcommand, diff --git a/crates/trusted-server-cli/src/main.rs b/crates/trusted-server-cli/src/main.rs index 7cee5b1ca..0a325bd55 100644 --- a/crates/trusted-server-cli/src/main.rs +++ b/crates/trusted-server-cli/src/main.rs @@ -2,10 +2,21 @@ fn main() { use std::process; + // Dependencies such as chromiumoxide instrument their internals with + // `tracing`. Without a subscriber, tracing's log-compatibility fallback + // forwards tolerated CDP decode warnings into the CLI's user-facing logger. + // Trusted Server uses `log` for intentional operator output, so install a + // no-op tracing subscriber to keep dependency diagnostics out of stdout and + // stderr without changing the process-wide `log` level. + let _ = tracing::subscriber::set_global_default(tracing::subscriber::NoSubscriber::default()); edgezero_cli::init_cli_logger(); - if let Err(err) = trusted_server_cli::run_from_env() { - log::error!("[ts] {err}"); - process::exit(2); + match trusted_server_cli::run_from_env() { + Ok(outcome) if outcome.exit_code() != 0 => process::exit(outcome.exit_code()), + Ok(_) => {} + Err(err) => { + log::error!("[ts] {err}"); + process::exit(2); + } } } diff --git a/crates/trusted-server-cli/src/prebid_bundle.rs b/crates/trusted-server-cli/src/prebid_bundle.rs index 802d854ce..abc545926 100644 --- a/crates/trusted-server-cli/src/prebid_bundle.rs +++ b/crates/trusted-server-cli/src/prebid_bundle.rs @@ -559,7 +559,6 @@ mod tests { r#" [integrations.prebid] enabled = true -server_url = "https://prebid.example.com/openrtb2/auction" external_bundle_url = "https://assets.example.com/prebid/trusted-prebid-old.js" [integrations.prebid.bundle] @@ -595,7 +594,6 @@ user_id_modules = ["sharedIdSystem", "uid2IdSystem"] r#" [integrations.prebid] enabled = true -server_url = "https://prebid.example.com/openrtb2/auction" [integrations.prebid.bundle] adapters = ["rubicon"] @@ -626,7 +624,6 @@ adapters = ["rubicon"] r#" [integrations.prebid] enabled = true -server_url = "https://prebid.example.com/openrtb2/auction" "#, ); @@ -646,7 +643,6 @@ server_url = "https://prebid.example.com/openrtb2/auction" r#" [integrations.prebid] enabled = true -server_url = "https://prebid.example.com/openrtb2/auction" [integrations.prebid.bundle] adapters = [] @@ -667,7 +663,6 @@ adapters = [] r#" [integrations.prebid] enabled = true -server_url = "https://prebid.example.com/openrtb2/auction" [integrations.prebid.bundle] adapters = ["rubicon", 123] diff --git a/crates/trusted-server-cli/src/run.rs b/crates/trusted-server-cli/src/run.rs index 13009d448..ec56238c1 100644 --- a/crates/trusted-server-cli/src/run.rs +++ b/crates/trusted-server-cli/src/run.rs @@ -7,8 +7,8 @@ use edgezero_cli::args::{ }; use trusted_server_core::config::TrustedServerAppConfig; -use crate::commands::audit::AuditArgs; -use crate::commands::audit::browser_collector::BrowserAuditCollector; +use crate::commands::audit::{AuditArgs, run_audit}; +use crate::commands::config::ad_templates::{AdTemplatesCommand, run_ad_templates}; use crate::commands::config::init::{ConfigInitArgs, run_config_init}; use crate::prebid_bundle::{NpmPrebidBundleGenerator, PrebidBundleArgs, run_bundle}; @@ -23,8 +23,8 @@ struct Args { enum Command { /// Print the currently active deployment version for a target adapter. ActiveVersion(ActiveVersionArgs), - /// Audit a public page and write draft Trusted Server artifacts. - Audit(AuditArgs), + /// Browser-backed page and ad-template audits. + Audit(Box), /// Sign in / out / status against an `EdgeZero` adapter. Auth(AuthArgs), /// Build the project for a target adapter. @@ -51,6 +51,9 @@ enum Command { #[derive(Debug, Subcommand)] enum ConfigCommand { + /// Diagnose server-side ad-template configuration and path matching. + #[command(name = "ad-templates", subcommand)] + AdTemplates(AdTemplatesCommand), /// Initialize a Trusted Server config file from the example template. Init(ConfigInitArgs), /// Diff `trusted-server.toml` against the live `EdgeZero` config. @@ -75,44 +78,71 @@ enum PrebidCommand { Bundle(PrebidBundleArgs), } +/// Process-level outcome for commands that distinguish drift from tool errors. +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +pub enum RunOutcome { + /// Command completed without drift. + Success, + /// Command ran successfully and found assertion drift. + AssertionFailed, +} + +impl RunOutcome { + /// Stable process exit code for this outcome. + #[must_use] + pub const fn exit_code(self) -> i32 { + match self { + Self::Success => 0, + Self::AssertionFailed => 1, + } + } +} + /// Run the CLI using process arguments. /// /// # Errors /// /// Returns an error when command parsing, config validation, `EdgeZero` /// delegation, audit collection, config initialization, or Prebid bundle generation fails. -pub fn run_from_env() -> Result<(), String> { +pub fn run_from_env() -> Result { dispatch(Args::parse()) } -fn dispatch(args: Args) -> Result<(), String> { +fn dispatch(args: Args) -> Result { match args.command { - Command::ActiveVersion(args) => edgezero_cli::run_active_version(&args), - Command::Audit(args) => { - let stdout = std::io::stdout(); - let mut out = stdout.lock(); - let collector = BrowserAuditCollector; - crate::commands::audit::run_audit(&args, &collector, &mut out) + Command::ActiveVersion(args) => { + edgezero_cli::run_active_version(&args).map(|()| RunOutcome::Success) + } + Command::Auth(args) => edgezero_cli::run_auth(&args).map(|()| RunOutcome::Success), + Command::Audit(args) => run_audit(&args), + Command::Build(args) => edgezero_cli::run_build(&args).map(|()| RunOutcome::Success), + Command::Config(ConfigCommand::AdTemplates(args)) => run_ad_templates(&args), + Command::Config(ConfigCommand::Init(args)) => { + run_config_init(&args).map(|()| RunOutcome::Success) } - Command::Auth(args) => edgezero_cli::run_auth(&args), - Command::Build(args) => edgezero_cli::run_build(&args), - Command::Config(ConfigCommand::Init(args)) => run_config_init(&args), Command::Config(ConfigCommand::Diff(args)) => { match edgezero_cli::run_config_diff_typed::(&args) { - Ok(edgezero_cli::DiffExit { code: 0 }) => Ok(()), + Ok(edgezero_cli::DiffExit { code: 0 }) => Ok(RunOutcome::Success), + Ok(edgezero_cli::DiffExit { code: 1 }) => Ok(RunOutcome::AssertionFailed), Ok(edgezero_cli::DiffExit { code }) => process::exit(code), Err(err) => Err(err), } } - Command::Config(ConfigCommand::Gc(args)) => edgezero_cli::run_config_gc(&args), + Command::Config(ConfigCommand::Gc(args)) => { + edgezero_cli::run_config_gc(&args).map(|()| RunOutcome::Success) + } Command::Config(ConfigCommand::Push(args)) => { edgezero_cli::run_config_push_typed::(&args) + .map(|()| RunOutcome::Success) } Command::Config(ConfigCommand::Validate(args)) => { edgezero_cli::run_config_validate_typed::(&args) + .map(|()| RunOutcome::Success) + } + Command::Deploy(args) => edgezero_cli::run_deploy(&args).map(|()| RunOutcome::Success), + Command::Healthcheck(args) => { + edgezero_cli::run_healthcheck(&args).map(|()| RunOutcome::Success) } - Command::Deploy(args) => edgezero_cli::run_deploy(&args), - Command::Healthcheck(args) => edgezero_cli::run_healthcheck(&args), Command::Prebid(prebid) => { let mut generator = NpmPrebidBundleGenerator; let mut stdout = std::io::stdout(); @@ -120,13 +150,16 @@ fn dispatch(args: Args) -> Result<(), String> { match prebid.command { PrebidCommand::Bundle(args) => { run_bundle(&args, &mut generator, &mut stdout, &mut stderr) + .map(|()| RunOutcome::Success) } } } - Command::Provision(args) => edgezero_cli::run_provision(&args), - Command::Rollback(args) => edgezero_cli::run_rollback(&args), - Command::Serve(args) => edgezero_cli::run_serve(&args), - Command::Dev(command) => crate::commands::dev::run(command), + Command::Provision(args) => { + edgezero_cli::run_provision(&args).map(|()| RunOutcome::Success) + } + Command::Rollback(args) => edgezero_cli::run_rollback(&args).map(|()| RunOutcome::Success), + Command::Serve(args) => edgezero_cli::run_serve(&args).map(|()| RunOutcome::Success), + Command::Dev(command) => crate::commands::dev::run(command).map(|()| RunOutcome::Success), } } @@ -143,6 +176,12 @@ mod tests { Args::try_parse_from(args).expect("should parse args") } + #[test] + fn run_outcomes_use_documented_exit_codes() { + assert_eq!(RunOutcome::Success.exit_code(), 0); + assert_eq!(RunOutcome::AssertionFailed.exit_code(), 1); + } + #[test] fn top_level_version_flag_is_available() { let err = Args::try_parse_from(["ts", "--version"]) @@ -353,64 +392,6 @@ mod tests { ); } - #[test] - fn parses_audit_with_default_outputs() { - let args = parse(&["ts", "audit", "https://publisher.example"]); - let Command::Audit(audit) = args.command else { - panic!("expected audit command"); - }; - assert_eq!(audit.url, "https://publisher.example"); - assert_eq!(audit.js_assets, None); - assert_eq!(audit.config, None); - assert!(!audit.no_js_assets); - assert!(!audit.no_config); - assert!(!audit.force); - } - - #[test] - fn parses_audit_with_custom_outputs() { - let args = parse(&[ - "ts", - "audit", - "https://publisher.example", - "--js-assets", - "audit/js-assets.toml", - "--config", - "audit/trusted-server.toml", - "--no-js-assets", - "--no-config", - "--force", - ]); - let Command::Audit(audit) = args.command else { - panic!("expected audit command"); - }; - assert_eq!(audit.js_assets, Some(PathBuf::from("audit/js-assets.toml"))); - assert_eq!( - audit.config, - Some(PathBuf::from("audit/trusted-server.toml")) - ); - assert!(audit.no_js_assets); - assert!(audit.no_config); - assert!(audit.force); - } - - #[test] - fn audit_does_not_accept_adapter_option() { - let error = Args::try_parse_from([ - "ts", - "audit", - "https://publisher.example", - "--adapter", - "fastly", - ]) - .expect_err("should reject audit adapter option"); - assert!( - error.to_string().contains("unexpected argument") - || error.to_string().contains("Found argument"), - "error should explain unsupported option" - ); - } - #[test] fn parses_build_with_adapter_args() { let args = parse(&[ @@ -638,6 +619,420 @@ mod tests { assert_eq!(validate.manifest, default_validate.manifest); } + #[test] + fn config_ad_templates_match_parses_app_config_flags() { + let args = parse(&[ + "ts", + "config", + "ad-templates", + "match", + "--app-config", + "publisher-a.toml", + "--no-env", + "--details", + "/news/story", + ]); + let Command::Config(ConfigCommand::AdTemplates(AdTemplatesCommand::Match(match_args))) = + args.command + else { + panic!("expected ad-templates match command"); + }; + assert_eq!( + match_args.config.app_config, + Some(PathBuf::from("publisher-a.toml")) + ); + assert!(match_args.config.no_env); + assert!(match_args.details); + assert_eq!(match_args.path_or_url, "/news/story"); + } + + #[test] + fn config_ad_templates_check_parses_expected_slots() { + let args = parse(&[ + "ts", + "config", + "ad-templates", + "check", + "/sports/game", + "--expected-slot", + "atf", + "--expected-slot", + "sports-sidebar", + "--allow-extra-slots", + ]); + let Command::Config(ConfigCommand::AdTemplates(AdTemplatesCommand::Check(check_args))) = + args.command + else { + panic!("expected ad-templates check command"); + }; + assert_eq!(check_args.path_or_url, "/sports/game"); + assert_eq!(check_args.expected_slots, ["atf", "sports-sidebar"]); + assert!(check_args.allow_extra_slots); + assert!(!check_args.expect_no_slots); + } + + #[test] + fn config_ad_templates_check_requires_an_expectation_mode() { + assert!(Args::try_parse_from(["ts", "config", "ad-templates", "check", "/news"]).is_err()); + } + + #[test] + fn config_ad_templates_check_rejects_extra_slots_with_no_slots_mode() { + assert!( + Args::try_parse_from([ + "ts", + "config", + "ad-templates", + "check", + "/news", + "--expect-no-slots", + "--allow-extra-slots", + ]) + .is_err() + ); + } + + #[test] + fn config_ad_templates_explain_rejects_removed_edgezero_model() { + assert!( + Args::try_parse_from([ + "ts", + "config", + "ad-templates", + "explain", + "/news", + "--edgezero-enabled", + ]) + .is_err() + ); + } + + #[test] + fn cli_definition_is_valid() { + // clap validates `requires` / `conflicts_with` argument-id references + // only from an explicit `debug_assert`. Without this, renaming or + // typoing an id compiles and ships. + ::command().debug_assert(); + } + + #[test] + fn bare_audit_namespace_displays_help_as_an_error() { + let error = Args::try_parse_from(["ts", "audit"]).expect_err("should require audit mode"); + + assert_eq!( + error.kind(), + clap::error::ErrorKind::DisplayHelpOnMissingArgumentOrSubcommand + ); + } + + #[test] + fn audit_legacy_url_parses_with_artifact_generation_flags() { + let args = parse(&[ + "ts", + "audit", + "https://www.example.com/", + "--js-assets", + "audit/assets.toml", + "--config", + "audit/config.toml", + "--force", + "--cookie", + "session=example", + "--chrome", + "/tmp/test-chrome", + "--headful", + "--no-assume-consent", + "--browser-proxy", + "127.0.0.1:8080", + "--settle-quiet-ms", + "900", + "--settle-max-ms", + "13000", + "--danger-accept-invalid-certs", + ]); + let Command::Audit(audit) = args.command else { + panic!("expected audit command"); + }; + assert_eq!( + audit.legacy_generate.js_assets, + Some(PathBuf::from("audit/assets.toml")) + ); + assert_eq!( + audit.legacy_generate.config, + Some(PathBuf::from("audit/config.toml")) + ); + assert!(audit.legacy_generate.force); + assert_eq!( + audit.legacy_generate.cookies, + [("session".to_string(), "example".to_string())] + ); + assert_eq!( + audit.legacy_generate.browser.chrome, + Some(PathBuf::from("/tmp/test-chrome")) + ); + assert!(audit.legacy_generate.browser.headful); + assert!(audit.legacy_generate.browser.no_assume_consent); + assert_eq!( + audit.legacy_generate.browser.browser_proxy.as_deref(), + Some("127.0.0.1:8080") + ); + assert_eq!(audit.legacy_generate.browser.settle_quiet_ms, 900); + assert_eq!(audit.legacy_generate.browser.settle_max_ms, 13_000); + assert!(audit.legacy_generate.browser.danger_accept_invalid_certs); + } + + #[test] + fn audit_help_does_not_advertise_hidden_legacy_browser_flags() { + let error = + Args::try_parse_from(["ts", "audit", "--help"]).expect_err("should render audit help"); + let help = error.to_string(); + + for flag in [ + "--chrome", + "--headful", + "--no-assume-consent", + "--browser-proxy", + "--settle-quiet-ms", + "--settle-max-ms", + "--danger-accept-invalid-certs", + ] { + assert!( + !help.contains(flag), + "`{flag}` is a legacy-only alias flag and must stay hidden; got {help}" + ); + } + } + + #[test] + fn audit_rejects_parent_browser_flags_before_a_subcommand() { + // `is_err()` alone would also pass if `--chrome` were deleted from + // `LegacyBrowserOpts` (an `UnknownArgument`), which is the opposite of + // the invariant this pins: the flag exists but requires the legacy URL. + let error = Args::try_parse_from([ + "ts", + "audit", + "--chrome", + "/tmp/test-chrome", + "generate", + "https://www.example.com/", + ]) + .expect_err("a parent-level browser flag must not be silently ignored"); + + assert_eq!( + error.kind(), + clap::error::ErrorKind::MissingRequiredArgument, + "should reject the flag for lacking the legacy URL it requires" + ); + } + + #[test] + fn audit_page_subcommand_parses_with_page_settle_defaults() { + let args = parse(&["ts", "audit", "page", "https://www.example.com/"]); + let Command::Audit(audit) = args.command else { + panic!("expected audit command"); + }; + let Some(crate::commands::audit::AuditSubcommand::Page(page)) = audit.command else { + panic!("expected audit page command"); + }; + assert_eq!(page.browser.settle_quiet_ms, 750); + assert_eq!(page.browser.settle_max_ms, 10_000); + } + + #[test] + fn audit_generate_subcommands_use_generation_settle_defaults() { + let args = parse(&["ts", "audit", "generate", "https://www.example.com/"]); + let Command::Audit(audit) = args.command else { + panic!("expected audit command"); + }; + let Some(crate::commands::audit::AuditSubcommand::Generate(generate)) = audit.command + else { + panic!("expected audit generate command"); + }; + assert_eq!(generate.browser.settle_quiet_ms, 750); + assert_eq!(generate.browser.settle_max_ms, 12_000); + + let args = parse(&[ + "ts", + "audit", + "ad-templates", + "generate", + "https://www.example.com/", + ]); + let Command::Audit(audit) = args.command else { + panic!("expected audit command"); + }; + let Some(crate::commands::audit::AuditSubcommand::AdTemplates( + crate::commands::audit::AuditAdTemplatesCommand::Generate(generate), + )) = audit.command + else { + panic!("expected audit ad-templates generate command"); + }; + assert_eq!(generate.browser.settle_quiet_ms, 750); + assert_eq!(generate.browser.settle_max_ms, 12_000); + assert!(!generate.scroll, "generation should not scroll by default"); + } + + #[test] + fn audit_ad_templates_generate_parses_scroll() { + let args = parse(&[ + "ts", + "audit", + "ad-templates", + "generate", + "https://www.example.com/", + "--scroll", + ]); + let Command::Audit(audit) = args.command else { + panic!("expected audit command"); + }; + let Some(crate::commands::audit::AuditSubcommand::AdTemplates( + crate::commands::audit::AuditAdTemplatesCommand::Generate(generate), + )) = audit.command + else { + panic!("expected audit ad-templates generate command"); + }; + + assert!( + generate.scroll, + "--scroll should enable generation scrolling" + ); + } + + #[test] + fn audit_ad_templates_verify_parses() { + let args = parse(&[ + "ts", + "audit", + "ad-templates", + "verify", + "https://www.example.com/", + ]); + assert!(matches!(args.command, Command::Audit(_))); + } + + #[test] + fn audit_browser_options_are_shared_by_generate_and_verify() { + for mode in ["generate", "verify"] { + let args = parse(&[ + "ts", + "audit", + "ad-templates", + mode, + "https://www.example.com/", + "--chrome", + "/tmp/test-chrome", + "--headful", + "--browser-proxy", + "127.0.0.1:8080", + "--no-assume-consent", + "--settle-quiet-ms", + "100", + "--settle-max-ms", + "200", + ]); + let Command::Audit(audit) = args.command else { + panic!("expected audit command"); + }; + let (chrome, headful, no_assume_consent, browser_proxy, validation) = + match audit.command.expect("should parse audit subcommand") { + crate::commands::audit::AuditSubcommand::AdTemplates( + crate::commands::audit::AuditAdTemplatesCommand::Generate(args), + ) => { + let validation = args.browser.validate(); + ( + args.browser.chrome, + args.browser.headful, + args.browser.no_assume_consent, + args.browser.browser_proxy, + validation, + ) + } + crate::commands::audit::AuditSubcommand::AdTemplates( + crate::commands::audit::AuditAdTemplatesCommand::Verify(args), + ) => { + let validation = args.browser.validate(); + ( + args.browser.chrome, + args.browser.headful, + args.browser.no_assume_consent, + args.browser.browser_proxy, + validation, + ) + } + _ => panic!("expected ad-template mode"), + }; + assert_eq!(chrome, Some(PathBuf::from("/tmp/test-chrome"))); + assert!(headful); + assert!(no_assume_consent); + assert_eq!(browser_proxy.as_deref(), Some("127.0.0.1:8080")); + validation.expect("should validate settle bounds"); + } + } + + #[test] + fn audit_generate_does_not_expose_the_ignored_browser_profile_flag() { + assert!( + Args::try_parse_from([ + "ts", + "audit", + "ad-templates", + "generate", + "https://www.example.com/", + "--browser-profile", + "mobile", + ]) + .is_err(), + "generation device selection must use --profiles" + ); + } + + #[test] + fn browser_settle_quiet_cannot_exceed_maximum() { + let args = parse(&[ + "ts", + "audit", + "page", + "https://www.example.com/", + "--settle-quiet-ms", + "201", + "--settle-max-ms", + "200", + ]); + let Command::Audit(audit) = args.command else { + panic!("expected audit command"); + }; + let crate::commands::audit::AuditSubcommand::Page(page) = + audit.command.expect("should parse page subcommand") + else { + panic!("expected page audit"); + }; + assert!(page.browser.validate().is_err()); + } + + #[test] + fn audit_ad_templates_without_verify_is_error() { + assert!(Args::try_parse_from(["ts", "audit", "ad-templates"]).is_err()); + } + + #[test] + fn audit_rejects_non_http_url() { + assert!(Args::try_parse_from(["ts", "audit", "ftp://www.example.com/"]).is_err()); + } + + #[test] + fn audit_does_not_accept_adapter_option() { + let error = Args::try_parse_from([ + "ts", + "audit", + "page", + "https://www.example.com/", + "--adapter", + "fastly", + ]) + .expect_err("should reject audit adapter option"); + assert!(error.to_string().contains("unexpected argument")); + } + #[test] fn prebid_bundle_defaults_match_spec() { let args = parse(&["ts", "prebid", "bundle"]); diff --git a/crates/trusted-server-cli/tests/config_env_overlay.rs b/crates/trusted-server-cli/tests/config_env_overlay.rs index ed86e1b4c..45eab5cc9 100644 --- a/crates/trusted-server-cli/tests/config_env_overlay.rs +++ b/crates/trusted-server-cli/tests/config_env_overlay.rs @@ -31,6 +31,8 @@ const REWRITE_ENV: &str = "TRUSTED_SERVER__AUCTION__REWRITE_CREATIVES"; const SANITIZE_ENV: &str = "TRUSTED_SERVER__AUCTION__SANITIZE_CREATIVES"; const GAM_ATTRIBUTION_ENV: &str = "TRUSTED_SERVER__INTEGRATIONS__GPT__GAM_ATTRIBUTION_ENABLED"; const AD_TEMPLATES_ENABLED_ENV: &str = "TRUSTED_SERVER__CREATIVE_OPPORTUNITIES__ENABLED"; +const PROVIDER_ENDPOINT_ENV: &str = "TRUSTED_SERVER__AUCTION__PROVIDERS__PBS-MAIN__ENDPOINT"; +const BIDDER_PROVIDER_ENV: &str = "TRUSTED_SERVER__AUCTION__BIDDERS__EXAMPLE-BIDDER__PROVIDER"; struct MigratedProject { directory: TempDir, @@ -38,7 +40,7 @@ struct MigratedProject { manifest_path: std::path::PathBuf, } -fn migrated_legacy_project() -> MigratedProject { +fn migrated_project() -> MigratedProject { let directory = tempfile::tempdir().expect("should create temporary config directory"); let config_path = directory.path().join("trusted-server.toml"); let manifest_path = directory.path().join("edgezero.toml"); @@ -52,6 +54,13 @@ fn migrated_legacy_project() -> MigratedProject { document["auction"]["sanitize_creatives"] = value(false); document["creative_opportunities"]["enabled"] = value(true); document["creative_opportunities"]["gam_network_id"] = value("123456789"); + document["auction"]["providers"]["pbs-main"] = toml_edit::table(); + document["auction"]["providers"]["pbs-main"]["protocol"] = value("openrtb-2.6"); + document["auction"]["providers"]["pbs-main"]["profile"] = value("standard"); + document["auction"]["providers"]["pbs-main"]["endpoint"] = + value("https://original.example/openrtb2/auction"); + document["auction"]["bidders"]["example-bidder"] = toml_edit::table(); + document["auction"]["bidders"]["example-bidder"]["provider"] = value("pbs-main"); fs::write(&config_path, document.to_string()).expect("should write migrated config"); fs::write(&manifest_path, MANIFEST).expect("should write test manifest"); MigratedProject { @@ -74,8 +83,8 @@ fn validate_with_overlay(project: &MigratedProject, raw_value: &str) -> Output { } #[test] -fn migrated_legacy_config_applies_rewrite_creatives_environment_override() { - let project = migrated_legacy_project(); +fn map_config_applies_rewrite_creatives_environment_override() { + let project = migrated_project(); let output = Command::new(env!("CARGO_BIN_EXE_ts")) .args(["config", "push", "--adapter", "axum", "--manifest"]) .arg(&project.manifest_path) @@ -117,8 +126,8 @@ fn migrated_legacy_config_applies_rewrite_creatives_environment_override() { } #[test] -fn migrated_legacy_config_applies_boolean_environment_overrides() { - let project = migrated_legacy_project(); +fn migrated_config_applies_boolean_environment_overrides() { + let project = migrated_project(); let output = Command::new(env!("CARGO_BIN_EXE_ts")) .args(["config", "push", "--adapter", "axum", "--manifest"]) .arg(&project.manifest_path) @@ -166,8 +175,8 @@ fn migrated_legacy_config_applies_boolean_environment_overrides() { } #[test] -fn migrated_legacy_config_applies_sanitize_creatives_environment_override() { - let project = migrated_legacy_project(); +fn migrated_config_applies_sanitize_creatives_environment_override() { + let project = migrated_project(); let output = Command::new(env!("CARGO_BIN_EXE_ts")) .args(["config", "push", "--adapter", "axum", "--manifest"]) .arg(&project.manifest_path) @@ -209,8 +218,8 @@ fn migrated_legacy_config_applies_sanitize_creatives_environment_override() { } #[test] -fn migrated_legacy_config_default_rewrite_creatives_has_no_local_diff() { - let project = migrated_legacy_project(); +fn map_config_default_rewrite_creatives_has_no_local_diff() { + let project = migrated_project(); let push = Command::new(env!("CARGO_BIN_EXE_ts")) .args(["config", "push", "--adapter", "axum", "--manifest"]) .arg(&project.manifest_path) @@ -279,8 +288,8 @@ fn migrated_legacy_config_default_rewrite_creatives_has_no_local_diff() { } #[test] -fn migrated_legacy_config_rejects_invalid_rewrite_creatives_environment_override() { - let project = migrated_legacy_project(); +fn map_config_rejects_invalid_rewrite_creatives_environment_override() { + let project = migrated_project(); let output = validate_with_overlay(&project, "not-a-boolean"); let stderr = String::from_utf8_lossy(&output.stderr); @@ -293,3 +302,52 @@ fn migrated_legacy_config_rejects_invalid_rewrite_creatives_environment_override "error should identify the invalid boolean overlay: {stderr}" ); } + +#[test] +fn map_shaped_provider_and_bidder_environment_overlays_apply() { + let project = migrated_project(); + let output = Command::new(env!("CARGO_BIN_EXE_ts")) + .args(["config", "push", "--adapter", "axum", "--manifest"]) + .arg(&project.manifest_path) + .arg("--app-config") + .arg(&project.config_path) + .args(["--yes", "--no-diff"]) + .current_dir(project.directory.path()) + .env( + PROVIDER_ENDPOINT_ENV, + "https://overlay.example/openrtb2/auction", + ) + .env(BIDDER_PROVIDER_ENV, "pbs-main") + .output() + .expect("should run ts config push with map overlays"); + + assert!( + output.status.success(), + "map-shaped overlays should push successfully: {}", + String::from_utf8_lossy(&output.stderr) + ); + let local_store_path = project + .directory + .path() + .join(".edgezero/local-config-trusted_server_config.json"); + let local_store: serde_json::Value = serde_json::from_str( + &fs::read_to_string(local_store_path).expect("should read pushed local config"), + ) + .expect("should parse local config store"); + let envelope_json = local_store + .as_object() + .and_then(|entries| entries.values().next()) + .and_then(serde_json::Value::as_str) + .expect("should contain a blob envelope"); + let envelope: serde_json::Value = + serde_json::from_str(envelope_json).expect("should parse blob envelope"); + + assert_eq!( + envelope["data"]["auction"]["providers"]["pbs-main"]["endpoint"], + "https://overlay.example/openrtb2/auction" + ); + assert_eq!( + envelope["data"]["auction"]["bidders"]["example-bidder"]["provider"], + "pbs-main" + ); +} diff --git a/crates/trusted-server-core/examples/local_dev_config.rs b/crates/trusted-server-core/examples/local_dev_config.rs new file mode 100644 index 000000000..acbb61b6c --- /dev/null +++ b/crates/trusted-server-core/examples/local_dev_config.rs @@ -0,0 +1,129 @@ +//! Generate a ready-to-use local dev config envelope for the Axum adapter. +//! +//! Reads `trusted-server.example.toml`, replaces the placeholder secrets with +//! random values, flips the flags a local smoke test needs, validates the +//! result through [`trusted_server_core::settings::Settings::from_toml`], and +//! prints the blob envelope JSON that the Axum adapter's +//! `TRUSTED_SERVER_CONFIG_{STORE}_{KEY}` environment variable expects. With +//! the default store and key both named `trusted_server_config`, the +//! concrete variable resolves (not a typo) to +//! `TRUSTED_SERVER_CONFIG_TRUSTED_SERVER_CONFIG_TRUSTED_SERVER_CONFIG`. +//! +//! The random values are time-and-pid seeded, not cryptographic. This tool +//! exists for throwaway local test instances only; never use its output for a +//! deployed service. +//! +//! Usage: +//! +//! ```text +//! cargo run -p trusted-server-core --example local_dev_config \ +//! --target -- [origin-url] [--realistic] +//! ``` +//! +//! `origin-url` defaults to `https://www.example.com`. By default every +//! response is forced `Cache-Control: private, no-store` so the Server-Timing +//! header is visible on all routes; pass `--realistic` to keep the origin's +//! own cache policy instead. + +use std::time::{SystemTime, UNIX_EPOCH}; + +/// Deliberately non-cryptographic generator for local placeholder secrets. +struct WeakRandom(u64); + +impl WeakRandom { + fn from_environment() -> Self { + let nanos = SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("should compute epoch time") + .subsec_nanos() as u64; + let secs = SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("should compute epoch time") + .as_secs(); + let pid = std::process::id() as u64; + Self(nanos ^ (secs << 20) ^ (pid << 40) ^ 0x9e37_79b9_7f4a_7c15) + } + + fn next(&mut self) -> u64 { + let mut x = self.0; + x ^= x << 13; + x ^= x >> 7; + x ^= x << 17; + self.0 = x; + x + } + + fn hex(&mut self, chars: usize) -> String { + let mut out = String::with_capacity(chars); + while out.len() < chars { + out.push_str(&format!("{:016x}", self.next())); + } + out.truncate(chars); + out + } +} + +#[allow(clippy::print_stdout, clippy::print_stderr)] +fn main() { + let args: Vec = std::env::args().skip(1).collect(); + let realistic = args.iter().any(|a| a == "--realistic"); + let origin = args + .iter() + .find(|a| !a.starts_with("--")) + .cloned() + .unwrap_or_else(|| "https://www.example.com".to_string()); + + let template = std::fs::read_to_string("trusted-server.example.toml") + .expect("should read trusted-server.example.toml from the repo root"); + + let mut random = WeakRandom::from_environment(); + let mut config = template + .replace( + "password = \"replace-with-admin-password-32-bytes\"", + &format!("password = \"{}\"", random.hex(48)), + ) + .replace( + "proxy_secret = \"change-me-proxy-secret\"", + &format!("proxy_secret = \"{}\"", random.hex(48)), + ) + .replace( + "passphrase = \"trusted-server-placeholder-secret\"", + &format!("passphrase = \"{}\"", random.hex(48)), + ) + .replace( + "server_timing_enabled = false", + "server_timing_enabled = true", + ); + + let origin_line = config + .lines() + .find(|line| line.starts_with("origin_url = ")) + .expect("should find the origin_url line in the template") + .to_string(); + config = config.replace(&origin_line, &format!("origin_url = \"{origin}\"")); + + if !realistic { + config = config.replace( + "# [response_headers]", + "[response_headers]\n\"Cache-Control\" = \"private, no-store\"", + ); + } + + let settings = trusted_server_core::settings::Settings::from_toml(&config) + .expect("should validate the generated local config"); + let data = serde_json::to_value(&settings).expect("should serialize settings"); + let generated_at = SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("should compute epoch time") + .as_secs() + .to_string(); + let envelope = edgezero_core::blob_envelope::BlobEnvelope::new(data, generated_at); + println!( + "{}", + serde_json::to_string(&envelope).expect("should serialize the envelope") + ); + eprintln!( + "local dev envelope generated: origin={origin} force_private={}", + !realistic + ); +} diff --git a/crates/trusted-server-core/src/access_telemetry.rs b/crates/trusted-server-core/src/access_telemetry.rs new file mode 100644 index 000000000..ba18c869a --- /dev/null +++ b/crates/trusted-server-core/src/access_telemetry.rs @@ -0,0 +1,627 @@ +//! Access telemetry: route classification and the per-request access log row. +//! +//! Extends the reserved `access_logs_raw` Tinybird datasource with bounded, +//! content-free route identity (see [`RouteClass`] and +//! [`publisher_route_template`]) instead of the raw request path, which would +//! otherwise carry identifiers, search terms, and other user-generated +//! content into a 30-day dataset. See the design spec +//! `docs/superpowers/specs/2026-08-24-request-phase-timing-design.md` +//! section 9. + +use serde_json::json; + +use crate::request_timing::{AuctionWaitPlacement, TimingSnapshot}; + +/// Maximum length of a publisher path's first segment before +/// [`publisher_route_template`] rejects it to `/other/*`. Longer segments +/// are opaque-identifier or slug shaped (a UUID is 36 characters), and a +/// truncated prefix of either would still be identifying, so the segment +/// is rejected whole rather than truncated. +const MAX_SEGMENT_LEN: usize = 32; + +/// Maximum number of ASCII digits in a publisher path's first segment +/// before [`publisher_route_template`] rejects it to `/other/*`. Hex ids, +/// base36 ids, and reset tokens are digit-heavy; real section names carry +/// at most a year (`2026`) or a small version number, so a segment with +/// more digits than this is treated as an identifier, not a name. +const MAX_SEGMENT_DIGITS: usize = 7; + +/// Normalizes an HTTP method token into the bounded set of values stored in +/// the `method` `LowCardinality` column. +/// +/// HTTP permits arbitrary extension-method tokens (`PROPFIND`, `MKCOL`, or +/// any client-supplied garbage), and the token on an inbound request is +/// entirely client controlled. Capturing one verbatim into a 30-day +/// `LowCardinality(String)` column would let a single caller inflate that +/// column's cardinality without bound and would violate this dataset's +/// bounded-dimension privacy rule (see the module doc). Every standard +/// method maps to its uppercase form; anything else maps to `"other"`. Runs +/// inside [`access_event_row`] rather than at each capture site, so every +/// row-building path is covered regardless of how `method` was populated. +/// +/// # Examples +/// +/// ``` +/// use trusted_server_core::access_telemetry::normalize_method; +/// +/// assert_eq!(normalize_method("get"), "GET"); +/// assert_eq!(normalize_method("PROPFIND"), "other"); +/// assert_eq!(normalize_method(""), + "other", + "an unbounded client-controlled token must not reach the row verbatim" + ); + } + + #[test] + fn row_normalizes_method_even_when_snapshot_carries_a_raw_token() { + // The normalizer runs inside `access_event_row` so every row-building + // path is covered, regardless of what the snapshot's `method` field + // holds — a caller-controlled extension method must never leak into + // the row unnormalized. + let mut snapshot = unknown_snapshot(RouteClass::Other, "/other/*"); + snapshot.method = "PROPFIND".to_owned(); + let row = access_event_row(&snapshot, &TimingSnapshot::default(), 0); + let parsed: serde_json::Value = + serde_json::from_str(&row).expect("should serialize valid JSON"); + + assert_eq!(parsed["method"], "other"); + } +} diff --git a/crates/trusted-server-core/src/auction/README.md b/crates/trusted-server-core/src/auction/README.md index 69c19475b..8a745eabc 100644 --- a/crates/trusted-server-core/src/auction/README.md +++ b/crates/trusted-server-core/src/auction/README.md @@ -1,583 +1,181 @@ -# Auction Orchestration System +# Auction orchestration + +The auction module compiles operator configuration into one immutable plan, +routes browser demand to providers, runs provider requests concurrently where +the adapter permits it, and returns normalized OpenRTB bids. -A flexible, extensible framework for managing multi-provider header bidding auctions with support for parallel execution and mediation. +The maintained operator guide is +[`docs/guide/auction-orchestration.md`](../../../../docs/guide/auction-orchestration.md). +This file describes the code layout and runtime flow for contributors. -## Overview - -The auction orchestration system allows you to: -- Run multiple auction providers (Prebid, Amazon APS, etc.) in parallel or sequentially -- Implement mediation strategies where a primary ad server makes the final decision -- Configure different auction flows for different scenarios -- Easily add new auction providers - -## Architecture - -``` -┌─────────────────────────────────────────────────────────┐ -│ Auction Orchestrator │ -│ - Manages auction workflow & sequencing │ -│ - Combines bids from multiple sources │ -│ - Applies business logic │ -└─────────────────────────────────────────────────────────┘ - │ - │ uses - ▼ -┌─────────────────────────────────────────────────────────┐ -│ AuctionProvider Trait │ -│ - request_bids() async │ -│ - parse_response() │ -│ - provider_name() │ -│ - timeout_ms() │ -│ - is_enabled() │ -└─────────────────────────────────────────────────────────┘ - │ - ┌─────────────────┼─────────────────┐ - │ │ │ - ▼ ▼ ▼ - ┌──────────┐ ┌──────────┐ ┌──────────┐ - │ Prebid │ │ Amazon │ │ AdServer │ - │ Provider │ │ APS │ │ Mock │ - └──────────┘ └──────────┘ └──────────┘ +## Runtime flow + +```mermaid +flowchart TB + A[Adapter app.rs routes POST /auction] --> B[endpoints::handle_auction] + B --> C[endpoints::convert_tsjs_to_auction_request] + C --> D[routing::route_auction] + D --> E[provider::GenericOpenRtbProvider builds requests] + E --> F[orchestrator::AuctionOrchestrator dispatches providers] + F --> G[Provider responses are normalized] + G --> H{Mediator configured?} + H -->|Yes| I[Mediator selects bids] + H -->|No| J[Orchestrator ranks bids locally] + I --> K[formats::convert_to_openrtb_response] + J --> K + K --> L[HTTP 200 OpenRTB response] ``` -## Request Flow +Each adapter owns transport routing in its `app.rs`. Core request handling stays +in `auction::endpoints`, so no provider or profile depends on Fastly types. -When a request arrives at the `/auction` endpoint, it goes through the following steps: +`handle_auction` performs these steps: -``` -┌──────────────────────────────────────────────────────────────────────┐ -│ 1. HTTP POST /auction │ -│ - Body: AdRequest (Prebid.js/tsjs format) │ -│ - Headers: User-Agent, cookies, etc. │ -└──────────────────────────────────────────────────────────────────────┘ - │ - ▼ -┌──────────────────────────────────────────────────────────────────────┐ -│ 2. Route Matching (crates/trusted-server-adapter-fastly/src/main.rs)│ -│ - Pattern: (Method::POST, "/auction") │ -│ - Handler: handle_auction(settings, &orchestrator, │ -│ &runtime_services, req) │ -└──────────────────────────────────────────────────────────────────────┘ - │ - ▼ -┌──────────────────────────────────────────────────────────────────────┐ -│ 3. Parse Request Body (mod.rs:149) │ -│ - Deserialize JSON → AdRequest struct │ -│ - Extract ad units with media types │ -└──────────────────────────────────────────────────────────────────────┘ - │ - ▼ -┌──────────────────────────────────────────────────────────────────────┐ -│ 4. Generate User IDs (mod.rs:206-214) │ -│ - Create/retrieve EC ID (persistent) │ -│ - Generate fresh ID (per-request) │ -└──────────────────────────────────────────────────────────────────────┘ - │ - ▼ -┌──────────────────────────────────────────────────────────────────────┐ -│ 5. Transform Request Format (mod.rs:216-240) │ -│ - AdRequest → AuctionRequest │ -│ - AdUnit.code → AdSlot.id │ -│ - mediaTypes.banner.sizes → AdFormat[] │ -│ - Build PublisherInfo, UserInfo, DeviceInfo │ -└──────────────────────────────────────────────────────────────────────┘ - │ - ▼ -┌──────────────────────────────────────────────────────────────────────┐ -│ 6. Use Provided Orchestrator (mod.rs:150) │ -│ - Reused across requests from startup construction │ -│ - Contains all registered providers (APS, Prebid, etc.) │ -└──────────────────────────────────────────────────────────────────────┘ - │ - ▼ -┌──────────────────────────────────────────────────────────────────────┐ -│ 7. Create Auction Context (mod.rs:172-176) │ -│ - Attach settings │ -│ - Attach original request │ -│ - Set timeout from config │ -└──────────────────────────────────────────────────────────────────────┘ - │ - ▼ -┌──────────────────────────────────────────────────────────────────────┐ -│ 8. Run Auction Strategy (orchestrator.rs:42) │ -│ ┌────────────────────────────────────────────────────────────┐ │ -│ │ Strategy: parallel_only │ │ -│ │ 1. Launch all bidders concurrently │ │ -│ │ 2. Wait for all responses │ │ -│ │ 3. Select highest bid per slot │ │ -│ └────────────────────────────────────────────────────────────┘ │ -│ ┌────────────────────────────────────────────────────────────┐ │ -│ │ Strategy: parallel_mediation │ │ -│ │ 1. Launch all bidders concurrently │ │ -│ │ 2. Collect all bids │ │ -│ │ 3. Send to mediator for final decision │ │ -│ └────────────────────────────────────────────────────────────┘ │ -└──────────────────────────────────────────────────────────────────────┘ - │ - ▼ -┌──────────────────────────────────────────────────────────────────────┐ -│ 9. Each Provider Processes Request │ -│ - Transform AuctionRequest → Provider OpenRTB request │ -│ - Send HTTP request to provider endpoint │ -│ - Parse provider response │ -│ - Transform → AuctionResponse with Bid[] │ -│ - Return to orchestrator │ -└──────────────────────────────────────────────────────────────────────┘ - │ - ▼ -┌──────────────────────────────────────────────────────────────────────┐ -│ 10. Select Winning Bids (orchestrator.rs:363-385) │ -│ - For each slot, find highest CPM bid │ -│ - Create HashMap │ -│ - Log winning selections │ -└──────────────────────────────────────────────────────────────────────┘ - │ - ▼ -┌──────────────────────────────────────────────────────────────────────┐ -│ 11. Transform to OpenRTB Response (mod.rs:274-322) │ -│ - Build seatbid array (one per winning bid) │ -│ - Sanitize creative HTML when enabled (opt-in) │ -│ - Rewrite creative HTML when enabled (default) │ -│ - Add orchestrator metadata (timing, strategy, bid count) │ -└──────────────────────────────────────────────────────────────────────┘ - │ - ▼ -┌──────────────────────────────────────────────────────────────────────┐ -│ 12. Return HTTP Response │ -│ - Status: 200 OK │ -│ - Content-Type: application/json │ -│ - Body: OpenRTB BidResponse │ -└──────────────────────────────────────────────────────────────────────┘ -``` +1. Enforce the body limit and parse the Trusted Server ad-unit request. +2. Apply the disabled-auction and consent gates before provider work. +3. Consume the request's existing EC and consent context. The endpoint does not + generate an EC ID. +4. Convert the request with `convert_tsjs_to_auction_request`. +5. Route slots and bidder params through the compiled `AuctionPlan`. +6. Run the plan-backed orchestrator and optional mediator. +7. Build the OpenRTB response with `convert_to_openrtb_response`. -### Step-by-Step Breakdown - -#### 1. Request Arrival -Client (browser, Prebid.js, tsjs) sends a POST request to `/auction` with ad unit definitions: - -```json -{ - "adUnits": [ - { - "code": "header-banner", - "mediaTypes": { - "banner": { - "sizes": [[728, 90], [970, 250]] - } - } - } - ] -} -``` +## Configuration boundary -#### 2. Format Transformation -The system transforms the Prebid.js format into an internal `AuctionRequest`: - -```rust -// From: AdUnit with sizes [[728, 90], [970, 250]] -// To: AdSlot with formats -AdSlot { - id: "header-banner", - formats: vec![ - AdFormat { width: 728, height: 90, media_type: Banner }, - AdFormat { width: 970, height: 250, media_type: Banner }, - ], - floor_price: None, - targeting: HashMap::new(), -} -``` +`auction::compile_auction_plan` is the single settings-to-plan boundary used by +startup and operator validation. It validates: -#### 3. Provider Execution -Each registered provider (APS, Prebid, etc.) receives the `AuctionRequest` and: -- Transforms it to the provider's OpenRTB request format -- Makes HTTP request to their endpoint -- Parses the response -- Returns `AuctionResponse` with `Bid[]` - -For example, APS provider: -```rust -// Transform AuctionRequest → APS OpenRTB request -// - ext.account = configured account_id -// - ext.sdk = { source: "prebid", version: "2.2.0" } -// - banner slots become secure impressions with matching formats/floors -// - existing consent, identity, device, and geo privacy gates apply - -// HTTP POST to https://aps.example.com/e/pb/bid -// Parse decoded-price response → AuctionResponse with a typed renderer -``` +- provider IDs, protocols, profiles, endpoints, timeouts, and routing modes; +- bidder-to-provider ownership; +- profile-specific configuration; +- mediator and request-signing references; +- bounded configuration values. -#### 4. Response Assembly -The orchestrator collects all bids and creates an OpenRTB response: - -```json -{ - "id": "auction-response", - "seatbid": [ - { - "seat": "aps", - "bid": [ - { - "id": "fictional-selected-bid-id", - "impid": "header-banner", - "price": 2.5, - "w": 728, - "h": 90, - "ext": { - "trusted_server": { - "renderer": { - "type": "aps", - "version": 1, - "accountId": "example-account", - "bidId": "fictional-selected-bid-id", - "tagType": "iframe", - "creativeUrl": "https://creative.example/render", - "aaxResponse": "", - "width": 728, - "height": 90 - } - } - } - } - ] - } - ], - "ext": { - "orchestrator": { - "strategy": "parallel_only", - "bidders": 1, - "total_bids": 1, - "time_ms": 5 - } - } -} -``` +Adapters then call `AuctionPlan::validate_for_target` for backend naming, +fan-out support, and target resource limits. -With `[auction].sanitize_creatives = true` (opt-in, default `false`), -executable markup is stripped with its inner content before delivery. With -`[auction].rewrite_creatives = true` (the default), each auction delivery path -rewrites eligible URLs through the first-party proxy (`/first-party/proxy`) and -removes bidder `` elements. The `POST /auction` response also injects the -creative runtime; the publisher SSAT inline path uses absolute first-party URLs -without injecting that bundle. With both disabled, the creative ships exactly -as the bidder returned it. In every mode, creatives over the 1 MiB cap are -rejected. - -## Route Registration & Endpoints - -### Auction-Related Routes - -The trusted-server handles several types of routes defined in `crates/trusted-server-adapter-fastly/src/main.rs`: - -| Route | Method | Handler | Purpose | Line | -|---------------------------|--------|--------------------------------|--------------------------------------------------|------| -| `/auction` | POST | `handle_auction()` | Main auction endpoint (Prebid.js/tsjs format) | 84 | -| `/first-party/proxy` | GET | `handle_first_party_proxy()` | Proxy creatives through first-party domain | 84 | -| `/first-party/click` | GET | `handle_first_party_click()` | Track clicks on ads | 85 | -| `/first-party/sign` | GET/POST | `handle_first_party_proxy_sign()` | Generate signed URLs for creatives | 86 | -| `/first-party/proxy-rebuild` | GET/POST | `handle_first_party_proxy_rebuild()` | Re-sign mutated click URLs (GET 302s for the opaque-origin click guard) | 89 | -| `/static/tsjs=*` | GET | `handle_tsjs_dynamic()` | Serve tsjs library (Prebid.js alternative) | 66 | -| `/.well-known/ts.jwks.json` | GET | `handle_jwks_endpoint()` | Public key distribution for request signing | 71 | -| `/verify-signature` | POST | `handle_verify_signature()` | Verify signed requests | 74 | -| `/_ts/admin/keys/rotate` | POST | `handle_rotate_key()` | Rotate signing keys (admin only) | 77 | -| `/_ts/admin/keys/deactivate` | POST | `handle_deactivate_key()` | Deactivate signing keys (admin only) | 78 | -| `/integrations/*` | * | Integration Registry | Provider-specific endpoints (Prebid, etc.) | 92 | -| `*` (fallback) | * | `handle_publisher_request()` | Proxy to publisher origin | 108 | - -### How Routing Works - -#### 1. Main Router (main.rs) -The Fastly Compute entrypoint uses pattern matching on `(Method, path)` tuples: - -```rust -let result = match (method, path.as_str()) { - // Auction endpoint - (Method::POST, "/auction") => { - handle_auction(&settings, &orchestrator, &runtime_services, req).await - }, - - // First-party endpoints - (Method::GET, "/first-party/proxy") => handle_first_party_proxy(&settings, req).await, - - // Integration registry (dynamic routes) - (m, path) if integration_registry.has_route(&m, path) => { - integration_registry.handle_proxy(&m, path, &settings, req).await - }, - - // Fallback to publisher origin - _ => handle_publisher_request(&settings, &integration_registry, &runtime_services, req), -} -``` +A plan-backed orchestrator contains generic OpenRTB providers compiled from the +plan. `AuctionOrchestrator::register_provider` and the old concrete Prebid +provider remain test-only parity code. They are not extension APIs. -#### 2. Integration Registry (Dynamic Routes) -Some integrations register their own routes dynamically. For example, Prebid registers `/integrations/prebid/auction`: - -```rust -// In integrations/prebid.rs -impl Integration for PrebidIntegration { - fn routes(&self) -> Vec { - vec![ - IntegrationRoute { - path: "/integrations/prebid/auction", - method: Method::POST, - handler: handle_prebid_auction, - } - ] - } -} -``` +## Routing -The integration registry checks if a route matches any registered integration routes before falling back to the publisher origin. - -#### 3. Route Priority -Routes are matched in this order: -1. **Exact top-level routes** (`/auction`, `/first-party/proxy`, etc.) -2. **Admin routes** (`/_ts/admin/*`) -3. **Integration routes** (`/integrations/*`) -4. **Fallback to publisher origin** (all other paths) - -This ensures auction and first-party endpoints take precedence over publisher content. - -### Auction Endpoint Deep Dive - -The `/auction` endpoint is the primary entry point for auctions: - -**Input Format (Prebid.js compatible):** -```json -{ - "adUnits": [ - { - "code": "div-id", - "mediaTypes": { - "banner": { - "sizes": [[300, 250], [728, 90]] - } - } - } - ], - "config": { /* optional Prebid.js config */ } -} -``` +`routing::route_auction` normalizes the browser `trustedServer` envelope and +produces one `ProviderAuctionInput` per provider. -**Output Format (OpenRTB 2.x):** -```json -{ - "id": "auction-response", - "seatbid": [ - { - "seat": "bidder-name", - "bid": [ - { - "id": "bid-id", - "impid": "div-id", - "price": 2.5, - "adm": "", - "w": 300, - "h": 250 - } - ] - } - ], - "ext": { - "orchestrator": { - "strategy": "parallel_only", - "bidders": 2, - "total_bids": 3, - "time_ms": 150 - } - } -} -``` +- `explicit` sends a slot only when it has bidder demand assigned to that + provider, or trusted stored-request demand where the profile supports it. +- `all_eligible` sends every compatible banner slot without copying another + provider's bidder params. +- `prebid-server` requires `explicit`. PBS rejects impressions that have neither + bidder demand nor a stored-request reference. +- APS normally uses `all_eligible` because APS participates across eligible + inventory without browser bidder params. -**Key Transformations:** -- `adUnits[].code` → `seatbid[].bid[].impid` (slot identifier) -- `mediaTypes.banner.sizes` → evaluated by providers, winning size in `bid.w` and `bid.h` -- Creative HTML: `[auction].sanitize_creatives = true` (opt-in) strips executable markup; `[auction].rewrite_creatives = true` (default) rewrites eligible URLs to `/first-party/proxy` in both delivery paths (with creative runtime injection on `POST /auction` only); with both disabled the creative ships as the bidder returned it -- Multiple bids per slot become separate `seatbid` entries -- Orchestrator metadata added in `ext.orchestrator` +Each `[auction.bidders.]` route has one provider owner. Unlisted page +bidders remain browser demand. -## Key Concepts +## Provider execution -### Auction Provider -Implements the `AuctionProvider` trait to integrate with a specific SSP/ad exchange. +`provider::GenericOpenRtbProvider` owns the shared transport path for the +`standard`, `prebid-server`, and `aps` profiles. Profiles receive routed and +privacy-approved facts, not the raw inbound request. -### Auction Flow -A named configuration that defines: -- Which providers participate -- Execution strategy (parallel mediation or parallel only) -- Timeout settings -- Optional mediator +The orchestrator launches all eligible providers before collecting responses. +It uses adapter `PlatformHttpClient` handles and predicted backend names for +correlation. Provider launch, transport, HTTP, parse, and admission failures are +provider-local when another provider can continue. -### Orchestrator -Manages the execution of an auction flow, coordinates providers, and collects results. +When no mediator is configured, the orchestrator selects the highest decoded +CPM per slot and applies floors locally. When a mediator is configured, it sends +normalized provider responses to the separately registered mediator and falls +back to local ranking when mediation cannot run. -## Auction Strategies +## Response admission -### 1. Parallel + Mediation (Recommended) -**Use case:** Header bidding with ad server mediation +Providers normalize successful upstream bids into `auction::types::Bid`. +Admission checks keep malformed or unrequested bids out of ranking. Aggregate +metadata reports bounded rejection counts without retaining raw upstream bid +payloads. -```toml -[auction] -enabled = true -providers = ["prebid", "aps"] -mediator = "adserver_mock" # Setting mediator enables parallel mediation strategy -timeout_ms = 2000 -``` - -**Flow:** -1. Prebid and APS run in parallel -2. Both return their bids simultaneously -3. Bids are sent to the mediator for final decision -4. Mediator competes house inventory and returns winning creative - -### 2. Parallel Only -**Use case:** Client-side auction, no mediation - -```toml -[auction] -enabled = true -providers = ["prebid", "aps"] -# No mediator = parallel only strategy (highest CPM wins) -timeout_ms = 2000 -``` +Notification suppression runs after normalization and matches exact returned +OpenRTB seats. Provider response identity uses the configured provider ID, such +as `pbs-main`. -**Flow:** -1. All providers run in parallel -2. Highest bid wins -3. No mediation server involved +## Creative delivery -## Configuration +`formats::convert_to_openrtb_response` assembles the direct `POST /auction` +response. -### Configuration +- `sanitize_creatives = true` strips executable markup. It is opt-in. +- `rewrite_creatives = true` rewrites eligible URLs through first-party routes + and removes bidder `` elements. It is enabled by default. +- The publisher inline delivery path uses absolute first-party URLs without + injecting the direct endpoint's creative runtime. +- Creatives over the configured hard cap are rejected. -All auction settings are configured directly under `[auction]`: +## Example plan ```toml [auction] -enabled = true # Enable/disable auction orchestration -providers = ["prebid", "aps"] # List of bidder providers -mediator = "adserver_mock" # Optional: if set, uses mediation; if omitted, highest bid wins -timeout_ms = 2000 # Overall auction timeout -``` - -**Strategy Auto-Detection:** -- When `mediator` is configured → Runs **parallel mediation** (providers in parallel, mediator decides winner) -- When `mediator` is omitted → Runs **parallel only** (providers in parallel, highest CPM wins) - -### Provider Configuration - -Each provider has its own configuration section: - -```toml -[integrations.prebid] enabled = true -server_url = "https://prebid-server.example.com" -timeout_ms = 1000 - -[integrations.aps] -enabled = true -mock = true # Set to false for real integration -timeout_ms = 800 - -[integrations.adserver_mock] -enabled = true -endpoint = "http://localhost:6767/adserver/mediate" -timeout_ms = 500 -``` - -## Adding a New Provider - -1. Create a new file in `src/auction/providers/your_provider.rs` - -```rust -use async_trait::async_trait; -use crate::auction::provider::{AuctionProvider, ProviderRequestOutcome}; -use crate::auction::types::{AuctionContext, AuctionRequest, AuctionResponse}; -use crate::platform::PlatformResponse; - -pub struct YourAuctionProvider { - config: YourConfig, -} - -#[async_trait(?Send)] -impl AuctionProvider for YourAuctionProvider { - fn provider_name(&self) -> &'static str { - "your_provider" - } - - async fn request_bids( - &self, - request: &AuctionRequest, - _context: &AuctionContext<'_>, - ) -> Result> { - // 1. Transform AuctionRequest to your provider's format - // 2. Launch through services.http_client().send_async(...) - // 3. Wrap the handle with ProviderRequestOutcome::pending(...) - todo!() - } - - async fn parse_response( - &self, - response: PlatformResponse, - response_time_ms: u64, - ) -> Result> { - // 4. Parse PlatformResponse into AuctionResponse - todo!() - } - - fn timeout_ms(&self) -> u32 { - self.config.timeout_ms - } - - fn is_enabled(&self) -> bool { - self.config.enabled - } -} -``` - -2. Register the provider in `src/auction/providers/mod.rs` +timeout_ms = 2000 -3. Configure it in `trusted-server.toml` +[auction.providers.pbs-main] +protocol = "openrtb-2.6" +profile = "prebid-server" +endpoint = "https://prebid.example.com/openrtb2/auction" +timeout_ms = 900 +routing = "explicit" + +[auction.providers.pbs-main.profile_config] +debug = false +test_mode = false +consent_forwarding = "both" + +[auction.providers.pbs-main.notifications] +suppress_all = false +suppress_seats = ["example-seat"] + +[auction.bidders.example-server] +provider = "pbs-main" + +[auction.providers.aps-main] +protocol = "openrtb-2.6" +profile = "aps" +endpoint = "https://aps.example.com/e/pb/bid" +routing = "all_eligible" +profile_config = { account_id = "example-account" } +``` + +Provider endpoints must be absolute HTTPS URLs. Replace all example values +before enabling an auction. + +## Code map + +- `mod.rs` compiles plans and builds the shared orchestrator. +- `endpoints.rs` handles `POST /auction` and converts the browser request. +- `plan.rs` owns plan validation and target capability checks. +- `profile.rs` owns typed OpenRTB profile configuration. +- `routing.rs` assigns slots and bidder params to providers. +- `openrtb.rs` builds shared requests and parses standard responses. +- `provider.rs` runs plan-backed provider requests and profile-specific parsing. +- `orchestrator.rs` owns fan-out, deadlines, mediation, and local ranking. +- `formats.rs` builds direct endpoint responses and processes creatives. +- `types.rs` contains normalized auction request, response, slot, and bid types. ## Testing -### Mock Providers - -APS and adserver_mock providers are used for testing the orchestration pattern: - -- **APS Mock**: Returns mock bids with Amazon branding -- **AdServer Mock**: Acts as mediator by calling mocktioneer's mediation endpoint, selects winning bids based on highest CPM - -Set `mock = false` in APS config when real APS integration is ready. - -### Example Test Flow - -```rust -let orchestrator = AuctionOrchestrator::new(config); -orchestrator.register_provider(Arc::new(PrebidAuctionProvider::try_new(prebid_config)?)); -orchestrator.register_provider(Arc::new(ApsAuctionProvider::new(aps_config))); +Use `compile_auction_plan` in tests, then construct the orchestrator and +integration registry from the same `Arc`. Profile tests should +cover typed configuration, exact request output, response admission, routing, +provider-local failures, and target validation. -let result = orchestrator.run_auction(&request, &context, &services).await?; +Run target-matched aliases rather than bare workspace tests: -// Check results -assert_eq!(result.winning_bids.len(), 2); -assert!(result.total_time_ms < 2000); +```bash +cargo test-fastly +cargo test-axum +cargo test-cloudflare +cargo test-spin ``` - -## Performance Considerations - -- **Parallel Execution**: Providers are launched concurrently via `select()` over `PendingRequest`s; responses are processed as they become ready within the auction deadline -- **Timeouts**: Each provider has independent timeout; global timeout enforced at flow level -- **Error Handling**: Provider failures don't fail entire auction; partial results returned - -## Related Files - -- `src/auction/mod.rs` - Module exports -- `src/auction/types.rs` - Core auction types -- `src/auction/provider.rs` - Provider trait definition -- `src/auction/orchestrator.rs` - Orchestration logic -- `src/auction/config.rs` - Configuration types -- `src/auction/providers/` - Provider implementations - -## Questions? - -See the main project [README](../../../../README.md) or [integration guide](../../../../docs/guide/integration-guide.md). diff --git a/crates/trusted-server-core/src/auction/endpoints.rs b/crates/trusted-server-core/src/auction/endpoints.rs index fdf387e93..c0c0a7792 100644 --- a/crates/trusted-server-core/src/auction/endpoints.rs +++ b/crates/trusted-server-core/src/auction/endpoints.rs @@ -13,10 +13,10 @@ use crate::consent::{consent_allows_server_side_auction, gate_eids_by_consent}; use crate::constants::COOKIE_TS_EIDS; use crate::cookies::extract_cookie_value; use crate::ec::EcContext; +use crate::ec::EcKvSnapshot; use crate::ec::eids::{resolve_partner_ids, to_eids}; use crate::ec::kv::KvIdentityGraph; use crate::ec::kv_types::MAX_UID_LENGTH; -use crate::ec::log_id; use crate::ec::prebid_eids::parse_prebid_eids_cookie; use crate::ec::registry::PartnerRegistry; use crate::error::TrustedServerError; @@ -119,7 +119,7 @@ pub async fn handle_auction( orchestrator: &AuctionOrchestrator, kv: Option<&KvIdentityGraph>, registry: Option<&PartnerRegistry>, - ec_context: &EcContext, + ec_context: &mut EcContext, services: &RuntimeServices, req: Request, ) -> Result, Report> { @@ -172,13 +172,58 @@ pub async fn handle_auction( // Story 5 middleware contract: auction is a read-only EC route. // It must not generate EC IDs; it only consumes pre-routed context. // Only forward the EC ID to auction partners when consent allows it. + // Owned so the identity-graph snapshot can be stored back on `ec_context` + // below without holding a borrow of it across the mutation. let ec_id = if ec_context.ec_allowed() { - ec_context.ec_value() + ec_context.ec_value().map(str::to_owned) } else { None }; let consent_context = ec_context.consent().clone(); + if !orchestrator.is_enabled() { + log::info!("/auction: auction is disabled; returning no-bid response"); + let auction_request = convert_tsjs_to_auction_request( + &body, + settings, + services, + &http_req, + consent_context, + ec_id.as_deref(), + None, + )?; + let observation = AuctionObservationContext::from_auction_request( + AuctionSource::AuctionApi, + &auction_request, + ec_context, + ); + let elapsed_ms = observation.elapsed_ms(); + emit_auction_events_best_effort_lazy(services, || { + build_auction_events( + observation, + AuctionTerminalOutcome::Skipped { + reason: "auction_disabled", + elapsed_ms, + }, + ) + }) + .await; + + let empty_result = OrchestrationResult { + provider_responses: Vec::new(), + mediator_response: None, + winning_bids: HashMap::new(), + total_time_ms: 0, + metadata: HashMap::new(), + }; + return convert_to_openrtb_response( + &empty_result, + settings, + &auction_request, + ec_context.ec_allowed(), + ); + } + // Server-side auction consent gate. The publisher-navigation and // `/_ts/page-bids` paths fail closed for GDPR/unknown jurisdictions that // lack effective TCF Purpose 1. `/auction` is the programmatic entry point @@ -198,7 +243,7 @@ pub async fn handle_auction( services, &http_req, consent_context, - ec_id, + ec_id.as_deref(), None, )?; let observation = AuctionObservationContext::from_auction_request( @@ -254,9 +299,22 @@ pub async fn handle_auction( None }; - // Resolve partner EIDs from the KV identity graph when the user has - // a valid EC and both KV and partner stores are available. - let eids = resolve_auction_eids(kv, registry, ec_context); + // Resolve partner EIDs from the KV identity graph when the user has a valid + // EC and both KV and partner stores are available. Gate the read on a + // present registry: without one, `resolve_auction_eids` yields no + // server-side EIDs, so the snapshot would be an unused billable KV read. + let auction_kv_snapshot = match (kv, ec_id.as_deref(), registry) { + (Some(graph), Some(ec_id), Some(_)) => graph.load_snapshot(ec_id), + _ => EcKvSnapshot::NotRead, + }; + // Hand the loaded row to the request context so response finalization — + // which runs on an EC context the adapter owns, after this handler returns + // — ingests `ts-eids`/`sharedId` updates from this read instead of paying + // for a second lookup. + if !matches!(auction_kv_snapshot, EcKvSnapshot::NotRead) { + ec_context.set_kv_snapshot(auction_kv_snapshot.clone()); + } + let eids = resolve_auction_eids(&auction_kv_snapshot, registry, ec_context); // Look up geo for device info. let geo = services @@ -274,7 +332,7 @@ pub async fn handle_auction( services, &http_req, consent_context, - ec_id, + ec_id.as_deref(), geo, )?; @@ -293,6 +351,7 @@ pub async fn handle_auction( settings, request: &http_req, timeout_ms: settings.auction.timeout_ms, + transport_timeout_ms: settings.auction.timeout_ms, provider_responses: None, services, }; @@ -380,11 +439,10 @@ pub async fn handle_auction( /// store, no EC, consent denied). On KV or partner-resolution errors, logs a /// warning and returns empty EIDs so the auction can proceed in degraded mode. pub(crate) fn resolve_auction_eids( - kv: Option<&KvIdentityGraph>, + snapshot: &EcKvSnapshot, registry: Option<&PartnerRegistry>, ec_context: &EcContext, ) -> Option> { - let kv = kv?; let registry = registry?; if !ec_context.ec_allowed() { @@ -393,19 +451,15 @@ pub(crate) fn resolve_auction_eids( let ec_id = ec_context.ec_value()?; - let entry = match kv.get(ec_id) { - Ok(Some((entry, _generation))) => entry, - Ok(None) => return Some(Vec::new()), - Err(err) => { - log::warn!( - "Auction KV read failed for EC ID '{}': {err:?}", - log_id(ec_id) - ); - return Some(Vec::new()); - } + let Some(entry) = snapshot.entry_for(ec_id) else { + return Some(Vec::new()); }; - let resolved = resolve_partner_ids(registry, &entry); + if !entry.consent.ok { + return Some(Vec::new()); + } + + let resolved = resolve_partner_ids(registry, entry); Some(to_eids(&resolved)) } @@ -570,6 +624,7 @@ mod tests { use crate::auction::types::{AuctionRequest, AuctionResponse}; use crate::consent::jurisdiction::Jurisdiction; use crate::consent::types::ConsentContext; + use crate::error::IntoHttpResponse as _; use crate::openrtb::Uid; use crate::platform::test_support::{ NoopBackend, NoopConfigStore, NoopGeo, NoopHttpClient, NoopSecretStore, StubHttpClient, @@ -626,13 +681,128 @@ mod tests { ) } + fn counting_test_partner(source_domain: &str) -> crate::settings::EcPartner { + crate::settings::EcPartner { + name: format!("Partner {source_domain}"), + source_domain: source_domain.to_owned(), + openrtb_atype: crate::settings::EcPartner::default_openrtb_atype(), + bidstream_enabled: true, + api_token: Some(crate::redacted::Redacted::new(format!( + "token-{source_domain}-32-bytes-minimum-value" + ))), + batch_rate_limit: crate::settings::EcPartner::default_batch_rate_limit(), + pull_sync_enabled: false, + pull_sync_url: None, + pull_sync_allowed_domains: vec![], + pull_sync_ttl_sec: crate::settings::EcPartner::default_pull_sync_ttl_sec(), + pull_sync_rate_limit: crate::settings::EcPartner::default_pull_sync_rate_limit(), + ts_pull_token: None, + } + } + + #[tokio::test] + async fn auction_endpoint_snapshot_is_reused_by_response_finalization() { + // `/auction` loads the identity-graph row to resolve server-side EIDs. + // Finalization runs afterwards on the same EC context and ingests + // `ts-eids`/`sharedId` updates. Both must be served by a single billable + // read: before the snapshot was shared, finalization saw `NotRead` and + // paid for a second lookup. + let settings = create_test_settings(); + let mut orchestrator = AuctionOrchestrator::new(AuctionConfig { + enabled: true, + providers: AuctionConfig::legacy_provider_map(&["eid_capturing_provider"]), + timeout_ms: 2000, + mediator: None, + ..Default::default() + }); + orchestrator.register_provider(Arc::new(EidCapturingProvider { + had_eids: Arc::new(std::sync::Mutex::new(None)), + })); + let registry = PartnerRegistry::from_config(&[counting_test_partner("sharedid.org")]) + .expect("should build partner registry"); + + let lookups = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let graph = KvIdentityGraph::counting("counting-store", Arc::clone(&lookups)); + let ec_id = format!("{}.ABC123", "a".repeat(64)); + let mut live = crate::ec::kv_types::KvEntry::tombstone(1000); + live.consent.ok = true; + graph.create(&ec_id, &live).expect("should seed live row"); + lookups.store(0, std::sync::atomic::Ordering::Relaxed); + + let mut ec_context = make_ec_context(Jurisdiction::NonRegulated, Some(&ec_id)); + let req = Request::builder() + .method("POST") + .uri("https://test-publisher.com/auction") + .body(EdgeBody::from( + serde_json::to_vec(&json!({ + "adUnits": [ + { + "code": "div-gpt-ad-1", + "mediaTypes": { "banner": { "sizes": [[300, 250]] } } + } + ] + })) + .expect("should serialize body"), + )) + .expect("should build auction request"); + + // The capturing provider deliberately fails its launch; identity + // resolution — the subject of this test — completes before dispatch. + let _ = handle_auction( + &settings, + &orchestrator, + Some(&graph), + Some(®istry), + &mut ec_context, + &noop_services(), + req, + ) + .await; + + assert_eq!( + lookups.load(std::sync::atomic::Ordering::Relaxed), + 1, + "the endpoint should read the identity row exactly once" + ); + assert!( + ec_context.kv_snapshot().entry_for(&ec_id).is_some(), + "the endpoint must hand its snapshot to the request context" + ); + + let mut response = http::Response::new(EdgeBody::empty()); + crate::ec::finalize::ec_finalize_response( + &settings, + &mut ec_context, + Some(&graph), + ®istry, + None, + Some("shared-cookie-id"), + &mut response, + ); + + assert_eq!( + lookups.load(std::sync::atomic::Ordering::Relaxed), + 1, + "finalization must reuse the endpoint snapshot instead of reading again" + ); + let (stored, _) = graph + .get(&ec_id) + .expect("should read store") + .expect("row should exist"); + assert_eq!( + stored.ids.get("sharedid.org").map(|id| id.uid.as_str()), + Some("shared-cookie-id"), + "the sharedId update must still be ingested from the shared snapshot" + ); + } + /// Provider that fails the test if it is ever contacted. Used to prove the /// `/auction` consent gate short-circuits before any outbound bid request. struct PanicOnBidProvider; #[async_trait::async_trait(?Send)] impl AuctionProvider for PanicOnBidProvider { - fn provider_name(&self) -> &'static str { + fn provider_name(&self) -> &str { "panic_provider" } @@ -670,7 +840,7 @@ mod tests { #[async_trait::async_trait(?Send)] impl AuctionProvider for TemplateSwitchProbeProvider { fn provider_name(&self) -> &'static str { - "template_switch_probe" + "template-switch-probe" } async fn request_bids( @@ -722,7 +892,7 @@ mod tests { #[tokio::test] async fn direct_auction_remains_available_when_templates_are_disabled() { let settings_toml = format!( - "{}\n[auction]\nenabled = true\nproviders = [\"template_switch_probe\"]\n\n[creative_opportunities]\nenabled = false\ngam_network_id = \"12345\"\n", + "{}\n[auction]\nenabled = true\n\n[auction.providers.template-switch-probe]\nprotocol = \"openrtb-2.6\"\nendpoint = \"https://bidder.example/auction\"\nrouting = \"all_eligible\"\n\n[creative_opportunities]\nenabled = false\ngam_network_id = \"12345\"\n", crate_test_settings_str() ); let settings = Settings::from_toml(&settings_toml) @@ -744,7 +914,7 @@ mod tests { .geo(Arc::new(NoopGeo)) .client_info(ClientInfo::default()) .build(); - let ec_context = make_ec_context(Jurisdiction::NonRegulated, None); + let mut ec_context = make_ec_context(Jurisdiction::NonRegulated, None); let body = json!({ "adUnits": [{ "code": "div-gpt-ad-1", @@ -764,7 +934,7 @@ mod tests { &orchestrator, None, None, - &ec_context, + &mut ec_context, &services, req, ) @@ -779,6 +949,126 @@ mod tests { assert_eq!(response.status(), StatusCode::OK); } + #[tokio::test] + async fn disabled_auction_endpoint_emits_skipped_telemetry_without_provider_work() { + let settings = create_test_settings(); + let config = AuctionConfig { + enabled: false, + providers: AuctionConfig::legacy_provider_map(&["panic_provider"]), + timeout_ms: 2000, + mediator: None, + ..Default::default() + }; + let mut orchestrator = AuctionOrchestrator::new(config); + orchestrator.register_provider(Arc::new(PanicOnBidProvider)); + let telemetry_sink = Arc::new(RecordingTelemetrySink::default()); + let services = services_with_telemetry(Arc::clone(&telemetry_sink)); + let mut ec_context = make_ec_context(Jurisdiction::NonRegulated, None); + let body = json!({ + "adUnits": [{ + "code": "div-gpt-ad-1", + "mediaTypes": { "banner": { "sizes": [[300, 250]] } } + }] + }); + let request = Request::builder() + .method("POST") + .uri("https://test-publisher.example/auction") + .body(EdgeBody::from( + serde_json::to_vec(&body).expect("should serialize disabled-auction body"), + )) + .expect("should build disabled-auction request"); + + let response = handle_auction( + &settings, + &orchestrator, + None, + None, + &mut ec_context, + &services, + request, + ) + .await + .expect("disabled auction should return a no-bid response"); + + assert_eq!( + response.status(), + StatusCode::OK, + "disabled auction should return a 200 no-bid response" + ); + let batches = telemetry_sink + .batches + .lock() + .expect("should lock telemetry batches"); + assert_eq!(batches.len(), 1, "should emit one telemetry batch"); + let rows = batches[0].rows(); + assert_eq!(rows.len(), 1, "should emit one skipped summary row"); + assert_eq!(rows[0].event_kind, "summary", "should emit a summary row"); + assert_eq!(rows[0].terminal_status.as_deref(), Some("skipped")); + assert_eq!( + rows[0].terminal_reason.as_deref(), + Some("auction_disabled"), + "should identify the disabled auction policy" + ); + } + + #[tokio::test] + async fn all_planned_launch_failures_return_bad_gateway_and_execution_failed_telemetry() { + let settings_toml = format!( + "{}\n[auction]\nenabled = true\n\n[auction.providers.launch-fail]\nprotocol = \"openrtb-2.6\"\nprofile = \"standard\"\nendpoint = \"https://bidder.example/auction\"\nrouting = \"all_eligible\"\n", + crate_test_settings_str() + ); + let settings = + Settings::from_toml(&settings_toml).expect("should parse launch-failure settings"); + let plan = Arc::new( + crate::auction::compile_auction_plan(&settings) + .expect("should compile launch-failure plan"), + ); + let orchestrator = AuctionOrchestrator::from_plan(plan, None); + let telemetry_sink = Arc::new(RecordingTelemetrySink::default()); + let services = services_with_telemetry(Arc::clone(&telemetry_sink)); + let mut ec_context = make_ec_context(Jurisdiction::NonRegulated, None); + let body = json!({ + "adUnits": [{ + "code": "div-gpt-ad-1", + "mediaTypes": { "banner": { "sizes": [[300, 250]] } } + }] + }); + let request = Request::builder() + .method("POST") + .uri("https://test-publisher.example/auction") + .body(EdgeBody::from( + serde_json::to_vec(&body).expect("should serialize launch-failure body"), + )) + .expect("should build launch-failure request"); + + let error = handle_auction( + &settings, + &orchestrator, + None, + None, + &mut ec_context, + &services, + request, + ) + .await + .expect_err("all planned launch failures should fail the auction endpoint"); + + assert_eq!( + error.current_context().status_code(), + StatusCode::BAD_GATEWAY + ); + let batches = telemetry_sink + .batches + .lock() + .expect("should lock telemetry batches"); + assert_eq!(batches.len(), 1, "should emit one telemetry batch"); + let rows = batches[0].rows(); + assert_eq!(rows.len(), 1, "should emit one execution-failure summary"); + assert_eq!(rows[0].event_kind, "summary"); + assert_eq!(rows[0].terminal_status.as_deref(), Some("execution_failed")); + assert_eq!(rows[0].terminal_reason.as_deref(), Some("execution_failed")); + } + #[tokio::test] async fn auction_endpoint_consent_gate_returns_no_bid_without_contacting_providers() { // GDPR/unknown jurisdiction lacking effective TCF Purpose 1 must not run @@ -788,7 +1078,7 @@ mod tests { let settings = create_test_settings(); let config = AuctionConfig { enabled: true, - providers: vec!["panic_provider".to_string()], + providers: AuctionConfig::legacy_provider_map(&["panic_provider"]), timeout_ms: 2000, mediator: None, ..Default::default() @@ -798,7 +1088,7 @@ mod tests { let telemetry_sink = Arc::new(RecordingTelemetrySink::default()); let services = services_with_telemetry(Arc::clone(&telemetry_sink)); let ec_id = format!("{}.ABC123", "a".repeat(64)); - let ec_context = make_ec_context(Jurisdiction::Unknown, Some(&ec_id)); + let mut ec_context = make_ec_context(Jurisdiction::Unknown, Some(&ec_id)); let body = json!({ "adUnits": [ @@ -821,7 +1111,7 @@ mod tests { &orchestrator, None, None, - &ec_context, + &mut ec_context, &services, req, ) @@ -872,7 +1162,7 @@ mod tests { #[async_trait::async_trait(?Send)] impl AuctionProvider for EidCapturingProvider { - fn provider_name(&self) -> &'static str { + fn provider_name(&self) -> &str { "eid_capturing_provider" } @@ -916,7 +1206,7 @@ mod tests { let settings = create_test_settings(); let config = AuctionConfig { enabled: true, - providers: vec!["eid_capturing_provider".to_string()], + providers: AuctionConfig::legacy_provider_map(&["eid_capturing_provider"]), timeout_ms: 2000, mediator: None, ..Default::default() @@ -930,7 +1220,7 @@ mod tests { // US-state jurisdiction with an explicit GPC opt-out: auction allowed, // EC identity denied. - let ec_context = EcContext::new_for_test( + let mut ec_context = EcContext::new_for_test( None, ConsentContext { jurisdiction: Jurisdiction::UsState("CA".to_owned()), @@ -978,7 +1268,7 @@ mod tests { &orchestrator, None, None, - &ec_context, + &mut ec_context, &services, req, ) @@ -992,22 +1282,24 @@ mod tests { } #[test] - fn resolve_auction_eids_returns_none_without_kv() { + fn resolve_auction_eids_returns_empty_without_snapshot() { let registry = PartnerRegistry::empty(); let ec_id = format!("{}.ABC123", "a".repeat(64)); let ec_context = make_ec_context(Jurisdiction::NonRegulated, Some(&ec_id)); - let result = resolve_auction_eids(None, Some(®istry), &ec_context); - assert!(result.is_none(), "should return None when KV is missing"); + let result = resolve_auction_eids(&EcKvSnapshot::NotRead, Some(®istry), &ec_context); + assert!( + result.is_some_and(|eids| eids.is_empty()), + "should degrade to empty EIDs without a snapshot" + ); } #[test] fn resolve_auction_eids_returns_none_without_registry() { - let kv = KvIdentityGraph::failing("test_store"); let ec_id = format!("{}.ABC123", "a".repeat(64)); let ec_context = make_ec_context(Jurisdiction::NonRegulated, Some(&ec_id)); - let result = resolve_auction_eids(Some(&kv), None, &ec_context); + let result = resolve_auction_eids(&EcKvSnapshot::NotRead, None, &ec_context); assert!( result.is_none(), "should return None when registry is missing" @@ -1016,12 +1308,11 @@ mod tests { #[test] fn resolve_auction_eids_returns_none_when_consent_denied() { - let kv = KvIdentityGraph::failing("test_store"); let registry = PartnerRegistry::empty(); let ec_id = format!("{}.ABC123", "a".repeat(64)); let ec_context = make_ec_context(Jurisdiction::Unknown, Some(&ec_id)); - let result = resolve_auction_eids(Some(&kv), Some(®istry), &ec_context); + let result = resolve_auction_eids(&EcKvSnapshot::NotRead, Some(®istry), &ec_context); assert!( result.is_none(), "should return None when consent is denied" @@ -1030,11 +1321,10 @@ mod tests { #[test] fn resolve_auction_eids_returns_none_when_no_ec() { - let kv = KvIdentityGraph::failing("test_store"); let registry = PartnerRegistry::empty(); let ec_context = make_ec_context(Jurisdiction::NonRegulated, None); - let result = resolve_auction_eids(Some(&kv), Some(®istry), &ec_context); + let result = resolve_auction_eids(&EcKvSnapshot::NotRead, Some(®istry), &ec_context); assert!( result.is_none(), "should return None when no EC value is present" @@ -1043,14 +1333,14 @@ mod tests { #[test] fn resolve_auction_eids_returns_empty_on_kv_miss() { - let kv = KvIdentityGraph::failing("nonexistent_store"); let registry = PartnerRegistry::empty(); let ec_id = format!("{}.ABC123", "a".repeat(64)); let ec_context = make_ec_context(Jurisdiction::NonRegulated, Some(&ec_id)); - // KV store doesn't exist, so the get() call will error — should return - // empty Vec (degraded mode), not None. - let result = resolve_auction_eids(Some(&kv), Some(®istry), &ec_context); + let snapshot = EcKvSnapshot::Failed { + ec_id: ec_id.clone(), + }; + let result = resolve_auction_eids(&snapshot, Some(®istry), &ec_context); let eids = result.expect("should return Some on KV error (degraded mode)"); assert!( eids.is_empty(), @@ -1377,7 +1667,7 @@ mod tests { let settings = create_test_settings(); let orchestrator = build_orchestrator(&settings).expect("should build orchestrator"); let services = noop_services(); - let ec_context = EcContext::new_for_test(None, ConsentContext::default()); + let mut ec_context = EcContext::new_for_test(None, ConsentContext::default()); let oversized = vec![b'x'; MAX_AUCTION_BODY_SIZE + 1]; let req = HttpRequest::builder() .method(Method::POST) @@ -1389,7 +1679,7 @@ mod tests { &orchestrator, None, None, - &ec_context, + &mut ec_context, &services, req, ) @@ -1420,7 +1710,7 @@ mod tests { let settings = create_test_settings(); let orchestrator = build_orchestrator(&settings).expect("should build orchestrator"); let services = noop_services(); - let ec_context = EcContext::new_for_test(None, ConsentContext::default()); + let mut ec_context = EcContext::new_for_test(None, ConsentContext::default()); let stream = futures::stream::iter([Bytes::from_static(br#"{}"#)]); let req = HttpRequest::builder() .method(Method::POST) @@ -1433,7 +1723,7 @@ mod tests { &orchestrator, None, None, - &ec_context, + &mut ec_context, &services, req, ) diff --git a/crates/trusted-server-core/src/auction/formats.rs b/crates/trusted-server-core/src/auction/formats.rs index e09912aab..571d9d484 100644 --- a/crates/trusted-server-core/src/auction/formats.rs +++ b/crates/trusted-server-core/src/auction/formats.rs @@ -552,6 +552,10 @@ pub(crate) fn convert_to_openrtb_response_with_report( #[cfg(test)] mod tests { use super::*; + use crate::auction::plan::{ + AuctionPlan, AuctionPlanConfig, NotificationConfig, ProviderConfig, ProviderId, RoutingMode, + }; + use crate::auction::routing::route_auction; use crate::auction::types::{ ApsRendererV1, ApsTagType, AuctionResponse, Bid, BidRenderer, BidStatus, }; @@ -560,7 +564,8 @@ mod tests { use crate::test_support::tests::create_test_settings; use http::Method; use serde_json::json; - use std::collections::HashSet; + use std::collections::{BTreeMap, HashSet}; + use std::str::FromStr as _; fn make_request() -> Request { Request::builder() @@ -575,6 +580,28 @@ mod tests { create_test_settings() } + fn single_prebid_plan() -> AuctionPlan { + AuctionPlan::compile(AuctionPlanConfig { + timeout_ms: 900, + providers: BTreeMap::from([( + ProviderId::from_str("pbs-primary").expect("should parse provider ID"), + ProviderConfig { + protocol: "openrtb-2.6".to_string(), + profile: "prebid-server".to_string(), + endpoint: "https://pbs.example.test/openrtb".to_string(), + timeout_ms: None, + routing: RoutingMode::Explicit, + notifications: NotificationConfig::default(), + profile_config: json!({}), + }, + )]), + bidders: BTreeMap::new(), + mediator: None, + request_signing: None, + }) + .expect("should compile plan") + } + fn make_auction_request() -> AuctionRequest { AuctionRequest { id: "auction-1".to_string(), @@ -622,6 +649,7 @@ mod tests { creative: Some("
Ad
".to_string()), adomain: Some(vec!["advertiser.example.com".to_string()]), bidder: bidder.to_string(), + returned_seat: None, width: 300, height: 250, nurl: None, @@ -715,6 +743,31 @@ mod tests { .expect("should convert banner request") } + #[test] + fn canonical_tsjs_request_without_bids_feeds_stored_request_router() { + let body = AdRequest { + ad_units: vec![AdUnit { + code: "stored-slot".to_string(), + media_types: Some(MediaTypes { + banner: Some(BannerUnit { + sizes: vec![vec![300, 250]], + }), + }), + bids: None, + }], + config: None, + eids: None, + }; + let request = convert_body_to_auction_request(&body, &make_settings()); + let routed = route_auction(request, &make_request(), &single_prebid_plan(), None); + + assert_eq!(routed.inputs().len(), 1); + assert!( + routed.inputs()[0].slots()[0].has_trusted_stored_request(), + "canonical empty bidder map should preserve stored-request intent" + ); + } + #[test] fn response_serializes_prebid_immediate_no_bid_without_error_metadata() { let request = make_auction_request(); @@ -1630,6 +1683,7 @@ mod tests { "should omit adm for renderer bids" ); assert_eq!(bid["id"], json!("fictional-bid")); + assert_eq!(json["seatbid"][0]["seat"], json!("aps")); assert_eq!(bid["adid"], json!("fictional-ad")); assert_eq!(bid["crid"], json!("fictional-creative")); assert_eq!( diff --git a/crates/trusted-server-core/src/auction/mod.rs b/crates/trusted-server-core/src/auction/mod.rs index 986beb984..432303928 100644 --- a/crates/trusted-server-core/src/auction/mod.rs +++ b/crates/trusted-server-core/src/auction/mod.rs @@ -17,8 +17,12 @@ pub mod config; pub mod context; pub mod endpoints; pub mod formats; +pub(crate) mod openrtb; pub mod orchestrator; +pub mod plan; +pub(crate) mod profile; pub mod provider; +pub(crate) mod routing; pub mod telemetry; #[cfg(test)] pub(crate) mod test_support; @@ -27,6 +31,10 @@ pub mod types; pub use config::AuctionConfig; pub use context::{ContextQueryParams, ContextValue, build_url_with_context_params}; pub use orchestrator::AuctionOrchestrator; +pub use plan::{ + AuctionPlan, BidderId, BidderRouteConfig, NotificationConfig, ProviderConfig, ProviderId, + RoutingMode, +}; pub use provider::AuctionProvider; pub use telemetry::{ AbandonedProviderCall, AuctionEventBatch, AuctionEventRow, AuctionObservationContext, @@ -37,132 +45,189 @@ pub use types::{ AdFormat, AuctionContext, AuctionRequest, AuctionResponse, Bid, BidStatus, MediaType, }; -/// Type alias for provider builder functions. -type ProviderBuilder = - fn(&Settings) -> Result>, Report>; - -/// Returns the list of all available provider builder functions. +/// Compile the canonical target-independent auction plan for [`Settings`]. /// -/// This list is used to auto-discover and register auction providers from settings. -/// Each builder function checks the settings for its specific provider configuration -/// and returns any enabled providers. -fn provider_builders() -> &'static [ProviderBuilder] { - &[ - crate::integrations::prebid::register_auction_provider, - crate::integrations::aps::register_providers, - crate::integrations::adserver_mock::register_providers, - ] +/// This is the single settings-to-plan boundary used by deploy validation, +/// adapter startup, and operator tooling. Global request signing remains owned +/// by [`Settings`] and is copied into compiler input only at this boundary. +/// +/// # Errors +/// +/// Returns an error when auction provider, bidder route, signing, or mediator +/// configuration is invalid. +pub fn compile_auction_plan( + settings: &Settings, +) -> Result> { + AuctionPlan::compile(plan::AuctionPlanConfig { + timeout_ms: settings.auction.timeout_ms, + providers: settings.auction.providers.clone(), + bidders: settings.auction.bidders.clone(), + mediator: settings.auction.mediator.clone(), + request_signing: settings.request_signing.clone(), + }) + .map(|plan| plan.with_enabled(settings.auction.enabled)) } -/// Build a new auction orchestrator for the current settings. +/// Build a new auction orchestrator from one shared compiled plan. /// /// This constructor registers all auction providers discovered from the provided settings. /// Callers can reuse the returned [`AuctionOrchestrator`] across requests. /// /// # Arguments -/// * `settings` - Application settings used to configure the orchestrator and providers +/// * `plan` - Shared immutable compiled plan +/// * `settings` - Application settings used only for the separately registered mediator /// /// # Errors /// /// Returns an error when an enabled auction provider has invalid configuration. -pub fn build_orchestrator( +pub fn build_orchestrator_with_plan( + plan: Arc, settings: &Settings, ) -> Result> { - log::info!("Building auction orchestrator"); - - let mut orchestrator = AuctionOrchestrator::new(settings.auction.clone()); - - // Auto-discover and register all auction providers from settings - for builder in provider_builders() { - for provider in builder(settings)? { - orchestrator.register_provider(provider); - } - } - - orchestrator.validate_configured_provider_names()?; + log::info!("Building plan-backed auction orchestrator"); + + let mediator = if let Some(expected_id) = plan.mediator() { + let provider = crate::integrations::adserver_mock::register_providers(settings)? + .into_iter() + .find(|provider| provider.provider_name() == expected_id) + .ok_or_else(|| { + Report::new(TrustedServerError::Configuration { + message: format!( + "auction mediator `{expected_id}` must reference a separately registered enabled integration with the exact same ID" + ), + }) + })?; + Some(provider) + } else { + None + }; + let orchestrator = AuctionOrchestrator::from_plan(plan, mediator); log::info!( - "Auction orchestrator built with {} providers", + "Auction orchestrator built with {} bidder providers", orchestrator.provider_count() ); Ok(orchestrator) } +/// Test convenience constructor that compiles a plan before construction. +/// +/// # Errors +/// +/// Returns an error when plan compilation or mediator construction fails. #[cfg(test)] -mod tests { - use crate::settings::Settings; - use crate::test_support::tests::crate_test_settings_str; +pub fn build_orchestrator( + settings: &Settings, +) -> Result> { + let plan = Arc::new(compile_auction_plan(settings)?); + build_orchestrator_with_plan(plan, settings) +} - use super::build_orchestrator; +#[cfg(test)] +mod plan_sharing_tests { + use super::*; + use crate::integrations::IntegrationRegistry; + use crate::test_support::tests::create_test_settings; - fn settings_with_auction_config(auction_config: &str) -> Settings { - let settings_str = format!("{}\n{auction_config}", crate_test_settings_str()); - let mut settings = Settings::from_toml(&settings_str) - .expect("should parse auction provider validation test settings"); - settings.proxy.allowed_domains = vec!["*.example".to_string(), "*.example.com".to_string()]; - settings + #[test] + fn orchestrator_and_registry_share_the_compiled_plan_allocation() { + let settings = create_test_settings(); + let plan = Arc::new(compile_auction_plan(&settings).expect("should compile auction plan")); + let orchestrator = build_orchestrator_with_plan(Arc::clone(&plan), &settings) + .expect("should build orchestrator"); + let registry = IntegrationRegistry::with_plan(&settings, Arc::clone(&plan)) + .expect("should build integration registry"); + + assert!(orchestrator.shares_plan(&plan)); + assert!(registry.shares_plan(&plan)); } - fn assert_orchestrator_error_contains(settings: &Settings, expected: &str) { - let Err(err) = build_orchestrator(settings) else { - panic!("build_orchestrator should reject invalid auction providers"); - }; - assert!( - err.to_string().contains(expected), - "should include expected validation message: {expected}" - ); + #[test] + fn configured_mediator_requires_enabled_exact_registration() { + for mediator_config in [None, Some(serde_json::json!({"enabled": false}))] { + let mut settings = create_test_settings(); + settings.auction.mediator = Some("adserver_mock".to_string()); + if let Some(config) = mediator_config { + settings + .integrations + .insert_config("adserver_mock", &config) + .expect("should insert mediator config"); + } else { + settings.integrations.remove("adserver_mock"); + } + let plan = Arc::new(compile_auction_plan(&settings).expect("should compile plan")); + + let error = match build_orchestrator_with_plan(plan, &settings) { + Ok(_) => panic!("should require enabled mediator registration"), + Err(error) => error, + }; + assert!(error.to_string().contains("adserver_mock")); + } } #[test] - fn configured_unregistered_provider_fails_startup() { - let settings = settings_with_auction_config( - r#" - [auction] - enabled = true - providers = ["missing-provider"] - timeout_ms = 2000 - "#, - ); - - assert_orchestrator_error_contains( - &settings, - "Auction provider `missing-provider` is listed in [auction] but no enabled integration provides it", - ); + fn configured_mediator_builds_when_exact_registration_is_enabled() { + let mut settings = create_test_settings(); + settings.auction.mediator = Some("adserver_mock".to_string()); + settings + .integrations + .insert_config( + "adserver_mock", + &serde_json::json!({ + "enabled": true, + "endpoint": "https://mediator.example/mediate" + }), + ) + .expect("should insert mediator config"); + let plan = Arc::new(compile_auction_plan(&settings).expect("should compile plan")); + + build_orchestrator_with_plan(plan, &settings) + .expect("should build with enabled exact mediator registration"); } #[test] - fn mixed_registered_and_unregistered_providers_fail_startup() { - let settings = settings_with_auction_config( - r#" - [auction] - enabled = true - providers = ["prebid", "missing-provider"] - timeout_ms = 2000 - "#, - ); - - assert_orchestrator_error_contains( - &settings, - "Auction provider `missing-provider` is listed in [auction] but no enabled integration provides it", - ); + fn cloudflare_and_spin_reject_multi_provider_plans_before_runtime_construction() { + let mut settings = create_test_settings(); + settings.auction.enabled = true; + settings.auction.providers = + AuctionConfig::legacy_provider_map(&["provider-a", "provider-b"]); + let plan = compile_auction_plan(&settings).expect("should compile target-independent plan"); + + for target in [ + crate::platform::AuctionTargetId::Cloudflare, + crate::platform::AuctionTargetId::Spin, + ] { + let error = plan + .validate_for_target(target) + .expect_err("should reject unsupported multi-provider fanout"); + assert!( + error + .to_string() + .contains("does not support concurrent provider fanout") + ); + } } #[test] - fn configured_unregistered_mediator_fails_startup() { - let settings = settings_with_auction_config( - r#" - [auction] - enabled = true - providers = ["prebid"] - mediator = "missing-mediator" - timeout_ms = 2000 - "#, - ); - - assert_orchestrator_error_contains( - &settings, - "Auction provider `missing-mediator` is listed in [auction] but no enabled integration provides it", - ); + fn aps_profile_registers_renderer_without_browser_aps_config() { + let mut settings = create_test_settings(); + settings.auction.providers = std::collections::BTreeMap::from([( + "aps-main".parse().expect("should parse APS provider ID"), + ProviderConfig { + protocol: "openrtb-2.6".to_string(), + profile: "aps".to_string(), + endpoint: "https://aps.example/e/pb/bid".to_string(), + timeout_ms: None, + routing: RoutingMode::AllEligible, + notifications: NotificationConfig::default(), + profile_config: serde_json::json!({"account_id":"example-account"}), + }, + )]); + let plan = Arc::new(compile_auction_plan(&settings).expect("should compile APS plan")); + let registry = IntegrationRegistry::with_plan(&settings, plan) + .expect("should build APS renderer registry"); + + assert!(registry.has_route(&http::Method::GET, "/integrations/aps/renderer")); } } diff --git a/crates/trusted-server-core/src/auction/openrtb.rs b/crates/trusted-server-core/src/auction/openrtb.rs new file mode 100644 index 000000000..0a142c1de --- /dev/null +++ b/crates/trusted-server-core/src/auction/openrtb.rs @@ -0,0 +1,888 @@ +//! Shared `OpenRTB` 2.6 request/response support for config-first providers. +//! +//! Profiles receive only routed, privacy-approved facts and never the raw +//! downstream request or unrestricted runtime services. + +use std::collections::{BTreeMap, BTreeSet, HashMap}; + +use error_stack::Report; +use serde_json::{Map, Value, json}; +use url::Url; + +use super::plan::{NotificationPolicy, ProviderPlan}; +use super::profile::{ + ApsProfilePlan, CompiledOpenRtbProfile, PrebidProfilePlan, StandardProfilePlan, +}; +use super::routing::{ + PrebidTransportHeaders, ProviderAuctionInput, ProviderSlotInput, RoutedAuction, +}; +use super::types::{AuctionResponse, Bid}; +use crate::consent::ConsentSource; +use crate::error::TrustedServerError; +use crate::openrtb::{ + Banner, ConsentedProvidersSettings, Device, Format, Geo, Imp, OpenRtbRequest, Publisher, Regs, + RegsExt, Site, ToExt as _, TrustedServerExt, User, UserExt, to_openrtb_i32, +}; +use crate::request_signing::{RequestSigner, SIGNING_VERSION, SigningParams}; + +const DEFAULT_CURRENCY: &str = "USD"; +const APS_SDK_SOURCE: &str = "prebid"; +const APS_SDK_VERSION: &str = "2.2.0"; +const MAX_CONSERVATIVE_LANGUAGE_BYTES: usize = 8; + +/// Fixed reasons why an upstream bid failed response admission. +#[derive(Debug, Clone, Copy, Eq, Ord, PartialEq, PartialOrd)] +pub(crate) enum BidRejectionReason { + InvalidBid, + UnrequestedImpression, + DimensionMismatch, + AmbiguousDimensions, +} + +impl BidRejectionReason { + fn as_str(self) -> &'static str { + match self { + Self::InvalidBid => "invalid_bid", + Self::UnrequestedImpression => "unrequested_impression", + Self::DimensionMismatch => "dimension_mismatch", + Self::AmbiguousDimensions => "ambiguous_dimensions", + } + } +} + +/// Bounded aggregate diagnostics for rejected upstream bids. +#[derive(Debug, Default)] +pub(crate) struct ResponseAdmissionDiagnostics { + rejected_bid_count: u32, + reason_counts: BTreeMap, +} + +impl ResponseAdmissionDiagnostics { + /// Record one rejected bid without retaining upstream payload data. + pub(crate) fn record(&mut self, reason: BidRejectionReason) { + self.rejected_bid_count = self.rejected_bid_count.saturating_add(1); + let count = self.reason_counts.entry(reason).or_default(); + *count = count.saturating_add(1); + } + + /// Attach fixed-cardinality rejection counts to a provider response. + pub(crate) fn attach_to(self, response: &mut AuctionResponse) { + if self.rejected_bid_count == 0 { + return; + } + let reasons = self + .reason_counts + .into_iter() + .map(|(reason, count)| (reason.as_str().to_string(), json!(count))) + .collect::>(); + response.metadata.insert( + "response_admission".to_string(), + json!({ + "rejected_bid_count": self.rejected_bid_count, + "rejection_reasons": reasons, + }), + ); + } +} + +/// Parse an optional positive `OpenRTB` bid dimension. +pub(crate) fn parse_optional_bid_dimension( + value: &Value, + key: &str, +) -> Result, BidRejectionReason> { + let Some(raw) = value.get(key) else { + return Ok(None); + }; + raw.as_u64() + .and_then(|dimension| u32::try_from(dimension).ok()) + .filter(|dimension| *dimension > 0) + .map(Some) + .ok_or(BidRejectionReason::InvalidBid) +} + +/// Validate explicit dimensions or infer them from one routed banner format. +pub(crate) fn resolve_bid_dimensions( + input: &ProviderAuctionInput, + slot_id: &str, + width: Option, + height: Option, +) -> Result<(u32, u32), BidRejectionReason> { + let slot = input + .slots() + .iter() + .find(|slot| slot.slot().id == slot_id) + .ok_or(BidRejectionReason::UnrequestedImpression)?; + let dimensions = slot + .slot() + .formats + .iter() + .map(|format| (format.width, format.height)) + .collect::>(); + + if let (Some(width), Some(height)) = (width, height) { + return dimensions + .contains(&(width, height)) + .then_some((width, height)) + .ok_or(BidRejectionReason::DimensionMismatch); + } + + if dimensions.len() != 1 { + return Err(BidRejectionReason::AmbiguousDimensions); + } + let inferred = dimensions + .first() + .copied() + .expect("should have one routed banner format"); + if width.is_some_and(|width| width != inferred.0) + || height.is_some_and(|height| height != inferred.1) + { + return Err(BidRejectionReason::DimensionMismatch); + } + Ok(inferred) +} + +/// Result of request construction before transport. +#[derive(Debug)] +#[allow( + clippy::large_enum_variant, + reason = "Ready carries the full built request by design" +)] +pub(crate) enum OpenRtbBuildOutcome { + Ready(OpenRtbRequest), + NoImpressions, +} + +/// Explicit, deterministic signing input. No signer is loaded by this driver. +pub(crate) struct RequestFinalization<'a> { + pub(crate) signer: Option<&'a RequestSigner>, + pub(crate) signing_params: SigningParams, +} + +/// Build one provider request from its immutable routed input. +/// +/// # Errors +/// +/// Returns an auction error when static/profile extensions cannot be merged or +/// the supplied signing input does not bind the already-fixed request ID. +pub(crate) fn build_request( + input: &ProviderAuctionInput, + routed: &RoutedAuction, + provider: &ProviderPlan, + effective_timeout_ms: u32, + finalization: &RequestFinalization<'_>, +) -> Result> { + let policy = ProfilePolicy::from(&provider.profile); + let mut request = build_common_request(input, routed, policy, effective_timeout_ms); + if request.imp.is_empty() { + return Ok(OpenRtbBuildOutcome::NoImpressions); + } + policy.augment_request(&mut request, input, routed)?; + finalize_request(&mut request, policy, finalization)?; + Ok(OpenRtbBuildOutcome::Ready(request)) +} + +#[derive(Clone, Copy)] +enum ProfilePolicy<'a> { + Standard(&'a StandardProfilePlan), + Prebid(&'a PrebidProfilePlan), + Aps(&'a ApsProfilePlan), +} + +impl<'a> From<&'a CompiledOpenRtbProfile> for ProfilePolicy<'a> { + fn from(profile: &'a CompiledOpenRtbProfile) -> Self { + match profile { + CompiledOpenRtbProfile::Standard(plan) => Self::Standard(plan), + CompiledOpenRtbProfile::PrebidServer(plan) => Self::Prebid(plan), + CompiledOpenRtbProfile::Aps(plan) => Self::Aps(plan), + } + } +} + +impl ProfilePolicy<'_> { + fn augment_request( + self, + request: &mut OpenRtbRequest, + input: &ProviderAuctionInput, + routed: &RoutedAuction, + ) -> Result<(), Report> { + match self { + Self::Standard(plan) => apply_standard(request, plan), + Self::Prebid(plan) => apply_prebid(request, input, routed, plan), + Self::Aps(plan) => apply_aps(request, plan), + } + } + + fn keeps_pbs_identity_when_unsigned(self) -> bool { + matches!(self, Self::Prebid(_)) + } +} + +fn build_common_request( + input: &ProviderAuctionInput, + routed: &RoutedAuction, + policy: ProfilePolicy<'_>, + effective_timeout_ms: u32, +) -> OpenRtbRequest { + let common = input.common_request(); + let imps = input + .slots() + .iter() + .filter_map(|slot| build_imp(slot, policy)) + .collect(); + let site_domain = match policy { + ProfilePolicy::Aps(plan) => plan + .inventory_domain + .clone() + .unwrap_or_else(|| common.publisher.domain.clone()), + _ => common.publisher.domain.clone(), + }; + let page = match policy { + ProfilePolicy::Aps(plan) => { + aps_inventory_page(plan, common.publisher.page_url.as_deref(), &site_domain) + } + ProfilePolicy::Prebid(plan) => common.publisher.page_url.as_deref().map(|page| { + plan.debug_query_params.as_deref().map_or_else( + || page.to_string(), + |query| append_query_fragment(page, query), + ) + }), + ProfilePolicy::Standard(_) => common.publisher.page_url.clone(), + }; + let consent = common.user.consent.as_ref(); + let body_consent = match policy { + ProfilePolicy::Prebid(plan) => consent.filter(|value| { + plan.consent_forwarding.includes_body_consent() + || !matches!(value.source, ConsentSource::Cookie) + }), + _ => consent, + }; + let raw_tc = body_consent.and_then(|value| value.raw_tc_string.clone()); + let user = Some(User { + id: common.user.id.clone(), + consent: raw_tc.clone(), + ext: UserExt { + consent: raw_tc, + consented_providers_settings: matches!(policy, ProfilePolicy::Prebid(_)) + .then(|| { + body_consent + .and_then(|value| value.raw_ac_string.clone()) + .map(|consented_providers| ConsentedProvidersSettings { + consented_providers: Some(consented_providers), + }) + }) + .flatten(), + eids: common.user.eids.clone(), + } + .to_ext(), + ..Default::default() + }); + let language = normalized_language(routed.prebid_transport_headers(), policy); + let device = common + .device + .as_ref() + .map(|device| Device { + ua: device.user_agent.clone(), + ip: device.ip.clone(), + geo: device.geo.as_ref().map(|geo| Geo { + country: Some(geo.country.clone()), + region: geo.region.clone(), + city: Some(geo.city.clone()), + lat: matches!(policy, ProfilePolicy::Prebid(_)).then_some(geo.latitude), + lon: matches!(policy, ProfilePolicy::Prebid(_)).then_some(geo.longitude), + metro: (geo.metro_code > 0).then(|| geo.metro_code.to_string()), + r#type: Some(2), + ..Default::default() + }), + dnt: routed.dnt(), + language: language.clone(), + ..Default::default() + }) + .or_else(|| { + (routed.dnt().is_some() || language.is_some()).then_some(Device { + dnt: routed.dnt(), + language, + ..Default::default() + }) + }); + + OpenRtbRequest { + id: Some(common.id.clone()), + imp: imps, + site: Some(Site { + domain: Some(site_domain.clone()), + page, + r#ref: matches!(policy, ProfilePolicy::Prebid(_)) + .then(|| header_string(routed.prebid_transport_headers().referer())) + .flatten(), + publisher: Some(Publisher { + domain: Some(site_domain), + ..Default::default() + }), + ..Default::default() + }), + user, + device, + regs: build_regs(body_consent, policy), + test: match policy { + ProfilePolicy::Prebid(plan) => plan.test_mode.then_some(true), + _ => None, + }, + tmax: to_openrtb_i32( + effective_timeout_ms, + "tmax", + "config-first provider request", + ), + cur: vec![DEFAULT_CURRENCY.to_string()], + ..Default::default() + } +} + +fn build_imp(slot: &ProviderSlotInput, policy: ProfilePolicy<'_>) -> Option { + let formats = slot + .slot() + .formats + .iter() + .filter_map(|format| { + Some(Format { + w: to_openrtb_i32(format.width, "format.w", "routed slot"), + h: to_openrtb_i32(format.height, "format.h", "routed slot"), + ..Default::default() + }) + .filter(|value| value.w.is_some() && value.h.is_some()) + }) + .collect::>(); + let first_width = formats.first()?.w; + let first_height = formats.first()?.h; + let aps_banner = matches!(policy, ProfilePolicy::Aps(_)); + Some(Imp { + id: Some(slot.slot().id.clone()), + banner: Some(Banner { + format: formats, + w: aps_banner.then_some(first_width).flatten(), + h: aps_banner.then_some(first_height).flatten(), + topframe: aps_banner.then_some(false), + ..Default::default() + }), + tagid: matches!(policy, ProfilePolicy::Prebid(_)).then(|| slot.slot().id.clone()), + bidfloor: slot.slot().floor_price, + bidfloorcur: slot + .slot() + .floor_price + .map(|_| DEFAULT_CURRENCY.to_string()), + secure: Some(true), + ..Default::default() + }) +} + +fn apply_standard( + request: &mut OpenRtbRequest, + plan: &StandardProfilePlan, +) -> Result<(), Report> { + request.ext = nonempty_map(plan.request_ext.as_object().clone()); + for imp in &mut request.imp { + imp.ext = nonempty_map(plan.imp_ext.as_object().clone()); + } + Ok(()) +} + +fn apply_prebid( + request: &mut OpenRtbRequest, + input: &ProviderAuctionInput, + _routed: &RoutedAuction, + plan: &PrebidProfilePlan, +) -> Result<(), Report> { + debug_assert_eq!( + request.imp.len(), + input.slots().len(), + "should keep one impression per routed slot" + ); + for (imp, slot) in request.imp.iter_mut().zip(input.slots()) { + let bidder = slot + .bidder_params() + .iter() + .filter_map(|(bidder, params)| { + let mut params = params.clone(); + plan.override_engine + .apply_routed(bidder.as_str(), slot.prebid_zone(), &mut params); + params + .as_object() + .is_some_and(|params| !params.is_empty()) + .then(|| (bidder.as_str().to_string(), params)) + }) + .collect::>(); + let mut prebid = Map::new(); + if !bidder.is_empty() { + prebid.insert("bidder".to_string(), Value::Object(bidder)); + } else if slot.has_trusted_stored_request() || !slot.bidder_params().is_empty() { + prebid.insert("storedrequest".to_string(), json!({"id": slot.slot().id})); + } + imp.ext = Some(Map::from_iter([( + "prebid".to_string(), + Value::Object(prebid), + )])); + } + let mut prebid_request = Map::new(); + if plan.debug { + prebid_request.insert("debug".to_string(), Value::Bool(true)); + prebid_request.insert("returnallbidstatus".to_string(), Value::Bool(true)); + } + request.ext = Some(Map::from_iter([( + "prebid".to_string(), + Value::Object(prebid_request), + )])); + Ok(()) +} + +fn apply_aps( + request: &mut OpenRtbRequest, + plan: &ApsProfilePlan, +) -> Result<(), Report> { + request.ext = Some(Map::from_iter([ + ( + "account".to_string(), + Value::String(plan.account_id.clone()), + ), + ( + "sdk".to_string(), + json!({"source": APS_SDK_SOURCE, "version": APS_SDK_VERSION}), + ), + ])); + Ok(()) +} + +fn finalize_request( + request: &mut OpenRtbRequest, + policy: ProfilePolicy<'_>, + finalization: &RequestFinalization<'_>, +) -> Result<(), Report> { + let request_id = request.id.as_deref().ok_or_else(|| { + Report::new(TrustedServerError::Auction { + message: "OpenRTB request ID must be fixed before signing".to_string(), + }) + })?; + if request_id != finalization.signing_params.request_id { + return Err(Report::new(TrustedServerError::Auction { + message: "OpenRTB signing params do not bind the fixed request ID".to_string(), + })); + } + let trusted_server = if let Some(signer) = finalization.signer { + let signature = signer.sign_request(&finalization.signing_params)?; + Some(TrustedServerExt { + version: Some(SIGNING_VERSION.to_string()), + signature: Some(signature), + kid: Some(signer.kid.clone()), + request_host: Some(finalization.signing_params.request_host.clone()), + request_scheme: Some(finalization.signing_params.request_scheme.clone()), + ts: Some(finalization.signing_params.timestamp), + }) + } else if policy.keeps_pbs_identity_when_unsigned() { + Some(TrustedServerExt { + version: None, + signature: None, + kid: None, + request_host: Some(finalization.signing_params.request_host.clone()), + request_scheme: Some(finalization.signing_params.request_scheme.clone()), + ts: None, + }) + } else { + None + }; + if let Some(trusted_server) = trusted_server { + let ext = request.ext.get_or_insert_with(Map::new); + let serialized = serde_json::to_value(trusted_server).map_err(|error| { + Report::new(TrustedServerError::Auction { + message: format!("Failed to serialize Trusted Server extension: {error}"), + }) + })?; + ext.insert("trusted_server".to_string(), serialized); + } + Ok(()) +} + +fn build_regs( + consent: Option<&crate::consent::ConsentContext>, + policy: ProfilePolicy<'_>, +) -> Option { + let consent = consent?; + if matches!(policy, ProfilePolicy::Aps(_)) { + // Preserve APS exactly: any admitted context produces regs and GDPR is + // derived only from the applicability bit, without jurisdiction rules. + let ext = RegsExt { + gdpr: Some(u8::from(consent.gdpr_applies)), + us_privacy: consent.raw_us_privacy.clone(), + gpp: consent.raw_gpp_string.clone(), + gpp_sid: consent.gpp_section_ids.clone(), + }; + return Some(Regs { + coppa: None, + gdpr: Some(consent.gdpr_applies), + us_privacy: ext.us_privacy.clone(), + gpp: ext.gpp.clone(), + gpp_sid: ext + .gpp_sid + .as_ref() + .map(|ids| ids.iter().copied().map(i32::from).collect()) + .unwrap_or_default(), + ext: ext.to_ext(), + }); + } + + // Standard deliberately shares PBS's conservative consent baseline. Keep + // the legacy PBS empty-context and jurisdiction behavior byte-for-byte. + let has_data = consent.gdpr_applies + || consent.raw_us_privacy.is_some() + || consent.raw_gpp_string.is_some() + || consent.gpp_section_ids.is_some() + || consent.gpc; + if !has_data { + return None; + } + let gdpr = if consent.gdpr_applies + || matches!( + consent.jurisdiction, + crate::consent::jurisdiction::Jurisdiction::Gdpr + ) { + Some(true) + } else if matches!( + consent.jurisdiction, + crate::consent::jurisdiction::Jurisdiction::Unknown + ) { + None + } else { + Some(false) + }; + let us_privacy = consent.raw_us_privacy.clone(); + let gpp = consent.raw_gpp_string.clone(); + let gpp_sid = consent.gpp_section_ids.clone(); + let ext = RegsExt { + gdpr: gdpr.map(u8::from), + us_privacy: us_privacy.clone(), + gpp: gpp.clone(), + gpp_sid: gpp_sid.clone(), + }; + Some(Regs { + coppa: None, + gdpr, + us_privacy, + gpp, + gpp_sid: gpp_sid + .map(|ids| ids.into_iter().map(i32::from).collect()) + .unwrap_or_default(), + ext: ext.to_ext(), + }) +} + +fn normalized_language( + headers: &PrebidTransportHeaders, + policy: ProfilePolicy<'_>, +) -> Option { + let value = header_string(headers.accept_language()) + .and_then(|value| value.split(',').next().map(str::to_string)) + .and_then(|value| value.split(';').next().map(str::to_string)) + .and_then(|value| value.split('-').next().map(str::to_string)) + .map(|value| value.trim().to_string()) + .filter(|value| !value.is_empty())?; + match policy { + ProfilePolicy::Prebid(_) => Some(value), + ProfilePolicy::Aps(_) | ProfilePolicy::Standard(_) => { + (value.len() <= MAX_CONSERVATIVE_LANGUAGE_BYTES).then_some(value) + } + } +} + +fn header_string(value: Option<&http::HeaderValue>) -> Option { + value + .and_then(|value| value.to_str().ok()) + .map(str::to_string) +} + +fn aps_inventory_page( + plan: &ApsProfilePlan, + publisher_page: Option<&str>, + domain: &str, +) -> Option { + let fallback = publisher_page + .and_then(valid_aps_page_url) + .unwrap_or_else(|| format!("https://{domain}")); + let Some(origin) = plan.inventory_page_origin.as_deref() else { + return Some(fallback); + }; + let (Ok(mut canonical), Ok(current)) = (Url::parse(origin), Url::parse(&fallback)) else { + return Some(fallback); + }; + canonical.set_path(current.path()); + canonical.set_query(current.query()); + canonical.set_fragment(None); + Some(canonical.to_string()) +} + +fn append_query_fragment(url: &str, query: &str) -> String { + if query.is_empty() || url.contains(query) { + return url.to_string(); + } + let separator = if url.contains('?') { '&' } else { '?' }; + format!("{url}{separator}{query}") +} + +fn valid_aps_page_url(value: &str) -> Option { + const MAX_APS_PAGE_URL_BYTES: usize = 8192; + + if value.len() > MAX_APS_PAGE_URL_BYTES { + return None; + } + let parsed = Url::parse(value).ok()?; + (matches!(parsed.scheme(), "http" | "https") + && parsed.host_str().is_some() + && parsed.username().is_empty() + && parsed.password().is_none()) + .then(|| parsed.to_string()) +} + +fn nonempty_map(value: Map) -> Option> { + (!value.is_empty()).then_some(value) +} + +/// Suppress notification URLs using exact returned-seat identity. +pub(crate) fn apply_notification_policy(bids: &mut [Bid], policy: &NotificationPolicy) { + for bid in bids { + let suppress = policy.suppress_all + || bid + .returned_seat + .as_ref() + .is_some_and(|seat| policy.suppress_seats.contains(seat)); + if suppress { + bid.nurl = None; + bid.burl = None; + } + } +} + +/// Parse ordinary `OpenRTB` bids independently. Response ID is informational. +pub(crate) fn extract_standard_response( + provider_id: &str, + input: &ProviderAuctionInput, + value: &Value, + response_time_ms: u64, +) -> AuctionResponse { + let Some(response) = value.as_object() else { + return AuctionResponse::error(provider_id, response_time_ms) + .with_metadata("error_type", json!("parse_response")); + }; + match response.get("cur") { + None => {} + Some(Value::String(currency)) if currency.eq_ignore_ascii_case(DEFAULT_CURRENCY) => {} + Some(Value::String(currency)) => { + return AuctionResponse::no_bid(provider_id, response_time_ms) + .with_metadata("unsupported_currency", json!(currency)); + } + Some(_) => { + return AuctionResponse::error(provider_id, response_time_ms) + .with_metadata("error_type", json!("parse_response")); + } + } + let mut diagnostics = ResponseAdmissionDiagnostics::default(); + let mut bids = Vec::new(); + for seatbid in response + .get("seatbid") + .and_then(Value::as_array) + .into_iter() + .flatten() + { + let returned_seat = seatbid + .get("seat") + .and_then(Value::as_str) + .filter(|seat| !seat.is_empty()); + let Some(entries) = seatbid.get("bid").and_then(Value::as_array) else { + continue; + }; + for value in entries { + match extract_standard_bid(value, returned_seat, input) { + Ok(bid) => bids.push(bid), + Err(reason) => diagnostics.record(reason), + } + } + } + let mut parsed = if bids.is_empty() { + AuctionResponse::no_bid(provider_id, response_time_ms) + } else { + AuctionResponse::success(provider_id, bids, response_time_ms) + }; + diagnostics.attach_to(&mut parsed); + parsed +} + +fn extract_standard_bid( + value: &Value, + returned_seat: Option<&str>, + input: &ProviderAuctionInput, +) -> Result { + let slot_id = value + .get("impid") + .and_then(Value::as_str) + .filter(|slot_id| !slot_id.is_empty()) + .ok_or(BidRejectionReason::InvalidBid)? + .to_string(); + let width = parse_optional_bid_dimension(value, "w")?; + let height = parse_optional_bid_dimension(value, "h")?; + let (width, height) = resolve_bid_dimensions(input, &slot_id, width, height)?; + let price = value + .get("price") + .and_then(Value::as_f64) + .filter(|price| price.is_finite() && *price >= 0.0) + .ok_or(BidRejectionReason::InvalidBid)?; + let creative = value + .get("adm") + .and_then(Value::as_str) + .filter(|creative| !creative.is_empty()) + .map(str::to_string) + .ok_or(BidRejectionReason::InvalidBid)?; + Ok(Bid { + slot_id, + price: Some(price), + currency: DEFAULT_CURRENCY.to_string(), + creative: Some(creative), + adomain: value + .get("adomain") + .and_then(Value::as_array) + .map(|domains| { + domains + .iter() + .filter_map(Value::as_str) + .map(str::to_string) + .collect() + }), + bidder: returned_seat.unwrap_or("unknown").to_string(), + returned_seat: returned_seat.map(str::to_string), + width, + height, + nurl: value + .get("nurl") + .and_then(Value::as_str) + .map(str::to_string), + burl: value + .get("burl") + .and_then(Value::as_str) + .map(str::to_string), + bid_id: value + .get("id") + .and_then(Value::as_str) + .filter(|id| !id.is_empty()) + .map(str::to_string), + ad_id: value + .get("adid") + .and_then(Value::as_str) + .map(str::to_string), + creative_id: value + .get("crid") + .and_then(Value::as_str) + .map(str::to_string), + renderer: None, + cache_id: None, + cache_host: None, + cache_path: None, + metadata: HashMap::new(), + }) +} + +/// Count bidder parameter objects a profile did not consume. +#[must_use] +pub(crate) fn unused_bidder_params_count( + profile: &CompiledOpenRtbProfile, + input: &ProviderAuctionInput, +) -> u32 { + if profile.is_prebid_server() { + return 0; + } + ignored_bidder_params_count(input) +} + +/// Count routed bidder params for a profile known to ignore them. +#[must_use] +pub(crate) fn ignored_bidder_params_count(input: &ProviderAuctionInput) -> u32 { + saturating_bidder_param_counts(input.slots().iter().map(|slot| slot.bidder_params().len())) +} + +fn saturating_bidder_param_counts(counts: impl IntoIterator) -> u32 { + counts.into_iter().fold(0_u32, |count, slot_count| { + count.saturating_add(u32::try_from(slot_count).unwrap_or(u32::MAX)) + }) +} + +#[cfg(test)] +mod routing_metadata_tests { + use std::collections::BTreeMap; + use std::str::FromStr as _; + + use serde_json::json; + + use super::{saturating_bidder_param_counts, unused_bidder_params_count}; + use crate::auction::plan::{ + AuctionPlan, AuctionPlanConfig, BidderId, BidderRouteConfig, NotificationConfig, + ProviderConfig, ProviderId, RoutingMode, + }; + use crate::auction::routing::route_auction; + use crate::auction::test_support::canonical_parity_auction_request; + + #[test] + fn unused_bidder_param_count_saturates_across_slots_and_large_values() { + assert_eq!(saturating_bidder_param_counts([1, 2, 3]), 6); + assert_eq!( + saturating_bidder_param_counts([usize::try_from(u32::MAX).unwrap_or(usize::MAX), 1]), + u32::MAX + ); + assert_eq!(saturating_bidder_param_counts([usize::MAX]), u32::MAX); + } + + #[test] + fn unused_bidder_param_count_is_profile_aware() { + for (profile, profile_config, expected) in [ + ("prebid-server", json!({}), 0), + ("standard", json!({}), 1), + ("aps", json!({"account_id":"example-account"}), 1), + ] { + let provider_id = + ProviderId::from_str("fictional-provider").expect("should parse provider ID"); + let plan = AuctionPlan::compile(AuctionPlanConfig { + timeout_ms: 1_000, + providers: BTreeMap::from([( + provider_id.clone(), + ProviderConfig { + protocol: "openrtb-2.6".to_string(), + profile: profile.to_string(), + endpoint: if profile == "aps" { + "https://aps.example/e/pb/bid".to_string() + } else { + "https://provider.example/openrtb".to_string() + }, + timeout_ms: None, + routing: RoutingMode::Explicit, + notifications: NotificationConfig::default(), + profile_config, + }, + )]), + bidders: BTreeMap::from([( + BidderId::from_str("exampleBidder").expect("should parse bidder ID"), + BidderRouteConfig { + provider: provider_id, + }, + )]), + mediator: None, + request_signing: None, + }) + .expect("should compile profile plan"); + let inbound = http::Request::new(edgezero_core::body::Body::empty()); + let routed = route_auction(canonical_parity_auction_request(), &inbound, &plan, None); + + assert_eq!( + unused_bidder_params_count(&plan.providers()[0].profile, &routed.inputs()[0]), + expected, + "{profile} should report only bidder params it ignores" + ); + } + } +} + +#[cfg(test)] +mod test_executor; +#[cfg(test)] +mod tests; diff --git a/crates/trusted-server-core/src/auction/openrtb/test_executor.rs b/crates/trusted-server-core/src/auction/openrtb/test_executor.rs new file mode 100644 index 000000000..d322102e6 --- /dev/null +++ b/crates/trusted-server-core/src/auction/openrtb/test_executor.rs @@ -0,0 +1,107 @@ +//! Fictional standard-profile executor compiled only for automated tests. + +use edgezero_core::body::Body as EdgeBody; +use error_stack::{Report, ResultExt as _}; +use http::{Method, Request, StatusCode, header}; +use serde_json::{Value, json}; + +use super::{apply_notification_policy, extract_standard_response, unused_bidder_params_count}; +use crate::auction::plan::ProviderPlan; +use crate::auction::routing::ProviderAuctionInput; +use crate::auction::types::AuctionResponse; +use crate::error::TrustedServerError; +use crate::platform::{PlatformBackend, PlatformHttpClient, PlatformHttpRequest}; + +const MAX_STANDARD_RESPONSE_BYTES: usize = 1024 * 1024; + +/// Execute one fictional standard-profile request through a supplied test client. +/// +/// The HTTP client receives exactly one request. Redirect statuses are +/// classified as the original provider error and never followed. +pub(super) async fn execute_standard_fixture( + provider: &ProviderPlan, + input: &ProviderAuctionInput, + request: &trusted_server_openrtb::BidRequest, + backend: &dyn PlatformBackend, + http_client: &dyn PlatformHttpClient, +) -> Result> { + let spec = provider.backend_spec(); + let predicted_name = + backend + .predict_name(&spec) + .change_context(TrustedServerError::Auction { + message: "Failed to predict fictional standard backend".to_string(), + })?; + let backend_name = backend + .ensure(&spec) + .change_context(TrustedServerError::Auction { + message: "Failed to ensure fictional standard backend".to_string(), + })?; + if backend_name != predicted_name { + return Err(Report::new(TrustedServerError::Auction { + message: "Fictional standard backend ensure did not match prediction".to_string(), + })); + } + let body = serde_json::to_vec(request).change_context(TrustedServerError::Auction { + message: "Failed to serialize fictional standard request".to_string(), + })?; + let outbound = Request::builder() + .method(Method::POST) + .uri(provider.endpoint.as_str()) + .header(header::CONTENT_TYPE, "application/json") + .header(header::ACCEPT, "application/json") + .body(EdgeBody::from(body)) + .change_context(TrustedServerError::Auction { + message: "Failed to build fictional standard request".to_string(), + })?; + let response = http_client + .send(PlatformHttpRequest::new(outbound, backend_name)) + .await + .change_context(TrustedServerError::Auction { + message: "Fictional standard transport failed".to_string(), + })? + .response; + let status = response.status(); + if status == StatusCode::NO_CONTENT { + return Ok( + AuctionResponse::no_bid(provider.id.as_str(), 0).with_metadata( + "routing", + json!({"unused_bidder_params_count": unused_bidder_params_count(&provider.profile, input)}), + ), + ); + } + if !status.is_success() { + return Ok(AuctionResponse::error(provider.id.as_str(), 0) + .with_metadata("error_type", json!("http_status")) + .with_metadata("http_status", json!(status.as_u16())) + .with_metadata( + "routing", + json!({"unused_bidder_params_count": unused_bidder_params_count(&provider.profile, input)}), + )); + } + let body = response + .into_body() + .into_bytes_bounded(MAX_STANDARD_RESPONSE_BYTES) + .await + .change_context(TrustedServerError::Auction { + message: "Fictional standard response exceeded its limit".to_string(), + })?; + let value: Value = match serde_json::from_slice(&body) { + Ok(value) => value, + Err(_) => { + return Ok(AuctionResponse::error(provider.id.as_str(), 0) + .with_metadata("error_type", json!("parse_response")) + .with_metadata( + "routing", + json!({"unused_bidder_params_count": unused_bidder_params_count(&provider.profile, input)}), + )); + } + }; + let mut parsed = extract_standard_response(provider.id.as_str(), input, &value, 0); + apply_notification_policy(&mut parsed.bids, &provider.notifications); + parsed.metadata.insert( + "routing".to_string(), + json!({"unused_bidder_params_count": unused_bidder_params_count(&provider.profile, input)}), + ); + Ok(parsed) +} diff --git a/crates/trusted-server-core/src/auction/openrtb/tests.rs b/crates/trusted-server-core/src/auction/openrtb/tests.rs new file mode 100644 index 000000000..83ed9e77e --- /dev/null +++ b/crates/trusted-server-core/src/auction/openrtb/tests.rs @@ -0,0 +1,1243 @@ +use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; +use std::str::FromStr as _; +use std::sync::Arc; + +use base64::Engine as _; +use edgezero_core::body::Body as EdgeBody; +use http::{Request, header}; +use serde_json::{Value, json}; + +use super::test_executor::execute_standard_fixture; +use super::*; +use crate::auction::plan::{ + AuctionPlan, AuctionPlanConfig, BidderId, BidderRouteConfig, NotificationConfig, + ProviderConfig, ProviderId, RoutingMode, +}; +use crate::auction::provider::{GenericOpenRtbProvider, ProviderRequestOutcome}; +use crate::auction::routing::route_auction; +use crate::auction::test_support::canonical_parity_auction_request; +use crate::auction::types::{AdFormat, AdSlot, BidStatus, MediaType}; +use crate::consent::jurisdiction::Jurisdiction; +use crate::consent::{ConsentContext, ConsentSource}; +use crate::platform::PlatformHttpClient; +use crate::platform::test_support::{ + HashMapConfigStore, HashMapSecretStore, NoopHttpClient, StubBackend, StubHttpClient, + build_services_with_backend_and_http_client, build_services_with_config_secret_and_http_client, +}; +use crate::request_signing::RequestSigner; + +fn config(profile: &str, profile_config: Value) -> AuctionPlanConfig { + config_with_endpoint( + profile, + profile_config, + "https://exchange.example.test/openrtb", + ) +} + +fn config_with_endpoint(profile: &str, profile_config: Value, endpoint: &str) -> AuctionPlanConfig { + let provider_id = ProviderId::from_str("fictional-provider").expect("should parse provider ID"); + let prebid_server = profile == "prebid-server"; + AuctionPlanConfig { + timeout_ms: 321, + providers: BTreeMap::from([( + provider_id.clone(), + ProviderConfig { + protocol: "openrtb-2.6".to_string(), + profile: profile.to_string(), + endpoint: endpoint.to_string(), + timeout_ms: Some(321), + routing: if prebid_server { + RoutingMode::Explicit + } else { + RoutingMode::AllEligible + }, + notifications: NotificationConfig::default(), + profile_config, + }, + )]), + bidders: if prebid_server { + BTreeMap::from([( + BidderId::from_str("exampleBidder").expect("should parse bidder ID"), + BidderRouteConfig { + provider: provider_id, + }, + )]) + } else { + BTreeMap::new() + }, + mediator: None, + request_signing: None, + } +} + +fn routed(profile: &str, profile_config: Value) -> (AuctionPlan, RoutedAuction) { + let plan = AuctionPlan::compile(config(profile, profile_config)).expect("should compile plan"); + let inbound = Request::builder() + .uri("https://publisher.example/auction") + .header( + header::REFERER, + "https://referrer.example/story?fictional=1", + ) + .header(header::ACCEPT_LANGUAGE, "en-US,en;q=0.9") + .header("dnt", "1") + .body(EdgeBody::empty()) + .expect("should build inbound request"); + let routed = route_auction(canonical_parity_auction_request(), &inbound, &plan, None); + (plan, routed) +} + +fn routed_with_request( + profile: &str, + profile_config: Value, + request: crate::auction::types::AuctionRequest, + accept_language: Option<&str>, +) -> (AuctionPlan, RoutedAuction) { + let plan = AuctionPlan::compile(config(profile, profile_config)).expect("should compile plan"); + let mut builder = Request::builder().uri("https://publisher.example/auction"); + if let Some(language) = accept_language { + builder = builder.header(header::ACCEPT_LANGUAGE, language); + } + let inbound = builder + .body(EdgeBody::empty()) + .expect("should build inbound request"); + let routed = route_auction(request, &inbound, &plan, None); + (plan, routed) +} + +fn build_with_request( + profile: &str, + profile_config: Value, + request: crate::auction::types::AuctionRequest, + accept_language: Option<&str>, +) -> OpenRtbRequest { + let (plan, routed) = routed_with_request(profile, profile_config, request, accept_language); + match build_request( + &routed.inputs()[0], + &routed, + &plan.providers()[0], + 321, + &finalization(None), + ) + .expect("should build request") + { + OpenRtbBuildOutcome::Ready(request) => request, + OpenRtbBuildOutcome::NoImpressions => panic!("should retain impression"), + } +} + +fn finalization<'a>(signer: Option<&'a RequestSigner>) -> RequestFinalization<'a> { + RequestFinalization { + signer, + signing_params: SigningParams { + request_id: "fictional-auction".to_string(), + request_host: "publisher.example".to_string(), + request_scheme: "https".to_string(), + timestamp: 1_706_900_000, + }, + } +} + +fn build(profile: &str, profile_config: Value, signer: Option<&RequestSigner>) -> OpenRtbRequest { + let (plan, routed) = routed(profile, profile_config); + match build_request( + &routed.inputs()[0], + &routed, + &plan.providers()[0], + 321, + &finalization(signer), + ) + .expect("should build request") + { + OpenRtbBuildOutcome::Ready(request) => request, + OpenRtbBuildOutcome::NoImpressions => panic!("should retain impression"), + } +} + +fn deterministic_signer() -> RequestSigner { + let mut config_data = HashMap::new(); + config_data.insert("current-kid".to_string(), "fictional-kid".to_string()); + let mut secret_data = HashMap::new(); + secret_data.insert( + "fictional-kid".to_string(), + base64::engine::general_purpose::STANDARD + .encode([7_u8; 32]) + .into_bytes(), + ); + let services = build_services_with_config_secret_and_http_client( + HashMapConfigStore::new(config_data), + HashMapSecretStore::new(secret_data), + Arc::new(NoopHttpClient), + ); + RequestSigner::from_services(&services).expect("should load deterministic signer") +} + +#[test] +fn consent_matrix_preserves_pbs_standard_and_aps_policies() { + let cases = [ + ("empty", ConsentContext::default()), + ( + "gdpr", + ConsentContext { + gdpr_applies: true, + raw_tc_string: Some("tc-string".to_string()), + jurisdiction: Jurisdiction::Gdpr, + ..Default::default() + }, + ), + ( + "unknown-gpc", + ConsentContext { + gpc: true, + jurisdiction: Jurisdiction::Unknown, + ..Default::default() + }, + ), + ( + "nonregulated-gpc", + ConsentContext { + gpc: true, + jurisdiction: Jurisdiction::NonRegulated, + ..Default::default() + }, + ), + ( + "usp-gpp", + ConsentContext { + raw_us_privacy: Some("1YNN".to_string()), + raw_gpp_string: Some("gpp-string".to_string()), + gpp_section_ids: Some(vec![7, 8]), + jurisdiction: Jurisdiction::NonRegulated, + ..Default::default() + }, + ), + ]; + for (name, consent) in cases { + for profile in ["standard", "prebid-server", "aps"] { + let mut canonical = canonical_parity_auction_request(); + canonical.user.consent = Some(consent.clone()); + let config = if profile == "aps" { + json!({"account_id": "example-account-id"}) + } else { + json!({}) + }; + let value = serde_json::to_value(build_with_request(profile, config, canonical, None)) + .expect("should serialize request"); + let regs = value.get("regs"); + if profile == "aps" { + let regs = regs.expect("APS should preserve empty admitted context"); + assert_eq!( + regs["gdpr"], + json!(u8::from(consent.gdpr_applies)), + "{name}" + ); + } else if name == "empty" { + assert!(regs.is_none(), "{profile} should omit empty regs"); + } else { + let regs = regs.expect("should emit actionable regs"); + let expected_gdpr = match consent.jurisdiction { + Jurisdiction::Gdpr => Some(true), + Jurisdiction::Unknown if !consent.gdpr_applies => None, + _ => Some(consent.gdpr_applies), + }; + assert_eq!( + regs.get("gdpr"), + expected_gdpr.map(|value| json!(u8::from(value))).as_ref(), + "{name} {profile}" + ); + } + let serialized = value.to_string(); + assert!( + !serialized.contains("1YYY"), + "must never synthesize USP from GPC" + ); + if name == "usp-gpp" { + let regs = regs.expect("should have explicit fields"); + assert_eq!(regs["us_privacy"], "1YNN"); + assert_eq!(regs["gpp"], "gpp-string"); + assert_eq!(regs["gpp_sid"], json!([7, 8])); + assert_eq!(regs["ext"]["us_privacy"], "1YNN"); + assert_eq!(regs["ext"]["gpp"], "gpp-string"); + assert_eq!(regs["ext"]["gpp_sid"], json!([7, 8])); + } + } + } +} + +#[test] +fn pbs_body_consent_respects_source_and_forwarding_mode() { + for (mode, source, expected) in [ + ("cookies_only", ConsentSource::Cookie, false), + ("cookies_only", ConsentSource::KvStore, true), + ("cookies_only", ConsentSource::PolicyDefault, true), + ("openrtb_only", ConsentSource::Cookie, true), + ("both", ConsentSource::Cookie, true), + ] { + let mut canonical = canonical_parity_auction_request(); + canonical + .user + .consent + .as_mut() + .expect("should have consent context") + .source = source; + let value = serde_json::to_value(build_with_request( + "prebid-server", + json!({"consent_forwarding": mode}), + canonical, + None, + )) + .expect("should serialize request"); + assert_eq!( + value["user"].get("consent").is_some(), + expected, + "{mode:?} {source:?}" + ); + assert_eq!(value.get("regs").is_some(), expected, "{mode:?} {source:?}"); + } +} + +#[test] +fn language_limits_are_profile_specific() { + let language = "abcdefghijk"; + for (profile, expected) in [ + ("prebid-server", Some(language)), + ("aps", None), + ("standard", None), + ] { + let config = if profile == "aps" { + json!({"account_id": "example-account-id"}) + } else { + json!({}) + }; + let request = build_with_request( + profile, + config, + canonical_parity_auction_request(), + Some(language), + ); + assert_eq!( + request.device.and_then(|device| device.language).as_deref(), + expected + ); + } + for profile in ["prebid-server", "aps", "standard"] { + let config = if profile == "aps" { + json!({"account_id": "example-account-id"}) + } else { + json!({}) + }; + let request = build_with_request( + profile, + config, + canonical_parity_auction_request(), + Some("en-US,en;q=0.9"), + ); + assert_eq!( + request.device.and_then(|device| device.language).as_deref(), + Some("en") + ); + } +} + +#[test] +fn pbs_debug_query_fragment_preserves_exact_legacy_configured_semantics() { + for (page, fragment, expected) in [ + ( + "https://publisher.example/article", + "pbjs_debug=true", + "https://publisher.example/article?pbjs_debug=true", + ), + ( + "https://publisher.example/article?existing=1", + "pbjs_debug=true", + "https://publisher.example/article?existing=1&pbjs_debug=true", + ), + ( + "https://publisher.example/article", + "?pbjs_debug=true", + "https://publisher.example/article??pbjs_debug=true", + ), + ( + "https://publisher.example/article?pbjs_debug=true", + "pbjs_debug=true", + "https://publisher.example/article?pbjs_debug=true", + ), + ( + "https://publisher.example/article", + "", + "https://publisher.example/article", + ), + ] { + let mut request = canonical_parity_auction_request(); + request.publisher.page_url = Some(page.to_string()); + let built = build_with_request( + "prebid-server", + json!({"debug_query_params": fragment}), + request, + None, + ); + assert_eq!( + built.site.and_then(|site| site.page).as_deref(), + Some(expected), + "should preserve exact legacy query fragment semantics" + ); + } +} + +#[test] +fn pbs_routed_overrides_are_ordered_and_stored_request_is_trusted_fallback() { + let mut raw = config( + "prebid-server", + json!({ + "debug": true, + "test_mode": true, + "bid_param_overrides": {"exampleBidder": {"generic": 1, "shared": "generic"}}, + "bid_param_zone_overrides": {"exampleBidder": {"zone-a": {"zone": 2, "shared": "zone"}}}, + "bid_param_override_rules": [ + {"when":{"bidder":"exampleBidder"},"set":{"ordered":1,"shared":"rule-one"}}, + {"when":{"bidder":"exampleBidder","zone":"zone-a"},"set":{"ordered":2,"shared":"rule-two"}} + ] + }), + ); + raw.providers + .get_mut(&ProviderId::from_str("fictional-provider").expect("should parse provider")) + .expect("should find provider") + .routing = RoutingMode::Explicit; + raw.bidders.insert( + crate::auction::plan::BidderId::from_str("exampleBidder").expect("should parse bidder"), + BidderRouteConfig { + provider: ProviderId::from_str("fictional-provider").expect("should parse provider"), + }, + ); + let plan = AuctionPlan::compile(raw).expect("should compile PBS override plan"); + let mut request = canonical_parity_auction_request(); + request.slots[0].bidders = HashMap::from([( + "trustedServer".to_string(), + json!({"zone":"zone-a","bidderParams":{"exampleBidder":{"original":true,"shared":"original"}}}), + )]); + let inbound = Request::builder() + .uri("https://publisher.example/auction") + .body(EdgeBody::empty()) + .expect("should build inbound request"); + let routed = route_auction(request, &inbound, &plan, None); + let built = match build_request( + &routed.inputs()[0], + &routed, + &plan.providers()[0], + 321, + &finalization(None), + ) + .expect("should build request") + { + OpenRtbBuildOutcome::Ready(request) => request, + OpenRtbBuildOutcome::NoImpressions => panic!("should retain impression"), + }; + let value = serde_json::to_value(built).expect("should serialize request"); + assert_eq!( + value["imp"][0]["ext"]["prebid"]["bidder"]["exampleBidder"], + json!({"generic":1,"ordered":2,"original":true,"shared":"rule-two","zone":2}) + ); + assert_eq!(value["ext"]["prebid"]["debug"], true); + assert_eq!(value["ext"]["prebid"]["returnallbidstatus"], true); + assert_eq!(value["test"], 1); + + let mut empty_overridden = canonical_parity_auction_request(); + empty_overridden.slots[0].bidders = HashMap::from([( + "trustedServer".to_string(), + json!({"zone":"zone-a","bidderParams":{"exampleBidder":{}}}), + )]); + let routed = route_auction(empty_overridden, &inbound, &plan, None); + let built = match build_request( + &routed.inputs()[0], + &routed, + &plan.providers()[0], + 321, + &finalization(None), + ) + .expect("should build request after populating empty params") + { + OpenRtbBuildOutcome::Ready(request) => request, + OpenRtbBuildOutcome::NoImpressions => panic!("should retain overridden impression"), + }; + let value = serde_json::to_value(built).expect("should serialize overridden request"); + assert_eq!( + value["imp"][0]["ext"]["prebid"]["bidder"]["exampleBidder"], + json!({"generic":1,"ordered":2,"shared":"rule-two","zone":2}), + "should allow profile overrides to populate empty browser params" + ); + + let mut stored = canonical_parity_auction_request(); + stored.slots[0].bidders.clear(); + let routed = route_auction(stored, &inbound, &plan, None); + let built = match build_request( + &routed.inputs()[0], + &routed, + &plan.providers()[0], + 321, + &finalization(None), + ) + .expect("should build stored request") + { + OpenRtbBuildOutcome::Ready(request) => request, + OpenRtbBuildOutcome::NoImpressions => panic!("should retain impression"), + }; + let value = serde_json::to_value(built).expect("should serialize stored request"); + assert_eq!( + value["imp"][0]["ext"]["prebid"]["storedrequest"]["id"], + "fictional-slot" + ); +} + +#[test] +fn pbs_pairs_each_impression_with_its_routed_slot_params() { + let mut raw = config("prebid-server", json!({})); + raw.providers + .get_mut(&ProviderId::from_str("fictional-provider").expect("should parse provider")) + .expect("should find provider") + .routing = RoutingMode::Explicit; + raw.bidders.insert( + crate::auction::plan::BidderId::from_str("exampleBidder").expect("should parse bidder"), + BidderRouteConfig { + provider: ProviderId::from_str("fictional-provider").expect("should parse provider"), + }, + ); + let plan = AuctionPlan::compile(raw).expect("should compile PBS plan"); + let mut request = canonical_parity_auction_request(); + request.slots[0].id = "first-slot".to_string(); + request.slots[0].bidders = HashMap::from([( + "trustedServer".to_string(), + json!({"bidderParams":{"exampleBidder":{"placement":"first"}}}), + )]); + let mut second_slot = request.slots[0].clone(); + second_slot.id = "second-slot".to_string(); + second_slot.bidders = HashMap::from([( + "trustedServer".to_string(), + json!({"bidderParams":{"exampleBidder":{"placement":"second"}}}), + )]); + request.slots.push(second_slot); + let inbound = Request::builder() + .uri("https://publisher.example/auction") + .body(EdgeBody::empty()) + .expect("should build inbound request"); + let routed = route_auction(request, &inbound, &plan, None); + + let built = match build_request( + &routed.inputs()[0], + &routed, + &plan.providers()[0], + 321, + &finalization(None), + ) + .expect("should build request") + { + OpenRtbBuildOutcome::Ready(request) => request, + OpenRtbBuildOutcome::NoImpressions => panic!("should retain impressions"), + }; + let value = serde_json::to_value(built).expect("should serialize request"); + + assert_eq!(value["imp"][0]["id"], "first-slot"); + assert_eq!( + value["imp"][0]["ext"]["prebid"]["bidder"]["exampleBidder"], + json!({"placement":"first"}) + ); + assert_eq!(value["imp"][1]["id"], "second-slot"); + assert_eq!( + value["imp"][1]["ext"]["prebid"]["bidder"]["exampleBidder"], + json!({"placement":"second"}) + ); +} + +#[test] +fn pbs_empty_params_without_matching_override_fall_back_to_stored_request() { + let mut raw = config("prebid-server", json!({})); + raw.providers + .get_mut(&ProviderId::from_str("fictional-provider").expect("should parse provider")) + .expect("should find provider") + .routing = RoutingMode::Explicit; + raw.bidders.insert( + crate::auction::plan::BidderId::from_str("exampleBidder").expect("should parse bidder"), + BidderRouteConfig { + provider: ProviderId::from_str("fictional-provider").expect("should parse provider"), + }, + ); + let plan = AuctionPlan::compile(raw).expect("should compile PBS plan"); + let mut request = canonical_parity_auction_request(); + request.slots[0].bidders = HashMap::from([( + "trustedServer".to_string(), + json!({"bidderParams":{"exampleBidder":{}}}), + )]); + let inbound = Request::builder() + .uri("https://publisher.example/auction") + .body(EdgeBody::empty()) + .expect("should build inbound request"); + let routed = route_auction(request, &inbound, &plan, None); + let built = match build_request( + &routed.inputs()[0], + &routed, + &plan.providers()[0], + 321, + &finalization(None), + ) + .expect("should build stored request") + { + OpenRtbBuildOutcome::Ready(request) => request, + OpenRtbBuildOutcome::NoImpressions => panic!("should retain stored impression"), + }; + let value = serde_json::to_value(built).expect("should serialize stored request"); + assert_eq!( + value["imp"][0]["ext"]["prebid"]["storedrequest"]["id"], + "fictional-slot" + ); + assert!(value["imp"][0]["ext"]["prebid"].get("bidder").is_none()); +} + +#[test] +fn pbs_driver_exact_golden_preserves_profile_policy() { + let mut raw = config("prebid-server", json!({"consent_forwarding": "both"})); + raw.providers + .get_mut(&ProviderId::from_str("fictional-provider").expect("should parse provider")) + .expect("should find provider") + .routing = RoutingMode::Explicit; + raw.bidders.insert( + crate::auction::plan::BidderId::from_str("exampleBidder").expect("should parse bidder"), + BidderRouteConfig { + provider: ProviderId::from_str("fictional-provider").expect("should parse provider"), + }, + ); + let plan = AuctionPlan::compile(raw).expect("should compile PBS plan"); + let mut common = canonical_parity_auction_request(); + common.slots[0].bidders = HashMap::from([( + "exampleBidder".to_string(), + json!({"placement": "fictional-placement"}), + )]); + let inbound = Request::builder() + .uri("https://publisher.example/auction") + .header( + header::REFERER, + "https://referrer.example/story?fictional=1", + ) + .header(header::ACCEPT_LANGUAGE, "en-US,en;q=0.9") + .header("dnt", "1") + .body(EdgeBody::empty()) + .expect("should build inbound request"); + let routed = route_auction(common, &inbound, &plan, None); + let request = match build_request( + &routed.inputs()[0], + &routed, + &plan.providers()[0], + 321, + &finalization(None), + ) + .expect("should build PBS request") + { + OpenRtbBuildOutcome::Ready(request) => request, + OpenRtbBuildOutcome::NoImpressions => panic!("should retain impression"), + }; + assert_eq!( + serde_json::to_string(&request).expect("should serialize PBS driver request"), + r#"{"id":"fictional-auction","imp":[{"id":"fictional-slot","banner":{"format":[{"w":300,"h":250},{"w":728,"h":90}]},"tagid":"fictional-slot","bidfloor":1.0,"bidfloorcur":"USD","secure":1,"ext":{"prebid":{"bidder":{"exampleBidder":{"placement":"fictional-placement"}}}}}],"site":{"domain":"publisher.example","page":"https://publisher.example/article","ref":"https://referrer.example/story?fictional=1","publisher":{"domain":"publisher.example"}},"device":{"geo":{"lat":12.34,"lon":56.78,"type":2,"country":"US","region":"CA","metro":"501","city":"Example City"},"dnt":1,"ua":"Fictional Browser","ip":"192.0.2.10","language":"en"},"user":{"id":"fictional-user","consent":"fictional-tcf","ext":{"ConsentedProvidersSettings":{"consented_providers":"fictional-ac"},"consent":"fictional-tcf","eids":[{"source":"identity.example","uids":[{"atype":1,"id":"fictional-uid"}]}]}},"tmax":321,"cur":["USD"],"regs":{"gdpr":1,"us_privacy":"1YNN","gpp":"fictional-gpp","gpp_sid":[2,6],"ext":{"gdpr":1,"gpp":"fictional-gpp","gpp_sid":[2,6],"us_privacy":"1YNN"}},"ext":{"prebid":{},"trusted_server":{"request_host":"publisher.example","request_scheme":"https"}}}"#, + "should preserve PBS parity differences" + ); +} + +#[test] +fn aps_inventory_identity_and_page_fallback_preserve_legacy_policy() { + let mut request = canonical_parity_auction_request(); + request.publisher.domain = "deployment.example".to_string(); + request.publisher.page_url = + Some("https://deployment.example/news/story?edition=fictional#section".to_string()); + let built = build_with_request( + "aps", + json!({ + "account_id": "example-account-id", + "inventory_domain": "publisher.example", + "inventory_page_origin": "https://www.publisher.example" + }), + request, + None, + ); + let site = built.site.expect("should include APS site"); + assert_eq!(site.domain.as_deref(), Some("publisher.example")); + assert_eq!( + site.page.as_deref(), + Some("https://www.publisher.example/news/story?edition=fictional") + ); + assert_eq!( + site.publisher + .and_then(|publisher| publisher.domain) + .as_deref(), + Some("publisher.example") + ); + + for unsafe_page in [ + "https://user:password@publisher.example/private", + "data:text/html,fictional", + ] { + let mut request = canonical_parity_auction_request(); + request.publisher.page_url = Some(unsafe_page.to_string()); + let built = build_with_request( + "aps", + json!({"account_id":"example-account-id"}), + request, + None, + ); + assert_eq!( + built.site.and_then(|site| site.page).as_deref(), + Some("https://publisher.example"), + "unsafe page should fall back to publisher domain" + ); + } +} + +#[test] +fn aps_driver_exact_golden_preserves_profile_policy() { + let request = build("aps", json!({"account_id": "example-account-id"}), None); + assert_eq!( + serde_json::to_string(&request).expect("should serialize APS driver request"), + r#"{"id":"fictional-auction","imp":[{"id":"fictional-slot","banner":{"format":[{"w":300,"h":250},{"w":728,"h":90}],"w":300,"h":250,"topframe":0},"bidfloor":1.0,"bidfloorcur":"USD","secure":1}],"site":{"domain":"publisher.example","page":"https://publisher.example/article","publisher":{"domain":"publisher.example"}},"device":{"geo":{"type":2,"country":"US","region":"CA","metro":"501","city":"Example City"},"dnt":1,"ua":"Fictional Browser","ip":"192.0.2.10","language":"en"},"user":{"id":"fictional-user","consent":"fictional-tcf","ext":{"consent":"fictional-tcf","eids":[{"source":"identity.example","uids":[{"atype":1,"id":"fictional-uid"}]}]}},"tmax":321,"cur":["USD"],"regs":{"gdpr":1,"us_privacy":"1YNN","gpp":"fictional-gpp","gpp_sid":[2,6],"ext":{"gdpr":1,"gpp":"fictional-gpp","gpp_sid":[2,6],"us_privacy":"1YNN"}},"ext":{"account":"example-account-id","sdk":{"source":"prebid","version":"2.2.0"}}}"#, + "should preserve APS parity differences" + ); +} + +#[test] +fn signing_finalization_is_after_profiles_and_asserts_every_owned_key() { + let signer = deterministic_signer(); + for (profile, config) in [ + ("standard", json!({"request_ext": {"fictional": true}})), + ("prebid-server", json!({})), + ("aps", json!({"account_id": "example-account-id"})), + ] { + let unsigned = serde_json::to_value(build(profile, config.clone(), None)) + .expect("should serialize unsigned request"); + let unsigned_ts = unsigned["ext"].get("trusted_server"); + if profile == "prebid-server" { + assert_eq!( + unsigned_ts, + Some(&json!({"request_host": "publisher.example", "request_scheme": "https"})), + "should retain only PBS host and scheme when unsigned" + ); + } else { + assert!( + unsigned_ts.is_none(), + "should omit unsigned non-PBS extension" + ); + } + + let signed = serde_json::to_value(build(profile, config, Some(&signer))) + .expect("should serialize signed request"); + let extension = &signed["ext"]["trusted_server"]; + assert_eq!(extension["version"], "1.1", "should set signing version"); + assert_eq!(extension["kid"], "fictional-kid", "should set key ID"); + assert_eq!( + extension["request_host"], "publisher.example", + "should set host" + ); + assert_eq!(extension["request_scheme"], "https", "should set scheme"); + assert_eq!( + extension["ts"], 1_706_900_000_u64, + "should set explicit time" + ); + assert!( + extension["signature"] + .as_str() + .is_some_and(|value| !value.is_empty()), + "should set signature" + ); + } +} + +#[test] +fn signed_profiles_and_unsigned_standard_have_exact_full_goldens() { + let signer = deterministic_signer(); + let cases = [ + ( + "standard", + json!({"request_ext": {"fictional": true}}), + r#"{"id":"fictional-auction","imp":[{"id":"fictional-slot","banner":{"format":[{"w":300,"h":250},{"w":728,"h":90}]},"bidfloor":1.0,"bidfloorcur":"USD","secure":1}],"site":{"domain":"publisher.example","page":"https://publisher.example/article","publisher":{"domain":"publisher.example"}},"device":{"geo":{"type":2,"country":"US","region":"CA","metro":"501","city":"Example City"},"dnt":1,"ua":"Fictional Browser","ip":"192.0.2.10","language":"en"},"user":{"id":"fictional-user","consent":"fictional-tcf","ext":{"consent":"fictional-tcf","eids":[{"source":"identity.example","uids":[{"atype":1,"id":"fictional-uid"}]}]}},"tmax":321,"cur":["USD"],"regs":{"gdpr":1,"us_privacy":"1YNN","gpp":"fictional-gpp","gpp_sid":[2,6],"ext":{"gdpr":1,"gpp":"fictional-gpp","gpp_sid":[2,6],"us_privacy":"1YNN"}},"ext":{"fictional":true,"trusted_server":{"kid":"fictional-kid","request_host":"publisher.example","request_scheme":"https","signature":"LU_JUIA1BT80ShZNjSa4PIF5T-uMjEeodwKrV_6bXgh0hi1SYVtCKn9g_DTW62krmjCOFgoFYPHsu6L0nAcuDg","ts":1706900000,"version":"1.1"}}}"#, + ), + ( + "prebid-server", + json!({}), + r#"{"id":"fictional-auction","imp":[{"id":"fictional-slot","banner":{"format":[{"w":300,"h":250},{"w":728,"h":90}]},"tagid":"fictional-slot","bidfloor":1.0,"bidfloorcur":"USD","secure":1,"ext":{"prebid":{"bidder":{"exampleBidder":{"placement":"fictional-placement"}}}}}],"site":{"domain":"publisher.example","page":"https://publisher.example/article","ref":"https://referrer.example/story?fictional=1","publisher":{"domain":"publisher.example"}},"device":{"geo":{"lat":12.34,"lon":56.78,"type":2,"country":"US","region":"CA","metro":"501","city":"Example City"},"dnt":1,"ua":"Fictional Browser","ip":"192.0.2.10","language":"en"},"user":{"id":"fictional-user","consent":"fictional-tcf","ext":{"ConsentedProvidersSettings":{"consented_providers":"fictional-ac"},"consent":"fictional-tcf","eids":[{"source":"identity.example","uids":[{"atype":1,"id":"fictional-uid"}]}]}},"tmax":321,"cur":["USD"],"regs":{"gdpr":1,"us_privacy":"1YNN","gpp":"fictional-gpp","gpp_sid":[2,6],"ext":{"gdpr":1,"gpp":"fictional-gpp","gpp_sid":[2,6],"us_privacy":"1YNN"}},"ext":{"prebid":{},"trusted_server":{"kid":"fictional-kid","request_host":"publisher.example","request_scheme":"https","signature":"LU_JUIA1BT80ShZNjSa4PIF5T-uMjEeodwKrV_6bXgh0hi1SYVtCKn9g_DTW62krmjCOFgoFYPHsu6L0nAcuDg","ts":1706900000,"version":"1.1"}}}"#, + ), + ( + "aps", + json!({"account_id": "example-account-id"}), + r#"{"id":"fictional-auction","imp":[{"id":"fictional-slot","banner":{"format":[{"w":300,"h":250},{"w":728,"h":90}],"w":300,"h":250,"topframe":0},"bidfloor":1.0,"bidfloorcur":"USD","secure":1}],"site":{"domain":"publisher.example","page":"https://publisher.example/article","publisher":{"domain":"publisher.example"}},"device":{"geo":{"type":2,"country":"US","region":"CA","metro":"501","city":"Example City"},"dnt":1,"ua":"Fictional Browser","ip":"192.0.2.10","language":"en"},"user":{"id":"fictional-user","consent":"fictional-tcf","ext":{"consent":"fictional-tcf","eids":[{"source":"identity.example","uids":[{"atype":1,"id":"fictional-uid"}]}]}},"tmax":321,"cur":["USD"],"regs":{"gdpr":1,"us_privacy":"1YNN","gpp":"fictional-gpp","gpp_sid":[2,6],"ext":{"gdpr":1,"gpp":"fictional-gpp","gpp_sid":[2,6],"us_privacy":"1YNN"}},"ext":{"account":"example-account-id","sdk":{"source":"prebid","version":"2.2.0"},"trusted_server":{"kid":"fictional-kid","request_host":"publisher.example","request_scheme":"https","signature":"LU_JUIA1BT80ShZNjSa4PIF5T-uMjEeodwKrV_6bXgh0hi1SYVtCKn9g_DTW62krmjCOFgoFYPHsu6L0nAcuDg","ts":1706900000,"version":"1.1"}}}"#, + ), + ]; + for (profile, config, expected) in cases { + assert_eq!( + serde_json::to_string(&build(profile, config, Some(&signer))) + .expect("should serialize signed request"), + expected, + "{profile} signed wire fixture should stay exact" + ); + } + + assert_eq!( + serde_json::to_string(&build( + "standard", + json!({"request_ext": {"fictional": true}}), + None, + )) + .expect("should serialize unsigned standard request"), + r#"{"id":"fictional-auction","imp":[{"id":"fictional-slot","banner":{"format":[{"w":300,"h":250},{"w":728,"h":90}]},"bidfloor":1.0,"bidfloorcur":"USD","secure":1}],"site":{"domain":"publisher.example","page":"https://publisher.example/article","publisher":{"domain":"publisher.example"}},"device":{"geo":{"type":2,"country":"US","region":"CA","metro":"501","city":"Example City"},"dnt":1,"ua":"Fictional Browser","ip":"192.0.2.10","language":"en"},"user":{"id":"fictional-user","consent":"fictional-tcf","ext":{"consent":"fictional-tcf","eids":[{"source":"identity.example","uids":[{"atype":1,"id":"fictional-uid"}]}]}},"tmax":321,"cur":["USD"],"regs":{"gdpr":1,"us_privacy":"1YNN","gpp":"fictional-gpp","gpp_sid":[2,6],"ext":{"gdpr":1,"gpp":"fictional-gpp","gpp_sid":[2,6],"us_privacy":"1YNN"}},"ext":{"fictional":true}}"#, + "unsigned standard wire fixture should stay exact" + ); +} + +#[test] +fn standard_static_extensions_have_no_invented_bidder_param_location() { + let request = build( + "standard", + json!({ + "request_ext": {"fictional_request": {"enabled": true}}, + "imp_ext": {"fictional_imp": "value"} + }), + None, + ); + let value = serde_json::to_value(request).expect("should serialize request"); + assert_eq!(value["ext"]["fictional_request"]["enabled"], true); + assert_eq!(value["imp"][0]["ext"]["fictional_imp"], "value"); + assert!( + !value.to_string().contains("exampleBidder"), + "standard profile must not invent bidder params placement" + ); +} + +#[test] +fn defensive_no_impression_outcome_does_not_build_transportable_request() { + let (plan, mut routed) = routed("standard", json!({})); + let mut common = routed.inputs()[0].common_request().clone(); + common.slots = vec![AdSlot { + id: "video-only".to_string(), + formats: vec![AdFormat { + media_type: MediaType::Video, + width: 640, + height: 480, + }], + floor_price: None, + targeting: HashMap::new(), + bidders: HashMap::new(), + }]; + let inbound = Request::builder() + .uri("https://publisher.example/auction") + .body(EdgeBody::empty()) + .expect("should build inbound request"); + routed = route_auction(common, &inbound, &plan, None); + assert!( + routed.inputs().is_empty(), + "should omit provider input before build" + ); +} + +fn standard_fixture_with_formats( + formats: Vec, +) -> (AuctionPlan, RoutedAuction, OpenRtbRequest) { + let mut raw = config( + "standard", + json!({"request_ext": {"fixture": true}, "imp_ext": {"slot_fixture": true}}), + ); + raw.providers + .get_mut(&ProviderId::from_str("fictional-provider").expect("should parse provider")) + .expect("should find provider") + .routing = RoutingMode::Explicit; + raw.bidders.insert( + crate::auction::plan::BidderId::from_str("exampleBidder").expect("should parse bidder"), + BidderRouteConfig { + provider: ProviderId::from_str("fictional-provider").expect("should parse provider"), + }, + ); + let plan = AuctionPlan::compile(raw).expect("should compile standard fixture plan"); + let inbound = Request::builder() + .uri("https://publisher.example/auction") + .body(EdgeBody::empty()) + .expect("should build inbound request"); + let mut auction_request = canonical_parity_auction_request(); + auction_request.slots[0].formats = formats; + let routed = route_auction(auction_request, &inbound, &plan, None); + let request = match build_request( + &routed.inputs()[0], + &routed, + &plan.providers()[0], + 321, + &finalization(None), + ) + .expect("should build standard fixture request") + { + OpenRtbBuildOutcome::Ready(request) => request, + OpenRtbBuildOutcome::NoImpressions => panic!("should retain impression"), + }; + (plan, routed, request) +} + +fn standard_fixture() -> (AuctionPlan, RoutedAuction, OpenRtbRequest) { + standard_fixture_with_formats(vec![AdFormat { + media_type: MediaType::Banner, + width: 300, + height: 250, + }]) +} + +#[test] +fn standard_response_extraction_isolates_malformed_siblings_and_ignores_response_id() { + let (_plan, routed, _request) = standard_fixture(); + let response = extract_standard_response( + "fictional-provider", + &routed.inputs()[0], + &json!({ + "id": "informational-mismatch", + "seatbid": [{"seat": "fictional-seat", "bid": [ + {"id": "good", "impid": "fictional-slot", "price": 1.5, "adm": "
ok
", "w": 300, "h": 250}, + {"id": "bad", "impid": "fictional-slot", "price": "bad", "adm": "
bad
", "w": 300, "h": 250} + ]}] + }), + 9, + ); + assert_eq!(response.status, BidStatus::Success); + assert_eq!(response.bids.len(), 1, "should isolate malformed sibling"); + assert_eq!( + response.bids[0].returned_seat.as_deref(), + Some("fictional-seat") + ); +} + +#[test] +fn standard_response_currency_accepts_omitted_and_usd_but_rejects_other_or_malformed_values() { + let (_plan, routed, _request) = standard_fixture(); + let bid = json!({"seatbid": [{"seat": "seat", "bid": [ + {"id":"good","impid":"fictional-slot","price":1.0,"adm":"ok","w":300,"h":250} + ]}]}); + + for currency in [None, Some(json!("USD")), Some(json!("usd"))] { + let mut value = bid.clone(); + if let Some(currency) = currency { + value["cur"] = currency; + } + let response = + extract_standard_response("fictional-provider", &routed.inputs()[0], &value, 0); + assert_eq!( + response.status, + BidStatus::Success, + "should accept omitted or USD currency" + ); + assert_eq!(response.bids[0].currency, "USD"); + } + + let mut eur = bid.clone(); + eur["cur"] = json!("EUR"); + let response = extract_standard_response("fictional-provider", &routed.inputs()[0], &eur, 0); + assert_eq!(response.status, BidStatus::NoBid); + assert_eq!(response.metadata["unsupported_currency"], "EUR"); + + let mut malformed = bid; + malformed["cur"] = json!(["USD"]); + let response = + extract_standard_response("fictional-provider", &routed.inputs()[0], &malformed, 0); + assert_eq!(response.status, BidStatus::Error); + assert_eq!(response.metadata["error_type"], "parse_response"); +} + +#[test] +fn standard_response_rejects_unknown_impressions_and_dimensions_but_keeps_siblings() { + let (_plan, routed, _request) = standard_fixture(); + let response = extract_standard_response( + "fictional-provider", + &routed.inputs()[0], + &json!({"seatbid": [{"seat": "seat", "bid": [ + {"id":"good","impid":"fictional-slot","price":1.0,"adm":"ok","w":300,"h":250}, + {"id":"unknown","impid":"unknown-slot","price":2.0,"adm":"bad","w":300,"h":250}, + {"id":"dimension","impid":"fictional-slot","price":3.0,"adm":"bad","w":320,"h":50} + ]}]}), + 0, + ); + assert_eq!(response.status, BidStatus::Success); + assert_eq!(response.bids.len(), 1); + assert_eq!(response.bids[0].bid_id.as_deref(), Some("good")); + assert_eq!( + response.metadata["response_admission"]["rejected_bid_count"], + 2 + ); + assert_eq!( + response.metadata["response_admission"]["rejection_reasons"]["unrequested_impression"], + 1 + ); + assert_eq!( + response.metadata["response_admission"]["rejection_reasons"]["dimension_mismatch"], + 1 + ); +} + +#[test] +fn standard_response_infers_only_unambiguous_missing_dimensions() { + let (_plan, routed, _request) = standard_fixture(); + let inferred = extract_standard_response( + "fictional-provider", + &routed.inputs()[0], + &json!({"seatbid": [{"bid": [ + {"id":"inferred","impid":"fictional-slot","price":1.0,"adm":"ok"} + ]}]}), + 0, + ); + assert_eq!(inferred.status, BidStatus::Success); + assert_eq!(inferred.bids[0].width, 300); + assert_eq!(inferred.bids[0].height, 250); + + let formats = vec![ + AdFormat { + media_type: MediaType::Banner, + width: 300, + height: 250, + }, + AdFormat { + media_type: MediaType::Banner, + width: 320, + height: 50, + }, + ]; + let (_plan, routed, _request) = standard_fixture_with_formats(formats); + let ambiguous = extract_standard_response( + "fictional-provider", + &routed.inputs()[0], + &json!({"seatbid": [{"bid": [ + {"id":"ambiguous","impid":"fictional-slot","price":1.0,"adm":"ok"} + ]}]}), + 0, + ); + assert_eq!(ambiguous.status, BidStatus::NoBid); + assert_eq!( + ambiguous.metadata["response_admission"]["rejected_bid_count"], + 1 + ); + assert_eq!( + ambiguous.metadata["response_admission"]["rejection_reasons"]["ambiguous_dimensions"], + 1 + ); +} + +#[test] +fn notification_suppression_matrix_uses_only_exact_valid_returned_seat() { + let (_plan, routed, _request) = standard_fixture(); + let response = extract_standard_response( + "fictional-provider", + &routed.inputs()[0], + &json!({"seatbid": [ + {"seat": "exact", "bid": [{"id":"exact","impid":"fictional-slot","price":1.0,"adm":"ok","w":300,"h":250,"nurl":"https://n.example","burl":"https://b.example"}]}, + {"seat": "Exact", "bid": [{"id":"case","impid":"fictional-slot","price":1.0,"adm":"ok","w":300,"h":250,"nurl":"https://n.example","burl":"https://b.example"}]}, + {"bid": [{"id":"missing","impid":"fictional-slot","price":1.0,"adm":"ok","w":300,"h":250,"nurl":"https://n.example","burl":"https://b.example"}]}, + {"seat": 7, "bid": [{"id":"nonstring","impid":"fictional-slot","price":1.0,"adm":"ok","w":300,"h":250,"nurl":"https://n.example","burl":"https://b.example"}]} + ]}), + 0, + ); + let mut exact = response.bids.clone(); + apply_notification_policy( + &mut exact, + &NotificationPolicy { + suppress_all: false, + suppress_seats: BTreeSet::from(["exact".to_string(), "unknown".to_string()]), + }, + ); + assert!(exact[0].nurl.is_none(), "should suppress exact seat"); + assert!(exact[1].nurl.is_some(), "matching should be case-sensitive"); + assert!(exact[2].nurl.is_some(), "missing seat must not match"); + assert!(exact[3].nurl.is_some(), "non-string seat must not match"); + assert_eq!(exact[2].bidder, "unknown"); + assert_eq!(exact[3].bidder, "unknown"); + + let mut all = response.bids; + apply_notification_policy( + &mut all, + &NotificationPolicy { + suppress_all: true, + suppress_seats: BTreeSet::new(), + }, + ); + assert!( + all.iter() + .all(|bid| bid.nurl.is_none() && bid.burl.is_none()), + "suppress_all should remove every notification" + ); +} + +#[test] +fn fictional_standard_executor_covers_bid_no_bid_malformed_unused_and_redirect() { + futures::executor::block_on(async { + let (plan, routed, request) = standard_fixture(); + let provider = &plan.providers()[0]; + let client = Arc::new(StubHttpClient::new()); + client.push_response( + 200, + serde_json::to_vec(&json!({"id":"mismatch","seatbid":[{"seat":"fictional-seat","bid":[ + {"id":"good","impid":"fictional-slot","price":1.25,"adm":"
fictional
","w":300,"h":250}, + {"id":"bad","impid":"fictional-slot","price":null,"adm":"bad","w":300,"h":250} + ]}]})).expect("should serialize response"), + ); + let response = execute_standard_fixture( + provider, + &routed.inputs()[0], + &request, + &StubBackend, + client.as_ref(), + ) + .await + .expect("should execute ordinary fixture"); + assert_eq!(response.status, BidStatus::Success); + assert_eq!(response.bids.len(), 1, "should isolate malformed sibling"); + assert_eq!( + response.metadata["routing"]["unused_bidder_params_count"], + 1 + ); + assert_eq!( + client.recorded_request_uris(), + vec![provider.endpoint.as_str()] + ); + let headers = &client.recorded_request_headers()[0]; + assert!( + !headers.iter().any(|(name, _)| name == "authorization"), + "fixture must add no authentication" + ); + let body: Value = serde_json::from_slice(&client.recorded_request_bodies()[0]) + .expect("should parse recorded body"); + assert_eq!(body["ext"]["fixture"], true, "should send static extension"); + + let no_bid = Arc::new(StubHttpClient::new()); + no_bid.push_response(204, Vec::new()); + let response = execute_standard_fixture( + provider, + &routed.inputs()[0], + &request, + &StubBackend, + no_bid.as_ref(), + ) + .await + .expect("should execute no-bid fixture"); + assert_eq!(response.status, BidStatus::NoBid); + + let malformed = Arc::new(StubHttpClient::new()); + malformed.push_response(200, b"not-json".to_vec()); + let response = execute_standard_fixture( + provider, + &routed.inputs()[0], + &request, + &StubBackend, + malformed.as_ref(), + ) + .await + .expect("should classify malformed response"); + assert_eq!(response.status, BidStatus::Error); + assert_eq!(response.metadata["error_type"], "parse_response"); + + let redirect = Arc::new(StubHttpClient::new()); + redirect.push_response_with_headers( + 302, + Vec::new(), + vec![("location", "https://redirect.example.test/openrtb")], + ); + let response = execute_standard_fixture( + provider, + &routed.inputs()[0], + &request, + &StubBackend, + redirect.as_ref(), + ) + .await + .expect("should classify redirect"); + assert_eq!(response.status, BidStatus::Error); + assert_eq!(response.metadata["error_type"], "http_status"); + assert_eq!(response.metadata["http_status"], 302); + assert_eq!( + redirect.recorded_request_uris(), + vec![provider.endpoint.as_str()], + "a 3xx Location must not trigger a second HTTP request" + ); + assert_eq!( + redirect.recorded_backend_names(), + vec!["stub-backend"], + "the common driver must perform exactly one underlying send for a 3xx" + ); + let spec = provider.backend_spec(); + assert_eq!(spec.host, "exchange.example.test"); + assert_eq!(spec.discriminator.as_deref(), Some("fictional-provider")); + }); +} + +#[test] +fn prebid_endpoint_normalization_reaches_generic_execution_and_preserves_custom_paths() { + futures::executor::block_on(async { + for (configured_endpoint, expected_endpoint) in [ + ( + "https://pbs.example", + "https://pbs.example/openrtb2/auction", + ), + ("https://pbs.example/bid", "https://pbs.example/bid"), + ] { + let plan = AuctionPlan::compile(config_with_endpoint( + "prebid-server", + json!({}), + configured_endpoint, + )) + .expect("should compile Prebid Server endpoint"); + let inbound = Request::builder() + .uri("https://publisher.example/auction") + .body(EdgeBody::empty()) + .expect("should build inbound request"); + let routed = route_auction(canonical_parity_auction_request(), &inbound, &plan, None); + let provider_plan = plan.providers()[0].clone(); + let provider = GenericOpenRtbProvider::new(provider_plan.clone()); + let client = Arc::new(StubHttpClient::new()); + client.push_response(204, Vec::new()); + let services = build_services_with_backend_and_http_client( + Arc::new(StubBackend), + Arc::clone(&client) as Arc, + ); + let mut reserved_backend_names = HashSet::new(); + + let outcome = provider + .request_bids_routed( + &routed.inputs()[0], + &routed, + 321, + 321, + None, + &services, + &mut reserved_backend_names, + ) + .await + .expect("should launch one Prebid Server request"); + let ProviderRequestOutcome::Pending { request, .. } = outcome else { + panic!("should launch a pending Prebid Server request"); + }; + let selected = services + .http_client() + .select(vec![request]) + .await + .expect("should select one Prebid Server response"); + let response = selected + .ready + .expect("should receive the Prebid Server response"); + + assert_eq!(response.response.status(), http::StatusCode::NO_CONTENT); + assert_eq!(client.recorded_backend_names(), vec!["stub-backend"]); + assert_eq!(client.recorded_request_methods(), vec!["POST"]); + assert_eq!(client.recorded_request_uris(), vec![expected_endpoint]); + assert_eq!(provider_plan.backend_spec().host, "pbs.example"); + let body: Value = serde_json::from_slice(&client.recorded_request_bodies()[0]) + .expect("should parse recorded Prebid Server body"); + assert!( + !body + .as_object() + .expect("should serialize an object") + .is_empty() + ); + } + }); +} + +#[test] +fn malformed_top_level_standard_response_is_error() { + let (_plan, routed, _request) = standard_fixture(); + let response = + extract_standard_response("fictional-provider", &routed.inputs()[0], &json!([]), 0); + assert_eq!(response.status, BidStatus::Error); + assert_eq!(response.metadata["error_type"], "parse_response"); +} diff --git a/crates/trusted-server-core/src/auction/orchestrator.rs b/crates/trusted-server-core/src/auction/orchestrator.rs index 728cc1efe..ee51de1b9 100644 --- a/crates/trusted-server-core/src/auction/orchestrator.rs +++ b/crates/trusted-server-core/src/auction/orchestrator.rs @@ -3,18 +3,26 @@ use edgezero_core::body::Body as EdgeBody; use error_stack::{Report, ResultExt}; use http::Request; -use std::collections::{HashMap, HashSet}; +use std::collections::{HashMap, HashSet, hash_map::Entry}; use std::sync::Arc; -use std::time::Duration; use web_time::Instant; use crate::error::TrustedServerError; use crate::platform::{PlatformPendingRequest, RuntimeServices}; +#[cfg(test)] use super::config::AuctionConfig; -use super::provider::{AuctionProvider, ProviderParseState, ProviderRequestOutcome}; +use super::openrtb::unused_bidder_params_count; +use super::plan::AuctionPlan; +use super::provider::{ + AuctionProvider, GenericOpenRtbProvider, ProviderParseState, ProviderRequestOutcome, +}; +#[cfg(test)] +use super::routing::RoutedAuction; +use super::routing::route_auction; use super::telemetry::AbandonedProviderCall; use super::types::{AuctionContext, AuctionRequest, AuctionResponse, Bid, BidStatus}; +use crate::request_signing::RequestSigner; /// In-flight auction requests dispatched to SSP backends. /// @@ -26,6 +34,7 @@ use super::types::{AuctionContext, AuctionRequest, AuctionResponse, Bid, BidStat pub struct DispatchedAuction { pending_requests: Vec, backend_to_provider: HashMap, + planned_backend_to_provider: HashMap, completed_responses: Vec, auction_start: Instant, timeout_ms: u32, @@ -33,6 +42,9 @@ pub struct DispatchedAuction { provider_request_context: Box>, /// Carried so the mediator call in collect can pass it as the auction request. request: AuctionRequest, + planned_unused_bidder_params: HashMap, + planned_unroutable_bidder_count: u32, + planned_provider_order: HashMap, } struct ProviderLaunchState { @@ -44,6 +56,10 @@ struct ProviderLaunchState { } /// Outcome of attempting to dispatch split-phase auction provider requests. +#[allow( + clippy::large_enum_variant, + reason = "Dispatched carries in-flight requests while failure preserves provider responses" +)] pub enum DispatchAuctionOutcome { /// No provider request was started and no provider failure was observed. NotStarted, @@ -53,6 +69,13 @@ pub enum DispatchAuctionOutcome { request: AuctionRequest, /// Provider launch-failure responses. provider_responses: Vec, + /// Fatal admission error that synchronous execution must propagate. + /// + /// Split publisher dispatch records the failure and continues without + /// attempting provider network I/O. + fatal_admission_error: Option>, + /// Auction-level metadata materialized before the failure. + metadata: HashMap, /// Elapsed dispatch time. elapsed_ms: u64, }, @@ -75,10 +98,16 @@ impl DispatchedAuction { let abandoned = self .backend_to_provider .into_values() - .map(|state| { + .map(|state| (state.provider_name, state.started_at)) + .chain( + self.planned_backend_to_provider + .into_values() + .map(|state| (state.provider.provider_name().to_string(), state.started_at)), + ) + .map(|(provider_name, started_at)| { AbandonedProviderCall::bidder( - state.provider_name, - Some(u32::try_from(state.started_at.elapsed().as_millis()).unwrap_or(u32::MAX)), + provider_name, + Some(u32::try_from(started_at.elapsed().as_millis()).unwrap_or(u32::MAX)), ) }) .collect(); @@ -97,12 +126,16 @@ impl DispatchedAuction { Self { pending_requests: Vec::new(), backend_to_provider: HashMap::new(), + planned_backend_to_provider: HashMap::new(), completed_responses: Vec::new(), auction_start: Instant::now(), timeout_ms, floor_prices: HashMap::new(), provider_request_context: Box::new(Request::new(EdgeBody::empty())), request, + planned_unused_bidder_params: HashMap::new(), + planned_unroutable_bidder_count: 0, + planned_provider_order: HashMap::new(), } } } @@ -182,6 +215,13 @@ fn provider_timeout_response(provider_name: &str, response_time_ms: u64) -> Auct .with_metadata("message", serde_json::json!("Provider request timed out")) } +fn provider_skipped_response(provider_name: &str) -> AuctionResponse { + AuctionResponse::no_bid(provider_name, 0).with_metadata( + "routing", + serde_json::json!({"skipped_no_eligible_slots": true}), + ) +} + /// Compute the remaining time budget from a deadline. /// /// Returns the number of milliseconds left before `timeout_ms` is exceeded, @@ -192,6 +232,68 @@ fn remaining_budget_ms(start: Instant, timeout_ms: u32) -> u32 { timeout_ms.saturating_sub(elapsed) } +/// Runtime policy for classifying responses that complete after the logical auction budget. +/// +/// Current adapters do not expose an enforceable total-request deadline. They +/// therefore drain already-launched work and accept completed late responses. +#[derive(Debug, Clone, Copy, Default)] +struct AuctionDeadlinePolicy { + enforceable_total_request_deadline: bool, +} + +impl AuctionDeadlinePolicy { + fn rejects_late_completion(self, start: Instant, timeout_ms: u32) -> bool { + self.enforceable_total_request_deadline && remaining_budget_ms(start, timeout_ms) == 0 + } + + fn for_runtime(services: &RuntimeServices) -> Self { + Self { + enforceable_total_request_deadline: services + .http_client() + .has_enforceable_total_request_deadline(), + } + } +} + +fn routing_metadata(unroutable_bidder_count: u32) -> HashMap { + HashMap::from([( + "routing".to_string(), + serde_json::json!({"unroutable_bidder_count": unroutable_bidder_count}), + )]) +} + +/// Attach only the count derived from the routed provider input at dispatch. +/// +/// This is intentionally applied after every provider outcome is materialized, +/// including failures produced before or during parsing. Skipped providers are +/// routed separately and retain their exclusive skipped diagnostic. +fn materialize_planned_response( + mut response: AuctionResponse, + unused_bidder_params_count: u32, +) -> AuctionResponse { + let routing = response + .metadata + .entry("routing".to_string()) + .or_insert_with(|| serde_json::json!({})); + if routing + .get("skipped_no_eligible_slots") + .is_some_and(|value| value == &serde_json::json!(true)) + { + return response; + } + if !routing.is_object() { + *routing = serde_json::json!({}); + } + routing + .as_object_mut() + .expect("should normalize planned routing metadata to an object") + .insert( + "unused_bidder_params_count".to_string(), + serde_json::json!(unused_bidder_params_count), + ); + response +} + fn snapshot_context_request(request: &Request) -> Request { let mut snapshot = Request::new(EdgeBody::empty()); *snapshot.method_mut() = request.method().clone(); @@ -203,22 +305,538 @@ fn snapshot_context_request(request: &Request) -> Request { /// Manages auction execution across multiple providers. pub struct AuctionOrchestrator { + enabled: bool, + plan_backed: bool, + plan: Arc, + planned_providers: Vec>, + mediator: Option>, + #[cfg(test)] config: AuctionConfig, + #[cfg(test)] providers: HashMap>, } +/// Test harness for the live plan-backed orchestrator semantics. +#[cfg(test)] +pub(crate) struct AuctionOrchestratorHarness { + plan: Arc, + providers: Vec>, + mediator: Option>, +} + +struct PlannedLaunchState { + provider: Arc, + started_at: Instant, + parse_state: Option, +} + +#[cfg(test)] +#[allow( + dead_code, + reason = "test harness exercises plan-backed runtime behavior" +)] +impl AuctionOrchestratorHarness { + pub(crate) fn new( + plan: impl Into>, + mediator: Option>, + ) -> Self { + let plan = plan.into(); + let providers = plan + .providers() + .iter() + .cloned() + .map(GenericOpenRtbProvider::new) + .map(Arc::new) + .collect(); + Self { + plan, + providers, + mediator, + } + } + + pub(crate) fn provider_count(&self) -> usize { + self.providers.len() + } + + pub(crate) fn mediator(&self) -> Option<&Arc> { + self.mediator.as_ref() + } + + /// Route and execute config-first bidder providers in deterministic order. + pub(crate) async fn run_auction( + &self, + request: &AuctionRequest, + context: &AuctionContext<'_>, + ) -> Result> { + // Admission, including signer-store reads and routing, consumes the same + // request-local deadline as provider transport and response collection. + let auction_start = Instant::now(); + let routed = route_auction( + request.clone(), + context.request, + &self.plan, + context.services.client_info().client_ip, + ); + if context.timeout_ms == 0 { + return self + .run_routed(request, &routed, context, None, auction_start) + .await; + } + if self.providers.len() > 1 && !context.services.http_client().supports_concurrent_fanout() + { + return Err(Report::new(TrustedServerError::Auction { + message: format!( + "{} auction providers configured, but this platform's HTTP client does not support concurrent fanout", + self.providers.len() + ), + })); + } + + // Signing admission deliberately precedes every backend call. + let signer = self + .plan + .signing_enabled() + .then(|| RequestSigner::from_services(context.services)) + .transpose()?; + self.run_routed(request, &routed, context, signer.as_ref(), auction_start) + .await + } + + async fn run_routed( + &self, + original_request: &AuctionRequest, + routed: &RoutedAuction, + context: &AuctionContext<'_>, + signer: Option<&RequestSigner>, + auction_start: Instant, + ) -> Result> { + let mut responses = routed + .skipped_no_eligible_provider_ids() + .iter() + .map(|id| provider_skipped_response(id.as_str())) + .collect::>(); + let planned_unused_bidder_params = routed + .inputs() + .iter() + .map(|input| { + ( + input.provider_id().as_str().to_string(), + unused_bidder_params_count( + &self + .plan + .provider(input.provider_id()) + .expect("should find routed provider in compiled plan") + .profile, + input, + ), + ) + }) + .collect::>(); + let mut pending = Vec::new(); + let mut launches = HashMap::new(); + let mut reserved_backend_names = HashSet::new(); + + for input in routed.inputs() { + let Some(provider) = self + .providers + .iter() + .find(|provider| provider.provider_name() == input.provider_id().as_str()) + .cloned() + else { + responses.push(provider_launch_failed_response( + input.provider_id().as_str(), + 0, + )); + continue; + }; + let remaining_ms = remaining_budget_ms(auction_start, context.timeout_ms); + let logical_budget_ms = remaining_ms.min(provider.timeout_ms()); + if logical_budget_ms == 0 { + responses.push(provider_timeout_response(provider.provider_name(), 0)); + continue; + } + let transport_timeout_ms = context + .services + .backend() + .canonicalize_transport_timeout_ms(logical_budget_ms, provider.timeout_ms()); + if transport_timeout_ms == 0 { + responses.push(provider_timeout_response(provider.provider_name(), 0)); + continue; + } + let started_at = Instant::now(); + match provider + .request_bids_routed( + input, + routed, + logical_budget_ms, + transport_timeout_ms, + signer, + context.services, + &mut reserved_backend_names, + ) + .await + { + Ok(ProviderRequestOutcome::Pending { + request: launched, + parse_state, + }) => { + let Some(backend_name) = launched.backend_name().map(str::to_string) else { + log::warn!( + "Planned provider '{}' pending request had no backend name", + provider.provider_name() + ); + responses.push(provider_launch_failed_response( + provider.provider_name(), + started_at.elapsed().as_millis() as u64, + )); + continue; + }; + match launches.entry(backend_name) { + Entry::Vacant(entry) => { + entry.insert(PlannedLaunchState { + provider, + started_at, + parse_state, + }); + pending.push(launched); + } + Entry::Occupied(entry) => { + log::warn!( + "Planned provider '{}' pending backend '{}' already belongs to another provider", + provider.provider_name(), + entry.key(), + ); + responses.push(provider_launch_failed_response( + provider.provider_name(), + started_at.elapsed().as_millis() as u64, + )); + } + } + } + Ok(ProviderRequestOutcome::Immediate(response)) => responses.push(response), + Err(error) => { + log::warn!( + "Planned provider '{}' failed to launch: {:?}", + provider.provider_name(), + error + ); + responses.push(provider_launch_failed_response( + provider.provider_name(), + started_at.elapsed().as_millis() as u64, + )); + } + } + } + + while !pending.is_empty() { + let select_result = match context.services.http_client().select(pending).await { + Ok(result) => result, + Err(error) => { + log::warn!("Planned provider select failed: {:?}", error); + let mut transport_failures = launches + .drain() + .map(|(_, state)| { + provider_transport_failed_response( + state.provider.provider_name(), + state.started_at.elapsed().as_millis() as u64, + ) + }) + .collect::>(); + transport_failures.sort_by(|left, right| left.provider.cmp(&right.provider)); + responses.extend(transport_failures); + break; + } + }; + pending = select_result.remaining; + match select_result.ready { + Ok(platform_response) => { + let backend_name = platform_response + .backend_name + .as_deref() + .unwrap_or_default() + .to_string(); + if let Some(state) = launches.remove(&backend_name) { + let elapsed_ms = state.started_at.elapsed().as_millis() as u64; + let deadline_policy = AuctionDeadlinePolicy::for_runtime(context.services); + if deadline_policy + .rejects_late_completion(auction_start, context.timeout_ms) + { + responses.push(provider_timeout_response( + state.provider.provider_name(), + elapsed_ms, + )); + continue; + } + match state + .provider + .parse_response_with_state( + platform_response, + elapsed_ms, + state.parse_state.as_deref(), + ) + .await + { + Ok(response) => responses.push(response), + Err(error) => responses.push(provider_error_response( + state.provider.provider_name(), + elapsed_ms, + ERROR_TYPE_PARSE_RESPONSE, + &error, + )), + } + } + } + Err(error) => { + if let Some(backend_name) = select_result.failed_backend_name + && let Some(state) = launches.remove(&backend_name) + { + let elapsed_ms = state.started_at.elapsed().as_millis() as u64; + log::warn!( + "Planned provider '{}' transport failed: {:?}", + state.provider.provider_name(), + error + ); + responses.push(provider_transport_failed_response( + state.provider.provider_name(), + elapsed_ms, + )); + } + } + } + } + for state in launches.into_values() { + responses.push(provider_timeout_response( + state.provider.provider_name(), + state.started_at.elapsed().as_millis() as u64, + )); + } + + for response in &mut responses { + if let Some(&unused_bidder_params_count) = + planned_unused_bidder_params.get(response.provider.as_str()) + { + *response = + materialize_planned_response(response.clone(), unused_bidder_params_count); + } + } + + let provider_order = self + .plan + .providers() + .iter() + .enumerate() + .map(|(index, provider)| (provider.id.as_str(), index)) + .collect::>(); + responses.sort_by_key(|response| { + provider_order + .get(response.provider.as_str()) + .copied() + .unwrap_or(usize::MAX) + }); + + let floor_prices = original_request + .slots + .iter() + .filter_map(|slot| slot.floor_price.map(|floor| (slot.id.clone(), floor))) + .collect::>(); + let helper = AuctionOrchestrator::new(AuctionConfig::default()); + let local_winners = || helper.select_winning_bids(&responses, &floor_prices); + let (mediator_response, winning_bids) = if let Some(mediator) = &self.mediator { + let remaining_ms = remaining_budget_ms(auction_start, context.timeout_ms); + let logical_budget_ms = remaining_ms.min(mediator.timeout_ms()); + if logical_budget_ms == 0 { + log::warn!( + "Auction deadline exhausted before planned mediator; using local ranking" + ); + (None, local_winners()) + } else { + let transport_timeout_ms = context + .services + .backend() + .canonicalize_transport_timeout_ms(logical_budget_ms, mediator.timeout_ms()); + if transport_timeout_ms == 0 { + log::warn!( + "Planned mediator transport budget canonicalized to zero; using local ranking" + ); + let winning_bids = local_winners(); + return Ok(OrchestrationResult { + provider_responses: responses, + mediator_response: None, + winning_bids, + total_time_ms: auction_start.elapsed().as_millis() as u64, + metadata: routing_metadata(routed.diagnostics().unroutable_bidder_count()), + }); + } + let mediator_context = AuctionContext { + settings: context.settings, + request: context.request, + timeout_ms: logical_budget_ms, + transport_timeout_ms, + provider_responses: Some(&responses), + services: context.services, + }; + let mediator_start = Instant::now(); + let mediated = match mediator + .request_bids(original_request, &mediator_context) + .await + { + Ok(ProviderRequestOutcome::Immediate(response)) => Some(response), + Ok(ProviderRequestOutcome::Pending { + request: pending, + parse_state, + }) => match context.services.http_client().wait(pending).await { + Ok(platform_response) => { + let response_time_ms = mediator_start.elapsed().as_millis() as u64; + if AuctionDeadlinePolicy::for_runtime(context.services) + .rejects_late_completion(auction_start, context.timeout_ms) + { + log::warn!( + "Planned mediator '{}' completed after the hard auction deadline; using local ranking ({}ms)", + mediator.provider_name(), + response_time_ms + ); + None + } else { + mediator + .parse_response_with_context_and_state( + platform_response, + response_time_ms, + original_request, + &mediator_context, + parse_state.as_deref(), + ) + .await + .map_err(|error| { + log::warn!( + "Planned mediator '{}' parse failed: {:?}", + mediator.provider_name(), + error + ); + }) + .ok() + } + } + Err(error) => { + log::warn!( + "Planned mediator '{}' request failed: {:?}", + mediator.provider_name(), + error + ); + None + } + }, + Err(error) => { + log::warn!( + "Planned mediator '{}' failed to launch: {:?}", + mediator.provider_name(), + error + ); + None + } + }; + if let Some(mediated) = mediated { + let winners = mediated + .bids + .iter() + .filter_map(|bid| { + if bid.price.is_none() { + log::warn!( + "Planned mediator returned a bid without a decoded price" + ); + None + } else { + Some((bid.slot_id.clone(), bid.clone())) + } + }) + .collect(); + ( + Some(mediated), + helper.apply_floor_prices(winners, &floor_prices), + ) + } else { + (None, local_winners()) + } + } + } else { + (None, local_winners()) + }; + let unroutable_bidder_count = routed.diagnostics().unroutable_bidder_count(); + // lgtm[rust/cleartext-logging] + // This logs only a bounded routing count, never request data or secrets. + log::info!( + "Auction routing diagnostics: unroutable_bidder_count={}", + unroutable_bidder_count + ); + Ok(OrchestrationResult { + provider_responses: responses, + mediator_response, + winning_bids, + total_time_ms: auction_start.elapsed().as_millis() as u64, + metadata: routing_metadata(unroutable_bidder_count), + }) + } +} + impl AuctionOrchestrator { - /// Create a new orchestrator with the given configuration. + /// Create a legacy orchestrator for parity tests. + #[cfg(test)] #[must_use] - pub fn new(config: AuctionConfig) -> Self { + pub(crate) fn new(config: AuctionConfig) -> Self { + let plan = Arc::new( + AuctionPlan::compile(super::plan::AuctionPlanConfig { + timeout_ms: config.timeout_ms, + providers: std::collections::BTreeMap::new(), + bidders: std::collections::BTreeMap::new(), + mediator: None, + request_signing: None, + }) + .expect("should compile empty legacy test plan") + .with_enabled(config.enabled), + ); Self { + enabled: config.enabled, + plan_backed: false, config, + plan, + planned_providers: Vec::new(), + mediator: None, + providers: HashMap::new(), + } + } + + /// Create the live orchestrator from one shared compiled auction plan. + #[must_use] + pub fn from_plan(plan: Arc, mediator: Option>) -> Self { + let planned_providers = plan + .providers() + .iter() + .cloned() + .map(GenericOpenRtbProvider::new) + .map(Arc::new) + .collect(); + Self { + enabled: plan.enabled(), + plan_backed: true, + plan, + planned_providers, + mediator, + #[cfg(test)] + config: AuctionConfig::default(), + #[cfg(test)] providers: HashMap::new(), } } - /// Register an auction provider. - pub fn register_provider(&mut self, provider: Arc) { + /// Return whether this orchestrator and another plan consumer share the same plan allocation. + #[must_use] + pub fn shares_plan(&self, plan: &Arc) -> bool { + Arc::ptr_eq(&self.plan, plan) + } + + /// Register an auction provider in the legacy parity harness. + #[cfg(test)] + pub(crate) fn register_provider(&mut self, provider: Arc) { let name = provider.provider_name().to_string(); log::info!("Registering auction provider: {}", name); self.providers.insert(name, provider); @@ -227,61 +845,44 @@ impl AuctionOrchestrator { /// Get the number of registered providers. #[must_use] pub fn provider_count(&self) -> usize { - self.providers.len() + self.planned_providers.len() } - /// Validate that every configured provider name has an enabled provider integration. - pub(crate) fn validate_configured_provider_names( + async fn run_planned_auction( &self, - ) -> Result<(), Report> { - if !self.config.enabled { - return Ok(()); - } - - let mut configured_providers = HashSet::new(); - for provider_name in &self.config.providers { - if !configured_providers.insert(provider_name.as_str()) { - return Err(Report::new(TrustedServerError::Configuration { - message: format!( - "Auction provider `{provider_name}` is listed more than once in [auction].providers; each provider may appear at most once" - ), - })); + request: &AuctionRequest, + context: &AuctionContext<'_>, + ) -> Result> { + match self.dispatch_auction(request, context).await { + DispatchAuctionOutcome::Dispatched(dispatched) => Ok(self + .collect_dispatched_auction(dispatched, context.services, context) + .await), + DispatchAuctionOutcome::DispatchFailed { + fatal_admission_error, + .. + } => { + if let Some(error) = fatal_admission_error { + return Err(error.change_context(TrustedServerError::Auction { + message: "Planned auction admission failed".to_string(), + })); + } + Err(Report::new(TrustedServerError::Auction { + message: "All eligible planned provider requests failed to launch".to_string(), + })) } - } - - if let Some(mediator_name) = &self.config.mediator - && configured_providers.contains(mediator_name.as_str()) - { - return Err(Report::new(TrustedServerError::Configuration { - message: format!( - "Auction mediator `{mediator_name}` is also listed in [auction].providers; a provider may not mediate its own auction" - ), - })); - } - - for provider_name in self - .config - .providers - .iter() - .chain(self.config.mediator.iter()) - { - if !self.providers.contains_key(provider_name) { - return Err(Report::new(TrustedServerError::Configuration { - message: format!( - "Auction provider `{provider_name}` is listed in [auction] but no enabled integration provides it" - ), - })); + DispatchAuctionOutcome::NotStarted => { + if self.planned_providers.is_empty() { + Ok(OrchestrationResult::no_bid()) + } else { + Err(Report::new(TrustedServerError::Auction { + message: "No planned provider request was started".to_string(), + })) + } } } - - Ok(()) } - /// Execute an auction using the auto-detected strategy. - /// - /// Strategy is determined by mediator configuration: - /// - If mediator is configured: runs parallel mediation (bidders → mediator decides) - /// - If no mediator: runs parallel only (bidders → highest CPM wins) + /// Execute an auction through the compiled plan. /// /// # Errors /// @@ -292,9 +893,20 @@ impl AuctionOrchestrator { request: &AuctionRequest, context: &AuctionContext<'_>, ) -> Result> { + if !self.enabled { + return Ok(OrchestrationResult::no_bid()); + } + #[cfg(not(test))] + return self.run_planned_auction(request, context).await; + #[cfg(test)] + if self.plan_backed { + return self.run_planned_auction(request, context).await; + } + #[cfg(test)] let start_time = Instant::now(); - // Auto-detect strategy based on mediator configuration + // Auto-detect strategy based on mediator configuration. + #[cfg(test)] let (strategy_name, result) = if self.config.has_mediator() { ( "parallel_mediation", @@ -307,11 +919,13 @@ impl AuctionOrchestrator { ) }; + #[cfg(test)] log::info!( "Running auction with strategy: {} (auto-detected from mediator config)", strategy_name ); + #[cfg(test)] Ok(OrchestrationResult { total_time_ms: start_time.elapsed().as_millis() as u64, ..result @@ -319,6 +933,7 @@ impl AuctionOrchestrator { } /// Run auction with parallel bidding + mediation. + #[cfg(test)] /// /// Flow: /// 1. Run all bidders in parallel @@ -368,6 +983,7 @@ impl AuctionOrchestrator { settings: context.settings, request: context.request, timeout_ms: mediator_timeout, + transport_timeout_ms: mediator_timeout, provider_responses: Some(&provider_responses), services: context.services, }; @@ -395,12 +1011,30 @@ impl AuctionOrchestrator { mediator.provider_name() ), })?; - - mediator - .parse_response_with_context_and_state( - platform_resp, - start_time.elapsed().as_millis() as u64, - request, + let response_time_ms = start_time.elapsed().as_millis() as u64; + if AuctionDeadlinePolicy::for_runtime(context.services) + .rejects_late_completion(mediation_start, context.timeout_ms) + { + log::warn!( + "Mediator '{}' completed after the hard auction deadline; using local ranking ({}ms)", + mediator.provider_name(), + response_time_ms + ); + let winning = self.select_winning_bids(&provider_responses, &floor_prices); + return Ok(OrchestrationResult { + provider_responses, + mediator_response: None, + winning_bids: winning, + total_time_ms: 0, + metadata: HashMap::new(), + }); + } + + mediator + .parse_response_with_context_and_state( + platform_resp, + response_time_ms, + request, &mediator_context, parse_state.as_deref(), ) @@ -449,6 +1083,7 @@ impl AuctionOrchestrator { } /// Run auction with only parallel bidding (no mediation). + #[cfg(test)] async fn run_parallel_only( &self, request: &AuctionRequest, @@ -468,6 +1103,7 @@ impl AuctionOrchestrator { } /// Run all providers in parallel and collect responses. + #[cfg(test)] /// /// Uses `PlatformHttpClient::select()` to process responses as they /// become ready, rather than waiting for each response sequentially. @@ -476,7 +1112,12 @@ impl AuctionOrchestrator { request: &AuctionRequest, context: &AuctionContext<'_>, ) -> Result, Report> { - let provider_names = self.config.provider_names(); + let provider_names = self + .config + .providers + .keys() + .map(super::plan::ProviderId::as_str) + .collect::>(); if provider_names.is_empty() { return Err(Report::new(TrustedServerError::Auction { @@ -515,10 +1156,12 @@ impl AuctionOrchestrator { let mut responses = Vec::new(); let mut immediate_response_count = 0usize; - for provider_name in provider_names { - let provider = match self.providers.get(provider_name) { + for provider_name in &provider_names { + let provider = match self.providers.get(*provider_name) { Some(p) => p, None => { + // lgtm[rust/cleartext-logging] + // This logs a configured provider identifier, not request data or secrets. log::warn!("Provider '{}' not registered, skipping", provider_name); continue; } @@ -568,10 +1211,13 @@ impl AuctionOrchestrator { settings: context.settings, request: context.request, timeout_ms: effective_timeout, + transport_timeout_ms: effective_timeout, provider_responses: context.provider_responses, services: context.services, }; + // lgtm[rust/cleartext-logging] + // This logs a configured provider identifier and timeout, not request data or secrets. log::info!( "Launching bid request to '{}' with a {}ms budget", provider.provider_name(), @@ -671,7 +1317,7 @@ impl AuctionOrchestrator { })); } - let deadline = Duration::from_millis(u64::from(context.timeout_ms)); + let deadline_policy = AuctionDeadlinePolicy::for_runtime(context.services); log::info!( "Launched {} concurrent provider request(s); waiting for responses", pending_requests.len() @@ -715,10 +1361,20 @@ impl AuctionOrchestrator { if let Some(state) = backend_to_provider.remove(&backend_name) { let response_time_ms = state.started_at.elapsed().as_millis() as u64; + if deadline_policy + .rejects_late_completion(auction_start, context.timeout_ms) + { + responses.push(provider_timeout_response( + &state.provider_name, + response_time_ms, + )); + continue; + } let provider_context = AuctionContext { settings: context.settings, request: context.request, timeout_ms: state.effective_timeout_ms, + transport_timeout_ms: state.effective_timeout_ms, provider_responses: context.provider_responses, services: context.services, }; @@ -796,16 +1452,10 @@ impl AuctionOrchestrator { } } - // Check auction deadline after processing each response. - // Remaining PendingRequests are dropped, which abandons the - // in-flight HTTP calls on the Fastly host. - if auction_start.elapsed() >= deadline && !remaining.is_empty() { - log::warn!( - "Auction timeout reached; dropping {} remaining request(s)", - remaining.len() - ); - break; - } + // Current adapters cannot enforce a hard total-request deadline, so + // drain already-launched handles and retain completed late responses. + // A future adapter that explicitly claims the capability classifies + // each late completion as a timeout instead. } for state in backend_to_provider.into_values() { @@ -920,6 +1570,7 @@ impl AuctionOrchestrator { } /// Get a provider by name. + #[cfg(test)] fn get_provider( &self, name: &str, @@ -936,6 +1587,254 @@ impl AuctionOrchestrator { }) } + async fn dispatch_planned_auction( + &self, + request: &AuctionRequest, + context: &AuctionContext<'_>, + ) -> DispatchAuctionOutcome { + let plan = &self.plan; + if self.planned_providers.is_empty() { + return DispatchAuctionOutcome::NotStarted; + } + if self.planned_providers.len() > 1 + && !context.services.http_client().supports_concurrent_fanout() + { + log::warn!( + "{} planned auction providers configured on a runtime without concurrent fanout", + self.planned_providers.len() + ); + return DispatchAuctionOutcome::NotStarted; + } + + let auction_start = Instant::now(); + let routed = route_auction( + request.clone(), + context.request, + plan, + context.services.client_info().client_ip, + ); + let planned_unused_bidder_params = routed + .inputs() + .iter() + .map(|input| { + ( + input.provider_id().as_str().to_string(), + unused_bidder_params_count( + &plan + .provider(input.provider_id()) + .expect("should find routed provider in compiled plan") + .profile, + input, + ), + ) + }) + .collect::>(); + let planned_unroutable_bidder_count = routed.diagnostics().unroutable_bidder_count(); + // A zero request budget is terminal before signing admission. In the + // split path, signer initialization would otherwise read config and + // secret stores even though every provider is materialized as timeout. + let signer_result = if context.timeout_ms == 0 { + Ok(None) + } else { + plan.signing_enabled() + .then(|| RequestSigner::from_services(context.services)) + .transpose() + }; + let signer = match signer_result { + Ok(signer) => signer, + Err(error) => { + log::warn!("Planned auction signer initialization failed: {error:?}"); + let mut provider_responses = routed + .skipped_no_eligible_provider_ids() + .iter() + .map(|id| provider_skipped_response(id.as_str())) + .chain(routed.inputs().iter().map(|input| { + materialize_planned_response( + provider_launch_failed_response(input.provider_id().as_str(), 0), + unused_bidder_params_count( + &plan + .provider(input.provider_id()) + .expect("should find routed provider in compiled plan") + .profile, + input, + ), + ) + })) + .collect::>(); + let provider_order = plan + .providers() + .iter() + .enumerate() + .map(|(index, provider)| (provider.id.as_str(), index)) + .collect::>(); + provider_responses.sort_by_key(|response| { + provider_order + .get(response.provider.as_str()) + .copied() + .unwrap_or(usize::MAX) + }); + return DispatchAuctionOutcome::DispatchFailed { + request: request.clone(), + provider_responses, + fatal_admission_error: Some(error), + metadata: routing_metadata(planned_unroutable_bidder_count), + elapsed_ms: auction_start.elapsed().as_millis() as u64, + }; + } + }; + let mut completed_responses = routed + .skipped_no_eligible_provider_ids() + .iter() + .map(|id| provider_skipped_response(id.as_str())) + .collect::>(); + let planned_provider_order = plan + .providers() + .iter() + .enumerate() + .map(|(index, provider)| (provider.id.as_str().to_string(), index)) + .collect::>(); + let mut pending_requests = Vec::new(); + let mut planned_backend_to_provider = HashMap::new(); + let mut reserved_backend_names = HashSet::new(); + let mut immediate_response_count = 0usize; + let mut launch_failure_count = 0usize; + + for input in routed.inputs() { + let Some(provider) = self + .planned_providers + .iter() + .find(|provider| provider.provider_name() == input.provider_id().as_str()) + .cloned() + else { + launch_failure_count += 1; + completed_responses.push(provider_launch_failed_response( + input.provider_id().as_str(), + 0, + )); + continue; + }; + let logical_budget_ms = + remaining_budget_ms(auction_start, context.timeout_ms).min(provider.timeout_ms()); + if logical_budget_ms == 0 { + completed_responses.push(provider_timeout_response(provider.provider_name(), 0)); + continue; + } + let transport_timeout_ms = context + .services + .backend() + .canonicalize_transport_timeout_ms(logical_budget_ms, provider.timeout_ms()); + if transport_timeout_ms == 0 { + completed_responses.push(provider_timeout_response(provider.provider_name(), 0)); + continue; + } + let started_at = Instant::now(); + match provider + .request_bids_routed( + input, + &routed, + logical_budget_ms, + transport_timeout_ms, + signer.as_ref(), + context.services, + &mut reserved_backend_names, + ) + .await + { + Ok(ProviderRequestOutcome::Pending { + request: pending, + parse_state, + }) => { + let Some(backend_name) = pending.backend_name().map(str::to_string) else { + launch_failure_count += 1; + completed_responses.push(provider_launch_failed_response( + provider.provider_name(), + started_at.elapsed().as_millis() as u64, + )); + continue; + }; + match planned_backend_to_provider.entry(backend_name.clone()) { + Entry::Vacant(entry) => { + entry.insert(PlannedLaunchState { + provider, + started_at, + parse_state, + }); + pending_requests.push(pending.with_backend_name(backend_name)); + } + Entry::Occupied(_) => { + launch_failure_count += 1; + completed_responses.push(provider_launch_failed_response( + provider.provider_name(), + started_at.elapsed().as_millis() as u64, + )) + } + } + } + Ok(ProviderRequestOutcome::Immediate(response)) => { + immediate_response_count += 1; + completed_responses.push(response); + } + Err(error) => { + log::warn!( + "Planned provider '{}' failed to dispatch: {error:?}", + provider.provider_name() + ); + launch_failure_count += 1; + completed_responses.push(provider_launch_failed_response( + provider.provider_name(), + started_at.elapsed().as_millis() as u64, + )); + } + } + } + + if pending_requests.is_empty() && immediate_response_count == 0 { + if launch_failure_count > 0 && launch_failure_count == routed.inputs().len() { + for response in &mut completed_responses { + if let Some(&count) = + planned_unused_bidder_params.get(response.provider.as_str()) + { + *response = materialize_planned_response(response.clone(), count); + } + } + completed_responses.sort_by_key(|response| { + planned_provider_order + .get(response.provider.as_str()) + .copied() + .unwrap_or(usize::MAX) + }); + return DispatchAuctionOutcome::DispatchFailed { + request: request.clone(), + provider_responses: completed_responses, + fatal_admission_error: None, + metadata: routing_metadata(planned_unroutable_bidder_count), + elapsed_ms: auction_start.elapsed().as_millis() as u64, + }; + } + if routed.inputs().is_empty() && completed_responses.is_empty() { + return DispatchAuctionOutcome::NotStarted; + } + } + + DispatchAuctionOutcome::Dispatched(DispatchedAuction { + pending_requests, + backend_to_provider: HashMap::new(), + planned_backend_to_provider, + completed_responses, + auction_start, + timeout_ms: context.timeout_ms, + floor_prices: self.floor_prices_by_slot(request), + // Planned providers carry their typed parse state, so collection + // does not need to retain the inbound client request. Keep the + // explicit request boundary empty across the origin wait. + provider_request_context: Box::new(Request::new(EdgeBody::empty())), + request: request.clone(), + planned_unused_bidder_params, + planned_unroutable_bidder_count, + planned_provider_order, + }) + } + /// Dispatch SSP bid requests without blocking WASM. /// /// Calls each enabled provider's [`AuctionProvider::request_bids`] (which @@ -953,7 +1852,20 @@ impl AuctionOrchestrator { request: &AuctionRequest, context: &AuctionContext<'_>, ) -> DispatchAuctionOutcome { - let provider_names = self.config.provider_names(); + if !self.enabled { + return DispatchAuctionOutcome::NotStarted; + } + if !cfg!(test) || self.plan_backed { + return self.dispatch_planned_auction(request, context).await; + } + #[cfg(test)] + let provider_names = self + .config + .providers + .keys() + .map(super::plan::ProviderId::as_str) + .collect::>(); + #[cfg(test)] if provider_names.is_empty() { return DispatchAuctionOutcome::NotStarted; } @@ -963,6 +1875,7 @@ impl AuctionOrchestrator { // (e.g. Cloudflare Workers, Spin). Sequential execution would accrue // the sum of provider latencies before the origin fetch and then fail // collection with empty bids. + #[cfg(test)] if provider_names.len() > 1 && !context.services.http_client().supports_concurrent_fanout() { log::warn!( @@ -976,13 +1889,26 @@ impl AuctionOrchestrator { } let auction_start = Instant::now(); + #[cfg(test)] let mut backend_to_provider: HashMap = HashMap::new(); + #[cfg(not(test))] + let backend_to_provider: HashMap = HashMap::new(); + #[cfg(test)] let mut pending_requests: Vec = Vec::new(); + #[cfg(not(test))] + let pending_requests: Vec = Vec::new(); + #[cfg(test)] let mut completed_responses: Vec = Vec::new(); + #[cfg(not(test))] + let completed_responses: Vec = Vec::new(); + #[cfg(test)] let mut immediate_response_count = 0usize; + #[cfg(not(test))] + let immediate_response_count = 0usize; - for provider_name in provider_names { - let provider = match self.providers.get(provider_name) { + #[cfg(test)] + for provider_name in &provider_names { + let provider = match self.providers.get(*provider_name) { Some(p) => p, None => { // lgtm[rust/cleartext-logging] @@ -1009,6 +1935,8 @@ impl AuctionOrchestrator { // Match the synchronous path's strict deadline semantics: do not // invoke even an immediate provider after the budget reaches zero. if effective_timeout == 0 { + // lgtm[rust/cleartext-logging] + // This logs a configured provider identifier and timeout, not request data or secrets. log::warn!( "Auction timeout ({}ms) exhausted before launching '{}' — skipping", context.timeout_ms, @@ -1038,6 +1966,7 @@ impl AuctionOrchestrator { settings: context.settings, request: context.request, timeout_ms: effective_timeout, + transport_timeout_ms: effective_timeout, provider_responses: context.provider_responses, services: context.services, }; @@ -1081,6 +2010,8 @@ impl AuctionOrchestrator { )); continue; } + // lgtm[rust/cleartext-logging] + // This logs configured provider and backend identifiers plus a timeout, not request data or secrets. log::info!( "Dispatching bid request to '{}' (backend: {}, budget: {}ms)", provider.provider_name(), @@ -1125,11 +2056,15 @@ impl AuctionOrchestrator { DispatchAuctionOutcome::DispatchFailed { request: request.clone(), provider_responses: completed_responses, + fatal_admission_error: None, + metadata: HashMap::new(), elapsed_ms: auction_start.elapsed().as_millis() as u64, } }; } + // lgtm[rust/cleartext-logging] + // This logs bounded request counts and a timeout, not request data or secrets. log::info!( "Dispatched {} SSP request(s) with {} immediate response(s) (timeout: {}ms)", pending_requests.len(), @@ -1140,12 +2075,16 @@ impl AuctionOrchestrator { DispatchAuctionOutcome::Dispatched(DispatchedAuction { pending_requests, backend_to_provider, + planned_backend_to_provider: HashMap::new(), completed_responses, auction_start, timeout_ms: context.timeout_ms, floor_prices: self.floor_prices_by_slot(request), provider_request_context: Box::new(snapshot_context_request(context.request)), request: request.clone(), + planned_unused_bidder_params: HashMap::new(), + planned_unroutable_bidder_count: 0, + planned_provider_order: HashMap::new(), }) } @@ -1168,12 +2107,16 @@ impl AuctionOrchestrator { let DispatchedAuction { pending_requests, mut backend_to_provider, + mut planned_backend_to_provider, completed_responses, auction_start, timeout_ms, floor_prices, provider_request_context, request, + planned_unused_bidder_params, + planned_unroutable_bidder_count, + planned_provider_order, } = dispatched; log::info!( @@ -1185,6 +2128,7 @@ impl AuctionOrchestrator { let mut responses: Vec = completed_responses; let mut remaining = pending_requests; + let deadline_policy = AuctionDeadlinePolicy::for_runtime(services); while !remaining.is_empty() { let select_result = match services @@ -1197,6 +2141,30 @@ impl AuctionOrchestrator { Ok(r) => r, Err(e) => { log::warn!("select() failed during auction collection: {:?}", e); + // An outer select failure means the platform could not poll + // any outstanding handle. Attribute every tracked launch as + // a transport failure rather than later relabeling it as a + // timeout. Drain through a sorted buffer because HashMap + // iteration order is intentionally nondeterministic. + let mut transport_failures = backend_to_provider + .drain() + .map(|(_, state)| { + let response_time_ms = state.started_at.elapsed().as_millis() as u64; + provider_transport_failed_response( + &state.provider_name, + response_time_ms, + ) + }) + .chain(planned_backend_to_provider.drain().map(|(_, state)| { + let response_time_ms = state.started_at.elapsed().as_millis() as u64; + provider_transport_failed_response( + state.provider.provider_name(), + response_time_ms, + ) + })) + .collect::>(); + transport_failures.sort_by(|left, right| left.provider.cmp(&right.provider)); + responses.extend(transport_failures); break; } }; @@ -1214,10 +2182,18 @@ impl AuctionOrchestrator { let backend_name = platform_response.backend_name.clone().unwrap_or_default(); if let Some(state) = backend_to_provider.remove(&backend_name) { let response_time_ms = state.started_at.elapsed().as_millis() as u64; + if deadline_policy.rejects_late_completion(auction_start, timeout_ms) { + responses.push(provider_timeout_response( + &state.provider_name, + response_time_ms, + )); + continue; + } let provider_context = AuctionContext { settings: context.settings, request: &provider_request_context, timeout_ms: state.effective_timeout_ms, + transport_timeout_ms: state.effective_timeout_ms, provider_responses: context.provider_responses, services: context.services, }; @@ -1232,28 +2208,39 @@ impl AuctionOrchestrator { ) .await { - Ok(auction_response) => { - log::info!( - "Provider '{}' returned {} bids ({}ms)", - auction_response.provider, - auction_response.bids.len(), - auction_response.response_time_ms - ); - responses.push(auction_response); - } - Err(e) => { - log::warn!( - "Provider '{}' parse failed: {:?}", - state.provider_name, - e - ); - responses.push(provider_error_response( - &state.provider_name, - response_time_ms, - ERROR_TYPE_PARSE_RESPONSE, - &e, - )); - } + Ok(auction_response) => responses.push(auction_response), + Err(error) => responses.push(provider_error_response( + &state.provider_name, + response_time_ms, + ERROR_TYPE_PARSE_RESPONSE, + &error, + )), + } + } else if let Some(state) = planned_backend_to_provider.remove(&backend_name) { + let response_time_ms = state.started_at.elapsed().as_millis() as u64; + if deadline_policy.rejects_late_completion(auction_start, timeout_ms) { + responses.push(provider_timeout_response( + state.provider.provider_name(), + response_time_ms, + )); + continue; + } + match state + .provider + .parse_response_with_state( + platform_response, + response_time_ms, + state.parse_state.as_deref(), + ) + .await + { + Ok(response) => responses.push(response), + Err(error) => responses.push(provider_error_response( + state.provider.provider_name(), + response_time_ms, + ERROR_TYPE_PARSE_RESPONSE, + &error, + )), } } else { log::warn!( @@ -1278,6 +2265,18 @@ impl AuctionOrchestrator { &state.provider_name, response_time_ms, )); + } else if let Some(state) = planned_backend_to_provider.remove(backend_name) + { + let response_time_ms = state.started_at.elapsed().as_millis() as u64; + log::warn!( + "Planned provider '{}' request failed: {:?}", + state.provider.provider_name(), + e + ); + responses.push(provider_transport_failed_response( + state.provider.provider_name(), + response_time_ms, + )); } else { log::warn!( "A provider request failed (backend '{}' not tracked): {:?}", @@ -1315,79 +2314,138 @@ impl AuctionOrchestrator { )); } backend_to_provider.clear(); + for state in planned_backend_to_provider.into_values() { + responses.push(provider_timeout_response( + state.provider.provider_name(), + state.started_at.elapsed().as_millis() as u64, + )); + } + for response in &mut responses { + if let Some(&count) = planned_unused_bidder_params.get(response.provider.as_str()) { + *response = materialize_planned_response(response.clone(), count); + } + } + if !planned_provider_order.is_empty() { + responses.sort_by_key(|response| { + planned_provider_order + .get(response.provider.as_str()) + .copied() + .unwrap_or(usize::MAX) + }); + } - let (mediator_response, winning_bids) = if let Some(mediator_name) = &self.config.mediator { - match self.providers.get(mediator_name.as_str()) { - Some(mediator) => { - // Cap the mediator at whichever is tighter: its own configured - // timeout or the remaining auction budget (A_deadline). Backend - // first-byte and between-bytes timeouts bound normal collection, but - // they are transport timers rather than absolute wall-clock limits: - // connection setup and byte-trickling can still consume more of the - // auction budget. Recomputing the remaining budget here prevents the - // mediator from extending that bounded response hold. - let remaining = remaining_budget_ms(auction_start, timeout_ms); - let mediator_timeout = services - .backend() - .canonicalize_transport_timeout_ms(remaining, mediator.timeout_ms()); - if mediator_timeout == 0 { - log::warn!( - "A_deadline exhausted before mediator '{}' — returning {} SSP bids without mediation", - mediator.provider_name(), - responses.len(), - ); - let winning = self.select_winning_bids(&responses, &floor_prices); - return OrchestrationResult { - provider_responses: responses, - mediator_response: None, - winning_bids: winning, - total_time_ms: auction_start.elapsed().as_millis() as u64, - metadata: HashMap::new(), - }; - } - let mediator_start = Instant::now(); - log::info!( - "Running mediator '{}' with {}ms budget (A_deadline remaining: {}ms, configured: {}ms)", + #[cfg(not(test))] + let mediator = self.mediator.as_ref(); + #[cfg(test)] + let mediator = self.mediator.as_ref().or_else(|| { + self.config + .mediator + .as_ref() + .and_then(|name| self.providers.get(name)) + }); + let (mediator_response, winning_bids) = if let Some(mediator) = mediator { + { + // Cap the mediator at whichever is tighter: its own configured + // timeout or the remaining auction budget (A_deadline). Backend + // first-byte and between-bytes timeouts bound normal collection, but + // they are transport timers rather than absolute wall-clock limits: + // connection setup and byte-trickling can still consume more of the + // auction budget. Recomputing the remaining budget here prevents the + // mediator from extending that bounded response hold. + let remaining = remaining_budget_ms(auction_start, timeout_ms); + let logical_budget_ms = remaining.min(mediator.timeout_ms()); + if logical_budget_ms == 0 { + log::warn!( + "A_deadline exhausted before mediator '{}' — returning {} SSP bids without mediation", mediator.provider_name(), - mediator_timeout, - remaining, - mediator.timeout_ms(), + responses.len(), ); - // The mediator runs on the collect path. See the doc-comment on - // `AuctionContext::request`: the real client request was already - // consumed by `send_async` during dispatch, so we substitute a - // canonical placeholder URL. Any future mediator that needs real - // client headers must snapshot them at dispatch time onto - // `DispatchedAuction` rather than reading `context.request` here. - let placeholder = http::Request::builder() - .uri(crate::auction::types::MEDIATOR_PLACEHOLDER_URL) - .body(edgezero_core::body::Body::empty()) - .unwrap_or_else(|_| http::Request::new(edgezero_core::body::Body::empty())); - let mediator_context = AuctionContext { - settings: context.settings, - request: &placeholder, - timeout_ms: mediator_timeout, - provider_responses: Some(&responses), - services: context.services, + let winning = self.select_winning_bids(&responses, &floor_prices); + return OrchestrationResult { + provider_responses: responses, + mediator_response: None, + winning_bids: winning, + total_time_ms: auction_start.elapsed().as_millis() as u64, + metadata: routing_metadata(planned_unroutable_bidder_count), }; - let mediator_response = - match mediator.request_bids(&request, &mediator_context).await { - Ok(ProviderRequestOutcome::Immediate(response)) => Some(response), - Ok(ProviderRequestOutcome::Pending { - request: pending, - parse_state, - }) => match services.http_client().wait(pending).await.change_context( - TrustedServerError::Auction { - message: format!( - "Mediator {} request failed", - mediator.provider_name() - ), - }, - ) { - Ok(platform_resp) => match mediator + } + let transport_timeout_ms = services + .backend() + .canonicalize_transport_timeout_ms(logical_budget_ms, mediator.timeout_ms()); + if transport_timeout_ms == 0 { + log::warn!( + "Mediator '{}' transport budget canonicalized to zero — returning {} SSP bids without mediation", + mediator.provider_name(), + responses.len(), + ); + let winning = self.select_winning_bids(&responses, &floor_prices); + return OrchestrationResult { + provider_responses: responses, + mediator_response: None, + winning_bids: winning, + total_time_ms: auction_start.elapsed().as_millis() as u64, + metadata: routing_metadata(planned_unroutable_bidder_count), + }; + } + let mediator_start = Instant::now(); + // lgtm[rust/cleartext-logging] + // This logs a configured mediator identifier and timeout values, not request data or secrets. + log::info!( + "Running mediator '{}' with {}ms logical budget and {}ms transport timeout (A_deadline remaining: {}ms, configured: {}ms)", + mediator.provider_name(), + logical_budget_ms, + transport_timeout_ms, + remaining, + mediator.timeout_ms(), + ); + // The mediator runs on the collect path. See the doc-comment on + // `AuctionContext::request`: the real client request was already + // consumed by `send_async` during dispatch, so we substitute a + // canonical placeholder URL. Any future mediator that needs real + // client headers must snapshot them at dispatch time onto + // `DispatchedAuction` rather than reading `context.request` here. + let placeholder = http::Request::builder() + .uri(crate::auction::types::MEDIATOR_PLACEHOLDER_URL) + .body(edgezero_core::body::Body::empty()) + .unwrap_or_else(|_| http::Request::new(edgezero_core::body::Body::empty())); + let mediator_context = AuctionContext { + settings: context.settings, + request: &placeholder, + timeout_ms: logical_budget_ms, + transport_timeout_ms, + provider_responses: Some(&responses), + services: context.services, + }; + let mediator_response = match mediator + .request_bids(&request, &mediator_context) + .await + { + Ok(ProviderRequestOutcome::Immediate(response)) => Some(response), + Ok(ProviderRequestOutcome::Pending { + request: pending, + parse_state, + }) => match services.http_client().wait(pending).await.change_context( + TrustedServerError::Auction { + message: format!( + "Mediator {} request failed", + mediator.provider_name() + ), + }, + ) { + Ok(platform_resp) => { + let response_time_ms = mediator_start.elapsed().as_millis() as u64; + if deadline_policy.rejects_late_completion(auction_start, timeout_ms) { + log::warn!( + "Mediator '{}' completed after the hard auction deadline; using local ranking ({}ms)", + mediator.provider_name(), + response_time_ms + ); + None + } else { + match mediator .parse_response_with_context_and_state( platform_resp, - mediator_start.elapsed().as_millis() as u64, + response_time_ms, &request, &mediator_context, parse_state.as_deref(), @@ -1403,24 +2461,26 @@ impl AuctionOrchestrator { ); None } - }, - Err(error) => { - log::warn!("Mediator request failed: {:?}", error); - None } - }, - Err(error) => { - log::warn!( - "Mediator '{}' failed to dispatch: {:?}", - mediator.provider_name(), - error - ); - None } - }; + } + Err(error) => { + log::warn!("Mediator request failed: {:?}", error); + None + } + }, + Err(error) => { + log::warn!( + "Mediator '{}' failed to dispatch: {:?}", + mediator.provider_name(), + error + ); + None + } + }; - if let Some(mediator_response) = mediator_response { - let winning = mediator_response + if let Some(mediator_response) = mediator_response { + let winning = mediator_response .bids .iter() .filter_map(|bid| { @@ -1436,16 +2496,9 @@ impl AuctionOrchestrator { } }) .collect(); - let winning = self.apply_floor_prices(winning, &floor_prices); - (Some(mediator_response), winning) - } else { - (None, self.select_winning_bids(&responses, &floor_prices)) - } - } - None => { - // lgtm[rust/cleartext-logging] - // The mediator name is a static config identifier, not a secret. - log::warn!("Mediator '{}' not registered", mediator_name); + let winning = self.apply_floor_prices(winning, &floor_prices); + (Some(mediator_response), winning) + } else { (None, self.select_winning_bids(&responses, &floor_prices)) } } @@ -1458,14 +2511,14 @@ impl AuctionOrchestrator { mediator_response, winning_bids, total_time_ms: auction_start.elapsed().as_millis() as u64, - metadata: HashMap::new(), + metadata: routing_metadata(planned_unroutable_bidder_count), } } /// Check if orchestrator is enabled. #[must_use] pub fn is_enabled(&self) -> bool { - self.config.enabled + self.enabled } } @@ -1485,6 +2538,16 @@ pub struct OrchestrationResult { } impl OrchestrationResult { + fn no_bid() -> Self { + Self { + provider_responses: Vec::new(), + mediator_response: None, + winning_bids: HashMap::new(), + total_time_ms: 0, + metadata: HashMap::new(), + } + } + /// Get the winning bid for a specific slot. #[must_use] pub fn get_winning_bid(&self, slot_id: &str) -> Option<&Bid> { @@ -1510,32 +2573,638 @@ impl OrchestrationResult { #[cfg(test)] mod tests { + use std::str::FromStr as _; use std::time::Duration; + + use base64::Engine as _; use web_time::Instant; use crate::auction::config::AuctionConfig; use crate::auction::orchestrator::DispatchAuctionOutcome; - use crate::auction::provider::{AuctionProvider, ProviderRequestOutcome}; + use crate::auction::plan::{ + AuctionPlan, AuctionPlanConfig, NotificationConfig, ProviderConfig, ProviderId, RoutingMode, + }; + use crate::auction::provider::{ + AuctionProvider, GenericOpenRtbProvider, ProviderRequestOutcome, + }; + use crate::auction::routing::{RoutingDiagnostics, route_auction}; use crate::auction::test_support::create_test_auction_context; use crate::auction::types::{ AdFormat, AdSlot, ApsRendererV1, ApsTagType, AuctionContext, AuctionRequest, AuctionResponse, Bid, BidRenderer, BidStatus, MediaType, PublisherInfo, UserInfo, }; use crate::error::TrustedServerError; + use crate::integrations::adserver_mock::{AdServerMockConfig, AdServerMockProvider}; use crate::platform::test_support::{ StubHttpClient, build_services_with_backend_and_http_client, build_services_with_http_client, noop_services, }; use crate::platform::{ - PlatformBackend, PlatformBackendSpec, PlatformError, PlatformHttpRequest, PlatformResponse, - RuntimeServices, + BackendNamingPolicy, PlatformBackend, PlatformBackendSpec, PlatformConfigStore, + PlatformError, PlatformHttpClient, PlatformHttpRequest, PlatformPendingRequest, + PlatformResponse, PlatformSecretStore, PlatformSelectResult, RuntimeServices, StoreId, + StoreName, }; use crate::test_support::tests::crate_test_settings_str; use error_stack::{Report, ResultExt}; - use std::collections::{HashMap, HashSet}; + use std::collections::{BTreeMap, HashMap, HashSet}; + use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::{Arc, Mutex}; - use super::AuctionOrchestrator; + use super::{ + AuctionOrchestrator, AuctionOrchestratorHarness, DispatchedAuction, + ERROR_TYPE_LAUNCH_FAILED, ERROR_TYPE_TIMEOUT, ERROR_TYPE_TRANSPORT, OrchestrationResult, + }; + + fn planned_config(providers: &[(&str, RoutingMode)], signing: bool) -> AuctionPlanConfig { + AuctionPlanConfig { + timeout_ms: 777, + providers: providers + .iter() + .map(|(id, routing)| { + ( + ProviderId::from_str(id).expect("should parse fictional provider ID"), + ProviderConfig { + protocol: "openrtb-2.6".to_string(), + profile: "standard".to_string(), + endpoint: "https://example.test/openrtb".to_string(), + timeout_ms: Some(1_000), + routing: *routing, + notifications: Default::default(), + profile_config: serde_json::json!({}), + }, + ) + }) + .collect(), + bidders: BTreeMap::new(), + mediator: None, + request_signing: signing.then(|| crate::settings::RequestSigning { + enabled: true, + config_store_id: "fictional-config-store".to_string(), + secret_store_id: "fictional-secret-store".to_string(), + }), + } + } + + fn planned_prebid_config( + providers: &[(&str, serde_json::Value, NotificationConfig)], + ) -> AuctionPlanConfig { + AuctionPlanConfig { + timeout_ms: 777, + providers: providers + .iter() + .map(|(id, profile_config, notifications)| { + ( + ProviderId::from_str(id).expect("should parse fictional provider ID"), + ProviderConfig { + protocol: "openrtb-2.6".to_string(), + profile: "prebid-server".to_string(), + endpoint: format!("https://{id}.example.test/openrtb"), + timeout_ms: Some(1_000), + routing: RoutingMode::Explicit, + notifications: notifications.clone(), + profile_config: profile_config.clone(), + }, + ) + }) + .collect(), + bidders: BTreeMap::new(), + mediator: None, + request_signing: None, + } + } + + fn planned_aps_config() -> AuctionPlanConfig { + planned_aps_instances_config(&[( + "aps-instance", + serde_json::json!({"account_id": "example-account"}), + NotificationConfig::default(), + )]) + } + + fn planned_aps_instances_config( + providers: &[(&str, serde_json::Value, NotificationConfig)], + ) -> AuctionPlanConfig { + AuctionPlanConfig { + timeout_ms: 777, + providers: providers + .iter() + .map(|(id, profile_config, notifications)| { + ( + ProviderId::from_str(id).expect("should parse fictional provider ID"), + ProviderConfig { + protocol: "openrtb-2.6".to_string(), + profile: "aps".to_string(), + endpoint: "https://aps.example/e/pb/bid".to_string(), + timeout_ms: Some(1_000), + routing: RoutingMode::AllEligible, + notifications: notifications.clone(), + profile_config: profile_config.clone(), + }, + ) + }) + .collect(), + bidders: BTreeMap::new(), + mediator: None, + request_signing: None, + } + } + + fn planned_request() -> AuctionRequest { + AuctionRequest { + id: "fictional-auction".to_string(), + slots: vec![AdSlot { + id: "fictional-slot".to_string(), + formats: vec![AdFormat { + media_type: MediaType::Banner, + width: 300, + height: 250, + }], + floor_price: Some(1.0), + targeting: HashMap::new(), + bidders: HashMap::new(), + }], + publisher: PublisherInfo { + domain: "publisher.example".to_string(), + page_url: Some("https://publisher.example/article".to_string()), + }, + user: UserInfo { + id: None, + consent: None, + eids: None, + }, + device: None, + site: None, + context: HashMap::new(), + } + } + + fn planned_prebid_request() -> AuctionRequest { + let mut request = planned_request(); + request.slots[0] + .bidders + .insert("trustedServer".to_string(), serde_json::json!({})); + request + } + + #[tokio::test] + async fn disabled_from_plan_is_a_no_work_kill_switch_for_sync_and_split_paths() { + let plan = Arc::new( + AuctionPlan::compile(planned_config( + &[("provider-a", RoutingMode::AllEligible)], + false, + )) + .expect("should compile plan") + .with_enabled(false), + ); + let orchestrator = AuctionOrchestrator::from_plan(plan, None); + let http = Arc::new(StubHttpClient::new()); + let services = build_services_with_http_client(Arc::clone(&http) as Arc<_>); + let settings = create_test_settings(); + let inbound = http::Request::new(edgezero_core::body::Body::empty()); + let context = AuctionContext { + settings: &settings, + request: &inbound, + timeout_ms: 777, + transport_timeout_ms: 777, + provider_responses: None, + services: &services, + }; + let request = planned_request(); + + let result = orchestrator + .run_auction(&request, &context) + .await + .expect("disabled auction should complete as no-bid"); + assert!(result.provider_responses.is_empty()); + assert!(result.winning_bids.is_empty()); + assert!(matches!( + orchestrator.dispatch_auction(&request, &context).await, + DispatchAuctionOutcome::NotStarted + )); + assert!(http.recorded_backend_names().is_empty()); + } + + #[tokio::test] + async fn enabled_empty_plan_is_successful_no_bid_without_dispatch() { + let plan = Arc::new( + AuctionPlan::compile(planned_config(&[], false)).expect("should compile empty plan"), + ); + let orchestrator = AuctionOrchestrator::from_plan(plan, None); + let http = Arc::new(StubHttpClient::new()); + let services = build_services_with_http_client(Arc::clone(&http) as Arc<_>); + let settings = create_test_settings(); + let inbound = http::Request::new(edgezero_core::body::Body::empty()); + let context = AuctionContext { + settings: &settings, + request: &inbound, + timeout_ms: 777, + transport_timeout_ms: 777, + provider_responses: None, + services: &services, + }; + let request = planned_request(); + + let result = orchestrator + .run_auction(&request, &context) + .await + .expect("empty auction should complete as no-bid"); + assert!(result.provider_responses.is_empty()); + assert!(result.winning_bids.is_empty()); + assert!(matches!( + orchestrator.dispatch_auction(&request, &context).await, + DispatchAuctionOutcome::NotStarted + )); + assert!(http.recorded_backend_names().is_empty()); + } + + #[tokio::test] + async fn all_planned_launch_failures_error_direct_and_surface_split_failure() { + let plan = Arc::new( + AuctionPlan::compile(planned_config( + &[("launch-fail", RoutingMode::AllEligible)], + false, + )) + .expect("should compile launch-failure plan"), + ); + let orchestrator = AuctionOrchestrator::from_plan(plan, None); + let backend = Arc::new(NamingBackend::new(BackendNamingPolicy::Axum)); + backend.fail_ensure_for("launch-fail"); + let http = Arc::new(StubHttpClient::new()); + let services = build_services_with_backend_and_http_client( + Arc::clone(&backend) as Arc<_>, + Arc::clone(&http) as Arc<_>, + ); + let settings = create_test_settings(); + let inbound = http::Request::new(edgezero_core::body::Body::empty()); + let context = AuctionContext { + settings: &settings, + request: &inbound, + timeout_ms: 777, + transport_timeout_ms: 777, + provider_responses: None, + services: &services, + }; + let request = planned_request(); + + let error = orchestrator + .run_auction(&request, &context) + .await + .expect_err("all planned launch failures should fail direct execution"); + assert!( + error + .to_string() + .contains("All eligible planned provider requests failed to launch"), + "should distinguish launch failure from an ordinary no-bid: {error:?}" + ); + + let DispatchAuctionOutcome::DispatchFailed { + provider_responses, + fatal_admission_error, + .. + } = orchestrator.dispatch_auction(&request, &context).await + else { + panic!("all planned launch failures should surface a split dispatch failure"); + }; + assert!(fatal_admission_error.is_none()); + assert_eq!(provider_responses.len(), 1); + assert_eq!(provider_responses[0].provider, "launch-fail"); + assert_eq!( + provider_responses[0].metadata["error_type"], + ERROR_TYPE_LAUNCH_FAILED + ); + assert!(http.recorded_backend_names().is_empty()); + } + + #[tokio::test] + async fn all_skipped_from_plan_completes_with_routing_metadata_in_sync_and_split_paths() { + for split in [false, true] { + let plan = Arc::new( + AuctionPlan::compile(planned_config(&[("skipped", RoutingMode::Explicit)], false)) + .expect("should compile all-skipped plan"), + ); + let orchestrator = AuctionOrchestrator::from_plan(plan, None); + let http = Arc::new(StubHttpClient::new()); + let services = build_services_with_http_client(Arc::clone(&http) as Arc<_>); + let settings = create_test_settings(); + let inbound = http::Request::new(edgezero_core::body::Body::empty()); + let context = AuctionContext { + settings: &settings, + request: &inbound, + timeout_ms: 777, + transport_timeout_ms: 777, + provider_responses: None, + services: &services, + }; + let request = planned_request(); + + let result = if split { + let DispatchAuctionOutcome::Dispatched(dispatched) = + orchestrator.dispatch_auction(&request, &context).await + else { + panic!("all-skipped auction should produce a completed dispatch token"); + }; + orchestrator + .collect_dispatched_auction(dispatched, &services, &context) + .await + } else { + orchestrator + .run_auction(&request, &context) + .await + .expect("all-skipped auction should complete") + }; + + assert_eq!(result.provider_responses.len(), 1); + assert_eq!( + result.provider_responses[0].metadata["routing"]["skipped_no_eligible_slots"], + true + ); + assert!(result.metadata.contains_key("routing")); + assert!(http.recorded_backend_names().is_empty()); + } + } + + struct NamingBackend { + policy: BackendNamingPolicy, + predicted: AtomicUsize, + ensured: AtomicUsize, + specs: Mutex>, + fail_ensure_for: Mutex>, + } + + impl NamingBackend { + fn new(policy: BackendNamingPolicy) -> Self { + Self { + policy, + predicted: AtomicUsize::new(0), + ensured: AtomicUsize::new(0), + specs: Mutex::new(Vec::new()), + fail_ensure_for: Mutex::new(HashSet::new()), + } + } + + fn fail_ensure_for(&self, provider_id: &str) { + self.fail_ensure_for + .lock() + .expect("should lock failing provider IDs") + .insert(provider_id.to_string()); + } + + fn name(&self, spec: &PlatformBackendSpec) -> Result> { + self.policy + .predict(spec) + .map(|prediction| prediction.name) + .change_context(PlatformError::Backend) + } + } + + impl PlatformBackend for NamingBackend { + fn naming_policy(&self) -> BackendNamingPolicy { + self.policy + } + + fn predict_name( + &self, + spec: &PlatformBackendSpec, + ) -> Result> { + self.predicted.fetch_add(1, Ordering::Relaxed); + self.name(spec) + } + + fn ensure(&self, spec: &PlatformBackendSpec) -> Result> { + self.ensured.fetch_add(1, Ordering::Relaxed); + if spec.discriminator.as_deref().is_some_and(|provider_id| { + self.fail_ensure_for + .lock() + .expect("should lock failing provider IDs") + .contains(provider_id) + }) { + return Err(Report::new(PlatformError::Backend)); + } + self.specs + .lock() + .expect("should lock planned backend specs") + .push(spec.clone()); + self.name(spec) + } + } + + struct ZeroCanonicalBackend { + predicted: AtomicUsize, + ensured: AtomicUsize, + } + + impl ZeroCanonicalBackend { + fn new() -> Self { + Self { + predicted: AtomicUsize::new(0), + ensured: AtomicUsize::new(0), + } + } + } + + impl PlatformBackend for ZeroCanonicalBackend { + fn naming_policy(&self) -> BackendNamingPolicy { + BackendNamingPolicy::Fastly + } + + fn predict_name( + &self, + _spec: &PlatformBackendSpec, + ) -> Result> { + self.predicted.fetch_add(1, Ordering::Relaxed); + Ok("zero-canonical-backend".to_string()) + } + + fn ensure(&self, _spec: &PlatformBackendSpec) -> Result> { + self.ensured.fetch_add(1, Ordering::Relaxed); + Ok("zero-canonical-backend".to_string()) + } + + fn canonicalize_transport_timeout_ms( + &self, + _remaining_ms: u32, + _configured_ms: u32, + ) -> u32 { + 0 + } + } + + struct CollidingBackend; + + impl PlatformBackend for CollidingBackend { + fn naming_policy(&self) -> BackendNamingPolicy { + BackendNamingPolicy::Axum + } + + fn predict_name( + &self, + _spec: &PlatformBackendSpec, + ) -> Result> { + Ok("colliding-backend".to_string()) + } + + fn ensure(&self, _spec: &PlatformBackendSpec) -> Result> { + Ok("colliding-backend".to_string()) + } + } + + struct FailingCountingConfigStore { + reads: AtomicUsize, + } + + impl PlatformConfigStore for FailingCountingConfigStore { + fn get( + &self, + _store_name: &StoreName, + _key: &str, + ) -> Result> { + self.reads.fetch_add(1, Ordering::Relaxed); + Err(Report::new(PlatformError::ConfigStore)) + } + + fn put( + &self, + _store_id: &StoreId, + _key: &str, + _value: &str, + ) -> Result<(), Report> { + Err(Report::new(PlatformError::Unsupported)) + } + + fn delete(&self, _store_id: &StoreId, _key: &str) -> Result<(), Report> { + Err(Report::new(PlatformError::Unsupported)) + } + } + + struct CountingConfigStore { + reads: AtomicUsize, + current_kid: String, + delay: Duration, + } + + impl PlatformConfigStore for CountingConfigStore { + fn get(&self, _store_name: &StoreName, key: &str) -> Result> { + self.reads.fetch_add(1, Ordering::Relaxed); + if !self.delay.is_zero() { + std::thread::sleep(self.delay); + } + (key == "current-kid") + .then(|| self.current_kid.clone()) + .ok_or_else(|| Report::new(PlatformError::ConfigStore)) + } + + fn put( + &self, + _store_id: &StoreId, + _key: &str, + _value: &str, + ) -> Result<(), Report> { + Err(Report::new(PlatformError::Unsupported)) + } + + fn delete(&self, _store_id: &StoreId, _key: &str) -> Result<(), Report> { + Err(Report::new(PlatformError::Unsupported)) + } + } + + struct CountingSecretStore { + reads: AtomicUsize, + key: Vec, + } + + impl PlatformSecretStore for CountingSecretStore { + fn get_bytes( + &self, + _store_name: &StoreName, + _key: &str, + ) -> Result, Report> { + self.reads.fetch_add(1, Ordering::Relaxed); + Ok(self.key.clone()) + } + + fn create( + &self, + _store_id: &StoreId, + _name: &str, + _value: &str, + ) -> Result<(), Report> { + Err(Report::new(PlatformError::Unsupported)) + } + + fn delete(&self, _store_id: &StoreId, _name: &str) -> Result<(), Report> { + Err(Report::new(PlatformError::Unsupported)) + } + } + + struct OuterSelectErrorHttpClient { + inner: StubHttpClient, + selected_pending: AtomicUsize, + } + + impl OuterSelectErrorHttpClient { + fn new() -> Self { + Self { + inner: StubHttpClient::new(), + selected_pending: AtomicUsize::new(0), + } + } + + fn push_response(&self, status: u16, body: Vec) { + self.inner.push_response(status, body); + } + } + + #[async_trait::async_trait(?Send)] + impl PlatformHttpClient for OuterSelectErrorHttpClient { + async fn send( + &self, + request: PlatformHttpRequest, + ) -> Result> { + self.inner.send(request).await + } + + async fn send_async( + &self, + request: PlatformHttpRequest, + ) -> Result> { + self.inner.send_async(request).await + } + + async fn select( + &self, + pending_requests: Vec, + ) -> Result> { + self.selected_pending + .store(pending_requests.len(), Ordering::Relaxed); + Err(Report::new(PlatformError::HttpClient)) + } + } + + struct UnusedSecretStore; + + impl PlatformSecretStore for UnusedSecretStore { + fn get_bytes( + &self, + _store_name: &StoreName, + _key: &str, + ) -> Result, Report> { + panic!("signing key should not be read after current-kid failure") + } + + fn create( + &self, + _store_id: &StoreId, + _name: &str, + _value: &str, + ) -> Result<(), Report> { + Err(Report::new(PlatformError::Unsupported)) + } + + fn delete(&self, _store_id: &StoreId, _name: &str) -> Result<(), Report> { + Err(Report::new(PlatformError::Unsupported)) + } + } // --------------------------------------------------------------------------- // Minimal test double for AuctionProvider @@ -1548,7 +3217,7 @@ mod tests { #[async_trait::async_trait(?Send)] impl AuctionProvider for StubAuctionProvider { - fn provider_name(&self) -> &'static str { + fn provider_name(&self) -> &str { self.name } @@ -1616,17 +3285,14 @@ mod tests { } } - struct RecordingTimeoutProvider { + struct DeadlineBidProvider { name: &'static str, backend: &'static str, - configured_timeout_ms: u32, - predicted: Arc>>, - requested: Arc>>, } #[async_trait::async_trait(?Send)] - impl AuctionProvider for RecordingTimeoutProvider { - fn provider_name(&self) -> &'static str { + impl AuctionProvider for DeadlineBidProvider { + fn provider_name(&self) -> &str { self.name } @@ -1635,16 +3301,12 @@ mod tests { _request: &AuctionRequest, context: &AuctionContext<'_>, ) -> Result> { - self.requested - .lock() - .expect("should lock requested timeouts") - .push(context.timeout_ms); let request = PlatformHttpRequest::new( http::Request::builder() .method("POST") .uri("https://example.com/bid") .body(edgezero_core::body::Body::empty()) - .expect("should build recording request"), + .expect("should build deadline test request"), self.backend, ); context @@ -1653,7 +3315,7 @@ mod tests { .send_async(request) .await .change_context(TrustedServerError::Auction { - message: "recording launch failed".to_string(), + message: "deadline test provider launch failed".to_string(), }) .map(ProviderRequestOutcome::pending) } @@ -1665,20 +3327,180 @@ mod tests { ) -> Result> { Ok(AuctionResponse::success( self.name, - vec![], + vec![auction_bid(self.name, 3.0)], response_time_ms, )) } fn timeout_ms(&self) -> u32 { - self.configured_timeout_ms + 1_000 } - fn backend_name(&self, _services: &RuntimeServices, timeout_ms: u32) -> Option { - self.predicted - .lock() - .expect("should lock predicted timeouts") - .push(timeout_ms); + fn backend_name(&self, _services: &RuntimeServices, _timeout_ms: u32) -> Option { + Some(self.backend.to_string()) + } + } + + type RecordedMediatorBudgets = Arc>>; + + struct DeadlineRecordingMediator { + launches: Arc, + budgets: Option, + } + + struct PendingDeadlineMediator; + + #[async_trait::async_trait(?Send)] + impl AuctionProvider for PendingDeadlineMediator { + fn provider_name(&self) -> &str { + "pending-deadline-mediator" + } + + async fn request_bids( + &self, + _request: &AuctionRequest, + context: &AuctionContext<'_>, + ) -> Result> { + let request = PlatformHttpRequest::new( + http::Request::builder() + .method("POST") + .uri("https://example.com/mediate") + .body(edgezero_core::body::Body::empty()) + .expect("should build pending mediator request"), + "pending-mediator-backend", + ); + context + .services + .http_client() + .send_async(request) + .await + .change_context(TrustedServerError::Auction { + message: "pending mediator launch failed".to_string(), + }) + .map(ProviderRequestOutcome::pending) + } + + async fn parse_response( + &self, + _response: PlatformResponse, + response_time_ms: u64, + ) -> Result> { + Ok(AuctionResponse::success( + self.provider_name(), + vec![auction_bid("mediated", 9.0)], + response_time_ms, + )) + } + + fn timeout_ms(&self) -> u32 { + 1_000 + } + + fn backend_name(&self, _services: &RuntimeServices, _timeout_ms: u32) -> Option { + Some("pending-mediator-backend".to_string()) + } + } + + #[async_trait::async_trait(?Send)] + impl AuctionProvider for DeadlineRecordingMediator { + fn provider_name(&self) -> &str { + "deadline-mediator" + } + + async fn request_bids( + &self, + _request: &AuctionRequest, + context: &AuctionContext<'_>, + ) -> Result> { + self.launches.fetch_add(1, Ordering::Relaxed); + if let Some(budgets) = &self.budgets { + budgets + .lock() + .expect("should lock mediator budgets") + .push((context.timeout_ms, context.transport_timeout_ms)); + } + Ok(ProviderRequestOutcome::Immediate(AuctionResponse::no_bid( + self.provider_name(), + 0, + ))) + } + + async fn parse_response( + &self, + _response: PlatformResponse, + _response_time_ms: u64, + ) -> Result> { + panic!("immediate mediator response should not be parsed"); + } + + fn timeout_ms(&self) -> u32 { + 1_000 + } + } + + struct RecordingTimeoutProvider { + name: &'static str, + backend: &'static str, + configured_timeout_ms: u32, + predicted: Arc>>, + requested: Arc>>, + } + + #[async_trait::async_trait(?Send)] + impl AuctionProvider for RecordingTimeoutProvider { + fn provider_name(&self) -> &str { + self.name + } + + async fn request_bids( + &self, + _request: &AuctionRequest, + context: &AuctionContext<'_>, + ) -> Result> { + self.requested + .lock() + .expect("should lock requested timeouts") + .push(context.transport_timeout_ms); + let request = PlatformHttpRequest::new( + http::Request::builder() + .method("POST") + .uri("https://example.com/bid") + .body(edgezero_core::body::Body::empty()) + .expect("should build recording request"), + self.backend, + ); + context + .services + .http_client() + .send_async(request) + .await + .change_context(TrustedServerError::Auction { + message: "recording launch failed".to_string(), + }) + .map(ProviderRequestOutcome::pending) + } + + async fn parse_response( + &self, + _response: PlatformResponse, + response_time_ms: u64, + ) -> Result> { + Ok(AuctionResponse::success( + self.name, + vec![], + response_time_ms, + )) + } + + fn timeout_ms(&self) -> u32 { + self.configured_timeout_ms + } + + fn backend_name(&self, _services: &RuntimeServices, timeout_ms: u32) -> Option { + self.predicted + .lock() + .expect("should lock predicted timeouts") + .push(timeout_ms); Some(self.backend.to_string()) } } @@ -1691,7 +3513,7 @@ mod tests { #[async_trait::async_trait(?Send)] impl AuctionProvider for DivergentBackendProvider { - fn provider_name(&self) -> &'static str { + fn provider_name(&self) -> &str { self.name } @@ -1746,6 +3568,10 @@ mod tests { } impl PlatformBackend for CanonicalTimeoutBackend { + fn naming_policy(&self) -> crate::platform::BackendNamingPolicy { + crate::platform::BackendNamingPolicy::Axum + } + fn predict_name( &self, _spec: &PlatformBackendSpec, @@ -1810,6 +3636,7 @@ mod tests { .then(|| "
ordinary
".to_string()), adomain: None, bidder: bidder.to_string(), + returned_seat: None, width: 300, height: 250, nurl: None, @@ -1833,6 +3660,7 @@ mod tests { creative: Some("
ad
".to_string()), adomain: None, bidder: "mediator".to_string(), + returned_seat: None, width: 728, height: 90, nurl: nurl.clone(), @@ -1850,7 +3678,7 @@ mod tests { #[async_trait::async_trait(?Send)] impl AuctionProvider for CacheRestoringMediator { - fn provider_name(&self) -> &'static str { + fn provider_name(&self) -> &str { "mediator" } @@ -1919,7 +3747,7 @@ mod tests { #[async_trait::async_trait(?Send)] impl AuctionProvider for ImmediateMediator { - fn provider_name(&self) -> &'static str { + fn provider_name(&self) -> &str { "immediate-mediator" } @@ -1965,7 +3793,7 @@ mod tests { let config = AuctionConfig { enabled: true, - providers: vec!["bidder".to_string()], + providers: AuctionConfig::legacy_provider_map(&["bidder"]), mediator: Some("mediator".to_string()), timeout_ms: 2000, ..Default::default() @@ -1988,6 +3816,7 @@ mod tests { settings: &settings, request: &req, timeout_ms: 2000, + transport_timeout_ms: 2000, provider_responses: None, services, }; @@ -2021,7 +3850,7 @@ mod tests { let services = build_services_with_http_client(stub); let config = AuctionConfig { enabled: true, - providers: vec!["bidder".to_string()], + providers: AuctionConfig::legacy_provider_map(&["bidder"]), mediator: Some("immediate-mediator".to_string()), timeout_ms: 2000, ..Default::default() @@ -2039,6 +3868,7 @@ mod tests { settings: &settings, request: &downstream, timeout_ms: 2000, + transport_timeout_ms: 2000, provider_responses: None, services: &services, }; @@ -2076,90 +3906,351 @@ mod tests { } } - fn create_test_auction_request() -> AuctionRequest { - AuctionRequest { - id: "test-auction-123".to_string(), - slots: vec![ - AdSlot { - id: "header-banner".to_string(), - formats: vec![AdFormat { - media_type: MediaType::Banner, - width: 728, - height: 90, - }], - floor_price: Some(1.50), - targeting: HashMap::new(), - bidders: HashMap::new(), - }, - AdSlot { - id: "sidebar".to_string(), - formats: vec![AdFormat { - media_type: MediaType::Banner, - width: 300, - height: 250, - }], - floor_price: Some(1.00), - targeting: HashMap::new(), - bidders: HashMap::new(), - }, - ], - publisher: PublisherInfo { - domain: "test.com".to_string(), - page_url: Some("https://test.com/article".to_string()), - }, - user: UserInfo { - id: Some("user-123".to_string()), - consent: None, - eids: None, - }, - device: None, - site: None, - context: HashMap::new(), - } - } - - fn create_test_settings() -> crate::settings::Settings { - let settings_str = crate_test_settings_str(); - crate::settings::Settings::from_toml(&settings_str).expect("should parse test settings") - } - - struct ImmediateNoBidProvider; - - #[async_trait::async_trait(?Send)] - impl AuctionProvider for ImmediateNoBidProvider { - fn provider_name(&self) -> &'static str { - "immediate" - } + async fn collect_deadline_test_result( + split: bool, + enforceable_total_request_deadline: bool, + ) -> OrchestrationResult { + let stub = Arc::new(StubHttpClient::new()); + stub.set_enforceable_total_request_deadline(enforceable_total_request_deadline); + stub.push_response(200, b"{}".to_vec()); + stub.push_response(200, b"{}".to_vec()); + stub.push_select_delay(Duration::from_millis(50)); + let services = build_services_with_http_client(Arc::clone(&stub) as Arc<_>); + let config = AuctionConfig { + enabled: true, + providers: AuctionConfig::legacy_provider_map(&["late-one", "late-two"]), + timeout_ms: 10, + ..Default::default() + }; + let mut orchestrator = AuctionOrchestrator::new(config); + orchestrator.register_provider(Arc::new(DeadlineBidProvider { + name: "late-one", + backend: "late-one-backend", + })); + orchestrator.register_provider(Arc::new(DeadlineBidProvider { + name: "late-two", + backend: "late-two-backend", + })); + let request = create_test_auction_request(); + let settings = create_test_settings(); + let downstream = http::Request::new(edgezero_core::body::Body::empty()); + let context = AuctionContext { + settings: &settings, + request: &downstream, + timeout_ms: 10, + transport_timeout_ms: 10, + provider_responses: None, + services: &services, + }; - async fn request_bids( - &self, - _request: &AuctionRequest, - _context: &AuctionContext<'_>, - ) -> Result> { - Ok(ProviderRequestOutcome::Immediate(AuctionResponse::no_bid( - "immediate", - 0, - ))) + if split { + let DispatchAuctionOutcome::Dispatched(dispatched) = + orchestrator.dispatch_auction(&request, &context).await + else { + panic!("deadline test providers should dispatch"); + }; + orchestrator + .collect_dispatched_auction(dispatched, &services, &context) + .await + } else { + orchestrator + .run_auction(&request, &context) + .await + .expect("deadline test auction should complete") } + } - async fn parse_response( - &self, - _response: PlatformResponse, - _response_time_ms: u64, - ) -> Result> { - panic!("immediate response should not be parsed"); + #[tokio::test] + async fn current_adapter_deadline_drains_late_responses_in_both_paths() { + for split in [false, true] { + let result = collect_deadline_test_result(split, false).await; + assert_eq!(result.provider_responses.len(), 2); + assert_eq!(result.provider_responses[0].provider, "late-one"); + assert_eq!(result.provider_responses[0].status, BidStatus::Success); + assert_eq!(result.provider_responses[1].provider, "late-two"); + assert_eq!(result.provider_responses[1].status, BidStatus::Success); + assert!( + result + .provider_responses + .iter() + .all(|response| response.response_time_ms >= 50), + "late response times should retain actual elapsed duration" + ); + assert_eq!( + result.winning_bids["slot-1"].bidder, "late-one", + "a completed response remains eligible after the logical deadline" + ); } + } - fn timeout_ms(&self) -> u32 { - 2000 + #[tokio::test] + async fn synthetic_hard_deadline_classifies_late_responses_in_both_paths() { + for split in [false, true] { + let result = collect_deadline_test_result(split, true).await; + assert_eq!(result.provider_responses.len(), 2); + assert!(result.provider_responses.iter().all(|response| { + response.status == BidStatus::Error + && response.metadata["error_type"] == ERROR_TYPE_TIMEOUT + && response.response_time_ms >= 50 + })); + assert!(result.winning_bids.is_empty()); } } - struct LaunchFailingProvider; - - #[async_trait::async_trait(?Send)] - impl AuctionProvider for LaunchFailingProvider { - fn provider_name(&self) -> &'static str { + async fn pending_mediator_deadline_test_result( + split: bool, + enforceable_total_request_deadline: bool, + ) -> OrchestrationResult { + let stub = Arc::new(StubHttpClient::new()); + stub.set_enforceable_total_request_deadline(enforceable_total_request_deadline); + stub.push_response(200, b"{}".to_vec()); + stub.push_response(200, b"{}".to_vec()); + stub.push_select_delay(Duration::ZERO); + stub.push_select_delay(Duration::from_millis(50)); + let services = build_services_with_http_client(Arc::clone(&stub) as Arc<_>); + let config = AuctionConfig { + enabled: true, + providers: AuctionConfig::legacy_provider_map(&["local"]), + mediator: Some("pending-deadline-mediator".to_string()), + timeout_ms: 20, + ..Default::default() + }; + let mut orchestrator = AuctionOrchestrator::new(config); + orchestrator.register_provider(Arc::new(DeadlineBidProvider { + name: "local", + backend: "local-backend", + })); + orchestrator.register_provider(Arc::new(PendingDeadlineMediator)); + let request = create_test_auction_request(); + let settings = create_test_settings(); + let downstream = http::Request::new(edgezero_core::body::Body::empty()); + let context = AuctionContext { + settings: &settings, + request: &downstream, + timeout_ms: 20, + transport_timeout_ms: 20, + provider_responses: None, + services: &services, + }; + + if split { + let DispatchAuctionOutcome::Dispatched(dispatched) = + orchestrator.dispatch_auction(&request, &context).await + else { + panic!("deadline test provider should dispatch"); + }; + orchestrator + .collect_dispatched_auction(dispatched, &services, &context) + .await + } else { + orchestrator + .run_auction(&request, &context) + .await + .expect("deadline test auction should complete") + } + } + + #[tokio::test] + async fn pending_mediator_late_completion_policy_is_equivalent_in_sync_and_split_paths() { + for split in [false, true] { + let current = pending_mediator_deadline_test_result(split, false).await; + let current_mediator = current + .mediator_response + .as_ref() + .expect("current adapters should accept completed late mediator responses"); + assert!( + current_mediator.response_time_ms >= 50, + "mediator timing should preserve actual elapsed duration, got {} ms", + current_mediator.response_time_ms + ); + assert_eq!(current.winning_bids["slot-1"].bidder, "mediated"); + + let hard = pending_mediator_deadline_test_result(split, true).await; + assert!(hard.mediator_response.is_none()); + assert_eq!(hard.winning_bids["slot-1"].bidder, "local"); + assert!( + hard.total_time_ms >= 50, + "discarding a late mediator must retain actual total elapsed time" + ); + } + } + + #[tokio::test] + async fn split_deadline_skips_mediator_and_falls_back_to_provider_winner() { + let stub = Arc::new(StubHttpClient::new()); + stub.push_response(200, b"{}".to_vec()); + stub.push_select_delay(Duration::from_millis(50)); + let services = build_services_with_http_client(Arc::clone(&stub) as Arc<_>); + let launches = Arc::new(AtomicUsize::new(0)); + let config = AuctionConfig { + enabled: true, + providers: AuctionConfig::legacy_provider_map(&["late-one"]), + mediator: Some("deadline-mediator".to_string()), + timeout_ms: 10, + ..Default::default() + }; + let mut orchestrator = AuctionOrchestrator::new(config); + orchestrator.register_provider(Arc::new(DeadlineBidProvider { + name: "late-one", + backend: "late-one-backend", + })); + orchestrator.register_provider(Arc::new(DeadlineRecordingMediator { + launches: Arc::clone(&launches), + budgets: None, + })); + let request = create_test_auction_request(); + let settings = create_test_settings(); + let downstream = http::Request::new(edgezero_core::body::Body::empty()); + let context = AuctionContext { + settings: &settings, + request: &downstream, + timeout_ms: 10, + transport_timeout_ms: 10, + provider_responses: None, + services: &services, + }; + let DispatchAuctionOutcome::Dispatched(dispatched) = + orchestrator.dispatch_auction(&request, &context).await + else { + panic!("deadline test provider should dispatch"); + }; + let result = orchestrator + .collect_dispatched_auction(dispatched, &services, &context) + .await; + + assert_eq!(launches.load(Ordering::Relaxed), 0); + assert!(result.mediator_response.is_none()); + assert_eq!(result.winning_bids["slot-1"].bidder, "late-one"); + } + + #[tokio::test] + async fn synchronous_deadline_skips_mediator_and_falls_back_to_provider_winner() { + let stub = Arc::new(StubHttpClient::new()); + stub.push_response(200, b"{}".to_vec()); + stub.push_select_delay(Duration::from_millis(50)); + let services = build_services_with_http_client(Arc::clone(&stub) as Arc<_>); + let launches = Arc::new(AtomicUsize::new(0)); + let config = AuctionConfig { + enabled: true, + providers: AuctionConfig::legacy_provider_map(&["late-one"]), + mediator: Some("deadline-mediator".to_string()), + timeout_ms: 10, + ..Default::default() + }; + let mut orchestrator = AuctionOrchestrator::new(config); + orchestrator.register_provider(Arc::new(DeadlineBidProvider { + name: "late-one", + backend: "late-one-backend", + })); + orchestrator.register_provider(Arc::new(DeadlineRecordingMediator { + launches: Arc::clone(&launches), + budgets: None, + })); + let request = create_test_auction_request(); + let settings = create_test_settings(); + let downstream = http::Request::new(edgezero_core::body::Body::empty()); + let context = AuctionContext { + settings: &settings, + request: &downstream, + timeout_ms: 10, + transport_timeout_ms: 10, + provider_responses: None, + services: &services, + }; + let result = orchestrator + .run_auction(&request, &context) + .await + .expect("synchronous deadline test should complete"); + + assert_eq!(launches.load(Ordering::Relaxed), 0); + assert!(result.mediator_response.is_none()); + assert_eq!(result.winning_bids["slot-1"].bidder, "late-one"); + } + + fn create_test_auction_request() -> AuctionRequest { + AuctionRequest { + id: "test-auction-123".to_string(), + slots: vec![ + AdSlot { + id: "header-banner".to_string(), + formats: vec![AdFormat { + media_type: MediaType::Banner, + width: 728, + height: 90, + }], + floor_price: Some(1.50), + targeting: HashMap::new(), + bidders: HashMap::new(), + }, + AdSlot { + id: "sidebar".to_string(), + formats: vec![AdFormat { + media_type: MediaType::Banner, + width: 300, + height: 250, + }], + floor_price: Some(1.00), + targeting: HashMap::new(), + bidders: HashMap::new(), + }, + ], + publisher: PublisherInfo { + domain: "test.com".to_string(), + page_url: Some("https://test.com/article".to_string()), + }, + user: UserInfo { + id: Some("user-123".to_string()), + consent: None, + eids: None, + }, + device: None, + site: None, + context: HashMap::new(), + } + } + + fn create_test_settings() -> crate::settings::Settings { + let settings_str = crate_test_settings_str(); + crate::settings::Settings::from_toml(&settings_str).expect("should parse test settings") + } + + struct ImmediateNoBidProvider; + + #[async_trait::async_trait(?Send)] + impl AuctionProvider for ImmediateNoBidProvider { + fn provider_name(&self) -> &str { + "immediate" + } + + async fn request_bids( + &self, + _request: &AuctionRequest, + _context: &AuctionContext<'_>, + ) -> Result> { + Ok(ProviderRequestOutcome::Immediate(AuctionResponse::no_bid( + "immediate", + 0, + ))) + } + + async fn parse_response( + &self, + _response: PlatformResponse, + _response_time_ms: u64, + ) -> Result> { + panic!("immediate response should not be parsed"); + } + + fn timeout_ms(&self) -> u32 { + 2000 + } + } + + struct LaunchFailingProvider; + + #[async_trait::async_trait(?Send)] + impl AuctionProvider for LaunchFailingProvider { + fn provider_name(&self) -> &str { "launch-failing" } @@ -2201,6 +4292,7 @@ mod tests { settings, request, timeout_ms: 2000, + transport_timeout_ms: 2000, provider_responses: None, services, } @@ -2210,7 +4302,7 @@ mod tests { async fn synchronous_auction_accepts_an_all_immediate_no_bid_result() { let config = AuctionConfig { enabled: true, - providers: vec!["immediate".to_string()], + providers: AuctionConfig::legacy_provider_map(&["immediate"]), timeout_ms: 2000, ..Default::default() }; @@ -2235,7 +4327,7 @@ mod tests { async fn split_auction_accepts_an_all_immediate_no_bid_result() { let config = AuctionConfig { enabled: true, - providers: vec!["immediate".to_string()], + providers: AuctionConfig::legacy_provider_map(&["immediate"]), timeout_ms: 2000, ..Default::default() }; @@ -2266,7 +4358,7 @@ mod tests { for split in [false, true] { let config = AuctionConfig { enabled: true, - providers: vec!["immediate".to_string(), "pending".to_string()], + providers: AuctionConfig::legacy_provider_map(&["immediate", "pending"]), timeout_ms: 2000, ..Default::default() }; @@ -2424,6 +4516,7 @@ mod tests { creative: Some("
Ad
".to_string()), adomain: None, bidder: "test-bidder".to_string(), + returned_seat: None, width: 300, height: 250, nurl: None, @@ -2447,6 +4540,7 @@ mod tests { creative: Some("
Ad
".to_string()), adomain: None, bidder: "test-bidder".to_string(), + returned_seat: None, width: 300, height: 250, nurl: None, @@ -2492,7 +4586,8 @@ mod tests { enabled: true, sanitize_creatives: true, rewrite_creatives: true, - providers: vec![], + providers: AuctionConfig::legacy_provider_map(&[]), + bidders: Default::default(), mediator: None, timeout_ms: 2000, creative_store: "creative_store".to_string(), @@ -2523,7 +4618,7 @@ mod tests { futures::executor::block_on(async { let config = AuctionConfig { enabled: true, - providers: vec!["launch-failing".to_string()], + providers: AuctionConfig::legacy_provider_map(&["launch-failing"]), timeout_ms: 2000, ..Default::default() }; @@ -2553,41 +4648,12 @@ mod tests { }); } - #[test] - fn rejects_duplicate_configured_providers() { - let config = AuctionConfig { - enabled: true, - providers: vec!["prebid".to_string(), "prebid".to_string()], - timeout_ms: 2000, - ..Default::default() - }; - let err = AuctionOrchestrator::new(config) - .validate_configured_provider_names() - .expect_err("should reject a provider listed more than once"); - assert!(err.to_string().contains("listed more than once")); - } - - #[test] - fn rejects_mediator_also_listed_as_provider() { - let config = AuctionConfig { - enabled: true, - providers: vec!["prebid".to_string()], - mediator: Some("prebid".to_string()), - timeout_ms: 2000, - ..Default::default() - }; - let err = AuctionOrchestrator::new(config) - .validate_configured_provider_names() - .expect_err("should reject a mediator also configured as a provider"); - assert!(err.to_string().contains("may not mediate its own auction")); - } - #[tokio::test] async fn duplicate_backend_name_fails_second_provider_attributably_in_both_paths() { for split in [false, true] { let config = AuctionConfig { enabled: true, - providers: vec!["provider-a".to_string(), "provider-b".to_string()], + providers: AuctionConfig::legacy_provider_map(&["provider-a", "provider-b"]), timeout_ms: 2000, ..Default::default() }; @@ -2702,7 +4768,7 @@ mod tests { let requested = Arc::new(Mutex::new(Vec::new())); let mut orchestrator = AuctionOrchestrator::new(AuctionConfig { enabled: true, - providers: vec!["bidder".to_string()], + providers: AuctionConfig::legacy_provider_map(&["bidder"]), timeout_ms: 2000, ..Default::default() }); @@ -2746,7 +4812,7 @@ mod tests { let requested = Arc::new(Mutex::new(Vec::new())); let mut orchestrator = AuctionOrchestrator::new(AuctionConfig { enabled: true, - providers: vec!["bidder".to_string()], + providers: AuctionConfig::legacy_provider_map(&["bidder"]), timeout_ms: 2000, ..Default::default() }); @@ -2789,7 +4855,7 @@ mod tests { let requested = Arc::new(Mutex::new(Vec::new())); let mut orchestrator = AuctionOrchestrator::new(AuctionConfig { enabled: true, - providers: vec!["bidder".to_string()], + providers: AuctionConfig::legacy_provider_map(&["bidder"]), mediator: Some("mediator".to_string()), timeout_ms: 2000, ..Default::default() @@ -2839,7 +4905,7 @@ mod tests { let mediator_requested = Arc::new(Mutex::new(Vec::new())); let mut orchestrator = AuctionOrchestrator::new(AuctionConfig { enabled: true, - providers: vec!["bidder".to_string()], + providers: AuctionConfig::legacy_provider_map(&["bidder"]), mediator: Some("mediator".to_string()), timeout_ms: 2000, ..Default::default() @@ -2893,6 +4959,64 @@ mod tests { }); } + #[test] + fn planned_collect_skips_mediator_with_zero_canonical_transport_budget() { + futures::executor::block_on(async { + let calls = Arc::new(Mutex::new(Vec::new())); + let services = build_services_with_backend_and_http_client( + Arc::new(CanonicalTimeoutBackend { + canonical_ms: 0, + calls: Arc::clone(&calls), + }), + Arc::new(StubHttpClient::new()), + ); + let launches = Arc::new(AtomicUsize::new(0)); + let budgets = Arc::new(Mutex::new(Vec::new())); + let plan = AuctionPlan::compile(AuctionPlanConfig { + timeout_ms: 49, + providers: BTreeMap::new(), + bidders: BTreeMap::new(), + mediator: Some("adserver_mock".to_string()), + request_signing: None, + }) + .expect("should compile mediator-only plan") + .with_enabled(true); + let orchestrator = AuctionOrchestrator::from_plan( + Arc::new(plan), + Some(Arc::new(DeadlineRecordingMediator { + launches: Arc::clone(&launches), + budgets: Some(Arc::clone(&budgets)), + })), + ); + let request = create_test_auction_request(); + let settings = create_test_settings(); + let downstream = http::Request::new(edgezero_core::body::Body::empty()); + let context = AuctionContext { + settings: &settings, + request: &downstream, + timeout_ms: 49, + transport_timeout_ms: 49, + provider_responses: None, + services: &services, + }; + let dispatched = DispatchedAuction::empty_for_test(request, 49); + + let result = orchestrator + .collect_dispatched_auction(dispatched, &services, &context) + .await; + + assert_eq!(launches.load(Ordering::Relaxed), 0); + assert!( + budgets + .lock() + .expect("should lock mediator budgets") + .is_empty() + ); + assert_eq!(calls.lock().expect("should lock calls").len(), 1); + assert!(result.mediator_response.is_none()); + }); + } + #[test] fn dispatched_resolved_backend_name_diverging_from_prediction_still_correlates() { futures::executor::block_on(async { @@ -2901,7 +5025,7 @@ mod tests { let services = build_services_with_http_client(stub); let mut orchestrator = AuctionOrchestrator::new(AuctionConfig { enabled: true, - providers: vec!["provider-a".to_string()], + providers: AuctionConfig::legacy_provider_map(&["provider-a"]), timeout_ms: 2000, ..Default::default() }); @@ -2939,7 +5063,7 @@ mod tests { let services = build_services_with_http_client(stub); let mut orchestrator = AuctionOrchestrator::new(AuctionConfig { enabled: true, - providers: vec!["provider-a".to_string(), "provider-b".to_string()], + providers: AuctionConfig::legacy_provider_map(&["provider-a", "provider-b"]), timeout_ms: 2000, ..Default::default() }); @@ -2994,7 +5118,7 @@ mod tests { let config = AuctionConfig { enabled: true, - providers: vec!["provider-a".to_string(), "provider-b".to_string()], + providers: AuctionConfig::legacy_provider_map(&["provider-a", "provider-b"]), timeout_ms: 2000, mediator: None, ..Default::default() @@ -3020,6 +5144,7 @@ mod tests { settings: &settings, request: &req, timeout_ms: 2000, + transport_timeout_ms: 2000, provider_responses: None, services, }; @@ -3061,46 +5186,205 @@ mod tests { }); } - #[test] - fn dispatched_collection_reuses_provider_launch_context() { - futures::executor::block_on(async { - let stub = Arc::new(StubHttpClient::new()); - stub.push_response(200, b"{}".to_vec()); - let services = build_services_with_http_client(stub); - let config = AuctionConfig { - enabled: true, - providers: vec!["provider-a".to_string()], - timeout_ms: 750, - mediator: None, - ..Default::default() - }; - let mut orchestrator = AuctionOrchestrator::new(config); - orchestrator.register_provider(Arc::new(StubAuctionProvider { - name: "provider-a", - backend: "backend-a", - })); - let request = create_test_auction_request(); - let settings = create_test_settings(); - let downstream = http::Request::builder() - .uri("https://publisher.example/article") - .header(http::header::REFERER, "https://referrer.example/source") - .body(edgezero_core::body::Body::empty()) - .expect("should build downstream request"); - let dispatch_context = AuctionContext { - settings: &settings, - request: &downstream, - timeout_ms: 750, - provider_responses: None, - services: &services, - }; - let dispatched = match orchestrator - .dispatch_auction(&request, &dispatch_context) - .await - { - DispatchAuctionOutcome::Dispatched(dispatched) => dispatched, - _ => panic!("should dispatch provider request"), - }; - let placeholder = http::Request::builder() + #[tokio::test] + async fn harness_outer_select_error_materializes_all_launches_as_transport_failures() { + let http = Arc::new(OuterSelectErrorHttpClient::new()); + http.push_response(204, Vec::new()); + let backend = Arc::new(NamingBackend::new(BackendNamingPolicy::Axum)); + let services = build_services_with_backend_and_http_client( + Arc::clone(&backend) as Arc<_>, + Arc::clone(&http) as Arc<_>, + ); + let plan = Arc::new( + AuctionPlan::compile(planned_config( + &[("provider-a", RoutingMode::AllEligible)], + false, + )) + .expect("should compile planned auction"), + ); + let harness = AuctionOrchestratorHarness::new(plan, None); + let request = planned_request(); + let settings = create_test_settings(); + let inbound = http::Request::new(edgezero_core::body::Body::empty()); + let context = AuctionContext { + settings: &settings, + request: &inbound, + timeout_ms: 777, + transport_timeout_ms: 777, + provider_responses: None, + services: &services, + }; + + let result = harness + .run_auction(&request, &context) + .await + .expect("should collect a harness auction after select failure"); + + assert_eq!( + result.provider_responses.len(), + 1, + "should report one provider response" + ); + assert_eq!( + result.provider_responses[0].status, + BidStatus::Error, + "outer select failure should report an error" + ); + assert_eq!( + result.provider_responses[0].metadata["error_type"], ERROR_TYPE_TRANSPORT, + "harness should match production transport classification" + ); + } + + #[tokio::test] + async fn planned_dispatch_does_not_snapshot_inbound_headers() { + let http = Arc::new(StubHttpClient::new()); + http.push_response(204, Vec::new()); + let backend = Arc::new(NamingBackend::new(BackendNamingPolicy::Axum)); + let services = build_services_with_backend_and_http_client( + Arc::clone(&backend) as Arc<_>, + Arc::clone(&http) as Arc<_>, + ); + let plan = Arc::new( + AuctionPlan::compile(planned_config( + &[("provider-a", RoutingMode::AllEligible)], + false, + )) + .expect("should compile planned auction"), + ); + let orchestrator = AuctionOrchestrator::from_plan(plan, None); + let request = planned_request(); + let settings = create_test_settings(); + let inbound = http::Request::builder() + .header(http::header::AUTHORIZATION, "Bearer should-not-be-retained") + .header(http::header::COOKIE, "session=should-not-be-retained") + .body(edgezero_core::body::Body::empty()) + .expect("should build inbound request"); + let context = AuctionContext { + settings: &settings, + request: &inbound, + timeout_ms: 777, + transport_timeout_ms: 777, + provider_responses: None, + services: &services, + }; + + let DispatchAuctionOutcome::Dispatched(dispatched) = + orchestrator.dispatch_auction(&request, &context).await + else { + panic!("should dispatch planned provider"); + }; + + assert!( + dispatched.provider_request_context.headers().is_empty(), + "planned dispatch should retain an empty request context" + ); + } + + #[tokio::test] + async fn outer_select_error_materializes_all_planned_launches_as_transport_failures() { + let http = Arc::new(OuterSelectErrorHttpClient::new()); + http.push_response(204, Vec::new()); + http.push_response(204, Vec::new()); + let backend = Arc::new(NamingBackend::new(BackendNamingPolicy::Axum)); + let services = build_services_with_backend_and_http_client( + Arc::clone(&backend) as Arc<_>, + Arc::clone(&http) as Arc<_>, + ); + let plan = Arc::new( + AuctionPlan::compile(planned_config( + &[ + ("provider-b", RoutingMode::AllEligible), + ("provider-a", RoutingMode::AllEligible), + ], + false, + )) + .expect("should compile planned auction"), + ); + let orchestrator = AuctionOrchestrator::from_plan(plan, None); + let request = planned_request(); + let settings = create_test_settings(); + let inbound = http::Request::new(edgezero_core::body::Body::empty()); + let context = AuctionContext { + settings: &settings, + request: &inbound, + timeout_ms: 777, + transport_timeout_ms: 777, + provider_responses: None, + services: &services, + }; + + let DispatchAuctionOutcome::Dispatched(dispatched) = + orchestrator.dispatch_auction(&request, &context).await + else { + panic!("should dispatch both planned providers"); + }; + tokio::time::sleep(Duration::from_millis(5)).await; + let result = orchestrator + .collect_dispatched_auction(dispatched, &services, &context) + .await; + + assert_eq!(http.selected_pending.load(Ordering::Relaxed), 2); + assert_eq!( + result + .provider_responses + .iter() + .map(|response| response.provider.as_str()) + .collect::>(), + vec!["provider-a", "provider-b"], + "outer select errors should retain deterministic plan order" + ); + for response in &result.provider_responses { + assert_eq!(response.status, BidStatus::Error); + assert_eq!(response.metadata["error_type"], "transport"); + assert!( + response.response_time_ms >= 5, + "transport failure should preserve launch elapsed time" + ); + } + } + + #[test] + fn dispatched_collection_reuses_provider_launch_context() { + futures::executor::block_on(async { + let stub = Arc::new(StubHttpClient::new()); + stub.push_response(200, b"{}".to_vec()); + let services = build_services_with_http_client(stub); + let config = AuctionConfig { + enabled: true, + providers: AuctionConfig::legacy_provider_map(&["provider-a"]), + timeout_ms: 750, + mediator: None, + ..Default::default() + }; + let mut orchestrator = AuctionOrchestrator::new(config); + orchestrator.register_provider(Arc::new(StubAuctionProvider { + name: "provider-a", + backend: "backend-a", + })); + let request = create_test_auction_request(); + let settings = create_test_settings(); + let downstream = http::Request::builder() + .uri("https://publisher.example/article") + .header(http::header::REFERER, "https://referrer.example/source") + .body(edgezero_core::body::Body::empty()) + .expect("should build downstream request"); + let dispatch_context = AuctionContext { + settings: &settings, + request: &downstream, + timeout_ms: 750, + transport_timeout_ms: 750, + provider_responses: None, + services: &services, + }; + let dispatched = match orchestrator + .dispatch_auction(&request, &dispatch_context) + .await + { + DispatchAuctionOutcome::Dispatched(dispatched) => dispatched, + _ => panic!("should dispatch provider request"), + }; + let placeholder = http::Request::builder() .uri("https://placeholder.invalid/") .body(edgezero_core::body::Body::empty()) .expect("should build placeholder request"); @@ -3108,6 +5392,7 @@ mod tests { settings: &settings, request: &placeholder, timeout_ms: 750, + transport_timeout_ms: 750, provider_responses: None, services: &services, }; @@ -3147,7 +5432,7 @@ mod tests { let config = AuctionConfig { enabled: true, - providers: vec!["provider-a".to_string(), "provider-b".to_string()], + providers: AuctionConfig::legacy_provider_map(&["provider-a", "provider-b"]), timeout_ms: 2000, mediator: None, ..Default::default() @@ -3173,6 +5458,7 @@ mod tests { settings: &settings, request: &req, timeout_ms: 2000, + transport_timeout_ms: 2000, provider_responses: None, services, }; @@ -3212,7 +5498,7 @@ mod tests { let config = AuctionConfig { enabled: true, - providers: vec!["provider-a".to_string(), "provider-b".to_string()], + providers: AuctionConfig::legacy_provider_map(&["provider-a", "provider-b"]), timeout_ms: 2000, mediator: None, ..Default::default() @@ -3238,6 +5524,7 @@ mod tests { settings: &settings, request: &req, timeout_ms: 2000, + transport_timeout_ms: 2000, provider_responses: None, services, }; @@ -3257,6 +5544,2118 @@ mod tests { }); } + #[tokio::test] + async fn from_plan_standard_provider_runs_direct_and_split_with_correlation_and_metadata() { + for split in [false, true] { + let http = Arc::new(StubHttpClient::new()); + http.push_response( + 200, + serde_json::to_vec(&serde_json::json!({ + "seatbid": [{"seat": "provider-seat", "bid": [{ + "id": "provider-bid", "impid": "fictional-slot", "price": 2.0, + "adm": "
provider
", "w": 300, "h": 250 + }]}] + })) + .expect("should serialize provider response"), + ); + let backend = Arc::new(NamingBackend::new(BackendNamingPolicy::Axum)); + let services = build_services_with_backend_and_http_client( + Arc::clone(&backend) as Arc<_>, + Arc::clone(&http) as Arc<_>, + ); + let mut config = planned_config(&[("provider-a", RoutingMode::AllEligible)], false); + config.bidders.insert( + "routed-bidder" + .parse() + .expect("should parse fictional bidder ID"), + crate::auction::plan::BidderRouteConfig { + provider: "provider-a" + .parse() + .expect("should parse fictional provider ID"), + }, + ); + let plan = Arc::new(AuctionPlan::compile(config).expect("should compile plan")); + let orchestrator = AuctionOrchestrator::from_plan(plan, None); + let mut request = planned_request(); + request.slots[0].bidders.insert( + "routed-bidder".to_string(), + serde_json::json!({"placement": 7}), + ); + request.slots[0].bidders.insert( + "unknown-private-id".to_string(), + serde_json::json!({"secret": 9}), + ); + let settings = create_test_settings(); + let inbound = http::Request::new(edgezero_core::body::Body::empty()); + let context = AuctionContext { + settings: &settings, + request: &inbound, + timeout_ms: 777, + transport_timeout_ms: 777, + provider_responses: None, + services: &services, + }; + + let result = if split { + let DispatchAuctionOutcome::Dispatched(dispatched) = + orchestrator.dispatch_auction(&request, &context).await + else { + panic!("standard provider should dispatch"); + }; + orchestrator + .collect_dispatched_auction(dispatched, &services, &context) + .await + } else { + orchestrator + .run_auction(&request, &context) + .await + .expect("standard provider should run") + }; + + assert_eq!(http.recorded_backend_names().len(), 1); + assert_eq!(backend.ensured.load(Ordering::Relaxed), 1); + assert_eq!(result.provider_responses.len(), 1); + assert_eq!(result.provider_responses[0].provider, "provider-a"); + assert_eq!(result.provider_responses[0].status, BidStatus::Success); + assert_eq!( + result.provider_responses[0].metadata["routing"]["unused_bidder_params_count"], + 1 + ); + assert_eq!(result.metadata["routing"]["unroutable_bidder_count"], 1); + assert_eq!( + result.winning_bids["fictional-slot"].bid_id.as_deref(), + Some("provider-bid") + ); + let metadata = + serde_json::to_string(&result.metadata).expect("should serialize auction metadata"); + assert!(!metadata.contains("unknown-private-id") && !metadata.contains("secret")); + } + } + + #[tokio::test] + async fn planned_executor_invokes_immediate_mediator_and_applies_floor() { + let http = Arc::new(StubHttpClient::new()); + http.push_response( + 200, + serde_json::to_vec(&serde_json::json!({ + "seatbid": [{"seat": "provider-seat", "bid": [{ + "id": "provider", "impid": "fictional-slot", "price": 2.0, + "adm": "
provider
", "w": 300, "h": 250 + }]}] + })) + .expect("should serialize provider response"), + ); + let backend = Arc::new(NamingBackend::new(BackendNamingPolicy::Axum)); + let services = build_services_with_backend_and_http_client( + Arc::clone(&backend) as Arc<_>, + Arc::clone(&http) as Arc<_>, + ); + let plan = AuctionPlan::compile(planned_config( + &[("provider-a", RoutingMode::AllEligible)], + false, + )) + .expect("should compile planned auction"); + let orchestrator = AuctionOrchestratorHarness::new(plan, Some(Arc::new(ImmediateMediator))); + let request = planned_request(); + let settings = create_test_settings(); + let inbound = http::Request::new(edgezero_core::body::Body::empty()); + let context = AuctionContext { + settings: &settings, + request: &inbound, + timeout_ms: 777, + transport_timeout_ms: 777, + provider_responses: None, + services: &services, + }; + + let result = orchestrator + .run_auction(&request, &context) + .await + .expect("should execute planned mediation"); + + assert_eq!( + result + .mediator_response + .as_ref() + .map(|response| response.provider.as_str()), + Some("immediate-mediator") + ); + assert_eq!( + result.winning_bids["header-banner"].nurl.as_deref(), + Some("https://nurl.example/immediate") + ); + assert!( + !result.winning_bids.contains_key("fictional-slot"), + "mediator output owns final selection" + ); + } + + async fn planned_pending_mediator_deadline_result( + enforceable_total_request_deadline: bool, + ) -> OrchestrationResult { + let http = Arc::new(StubHttpClient::new()); + http.set_enforceable_total_request_deadline(enforceable_total_request_deadline); + http.push_response( + 200, + serde_json::to_vec(&serde_json::json!({ + "seatbid": [{"seat": "provider-seat", "bid": [{ + "id": "provider", "impid": "fictional-slot", "price": 2.0, + "adm": "
provider
", "w": 300, "h": 250 + }]}] + })) + .expect("should serialize provider response"), + ); + http.push_response(200, b"{}".to_vec()); + http.push_select_delay(Duration::ZERO); + http.push_select_delay(Duration::from_millis(50)); + let backend = Arc::new(NamingBackend::new(BackendNamingPolicy::Axum)); + let services = build_services_with_backend_and_http_client( + Arc::clone(&backend) as Arc<_>, + Arc::clone(&http) as Arc<_>, + ); + let plan = AuctionPlan::compile(planned_config( + &[("provider-a", RoutingMode::AllEligible)], + false, + )) + .expect("should compile planned auction"); + let orchestrator = + AuctionOrchestratorHarness::new(plan, Some(Arc::new(PendingDeadlineMediator))); + let request = planned_request(); + let settings = create_test_settings(); + let inbound = http::Request::new(edgezero_core::body::Body::empty()); + let context = AuctionContext { + settings: &settings, + request: &inbound, + timeout_ms: 20, + transport_timeout_ms: 20, + provider_responses: None, + services: &services, + }; + + orchestrator + .run_auction(&request, &context) + .await + .expect("should execute planned pending mediator") + } + + #[tokio::test] + async fn planned_pending_mediator_applies_explicit_hard_deadline_policy() { + let current = planned_pending_mediator_deadline_result(false).await; + let current_mediator = current + .mediator_response + .as_ref() + .expect("current adapters should accept completed late mediator responses"); + assert!(current_mediator.response_time_ms >= 50); + assert_eq!(current.winning_bids["slot-1"].bidder, "mediated"); + + let hard = planned_pending_mediator_deadline_result(true).await; + assert!(hard.mediator_response.is_none()); + assert_eq!( + hard.winning_bids["fictional-slot"].bid_id.as_deref(), + Some("provider") + ); + assert!(hard.total_time_ms >= 50); + } + + #[tokio::test] + async fn planned_executor_mediator_transport_failure_falls_back_locally() { + let http = Arc::new(StubHttpClient::new()); + http.push_response( + 200, + serde_json::to_vec(&serde_json::json!({ + "seatbid": [{"seat": "provider-seat", "bid": [{ + "id": "provider", "impid": "fictional-slot", "price": 2.0, + "adm": "
provider
", "w": 300, "h": 250 + }]}] + })) + .expect("should serialize provider response"), + ); + http.push_response(200, b"{}".to_vec()); + http.push_select_success(); + http.push_wait_error(); + let backend = Arc::new(NamingBackend::new(BackendNamingPolicy::Axum)); + let services = build_services_with_backend_and_http_client( + Arc::clone(&backend) as Arc<_>, + Arc::clone(&http) as Arc<_>, + ); + let plan = AuctionPlan::compile(planned_config( + &[("provider-a", RoutingMode::AllEligible)], + false, + )) + .expect("should compile planned auction"); + let orchestrator = + AuctionOrchestratorHarness::new(plan, Some(Arc::new(CacheRestoringMediator))); + let request = planned_request(); + let settings = create_test_settings(); + let inbound = http::Request::new(edgezero_core::body::Body::empty()); + let context = AuctionContext { + settings: &settings, + request: &inbound, + timeout_ms: 777, + transport_timeout_ms: 777, + provider_responses: None, + services: &services, + }; + + let result = orchestrator + .run_auction(&request, &context) + .await + .expect("should fall back from mediator transport failure"); + + assert!(result.mediator_response.is_none()); + assert_eq!( + result.winning_bids["fictional-slot"].bid_id.as_deref(), + Some("provider") + ); + } + + #[tokio::test] + async fn planned_prebid_instances_preserve_headers_metadata_suppression_and_identity() { + let http = Arc::new(StubHttpClient::new()); + http.push_response( + 200, + serde_json::to_vec(&serde_json::json!({ + "seatbid": [{"seat": "suppress-exact", "bid": [ + {"id":"good-a","impid":"fictional-slot","price":1.25,"adm":"
a
","w":300,"h":250,"nurl":"https://notify.example/win","burl":"https://notify.example/bill","ext":{"prebid":{"cache":{"bids":{"cacheId":"cache-a","url":"https://cache-a.example/cache/path"}}}}}, + {"id":"bad-a","price":2.0} + ]}], + "ext": {"responsetimemillis":{"suppress-exact":4},"errors":{"other":["fictional"]},"warnings":{"other":["warning"]},"debug":{"httpcalls":[]},"prebid":{"bidstatus":{"suppress-exact":[{"bidid":"good-a"}]}}} + })) + .expect("should serialize PBS response a"), + ); + http.push_response( + 200, + serde_json::to_vec(&serde_json::json!({ + "seatbid": [{"seat": "keep-seat", "bid": [{ + "id":"good-b","impid":"fictional-slot","price":2.5,"adm":"
b
","w":300,"h":250,"nurl":"https://notify.example/win","burl":"https://notify.example/bill" + }]}] + })) + .expect("should serialize PBS response b"), + ); + let backend = Arc::new(NamingBackend::new(BackendNamingPolicy::Fastly)); + let services = build_services_with_backend_and_http_client( + Arc::clone(&backend) as Arc<_>, + Arc::clone(&http) as Arc<_>, + ); + let notifications = NotificationConfig { + suppress_all: false, + suppress_seats: vec!["suppress-exact".to_string()], + }; + let plan = AuctionPlan::compile(planned_prebid_config(&[ + ( + "pbs-a", + serde_json::json!({"debug":true,"test_mode":true,"consent_forwarding":"openrtb_only"}), + notifications, + ), + ("pbs-b", serde_json::json!({}), NotificationConfig::default()), + ])) + .expect("should compile planned PBS auction"); + let orchestrator = AuctionOrchestratorHarness::new(plan, None); + let request = planned_prebid_request(); + let settings = create_test_settings(); + let inbound = http::Request::builder() + .uri("https://publisher.example/auction") + .header( + http::header::COOKIE, + "consent=keep; euconsent-v2=drop; other=value", + ) + .header(http::header::USER_AGENT, "Fictional Browser/7") + .header(http::header::REFERER, "https://referrer.example/story") + .header(http::header::ACCEPT_LANGUAGE, "en-US,en;q=0.9") + .header("x-forwarded-for", "203.0.113.250") + .body(edgezero_core::body::Body::empty()) + .expect("should build inbound request"); + let context = AuctionContext { + settings: &settings, + request: &inbound, + timeout_ms: 777, + transport_timeout_ms: 777, + provider_responses: None, + services: &services, + }; + + let result = orchestrator + .run_auction(&request, &context) + .await + .expect("should execute planned PBS auction"); + + assert_eq!(result.provider_responses.len(), 2); + let first = &result.provider_responses[0]; + assert_eq!(first.provider, "pbs-a"); + assert_eq!(first.bids.len(), 1, "should isolate malformed sibling"); + assert_eq!( + first.bids[0].returned_seat.as_deref(), + Some("suppress-exact") + ); + assert_eq!(first.bids[0].bidder, "suppress-exact"); + assert!( + first.bids[0].nurl.is_none(), + "should suppress after normalization" + ); + assert!( + first.bids[0].burl.is_none(), + "should suppress billing notification" + ); + assert_eq!(first.bids[0].cache_id.as_deref(), Some("cache-a")); + assert_eq!(first.bids[0].cache_host.as_deref(), Some("cache-a.example")); + assert_eq!(first.bids[0].cache_path.as_deref(), Some("/cache/path")); + assert_eq!(first.metadata["responsetimemillis"]["suppress-exact"], 4); + assert!(first.metadata.contains_key("errors")); + assert!(first.metadata.contains_key("warnings")); + assert!(first.metadata.contains_key("debug")); + assert!(first.metadata.contains_key("bidstatus")); + let second = &result.provider_responses[1]; + assert_eq!(second.provider, "pbs-b"); + assert_eq!(second.bids[0].returned_seat.as_deref(), Some("keep-seat")); + assert!(second.bids[0].nurl.is_some()); + assert!(!second.metadata.contains_key("debug")); + assert!(!second.metadata.contains_key("bidstatus")); + + let headers = http.recorded_request_headers(); + assert_eq!(headers.len(), 2); + for request_headers in &headers { + assert!( + request_headers + .iter() + .any(|(name, value)| name == "user-agent" && value == "Fictional Browser/7") + ); + assert!(request_headers.iter().any( + |(name, value)| name == "referer" && value == "https://referrer.example/story" + )); + assert!( + request_headers + .iter() + .any(|(name, value)| name == "accept-language" && value == "en-US,en;q=0.9") + ); + assert!( + request_headers + .iter() + .all(|(name, _)| name != "x-forwarded-for"), + "must ignore inbound XFF without attestation" + ); + assert!( + request_headers.iter().all(|(name, _)| name != "accept"), + "planned PBS transport must not add Accept beyond legacy headers" + ); + } + let first_cookie = headers[0] + .iter() + .find(|(name, _)| name == "cookie") + .map(|(_, value)| value.as_str()); + assert_eq!(first_cookie, Some("consent=keep; other=value")); + let second_cookie = headers[1] + .iter() + .find(|(name, _)| name == "cookie") + .map(|(_, value)| value.as_str()); + assert_eq!( + second_cookie, + Some("consent=keep; euconsent-v2=drop; other=value") + ); + } + + #[tokio::test] + async fn planned_aps_mock_mediation_preserves_three_identities_and_renderer() { + let http = Arc::new(StubHttpClient::new()); + http.push_response( + 200, + serde_json::to_vec(&serde_json::json!({ + "seatbid": [{"seat": "upstream-seat", "bid": [{ + "id": "aps-bid", "impid": "fictional-slot", "price": 2.0, + "w": 300, "h": 250, + "ext": {"creativeurl": "https://creative.example/render", "tagtype": "iframe"} + }]}] + })) + .expect("should serialize APS response"), + ); + http.push_response( + 200, + serde_json::to_vec(&serde_json::json!({ + "seatbid": [{"seat": "aps-instance", "bid": [{ + "id": "mediated-aps", "impid": "fictional-slot", "price": 2.0, + "adm": "ignored", "w": 300, "h": 250, "crid": "aps-creative" + }]}] + })) + .expect("should serialize mediator response"), + ); + let backend = Arc::new(NamingBackend::new(BackendNamingPolicy::Axum)); + let services = build_services_with_backend_and_http_client( + Arc::clone(&backend) as Arc<_>, + Arc::clone(&http) as Arc<_>, + ); + let plan = + AuctionPlan::compile(planned_aps_config()).expect("should compile planned APS auction"); + let mediator = AdServerMockProvider::new(AdServerMockConfig { + enabled: true, + endpoint: "https://mediator.example/mediate".to_string(), + timeout_ms: 500, + ..AdServerMockConfig::default() + }); + let orchestrator = AuctionOrchestratorHarness::new(plan, Some(Arc::new(mediator))); + let request = planned_request(); + let settings = create_test_settings(); + let inbound = http::Request::new(edgezero_core::body::Body::empty()); + let context = AuctionContext { + settings: &settings, + request: &inbound, + timeout_ms: 777, + transport_timeout_ms: 777, + provider_responses: None, + services: &services, + }; + + let result = orchestrator + .run_auction(&request, &context) + .await + .expect("should mediate planned APS bid"); + + let provider_bid = &result.provider_responses[0].bids[0]; + assert_eq!(result.provider_responses[0].provider, "aps-instance"); + assert_eq!(provider_bid.returned_seat.as_deref(), Some("upstream-seat")); + assert_eq!(provider_bid.bidder, "aps"); + let winner = &result.winning_bids["fictional-slot"]; + assert_eq!(winner.returned_seat.as_deref(), Some("upstream-seat")); + assert_eq!(winner.bidder, "aps"); + assert!(winner.renderer.is_some()); + assert!(winner.creative.is_none()); + assert_eq!( + result + .mediator_response + .as_ref() + .map(|response| response.provider.as_str()), + Some("adserver_mock") + ); + } + + #[tokio::test] + async fn planned_aps_transport_omits_accept_header() { + let http = Arc::new(StubHttpClient::new()); + http.push_response(400, Vec::new()); + let backend = Arc::new(NamingBackend::new(BackendNamingPolicy::Fastly)); + let services = build_services_with_backend_and_http_client( + Arc::clone(&backend) as Arc<_>, + Arc::clone(&http) as Arc<_>, + ); + let plan = + AuctionPlan::compile(planned_aps_config()).expect("should compile planned APS auction"); + let orchestrator = AuctionOrchestratorHarness::new(plan, None); + let request = planned_request(); + let settings = create_test_settings(); + let inbound = http::Request::builder() + .uri("https://publisher.example/auction") + .body(edgezero_core::body::Body::empty()) + .expect("should build inbound request"); + let context = AuctionContext { + settings: &settings, + request: &inbound, + timeout_ms: 777, + transport_timeout_ms: 777, + provider_responses: None, + services: &services, + }; + + orchestrator + .run_auction(&request, &context) + .await + .expect("should execute planned APS auction"); + + let headers = http.recorded_request_headers(); + assert_eq!(headers.len(), 1); + assert!( + headers[0].iter().all(|(name, _)| name != "accept"), + "planned APS transport must not add Accept beyond legacy headers" + ); + } + + #[tokio::test] + async fn planned_aps_profile_normalizes_renderer_reduction_and_metadata() { + let http = Arc::new(StubHttpClient::new()); + http.push_response_with_headers( + 200, + serde_json::to_vec(&serde_json::json!({ + "cur": "USD", + "seatbid": [ + {"seat": "returned-seat", "bid": [ + {"id": "z-high", "impid": "fictional-slot", "price": 2.0, "w": 300, "h": 250, + "nurl": "https://notice.example/win", "burl": "https://notice.example/bill", + "crid": "fictional-creative", "adomain": ["advertiser.example"], + "ext": {"creativeurl": "https://creative.example/render", "tagtype": "iframe"}}, + {"id": "a-high", "impid": "fictional-slot", "price": 2.0, "w": 300, "h": 250, + "ext": {"creativeurl": "https://creative.example/render", "tagtype": "iframe"}}, + {"id": "bad-script", "impid": "fictional-slot", "price": 9.0, "w": 300, "h": 250, + "ext": {"creativeurl": "https://creative.example/render", "tagtype": "script"}}, + {"id": "bad-domain", "impid": "fictional-slot", "price": 8.0, "w": 300, "h": 250, + "ext": {"creativeurl": "https://publisher.example/render", "tagtype": "iframe"}}, + {"id": "bad-credentials", "impid": "fictional-slot", "price": 8.0, "w": 300, "h": 250, + "ext": {"creativeurl": "https://user:password@creative.example/render", "tagtype": "iframe"}}, + {"id": "bad-imp", "impid": "unknown-slot", "price": 8.0, "w": 300, "h": 250, + "ext": {"creativeurl": "https://creative.example/render", "tagtype": "iframe"}}, + {"id": "bad-dimensions", "impid": "fictional-slot", "price": 8.0, "w": 320, "h": 50, + "ext": {"creativeurl": "https://creative.example/render", "tagtype": "iframe"}}, + {"id": "bad-price", "impid": "fictional-slot", "price": "high", "w": 300, "h": 250, + "ext": {"creativeurl": "https://creative.example/render", "tagtype": "iframe"}}, + {"id": "bad-mtype", "impid": "fictional-slot", "price": 8.0, "mtype": 2, "w": 300, "h": 250, + "ext": {"creativeurl": "https://creative.example/render", "tagtype": "iframe"}}, + {"id": "bad-tag", "impid": "fictional-slot", "price": 8.0, "w": 300, "h": 250, + "ext": {"creativeurl": "https://creative.example/render", "tagtype": "native"}}, + {"id": "bad-crid", "impid": "fictional-slot", "price": 8.0, "w": 300, "h": 250, + "crid": "x".repeat(1025), + "ext": {"creativeurl": "https://creative.example/render", "tagtype": "iframe"}}, + {"impid": "fictional-slot", "price": 8.0, "w": 300, "h": 250, + "ext": {"creativeurl": "https://creative.example/render", "tagtype": "iframe"}} + ]}, + {"seat": 7, "bid": "bad-shape"} + ] + })) + .expect("should serialize APS profile response"), + vec![ + ("content-type", "application/json"), + ("authorization", "fictional-secret"), + ], + ); + let backend = Arc::new(NamingBackend::new(BackendNamingPolicy::Fastly)); + let services = build_services_with_backend_and_http_client( + Arc::clone(&backend) as Arc<_>, + Arc::clone(&http) as Arc<_>, + ); + let plan = AuctionPlan::compile(planned_aps_instances_config(&[( + "aps-instance", + serde_json::json!({"account_id": "example-account", "debug": true}), + NotificationConfig { + suppress_all: false, + suppress_seats: vec!["returned-seat".to_string()], + }, + )])) + .expect("should compile planned APS profile"); + let orchestrator = AuctionOrchestratorHarness::new(plan, None); + let request = planned_request(); + let settings = create_test_settings(); + let inbound = http::Request::new(edgezero_core::body::Body::empty()); + let context = AuctionContext { + settings: &settings, + request: &inbound, + timeout_ms: 777, + transport_timeout_ms: 777, + provider_responses: None, + services: &services, + }; + + let result = orchestrator + .run_auction(&request, &context) + .await + .expect("should execute planned APS profile"); + + let response = &result.provider_responses[0]; + assert_eq!(response.provider, "aps-instance"); + assert_eq!(response.status, BidStatus::Success); + assert_eq!( + response.bids.len(), + 1, + "should retain one bid per impression" + ); + let bid = &response.bids[0]; + assert_eq!(bid.bidder, "aps"); + assert_eq!(bid.returned_seat.as_deref(), Some("returned-seat")); + assert_eq!( + bid.bid_id.as_deref(), + Some("a-high"), + "lexical ID should break equal-price tie" + ); + assert!(bid.creative.is_none()); + assert!( + bid.nurl.is_none() && bid.burl.is_none(), + "APS must discard notification URLs" + ); + let renderer = bid + .renderer + .as_ref() + .and_then(BidRenderer::as_aps) + .expect("should construct typed APS renderer"); + assert_eq!(renderer.account_id, "example-account"); + let decoded = base64::engine::general_purpose::STANDARD + .decode(&renderer.aax_response) + .expect("should decode minimized APS response"); + assert_eq!( + serde_json::from_slice::(&decoded) + .expect("should parse minimized APS response"), + serde_json::json!({"seatbid":[{"bid":[{ + "id":"a-high","price":2.0,"w":300,"h":250, + "ext":{"creativeurl":"https://creative.example/render","tagtype":"iframe"} + }]}]}) + ); + assert_eq!(response.metadata["seatbid_count"], 2); + assert_eq!(response.metadata["accepted_bid_count"], 1); + assert_eq!(response.metadata["dropped_bid_count"], 12); + for reason in [ + "lost_to_higher_bid", + "script_rendering_disabled", + "unknown_impid", + "invalid_dimensions", + "invalid_price", + "unsupported_media_type", + "unsupported_tagtype", + "creative_id_too_large", + "missing_render_source", + "empty_seatbid_bids", + ] { + assert_eq!(response.metadata["drop_reasons"][reason], 1, "{reason}"); + } + assert_eq!( + response.metadata["drop_reasons"]["invalid_creative_url"], 2, + "same-publisher and credentialed URLs should both be rejected" + ); + assert_eq!( + response.metadata["routing"]["unused_bidder_params_count"], + 0 + ); + let debug = &response.metadata["debug"]["httpcalls"]["aps"][0]; + assert_eq!(debug["uri"], "https://aps.example/e/pb/bid"); + assert_eq!( + debug["responseheaders"], + serde_json::json!({"content-type": ["application/json"]}), + "async stub should preserve queued response headers" + ); + assert!( + debug["requestbody"] + .as_str() + .is_some_and(|body| body.contains("example-account")) + ); + assert!(debug["requestheaders"].get("authorization").is_none()); + assert!(debug["responseheaders"].get("authorization").is_none()); + } + + #[tokio::test] + async fn two_planned_aps_instances_correlate_independently() { + let http = Arc::new(StubHttpClient::new()); + for (seat, id, price) in [("seat-a", "bid-a", 1.0), ("seat-b", "bid-b", 2.0)] { + http.push_response( + 200, + serde_json::to_vec(&serde_json::json!({"seatbid":[{"seat":seat,"bid":[{ + "id":id,"impid":"fictional-slot","price":price,"w":300,"h":250, + "ext":{"creativeurl":"https://creative.example/render","tagtype":"iframe"} + }]}]})) + .expect("should serialize APS instance response"), + ); + } + let backend = Arc::new(NamingBackend::new(BackendNamingPolicy::Fastly)); + let services = build_services_with_backend_and_http_client( + Arc::clone(&backend) as Arc<_>, + Arc::clone(&http) as Arc<_>, + ); + let plan = AuctionPlan::compile(planned_aps_instances_config(&[ + ( + "aps-a", + serde_json::json!({"account_id":"account-a"}), + NotificationConfig::default(), + ), + ( + "aps-b", + serde_json::json!({"account_id":"account-b"}), + NotificationConfig::default(), + ), + ])) + .expect("should compile two APS instances"); + let orchestrator = AuctionOrchestratorHarness::new(plan, None); + let request = planned_request(); + let settings = create_test_settings(); + let inbound = http::Request::new(edgezero_core::body::Body::empty()); + let context = AuctionContext { + settings: &settings, + request: &inbound, + timeout_ms: 777, + transport_timeout_ms: 777, + provider_responses: None, + services: &services, + }; + + let result = orchestrator + .run_auction(&request, &context) + .await + .expect("should execute two APS instances"); + + assert_eq!(result.provider_responses.len(), 2); + assert_eq!(result.provider_responses[0].provider, "aps-a"); + assert_eq!( + result.provider_responses[0].bids[0].bid_id.as_deref(), + Some("bid-a") + ); + assert_eq!(result.provider_responses[1].provider, "aps-b"); + assert_eq!( + result.provider_responses[1].bids[0].bid_id.as_deref(), + Some("bid-b") + ); + assert_eq!(http.recorded_request_bodies().len(), 2); + assert_eq!( + result.winning_bids["fictional-slot"].bid_id.as_deref(), + Some("bid-b"), + "global ranking should remain orchestrator-owned" + ); + let specs = backend.specs.lock().expect("should lock specs"); + assert_eq!(specs.len(), 2); + assert_ne!(specs[0].discriminator, specs[1].discriminator); + } + + #[tokio::test] + async fn planned_aps_returned_seat_accepts_only_valid_nonempty_strings() { + let plan = AuctionPlan::compile(planned_aps_config()).expect("should compile APS plan"); + let routed = route_auction( + planned_request(), + &http::Request::new(edgezero_core::body::Body::empty()), + &plan, + None, + ); + let provider = GenericOpenRtbProvider::new(plan.providers()[0].clone()); + for (seat, expected) in [ + (serde_json::Value::Null, None), + (serde_json::json!(7), None), + (serde_json::json!(""), None), + (serde_json::json!("exact-seat"), Some("exact-seat")), + ] { + let state = provider.parse_state_for_test(routed.inputs()[0].clone()); + let response = PlatformResponse::new( + edgezero_core::http::response_builder() + .status(200) + .body(edgezero_core::body::Body::from( + serde_json::to_vec(&serde_json::json!({"seatbid":[{"seat":seat,"bid":[{ + "id":"bid","impid":"fictional-slot","price":1.0,"w":300,"h":250, + "nurl":"https://notice.example/win","burl":"https://notice.example/bill", + "ext":{"creativeurl":"https://creative.example/render","tagtype":"iframe"} + }]}]})) + .expect("should serialize seat identity response"), + )) + .expect("should build seat identity response"), + ); + let parsed = provider + .parse_response_with_state(response, 4, Some(state.as_ref())) + .await + .expect("should parse seat identity response"); + assert_eq!(parsed.bids[0].returned_seat.as_deref(), expected); + assert!(parsed.bids[0].nurl.is_none() && parsed.bids[0].burl.is_none()); + } + } + + #[tokio::test] + async fn planned_aps_response_status_shape_and_currency_matrix() { + let plan = AuctionPlan::compile(planned_aps_config()).expect("should compile APS plan"); + let routed = route_auction( + planned_request(), + &http::Request::new(edgezero_core::body::Body::empty()), + &plan, + None, + ); + let provider = GenericOpenRtbProvider::new(plan.providers()[0].clone()); + let cases = [ + (204, Vec::new(), BidStatus::NoBid, None, None), + (400, Vec::new(), BidStatus::Error, None, Some("http_status")), + ( + 200, + b"not-json".to_vec(), + BidStatus::Error, + Some("unexpected_response_shape"), + Some("parse_response"), + ), + ( + 200, + b"[]".to_vec(), + BidStatus::Error, + Some("unexpected_response_shape"), + Some("parse_response"), + ), + ( + 200, + br#"{"contextual":true}"#.to_vec(), + BidStatus::Error, + Some("unexpected_response_shape"), + Some("parse_response"), + ), + ( + 200, + br#"{"cur":"EUR","seatbid":[]}"#.to_vec(), + BidStatus::NoBid, + Some("unsupported_currency"), + None, + ), + ]; + for (status, body, expected, reason, error_type) in cases { + let state = provider.parse_state_for_test(routed.inputs()[0].clone()); + let response = PlatformResponse::new( + edgezero_core::http::response_builder() + .status(status) + .body(edgezero_core::body::Body::from(body)) + .expect("should build APS matrix response"), + ); + let parsed = provider + .parse_response_with_state(response, 4, Some(state.as_ref())) + .await + .expect("should classify APS matrix response"); + assert_eq!(parsed.status, expected, "status {status}"); + if let Some(reason) = reason { + assert_eq!( + parsed.metadata["drop_reasons"][reason], 1, + "status {status}" + ); + } + if let Some(error_type) = error_type { + assert_eq!(parsed.metadata["error_type"], error_type, "status {status}"); + } + } + } + + #[tokio::test] + async fn planned_provider_outcome_matrix_has_fixed_count_only_routing_metadata() { + let standard_plan = AuctionPlan::compile(planned_config( + &[("standard", RoutingMode::AllEligible)], + false, + )) + .expect("should compile standard plan"); + let prebid_plan = AuctionPlan::compile(planned_prebid_config(&[( + "pbs", + serde_json::json!({}), + NotificationConfig::default(), + )])) + .expect("should compile PBS plan"); + let cases = [ + (&standard_plan, 204, Vec::new(), BidStatus::NoBid), + (&standard_plan, 502, Vec::new(), BidStatus::Error), + (&standard_plan, 200, b"not-json".to_vec(), BidStatus::Error), + ( + &standard_plan, + 200, + br#"{"seatbid":[]}"#.to_vec(), + BidStatus::NoBid, + ), + (&prebid_plan, 204, b"{}".to_vec(), BidStatus::NoBid), + (&prebid_plan, 502, Vec::new(), BidStatus::Error), + (&prebid_plan, 200, b"not-json".to_vec(), BidStatus::Error), + ( + &prebid_plan, + 200, + br#"{"seatbid":[]}"#.to_vec(), + BidStatus::NoBid, + ), + ]; + + for (plan, status, body, expected) in cases { + let routed = route_auction( + planned_prebid_request(), + &http::Request::new(edgezero_core::body::Body::empty()), + plan, + None, + ); + let provider = GenericOpenRtbProvider::new(plan.providers()[0].clone()); + let state = provider.parse_state_for_test(routed.inputs()[0].clone()); + let response = PlatformResponse::new( + edgezero_core::http::response_builder() + .status(status) + .body(edgezero_core::body::Body::from(body)) + .expect("should build provider matrix response"), + ); + let parsed = provider + .parse_response_with_state(response, 4, Some(state.as_ref())) + .await + .expect("should classify provider matrix response"); + assert_eq!(parsed.status, expected, "status {status}"); + if expected == BidStatus::Error { + let expected_error_type = if (200..300).contains(&status) { + "parse_response" + } else { + "http_status" + }; + assert_eq!(parsed.metadata["error_type"], expected_error_type); + } + assert_eq!( + parsed.metadata["routing"], + serde_json::json!({"unused_bidder_params_count": 0}) + ); + let serialized = serde_json::to_string(&parsed.metadata["routing"]) + .expect("should serialize routing metadata"); + assert!(!serialized.contains("fictional-provider")); + assert!(!serialized.contains("fictional-slot")); + } + } + + #[tokio::test] + async fn planned_aps_script_opt_in_matches_shared_renderer_fixture() { + let plan = AuctionPlan::compile(planned_aps_instances_config(&[( + "aps-instance", + serde_json::json!({ + "account_id":"example-account-id", + "allow_script_creatives":true + }), + NotificationConfig::default(), + )])) + .expect("should compile script-enabled APS plan"); + let routed = route_auction( + planned_request(), + &http::Request::new(edgezero_core::body::Body::empty()), + &plan, + None, + ); + let provider = GenericOpenRtbProvider::new(plan.providers()[0].clone()); + let state = provider.parse_state_for_test(routed.inputs()[0].clone()); + let response = PlatformResponse::new( + edgezero_core::http::response_builder() + .status(200) + .body(edgezero_core::body::Body::from( + serde_json::to_vec(&serde_json::json!({"seatbid":[{"bid":[{ + "id":"fictional-selected-bid-id","impid":"fictional-slot","price":1.23, + "w":300,"h":250,"crid":"fictional-creative", + "ext":{"creativeurl":"https://creative.example/render","tagtype":"iframe"} + },{ + "id":"script-bid","impid":"fictional-slot","price":1.0, + "w":300,"h":250, + "ext":{"creativeurl":"https://creative.example/script","tagtype":"script"} + }]}]})) + .expect("should serialize APS renderer fixture response"), + )) + .expect("should build APS renderer fixture response"), + ); + + let parsed = provider + .parse_response_with_state(response, 3, Some(state.as_ref())) + .await + .expect("should parse APS renderer fixture response"); + + assert_eq!(parsed.status, BidStatus::Success); + assert_eq!( + parsed.metadata["drop_reasons"]["lost_to_higher_bid"], 1, + "enabled script creative should be eligible before reduction" + ); + let renderer = parsed.bids[0] + .renderer + .as_ref() + .and_then(BidRenderer::as_aps) + .expect("should construct APS renderer"); + let decoded = base64::engine::general_purpose::STANDARD + .decode(&renderer.aax_response) + .expect("should decode APS fixture envelope"); + let fixture: serde_json::Value = serde_json::from_str(include_str!( + "../../../trusted-server-js/lib/test/fixtures/aps-renderer-v1.json" + )) + .expect("should parse shared APS renderer fixture"); + assert_eq!( + serde_json::from_slice::(&decoded) + .expect("should parse decoded APS renderer"), + fixture + ); + } + + #[tokio::test] + async fn planned_aps_debug_response_headers_are_allowlisted() { + let plan = AuctionPlan::compile(planned_aps_instances_config(&[( + "aps-instance", + serde_json::json!({"account_id":"example-account","debug":true}), + NotificationConfig::default(), + )])) + .expect("should compile debug APS plan"); + let routed = route_auction( + planned_request(), + &http::Request::new(edgezero_core::body::Body::empty()), + &plan, + None, + ); + let provider = GenericOpenRtbProvider::new(plan.providers()[0].clone()); + let state = provider.parse_state_for_test(routed.inputs()[0].clone()); + let response = PlatformResponse::new( + edgezero_core::http::response_builder() + .status(200) + .header("content-type", "application/json") + .header("authorization", "fictional-secret") + .body(edgezero_core::body::Body::from("{}")) + .expect("should build debug APS response"), + ); + + let parsed = provider + .parse_response_with_state(response, 3, Some(state.as_ref())) + .await + .expect("should parse debug APS response"); + + let headers = &parsed.metadata["debug"]["httpcalls"]["aps"][0]["responseheaders"]; + assert_eq!( + headers, + &serde_json::json!({"content-type":["application/json"]}) + ); + assert!(headers.get("authorization").is_none()); + } + + #[tokio::test] + async fn planned_standard_instances_have_distinct_backends_and_independent_results() { + let http = Arc::new(StubHttpClient::new()); + http.push_response( + 200, + serde_json::to_vec(&serde_json::json!({ + "seatbid": [{"seat": "fictional-seat-a", "bid": [{ + "id": "bid-a", "impid": "fictional-slot", "price": 1.25, + "adm": "
a
", "w": 300, "h": 250 + }]}] + })) + .expect("should serialize response a"), + ); + http.push_response( + 200, + serde_json::to_vec(&serde_json::json!({ + "seatbid": [{"seat": "fictional-seat-b", "bid": [{ + "id": "bid-b", "impid": "fictional-slot", "price": 2.5, + "adm": "
b
", "w": 300, "h": 250 + }]}] + })) + .expect("should serialize response b"), + ); + let backend = Arc::new(NamingBackend::new(BackendNamingPolicy::Fastly)); + let services = build_services_with_backend_and_http_client( + Arc::clone(&backend) as Arc<_>, + Arc::clone(&http) as Arc<_>, + ); + let plan = AuctionPlan::compile(planned_config( + &[ + ("provider-a", RoutingMode::AllEligible), + ("provider-b", RoutingMode::AllEligible), + ], + false, + )) + .expect("should compile planned auction"); + let orchestrator = AuctionOrchestratorHarness::new(plan, None); + let request = planned_request(); + let settings = create_test_settings(); + let inbound = http::Request::builder() + .uri("https://publisher.example/auction") + .body(edgezero_core::body::Body::empty()) + .expect("should build inbound request"); + let context = AuctionContext { + settings: &settings, + request: &inbound, + timeout_ms: 777, + transport_timeout_ms: 777, + provider_responses: None, + services: &services, + }; + + let result = orchestrator + .run_auction(&request, &context) + .await + .expect("should execute planned auction"); + + assert_eq!(orchestrator.provider_count(), 2); + assert!(orchestrator.mediator().is_none()); + assert_eq!(result.provider_responses.len(), 2); + assert_eq!(result.provider_responses[0].provider, "provider-a"); + assert_eq!( + result.provider_responses[0].bids[0].bid_id.as_deref(), + Some("bid-a") + ); + assert_eq!( + result.provider_responses[0].bids[0] + .returned_seat + .as_deref(), + Some("fictional-seat-a") + ); + assert_eq!( + result.provider_responses[0].metadata["routing"]["unused_bidder_params_count"], + 0 + ); + assert_eq!(result.provider_responses[1].provider, "provider-b"); + assert_eq!( + result.provider_responses[1].bids[0].bid_id.as_deref(), + Some("bid-b") + ); + assert_eq!( + result.provider_responses[1].bids[0] + .returned_seat + .as_deref(), + Some("fictional-seat-b") + ); + assert_eq!( + result.provider_responses[1].metadata["routing"]["unused_bidder_params_count"], + 0 + ); + assert_eq!( + result.winning_bids["fictional-slot"].bid_id.as_deref(), + Some("bid-b") + ); + let request_headers = http.recorded_request_headers(); + assert_eq!(request_headers.len(), 2); + for headers in request_headers { + assert!( + headers + .iter() + .any(|(name, value)| name == "accept" && value == "application/json"), + "standard planned transport should retain its JSON Accept header" + ); + } + let backend_names = http.recorded_backend_names(); + assert_eq!(backend_names.len(), 2); + assert_ne!(backend_names[0], backend_names[1]); + let request_bodies = http.recorded_request_bodies(); + assert_eq!(request_bodies.len(), 2); + for body in request_bodies { + let value: serde_json::Value = + serde_json::from_slice(&body).expect("should parse planned request"); + let tmax = value["tmax"].as_u64().expect("should include logical tmax"); + assert!( + (750..=777).contains(&tmax), + "logical budget should remain near the auction budget, got {tmax}" + ); + assert_eq!(value["imp"].as_array().map(Vec::len), Some(1)); + } + let specs = backend.specs.lock().expect("should lock specs"); + assert_eq!(specs.len(), 2); + assert_eq!(specs[0].first_byte_timeout, Duration::from_millis(750)); + assert_eq!(specs[1].first_byte_timeout, Duration::from_millis(750)); + assert_ne!(specs[0].discriminator, specs[1].discriminator); + } + + #[tokio::test] + async fn planned_backend_collision_does_not_overwrite_first_launch_state() { + let http = Arc::new(StubHttpClient::new()); + http.push_response( + 200, + serde_json::to_vec(&serde_json::json!({ + "seatbid": [{"seat": "first", "bid": [{ + "id": "first-bid", "impid": "fictional-slot", "price": 2.0, + "adm": "
first
", "w": 300, "h": 250 + }]}] + })) + .expect("should serialize first response"), + ); + let backend = Arc::new(CollidingBackend); + let services = build_services_with_backend_and_http_client( + Arc::clone(&backend) as Arc<_>, + Arc::clone(&http) as Arc<_>, + ); + let plan = AuctionPlan::compile(planned_config( + &[ + ("provider-a", RoutingMode::AllEligible), + ("provider-b", RoutingMode::AllEligible), + ], + false, + )) + .expect("should compile planned auction"); + let orchestrator = AuctionOrchestratorHarness::new(plan, None); + let request = planned_request(); + let settings = create_test_settings(); + let inbound = http::Request::new(edgezero_core::body::Body::empty()); + let context = AuctionContext { + settings: &settings, + request: &inbound, + timeout_ms: 777, + transport_timeout_ms: 777, + provider_responses: None, + services: &services, + }; + + let result = orchestrator + .run_auction(&request, &context) + .await + .expect("should isolate backend collision"); + + assert_eq!(http.recorded_backend_names().len(), 1); + assert_eq!(result.provider_responses[0].provider, "provider-a"); + assert_eq!( + result.provider_responses[0].bids[0].bid_id.as_deref(), + Some("first-bid") + ); + assert_eq!(result.provider_responses[1].provider, "provider-b"); + assert_eq!( + result.provider_responses[1].metadata["error_type"], + "launch_failed" + ); + } + + #[tokio::test] + async fn planned_pending_backend_divergence_isolated_from_valid_provider() { + let http = Arc::new(StubHttpClient::new()); + http.push_response(204, Vec::new()); + http.push_response(204, Vec::new()); + http.push_pending_backend_name_override(Some("divergent-backend")); + let backend = Arc::new(NamingBackend::new(BackendNamingPolicy::Axum)); + let services = build_services_with_backend_and_http_client( + Arc::clone(&backend) as Arc<_>, + Arc::clone(&http) as Arc<_>, + ); + let plan = AuctionPlan::compile(planned_config( + &[ + ("provider-a", RoutingMode::AllEligible), + ("provider-b", RoutingMode::AllEligible), + ], + false, + )) + .expect("should compile planned auction"); + let orchestrator = AuctionOrchestratorHarness::new(plan, None); + let request = planned_request(); + let settings = create_test_settings(); + let inbound = http::Request::new(edgezero_core::body::Body::empty()); + let context = AuctionContext { + settings: &settings, + request: &inbound, + timeout_ms: 777, + transport_timeout_ms: 777, + provider_responses: None, + services: &services, + }; + + let result = orchestrator + .run_auction(&request, &context) + .await + .expect("should isolate divergent pending backend"); + + assert_eq!(result.provider_responses.len(), 2); + assert_eq!(result.provider_responses[0].provider, "provider-a"); + assert_eq!( + result.provider_responses[0].metadata["error_type"], + "launch_failed" + ); + assert_eq!(result.provider_responses[1].provider, "provider-b"); + assert_eq!(result.provider_responses[1].status, BidStatus::NoBid); + } + + #[tokio::test] + async fn planned_pending_backend_missing_isolated_from_valid_provider() { + let http = Arc::new(StubHttpClient::new()); + http.push_response(204, Vec::new()); + http.push_response(204, Vec::new()); + http.push_pending_backend_name_override(None); + let backend = Arc::new(NamingBackend::new(BackendNamingPolicy::Axum)); + let services = build_services_with_backend_and_http_client( + Arc::clone(&backend) as Arc<_>, + Arc::clone(&http) as Arc<_>, + ); + let plan = AuctionPlan::compile(planned_config( + &[ + ("provider-a", RoutingMode::AllEligible), + ("provider-b", RoutingMode::AllEligible), + ], + false, + )) + .expect("should compile planned auction"); + let orchestrator = AuctionOrchestratorHarness::new(plan, None); + let request = planned_request(); + let settings = create_test_settings(); + let inbound = http::Request::new(edgezero_core::body::Body::empty()); + let context = AuctionContext { + settings: &settings, + request: &inbound, + timeout_ms: 777, + transport_timeout_ms: 777, + provider_responses: None, + services: &services, + }; + + let result = orchestrator + .run_auction(&request, &context) + .await + .expect("should isolate missing pending backend"); + + assert_eq!(result.provider_responses.len(), 2); + assert_eq!(result.provider_responses[0].provider, "provider-a"); + assert_eq!( + result.provider_responses[0].metadata["error_type"], + "launch_failed" + ); + assert_eq!(result.provider_responses[1].provider, "provider-b"); + assert_eq!(result.provider_responses[1].status, BidStatus::NoBid); + } + + #[tokio::test] + async fn planned_same_profile_rejects_cross_provider_parse_state() { + let plan = AuctionPlan::compile(planned_config( + &[ + ("provider-a", RoutingMode::AllEligible), + ("provider-b", RoutingMode::AllEligible), + ], + false, + )) + .expect("should compile planned auction"); + let routed = route_auction( + planned_request(), + &http::Request::new(edgezero_core::body::Body::empty()), + &plan, + None, + ); + let provider_a = GenericOpenRtbProvider::new(plan.providers()[0].clone()); + let provider_b = GenericOpenRtbProvider::new(plan.providers()[1].clone()); + let parse_state = provider_a.parse_state_for_test(routed.inputs()[0].clone()); + let response = PlatformResponse::new( + edgezero_core::http::response_builder() + .status(204) + .body(edgezero_core::body::Body::empty()) + .expect("should build no-content response"), + ); + + let error = provider_b + .parse_response_with_state(response, 1, Some(parse_state.as_ref())) + .await + .expect_err("should reject another provider's parse state"); + + assert!( + error.to_string().contains("owned by provider provider-a"), + "should identify cross-provider state ownership" + ); + } + + #[tokio::test] + async fn planned_prebid_rejects_cross_provider_parse_state() { + let plan = AuctionPlan::compile(planned_prebid_config(&[ + ( + "pbs-a", + serde_json::json!({}), + NotificationConfig::default(), + ), + ( + "pbs-b", + serde_json::json!({}), + NotificationConfig::default(), + ), + ])) + .expect("should compile planned PBS auction"); + let routed = route_auction( + planned_prebid_request(), + &http::Request::new(edgezero_core::body::Body::empty()), + &plan, + None, + ); + let provider_a = GenericOpenRtbProvider::new(plan.providers()[0].clone()); + let provider_b = GenericOpenRtbProvider::new(plan.providers()[1].clone()); + let parse_state = provider_a.parse_state_for_test(routed.inputs()[0].clone()); + let response = PlatformResponse::new( + edgezero_core::http::response_builder() + .status(200) + .body(edgezero_core::body::Body::from_bytes(b"{}".as_slice())) + .expect("should build PBS response"), + ); + + let error = provider_b + .parse_response_with_state(response, 1, Some(parse_state.as_ref())) + .await + .expect_err("should reject another PBS provider's parse state"); + + assert!( + error.to_string().contains("owned by provider pbs-a"), + "should identify cross-provider PBS state ownership" + ); + } + + #[tokio::test] + async fn planned_skipped_provider_has_no_io_and_no_zero_impression_launch() { + let http = Arc::new(StubHttpClient::new()); + http.push_response(204, Vec::new()); + let backend = Arc::new(NamingBackend::new(BackendNamingPolicy::Axum)); + let services = build_services_with_backend_and_http_client( + Arc::clone(&backend) as Arc<_>, + Arc::clone(&http) as Arc<_>, + ); + let plan = AuctionPlan::compile(planned_config( + &[ + ("eligible", RoutingMode::AllEligible), + ("skipped", RoutingMode::Explicit), + ], + false, + )) + .expect("should compile planned auction"); + let orchestrator = AuctionOrchestratorHarness::new(plan, None); + let request = planned_request(); + let settings = create_test_settings(); + let inbound = http::Request::new(edgezero_core::body::Body::empty()); + let context = AuctionContext { + settings: &settings, + request: &inbound, + timeout_ms: 777, + transport_timeout_ms: 777, + provider_responses: None, + services: &services, + }; + + let result = orchestrator + .run_auction(&request, &context) + .await + .expect("should execute eligible provider only"); + + assert_eq!(http.recorded_backend_names().len(), 1); + assert_eq!(backend.ensured.load(Ordering::Relaxed), 1); + let skipped = result + .provider_responses + .iter() + .find(|response| response.provider == "skipped") + .expect("should materialize skipped provider"); + assert_eq!(skipped.status, BidStatus::NoBid); + assert_eq!( + skipped.metadata["routing"]["skipped_no_eligible_slots"], + true + ); + let request_body = &http.recorded_request_bodies()[0]; + let request_value: serde_json::Value = + serde_json::from_slice(request_body).expect("should parse request body"); + assert_eq!(request_value["imp"].as_array().map(Vec::len), Some(1)); + } + + #[tokio::test] + async fn zero_canonical_timeout_skips_plan_backed_direct_and_split_launches() { + for split in [false, true] { + let http = Arc::new(StubHttpClient::new()); + let backend = Arc::new(ZeroCanonicalBackend::new()); + let services = build_services_with_backend_and_http_client( + Arc::clone(&backend) as Arc<_>, + Arc::clone(&http) as Arc<_>, + ); + let mut config = planned_config(&[("provider", RoutingMode::AllEligible)], false); + config.mediator = Some("adserver_mock".to_string()); + let plan = AuctionPlan::compile(config).expect("should compile planned auction"); + let mediator_predicted = Arc::new(Mutex::new(Vec::new())); + let mediator_requested = Arc::new(Mutex::new(Vec::new())); + let mediator = Arc::new(recording_provider( + "adserver_mock", + "mediator-backend", + 777, + &mediator_predicted, + &mediator_requested, + )); + let orchestrator = AuctionOrchestrator::from_plan(Arc::new(plan), Some(mediator)); + let request = planned_request(); + let settings = create_test_settings(); + let inbound = http::Request::new(edgezero_core::body::Body::empty()); + let context = AuctionContext { + settings: &settings, + request: &inbound, + timeout_ms: 777, + transport_timeout_ms: 777, + provider_responses: None, + services: &services, + }; + + let result = if split { + let DispatchAuctionOutcome::Dispatched(dispatched) = + orchestrator.dispatch_auction(&request, &context).await + else { + panic!("zero canonical timeout should materialize a split response"); + }; + orchestrator + .collect_dispatched_auction(dispatched, &services, &context) + .await + } else { + orchestrator + .run_auction(&request, &context) + .await + .expect("should materialize direct timeout response") + }; + + assert_eq!(result.provider_responses.len(), 1); + assert_eq!( + result.provider_responses[0].metadata["error_type"], + "timeout" + ); + assert_eq!(backend.predicted.load(Ordering::Relaxed), 0); + assert_eq!(backend.ensured.load(Ordering::Relaxed), 0); + assert!(http.recorded_backend_names().is_empty()); + assert!( + mediator_predicted + .lock() + .expect("should lock mediator predictions") + .is_empty() + ); + assert!( + mediator_requested + .lock() + .expect("should lock mediator requests") + .is_empty() + ); + } + } + + #[tokio::test] + async fn planned_launch_transport_parse_failures_are_isolated_from_valid_winner_and_floor() { + let http = Arc::new(StubHttpClient::new()); + // BTreeMap plan order is alphabetical: below-floor, parse-fail, + // transport-fail, valid-winner. Queue responses in that exact order. + http.push_response( + 200, + serde_json::to_vec(&serde_json::json!({ + "seatbid": [{"seat": "below-floor", "bid": [{ + "id": "below", "impid": "fictional-slot", "price": 0.5, + "adm": "
below
", "w": 300, "h": 250 + }, { + "id": "below-only", "impid": "below-only-slot", "price": 0.5, + "adm": "
below only
", "w": 300, "h": 250 + }]}] + })) + .expect("should serialize below-floor response"), + ); + http.push_response(200, b"not-json".to_vec()); + http.push_response(200, b"{}".to_vec()); + http.push_response( + 200, + serde_json::to_vec(&serde_json::json!({ + "seatbid": [{"seat": "winner", "bid": [{ + "id": "winner", "impid": "fictional-slot", "price": 2.0, + "adm": "
winner
", "w": 300, "h": 250 + }]}] + })) + .expect("should serialize winner response"), + ); + http.push_select_success(); + http.push_select_success(); + http.push_select_error(); + let backend = Arc::new(NamingBackend::new(BackendNamingPolicy::Axum)); + backend.fail_ensure_for("launch-fail"); + let services = build_services_with_backend_and_http_client( + Arc::clone(&backend) as Arc<_>, + Arc::clone(&http) as Arc<_>, + ); + let plan = AuctionPlan::compile(planned_config( + &[ + ("launch-fail", RoutingMode::AllEligible), + ("transport-fail", RoutingMode::AllEligible), + ("parse-fail", RoutingMode::AllEligible), + ("below-floor", RoutingMode::AllEligible), + ("valid-winner", RoutingMode::AllEligible), + ], + false, + )) + .expect("should compile failure isolation plan"); + let orchestrator = AuctionOrchestratorHarness::new(plan, None); + let mut request = planned_request(); + let mut below_only_slot = request.slots[0].clone(); + below_only_slot.id = "below-only-slot".to_string(); + request.slots.push(below_only_slot); + let settings = create_test_settings(); + let inbound = http::Request::new(edgezero_core::body::Body::empty()); + let context = AuctionContext { + settings: &settings, + request: &inbound, + timeout_ms: 777, + transport_timeout_ms: 777, + provider_responses: None, + services: &services, + }; + + let result = orchestrator + .run_auction(&request, &context) + .await + .expect("should isolate planned provider failures"); + + let by_provider = result + .provider_responses + .iter() + .map(|response| (response.provider.as_str(), response)) + .collect::>(); + assert_eq!( + by_provider + .get("launch-fail") + .unwrap_or_else(|| panic!( + "should include launch-fail response; got {:?}", + by_provider.keys().collect::>() + )) + .metadata["error_type"], + "launch_failed" + ); + let below_floor = by_provider.get("below-floor").unwrap_or_else(|| { + panic!( + "should include below-floor response; got {:?}", + by_provider.keys().collect::>() + ) + }); + assert_eq!(below_floor.status, BidStatus::Success); + assert_eq!(below_floor.bids[0].bid_id.as_deref(), Some("below")); + assert_eq!(below_floor.bids[0].price, Some(0.5)); + assert_eq!(below_floor.bids[1].bid_id.as_deref(), Some("below-only")); + assert_eq!( + by_provider["transport-fail"].metadata["error_type"], + "transport" + ); + let parse_failure = by_provider.get("parse-fail").unwrap_or_else(|| { + panic!( + "should include parse-fail response; got {:?}", + by_provider.keys().collect::>() + ) + }); + assert_eq!(parse_failure.status, BidStatus::Error); + assert_eq!(parse_failure.metadata["error_type"], "parse_response"); + assert_eq!( + parse_failure.metadata["routing"]["unused_bidder_params_count"], + 0 + ); + for response in by_provider.values() { + assert_eq!( + response.metadata["routing"]["unused_bidder_params_count"], 0, + "every materialized planned provider response should have routing count" + ); + } + assert_eq!(by_provider["valid-winner"].status, BidStatus::Success); + assert_eq!( + result.winning_bids["fictional-slot"].bid_id.as_deref(), + Some("winner") + ); + assert!( + !result.winning_bids.contains_key("below-only-slot"), + "valid below-floor bid should be discarded when it is the only candidate" + ); + } + + #[tokio::test] + async fn planned_routing_count_survives_standard_and_aps_bounded_body_failures() { + for (profile, provider_id) in [("standard", "standard"), ("aps", "aps-instance")] { + let http = Arc::new(StubHttpClient::new()); + http.push_response(200, vec![b'x'; 1024 * 1024 + 1]); + let backend = Arc::new(NamingBackend::new(BackendNamingPolicy::Axum)); + let services = build_services_with_backend_and_http_client( + Arc::clone(&backend) as Arc<_>, + Arc::clone(&http) as Arc<_>, + ); + let mut config = if profile == "standard" { + planned_config(&[(provider_id, RoutingMode::AllEligible)], false) + } else { + planned_aps_config() + }; + config.bidders.insert( + "example-bidder" + .parse() + .expect("should parse fictional bidder ID"), + crate::auction::plan::BidderRouteConfig { + provider: provider_id + .parse() + .expect("should parse fictional provider ID"), + }, + ); + let plan = AuctionPlan::compile(config).expect("should compile bounded-body plan"); + let orchestrator = AuctionOrchestratorHarness::new(plan, None); + let mut request = planned_request(); + request.slots[0].bidders.insert( + "example-bidder".to_string(), + serde_json::json!({"private": "value"}), + ); + let settings = create_test_settings(); + let inbound = http::Request::new(edgezero_core::body::Body::empty()); + let context = AuctionContext { + settings: &settings, + request: &inbound, + timeout_ms: 777, + transport_timeout_ms: 777, + provider_responses: None, + services: &services, + }; + + let result = orchestrator + .run_auction(&request, &context) + .await + .expect("should materialize bounded-body failure"); + let response = &result.provider_responses[0]; + assert_eq!(response.status, BidStatus::Error, "{profile}"); + assert_eq!( + response.metadata["routing"]["unused_bidder_params_count"], 1, + "{profile} bounded-body failure should retain the input-derived count" + ); + let routing = serde_json::to_string(&response.metadata["routing"]) + .expect("should serialize routing metadata"); + assert!(!routing.contains("example-bidder") && !routing.contains("private")); + } + } + + #[tokio::test] + async fn planned_signer_admission_time_reduces_budget_and_total_time_includes_it() { + let config_store = Arc::new(CountingConfigStore { + reads: AtomicUsize::new(0), + current_kid: "test-kid".to_string(), + delay: Duration::from_millis(50), + }); + let secret_store = Arc::new(CountingSecretStore { + reads: AtomicUsize::new(0), + key: base64::Engine::encode(&base64::engine::general_purpose::STANDARD, [7_u8; 32]) + .into_bytes(), + }); + let backend = Arc::new(NamingBackend::new(BackendNamingPolicy::Axum)); + let http = Arc::new(StubHttpClient::new()); + http.push_response(204, Vec::new()); + let services = RuntimeServices::builder() + .config_store(Arc::clone(&config_store) as Arc<_>) + .secret_store(Arc::clone(&secret_store) as Arc<_>) + .kv_store(Arc::new(edgezero_core::key_value_store::NoopKvStore)) + .backend(Arc::clone(&backend) as Arc<_>) + .http_client(Arc::clone(&http) as Arc<_>) + .geo(Arc::new(crate::platform::test_support::NoopGeo)) + .auction_telemetry_sink(Arc::new( + crate::auction::telemetry::NoopAuctionTelemetrySink, + )) + .client_info(crate::platform::ClientInfo::default()) + .build(); + let plan = AuctionPlan::compile(planned_config( + &[("signed", RoutingMode::AllEligible)], + true, + )) + .expect("should compile signed plan"); + let orchestrator = AuctionOrchestratorHarness::new(plan, None); + let request = planned_request(); + let settings = create_test_settings(); + let inbound = http::Request::new(edgezero_core::body::Body::empty()); + let context = AuctionContext { + settings: &settings, + request: &inbound, + timeout_ms: 200, + transport_timeout_ms: 200, + provider_responses: None, + services: &services, + }; + + let result = orchestrator + .run_auction(&request, &context) + .await + .expect("should execute signed planned auction"); + + let body: serde_json::Value = serde_json::from_slice(&http.recorded_request_bodies()[0]) + .expect("should parse signed request"); + let tmax = body["tmax"].as_u64().expect("should include tmax"); + assert!( + (100..=175).contains(&tmax), + "signer delay should reduce logical budget, got {tmax}" + ); + assert!( + result.total_time_ms >= 50, + "total time should include signer admission" + ); + assert_eq!(config_store.reads.load(Ordering::Relaxed), 1); + assert_eq!(secret_store.reads.load(Ordering::Relaxed), 1); + } + + #[tokio::test] + async fn planned_signed_multi_provider_loads_signer_once_and_sends_twice() { + let config_store = Arc::new(CountingConfigStore { + reads: AtomicUsize::new(0), + current_kid: "test-kid".to_string(), + delay: Duration::ZERO, + }); + let secret_store = Arc::new(CountingSecretStore { + reads: AtomicUsize::new(0), + key: base64::Engine::encode(&base64::engine::general_purpose::STANDARD, [11_u8; 32]) + .into_bytes(), + }); + let backend = Arc::new(NamingBackend::new(BackendNamingPolicy::Axum)); + let http = Arc::new(StubHttpClient::new()); + http.push_response(204, Vec::new()); + http.push_response(204, Vec::new()); + let services = RuntimeServices::builder() + .config_store(Arc::clone(&config_store) as Arc<_>) + .secret_store(Arc::clone(&secret_store) as Arc<_>) + .kv_store(Arc::new(edgezero_core::key_value_store::NoopKvStore)) + .backend(Arc::clone(&backend) as Arc<_>) + .http_client(Arc::clone(&http) as Arc<_>) + .geo(Arc::new(crate::platform::test_support::NoopGeo)) + .auction_telemetry_sink(Arc::new( + crate::auction::telemetry::NoopAuctionTelemetrySink, + )) + .client_info(crate::platform::ClientInfo::default()) + .build(); + let plan = AuctionPlan::compile(planned_config( + &[ + ("provider-a", RoutingMode::AllEligible), + ("provider-b", RoutingMode::AllEligible), + ], + true, + )) + .expect("should compile signed plan"); + let orchestrator = AuctionOrchestratorHarness::new(plan, None); + let request = planned_request(); + let settings = create_test_settings(); + let inbound = http::Request::new(edgezero_core::body::Body::empty()); + let context = AuctionContext { + settings: &settings, + request: &inbound, + timeout_ms: 777, + transport_timeout_ms: 777, + provider_responses: None, + services: &services, + }; + + let result = orchestrator + .run_auction(&request, &context) + .await + .expect("should execute signed multi-provider auction"); + + assert_eq!(result.provider_responses.len(), 2); + assert_eq!(config_store.reads.load(Ordering::Relaxed), 1); + assert_eq!(secret_store.reads.load(Ordering::Relaxed), 1); + assert_eq!(http.recorded_backend_names().len(), 2); + for body in http.recorded_request_bodies() { + let value: serde_json::Value = + serde_json::from_slice(&body).expect("should parse signed provider request"); + assert!( + value["ext"]["trusted_server"]["signature"].is_string(), + "should sign every request" + ); + } + } + + #[tokio::test] + async fn from_plan_signing_failure_is_fatal_for_direct_and_safe_for_split() { + for split in [false, true] { + let config_store = Arc::new(FailingCountingConfigStore { + reads: AtomicUsize::new(0), + }); + let backend = Arc::new(NamingBackend::new(BackendNamingPolicy::Axum)); + let http = Arc::new(StubHttpClient::new()); + let services = RuntimeServices::builder() + .config_store(Arc::clone(&config_store) as Arc<_>) + .secret_store(Arc::new(UnusedSecretStore)) + .kv_store(Arc::new(edgezero_core::key_value_store::NoopKvStore)) + .backend(Arc::clone(&backend) as Arc<_>) + .http_client(Arc::clone(&http) as Arc<_>) + .geo(Arc::new(crate::platform::test_support::NoopGeo)) + .auction_telemetry_sink(Arc::new( + crate::auction::telemetry::NoopAuctionTelemetrySink, + )) + .client_info(crate::platform::ClientInfo::default()) + .build(); + let mut config = planned_config(&[("signed", RoutingMode::AllEligible)], true); + config.bidders.insert( + "unknown-private-id" + .parse() + .expect("should parse fictional bidder ID"), + crate::auction::plan::BidderRouteConfig { + provider: "signed" + .parse() + .expect("should parse fictional provider ID"), + }, + ); + let plan = Arc::new(AuctionPlan::compile(config).expect("should compile signed plan")); + let orchestrator = AuctionOrchestrator::from_plan(plan, None); + let mut request = planned_request(); + request.slots[0].bidders.insert( + "unroutable-private-id".to_string(), + serde_json::json!({"secret": 9}), + ); + let settings = create_test_settings(); + let inbound = http::Request::new(edgezero_core::body::Body::empty()); + let context = AuctionContext { + settings: &settings, + request: &inbound, + timeout_ms: 777, + transport_timeout_ms: 777, + provider_responses: None, + services: &services, + }; + + if split { + let DispatchAuctionOutcome::DispatchFailed { + provider_responses, + fatal_admission_error, + metadata, + .. + } = orchestrator.dispatch_auction(&request, &context).await + else { + panic!("split signer failure should be explicit"); + }; + let error = fatal_admission_error.expect("should carry fatal admission error"); + assert!(format!("{error:?}").contains("current-kid")); + assert_eq!(provider_responses.len(), 1); + assert_eq!( + provider_responses[0].metadata["routing"]["unused_bidder_params_count"], + 0 + ); + assert_eq!(metadata["routing"]["unroutable_bidder_count"], 1); + let serialized = + serde_json::to_string(&metadata).expect("should serialize routing metadata"); + assert!( + !serialized.contains("unroutable-private-id") && !serialized.contains("secret") + ); + } else { + let error = orchestrator + .run_auction(&request, &context) + .await + .expect_err("direct signer failure should propagate"); + let report = format!("{error:?}"); + assert!(report.contains("Planned auction admission failed")); + assert!(report.contains("current-kid")); + } + + assert_eq!(config_store.reads.load(Ordering::Relaxed), 1); + assert_eq!(backend.predicted.load(Ordering::Relaxed), 0); + assert_eq!(backend.ensured.load(Ordering::Relaxed), 0); + assert!(http.recorded_backend_names().is_empty()); + } + } + + #[tokio::test] + async fn planned_signing_failure_reads_store_once_before_backend_or_send() { + let config_store = Arc::new(FailingCountingConfigStore { + reads: AtomicUsize::new(0), + }); + let backend = Arc::new(NamingBackend::new(BackendNamingPolicy::Axum)); + let http = Arc::new(StubHttpClient::new()); + let services = RuntimeServices::builder() + .config_store(Arc::clone(&config_store) as Arc<_>) + .secret_store(Arc::new(UnusedSecretStore)) + .kv_store(Arc::new(edgezero_core::key_value_store::NoopKvStore)) + .backend(Arc::clone(&backend) as Arc<_>) + .http_client(Arc::clone(&http) as Arc<_>) + .geo(Arc::new(crate::platform::test_support::NoopGeo)) + .auction_telemetry_sink(Arc::new( + crate::auction::telemetry::NoopAuctionTelemetrySink, + )) + .client_info(crate::platform::ClientInfo::default()) + .build(); + let plan = AuctionPlan::compile(planned_config( + &[("signed", RoutingMode::AllEligible)], + true, + )) + .expect("should compile signed plan"); + let orchestrator = AuctionOrchestratorHarness::new(plan, None); + let request = planned_request(); + let settings = create_test_settings(); + let inbound = http::Request::new(edgezero_core::body::Body::empty()); + let context = AuctionContext { + settings: &settings, + request: &inbound, + timeout_ms: 777, + transport_timeout_ms: 777, + provider_responses: None, + services: &services, + }; + + let _error = orchestrator + .run_auction(&request, &context) + .await + .expect_err("should fail signer admission"); + + assert_eq!(config_store.reads.load(Ordering::Relaxed), 1); + assert_eq!(backend.predicted.load(Ordering::Relaxed), 0); + assert_eq!(backend.ensured.load(Ordering::Relaxed), 0); + assert!(http.recorded_backend_names().is_empty()); + } + + #[tokio::test] + async fn from_plan_split_zero_budget_does_no_signer_backend_or_network_work() { + let config_store = Arc::new(CountingConfigStore { + reads: AtomicUsize::new(0), + current_kid: "unused-kid".to_string(), + delay: Duration::ZERO, + }); + let secret_store = Arc::new(CountingSecretStore { + reads: AtomicUsize::new(0), + key: base64::Engine::encode(&base64::engine::general_purpose::STANDARD, [13_u8; 32]) + .into_bytes(), + }); + let backend = Arc::new(NamingBackend::new(BackendNamingPolicy::Axum)); + let http = Arc::new(StubHttpClient::new()); + let services = RuntimeServices::builder() + .config_store(Arc::clone(&config_store) as Arc<_>) + .secret_store(Arc::clone(&secret_store) as Arc<_>) + .kv_store(Arc::new(edgezero_core::key_value_store::NoopKvStore)) + .backend(Arc::clone(&backend) as Arc<_>) + .http_client(Arc::clone(&http) as Arc<_>) + .geo(Arc::new(crate::platform::test_support::NoopGeo)) + .auction_telemetry_sink(Arc::new( + crate::auction::telemetry::NoopAuctionTelemetrySink, + )) + .client_info(crate::platform::ClientInfo::default()) + .build(); + let plan = Arc::new( + AuctionPlan::compile(planned_config( + &[("signed", RoutingMode::AllEligible)], + true, + )) + .expect("should compile signed plan"), + ); + let mediator_launches = Arc::new(AtomicUsize::new(0)); + let orchestrator = AuctionOrchestrator::from_plan( + plan, + Some(Arc::new(DeadlineRecordingMediator { + launches: Arc::clone(&mediator_launches), + budgets: None, + })), + ); + let request = planned_request(); + let settings = create_test_settings(); + let inbound = http::Request::new(edgezero_core::body::Body::empty()); + let context = AuctionContext { + settings: &settings, + request: &inbound, + timeout_ms: 0, + transport_timeout_ms: 0, + provider_responses: None, + services: &services, + }; + + let DispatchAuctionOutcome::Dispatched(dispatched) = + orchestrator.dispatch_auction(&request, &context).await + else { + panic!("zero budget should materialize a completed split dispatch"); + }; + let result = orchestrator + .collect_dispatched_auction(dispatched, &services, &context) + .await; + + assert_eq!(result.provider_responses.len(), 1); + assert_eq!( + result.provider_responses[0].metadata["error_type"], + "timeout" + ); + assert_eq!(config_store.reads.load(Ordering::Relaxed), 0); + assert_eq!(secret_store.reads.load(Ordering::Relaxed), 0); + assert_eq!(backend.predicted.load(Ordering::Relaxed), 0); + assert_eq!(backend.ensured.load(Ordering::Relaxed), 0); + assert!(http.recorded_backend_names().is_empty()); + assert_eq!( + mediator_launches.load(Ordering::Relaxed), + 0, + "zero budget must not invoke even an immediate mediator" + ); + } + + #[tokio::test] + async fn planned_fanout_rejection_happens_before_backend_or_send() { + let http = Arc::new(StubHttpClient::new()); + http.set_concurrent_fanout(false); + let backend = Arc::new(NamingBackend::new(BackendNamingPolicy::Cloudflare)); + let services = build_services_with_backend_and_http_client( + Arc::clone(&backend) as Arc<_>, + Arc::clone(&http) as Arc<_>, + ); + let plan = AuctionPlan::compile(planned_config( + &[ + ("provider-a", RoutingMode::AllEligible), + ("provider-b", RoutingMode::AllEligible), + ], + false, + )) + .expect("should compile planned auction"); + let orchestrator = AuctionOrchestratorHarness::new(plan, None); + let request = planned_request(); + let settings = create_test_settings(); + let inbound = http::Request::new(edgezero_core::body::Body::empty()); + let context = AuctionContext { + settings: &settings, + request: &inbound, + timeout_ms: 777, + transport_timeout_ms: 777, + provider_responses: None, + services: &services, + }; + + let _error = orchestrator + .run_auction(&request, &context) + .await + .expect_err("should reject unsupported concurrent fanout"); + + assert_eq!(backend.predicted.load(Ordering::Relaxed), 0); + assert_eq!(backend.ensured.load(Ordering::Relaxed), 0); + assert!(http.recorded_backend_names().is_empty()); + } + + #[test] + fn routing_metadata_is_fixed_count_only_and_saturating() { + let metadata = super::routing_metadata( + RoutingDiagnostics::saturated_for_test().unroutable_bidder_count(), + ); + assert_eq!( + metadata, + HashMap::from([( + "routing".to_string(), + serde_json::json!({"unroutable_bidder_count": u32::MAX}), + )]) + ); + assert!( + !serde_json::to_string(&metadata) + .expect("should serialize routing metadata") + .contains("bidder_id"), + "routing metadata must not expose bidder identifiers" + ); + } + #[test] fn decoded_aps_bid_competes_directly_by_cpm() { let orchestrator = AuctionOrchestrator::new(AuctionConfig::default()); @@ -3308,6 +7707,7 @@ mod tests { creative: Some("
Ad
".to_string()), adomain: None, bidder: "aps".to_string(), + returned_seat: None, width: 300, height: 250, nurl: None, @@ -3352,6 +7752,7 @@ mod tests { creative: Some("
APS Ad
".to_string()), adomain: None, bidder: "aps".to_string(), + returned_seat: None, width: 300, height: 250, nurl: None, @@ -3391,6 +7792,7 @@ mod tests { creative: Some("
APS Ad
".to_string()), adomain: None, bidder: "aps".to_string(), + returned_seat: None, width: 300, height: 250, nurl: None, diff --git a/crates/trusted-server-core/src/auction/plan.rs b/crates/trusted-server-core/src/auction/plan.rs new file mode 100644 index 000000000..df36fc09b --- /dev/null +++ b/crates/trusted-server-core/src/auction/plan.rs @@ -0,0 +1,1372 @@ +//! Target-independent config-first auction plan compiler. + +use std::collections::{BTreeMap, BTreeSet}; +use std::str::FromStr; +use std::time::Duration; + +use error_stack::{Report, ResultExt as _}; +use serde::{Deserialize, Serialize}; +use serde_json::Value; +use url::Url; + +use super::profile::{CompiledOpenRtbProfile, ProfileTimeoutDefault, find_profile}; +use crate::error::TrustedServerError; +use crate::platform::{AuctionTargetId, PlatformBackendSpec}; +use crate::settings::RequestSigning; + +const MAX_ID_BYTES: usize = 128; +const MAX_SUPPRESS_SEATS: usize = 128; +const MAX_SUPPRESS_SEAT_BYTES: usize = 128; +const MOCK_MEDIATOR_ID: &str = "adserver_mock"; +const RESERVED_BROWSER_ENVELOPE_BIDDER_ID: &str = "trustedServer"; + +/// Validated operator-defined provider identifier. +#[derive(Debug, Clone, Eq, Hash, Ord, PartialEq, PartialOrd, derive_more::Display)] +pub struct ProviderId(String); + +impl ProviderId { + /// Borrow the validated identifier. + #[must_use] + pub fn as_str(&self) -> &str { + &self.0 + } + + #[cfg(test)] + pub(crate) fn unchecked_for_legacy_test(value: &str) -> Self { + Self(value.to_string()) + } +} + +impl FromStr for ProviderId { + type Err = Report; + + fn from_str(value: &str) -> Result { + let valid = !value.is_empty() + && value.len() <= 63 + && value.as_bytes().first().is_some_and(u8::is_ascii_lowercase) + && value + .as_bytes() + .iter() + .all(|byte| byte.is_ascii_lowercase() || byte.is_ascii_digit() || *byte == b'-'); + if !valid { + return Err(configuration_error(format!( + "provider ID `{value}` must match ^[a-z][a-z0-9-]{{0,62}}$" + ))); + } + Ok(Self(value.to_string())) + } +} + +impl<'de> Deserialize<'de> for ProviderId { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + let value = String::deserialize(deserializer)?; + Self::from_str(&value).map_err(serde::de::Error::custom) + } +} + +impl Serialize for ProviderId { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + serializer.serialize_str(self.as_str()) + } +} + +/// Validated client-visible bidder identifier. +#[derive(Debug, Clone, Eq, Hash, Ord, PartialEq, PartialOrd, derive_more::Display)] +pub struct BidderId(String); + +impl BidderId { + /// Borrow the validated identifier. + #[must_use] + pub fn as_str(&self) -> &str { + &self.0 + } +} + +impl FromStr for BidderId { + type Err = Report; + + fn from_str(value: &str) -> Result { + if value.is_empty() + || value.len() > MAX_ID_BYTES + || value.chars().any(char::is_control) + || value.trim() != value + { + return Err(configuration_error(format!( + "bidder ID {value:?} must be nonempty, at most {MAX_ID_BYTES} UTF-8 bytes, contain no control characters, and have no surrounding whitespace" + ))); + } + Ok(Self(value.to_string())) + } +} + +impl<'de> Deserialize<'de> for BidderId { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + let value = String::deserialize(deserializer)?; + Self::from_str(&value).map_err(serde::de::Error::custom) + } +} + +impl Serialize for BidderId { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + serializer.serialize_str(self.as_str()) + } +} + +/// Raw config-first provider declaration. +#[derive(Debug, Clone, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +pub struct ProviderConfig { + /// Protocol identifier. Version one accepts only `openrtb-2.6`. + pub protocol: String, + /// Registered profile identifier. + #[serde(default = "default_profile")] + pub profile: String, + /// Fixed provider endpoint. + pub endpoint: String, + /// Optional profile-default timeout override. + #[serde(default)] + pub timeout_ms: Option, + /// Slot routing mode. + #[serde(default)] + pub routing: RoutingMode, + /// Common `OpenRTB` notification policy. + #[serde(default)] + pub notifications: NotificationConfig, + /// Selected profile's typed configuration object. + #[serde(default = "empty_object")] + pub profile_config: Value, +} + +/// Raw central bidder route. +#[derive(Debug, Clone, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +pub struct BidderRouteConfig { + /// Referenced provider identifier. + pub provider: ProviderId, +} + +/// Provider slot routing behavior. +#[derive(Debug, Clone, Copy, Default, Deserialize, Eq, PartialEq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum RoutingMode { + /// Route only centrally assigned or trusted demand. + #[default] + Explicit, + /// Route every banner-compatible slot. + AllEligible, +} + +/// Common normalized-notification suppression configuration. +#[derive(Debug, Clone, Default, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +pub struct NotificationConfig { + /// Suppress notification URLs for every normalized bid. + #[serde(default)] + pub suppress_all: bool, + /// Suppress notification URLs for exact returned-seat matches. + #[serde(default)] + pub suppress_seats: Vec, +} + +/// Raw internal input for target-independent plan compilation. +#[derive(Debug, Clone, Default, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +pub struct AuctionPlanConfig { + /// Auction-wide logical timeout. + pub timeout_ms: u32, + /// Operator-defined provider instances. + #[serde(default)] + pub providers: BTreeMap, + /// Client bidder-to-provider routes. + #[serde(default)] + pub bidders: BTreeMap, + /// Existing separately registered mock mediator. + #[serde(default)] + pub mediator: Option, + /// Existing global Trusted Server signing configuration. + #[serde(default)] + pub request_signing: Option, +} + +/// Canonical absolute provider endpoint. +#[derive(Debug, Clone, Eq, PartialEq)] +pub struct CanonicalProviderEndpoint(Url); + +impl CanonicalProviderEndpoint { + /// Borrow the canonical endpoint string. + #[must_use] + pub fn as_str(&self) -> &str { + self.0.as_str() + } + + pub(crate) fn url(&self) -> &Url { + &self.0 + } +} + +/// Closed first-version protocol plan. +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +pub enum ProtocolPlan { + /// `OpenRTB` version 2.6 subset. + OpenRtb26, +} + +/// Immutable common notification policy. +#[derive(Debug, Clone, Default, Eq, PartialEq)] +pub struct NotificationPolicy { + /// Suppress notification URLs for every bid. + pub suppress_all: bool, + /// Exact returned seats whose notification URLs are suppressed. + pub suppress_seats: BTreeSet, +} + +/// Immutable compiled provider instance. +#[derive(Debug, Clone)] +pub struct ProviderPlan { + /// Provider identity. + pub id: ProviderId, + /// Canonical endpoint. + pub endpoint: CanonicalProviderEndpoint, + /// Resolved profile-default or explicit timeout. + pub timeout_ms: u32, + /// Slot routing mode. + pub routing: RoutingMode, + /// Common notification policy. + pub notifications: NotificationPolicy, + /// Compiled protocol behavior. + pub protocol: ProtocolPlan, + /// Compiled typed profile behavior. + pub profile: CompiledOpenRtbProfile, +} + +/// Immutable target-independent auction plan. +#[derive(Debug, Clone)] +pub struct AuctionPlan { + enabled: bool, + timeout_ms: u32, + providers: Vec, + bidder_routes: BTreeMap, + signing_enabled: bool, + mediator: Option, +} + +impl AuctionPlan { + /// Return whether auction execution is enabled. + #[must_use] + pub fn enabled(&self) -> bool { + self.enabled + } + + pub(crate) fn with_enabled(mut self, enabled: bool) -> Self { + self.enabled = enabled; + self + } + + /// Compile a deterministic plan without adapter-specific validation. + /// + /// # Errors + /// + /// Returns a configuration error for invalid identifiers, protocol/profile + /// declarations, endpoints, profile configuration, routes, notifications, + /// signing structure, or mediator selection. + pub fn compile(config: AuctionPlanConfig) -> Result> { + if config.timeout_ms == 0 { + return Err(configuration_error( + "auction timeout_ms must be greater than zero", + )); + } + validate_mediator(config.mediator.as_deref())?; + let signing_enabled = compile_signing_enabled(config.request_signing.as_ref())?; + let mut providers = Vec::with_capacity(config.providers.len()); + let mut provider_indices = BTreeMap::new(); + for (id, raw) in config.providers { + if raw.protocol != "openrtb-2.6" { + return Err(configuration_error(format!( + "provider `{id}` uses unsupported protocol `{}`", + raw.protocol + ))); + } + let registration = find_profile(&raw.profile).ok_or_else(|| { + configuration_error(format!( + "provider `{id}` uses unknown OpenRTB profile `{}`", + raw.profile + )) + })?; + if registration.id == "prebid-server" && raw.routing == RoutingMode::AllEligible { + return Err(configuration_error(format!( + "provider `{id}` cannot use routing `all_eligible` with profile `prebid-server`; configure explicit bidder routes" + ))); + } + if !raw.profile_config.is_object() { + return Err(configuration_error(format!( + "provider `{id}` profile_config must be an object" + ))); + } + let endpoint = canonicalize_endpoint(&id, registration.id, &raw.endpoint)?; + let timeout_ms = raw + .timeout_ms + .unwrap_or(match registration.default_timeout { + ProfileTimeoutDefault::Auction => config.timeout_ms, + ProfileTimeoutDefault::Fixed(value) => value, + }); + if timeout_ms == 0 { + return Err(configuration_error(format!( + "provider `{id}` timeout_ms must be greater than zero" + ))); + } + let notifications = compile_notifications(&id, raw.notifications)?; + let profile = registration.compile(&raw.profile_config)?; + let index = providers.len(); + provider_indices.insert(id.clone(), index); + providers.push(ProviderPlan { + id, + endpoint, + timeout_ms, + routing: raw.routing, + notifications, + protocol: ProtocolPlan::OpenRtb26, + profile, + }); + } + let mut bidder_routes = BTreeMap::new(); + for (bidder, route) in config.bidders { + if bidder.as_str() == RESERVED_BROWSER_ENVELOPE_BIDDER_ID { + return Err(configuration_error(format!( + "bidder ID `{RESERVED_BROWSER_ENVELOPE_BIDDER_ID}` is reserved for browser admission" + ))); + } + let provider_index = + provider_indices + .get(&route.provider) + .copied() + .ok_or_else(|| { + configuration_error(format!( + "bidder `{bidder}` references unknown provider `{}`", + route.provider + )) + })?; + bidder_routes.insert(bidder, provider_index); + } + Ok(Self { + enabled: true, + timeout_ms: config.timeout_ms, + providers, + bidder_routes, + signing_enabled, + mediator: config.mediator, + }) + } +} + +impl ProviderPlan { + /// Build the canonical backend specification with the configured timeout. + #[must_use] + pub(crate) fn backend_spec(&self) -> PlatformBackendSpec { + self.backend_spec_with_transport_timeout(self.timeout_ms) + } + + /// Build the canonical backend specification with request-local transport timers. + #[must_use] + pub(crate) fn backend_spec_with_transport_timeout( + &self, + transport_timeout_ms: u32, + ) -> PlatformBackendSpec { + let endpoint = self.endpoint.url(); + let timeout = Duration::from_millis(u64::from(transport_timeout_ms)); + PlatformBackendSpec { + scheme: endpoint.scheme().to_owned(), + host: endpoint + .host_str() + .expect("should retain validated provider endpoint host") + .to_owned(), + port: endpoint.port(), + host_header_override: None, + certificate_check: true, + first_byte_timeout: timeout, + between_bytes_timeout: timeout, + discriminator: Some(self.id.as_str().to_owned()), + } + } +} + +impl AuctionPlan { + /// Validate adapter capabilities and backend-name correlation before I/O. + /// + /// Each provider is predicted from a canonical backend specification using + /// its exact configured provider timeout as both transport timers and its + /// provider ID as the stable discriminator. These transport timers do not + /// replace the auction-wide logical budget. + /// + /// # Errors + /// + /// Returns a configuration error when the target cannot fan out to every + /// configured provider, backend prediction fails, or two predicted names + /// collide. + pub fn validate_for_target( + &self, + target_id: AuctionTargetId, + ) -> Result<(), Report> { + if !self.enabled { + return Ok(()); + } + let target = target_id.descriptor(); + if self.providers.len() > 1 && !target.capabilities().supports_concurrent_provider_fanout() + { + return Err(configuration_error(format!( + "auction target `{}` does not support concurrent provider fanout; configured {} providers", + target_id.adapter_id(), + self.providers.len() + ))); + } + + let naming_policy = target.naming_policy(); + let backend_budget = naming_policy.auction_dynamic_backend_budget(); + let mut required_backend_names = 0_usize; + let mut predicted_names = BTreeMap::::new(); + for provider in &self.providers { + let reachable_timeout_ms = provider.timeout_ms.min(self.timeout_ms); + required_backend_names = required_backend_names + .saturating_add(naming_policy.transport_timeout_bucket_count(reachable_timeout_ms)); + if let Some(budget) = backend_budget + && required_backend_names > budget + { + return Err(configuration_error(format!( + "auction target `{}` requires up to {required_backend_names} dynamic provider backends, exceeding its auction budget of {budget}", + target_id.adapter_id(), + ))); + } + let spec = provider.backend_spec(); + let prediction = + naming_policy + .predict(&spec) + .change_context(TrustedServerError::Configuration { + message: format!( + "provider `{}` backend prediction failed for target `{}`", + provider.id, + target_id.adapter_id() + ), + })?; + if let Some(existing) = predicted_names.insert(prediction.name.clone(), &provider.id) { + return Err(configuration_error(format!( + "providers `{existing}` and `{}` predict the same backend name `{}` for target `{}`", + provider.id, + prediction.name, + target_id.adapter_id() + ))); + } + } + Ok(()) + } + + /// Borrow compiled providers in deterministic provider-ID order. + #[must_use] + pub fn providers(&self) -> &[ProviderPlan] { + &self.providers + } + + /// Borrow a compiled provider by its validated identity. + #[must_use] + pub(crate) fn provider(&self, id: &ProviderId) -> Option<&ProviderPlan> { + self.providers.iter().find(|provider| provider.id == *id) + } + + /// Return whether any compiled provider uses the named profile. + /// + /// This narrow query allows capability activation to follow the validated + /// plan without exposing profile configuration. + #[must_use] + pub fn has_profile(&self, profile_id: &str) -> bool { + self.providers + .iter() + .any(|provider| provider.profile.id() == profile_id) + } + + /// Borrow validated client-visible bidder route codes in deterministic order. + /// + /// This intentionally exposes route keys rather than provider identities or + /// profile configuration for the browser Prebid injection boundary. + pub(crate) fn browser_bidder_codes(&self) -> impl Iterator { + self.bidder_routes.keys().map(BidderId::as_str) + } + + /// Resolve a bidder route to a compiled provider. + #[must_use] + pub fn provider_for_bidder(&self, bidder: &BidderId) -> Option<&ProviderPlan> { + self.bidder_routes + .get(bidder) + .and_then(|index| self.providers.get(*index)) + } + + /// Return whether auction-wide signing is enabled. + #[must_use] + pub fn signing_enabled(&self) -> bool { + self.signing_enabled + } + + /// Borrow the separately validated static mediator identifier. + #[must_use] + pub fn mediator(&self) -> Option<&str> { + self.mediator.as_deref() + } +} + +fn default_profile() -> String { + "standard".to_string() +} + +fn empty_object() -> Value { + Value::Object(serde_json::Map::new()) +} + +fn configuration_error(message: impl Into) -> Report { + Report::new(TrustedServerError::Configuration { + message: message.into(), + }) +} + +fn compile_signing_enabled( + request_signing: Option<&RequestSigning>, +) -> Result> { + let Some(request_signing) = request_signing else { + return Ok(false); + }; + if request_signing.enabled + && (request_signing.config_store_id.trim().is_empty() + || request_signing.secret_store_id.trim().is_empty()) + { + return Err(configuration_error( + "enabled request_signing requires nonblank config_store_id and secret_store_id", + )); + } + Ok(request_signing.enabled) +} + +fn validate_mediator(mediator: Option<&str>) -> Result<(), Report> { + if mediator.is_some_and(|value| value != MOCK_MEDIATOR_ID) { + return Err(configuration_error(format!( + "auction mediator must be `{MOCK_MEDIATOR_ID}` when configured" + ))); + } + Ok(()) +} + +fn canonicalize_endpoint( + provider_id: &ProviderId, + profile_id: &str, + value: &str, +) -> Result> { + let mut endpoint = Url::parse(value).map_err(|error| { + configuration_error(format!( + "provider `{provider_id}` endpoint must be an absolute HTTPS URL: {error}" + )) + })?; + if endpoint.scheme() != "https" + || endpoint.host_str().is_none() + || !endpoint.username().is_empty() + || endpoint.password().is_some() + || endpoint.fragment().is_some() + { + return Err(configuration_error(format!( + "provider `{provider_id}` endpoint must be absolute HTTPS with a host and no credentials or fragment" + ))); + } + if profile_id == "aps" + && endpoint + .path() + .trim_end_matches('/') + .ends_with("/e/dtb/bid") + { + return Err(configuration_error(format!( + "provider `{provider_id}` uses unsupported legacy APS endpoint `/e/dtb/bid`" + ))); + } + if profile_id == "prebid-server" { + normalize_prebid_server_endpoint(&mut endpoint); + } + Ok(CanonicalProviderEndpoint(endpoint)) +} + +fn normalize_prebid_server_endpoint(endpoint: &mut Url) { + match endpoint.path() { + "" | "/" => endpoint.set_path("/openrtb2/auction"), + "/openrtb2/auction/" => endpoint.set_path("/openrtb2/auction"), + _ => {} + } +} + +fn compile_notifications( + provider_id: &ProviderId, + config: NotificationConfig, +) -> Result> { + if config.suppress_seats.len() > MAX_SUPPRESS_SEATS { + return Err(configuration_error(format!( + "provider `{provider_id}` notifications.suppress_seats exceeds {MAX_SUPPRESS_SEATS} entries" + ))); + } + let mut seats = BTreeSet::new(); + for seat in config.suppress_seats { + if seat.is_empty() + || seat.len() > MAX_SUPPRESS_SEAT_BYTES + || seat.chars().any(|character| character.is_ascii_control()) + { + return Err(configuration_error(format!( + "provider `{provider_id}` notification seat must be nonempty, at most {MAX_SUPPRESS_SEAT_BYTES} UTF-8 bytes, and contain no ASCII control characters" + ))); + } + if !seats.insert(seat.clone()) { + return Err(configuration_error(format!( + "provider `{provider_id}` notification seat `{seat}` is duplicated" + ))); + } + } + Ok(NotificationPolicy { + suppress_all: config.suppress_all, + suppress_seats: seats, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::auction::profile::CompiledOpenRtbProfile; + + fn provider(profile: &str) -> ProviderConfig { + ProviderConfig { + protocol: "openrtb-2.6".to_string(), + profile: profile.to_string(), + endpoint: "https://bid.example/openrtb2/auction".to_string(), + timeout_ms: None, + routing: RoutingMode::Explicit, + notifications: NotificationConfig::default(), + profile_config: empty_object(), + } + } + + fn config(providers: BTreeMap) -> AuctionPlanConfig { + AuctionPlanConfig { + timeout_ms: 1500, + providers, + ..AuctionPlanConfig::default() + } + } + + fn id(value: &str) -> ProviderId { + ProviderId::from_str(value).expect("should parse provider ID") + } + + fn bidder(value: &str) -> BidderId { + BidderId::from_str(value).expect("should parse bidder ID") + } + + fn nested_object(levels: usize) -> Value { + let mut value = Value::String("leaf".to_string()); + for level in 0..levels { + value = Value::Object(serde_json::Map::from_iter([( + format!("level-{level}"), + value, + )])); + } + value + } + + fn nested_array(levels: usize) -> Value { + let mut value = Value::String("leaf".to_string()); + for _ in 0..levels { + value = Value::Array(vec![value]); + } + value + } + + #[test] + fn target_validation_accepts_fanout_and_rejects_unsupported_targets() { + let providers = BTreeMap::from([ + (id("provider-one"), provider("standard")), + (id("provider-two"), provider("standard")), + ]); + let plan = AuctionPlan::compile(config(providers)).expect("should compile plan"); + + assert!( + plan.validate_for_target(crate::platform::AuctionTargetId::Fastly) + .is_ok(), + "Fastly should accept provider fanout" + ); + assert!( + plan.validate_for_target(crate::platform::AuctionTargetId::Axum) + .is_ok(), + "Axum should accept provider fanout" + ); + for target in [ + crate::platform::AuctionTargetId::Cloudflare, + crate::platform::AuctionTargetId::Spin, + ] { + let error = plan + .validate_for_target(target) + .expect_err("should reject unsupported provider fanout"); + assert!( + error.to_string().contains("fanout"), + "should explain fanout rejection: {error:?}" + ); + } + } + + #[test] + fn fastly_target_validation_canonicalizes_url_derived_ipv6_prediction() { + let mut ipv6_provider = provider("standard"); + ipv6_provider.endpoint = "https://[2001:db8::5]:8443/openrtb".to_string(); + ipv6_provider.timeout_ms = Some(750); + let plan = AuctionPlan::compile(config(BTreeMap::from([( + id("ipv6-provider"), + ipv6_provider, + )]))) + .expect("should compile IPv6 provider plan"); + + plan.validate_for_target(crate::platform::AuctionTargetId::Fastly) + .expect("Fastly target validation should accept canonical IPv6 prediction"); + let bracketed_spec = plan.providers()[0].backend_spec(); + assert_eq!(bracketed_spec.host, "[2001:db8::5]"); + let mut bare_spec = bracketed_spec.clone(); + bare_spec.host = "2001:db8::5".to_string(); + let policy = crate::platform::BackendNamingPolicy::Fastly; + assert_eq!( + policy + .predict(&bracketed_spec) + .expect("should predict URL-derived bracketed IPv6 backend"), + policy + .predict(&bare_spec) + .expect("should predict runtime-normalized bare IPv6 backend"), + "startup target validation and Fastly runtime must hash the same backend spec" + ); + } + + #[test] + fn fastly_target_validation_reserves_dynamic_backends_outside_auction() { + let plan_with = |count: usize| { + let providers = (0..count) + .map(|index| { + let mut provider = provider("standard"); + provider.timeout_ms = Some(1000); + (id(&format!("provider-{index}")), provider) + }) + .collect(); + AuctionPlan::compile(config(providers)).expect("should compile provider plan") + }; + + plan_with(19) + .validate_for_target(crate::platform::AuctionTargetId::Fastly) + .expect("below-budget provider plan should validate"); + plan_with(20) + .validate_for_target(crate::platform::AuctionTargetId::Fastly) + .expect("at-budget provider plan should validate"); + let error = plan_with(21) + .validate_for_target(crate::platform::AuctionTargetId::Fastly) + .expect_err("over-budget provider plan should fail"); + assert!( + error + .to_string() + .contains("exceeding its auction budget of 160") + ); + } + + #[test] + fn fastly_backend_quota_uses_auction_timeout_as_reachable_bucket_ceiling() { + let providers = (0..21) + .map(|index| { + let mut provider = provider("standard"); + provider.timeout_ms = Some(1000); + (id(&format!("provider-{index}")), provider) + }) + .collect(); + let mut bounded = config(providers); + bounded.timeout_ms = 100; + let plan = AuctionPlan::compile(bounded).expect("should compile bounded provider plan"); + + assert!( + plan.providers() + .iter() + .all(|provider| provider.timeout_ms == 1000), + "quota validation must not rewrite configured provider timeouts" + ); + plan.validate_for_target(crate::platform::AuctionTargetId::Fastly) + .expect("21 providers reach only the 50ms and 100ms Fastly buckets"); + } + + #[test] + fn disabled_target_validation_skips_fanout_and_collision_checks() { + let providers = BTreeMap::from([ + (id("provider-one"), provider("standard")), + (id("provider-two"), provider("standard")), + ]); + let disabled = AuctionPlan::compile(config(providers)) + .expect("should compile plan") + .with_enabled(false); + + for target in [ + crate::platform::AuctionTargetId::Cloudflare, + crate::platform::AuctionTargetId::Spin, + ] { + disabled + .validate_for_target(target) + .expect("disabled dormant providers should skip target validation"); + } + + let provider = disabled.providers[0].clone(); + let disabled_collision = AuctionPlan { + enabled: false, + timeout_ms: disabled.timeout_ms, + providers: vec![provider.clone(), provider], + bidder_routes: BTreeMap::new(), + signing_enabled: false, + mediator: None, + }; + disabled_collision + .validate_for_target(crate::platform::AuctionTargetId::Axum) + .expect("disabled dormant providers should skip collision validation"); + } + + #[test] + fn target_validation_keeps_same_origin_timeout_profile_instances_distinct() { + let shared = ProviderConfig { + timeout_ms: Some(777), + ..provider("standard") + }; + let plan = AuctionPlan::compile(config(BTreeMap::from([ + (id("provider-one"), shared.clone()), + (id("provider-two"), shared), + ]))) + .expect("should compile same-origin provider instances"); + + for target in [ + crate::platform::AuctionTargetId::Fastly, + crate::platform::AuctionTargetId::Axum, + ] { + plan.validate_for_target(target) + .expect("provider ID discriminators should prevent predicted collisions"); + } + } + + #[test] + fn target_validation_rejects_predicted_name_collisions() { + let compiled = AuctionPlan::compile(config(BTreeMap::from([( + id("provider-a"), + provider("standard"), + )]))) + .expect("should compile plan"); + let provider = compiled.providers[0].clone(); + // The compiler prevents duplicate provider IDs. Construct the otherwise + // impossible duplicate internally to pin validation's defense-in-depth + // collision rejection independently of compiler invariants. + let collision_plan = AuctionPlan { + enabled: true, + timeout_ms: compiled.timeout_ms, + providers: vec![provider.clone(), provider], + bidder_routes: BTreeMap::new(), + signing_enabled: false, + mediator: None, + }; + + let error = collision_plan + .validate_for_target(crate::platform::AuctionTargetId::Axum) + .expect_err("should reject predicted backend collision"); + assert!(error.to_string().contains("same backend name")); + } + + #[test] + fn provider_id_enforces_exact_grammar_and_bounds() { + for valid in ["a", "pbs-primary", &format!("a{}", "0".repeat(62))] { + assert!(ProviderId::from_str(valid).is_ok(), "should accept {valid}"); + } + for invalid in [ + "", + "A", + "1provider", + "provider_name", + "provider.name", + "provider/one", + &format!("a{}", "0".repeat(63)), + ] { + assert!( + ProviderId::from_str(invalid).is_err(), + "should reject {invalid}" + ); + } + } + + #[test] + fn bidder_id_enforces_admission_bounds() { + assert!(BidderId::from_str("exampleBidder").is_ok()); + for invalid in ["", " bidder", "bidder\n", &"a".repeat(129)] { + let error = BidderId::from_str(invalid).expect_err("should reject invalid bidder ID"); + assert!( + error.to_string().contains(&format!("{invalid:?}")), + "should identify invalid bidder ID {invalid:?}: {error:?}" + ); + } + } + + #[test] + fn compiler_rejects_all_eligible_for_prebid_server_only() { + let mut prebid = provider("prebid-server"); + prebid.routing = RoutingMode::AllEligible; + let error = AuctionPlan::compile(config(BTreeMap::from([(id("pbs-main"), prebid)]))) + .expect_err("should reject all_eligible Prebid Server routing"); + let message = error.to_string(); + for expected in ["pbs-main", "all_eligible", "prebid-server"] { + assert!( + message.contains(expected), + "should identify provider, routing, and profile: {error:?}" + ); + } + + let mut standard = provider("standard"); + standard.routing = RoutingMode::AllEligible; + AuctionPlan::compile(config(BTreeMap::from([(id("standard-main"), standard)]))) + .expect("should retain all_eligible for non-Prebid profiles"); + } + + #[test] + fn compiler_rejects_exact_reserved_browser_envelope_bidder_id() { + let mut raw = config(BTreeMap::from([(id("one"), provider("standard"))])); + raw.bidders.insert( + bidder("trustedServer"), + BidderRouteConfig { + provider: id("one"), + }, + ); + assert!( + AuctionPlan::compile(raw).is_err(), + "exact reserved bidder ID should be rejected" + ); + + let mut case_distinct = config(BTreeMap::from([(id("one"), provider("standard"))])); + case_distinct.bidders.insert( + bidder("TrustedServer"), + BidderRouteConfig { + provider: id("one"), + }, + ); + assert!( + AuctionPlan::compile(case_distinct).is_ok(), + "reserved bidder comparison should remain case-sensitive" + ); + } + + #[test] + fn compiler_orders_providers_and_routes_deterministically() { + let mut providers = BTreeMap::new(); + providers.insert(id("z-provider"), provider("standard")); + providers.insert(id("a-provider"), provider("standard")); + let mut raw = config(providers); + raw.bidders.insert( + bidder("z-bidder"), + BidderRouteConfig { + provider: id("z-provider"), + }, + ); + raw.bidders.insert( + bidder("a-bidder"), + BidderRouteConfig { + provider: id("a-provider"), + }, + ); + let plan = AuctionPlan::compile(raw).expect("should compile deterministic plan"); + assert_eq!(plan.providers()[0].id.as_str(), "a-provider"); + assert_eq!(plan.providers()[1].id.as_str(), "z-provider"); + assert_eq!( + plan.browser_bidder_codes().collect::>(), + vec!["a-bidder", "z-bidder"], + "browser query should return deduplicated route codes in deterministic order" + ); + assert_eq!( + plan.provider_for_bidder(&bidder("a-bidder")) + .map(|provider| provider.id.as_str()), + Some("a-provider") + ); + } + + #[test] + fn compiler_supports_two_instances_of_the_same_profile() { + let mut providers = BTreeMap::new(); + providers.insert(id("pbs-a"), provider("prebid-server")); + providers.insert(id("pbs-b"), provider("prebid-server")); + let plan = AuctionPlan::compile(config(providers)).expect("should compile two PBS plans"); + assert_eq!(plan.providers().len(), 2); + assert!( + plan.providers().iter().all(|provider| matches!( + provider.profile, + CompiledOpenRtbProfile::PrebidServer(_) + )) + ); + } + + #[test] + fn profile_defaults_and_explicit_timeout_override_are_resolved() { + let mut providers = BTreeMap::new(); + providers.insert(id("standard-one"), provider("standard")); + providers.insert(id("pbs-one"), provider("prebid-server")); + providers.insert( + id("aps-one"), + ProviderConfig { + endpoint: "https://aps.example/e/pb/bid".to_string(), + profile_config: serde_json::json!({"account_id": "example-account"}), + ..provider("aps") + }, + ); + providers.insert( + id("pbs-override"), + ProviderConfig { + timeout_ms: Some(321), + ..provider("prebid-server") + }, + ); + let plan = AuctionPlan::compile(config(providers)).expect("should resolve timeouts"); + let timeouts = plan + .providers() + .iter() + .map(|provider| (provider.id.as_str(), provider.timeout_ms)) + .collect::>(); + assert_eq!(timeouts["standard-one"], 1500); + assert_eq!(timeouts["pbs-one"], 1000); + assert_eq!(timeouts["aps-one"], 800); + assert_eq!(timeouts["pbs-override"], 321); + } + + #[test] + fn profile_registry_is_independent_of_browser_configuration() { + let ids = crate::auction::profile::profile_registrations() + .iter() + .map(|registration| registration.id) + .collect::>(); + assert_eq!(ids, vec!["standard", "prebid-server", "aps"]); + let mut providers = BTreeMap::new(); + providers.insert(id("pbs"), provider("prebid-server")); + let plan = AuctionPlan::compile(config(providers)) + .expect("should compile without Settings or browser integration state"); + assert!(!plan.has_profile("aps")); + + let mut providers = BTreeMap::new(); + providers.insert( + id("aps-instance"), + ProviderConfig { + endpoint: "https://aps.example/e/pb/bid".to_string(), + profile_config: serde_json::json!({"account_id": "example-account"}), + ..provider("aps") + }, + ); + let plan = AuctionPlan::compile(config(providers)).expect("should compile APS plan"); + assert!( + plan.has_profile("aps"), + "validated plan should expose APS renderer capability" + ); + assert!(!plan.has_profile("prebid-server")); + } + + #[test] + fn compiler_rejects_unknown_protocol_profile_and_route() { + let mut unknown_protocol = provider("standard"); + unknown_protocol.protocol = "openrtb-2.5".to_string(); + assert!( + AuctionPlan::compile(config(BTreeMap::from([(id("one"), unknown_protocol)]))).is_err() + ); + assert!( + AuctionPlan::compile(config(BTreeMap::from([(id("one"), provider("unknown"))]))) + .is_err() + ); + let mut raw = config(BTreeMap::from([(id("one"), provider("standard"))])); + raw.bidders.insert( + bidder("example"), + BidderRouteConfig { + provider: id("missing"), + }, + ); + assert!(AuctionPlan::compile(raw).is_err()); + } + + #[test] + fn compiler_canonicalizes_https_endpoints_and_rejects_unsafe_forms() { + let mut canonical = provider("standard"); + canonical.endpoint = "https://BID.EXAMPLE:443/path".to_string(); + let plan = AuctionPlan::compile(config(BTreeMap::from([(id("one"), canonical)]))) + .expect("should canonicalize endpoint"); + assert_eq!( + plan.providers()[0].endpoint.as_str(), + "https://bid.example/path" + ); + for endpoint in [ + "http://bid.example/path", + "https://", + "https://user@bid.example/path", + "https://bid.example/path#fragment", + "/relative", + ] { + let mut raw_provider = provider("standard"); + raw_provider.endpoint = endpoint.to_string(); + assert!( + AuctionPlan::compile(config(BTreeMap::from([(id("one"), raw_provider)]))).is_err(), + "should reject {endpoint}" + ); + } + let mut aps = provider("aps"); + aps.endpoint = "https://aps.example/e/dtb/bid".to_string(); + aps.profile_config = serde_json::json!({"account_id": "example-account"}); + assert!(AuctionPlan::compile(config(BTreeMap::from([(id("aps"), aps)]))).is_err()); + } + + #[test] + fn compiler_normalizes_only_prebid_server_origin_and_canonical_paths() { + for (configured, expected) in [ + ( + "https://pbs.example", + "https://pbs.example/openrtb2/auction", + ), + ( + "https://pbs.example/", + "https://pbs.example/openrtb2/auction", + ), + ( + "https://pbs.example/openrtb2/auction", + "https://pbs.example/openrtb2/auction", + ), + ( + "https://pbs.example/openrtb2/auction/", + "https://pbs.example/openrtb2/auction", + ), + ( + "https://pbs.example?region=example", + "https://pbs.example/openrtb2/auction?region=example", + ), + ("https://pbs.example/bid", "https://pbs.example/bid"), + ( + "https://pbs.example/custom/pbs", + "https://pbs.example/custom/pbs", + ), + ] { + let mut pbs = provider("prebid-server"); + pbs.endpoint = configured.to_string(); + let plan = AuctionPlan::compile(config(BTreeMap::from([(id("pbs"), pbs)]))) + .expect("should compile Prebid Server endpoint"); + assert_eq!( + plan.providers()[0].endpoint.as_str(), + expected, + "{configured}" + ); + } + + let mut standard = provider("standard"); + standard.endpoint = "https://bid.example/".to_string(); + let plan = AuctionPlan::compile(config(BTreeMap::from([(id("standard"), standard)]))) + .expect("should compile standard root endpoint"); + assert_eq!( + plan.providers()[0].endpoint.as_str(), + "https://bid.example/" + ); + + let mut aps = provider("aps"); + aps.endpoint = "https://aps.example/e/pb/bid".to_string(); + aps.profile_config = serde_json::json!({"account_id": "example-account"}); + let plan = AuctionPlan::compile(config(BTreeMap::from([(id("aps"), aps)]))) + .expect("should compile APS endpoint"); + assert_eq!( + plan.providers()[0].endpoint.as_str(), + "https://aps.example/e/pb/bid" + ); + } + + #[test] + fn standard_extensions_are_typed_bounded_and_cannot_claim_reserved_fields() { + let mut valid = provider("standard"); + valid.profile_config = serde_json::json!({ + "request_ext": {"fictional_account": "example"}, + "imp_ext": {"placement_group": "display"} + }); + let plan = AuctionPlan::compile(config(BTreeMap::from([(id("one"), valid)]))) + .expect("should compile static extensions"); + let CompiledOpenRtbProfile::Standard(standard) = &plan.providers()[0].profile else { + panic!("should compile standard profile") + }; + assert_eq!( + standard.request_ext.as_object()["fictional_account"], + "example" + ); + + let mut standard_owned_fields = provider("standard"); + standard_owned_fields.profile_config = serde_json::json!({ + "request_ext": { + "account": "example-account", + "sdk": {"source": "example"}, + "prebid": {"example": true} + }, + "imp_ext": {"prebid": {"example": true}} + }); + AuctionPlan::compile(config(BTreeMap::from([( + id("standard-owned-fields"), + standard_owned_fields, + )]))) + .expect("should allow standard static extensions outside common-owned fields"); + + for profile_config in [ + serde_json::json!({"request_ext": "bad"}), + serde_json::json!({"request_ext": {"trusted_server": {}}}), + ] { + let mut invalid = provider("standard"); + invalid.profile_config = profile_config; + assert!(AuctionPlan::compile(config(BTreeMap::from([(id("one"), invalid)]))).is_err()); + } + let oversized = "x".repeat(16 * 1024); + let mut invalid = provider("standard"); + invalid.profile_config = serde_json::json!({"request_ext": {"value": oversized}}); + assert!(AuctionPlan::compile(config(BTreeMap::from([(id("one"), invalid)]))).is_err()); + + let too_many_keys = (0..257) + .map(|index| (format!("key-{index}"), Value::Bool(true))) + .collect::>(); + let mut invalid = provider("standard"); + invalid.profile_config = serde_json::json!({"imp_ext": too_many_keys}); + assert!(AuctionPlan::compile(config(BTreeMap::from([(id("one"), invalid)]))).is_err()); + } + + #[test] + fn standard_extension_depth_counts_container_levels() { + let mut object_valid = provider("standard"); + object_valid.profile_config = serde_json::json!({"request_ext": nested_object(8)}); + AuctionPlan::compile(config(BTreeMap::from([(id("object-valid"), object_valid)]))) + .expect("should accept eight nested object levels"); + + let mut object_invalid = provider("standard"); + object_invalid.profile_config = serde_json::json!({"request_ext": nested_object(9)}); + assert!( + AuctionPlan::compile(config(BTreeMap::from([( + id("object-invalid"), + object_invalid, + )]))) + .is_err(), + "should reject nine nested object levels" + ); + + let mut array_valid = provider("standard"); + array_valid.profile_config = serde_json::json!({"request_ext": {"value": nested_array(7)}}); + AuctionPlan::compile(config(BTreeMap::from([(id("array-valid"), array_valid)]))) + .expect("should accept one object plus seven nested array levels"); + + let mut array_invalid = provider("standard"); + array_invalid.profile_config = + serde_json::json!({"request_ext": {"value": nested_array(8)}}); + assert!( + AuctionPlan::compile(config(BTreeMap::from([( + id("array-invalid"), + array_invalid, + )]))) + .is_err(), + "should reject one object plus eight nested array levels" + ); + } + + #[test] + fn notification_policy_rejects_duplicates_and_bounds() { + let mut valid = provider("standard"); + valid.notifications = NotificationConfig { + suppress_all: true, + suppress_seats: vec!["seat-b".to_string(), "seat-a".to_string()], + }; + let plan = AuctionPlan::compile(config(BTreeMap::from([(id("one"), valid)]))) + .expect("should compile notifications"); + assert_eq!( + plan.providers()[0] + .notifications + .suppress_seats + .iter() + .map(String::as_str) + .collect::>(), + vec!["seat-a", "seat-b"] + ); + for seats in [ + vec!["same".to_string(), "same".to_string()], + vec![String::new()], + vec!["bad\nseat".to_string()], + vec!["x".repeat(129)], + (0..129).map(|index| format!("seat-{index}")).collect(), + ] { + let mut invalid = provider("standard"); + invalid.notifications.suppress_seats = seats; + assert!(AuctionPlan::compile(config(BTreeMap::from([(id("one"), invalid)]))).is_err()); + } + } + + #[test] + fn typed_profile_config_rejects_unknown_fields_and_validates_aps_pairing() { + let mut pbs = provider("prebid-server"); + pbs.profile_config = serde_json::json!({"browser_only": true}); + assert!(AuctionPlan::compile(config(BTreeMap::from([(id("pbs"), pbs)]))).is_err()); + + let mut non_object = provider("standard"); + non_object.profile_config = Value::Null; + assert!( + AuctionPlan::compile(config(BTreeMap::from([(id("standard"), non_object,)]))).is_err() + ); + + let mut aps = provider("aps"); + aps.endpoint = "https://aps.example/e/pb/bid".to_string(); + aps.profile_config = serde_json::json!({ + "account_id": "example-account", + "inventory_domain": "publisher.example" + }); + assert!(AuctionPlan::compile(config(BTreeMap::from([(id("aps"), aps)]))).is_err()); + } + + #[test] + fn enabled_signing_requires_nonblank_existing_global_store_ids() { + let mut raw = config(BTreeMap::from([(id("one"), provider("standard"))])); + raw.request_signing = Some(RequestSigning { + enabled: true, + config_store_id: " ".to_string(), + secret_store_id: "example-secret-store".to_string(), + }); + assert!( + AuctionPlan::compile(raw).is_err(), + "should reject enabled signing without a config store ID" + ); + + let mut raw = config(BTreeMap::from([(id("one"), provider("standard"))])); + raw.request_signing = Some(RequestSigning { + enabled: true, + config_store_id: "example-config-store".to_string(), + secret_store_id: "\t".to_string(), + }); + assert!( + AuctionPlan::compile(raw).is_err(), + "should reject enabled signing without a secret store ID" + ); + } + + #[test] + fn routing_signing_and_static_mediator_are_preserved_in_plan() { + let mut provider = provider("standard"); + provider.routing = RoutingMode::AllEligible; + let mut raw = config(BTreeMap::from([(id("one"), provider)])); + raw.request_signing = Some(RequestSigning { + enabled: true, + config_store_id: "example-config-store".to_string(), + secret_store_id: "example-secret-store".to_string(), + }); + raw.mediator = Some(MOCK_MEDIATOR_ID.to_string()); + let plan = AuctionPlan::compile(raw).expect("should compile common policies"); + assert_eq!(plan.providers()[0].routing, RoutingMode::AllEligible); + assert!(plan.signing_enabled()); + assert_eq!(plan.mediator(), Some(MOCK_MEDIATOR_ID)); + + let mut invalid = config(BTreeMap::new()); + invalid.mediator = Some("generic-mediator".to_string()); + assert!(AuctionPlan::compile(invalid).is_err()); + } +} diff --git a/crates/trusted-server-core/src/auction/profile.rs b/crates/trusted-server-core/src/auction/profile.rs new file mode 100644 index 000000000..51a91e69c --- /dev/null +++ b/crates/trusted-server-core/src/auction/profile.rs @@ -0,0 +1,325 @@ +//! Compile-time `OpenRTB` profile registry and typed profile plans. + +use std::collections::BTreeMap; + +use error_stack::Report; +use serde::Deserialize; +use serde_json::{Map, Value}; + +use crate::consent_config::ConsentForwardingMode; +use crate::error::TrustedServerError; +use crate::integrations::aps::compile_profile_config as compile_aps_profile_config; +use crate::integrations::prebid::{ + BidParamOverrideEngine, BidParamOverrideRule, compile_profile_override_rules, +}; + +const STANDARD_PROFILE_ID: &str = "standard"; +const PREBID_PROFILE_ID: &str = "prebid-server"; +const APS_PROFILE_ID: &str = "aps"; +const STATIC_EXTENSION_MAX_BYTES: usize = 16 * 1024; +const STATIC_EXTENSION_MAX_DEPTH: usize = 8; +const STATIC_EXTENSION_MAX_KEYS: usize = 256; + +/// A registered profile's provider-timeout default. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ProfileTimeoutDefault { + /// Inherit the configured auction timeout. + Auction, + /// Use this fixed profile timeout. + Fixed(u32), +} + +/// Compile-time profile registration. +#[derive(Clone, Copy)] +pub struct OpenRtbProfileRegistration { + /// Stable profile identifier used by configuration. + pub id: &'static str, + /// Profile timeout used when a provider has no explicit override. + pub default_timeout: ProfileTimeoutDefault, + compile: fn(&Value) -> Result>, +} + +impl core::fmt::Debug for OpenRtbProfileRegistration { + fn fmt(&self, formatter: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + formatter + .debug_struct("OpenRtbProfileRegistration") + .field("id", &self.id) + .field("default_timeout", &self.default_timeout) + .finish_non_exhaustive() + } +} + +impl OpenRtbProfileRegistration { + pub(crate) fn compile( + self, + config: &Value, + ) -> Result> { + (self.compile)(config) + } +} + +/// Immutable, typed profile behavior selected during plan compilation. +#[derive(Debug, Clone)] +pub enum CompiledOpenRtbProfile { + /// Generic `OpenRTB` 2.6 profile. + Standard(StandardProfilePlan), + /// Prebid Server compatibility profile. + PrebidServer(PrebidProfilePlan), + /// APS `OpenRTB` compatibility profile. + Aps(ApsProfilePlan), +} + +impl CompiledOpenRtbProfile { + /// Return the stable profile identifier. + #[must_use] + pub fn id(&self) -> &'static str { + match self { + Self::Standard(_) => STANDARD_PROFILE_ID, + Self::PrebidServer(_) => PREBID_PROFILE_ID, + Self::Aps(_) => APS_PROFILE_ID, + } + } + + /// Return whether this plan uses the Prebid Server profile. + #[must_use] + pub(crate) fn is_prebid_server(&self) -> bool { + matches!(self, Self::PrebidServer(_)) + } +} + +/// Validated static extension object. +#[derive(Debug, Clone, Default, PartialEq)] +pub struct StaticExtension(Map); + +impl StaticExtension { + /// Borrow the validated extension object. + #[must_use] + pub fn as_object(&self) -> &Map { + &self.0 + } +} + +/// Compiled generic `OpenRTB` profile configuration. +#[derive(Debug, Clone, Default)] +pub struct StandardProfilePlan { + /// Static request-level extension fields. + pub request_ext: StaticExtension, + /// Static impression-level extension fields. + pub imp_ext: StaticExtension, +} + +/// Compiled Prebid profile configuration. +#[derive(Debug, Clone)] +pub struct PrebidProfilePlan { + /// Include Prebid HTTP exchange diagnostics. + pub debug: bool, + /// Set `OpenRTB` test mode. + pub test_mode: bool, + /// Optional query fragment appended to the page URL under legacy rules. + pub debug_query_params: Option, + /// Compiled override matching and merge index. + pub(crate) override_engine: BidParamOverrideEngine, + /// Consent transport policy. + pub consent_forwarding: ConsentForwardingMode, +} + +/// Compiled APS profile configuration. +#[derive(Debug, Clone)] +pub struct ApsProfilePlan { + /// APS account identifier. + pub account_id: String, + /// Include APS request/response diagnostics. + pub debug: bool, + /// Permit APS script creatives. + pub allow_script_creatives: bool, + /// Optional authorized inventory domain. + pub inventory_domain: Option, + /// Optional canonical inventory page origin. + pub inventory_page_origin: Option, +} + +#[derive(Debug, Deserialize, Default)] +#[serde(deny_unknown_fields)] +struct StandardProfileConfig { + #[serde(default)] + request_ext: Option, + #[serde(default)] + imp_ext: Option, +} + +/// Typed operator configuration compiled into a [`PrebidProfilePlan`]. +#[derive(Debug, Deserialize, Default)] +#[serde(deny_unknown_fields)] +pub(crate) struct PrebidProfileConfig { + #[serde(default)] + debug: bool, + #[serde(default)] + test_mode: bool, + #[serde(default)] + debug_query_params: Option, + #[serde(default)] + bid_param_zone_overrides: BTreeMap>>, + #[serde(default)] + bid_param_overrides: BTreeMap>, + #[serde(default)] + bid_param_override_rules: Vec, + #[serde(default)] + consent_forwarding: ConsentForwardingMode, +} + +const PROFILE_REGISTRATIONS: [OpenRtbProfileRegistration; 3] = [ + OpenRtbProfileRegistration { + id: STANDARD_PROFILE_ID, + default_timeout: ProfileTimeoutDefault::Auction, + compile: compile_standard, + }, + OpenRtbProfileRegistration { + id: PREBID_PROFILE_ID, + default_timeout: ProfileTimeoutDefault::Fixed(1000), + compile: compile_prebid, + }, + OpenRtbProfileRegistration { + id: APS_PROFILE_ID, + default_timeout: ProfileTimeoutDefault::Fixed(800), + compile: compile_aps, + }, +]; + +/// Return the compile-time profile registry. +#[must_use] +pub fn profile_registrations() -> &'static [OpenRtbProfileRegistration] { + &PROFILE_REGISTRATIONS +} + +pub(crate) fn find_profile(id: &str) -> Option { + profile_registrations() + .iter() + .copied() + .find(|registration| registration.id == id) +} + +fn configuration_error(message: impl Into) -> Report { + Report::new(TrustedServerError::Configuration { + message: message.into(), + }) +} + +fn deserialize_profile(id: &str, value: &Value) -> Result> +where + T: for<'de> Deserialize<'de>, +{ + T::deserialize(value) + .map_err(|error| configuration_error(format!("invalid `{id}` profile_config: {error}"))) +} + +fn compile_standard(value: &Value) -> Result> { + let config: StandardProfileConfig = deserialize_profile(STANDARD_PROFILE_ID, value)?; + Ok(CompiledOpenRtbProfile::Standard(StandardProfilePlan { + request_ext: validate_static_extension("request_ext", config.request_ext)?, + imp_ext: validate_static_extension("imp_ext", config.imp_ext)?, + })) +} + +fn compile_prebid(value: &Value) -> Result> { + let config: PrebidProfileConfig = deserialize_profile(PREBID_PROFILE_ID, value)?; + let override_engine = compile_profile_override_rules( + &config.bid_param_zone_overrides, + &config.bid_param_overrides, + &config.bid_param_override_rules, + )?; + Ok(CompiledOpenRtbProfile::PrebidServer(PrebidProfilePlan { + debug: config.debug, + test_mode: config.test_mode, + debug_query_params: config.debug_query_params, + override_engine, + consent_forwarding: config.consent_forwarding, + })) +} + +fn compile_aps(value: &Value) -> Result> { + let config = compile_aps_profile_config(value.clone())?; + Ok(CompiledOpenRtbProfile::Aps(ApsProfilePlan { + account_id: config.account_id, + debug: config.debug, + allow_script_creatives: config.allow_script_creatives, + inventory_domain: config.inventory_domain, + inventory_page_origin: config.inventory_page_origin, + })) +} + +fn validate_static_extension( + field: &str, + value: Option, +) -> Result> { + let Some(value) = value else { + return Ok(StaticExtension::default()); + }; + let object = value.as_object().ok_or_else(|| { + configuration_error(format!("standard profile {field} must be an object")) + })?; + let size = serde_json::to_vec(&value) + .map_err(|error| configuration_error(format!("cannot serialize {field}: {error}")))? + .len(); + if size > STATIC_EXTENSION_MAX_BYTES { + return Err(configuration_error(format!( + "standard profile {field} exceeds {STATIC_EXTENSION_MAX_BYTES} bytes" + ))); + } + validate_extension_value(field, &value, 0)?; + reject_reserved_fields(field, object)?; + Ok(StaticExtension(object.clone())) +} + +fn validate_extension_value( + field: &str, + value: &Value, + container_depth: usize, +) -> Result<(), Report> { + match value { + Value::Object(object) => { + let container_depth = container_depth + 1; + if container_depth > STATIC_EXTENSION_MAX_DEPTH { + return Err(configuration_error(format!( + "standard profile {field} exceeds nesting depth {STATIC_EXTENSION_MAX_DEPTH}" + ))); + } + if object.len() > STATIC_EXTENSION_MAX_KEYS { + return Err(configuration_error(format!( + "standard profile {field} object exceeds {STATIC_EXTENSION_MAX_KEYS} keys" + ))); + } + for nested in object.values() { + validate_extension_value(field, nested, container_depth)?; + } + } + Value::Array(array) => { + let container_depth = container_depth + 1; + if container_depth > STATIC_EXTENSION_MAX_DEPTH { + return Err(configuration_error(format!( + "standard profile {field} exceeds nesting depth {STATIC_EXTENSION_MAX_DEPTH}" + ))); + } + for nested in array { + validate_extension_value(field, nested, container_depth)?; + } + } + _ => {} + } + Ok(()) +} + +fn reject_reserved_fields( + field: &str, + object: &Map, +) -> Result<(), Report> { + let reserved: &[&str] = match field { + "request_ext" => &["trusted_server"], + _ => &[], + }; + if let Some(key) = reserved.iter().find(|key| object.contains_key(**key)) { + return Err(configuration_error(format!( + "standard profile {field} cannot claim reserved field `{key}`" + ))); + } + Ok(()) +} diff --git a/crates/trusted-server-core/src/auction/provider.rs b/crates/trusted-server-core/src/auction/provider.rs index 766bd7a08..28f6dbd96 100644 --- a/crates/trusted-server-core/src/auction/provider.rs +++ b/crates/trusted-server-core/src/auction/provider.rs @@ -1,15 +1,45 @@ //! Trait definition for auction providers. use core::any::Any; +use std::collections::HashSet; use async_trait::async_trait; -use error_stack::Report; +use edgezero_core::body::Body as EdgeBody; +use error_stack::{Report, ResultExt as _}; +use http::{Method, Request, StatusCode, header}; +use serde_json::{Value, json}; + +use crate::integrations::aps::{ApsDebugRequest, parse_planned_aps_response}; +use crate::integrations::prebid::{apply_prebid_transport_headers, parse_planned_prebid_response}; use crate::error::TrustedServerError; -use crate::platform::{PlatformPendingRequest, PlatformResponse, RuntimeServices}; +use crate::platform::{ + PlatformHttpRequest, PlatformPendingRequest, PlatformResponse, RuntimeServices, +}; +use crate::request_signing::{RequestSigner, SigningParams}; +use super::openrtb::{ + OpenRtbBuildOutcome, RequestFinalization, apply_notification_policy, build_request, + extract_standard_response, unused_bidder_params_count, +}; +use super::plan::ProviderPlan; +use super::profile::CompiledOpenRtbProfile; +use super::routing::{ProviderAuctionInput, RoutedAuction}; use super::types::{AuctionContext, AuctionRequest, AuctionResponse}; +const MAX_PLANNED_RESPONSE_BYTES: usize = 1024 * 1024; + +fn attach_provider_routing_metadata( + response: &mut AuctionResponse, + profile: &CompiledOpenRtbProfile, + input: &ProviderAuctionInput, +) { + response.metadata.insert( + "routing".to_string(), + json!({"unused_bidder_params_count": unused_bidder_params_count(profile, input)}), + ); +} + /// Provider-local state carried from request dispatch to response parsing. pub type ProviderParseState = Box; @@ -52,8 +82,11 @@ impl ProviderRequestOutcome { /// Trait implemented by all auction providers (Prebid, APS, GAM, etc.). #[async_trait(?Send)] pub trait AuctionProvider: Send + Sync { - /// Unique identifier for this provider (e.g., "prebid", "aps", "gam"). - fn provider_name(&self) -> &'static str; + /// Borrow this provider instance's unique validated identifier. + /// + /// Legacy providers may return a string literal; config-first providers + /// return their owned operator-defined [`super::plan::ProviderId`]. + fn provider_name(&self) -> &str; /// Submit a bid request to this provider. /// @@ -155,3 +188,398 @@ pub trait AuctionProvider: Send + Sync { None } } + +/// One immutable config-first `OpenRTB` provider instance. +/// +/// Every instance owns its validated provider identity and carries only its own +/// typed response state across transport. +pub(crate) struct GenericOpenRtbProvider { + plan: ProviderPlan, +} + +/// Typed state created by and returned to one [`GenericOpenRtbProvider`]. +#[allow( + dead_code, + clippy::large_enum_variant, + reason = "typed Stage 6 state avoids provider-state confusion; Stage 7/8 replace profile variants" +)] +pub(crate) enum GenericOpenRtbParseState { + Standard { + provider_id: String, + input: ProviderAuctionInput, + }, + Prebid { + provider_id: String, + auction_id: String, + input: ProviderAuctionInput, + }, + Aps { + provider_id: String, + input: ProviderAuctionInput, + debug_request: Option, + }, +} + +impl GenericOpenRtbProvider { + pub(crate) fn new(plan: ProviderPlan) -> Self { + Self { plan } + } + + pub(crate) fn provider_name(&self) -> &str { + self.plan.id.as_str() + } + + pub(crate) fn timeout_ms(&self) -> u32 { + self.plan.timeout_ms + } + + #[cfg(test)] + pub(crate) fn parse_state_for_test(&self, input: ProviderAuctionInput) -> ProviderParseState { + let state = match &self.plan.profile { + CompiledOpenRtbProfile::Standard(_) => GenericOpenRtbParseState::Standard { + provider_id: self.provider_name().to_string(), + input, + }, + CompiledOpenRtbProfile::PrebidServer(_) => GenericOpenRtbParseState::Prebid { + provider_id: self.provider_name().to_string(), + auction_id: input.common_request().id.clone(), + input, + }, + CompiledOpenRtbProfile::Aps(_) => GenericOpenRtbParseState::Aps { + provider_id: self.provider_name().to_string(), + input, + debug_request: None, + }, + }; + Box::new(state) + } + + /// Build, register, and start exactly one routed provider request. + /// + /// The public [`AuctionProvider::request_bids`] seam cannot carry a + /// [`ProviderAuctionInput`], the complete [`RoutedAuction`], or an + /// auction-local [`RequestSigner`] without shared mutable provider state. + /// The plan-backed split dispatcher therefore supplies that explicit + /// execution context here, while this method reuses + /// [`ProviderRequestOutcome`] and [`ProviderParseState`] for the existing + /// request/parse token boundary. + #[allow( + clippy::too_many_arguments, + reason = "the internal driver keeps routed inputs, both budgets, signer, services, and collision state explicit" + )] + pub(crate) async fn request_bids_routed( + &self, + input: &ProviderAuctionInput, + routed: &RoutedAuction, + logical_budget_ms: u32, + transport_timeout_ms: u32, + signer: Option<&RequestSigner>, + services: &RuntimeServices, + reserved_backend_names: &mut HashSet, + ) -> Result> { + let signing_params = SigningParams::new( + input.common_request().id.clone(), + input.common_request().publisher.domain.clone(), + "https".to_string(), + ); + let request = match build_request( + input, + routed, + &self.plan, + logical_budget_ms, + &RequestFinalization { + signer, + signing_params, + }, + )? { + OpenRtbBuildOutcome::Ready(request) => request, + OpenRtbBuildOutcome::NoImpressions => { + return Ok(ProviderRequestOutcome::Immediate(AuctionResponse::no_bid( + self.provider_name(), + 0, + ))); + } + }; + + let spec = self + .plan + .backend_spec_with_transport_timeout(transport_timeout_ms); + let predicted_name = + services + .backend() + .predict_name(&spec) + .change_context(TrustedServerError::Auction { + message: format!( + "Provider {} backend prediction failed", + self.provider_name() + ), + })?; + let backend_name = + services + .backend() + .ensure(&spec) + .change_context(TrustedServerError::Auction { + message: format!( + "Provider {} backend registration failed", + self.provider_name() + ), + })?; + if backend_name != predicted_name { + return Err(Report::new(TrustedServerError::Auction { + message: format!( + "Provider {} backend registration did not match prediction", + self.provider_name() + ), + })); + } + if !reserved_backend_names.insert(backend_name.clone()) { + return Err(Report::new(TrustedServerError::Auction { + message: format!( + "Provider {} resolved an actual backend name already owned by another provider", + self.provider_name() + ), + })); + } + + let body = serde_json::to_vec(&request).change_context(TrustedServerError::Auction { + message: format!( + "Provider {} request serialization failed", + self.provider_name() + ), + })?; + let mut outbound = Request::builder() + .method(Method::POST) + .uri(self.plan.endpoint.as_str()) + .header(header::CONTENT_TYPE, "application/json"); + if matches!(&self.plan.profile, CompiledOpenRtbProfile::Standard(_)) { + outbound = outbound.header(header::ACCEPT, "application/json"); + } + let aps_debug_body = matches!( + &self.plan.profile, + CompiledOpenRtbProfile::Aps(profile) if profile.debug + ) + .then(|| body.clone()); + let mut outbound = + outbound + .body(EdgeBody::from(body)) + .change_context(TrustedServerError::Auction { + message: format!( + "Provider {} request construction failed", + self.provider_name() + ), + })?; + let aps_debug_request = aps_debug_body + .as_deref() + .map(|body| ApsDebugRequest::capture(body, outbound.headers())); + if let CompiledOpenRtbProfile::PrebidServer(profile) = &self.plan.profile { + apply_prebid_transport_headers( + routed.prebid_transport_headers(), + &mut outbound, + profile.consent_forwarding, + routed.attested_client_ip(), + ); + } + let pending = services + .http_client() + .send_async(PlatformHttpRequest::new(outbound, backend_name.clone())) + .await + .change_context(TrustedServerError::Auction { + message: format!("Provider {} request launch failed", self.provider_name()), + })?; + if pending.backend_name() != Some(backend_name.as_str()) { + return Err(Report::new(TrustedServerError::Auction { + message: format!( + "Provider {} pending request backend did not match registered backend", + self.provider_name() + ), + })); + } + let parse_state = match &self.plan.profile { + CompiledOpenRtbProfile::Standard(_) => GenericOpenRtbParseState::Standard { + provider_id: self.provider_name().to_string(), + input: input.clone(), + }, + CompiledOpenRtbProfile::PrebidServer(_) => GenericOpenRtbParseState::Prebid { + provider_id: self.provider_name().to_string(), + auction_id: input.common_request().id.clone(), + input: input.clone(), + }, + CompiledOpenRtbProfile::Aps(_) => GenericOpenRtbParseState::Aps { + provider_id: self.provider_name().to_string(), + input: input.clone(), + debug_request: aps_debug_request, + }, + }; + Ok(ProviderRequestOutcome::pending_with_state( + pending, + Box::new(parse_state), + )) + } + + /// Parse a response using state created by this exact provider instance. + pub(crate) async fn parse_response_with_state( + &self, + response: PlatformResponse, + response_time_ms: u64, + parse_state: Option<&(dyn Any + Send + Sync)>, + ) -> Result> { + let parse_state = parse_state + .and_then(|state| state.downcast_ref::()) + .ok_or_else(|| { + Report::new(TrustedServerError::Auction { + message: format!( + "Provider {} received missing or invalid response state", + self.provider_name() + ), + }) + })?; + let state_provider_id = match parse_state { + GenericOpenRtbParseState::Standard { provider_id, .. } + | GenericOpenRtbParseState::Prebid { provider_id, .. } + | GenericOpenRtbParseState::Aps { provider_id, .. } => provider_id, + }; + if state_provider_id != self.provider_name() { + return Err(Report::new(TrustedServerError::Auction { + message: format!( + "Provider {} received response state owned by provider {}", + self.provider_name(), + state_provider_id + ), + })); + } + + if let GenericOpenRtbParseState::Prebid { + auction_id, input, .. + } = parse_state + { + let CompiledOpenRtbProfile::PrebidServer(profile) = &self.plan.profile else { + return Err(Report::new(TrustedServerError::Auction { + message: format!( + "Provider {} received PBS response state for profile {}", + self.provider_name(), + self.plan.profile.id() + ), + })); + }; + let mut parsed = match parse_planned_prebid_response( + self.provider_name(), + profile, + input, + response, + response_time_ms, + auction_id, + ) + .await + { + Ok(parsed) => parsed, + Err(error) => { + log::warn!( + "Provider '{}' PBS response parse failed: {:?}", + self.provider_name(), + error + ); + AuctionResponse::error(self.provider_name(), response_time_ms) + .with_metadata("error_type", json!("parse_response")) + } + }; + apply_notification_policy(&mut parsed.bids, &self.plan.notifications); + attach_provider_routing_metadata(&mut parsed, &self.plan.profile, input); + return Ok(parsed); + } + + if let GenericOpenRtbParseState::Aps { + input, + debug_request, + .. + } = parse_state + { + let CompiledOpenRtbProfile::Aps(profile) = &self.plan.profile else { + return Err(Report::new(TrustedServerError::Auction { + message: format!( + "Provider {} received APS response state for profile {}", + self.provider_name(), + self.plan.profile.id() + ), + })); + }; + let mut parsed = match parse_planned_aps_response( + self.provider_name(), + profile, + self.plan.endpoint.as_str(), + input, + response, + response_time_ms, + debug_request.clone(), + ) + .await + { + Ok(parsed) => parsed, + Err(error) => { + log::warn!( + "Provider '{}' APS response parse failed: {:?}", + self.provider_name(), + error + ); + let mut parsed = AuctionResponse::error(self.provider_name(), response_time_ms) + .with_metadata("error_type", json!("parse_response")); + attach_provider_routing_metadata(&mut parsed, &self.plan.profile, input); + parsed + } + }; + apply_notification_policy(&mut parsed.bids, &self.plan.notifications); + return Ok(parsed); + } + + let response = response.response; + let status = response.status(); + let GenericOpenRtbParseState::Standard { input, .. } = parse_state else { + unreachable!("profile-specific states are handled before standard parsing"); + }; + if status == StatusCode::NO_CONTENT { + let mut parsed = AuctionResponse::no_bid(self.provider_name(), response_time_ms); + attach_provider_routing_metadata(&mut parsed, &self.plan.profile, input); + return Ok(parsed); + } + if !status.is_success() { + if status.is_redirection() { + log::warn!( + "Provider '{}' returned a redirect; generic OpenRTB redirects are refused", + self.provider_name() + ); + } + let mut parsed = AuctionResponse::error(self.provider_name(), response_time_ms) + .with_metadata("error_type", json!("http_status")) + .with_metadata("http_status", json!(status.as_u16())); + attach_provider_routing_metadata(&mut parsed, &self.plan.profile, input); + return Ok(parsed); + } + + let body = response + .into_body() + .into_bytes_bounded(MAX_PLANNED_RESPONSE_BYTES) + .await + .change_context(TrustedServerError::Auction { + message: format!("Provider {} response body failed", self.provider_name()), + })?; + let value: Value = match serde_json::from_slice(&body) { + Ok(value) => value, + Err(error) => { + log::warn!( + "Provider '{}' response JSON was invalid: {}", + self.provider_name(), + error + ); + let mut parsed = AuctionResponse::error(self.provider_name(), response_time_ms) + .with_metadata("error_type", json!("parse_response")); + attach_provider_routing_metadata(&mut parsed, &self.plan.profile, input); + return Ok(parsed); + } + }; + + let mut parsed = + extract_standard_response(self.provider_name(), input, &value, response_time_ms); + apply_notification_policy(&mut parsed.bids, &self.plan.notifications); + attach_provider_routing_metadata(&mut parsed, &self.plan.profile, input); + Ok(parsed) + } +} diff --git a/crates/trusted-server-core/src/auction/routing.rs b/crates/trusted-server-core/src/auction/routing.rs new file mode 100644 index 000000000..61b2302cc --- /dev/null +++ b/crates/trusted-server-core/src/auction/routing.rs @@ -0,0 +1,1200 @@ +//! Internal admission normalization and config-first provider routing. + +use std::collections::{BTreeMap, BTreeSet, HashMap}; +use std::net::IpAddr; + +use edgezero_core::body::Body as EdgeBody; +use http::{HeaderValue, Request, header}; +use serde_json::Value; + +use super::plan::{AuctionPlan, BidderId, ProviderId, RoutingMode}; +use super::types::{AdSlot, AuctionRequest, MediaType}; + +const TRUSTED_SERVER_ENVELOPE: &str = "trustedServer"; +const BIDDER_PARAMS_FIELD: &str = "bidderParams"; +const ZONE_FIELD: &str = "zone"; + +/// Maximum bidder entries admitted from one browser `bidderParams` envelope. +pub(crate) const MAX_BIDDER_ENTRIES: usize = 128; +/// Maximum UTF-8 byte length of the optional Prebid zone fact. +pub(crate) const MAX_PREBID_ZONE_BYTES: usize = 256; + +/// Immutable provider-local routing output in deterministic provider-ID order. +#[derive(Debug, Clone)] +pub(crate) struct RoutedAuction { + inputs: Vec, + skipped_no_eligible_provider_ids: Vec, + diagnostics: RoutingDiagnostics, + transport_headers: PrebidTransportHeaders, + attested_client_ip: Option, + dnt: Option, +} + +impl RoutedAuction { + pub(crate) fn inputs(&self) -> &[ProviderAuctionInput] { + &self.inputs + } + + /// Providers skipped because they had no eligible banner slots. + /// + /// IDs retain the compiled plan's deterministic provider-ID order. + pub(crate) fn skipped_no_eligible_provider_ids(&self) -> &[ProviderId] { + &self.skipped_no_eligible_provider_ids + } + + pub(crate) fn diagnostics(&self) -> RoutingDiagnostics { + self.diagnostics + } + + /// Request headers approved for later Prebid transport forwarding. + pub(crate) fn prebid_transport_headers(&self) -> &PrebidTransportHeaders { + &self.transport_headers + } + + /// Platform-attested client IP for transport forwarding. + pub(crate) fn attested_client_ip(&self) -> Option { + self.attested_client_ip + } + + /// Normalized Do Not Track fact; raw request headers are not exposed to profiles. + pub(crate) fn dnt(&self) -> Option { + self.dnt + } +} + +/// Saturating, count-only routing diagnostics. +#[derive(Debug, Clone, Copy, Default, Eq, PartialEq)] +pub(crate) struct RoutingDiagnostics { + unroutable_bidder_count: u32, + malformed_envelope_count: u32, + malformed_direct_demand_count: u32, + unroutable_trusted_provider_count: u32, +} + +impl RoutingDiagnostics { + pub(crate) fn unroutable_bidder_count(self) -> u32 { + self.unroutable_bidder_count + } + + #[cfg(test)] + pub(crate) fn malformed_envelope_count(self) -> u32 { + self.malformed_envelope_count + } + + #[cfg(test)] + pub(crate) fn malformed_direct_demand_count(self) -> u32 { + self.malformed_direct_demand_count + } + + #[cfg(test)] + pub(crate) fn unroutable_trusted_provider_count(self) -> u32 { + self.unroutable_trusted_provider_count + } + + fn record_unroutable_bidder(&mut self) { + self.unroutable_bidder_count = self.unroutable_bidder_count.saturating_add(1); + } + + fn record_malformed_envelope(&mut self) { + self.malformed_envelope_count = self.malformed_envelope_count.saturating_add(1); + } + + fn record_malformed_direct_demand(&mut self) { + self.malformed_direct_demand_count = self.malformed_direct_demand_count.saturating_add(1); + } + + fn record_unroutable_trusted_provider(&mut self) { + self.unroutable_trusted_provider_count = + self.unroutable_trusted_provider_count.saturating_add(1); + } + + #[cfg(test)] + pub(crate) fn saturated_for_test() -> Self { + let mut diagnostics = Self { + unroutable_bidder_count: u32::MAX, + ..Self::default() + }; + diagnostics.record_unroutable_bidder(); + diagnostics + } +} + +/// Provider-local immutable auction input. +#[derive(Debug, Clone)] +pub(crate) struct ProviderAuctionInput { + provider_id: ProviderId, + #[cfg_attr( + not(test), + allow( + dead_code, + reason = "retained in routed input to pin the provider budget invariant" + ) + )] + timeout_ms: u32, + common_request: AuctionRequest, + slots: Vec, +} + +impl ProviderAuctionInput { + pub(crate) fn provider_id(&self) -> &ProviderId { + &self.provider_id + } + + #[cfg(test)] + pub(crate) fn timeout_ms(&self) -> u32 { + self.timeout_ms + } + + /// Common privacy-approved request data. Its slot list is always empty. + pub(crate) fn common_request(&self) -> &AuctionRequest { + &self.common_request + } + + pub(crate) fn slots(&self) -> &[ProviderSlotInput] { + &self.slots + } +} + +/// One eligible slot with only the demand assigned to this provider. +#[derive(Debug, Clone)] +pub(crate) struct ProviderSlotInput { + slot: AdSlot, + bidder_params: BTreeMap, + prebid_zone: Option, + trusted_stored_request: bool, +} + +impl ProviderSlotInput { + /// Common slot facts. The legacy `bidders` map is always empty. + pub(crate) fn slot(&self) -> &AdSlot { + &self.slot + } + + pub(crate) fn bidder_params(&self) -> &BTreeMap { + &self.bidder_params + } + + pub(crate) fn prebid_zone(&self) -> Option<&str> { + self.prebid_zone.as_deref() + } + + pub(crate) fn has_trusted_stored_request(&self) -> bool { + self.trusted_stored_request + } +} + +/// Request headers approved for later Prebid transport forwarding. +/// +/// Values remain as raw [`HeaderValue`] instances so non-ASCII bytes retain +/// the same legacy handling. Client-supplied `X-Forwarded-For` is never read. +#[derive(Debug, Clone, Default)] +pub(crate) struct PrebidTransportHeaders { + cookie: Option, + user_agent: Option, + referer: Option, + accept_language: Option, +} + +impl PrebidTransportHeaders { + pub(crate) fn cookie(&self) -> Option<&HeaderValue> { + self.cookie.as_ref() + } + + pub(crate) fn user_agent(&self) -> Option<&HeaderValue> { + self.user_agent.as_ref() + } + + pub(crate) fn referer(&self) -> Option<&HeaderValue> { + self.referer.as_ref() + } + + pub(crate) fn accept_language(&self) -> Option<&HeaderValue> { + self.accept_language.as_ref() + } + + fn snapshot(request: &Request) -> Self { + Self { + cookie: request.headers().get(header::COOKIE).cloned(), + user_agent: request.headers().get(header::USER_AGENT).cloned(), + referer: request.headers().get(header::REFERER).cloned(), + accept_language: request.headers().get(header::ACCEPT_LANGUAGE).cloned(), + } + } +} + +/// Server-owned explicit provider routes aligned with canonical auction slots. +/// +/// This internal-only type has no deserializer, so browser input cannot select +/// a provider ID. The caller supplies one route list for each request slot. +#[derive(Debug, Default)] +pub(crate) struct TrustedProviderRoutes { + routes_by_slot: Vec>, +} + +impl TrustedProviderRoutes { + #[cfg(test)] + pub(crate) fn new(routes_by_slot: Vec>) -> Self { + Self { routes_by_slot } + } + + fn for_slot(&self, slot_index: usize) -> &[ProviderId] { + self.routes_by_slot + .get(slot_index) + .map_or(&[], Vec::as_slice) + } +} + +#[derive(Debug, Default)] +struct NormalizedSlotDemand { + bidder_params: BTreeMap, + stored_request: bool, + prebid_zone: Option, +} + +#[derive(Debug)] +struct ProviderInputBuilder { + provider_id: ProviderId, + timeout_ms: u32, + is_prebid: bool, + routing: RoutingMode, + slots: Vec, +} + +/// Normalize admitted demand and build deterministic provider-local inputs. +/// +/// This helper consumes the canonical request so the common request retained by +/// each provider can be scrubbed of slots. It performs no I/O. +pub(crate) fn route_auction( + request: AuctionRequest, + inbound_request: &Request, + plan: &AuctionPlan, + attested_client_ip: Option, +) -> RoutedAuction { + route_auction_with_trusted_routes( + request, + inbound_request, + plan, + attested_client_ip, + &TrustedProviderRoutes::default(), + ) +} + +/// Route an auction with server-owned explicit provider routes. +/// +/// Only server-generated entry points may construct [`TrustedProviderRoutes`]. +/// Browser/default admission must use [`route_auction`]. +pub(crate) fn route_auction_with_trusted_routes( + mut request: AuctionRequest, + inbound_request: &Request, + plan: &AuctionPlan, + attested_client_ip: Option, + trusted_routes: &TrustedProviderRoutes, +) -> RoutedAuction { + let slots = std::mem::take(&mut request.slots); + let common_request = request; + let transport_headers = PrebidTransportHeaders::snapshot(inbound_request); + let dnt = inbound_request + .headers() + .get("dnt") + .and_then(|value| value.to_str().ok()) + .is_some_and(|value| value.trim() == "1") + .then_some(true); + let mut diagnostics = RoutingDiagnostics::default(); + let mut builders = plan + .providers() + .iter() + .map(|provider| ProviderInputBuilder { + provider_id: provider.id.clone(), + timeout_ms: provider.timeout_ms, + is_prebid: provider.profile.is_prebid_server(), + routing: provider.routing, + slots: Vec::new(), + }) + .collect::>(); + let provider_indices = builders + .iter() + .enumerate() + .map(|(index, provider)| (provider.provider_id.clone(), index)) + .collect::>(); + + for (slot_index, slot) in slots.into_iter().enumerate() { + let Some(common_slot) = eligible_banner_slot(&slot) else { + continue; + }; + let demand = normalize_slot_demand(&slot.bidders, &mut diagnostics); + let mut routed_params = vec![BTreeMap::new(); builders.len()]; + for (bidder, params) in demand.bidder_params { + let Some(provider) = plan.provider_for_bidder(&bidder) else { + diagnostics.record_unroutable_bidder(); + continue; + }; + let provider_index = *provider_indices + .get(&provider.id) + .expect("should resolve compiled provider index"); + routed_params[provider_index].insert(bidder, params); + } + let mut trusted_provider_indices = BTreeSet::new(); + for provider_id in trusted_routes.for_slot(slot_index) { + let Some(provider_index) = provider_indices.get(provider_id).copied() else { + diagnostics.record_unroutable_trusted_provider(); + continue; + }; + trusted_provider_indices.insert(provider_index); + } + + for (provider_index, builder) in builders.iter_mut().enumerate() { + let bidder_params = std::mem::take(&mut routed_params[provider_index]); + let trusted_route = trusted_provider_indices.contains(&provider_index); + let trusted_stored_request = + builder.is_prebid && demand.stored_request && bidder_params.is_empty(); + let include = builder.routing == RoutingMode::AllEligible + || !bidder_params.is_empty() + || trusted_stored_request + || trusted_route; + if !include { + continue; + } + builder.slots.push(ProviderSlotInput { + slot: common_slot.clone(), + bidder_params, + prebid_zone: builder + .is_prebid + .then(|| demand.prebid_zone.clone()) + .flatten(), + trusted_stored_request, + }); + } + } + + let mut skipped_no_eligible_provider_ids = Vec::new(); + let inputs = builders + .into_iter() + .filter_map(|builder| { + if builder.slots.is_empty() { + skipped_no_eligible_provider_ids.push(builder.provider_id); + return None; + } + Some(ProviderAuctionInput { + provider_id: builder.provider_id, + timeout_ms: builder.timeout_ms, + common_request: common_request.clone(), + slots: builder.slots, + }) + }) + .collect(); + + RoutedAuction { + inputs, + skipped_no_eligible_provider_ids, + diagnostics, + transport_headers, + attested_client_ip, + dnt, + } +} + +fn eligible_banner_slot(slot: &AdSlot) -> Option { + let formats = slot + .formats + .iter() + .filter(|format| { + format.media_type == MediaType::Banner + && i32::try_from(format.width).is_ok_and(|width| width > 0) + && i32::try_from(format.height).is_ok_and(|height| height > 0) + }) + .cloned() + .collect::>(); + if formats.is_empty() { + return None; + } + Some(AdSlot { + id: slot.id.clone(), + formats, + floor_price: slot.floor_price, + targeting: slot.targeting.clone(), + bidders: HashMap::new(), + }) +} + +fn normalize_slot_demand( + bidders: &HashMap, + diagnostics: &mut RoutingDiagnostics, +) -> NormalizedSlotDemand { + if bidders.is_empty() { + return NormalizedSlotDemand { + stored_request: true, + ..Default::default() + }; + } + + let mut demand = NormalizedSlotDemand::default(); + if let Some(envelope) = bidders.get(TRUSTED_SERVER_ENVELOPE) { + match normalize_envelope(envelope) { + Some(normalized) => demand = normalized, + None => diagnostics.record_malformed_envelope(), + } + } + + let mut direct = bidders + .iter() + .filter(|(key, _)| key.as_str() != TRUSTED_SERVER_ENVELOPE) + .collect::>(); + direct.sort_by_key(|(left, _)| *left); + for (raw_bidder, params) in direct { + let Ok(bidder) = raw_bidder.parse::() else { + diagnostics.record_malformed_direct_demand(); + continue; + }; + if bidder.as_str() == TRUSTED_SERVER_ENVELOPE || !is_usable_params(params) { + diagnostics.record_malformed_direct_demand(); + continue; + } + demand.bidder_params.insert(bidder, params.clone()); + } + demand +} + +fn normalize_envelope(envelope: &Value) -> Option { + let object = envelope.as_object()?; + if object + .keys() + .any(|key| !matches!(key.as_str(), BIDDER_PARAMS_FIELD | ZONE_FIELD)) + { + return None; + } + let prebid_zone = match object.get(ZONE_FIELD) { + None => None, + Some(Value::String(zone)) if zone.len() <= MAX_PREBID_ZONE_BYTES => Some(zone.clone()), + Some(_) => return None, + }; + let Some(raw_params) = object.get(BIDDER_PARAMS_FIELD) else { + return Some(NormalizedSlotDemand { + stored_request: true, + prebid_zone, + ..Default::default() + }); + }; + if raw_params.is_null() { + return Some(NormalizedSlotDemand { + stored_request: true, + prebid_zone, + ..Default::default() + }); + } + let params = raw_params.as_object()?; + if params.is_empty() { + return Some(NormalizedSlotDemand { + stored_request: true, + prebid_zone, + ..Default::default() + }); + } + if params.len() > MAX_BIDDER_ENTRIES { + return None; + } + + let mut bidder_params = BTreeMap::new(); + for (raw_bidder, value) in params { + let bidder = raw_bidder.parse::().ok()?; + if bidder.as_str() == TRUSTED_SERVER_ENVELOPE || !value.is_object() { + return None; + } + bidder_params.insert(bidder, value.clone()); + } + Some(NormalizedSlotDemand { + bidder_params, + stored_request: false, + prebid_zone, + }) +} + +fn is_usable_params(value: &Value) -> bool { + value.as_object().is_some_and(|object| !object.is_empty()) +} + +#[cfg(test)] +mod tests { + use std::str::FromStr as _; + + use super::*; + use crate::auction::plan::{ + AuctionPlanConfig, BidderRouteConfig, NotificationConfig, ProviderConfig, + }; + use crate::auction::types::{AdFormat, DeviceInfo, PublisherInfo, SiteInfo, UserInfo}; + use http::HeaderName; + use serde_json::{Map, json}; + + fn provider(profile: &str, routing: RoutingMode) -> ProviderConfig { + ProviderConfig { + protocol: "openrtb-2.6".to_string(), + profile: profile.to_string(), + endpoint: format!("https://{profile}.example.test/openrtb"), + timeout_ms: None, + routing, + notifications: NotificationConfig::default(), + profile_config: if profile == "aps" { + json!({"account_id": "example-account"}) + } else { + json!({}) + }, + } + } + + fn plan() -> AuctionPlan { + AuctionPlan::compile(AuctionPlanConfig { + timeout_ms: 900, + providers: BTreeMap::from([ + ( + ProviderId::from_str("aps-primary").expect("should parse provider"), + provider("aps", RoutingMode::AllEligible), + ), + ( + ProviderId::from_str("pbs-a").expect("should parse provider"), + provider("prebid-server", RoutingMode::Explicit), + ), + ( + ProviderId::from_str("pbs-b").expect("should parse provider"), + provider("prebid-server", RoutingMode::Explicit), + ), + ( + ProviderId::from_str("standard-direct").expect("should parse provider"), + provider("standard", RoutingMode::Explicit), + ), + ]), + bidders: BTreeMap::from([ + ( + BidderId::from_str("alpha").expect("should parse bidder"), + BidderRouteConfig { + provider: ProviderId::from_str("pbs-a").expect("should parse provider"), + }, + ), + ( + BidderId::from_str("beta").expect("should parse bidder"), + BidderRouteConfig { + provider: ProviderId::from_str("standard-direct") + .expect("should parse provider"), + }, + ), + ]), + mediator: None, + request_signing: None, + }) + .expect("should compile plan") + } + + fn explicit_plan() -> AuctionPlan { + AuctionPlan::compile(AuctionPlanConfig { + timeout_ms: 900, + providers: BTreeMap::from([ + ( + ProviderId::from_str("aps-primary").expect("should parse provider"), + provider("aps", RoutingMode::Explicit), + ), + ( + ProviderId::from_str("pbs-a").expect("should parse provider"), + provider("prebid-server", RoutingMode::Explicit), + ), + ( + ProviderId::from_str("pbs-b").expect("should parse provider"), + provider("prebid-server", RoutingMode::Explicit), + ), + ( + ProviderId::from_str("standard-direct").expect("should parse provider"), + provider("standard", RoutingMode::Explicit), + ), + ]), + bidders: BTreeMap::new(), + mediator: None, + request_signing: None, + }) + .expect("should compile explicit plan") + } + + fn slot(bidders: HashMap) -> AdSlot { + AdSlot { + id: "slot-1".to_string(), + formats: vec![AdFormat { + media_type: MediaType::Banner, + width: 300, + height: 250, + }], + floor_price: Some(0.5), + targeting: HashMap::new(), + bidders, + } + } + + fn request(slots: Vec) -> AuctionRequest { + AuctionRequest { + id: "auction-1".to_string(), + slots, + publisher: PublisherInfo { + domain: "publisher.example.test".to_string(), + page_url: Some("https://publisher.example.test/article".to_string()), + }, + user: UserInfo { + id: None, + consent: None, + eids: None, + }, + device: Some(DeviceInfo { + user_agent: None, + ip: None, + geo: None, + }), + site: Some(SiteInfo { + domain: "publisher.example.test".to_string(), + page: "https://publisher.example.test/article".to_string(), + }), + context: HashMap::new(), + } + } + + fn inbound() -> Request { + Request::builder() + .uri("https://publisher.example.test/auction") + .body(EdgeBody::empty()) + .expect("should build request") + } + + fn envelope(bidder_params: Option) -> Value { + let mut value = Map::new(); + if let Some(params) = bidder_params { + value.insert(BIDDER_PARAMS_FIELD.to_string(), params); + } + Value::Object(value) + } + + fn input<'a>(routed: &'a RoutedAuction, provider_id: &str) -> &'a ProviderAuctionInput { + routed + .inputs() + .iter() + .find(|input| input.provider_id().as_str() == provider_id) + .expect("should find provider input") + } + + #[test] + fn missing_null_and_empty_envelope_params_fan_out_stored_routes() { + let cases = [ + ("missing", envelope(None)), + ("null", envelope(Some(Value::Null))), + ("empty", envelope(Some(json!({})))), + ]; + for (name, trusted_server) in cases { + let routed = route_auction( + request(vec![slot(HashMap::from([( + TRUSTED_SERVER_ENVELOPE.to_string(), + trusted_server, + )]))]), + &inbound(), + &plan(), + None, + ); + let ids = routed + .inputs() + .iter() + .map(|input| input.provider_id().as_str()) + .collect::>(); + assert_eq!( + ids, + vec!["aps-primary", "pbs-a", "pbs-b"], + "{name} should fan out to both PBS providers while APS remains all-eligible" + ); + assert!( + input(&routed, "pbs-a").slots()[0].has_trusted_stored_request(), + "{name} should create stored intent" + ); + assert!( + input(&routed, "pbs-b").slots()[0].has_trusted_stored_request(), + "{name} should create stored intent for every PBS provider" + ); + } + } + + #[test] + fn entirely_empty_bidder_map_is_trusted_stored_intent() { + let routed = route_auction( + request(vec![slot(HashMap::new())]), + &inbound(), + &plan(), + None, + ); + assert!( + input(&routed, "pbs-a").slots()[0].has_trusted_stored_request(), + "empty canonical demand should preserve stored-request behavior" + ); + assert!( + input(&routed, "pbs-b").slots()[0].has_trusted_stored_request(), + "empty canonical demand should fan out to same-profile PBS plans" + ); + } + + #[test] + fn malformed_envelopes_are_atomic_and_do_not_trigger_stored_routes() { + let too_many = Value::Object( + (0..=MAX_BIDDER_ENTRIES) + .map(|index| (format!("bidder-{index}"), json!({"placement": index}))) + .collect(), + ); + let cases = vec![ + ("nonobject envelope", json!("bad")), + ("nonobject params", envelope(Some(json!(true)))), + ("invalid key", envelope(Some(json!({" bad": {"x": 1}})))), + ( + "reserved key", + envelope(Some(json!({"trustedServer": {"x": 1}}))), + ), + ("nonobject value", envelope(Some(json!({"alpha": 1})))), + ( + "partial", + envelope(Some(json!({"alpha": {"x": 1}, "beta": null}))), + ), + ( + "unknown field", + json!({"bidderParams": {"alpha": {"x": 1}}, "endpoint": "https://bad.example"}), + ), + ("too many bidders", envelope(Some(too_many))), + ( + "oversized zone", + json!({"bidderParams": {}, "zone": "z".repeat(MAX_PREBID_ZONE_BYTES + 1)}), + ), + ("nonstring zone", json!({"bidderParams": {}, "zone": 1})), + ]; + for (name, malformed) in cases { + let bidders = HashMap::from([ + (TRUSTED_SERVER_ENVELOPE.to_string(), malformed), + ("beta".to_string(), json!({"placement": "direct"})), + ]); + let routed = route_auction(request(vec![slot(bidders)]), &inbound(), &plan(), None); + assert_eq!( + routed.diagnostics().malformed_envelope_count(), + 1, + "{name} should record one malformed envelope" + ); + assert!( + routed.inputs().iter().all(|provider| { + provider.provider_id().as_str() != "pbs-a" + && provider.provider_id().as_str() != "pbs-b" + }), + "{name} should not produce stored or inline PBS demand" + ); + assert_eq!( + input(&routed, "standard-direct").slots()[0] + .bidder_params() + .len(), + 1, + "{name} should preserve independent valid direct demand" + ); + assert!( + routed + .inputs() + .iter() + .any(|provider| provider.provider_id().as_str() == "aps-primary"), + "{name} should preserve independent all-eligible participation" + ); + } + } + + #[test] + fn envelope_preserves_empty_bidder_params_without_rejecting_valid_siblings() { + let normalized = normalize_envelope(&envelope(Some(json!({ + "alpha": {}, + "beta": {"placement": 42} + })))) + .expect("should preserve object-valued bidder params"); + + assert_eq!( + normalized + .bidder_params + .get(&BidderId::from_str("alpha").expect("should parse bidder")), + Some(&json!({})), + "should preserve empty params for profile overrides" + ); + assert_eq!( + normalized + .bidder_params + .get(&BidderId::from_str("beta").expect("should parse bidder")), + Some(&json!({"placement": 42})), + "should preserve valid sibling params" + ); + } + + #[test] + fn exact_envelope_bidder_entry_bound_is_accepted_and_next_entry_is_rejected() { + let accepted = Value::Object( + (0..MAX_BIDDER_ENTRIES) + .map(|index| (format!("bidder-{index}"), json!({"placement": index}))) + .collect(), + ); + let rejected = Value::Object( + (0..=MAX_BIDDER_ENTRIES) + .map(|index| (format!("bidder-{index}"), json!({"placement": index}))) + .collect(), + ); + assert!( + normalize_envelope(&envelope(Some(accepted))).is_some(), + "exactly 128 envelope bidder entries should be admitted" + ); + assert!( + normalize_envelope(&envelope(Some(rejected))).is_none(), + "129 envelope bidder entries should be rejected" + ); + } + + #[test] + fn unknown_bidder_is_counted_without_fallback() { + let routed = route_auction( + request(vec![slot(HashMap::from([( + TRUSTED_SERVER_ENVELOPE.to_string(), + envelope(Some(json!({"unknown": {"placement": 1}}))), + )]))]), + &inbound(), + &plan(), + None, + ); + assert_eq!( + routed.diagnostics().unroutable_bidder_count(), + 1, + "unknown bidder should increment bounded diagnostics" + ); + assert_eq!( + routed.inputs().len(), + 1, + "only all-eligible APS should remain" + ); + assert_eq!( + routed.inputs()[0].provider_id().as_str(), + "aps-primary", + "unknown demand should not cause PBS fallback" + ); + } + + #[test] + fn direct_usable_params_win_collision_and_unusable_direct_does_not_overwrite() { + let cases = [ + ("usable direct", json!({"source": "direct"}), "direct", 0), + ("empty direct", json!({}), "envelope", 1), + ("null direct", Value::Null, "envelope", 1), + ]; + for (name, direct, expected_source, malformed_count) in cases { + let routed = route_auction( + request(vec![slot(HashMap::from([ + ( + TRUSTED_SERVER_ENVELOPE.to_string(), + envelope(Some(json!({"alpha": {"source": "envelope"}}))), + ), + ("alpha".to_string(), direct), + ]))]), + &inbound(), + &plan(), + None, + ); + assert_eq!( + input(&routed, "pbs-a").slots()[0].bidder_params() + [&BidderId::from_str("alpha").expect("should parse bidder")]["source"], + expected_source, + "{name} should follow deterministic collision semantics" + ); + assert_eq!( + routed.diagnostics().malformed_direct_demand_count(), + malformed_count, + "{name} should record only unusable direct demand" + ); + } + } + + #[test] + fn hash_map_insertion_order_does_not_change_routing() { + let entries = [ + ("alpha".to_string(), json!({"a": 1})), + ("unknown".to_string(), json!({"u": 1})), + ( + TRUSTED_SERVER_ENVELOPE.to_string(), + envelope(Some(json!({"alpha": {"a": 0}}))), + ), + ]; + let forward = HashMap::from(entries.clone()); + let reverse = entries.into_iter().rev().collect::>(); + let first = route_auction(request(vec![slot(forward)]), &inbound(), &plan(), None); + let second = route_auction(request(vec![slot(reverse)]), &inbound(), &plan(), None); + let summarize = |routed: &RoutedAuction| { + routed + .inputs() + .iter() + .map(|provider| { + ( + provider.provider_id().as_str().to_string(), + provider.slots()[0] + .bidder_params() + .keys() + .map(|bidder| bidder.as_str().to_string()) + .collect::>(), + ) + }) + .collect::>() + }; + assert_eq!(summarize(&first), summarize(&second)); + assert_eq!(first.diagnostics(), second.diagnostics()); + } + + #[test] + fn mixed_routing_filters_params_per_provider_and_inline_wins_over_stored() { + let routed = route_auction( + request(vec![slot(HashMap::from([ + ( + TRUSTED_SERVER_ENVELOPE.to_string(), + json!({"zone": "home", "bidderParams": null}), + ), + ("alpha".to_string(), json!({"placement": "pbs"})), + ("beta".to_string(), json!({"placement": "direct"})), + ]))]), + &inbound(), + &plan(), + None, + ); + let ids = routed + .inputs() + .iter() + .map(|provider| provider.provider_id().as_str()) + .collect::>(); + assert_eq!( + ids, + vec!["aps-primary", "pbs-a", "pbs-b", "standard-direct"], + "inputs should follow deterministic provider-ID order" + ); + let aps = input(&routed, "aps-primary") + .slots() + .first() + .expect("should have slot"); + assert!( + aps.bidder_params().is_empty(), + "APS must receive no foreign params" + ); + assert_eq!(aps.prebid_zone(), None, "APS must receive no Prebid zone"); + let pbs_a = &input(&routed, "pbs-a").slots()[0]; + assert_eq!(pbs_a.bidder_params().len(), 1); + assert!( + !pbs_a.has_trusted_stored_request(), + "inline params should win for this PBS provider" + ); + assert_eq!(pbs_a.prebid_zone(), Some("home")); + let pbs_b = &input(&routed, "pbs-b").slots()[0]; + assert!(pbs_b.bidder_params().is_empty()); + assert!(pbs_b.has_trusted_stored_request()); + let direct = &input(&routed, "standard-direct").slots()[0]; + assert_eq!(direct.bidder_params().len(), 1); + assert!(direct.prebid_zone().is_none()); + for provider in routed.inputs() { + assert!(provider.common_request().slots.is_empty()); + assert!(provider.slots()[0].slot().bidders.is_empty()); + } + } + + #[test] + fn nonbanner_and_zero_sized_formats_are_removed_and_empty_slots_are_omitted() { + let mut mixed = slot(HashMap::new()); + mixed.formats = vec![ + AdFormat { + media_type: MediaType::Video, + width: 640, + height: 360, + }, + AdFormat { + media_type: MediaType::Banner, + width: 0, + height: 250, + }, + AdFormat { + media_type: MediaType::Banner, + width: u32::MAX, + height: 250, + }, + AdFormat { + media_type: MediaType::Banner, + width: 300, + height: 250, + }, + ]; + let mut invalid = slot(HashMap::new()); + invalid.id = "invalid".to_string(); + invalid.formats = vec![ + AdFormat { + media_type: MediaType::Native, + width: 1, + height: 1, + }, + AdFormat { + media_type: MediaType::Banner, + width: 300, + height: 0, + }, + ]; + let routed = route_auction(request(vec![mixed, invalid]), &inbound(), &plan(), None); + assert_eq!( + routed.inputs().len(), + 3, + "APS and two PBS providers should receive the eligible slot" + ); + for provider in routed.inputs() { + assert_eq!(provider.slots().len(), 1); + assert_eq!(provider.slots()[0].slot().id, "slot-1"); + assert_eq!(provider.slots()[0].slot().formats.len(), 1); + assert_eq!(provider.slots()[0].slot().formats[0].width, 300); + } + let none = route_auction( + request(vec![slot_with_formats(vec![AdFormat { + media_type: MediaType::Video, + width: 640, + height: 360, + }])]), + &inbound(), + &plan(), + None, + ); + assert!( + none.inputs().is_empty(), + "no eligible slots should omit every provider input" + ); + assert_eq!( + none.skipped_no_eligible_provider_ids() + .iter() + .map(ProviderId::as_str) + .collect::>(), + vec!["aps-primary", "pbs-a", "pbs-b", "standard-direct"], + "no-banner auction should retain every provider's deterministic skip outcome" + ); + } + + #[test] + fn explicit_no_demand_retains_deterministic_skip_outcomes() { + let routed = route_auction( + request(vec![slot(HashMap::from([( + "unknown".to_string(), + json!({"placement": "none"}), + )]))]), + &inbound(), + &plan(), + None, + ); + assert_eq!( + routed + .skipped_no_eligible_provider_ids() + .iter() + .map(ProviderId::as_str) + .collect::>(), + vec!["pbs-a", "pbs-b", "standard-direct"], + "explicit providers with no routed demand should be retained as skipped" + ); + } + + #[test] + fn trusted_routes_admit_explicit_aps_and_standard_and_ignore_unknown_provider() { + let trusted_routes = TrustedProviderRoutes::new(vec![vec![ + ProviderId::from_str("aps-primary").expect("should parse provider"), + ProviderId::from_str("standard-direct").expect("should parse provider"), + ProviderId::from_str("unknown-provider").expect("should parse provider"), + ]]); + let routed = route_auction_with_trusted_routes( + request(vec![slot(HashMap::from([( + "unknown".to_string(), + json!({"placement": "none"}), + )]))]), + &inbound(), + &explicit_plan(), + None, + &trusted_routes, + ); + assert_eq!( + routed + .inputs() + .iter() + .map(|input| input.provider_id().as_str()) + .collect::>(), + vec!["aps-primary", "standard-direct"], + "only known server-owned provider routes should admit explicit providers" + ); + assert_eq!( + routed.diagnostics().unroutable_trusted_provider_count(), + 1, + "unknown trusted provider should be ignored and counted" + ); + assert_eq!( + routed + .skipped_no_eligible_provider_ids() + .iter() + .map(ProviderId::as_str) + .collect::>(), + vec!["pbs-a", "pbs-b"], + "unrouted explicit providers should retain skip outcomes" + ); + } + + fn slot_with_formats(formats: Vec) -> AdSlot { + let mut value = slot(HashMap::new()); + value.formats = formats; + value + } + + #[test] + fn snapshots_first_headers_retains_raw_bytes_and_ignores_inbound_xff() { + let mut inbound = inbound(); + inbound + .headers_mut() + .append(header::COOKIE, HeaderValue::from_static("first=1")); + inbound + .headers_mut() + .append(header::COOKIE, HeaderValue::from_static("second=2")); + inbound.headers_mut().append( + header::USER_AGENT, + HeaderValue::from_bytes(b"agent-\x80").expect("should accept raw header"), + ); + inbound.headers_mut().append( + header::REFERER, + HeaderValue::from_static("https://publisher.example.test/article"), + ); + inbound + .headers_mut() + .append(header::ACCEPT_LANGUAGE, HeaderValue::from_static("en-US")); + inbound.headers_mut().append( + HeaderName::from_static("x-forwarded-for"), + HeaderValue::from_static("203.0.113.250"), + ); + inbound.headers_mut().append( + HeaderName::from_static("dnt"), + HeaderValue::from_static(" 1 "), + ); + let attested = IpAddr::from_str("192.0.2.10").expect("should parse IP"); + let routed = route_auction( + request(vec![slot(HashMap::new())]), + &inbound, + &plan(), + Some(attested), + ); + let headers = routed.prebid_transport_headers(); + assert_eq!(headers.cookie(), Some(&HeaderValue::from_static("first=1"))); + assert_eq!( + headers.user_agent().expect("should retain UA").as_bytes(), + b"agent-\x80" + ); + assert_eq!( + headers.referer(), + Some(&HeaderValue::from_static( + "https://publisher.example.test/article" + )) + ); + assert_eq!( + headers.accept_language(), + Some(&HeaderValue::from_static("en-US")) + ); + assert_eq!(routed.attested_client_ip(), Some(attested)); + assert_eq!(routed.dnt(), Some(true)); + for provider in routed.inputs() { + let expected_timeout = match provider.provider_id().as_str() { + "aps-primary" => 800, + id if id.starts_with("pbs-") => 1000, + _ => 900, + }; + assert_eq!(provider.timeout_ms(), expected_timeout); + } + } +} diff --git a/crates/trusted-server-core/src/auction/telemetry.rs b/crates/trusted-server-core/src/auction/telemetry.rs index b3e049eaf..9b639025a 100644 --- a/crates/trusted-server-core/src/auction/telemetry.rs +++ b/crates/trusted-server-core/src/auction/telemetry.rs @@ -1,7 +1,7 @@ //! Auction telemetry row construction and sink abstraction. //! -//! Core owns the privacy-preserving auction observation model and pure row -//! builder. Platform adapters provide the concrete sink implementation. +//! Core owns the auction observation model and pure row builder. Platform +//! adapters provide the concrete sink implementation. use std::collections::HashSet; use std::time::Instant; @@ -19,6 +19,9 @@ use crate::platform::RuntimeServices; const MAX_PAGE_PATH_BYTES: usize = 256; const DYNAMIC_SEGMENT_REPLACEMENT: &str = ":id"; +#[cfg(test)] +const TEST_USER_AGENT: &str = + "FictionalBrowser/123.4 (FictionalOS 10.2; FictionalDevice) ExampleRenderer/567.8"; /// Source path that initiated an auction candidate. #[derive(Debug, Clone, Copy, Eq, PartialEq)] @@ -91,7 +94,7 @@ impl AbandonedProviderCall { } } -/// Privacy-preserving context shared by all rows in one auction observation. +/// Context shared by all rows in one auction observation. #[derive(Debug, Clone)] pub struct AuctionObservationContext { /// Fresh telemetry UUID, independent of EC and internal auction IDs. @@ -110,6 +113,8 @@ pub struct AuctionObservationContext { pub is_mobile: u8, /// `0` = bot, `1` = browser, `2` = unknown. pub is_known_browser: u8, + /// Complete User-Agent supplied by the auction request, when present. + pub user_agent: Option, /// Whether GDPR applies. pub gdpr_applies: bool, /// Whether any consent signal was present. @@ -134,11 +139,16 @@ impl AuctionObservationContext { .and_then(|page_url| url::Url::parse(page_url).ok()) .map(|url| url.path().to_owned()) .unwrap_or_else(|| "/".to_owned()); + let user_agent = request + .device + .as_ref() + .and_then(|device| device.user_agent.as_deref()); Self::from_parts( auction_source, &request.publisher.domain, &raw_path, request.slots.len(), + user_agent, ec_context, ) } @@ -150,6 +160,7 @@ impl AuctionObservationContext { publisher_domain: &str, raw_page_path: &str, slot_count: usize, + user_agent: Option<&str>, ec_context: &EcContext, ) -> Self { let device = ec_context.device_signals(); @@ -172,6 +183,7 @@ impl AuctionObservationContext { Some(false) => 0, None => 2, }, + user_agent: user_agent.map(str::to_owned), gdpr_applies: consent.gdpr_applies, consent_present: !consent.is_empty(), slot_count, @@ -196,6 +208,7 @@ impl AuctionObservationContext { region: Some("CA".to_owned()), is_mobile: 0, is_known_browser: 1, + user_agent: Some(TEST_USER_AGENT.to_owned()), gdpr_applies: false, consent_present: false, slot_count, @@ -282,6 +295,8 @@ pub struct AuctionEventRow { pub is_mobile: u8, /// `0` = bot, `1` = browser, `2` = unknown. pub is_known_browser: u8, + /// Complete User-Agent supplied by the auction request, when present. + pub user_agent: Option, /// `0` or `1`. pub gdpr_applies: u8, /// `0` or `1`. @@ -341,6 +356,7 @@ impl AuctionEventRow { region: observation.region.clone(), is_mobile: observation.is_mobile, is_known_browser: observation.is_known_browser, + user_agent: observation.user_agent.clone(), gdpr_applies: u8::from(observation.gdpr_applies), consent_present: u8::from(observation.consent_present), terminal_status: None, @@ -770,7 +786,11 @@ fn bid_row( row.slot_w = Some(u16::try_from(bid.width).unwrap_or(u16::MAX)); row.slot_h = Some(u16::try_from(bid.height).unwrap_or(u16::MAX)); row.media_type = media_type_for_slot(request, &bid.slot_id).map(str::to_owned); - row.seat = Some(bid.bidder.clone()); + row.seat = Some( + bid.returned_seat + .clone() + .unwrap_or_else(|| bid.bidder.clone()), + ); row.price_cpm = price; row.currency = Some(bid.currency.clone()); row.is_win = Some(is_win); @@ -933,7 +953,7 @@ mod tests { use serde_json::json; - use crate::auction::types::{AdFormat, AdSlot, PublisherInfo, UserInfo}; + use crate::auction::types::{AdFormat, AdSlot, DeviceInfo, PublisherInfo, UserInfo}; use super::*; @@ -974,6 +994,7 @@ mod tests { creative: None, adomain: Some(vec!["advertiser.example".to_owned()]), bidder: bidder.to_owned(), + returned_seat: None, width: 300, height: 250, nurl: None, @@ -1005,6 +1026,52 @@ mod tests { ); } + #[test] + fn observation_sources_preserve_complete_user_agent() { + let ec_context = + EcContext::new_for_test(None, crate::consent::types::ConsentContext::default()); + let mut request = test_request("request-id"); + request.device = Some(DeviceInfo { + user_agent: Some(TEST_USER_AGENT.to_owned()), + ip: None, + geo: None, + }); + + let from_request = AuctionObservationContext::from_auction_request( + AuctionSource::AuctionApi, + &request, + &ec_context, + ); + let from_parts = AuctionObservationContext::from_parts( + AuctionSource::InitialNavigation, + "test-publisher.example", + "/article", + 1, + Some(TEST_USER_AGENT), + &ec_context, + ); + let without_user_agent = AuctionObservationContext::from_auction_request( + AuctionSource::AuctionApi, + &test_request("request-without-user-agent"), + &ec_context, + ); + + assert_eq!( + from_request.user_agent.as_deref(), + Some(TEST_USER_AGENT), + "should preserve the complete user agent from an auction request" + ); + assert_eq!( + from_parts.user_agent.as_deref(), + Some(TEST_USER_AGENT), + "should preserve the complete user agent from publisher request parts" + ); + assert_eq!( + without_user_agent.user_agent, None, + "should omit a missing user agent" + ); + } + #[test] fn normalize_page_path_strips_query_and_redacts_dynamic_segments() { assert_eq!( @@ -1065,6 +1132,11 @@ mod tests { ); let rows = batch.rows(); + assert!( + rows.iter() + .all(|row| row.user_agent.as_deref() == Some(TEST_USER_AGENT)), + "should copy the complete user agent to every row kind" + ); assert_eq!( rows.iter() .filter(|row| row.event_kind == "summary") @@ -1100,6 +1172,112 @@ mod tests { ); } + #[test] + fn bid_rows_prefer_returned_seat_over_delivery_bidder() { + let request = test_request("ts-ec-derived-id"); + let mut aps_bid = bid("slot-1", "aps", Some("ad-1"), Some(1.25)); + aps_bid.returned_seat = Some("upstream-seat".to_string()); + let provider = AuctionResponse::success("aps-primary", vec![aps_bid.clone()], 12); + let result = OrchestrationResult { + provider_responses: vec![provider], + mediator_response: None, + winning_bids: HashMap::from([("slot-1".to_owned(), aps_bid.clone())]), + total_time_ms: 12, + metadata: HashMap::new(), + }; + let batch = build_auction_events( + AuctionObservationContext::for_test(AuctionSource::AuctionApi, "/auction", 1), + AuctionTerminalOutcome::Completed { + request: &request, + result: &result, + delivered_winner_slots: None, + }, + ); + + let provider_row = batch + .rows() + .iter() + .find(|row| row.event_kind == "provider_call") + .expect("should emit provider row"); + assert_eq!(provider_row.provider.as_deref(), Some("aps-primary")); + let bid_row = batch + .rows() + .iter() + .find(|row| row.event_kind == "bid") + .expect("should emit bid row"); + assert_eq!(bid_row.provider.as_deref(), Some("aps-primary")); + assert_eq!(bid_row.seat.as_deref(), Some("upstream-seat")); + + let mut fallback_bid = aps_bid; + fallback_bid.returned_seat = None; + let fallback = OrchestrationResult { + provider_responses: vec![AuctionResponse::success( + "aps-primary", + vec![fallback_bid.clone()], + 12, + )], + mediator_response: None, + winning_bids: HashMap::from([("slot-1".to_owned(), fallback_bid)]), + total_time_ms: 12, + metadata: HashMap::new(), + }; + let fallback_batch = build_auction_events( + AuctionObservationContext::for_test(AuctionSource::AuctionApi, "/auction", 1), + AuctionTerminalOutcome::Completed { + request: &request, + result: &fallback, + delivered_winner_slots: None, + }, + ); + assert_eq!( + fallback_batch + .rows() + .iter() + .find(|row| row.event_kind == "bid") + .and_then(|row| row.seat.as_deref()), + Some("aps") + ); + } + + #[test] + fn mediated_aps_telemetry_retains_provider_upstream_seat_and_delivery_identity() { + let request = test_request("ts-ec-derived-id"); + let mut aps_bid = bid("slot-1", "aps", Some("ad-1"), Some(1.25)); + aps_bid.returned_seat = Some("upstream-seat".to_string()); + let provider = AuctionResponse::success("aps-primary", vec![aps_bid.clone()], 12); + let mediator = AuctionResponse::success("adserver_mock", vec![aps_bid.clone()], 3); + let result = OrchestrationResult { + provider_responses: vec![provider], + mediator_response: Some(mediator), + winning_bids: HashMap::from([("slot-1".to_owned(), aps_bid.clone())]), + total_time_ms: 15, + metadata: HashMap::new(), + }; + let batch = build_auction_events( + AuctionObservationContext::for_test(AuctionSource::AuctionApi, "/auction", 1), + AuctionTerminalOutcome::Completed { + request: &request, + result: &result, + delivered_winner_slots: None, + }, + ); + + let provider_row = batch + .rows() + .iter() + .find(|row| row.event_kind == "provider_call") + .expect("should emit provider call"); + assert_eq!(provider_row.provider.as_deref(), Some("aps-primary")); + let bid_row = batch + .rows() + .iter() + .find(|row| row.event_kind == "bid") + .expect("should emit provider bid"); + assert_eq!(bid_row.provider.as_deref(), Some("aps-primary")); + assert_eq!(bid_row.seat.as_deref(), Some("upstream-seat")); + assert_eq!(aps_bid.bidder, "aps", "delivery identity remains distinct"); + } + #[test] fn completed_events_do_not_mark_dropped_winners_as_delivered() { let request = test_request("ts-ec-derived-id"); @@ -1249,6 +1427,10 @@ mod tests { .expect("should serialize ndjson"); assert!(body.ends_with('\n'), "should end each row with newline"); + assert!( + body.contains(TEST_USER_AGENT), + "should preserve the complete user agent in serialized rows" + ); for line in body.lines() { let parsed: serde_json::Value = serde_json::from_str(line).expect("should parse row"); assert_eq!(parsed["event_kind"], "summary"); diff --git a/crates/trusted-server-core/src/auction/test_support.rs b/crates/trusted-server-core/src/auction/test_support.rs index e4b953e05..a83f8899a 100644 --- a/crates/trusted-server-core/src/auction/test_support.rs +++ b/crates/trusted-server-core/src/auction/test_support.rs @@ -1,9 +1,17 @@ +use std::collections::HashMap; use std::sync::LazyLock; use edgezero_core::body::Body as EdgeBody; use http::Request; +use serde_json::json; use super::AuctionContext; +use crate::auction::types::{ + AdFormat, AdSlot, AuctionRequest, DeviceInfo, MediaType, PublisherInfo, UserInfo, +}; +use crate::consent::ConsentContext; +use crate::geo::GeoInfo; +use crate::openrtb::{Eid, Uid}; use crate::platform::{RuntimeServices, test_support::noop_services}; use crate::settings::Settings; @@ -19,7 +27,95 @@ pub(crate) fn create_test_auction_context<'a>( settings, request, timeout_ms, + transport_timeout_ms: timeout_ms, provider_responses: None, services, } } + +/// Build canonical request facts shared by the PBS and APS Stage 1 wire goldens. +/// +/// The supported and unsupported formats deliberately exercise each profile's +/// existing filtering and field-ownership policy. `trustedServer` bidder +/// parameters are included to pin that PBS consumes them while APS ignores +/// them. +pub(crate) fn canonical_parity_auction_request() -> AuctionRequest { + AuctionRequest { + id: "fictional-auction".to_string(), + slots: vec![AdSlot { + id: "fictional-slot".to_string(), + formats: vec![ + AdFormat { + media_type: MediaType::Banner, + width: 300, + height: 250, + }, + AdFormat { + media_type: MediaType::Video, + width: 640, + height: 480, + }, + AdFormat { + media_type: MediaType::Banner, + width: u32::MAX, + height: 90, + }, + AdFormat { + media_type: MediaType::Banner, + width: 728, + height: 90, + }, + ], + floor_price: Some(1.0), + targeting: HashMap::new(), + bidders: HashMap::from([( + "trustedServer".to_string(), + json!({ + "bidderParams": { + "exampleBidder": { "placement": "fictional-placement" } + } + }), + )]), + }], + publisher: PublisherInfo { + domain: "publisher.example".to_string(), + page_url: Some("https://publisher.example/article".to_string()), + }, + user: UserInfo { + id: Some("fictional-user".to_string()), + consent: Some(ConsentContext { + gdpr_applies: true, + raw_tc_string: Some("fictional-tcf".to_string()), + raw_us_privacy: Some("1YNN".to_string()), + raw_gpp_string: Some("fictional-gpp".to_string()), + gpp_section_ids: Some(vec![2, 6]), + raw_ac_string: Some("fictional-ac".to_string()), + ..Default::default() + }), + eids: Some(vec![Eid { + source: "identity.example".to_string(), + uids: vec![Uid { + id: "fictional-uid".to_string(), + atype: Some(1), + ext: None, + }], + }]), + }, + device: Some(DeviceInfo { + user_agent: Some("Fictional Browser".to_string()), + ip: Some("192.0.2.10".to_string()), + geo: Some(GeoInfo { + city: "Example City".to_string(), + country: "US".to_string(), + continent: "NA".to_string(), + latitude: 12.34, + longitude: 56.78, + metro_code: 501, + region: Some("CA".to_string()), + asn: None, + }), + }), + site: None, + context: HashMap::new(), + } +} diff --git a/crates/trusted-server-core/src/auction/types.rs b/crates/trusted-server-core/src/auction/types.rs index f61334787..406915706 100644 --- a/crates/trusted-server-core/src/auction/types.rs +++ b/crates/trusted-server-core/src/auction/types.rs @@ -146,7 +146,14 @@ pub struct SiteInfo { pub struct AuctionContext<'a> { pub settings: &'a Settings, pub request: &'a Request, + /// Exact logical provider budget used by auction policy and payloads. pub timeout_ms: u32, + /// Canonical backend transport timeout used for provider registration. + /// + /// This can be lower than `timeout_ms` on runtimes whose backend names + /// encode timers. Providers that register a backend should use this value + /// for transport timers while retaining `timeout_ms` for logical policy. + pub transport_timeout_ms: u32, /// Provider responses from the bidding phase, used by mediators. /// This is `None` for regular bidders and `Some` when calling a mediator. pub provider_responses: Option<&'a [AuctionResponse]>, @@ -243,8 +250,11 @@ pub struct Bid { pub creative: Option, /// Advertiser domain pub adomain: Option>, - /// Bidder/seat identifier + /// Browser-facing delivery bidder code. pub bidder: String, + /// Exact valid upstream `seatbid.seat`, independent of delivery identity. + #[serde(skip)] + pub returned_seat: Option, /// Width of creative pub width: u32, /// Height of creative @@ -409,6 +419,7 @@ mod tests { creative: None, adomain: None, bidder: bidder.to_owned(), + returned_seat: None, width: 300, height: 250, nurl: None, @@ -532,6 +543,20 @@ mod tests { ); } + #[test] + fn returned_seat_is_internal_and_not_serialized() { + let mut bid = make_bid("aps"); + bid.returned_seat = Some("upstream-seat".to_string()); + + let serialized = serde_json::to_value(&bid).expect("should serialize bid"); + assert!( + serialized.get("returned_seat").is_none(), + "returned seat must not change client/debug wire shapes" + ); + let decoded: Bid = serde_json::from_value(serialized).expect("should deserialize bid"); + assert!(decoded.returned_seat.is_none()); + } + #[test] fn bid_with_cache_fields_round_trips_through_json() { let bid = Bid { @@ -541,6 +566,7 @@ mod tests { creative: None, adomain: None, bidder: "thetradedesk".to_string(), + returned_seat: None, width: 300, height: 250, nurl: None, @@ -647,6 +673,7 @@ mod tests { creative: None, adomain: None, bidder: "kargo".to_string(), + returned_seat: None, width: 300, height: 250, nurl: None, diff --git a/crates/trusted-server-core/src/auction_config_types.rs b/crates/trusted-server-core/src/auction_config_types.rs index af52b7ad2..caee91cf2 100644 --- a/crates/trusted-server-core/src/auction_config_types.rs +++ b/crates/trusted-server-core/src/auction_config_types.rs @@ -1,9 +1,13 @@ -//! Auction configuration types (separated to avoid circular deps in build.rs). +//! Auction configuration types shared by settings and auction planning. use serde::{Deserialize, Serialize}; -use std::collections::HashSet; +use std::collections::{BTreeMap, HashSet}; use validator::Validate; +pub use crate::auction::plan::{ + BidderId, BidderRouteConfig, NotificationConfig, ProviderConfig, ProviderId, RoutingMode, +}; + /// Auction orchestration configuration. #[derive(Debug, Clone, Deserialize, Serialize, Validate)] #[serde(deny_unknown_fields)] @@ -31,22 +35,25 @@ pub struct AuctionConfig { /// Rewrite winning-bid creative HTML to first-party endpoints (applied /// after sanitization when [`Self::sanitize_creatives`] is enabled). /// - /// The default must stay omitted from serialized config blobs: older - /// [`AuctionConfig`] schemas reject unknown fields during binary rollback. - /// An explicit `false` remains serialized and requires restoring a - /// compatible blob before rolling back. + /// The default stays omitted from serialized config blobs to avoid adding + /// this field when it has no effect. Any rollback across schema versions + /// still requires restoring the matching old-schema blob with the old + /// binary. #[serde( default = "default_rewrite_creatives", skip_serializing_if = "is_default_rewrite_creatives" )] pub rewrite_creatives: bool, - /// Provider names that participate in bidding - /// Simply list the provider names (e.g., ["prebid", "aps"]) - #[serde(default, deserialize_with = "crate::settings::vec_from_seq_or_map")] - pub providers: Vec, + /// Operator-defined bidder-provider instances, keyed by provider ID. + #[serde(default)] + pub providers: BTreeMap, + + /// Client-visible bidder routes, keyed by bidder code. + #[serde(default)] + pub bidders: BTreeMap, - /// Optional mediator provider name (e.g., "gam") + /// Optional separately registered mediator provider name. /// When set, runs parallel mediation strategy (bidders in parallel, then mediator decides) /// When omitted, runs parallel only strategy (bidders in parallel, highest CPM wins) pub mediator: Option, @@ -74,7 +81,8 @@ impl Default for AuctionConfig { enabled: false, sanitize_creatives: default_sanitize_creatives(), rewrite_creatives: default_rewrite_creatives(), - providers: Vec::new(), + providers: BTreeMap::new(), + bidders: BTreeMap::new(), mediator: None, timeout_ms: default_timeout(), creative_store: default_creative_store(), @@ -95,7 +103,7 @@ fn default_rewrite_creatives() -> bool { true } -// This predicate preserves rollback compatibility by omitting the default field. +// Omit the default field when it has no effect on the serialized config. fn is_default_rewrite_creatives(value: &bool) -> bool { *value == default_rewrite_creatives() } @@ -112,15 +120,27 @@ fn default_allowed_context_keys() -> HashSet { HashSet::new() } -#[allow( - dead_code, - reason = "methods are used by the runtime crate but not by build.rs path inclusion" -)] impl AuctionConfig { - /// Get all provider names. - #[must_use] - pub fn provider_names(&self) -> &[String] { - &self.providers + #[cfg(test)] + pub(crate) fn legacy_provider_map(names: &[&str]) -> BTreeMap { + names + .iter() + .map(|name| { + let id = ProviderId::unchecked_for_legacy_test(name); + ( + id, + ProviderConfig { + protocol: "openrtb-2.6".to_string(), + profile: "standard".to_string(), + endpoint: format!("https://{name}.example/openrtb2/auction"), + timeout_ms: None, + routing: RoutingMode::AllEligible, + notifications: NotificationConfig::default(), + profile_config: serde_json::json!({}), + }, + ) + }) + .collect() } /// Check if this config has a mediator configured. @@ -222,4 +242,37 @@ mod tests { "should preserve an explicit sanitize opt-in" ); } + + #[test] + fn provider_list_shape_is_rejected() { + let error = serde_json::from_value::(serde_json::json!({ + "providers": ["prebid"] + })) + .expect_err("should reject the removed provider-list schema"); + + assert!( + error.to_string().contains("map") || error.to_string().contains("object"), + "should require map-shaped providers: {error}" + ); + } + + #[test] + fn map_schema_round_trips_provider_and_bidder_routes() { + let config: AuctionConfig = serde_json::from_value(serde_json::json!({ + "providers": { + "pbs-main": { + "protocol": "openrtb-2.6", + "profile": "prebid-server", + "endpoint": "https://prebid.example/openrtb2/auction" + } + }, + "bidders": { + "example-bidder": { "provider": "pbs-main" } + } + })) + .expect("should parse map-shaped auction config"); + + assert_eq!(config.providers.len(), 1); + assert_eq!(config.bidders.len(), 1); + } } diff --git a/crates/trusted-server-core/src/config.rs b/crates/trusted-server-core/src/config.rs index ad4f66460..b32f19c3f 100644 --- a/crates/trusted-server-core/src/config.rs +++ b/crates/trusted-server-core/src/config.rs @@ -7,8 +7,8 @@ //! `EdgeZero`'s typed config push path. use std::borrow::Cow; -use std::collections::HashSet; +use edgezero_core::app_config::{SecretField, SecretKind, SecretPathSegment}; use error_stack::Report; use serde::{Deserialize, Deserializer, Serialize, Serializer}; use validator::{Validate, ValidationError, ValidationErrors}; @@ -16,13 +16,23 @@ use validator::{Validate, ValidationError, ValidationErrors}; use crate::ec::registry::PartnerRegistry; use crate::error::TrustedServerError; use crate::integrations::{ - adserver_mock::AdServerMockConfig, aps::ApsConfig, datadome::DataDomeConfig, - didomi::DidomiIntegrationConfig, google_tag_manager::GoogleTagManagerConfig, gpt::GptConfig, - gpt_diagnostics::GptDiagnosticsConfig, lockr::LockrConfig, nextjs::NextJsIntegrationConfig, - osano::OsanoConfig, permutive::PermutiveConfig, prebid, sourcepoint::SourcepointConfig, + adserver_mock::AdServerMockConfig, + aps::ApsConfig, + datadome::DataDomeConfig, + didomi::DidomiIntegrationConfig, + google_tag_manager::GoogleTagManagerConfig, + gpt::GptConfig, + gpt_diagnostics::GptDiagnosticsConfig, + js_asset_proxy::{JS_ASSET_PROXY_INTEGRATION_ID, JsAssetProxyConfig}, + lockr::LockrConfig, + nextjs::NextJsIntegrationConfig, + osano::OsanoConfig, + permutive::PermutiveConfig, + prebid, + sourcepoint::SourcepointConfig, testlight::TestlightConfig, }; -use crate::settings::{IntegrationConfig, Settings}; +use crate::settings::{AssetOriginAuth, IntegrationConfig, Settings}; const DEPLOY_VALIDATION_FIELD: &str = "trusted_server"; #[cfg(test)] @@ -41,6 +51,7 @@ const DEPLOY_VALIDATED_INTEGRATION_IDS: &[&str] = &[ "datadome", "gpt", "gpt_diagnostics", + JS_ASSET_PROXY_INTEGRATION_ID, ]; /// Typed app-config root used by the `ts` CLI. @@ -54,15 +65,20 @@ pub struct TrustedServerAppConfig { } impl TrustedServerAppConfig { - /// Creates a validated app-config wrapper from [`Settings`]. + /// Creates a push-valid app-config wrapper from [`Settings`]. /// /// # Errors /// - /// Returns [`TrustedServerError::Configuration`] when deploy validation + /// Returns [`TrustedServerError::Configuration`] when push-safe validation /// fails. pub fn new(settings: Settings) -> Result> { - validate_settings_for_deploy(&settings)?; - Ok(Self { settings }) + let app_config = Self { settings }; + edgezero_core::app_config::validate_excluding_secrets(&app_config).map_err(|errors| { + Report::new(TrustedServerError::Configuration { + message: format!("Configuration validation failed: {errors}"), + }) + })?; + Ok(app_config) } /// Consumes the wrapper and returns the inner [`Settings`]. @@ -92,64 +108,201 @@ impl<'de> Deserialize<'de> for TrustedServerAppConfig { where D: Deserializer<'de>, { - let settings = Settings::deserialize(deserializer)?; - let settings = Settings::finalize_deserialized(settings, "Configuration") - .map_err(serde::de::Error::custom)?; + let mut settings = Settings::deserialize(deserializer)?; + settings.normalize_deserialized(); Ok(Self { settings }) } } impl Validate for TrustedServerAppConfig { fn validate(&self) -> Result<(), ValidationErrors> { - validate_settings_for_deploy(&self.settings) - .map_err(|report| report_to_validation_errors(&report)) + let mut errors = self.settings.validate().err().unwrap_or_default(); + if let Err(report) = validate_settings_for_deploy(&self.settings) { + errors.add( + DEPLOY_VALIDATION_FIELD, + report_to_validation_error(&report, "trusted_server_deploy_validation"), + ); + } + if errors.errors().is_empty() { + Ok(()) + } else { + Err(errors) + } } } impl edgezero_core::app_config::AppConfigMeta for TrustedServerAppConfig { - // Phase 1 intentionally preserves the existing inline-settings model: - // `ts config push` publishes the validated Trusted Server config as one - // app-config blob. Migrating app-level secrets to `EdgeZero` secret-store - // references needs the secret-field paths spelled out here (this - // hand-written impl does not inherit the derive's nested/array paths) - // plus operator migration work tracked separately. - fn secret_fields() -> Vec { - Vec::new() + fn secret_fields() -> Vec { + let field = |path: Vec, optional| SecretField { + kind: SecretKind::KeyInDefault, + optional, + path, + }; + let object = |name: &'static str| SecretPathSegment::Field(Cow::Borrowed(name)); + let optional_object = + |name: &'static str| SecretPathSegment::OptionalField(Cow::Borrowed(name)); + + vec![ + field(vec![object("publisher"), object("proxy_secret")], false), + field(vec![object("ec"), object("passphrase")], false), + field( + vec![ + object("ec"), + optional_object("partners"), + SecretPathSegment::ArrayEach, + object("api_token"), + ], + true, + ), + field( + vec![ + object("ec"), + optional_object("partners"), + SecretPathSegment::ArrayEach, + object("ts_pull_token"), + ], + true, + ), + field( + vec![ + object("handlers"), + SecretPathSegment::ArrayEach, + object("password"), + ], + false, + ), + field( + vec![optional_object("tinybird"), object("auction_token_secret")], + true, + ), + field( + vec![optional_object("tinybird"), object("access_token_secret")], + true, + ), + field( + vec![ + optional_object("integrations"), + optional_object("datadome"), + object("server_side_key_secret_name"), + ], + true, + ), + field( + vec![ + optional_object("integrations"), + optional_object("datadome"), + optional_object("protection_test_bypass"), + object("credential_secret_name"), + ], + true, + ), + field( + vec![ + optional_object("proxy"), + optional_object("asset_routes"), + SecretPathSegment::ArrayEach, + optional_object("auth"), + object("access_key_id"), + ], + true, + ), + field( + vec![ + optional_object("proxy"), + optional_object("asset_routes"), + SecretPathSegment::ArrayEach, + optional_object("auth"), + object("secret_access_key"), + ], + true, + ), + field( + vec![ + optional_object("proxy"), + optional_object("asset_routes"), + SecretPathSegment::ArrayEach, + optional_object("auth"), + object("session_token"), + ], + true, + ), + ] } } -/// Runs Trusted Server deploy-time validation for pushed app config. +/// Runs Trusted Server push-time validation for app config. /// -/// This supplements [`Settings`] structural validation with checks that should -/// fail before an operator publishes a config blob: placeholder secrets, -/// enabled integration startup checks, auction provider references, and EC -/// partner registry construction. +/// Secret fields contain secret-store key names at this stage, so this function +/// deliberately excludes checks that require resolved values. The `EdgeZero` CLI +/// additionally calls [`edgezero_core::app_config::validate_excluding_secrets`] +/// to remove validators attached to those leaves. /// /// # Errors /// -/// Returns [`TrustedServerError`] when the config should not be deployed. +/// Returns [`TrustedServerError`] when non-secret configuration or a secret key +/// reference is invalid. pub fn validate_settings_for_deploy(settings: &Settings) -> Result<(), Report> { + validate_secret_key_references(settings)?; + validate_non_secret_deploy_placeholders(settings)?; + + let mut structural_settings = settings.clone(); + structural_settings.prepare_runtime()?; + structural_settings.validate_admin_coverage()?; + validate_js_asset_proxy_config(settings)?; + + let plan = crate::auction::compile_auction_plan(settings)?; + validate_enabled_integrations(settings, &plan, false)?; + PartnerRegistry::validate_config_for_deploy(&settings.ec.partners)?; + Ok(()) +} + +/// Runs Trusted Server runtime validation after secret references are resolved. +/// +/// # Errors +/// +/// Returns [`TrustedServerError`] when resolved secrets or runtime-only +/// configuration checks are invalid. +pub fn validate_settings_for_runtime( + settings: &Settings, +) -> Result<(), Report> { settings.reject_placeholder_secrets()?; - let enabled_auction_providers = validate_enabled_integrations(settings)?; - validate_auction_provider_names(settings, &enabled_auction_providers)?; + settings.validate_admin_handler_passwords()?; + validate_js_asset_proxy_config(settings)?; + let plan = crate::auction::compile_auction_plan(settings)?; + validate_enabled_integrations(settings, &plan, true)?; PartnerRegistry::from_config(&settings.ec.partners).map(|_| ())?; Ok(()) } +fn validate_js_asset_proxy_config(settings: &Settings) -> Result<(), Report> { + let Some(raw_config) = settings.integrations.get(JS_ASSET_PROXY_INTEGRATION_ID) else { + return Ok(()); + }; + + let config: JsAssetProxyConfig = serde_json::from_value(raw_config.clone()).map_err(|error| { + Report::new(TrustedServerError::Configuration { + message: format!( + "integration startup failed for `{JS_ASSET_PROXY_INTEGRATION_ID}`: configuration could not be parsed: {error}" + ), + }) + })?; + config.validate().map_err(|error| { + Report::new(TrustedServerError::Configuration { + message: format!( + "integration startup failed for `{JS_ASSET_PROXY_INTEGRATION_ID}`: {error}" + ), + }) + }) +} + fn validate_enabled_integrations( settings: &Settings, -) -> Result, Report> { - let mut enabled_auction_providers = HashSet::new(); - - if validate_prebid(settings)? { - enabled_auction_providers.insert("prebid"); - } - if validate_integration::(settings, "aps")? { - enabled_auction_providers.insert("aps"); - } - if validate_integration::(settings, "adserver_mock")? { - enabled_auction_providers.insert("adserver_mock"); - } + plan: &crate::auction::AuctionPlan, + resolved_secrets: bool, +) -> Result<(), Report> { + validate_prebid(settings, plan)?; + validate_integration::(settings, "aps")?; + validate_integration::(settings, "adserver_mock")?; validate_integration::(settings, "testlight")?; validate_integration::(settings, "nextjs")?; validate_integration::(settings, "permutive")?; @@ -159,16 +312,30 @@ fn validate_enabled_integrations( validate_integration::(settings, "osano")?; validate_integration::(settings, "google_tag_manager")?; if let Some(config) = settings.integration_config::("datadome")? { - crate::integrations::datadome::DataDomeIntegration::validate_config_for_startup(config)?; + if resolved_secrets { + crate::integrations::datadome::DataDomeIntegration::validate_config_for_startup( + config, + )?; + } else { + crate::integrations::datadome::DataDomeIntegration::validate_config_for_deploy(config)?; + } } validate_integration::(settings, "gpt")?; validate_integration::(settings, "gpt_diagnostics")?; - Ok(enabled_auction_providers) + Ok(()) } -fn validate_prebid(settings: &Settings) -> Result> { - prebid::validate_config_for_startup(settings).map(|config| config.is_some()) +fn validate_prebid( + settings: &Settings, + plan: &crate::auction::AuctionPlan, +) -> Result<(), Report> { + let Some(config) = settings.integration_config::("prebid")? + else { + return Ok(()); + }; + prebid::validate_browser_config_for_startup(&config, &settings.proxy.allowed_domains)?; + prebid::validate_browser_bidder_ownership(&config, plan) } fn validate_integration( @@ -183,45 +350,173 @@ where .map(|config| config.is_some()) } -fn validate_auction_provider_names( +fn validate_non_secret_deploy_placeholders( settings: &Settings, - enabled_auction_providers: &HashSet<&'static str>, ) -> Result<(), Report> { - if !settings.auction.enabled { + let mut insecure_fields = Vec::new(); + + if crate::settings::Publisher::is_placeholder_domain(&settings.publisher.domain) { + insecure_fields.push("publisher.domain"); + } + if crate::settings::Publisher::is_placeholder_cookie_domain(&settings.publisher.cookie_domain) { + insecure_fields.push("publisher.cookie_domain"); + } + if crate::settings::Publisher::is_placeholder_origin_url(&settings.publisher.origin_url) { + insecure_fields.push("publisher.origin_url"); + } + if let Some(request_signing) = &settings.request_signing { + if crate::settings::RequestSigning::is_unusable_store_id(&request_signing.config_store_id) { + insecure_fields.push("request_signing.config_store_id"); + } + if crate::settings::RequestSigning::is_unusable_store_id(&request_signing.secret_store_id) { + insecure_fields.push("request_signing.secret_store_id"); + } + } + + if insecure_fields.is_empty() { return Ok(()); } - for provider_name in settings - .auction - .providers - .iter() - .chain(settings.auction.mediator.iter()) - { - if !enabled_auction_providers.contains(provider_name.as_str()) { - return Err(Report::new(TrustedServerError::Configuration { - message: format!( - "auction provider `{provider_name}` is listed in [auction] but no enabled integration provides it" - ), - })); + Err(Report::new(TrustedServerError::InsecureDefault { + field: insecure_fields.join(", "), + })) +} + +fn validate_secret_key_references(settings: &Settings) -> Result<(), Report> { + validate_secret_key_reference( + "publisher.proxy_secret", + settings.publisher.proxy_secret.expose(), + )?; + validate_secret_key_reference("ec.passphrase", settings.ec.passphrase.expose())?; + + for (index, partner) in settings.ec.partners.iter().enumerate() { + if let Some(token) = &partner.api_token { + validate_secret_key_reference( + &format!("ec.partners[{index}].api_token"), + token.expose(), + )?; + } + if let Some(token) = &partner.ts_pull_token { + validate_secret_key_reference( + &format!("ec.partners[{index}].ts_pull_token"), + token.expose(), + )?; + } + } + + for (index, handler) in settings.handlers.iter().enumerate() { + validate_secret_key_reference( + &format!("handlers[{index}].password"), + handler.password.expose(), + )?; + } + + if settings.tinybird.enabled && settings.tinybird.auction_enabled { + let token = settings + .tinybird + .auction_token_secret + .as_ref() + .ok_or_else(|| missing_secret_key_reference("tinybird.auction_token_secret"))?; + validate_secret_key_reference("tinybird.auction_token_secret", token.expose())?; + } + if settings.tinybird.enabled && settings.tinybird.access_enabled { + let token = settings + .tinybird + .access_token_secret + .as_ref() + .ok_or_else(|| missing_secret_key_reference("tinybird.access_token_secret"))?; + validate_secret_key_reference("tinybird.access_token_secret", token.expose())?; + } + + if let Some(datadome) = settings.integration_config::("datadome")? { + if datadome.enable_protection { + let key = datadome + .server_side_key_secret_name + .as_ref() + .ok_or_else(|| { + missing_secret_key_reference( + "integrations.datadome.server_side_key_secret_name", + ) + })?; + validate_secret_key_reference( + "integrations.datadome.server_side_key_secret_name", + key.expose(), + )?; + } + if let Some(bypass) = datadome + .protection_test_bypass + .as_ref() + .filter(|bypass| bypass.enabled) + { + let credential = bypass.credential_secret_name.as_ref().ok_or_else(|| { + missing_secret_key_reference( + "integrations.datadome.protection_test_bypass.credential_secret_name", + ) + })?; + validate_secret_key_reference( + "integrations.datadome.protection_test_bypass.credential_secret_name", + credential.expose(), + )?; + } + } + + for (index, route) in settings.proxy.asset_routes.iter().enumerate() { + let Some(AssetOriginAuth::S3SigV4(auth)) = route.auth.as_ref() else { + continue; + }; + validate_secret_key_reference( + &format!("proxy.asset_routes[{index}].auth.access_key_id"), + auth.access_key_id.expose(), + )?; + validate_secret_key_reference( + &format!("proxy.asset_routes[{index}].auth.secret_access_key"), + auth.secret_access_key.expose(), + )?; + if let Some(token) = &auth.session_token { + validate_secret_key_reference( + &format!("proxy.asset_routes[{index}].auth.session_token"), + token.expose(), + )?; } } Ok(()) } -fn report_to_validation_errors(report: &Report) -> ValidationErrors { - let mut error = ValidationError::new("trusted_server_deploy_validation"); - error.message = Some(Cow::Owned(report.to_string())); +fn validate_secret_key_reference( + path: &str, + key_name: &str, +) -> Result<(), Report> { + if key_name.is_empty() { + return Err(missing_secret_key_reference(path)); + } + Ok(()) +} + +fn missing_secret_key_reference(path: &str) -> Report { + Report::new(TrustedServerError::Configuration { + message: format!("secret key reference at `{path}` must not be empty"), + }) +} - let mut errors = ValidationErrors::new(); - errors.add(DEPLOY_VALIDATION_FIELD, error); - errors +fn report_to_validation_error( + report: &Report, + code: &'static str, +) -> ValidationError { + let mut error = ValidationError::new(code); + error.message = Some(Cow::Owned(report.to_string())); + error } #[cfg(test)] mod tests { + use std::collections::HashSet; + use super::*; + use crate::redacted::Redacted; + use crate::settings::{ProxyAssetRoute, S3SigV4AuthConfig}; use crate::test_support::tests::crate_test_settings_str; + use edgezero_core::app_config::AppConfigMeta; #[derive(Debug, Deserialize)] #[serde(deny_unknown_fields)] @@ -236,7 +531,9 @@ mod tests { slot: Vec, } - fn serialized_creative_opportunities(gam_unit_path: Option<&str>) -> serde_json::Value { + fn app_config_with_creative_opportunities( + gam_unit_path: Option<&str>, + ) -> TrustedServerAppConfig { let mut toml = crate_test_settings_str(); toml.push_str( r#" @@ -254,9 +551,15 @@ formats = [{ width = 300, height = 250 }] toml.push_str(&format!("gam_unit_path = {gam_unit_path:?}\n")); } - let app_config: TrustedServerAppConfig = + let mut app_config: TrustedServerAppConfig = toml::from_str(&toml).expect("should deserialize app config wrapper"); - serde_json::to_value(app_config) + app_config.settings.proxy.allowed_domains = + vec!["*.example".to_owned(), "*.example.com".to_owned()]; + app_config + } + + fn serialized_creative_opportunities(gam_unit_path: Option<&str>) -> serde_json::Value { + serde_json::to_value(app_config_with_creative_opportunities(gam_unit_path)) .expect("should serialize app config wrapper") .get("creative_opportunities") .cloned() @@ -276,14 +579,23 @@ formats = [{ width = 300, height = 250 }] "/../../trusted-server.example.toml" )); - /// Returns the template with its deliberately-invalid placeholder admin - /// password swapped for a valid one, so parse-time validation succeeds and - /// the test can exercise the optional blocks it uncomments. - fn template_with_valid_admin_password() -> String { - EXAMPLE_TEMPLATE.replace( - "password = \"replace-with-admin-password-32-bytes\"", - "password = \"unit-test-admin-password-that-is-long-enough\"", - ) + /// Returns the template with required secret-store key references replaced + /// by resolved test values, so direct [`Settings`] parsing can exercise the + /// optional blocks this module uncomments. + fn template_with_resolved_required_secrets() -> String { + EXAMPLE_TEMPLATE + .replace( + "password = \"handler_password\"", + "password = \"unit-test-resolved-handler-password-0001\"", + ) + .replace( + "proxy_secret = \"publisher_proxy_secret\"", + "proxy_secret = \"unit-test-resolved-publisher-proxy-secret-0001\"", + ) + .replace( + "passphrase = \"ec_passphrase\"", + "passphrase = \"unit-test-resolved-ec-passphrase-secret-0001\"", + ) } /// Uncomments the contiguous `#`-prefixed block that begins at the line @@ -317,11 +629,11 @@ formats = [{ width = 300, height = 250 }] /// Every documented block should be push-ready: uncommenting it and setting /// the shown values must parse and pass field validation. Blocks that ship - /// a deliberately-invalid placeholder (admin password, `ec.passphrase`, GTM - /// `container_id`, `request_signing` store ids) are excluded. + /// a deliberately-invalid non-secret placeholder (GTM `container_id` and + /// `request_signing` store ids) are excluded. #[test] fn documented_integration_blocks_validate_when_uncommented() { - let base = template_with_valid_admin_password(); + let base = template_with_resolved_required_secrets(); for (header, id) in [ ("[integrations.permutive]", "permutive"), @@ -363,7 +675,7 @@ formats = [{ width = 300, height = 250 }] /// uncommenting it with the documented `api_host` must parse cleanly. #[test] fn documented_tinybird_block_validates_when_uncommented() { - let toml = uncomment_block(&template_with_valid_admin_password(), "[tinybird]"); + let toml = uncomment_block(&template_with_resolved_required_secrets(), "[tinybird]"); let settings = Settings::from_toml(&toml) .expect("uncommented [tinybird] with documented api_host should parse and validate"); assert!( @@ -402,20 +714,185 @@ formats = [{ width = 300, height = 250 }] } #[test] - fn dynamic_gam_unit_templates_are_rejected_by_legacy_schema() { - for gam_unit_path in ["/{network_id}/example", "/example/{slot_id}"] { - let creative_opportunities = serialized_creative_opportunities(Some(gam_unit_path)); - let err = - serde_json::from_value::(creative_opportunities) - .expect_err("should reject dynamic GAM unit template"); + fn push_validation_accepts_secret_key_names() { + let mut settings = valid_settings(); + settings.publisher.proxy_secret = Redacted::new("publisher_proxy".to_owned()); + settings.ec.passphrase = Redacted::new("ec_key".to_owned()); + settings.handlers[0].password = Redacted::new("handler_password".to_owned()); + settings.handlers[1].password = Redacted::new("admin_password".to_owned()); + let app_config = TrustedServerAppConfig::new(settings) + .expect("should validate key names without values"); + + let serialized = + serde_json::to_string(&app_config).expect("should serialize key-name-only app config"); + assert!(serialized.contains("publisher_proxy")); + assert!(!serialized.contains("unit-test-proxy-secret")); + } + + #[test] + fn secret_metadata_lists_all_secret_paths_and_optionality() { + let fields = TrustedServerAppConfig::secret_fields(); + let paths = fields + .iter() + .map(|field| (field.dotted_path(), field.optional)) + .collect::>(); + + assert_eq!( + paths, + vec![ + ("publisher.proxy_secret".to_owned(), false), + ("ec.passphrase".to_owned(), false), + ("ec.partners[*].api_token".to_owned(), true), + ("ec.partners[*].ts_pull_token".to_owned(), true), + ("handlers[*].password".to_owned(), false), + ("tinybird.auction_token_secret".to_owned(), true), + ("tinybird.access_token_secret".to_owned(), true), + ( + "integrations.datadome.server_side_key_secret_name".to_owned(), + true, + ), + ( + "integrations.datadome.protection_test_bypass.credential_secret_name" + .to_owned(), + true, + ), + ("proxy.asset_routes[*].auth.access_key_id".to_owned(), true), + ( + "proxy.asset_routes[*].auth.secret_access_key".to_owned(), + true, + ), + ("proxy.asset_routes[*].auth.session_token".to_owned(), true), + ], + "should expose the native EdgeZero secret metadata contract" + ); + assert!( + fields.iter().all(|field| matches!( + field.kind, + edgezero_core::app_config::SecretKind::KeyInDefault + )), + "all Trusted Server app secrets should use the default secret store" + ); + } + + #[test] + fn partner_secret_metadata_makes_the_defaulted_array_optional() { + let fields = TrustedServerAppConfig::secret_fields(); + + for field in fields.iter().filter(|field| { + matches!( + field.dotted_path().as_str(), + "ec.partners[*].api_token" | "ec.partners[*].ts_pull_token" + ) + }) { + assert!(matches!( + &field.path[1], + SecretPathSegment::OptionalField(name) if name == "partners" + )); + } + } + + #[test] + fn omitted_s3_secret_references_materialize_as_defaults() { + let auth: S3SigV4AuthConfig = + toml::from_str("region = \"us-east-1\"").expect("should apply S3 secret defaults"); + + assert_eq!(auth.access_key_id.expose(), "access_key_id"); + assert_eq!(auth.secret_access_key.expose(), "secret_access_key"); + + let serialized = serde_json::to_value(auth).expect("should serialize S3 auth"); + assert_eq!(serialized["access_key_id"], "access_key_id"); + assert_eq!(serialized["secret_access_key"], "secret_access_key"); + } + #[test] + fn legacy_static_secret_store_selectors_are_accepted_but_not_serialized() { + let mut settings = valid_settings(); + settings.tinybird.secret_store = Some("legacy-tinybird-store".to_string()); + settings + .integrations + .insert_config( + "datadome", + &serde_json::json!({ + "enabled": true, + "server_side_key_secret_store": "legacy-datadome-store", + "protection_test_bypass": { + "enabled": false, + "credential_secret_store": "legacy-bypass-store", + }, + }), + ) + .expect("should insert legacy DataDome selectors"); + let mut route = ProxyAssetRoute::new( + "/assets/", + "https://examplebucket.s3.us-east-1.amazonaws.com", + ); + route.auth = Some(AssetOriginAuth::S3SigV4(S3SigV4AuthConfig { + region: "us-east-1".to_string(), + secret_store: Some("legacy-s3-store".to_string()), + access_key_id: Redacted::new("s3-access-key".to_string()), + secret_access_key: Redacted::new("s3-secret-key".to_string()), + session_token: None, + origin_query: None, + })); + settings.proxy.asset_routes.push(route); + + settings.normalize_deserialized(); + let serialized = serde_json::to_string(&settings).expect("should serialize settings"); + + for legacy_store in [ + "legacy-tinybird-store", + "legacy-datadome-store", + "legacy-bypass-store", + "legacy-s3-store", + ] { assert!( - err.to_string().contains("section_segment"), - "legacy error should name section_segment: {err}" + !serialized.contains(legacy_store), + "serialized config should omit deprecated selector {legacy_store}" ); } } + #[test] + fn settings_debug_redacts_resolved_static_credentials() { + let mut settings = valid_settings(); + settings.tinybird.auction_token_secret = + Some(Redacted::new("resolved-tinybird-secret".to_string())); + settings + .integrations + .insert_config( + "datadome", + &serde_json::json!({ + "enabled": true, + "server_side_key_secret_name": "resolved-datadome-secret", + }), + ) + .expect("should insert resolved DataDome config"); + + let debug = format!("{settings:?}"); + + assert!(!debug.contains("resolved-tinybird-secret")); + assert!(!debug.contains("resolved-datadome-secret")); + assert!(debug.contains("datadome")); + } + + #[test] + fn app_config_deserialization_does_not_finalize_runtime_templates() { + let creative_opportunities = + serialized_creative_opportunities(Some("/{network_id}/example")); + let slot = creative_opportunities["slot"][0] + .as_object() + .expect("should serialize creative opportunity slot"); + + assert!( + slot.contains_key("gam_unit_path"), + "push deserialization should preserve the operator config field" + ); + assert!( + !slot.contains_key("section_segment"), + "push deserialization should not add runtime-only compiled fields" + ); + } + #[test] fn static_gam_unit_template_is_accepted_by_legacy_schema() { let creative_opportunities = serialized_creative_opportunities(Some("/99999/example/home")); @@ -460,7 +937,35 @@ gam_network_id = "99999" } #[test] - fn deploy_validation_rejects_placeholders() { + fn app_config_new_rejects_empty_secret_key_reference() { + let mut settings = valid_settings(); + settings.publisher.proxy_secret = Redacted::new(String::new()); + + let err = TrustedServerAppConfig::new(settings) + .expect_err("should reject an empty secret key reference"); + + assert!( + err.to_string().contains("publisher.proxy_secret"), + "error should identify the empty secret reference: {err:?}" + ); + } + + #[test] + fn app_config_new_rejects_invalid_non_secret_settings() { + let mut settings = valid_settings(); + settings.publisher.domain = "invalid/domain".to_owned(); + + let err = TrustedServerAppConfig::new(settings) + .expect_err("should reject invalid publisher domain before creating an app config"); + + assert!( + err.to_string().contains("invalid_publisher_domain"), + "error should identify the structural validation failure: {err:?}" + ); + } + + #[test] + fn runtime_validation_rejects_placeholders() { let settings = Settings::from_toml( r#" [publisher] @@ -478,10 +983,10 @@ username = "admin" password = "production-admin-password-32-bytes" "#, ) - .expect("should parse placeholder settings before deploy validation"); + .expect("should parse placeholder settings before runtime validation"); - let err = - validate_settings_for_deploy(&settings).expect_err("should reject placeholder secrets"); + let err = validate_settings_for_runtime(&settings) + .expect_err("should reject placeholder secrets at runtime"); assert!( err.to_string().contains("Insecure default"), @@ -572,55 +1077,6 @@ password = "production-admin-password-32-bytes" ); } - #[test] - fn deploy_validation_rejects_blank_aps_account_id() { - // `deserialize_account_id` trims then rejects an empty result, so blank - // and whitespace-only ids fail at parse time. - for (label, account_id) in [("empty", ""), ("whitespace-only", " ")] { - let mut settings = valid_settings(); - settings - .integrations - .insert_config( - "aps", - &serde_json::json!({ - "enabled": true, - "account_id": account_id, - "endpoint": "https://aps.example.com/e/pb/bid" - }), - ) - .expect("should insert APS config"); - - let err = validate_settings_for_deploy(&settings) - .expect_err("should reject blank APS account_id when enabled"); - - assert!( - format!("{err:?}").contains("aps"), - "should mention the APS integration for {label} account_id: {err:?}" - ); - } - } - - #[test] - fn deploy_validation_normalizes_padded_aps_account_id() { - // Surrounding whitespace is normalized (trimmed) at deserialization, so - // a padded-but-otherwise-valid id deploys and reaches APS trimmed. - let mut settings = valid_settings(); - settings - .integrations - .insert_config( - "aps", - &serde_json::json!({ - "enabled": true, - "account_id": " example-account ", - "endpoint": "https://aps.example.com/e/pb/bid" - }), - ) - .expect("should insert APS config"); - - validate_settings_for_deploy(&settings) - .expect("should accept a padded-but-valid APS account_id (trimmed at deserialization)"); - } - #[test] fn deploy_validation_rejects_padded_request_signing_store_ids() { let mut settings = valid_settings(); @@ -641,22 +1097,11 @@ password = "production-admin-password-32-bytes" ); } - /// `enabled` defaults to `false` for APS, so a section that omits the flag - /// resolves to disabled and must not have its fields validated — otherwise - /// the documented template placeholder breaks existing configs on upgrade. + /// `enabled` defaults to `false`, so a section that omits the flag resolves + /// to disabled and must not have its fields validated. #[test] fn deploy_validation_skips_field_validation_for_integrations_with_omitted_enabled() { let mut settings = valid_settings(); - settings - .integrations - .insert_config( - "aps", - &serde_json::json!({ - "pub_id": "your-aps-publisher-id", - "endpoint": "https://aps.example.com/e/dtb/bid" - }), - ) - .expect("should insert APS config"); // `endpoint` parses as a plain string but would fail the `url` // validator, so this section only survives if validation is skipped for // integrations that resolve to disabled. @@ -687,6 +1132,114 @@ password = "production-admin-password-32-bytes" ); } + #[test] + fn deploy_validation_requires_external_bundle_url_for_enabled_prebid() { + let mut settings = valid_settings(); + settings + .integrations + .insert_config( + "prebid", + &serde_json::json!({ + "enabled": true, + "bundle": { "adapters": ["exampleBidder"] } + }), + ) + .expect("should insert enabled Prebid config"); + + let error = validate_settings_for_deploy(&settings) + .expect_err("should require enabled Prebid external bundle URL"); + assert!(error.to_string().contains("external_bundle_url")); + } + + #[test] + fn deploy_validation_rejects_conflicting_prebid_browser_bidder_ownership() { + let mut settings = valid_settings(); + settings.auction.enabled = true; + settings.auction.providers = crate::auction::AuctionConfig::legacy_provider_map(&["pbs"]); + settings.auction.bidders.insert( + "exampleBidder" + .parse() + .expect("should parse server-side bidder"), + crate::auction::BidderRouteConfig { + provider: "pbs".parse().expect("should parse provider"), + }, + ); + let mut prebid = settings + .integration_config::("prebid") + .expect("should parse Prebid config") + .expect("should have enabled Prebid config"); + prebid.client_side_bidders = vec!["exampleBidder".to_string()]; + settings + .integrations + .insert_config("prebid", &prebid) + .expect("should replace Prebid config"); + + let error = validate_settings_for_deploy(&settings) + .expect_err("should reject conflicting browser bidder ownership"); + assert!(error.to_string().contains("exampleBidder")); + assert!( + error + .to_string() + .contains("both client-side and server-side") + ); + } + + #[test] + fn validate_rejects_invalid_enabled_js_asset_proxy_config() { + let mut settings = valid_settings(); + settings.integrations.insert( + "js_asset_proxy".to_string(), + serde_json::json!({ "enabled": true }), + ); + + let err = validate_settings_for_deploy(&settings) + .expect_err("should reject invalid JS asset proxy config"); + let message = err.to_string(); + assert!( + message.contains("js_asset_proxy"), + "error should mention JS asset proxy validation" + ); + assert!( + message.contains("empty_assets") || message.contains("assets"), + "error should mention the missing assets" + ); + } + + #[test] + fn deploy_validation_accepts_dormant_prebid_browser_bidder_overlap() { + let mut settings = valid_settings(); + settings.auction.enabled = false; + settings.auction.providers = crate::auction::AuctionConfig::legacy_provider_map(&["pbs"]); + settings.auction.bidders.insert( + "exampleBidder" + .parse() + .expect("should parse server-side bidder"), + crate::auction::BidderRouteConfig { + provider: "pbs".parse().expect("should parse provider"), + }, + ); + let mut prebid = settings + .integration_config::("prebid") + .expect("should parse Prebid config") + .expect("should have enabled Prebid config"); + prebid.client_side_bidders = vec!["exampleBidder".to_string()]; + settings + .integrations + .insert_config("prebid", &prebid) + .expect("should replace Prebid config"); + + validate_settings_for_deploy(&settings) + .expect("disabled plan overlap should remain deployable"); + } + + #[test] + fn deploy_validation_accepts_typed_prebid_bundle_build_table() { + let settings = valid_settings(); + + validate_settings_for_deploy(&settings) + .expect("test config with typed Prebid bundle build table should validate"); + } + #[test] fn deploy_validation_covers_registered_integration_builders() { let validated_ids: HashSet<&'static str> = @@ -724,15 +1277,9 @@ password = "production-admin-password-32-bytes" #[test] fn deploy_validation_rejects_invalid_datadome_test_bypass() { - for (enable_protection, store, name, expected_message) in [ - ( - false, - "ts_secrets", - "datadome_test_bypass", - "requires enable_protection", - ), - (true, "", "datadome_test_bypass", "credential_secret_store"), - (true, "ts_secrets", "", "credential_secret_name"), + for (enable_protection, name, expected_message) in [ + (false, "datadome_test_bypass", "requires enable_protection"), + (true, "", "credential_secret_name"), ] { let mut settings = valid_settings(); settings @@ -742,9 +1289,9 @@ password = "production-admin-password-32-bytes" &serde_json::json!({ "enabled": true, "enable_protection": enable_protection, + "server_side_key_secret_name": "datadome_server_side_key", "protection_test_bypass": { "enabled": true, - "credential_secret_store": store, "credential_secret_name": name, }, }), @@ -760,11 +1307,47 @@ password = "production-admin-password-32-bytes" } } + #[test] + fn validate_rejects_invalid_disabled_js_asset_proxy_assets() { + let mut settings = valid_settings(); + settings.integrations.insert( + JS_ASSET_PROXY_INTEGRATION_ID.to_string(), + serde_json::json!({ + "enabled": false, + "assets": [{ + "path": "bad path", + "origin_url": "not-a-url", + "proxy": "disabled" + }] + }), + ); + + let err = validate_settings_for_deploy(&settings) + .expect_err("should reject invalid disabled asset inventory"); + let message = err.to_string(); + assert!( + message.contains(JS_ASSET_PROXY_INTEGRATION_ID), + "error should mention JS asset proxy validation" + ); + assert!( + message.contains("path") || message.contains("origin_url"), + "error should mention the invalid asset fields" + ); + } + #[test] fn validate_trait_reports_deploy_errors() { let mut settings = valid_settings(); settings.auction.enabled = true; - settings.auction.providers = vec!["missing-provider".to_string()]; + settings.auction.providers = + crate::auction::AuctionConfig::legacy_provider_map(&["missing-provider"]); + settings + .auction + .providers + .values_mut() + .next() + .expect("should have provider") + .protocol = "unsupported".to_string(); let app_config = TrustedServerAppConfig { settings }; let err = app_config diff --git a/crates/trusted-server-core/src/config_payload.rs b/crates/trusted-server-core/src/config_payload.rs index 6ede36e9c..006775363 100644 --- a/crates/trusted-server-core/src/config_payload.rs +++ b/crates/trusted-server-core/src/config_payload.rs @@ -8,20 +8,32 @@ use edgezero_core::blob_envelope::BlobEnvelope; use error_stack::Report; +use crate::config::TrustedServerAppConfig; use crate::error::TrustedServerError; +use crate::platform::{PlatformSecretStore, StoreName}; +use crate::secret_resolution::resolve_secret_references; use crate::settings::Settings; +/// Canonical logical secret store used by Trusted Server app-config secrets. +pub const DEFAULT_SECRET_STORE_ID: &str = "trusted_server_secrets"; + /// Default config-store key containing the Trusted Server app-config blob. pub const CONFIG_BLOB_KEY: &str = "trusted_server_config"; -/// Reconstruct validated [`Settings`] from a serialized config blob envelope. +/// Reconstruct runtime [`Settings`] from a serialized config blob envelope. +/// +/// Secret references are resolved after envelope verification and before +/// deserialization. The envelope data itself is never mutated or rewritten. /// /// # Errors /// /// Returns [`TrustedServerError::Configuration`] when the envelope cannot be -/// parsed, fails integrity verification, or contains invalid settings data. +/// parsed, fails integrity verification, secret resolution fails, or resolved +/// settings are invalid. pub fn settings_from_config_blob( envelope_json: &str, + secret_store: &dyn PlatformSecretStore, + default_secret_store_name: &StoreName, ) -> Result> { let envelope: BlobEnvelope = serde_json::from_str(envelope_json).map_err(|error| { Report::new(TrustedServerError::Configuration { @@ -36,40 +48,198 @@ pub fn settings_from_config_blob( .attach(error.to_string()) })?; - let settings = Settings::from_json_value(envelope.into_data())?; - settings.reject_placeholder_secrets()?; + let mut data = envelope.into_data(); + remove_inactive_secret_references(&mut data); + resolve_secret_references::( + &mut data, + secret_store, + default_secret_store_name, + )?; + let settings = Settings::from_json_value(data)?; + crate::config::validate_settings_for_runtime(&settings)?; Ok(settings) } +fn remove_inactive_secret_references(data: &mut serde_json::Value) { + if let Some(tinybird) = data + .get_mut("tinybird") + .and_then(serde_json::Value::as_object_mut) + { + let enabled = tinybird.get("enabled").and_then(serde_json::Value::as_bool) == Some(true); + if !enabled { + tinybird.remove("auction_token_secret"); + tinybird.remove("access_token_secret"); + } else { + if tinybird + .get("auction_enabled") + .and_then(serde_json::Value::as_bool) + == Some(false) + { + tinybird.remove("auction_token_secret"); + } + if tinybird + .get("access_enabled") + .and_then(serde_json::Value::as_bool) + != Some(true) + { + tinybird.remove("access_token_secret"); + } + } + } + + if let Some(partners) = data + .pointer_mut("/ec/partners") + .and_then(serde_json::Value::as_array_mut) + { + for partner in partners { + let Some(partner) = partner.as_object_mut() else { + continue; + }; + if partner + .get("pull_sync_enabled") + .and_then(serde_json::Value::as_bool) + != Some(true) + { + partner.remove("ts_pull_token"); + } + } + } + + let Some(datadome) = data + .pointer_mut("/integrations/datadome") + .and_then(serde_json::Value::as_object_mut) + else { + return; + }; + let integration_enabled = + datadome.get("enabled").and_then(serde_json::Value::as_bool) == Some(true); + let protection_enabled = integration_enabled + && datadome + .get("enable_protection") + .and_then(serde_json::Value::as_bool) + == Some(true); + if !protection_enabled { + datadome.remove("server_side_key_secret_name"); + } + + let bypass_enabled = protection_enabled + && datadome + .get("protection_test_bypass") + .and_then(serde_json::Value::as_object) + .and_then(|bypass| bypass.get("enabled")) + .and_then(serde_json::Value::as_bool) + == Some(true); + if !bypass_enabled + && let Some(bypass) = datadome + .get_mut("protection_test_bypass") + .and_then(serde_json::Value::as_object_mut) + { + bypass.remove("credential_secret_name"); + } +} + #[cfg(test)] mod tests { + use std::sync::Arc; + use super::*; + use crate::integrations::IntegrationRegistry; + use crate::platform::{PlatformError, StoreId}; use crate::redacted::Redacted; + use crate::settings::{AssetOriginAuth, EcPartner, ProxyAssetRoute, S3SigV4AuthConfig}; use crate::test_support::tests::crate_test_settings_str; - use serde::Deserialize; - - // Intentionally mirrors `AuctionConfig` before `rewrite_creatives` existed. - // Do not add fields introduced after that snapshot: this test proves a - // default payload remains readable by the previous binary schema. - #[derive(Deserialize)] - #[serde(deny_unknown_fields)] - struct LegacyAuctionConfig { - #[serde(rename = "enabled")] - _enabled: bool, - #[serde(rename = "providers")] - _providers: Vec, - #[serde(rename = "mediator")] - _mediator: Option, - #[serde(rename = "timeout_ms")] - _timeout_ms: u32, - #[serde(rename = "creative_store")] - _creative_store: String, - #[serde(rename = "allowed_context_keys")] - _allowed_context_keys: std::collections::HashSet, - } fn test_settings() -> Settings { - Settings::from_toml(&crate_test_settings_str()).expect("should parse test settings") + let mut settings = + Settings::from_toml(&crate_test_settings_str()).expect("should parse test settings"); + settings.proxy.allowed_domains = vec!["*.example".to_owned(), "*.example.com".to_owned()]; + settings + } + + struct EchoSecretStore; + + impl PlatformSecretStore for EchoSecretStore { + fn get_bytes( + &self, + _store_name: &StoreName, + key: &str, + ) -> Result, Report> { + let value = match key { + "placeholder_proxy" => "change-me-proxy-secret", + "unit-test-proxy-secret" => "unit-test-proxy-secret-32-bytes-ok", + _ => key, + }; + Ok(value.as_bytes().to_vec()) + } + + fn create( + &self, + _store_id: &StoreId, + _name: &str, + _value: &str, + ) -> Result<(), Report> { + Ok(()) + } + + fn delete(&self, _store_id: &StoreId, _name: &str) -> Result<(), Report> { + Ok(()) + } + } + + struct UnifiedSecretStore; + + impl PlatformSecretStore for UnifiedSecretStore { + fn get_bytes( + &self, + store_name: &StoreName, + key: &str, + ) -> Result, Report> { + if store_name.as_ref() != "ts_secrets" || key.starts_with("unused-") { + return Err(Report::new(PlatformError::SecretStore)); + } + let value = match key { + "unit-test-proxy-secret" => "unit-test-proxy-secret-32-bytes-ok", + "tinybird-token-key" => "resolved-tinybird-token", + "datadome-server-key" => "resolved-datadome-server-key", + "datadome-bypass-key" => "resolved-datadome-bypass-credential-32-bytes", + "access_key_id" | "s3-access-key" => "AKIAIOSFODNN7EXAMPLE", + "secret_access_key" | "s3-secret-key" => "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY", + "s3-session-key" => "resolved-session-token", + "partner-api-token-key" => "resolved-partner-api-token-32-bytes-ok", + "partner-pull-token-key" => "resolved-partner-pull-token-32-bytes-ok", + _ => key, + }; + Ok(value.as_bytes().to_vec()) + } + + fn create( + &self, + _store_id: &StoreId, + _name: &str, + _value: &str, + ) -> Result<(), Report> { + Ok(()) + } + + fn delete(&self, _store_id: &StoreId, _name: &str) -> Result<(), Report> { + Ok(()) + } + } + + fn partner_with_pull_sync(enabled: bool, token_key: &str) -> EcPartner { + let mut value = serde_json::json!({ + "name": "Example Partner", + "source_domain": "partner.example.com", + "api_token": "partner-api-token-key", + "pull_sync_enabled": enabled, + "ts_pull_token": token_key, + }); + if enabled { + value["pull_sync_url"] = + serde_json::Value::String("https://partner.example.com/sync".to_string()); + value["pull_sync_allowed_domains"] = serde_json::json!(["partner.example.com"]); + } + serde_json::from_value(value).expect("should build pull-sync partner") } fn envelope_json(settings: &Settings) -> String { @@ -78,11 +248,44 @@ mod tests { serde_json::to_string(&envelope).expect("should serialize envelope") } + fn load_settings(envelope_json: &str) -> Result> { + settings_from_config_blob( + envelope_json, + &EchoSecretStore, + &StoreName::from("trusted_server_secrets"), + ) + } + + fn settings_with_browser_bidder_overlap(auction_enabled: bool) -> Settings { + let mut settings = test_settings(); + settings.proxy.allowed_domains = vec!["*.example".to_string()]; + settings.auction.enabled = auction_enabled; + settings.auction.providers = crate::auction::AuctionConfig::legacy_provider_map(&["pbs"]); + settings.auction.bidders.insert( + "exampleBidder" + .parse() + .expect("should parse server-side bidder"), + crate::auction::BidderRouteConfig { + provider: "pbs".parse().expect("should parse provider"), + }, + ); + let mut prebid = settings + .integration_config::("prebid") + .expect("should parse Prebid config") + .expect("should have enabled Prebid config"); + prebid.client_side_bidders = vec!["exampleBidder".to_string()]; + settings + .integrations + .insert_config("prebid", &prebid) + .expect("should replace Prebid config"); + settings + } + #[test] fn payload_round_trips_through_blob_envelope() { let original = test_settings(); - let reconstructed = settings_from_config_blob(&envelope_json(&original)) - .expect("should reconstruct settings"); + let reconstructed = + load_settings(&envelope_json(&original)).expect("should reconstruct settings"); assert_eq!( reconstructed.publisher.domain, original.publisher.domain, @@ -99,6 +302,274 @@ mod tests { ); } + #[test] + fn resolves_all_static_credentials_from_the_mapped_default_store() { + let mut original = test_settings(); + original.tinybird.enabled = true; + original.tinybird.api_host = "api.example.com".to_string(); + original.tinybird.auction_token_secret = + Some(Redacted::new("tinybird-token-key".to_string())); + original + .integrations + .insert_config( + "datadome", + &serde_json::json!({ + "enabled": true, + "enable_protection": true, + "server_side_key_secret_name": "datadome-server-key", + "protection_test_bypass": { + "enabled": true, + "credential_secret_name": "datadome-bypass-key", + }, + }), + ) + .expect("should configure DataDome references"); + let mut route = ProxyAssetRoute::new( + "/assets/", + "https://examplebucket.s3.us-east-1.amazonaws.com", + ); + route.auth = Some(AssetOriginAuth::S3SigV4(S3SigV4AuthConfig { + region: "us-east-1".to_string(), + secret_store: Some("legacy-s3-store".to_string()), + access_key_id: Redacted::new("s3-access-key".to_string()), + secret_access_key: Redacted::new("s3-secret-key".to_string()), + session_token: Some(Redacted::new("s3-session-key".to_string())), + origin_query: None, + })); + original.proxy.asset_routes.push(route); + original + .ec + .partners + .push(partner_with_pull_sync(true, "partner-pull-token-key")); + + let reconstructed = settings_from_config_blob( + &envelope_json(&original), + &UnifiedSecretStore, + &StoreName::from("ts_secrets"), + ) + .expect("should resolve every static credential from the mapped store"); + + assert_eq!( + reconstructed + .tinybird + .auction_token_secret + .as_ref() + .map(Redacted::expose) + .map(String::as_str), + Some("resolved-tinybird-token") + ); + let datadome = reconstructed + .integration_config::("datadome") + .expect("should parse DataDome config") + .expect("should enable DataDome"); + assert_eq!( + datadome + .server_side_key_secret_name + .as_ref() + .map(Redacted::expose) + .map(String::as_str), + Some("resolved-datadome-server-key") + ); + let bypass = datadome + .protection_test_bypass + .as_ref() + .expect("should configure bypass"); + assert_eq!( + bypass + .credential_secret_name + .as_ref() + .map(Redacted::expose) + .map(String::as_str), + Some("resolved-datadome-bypass-credential-32-bytes") + ); + let auth = reconstructed.proxy.asset_routes[0] + .auth + .as_ref() + .expect("should preserve S3 auth"); + let AssetOriginAuth::S3SigV4(auth) = auth; + assert_eq!(auth.access_key_id.expose(), "AKIAIOSFODNN7EXAMPLE"); + assert_eq!( + auth.secret_access_key.expose(), + "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY" + ); + assert_eq!( + auth.session_token + .as_ref() + .map(Redacted::expose) + .map(String::as_str), + Some("resolved-session-token") + ); + assert!(auth.secret_store.is_none()); + assert_eq!( + reconstructed.ec.partners[0] + .api_token + .as_ref() + .map(Redacted::expose) + .map(String::as_str), + Some("resolved-partner-api-token-32-bytes-ok") + ); + assert_eq!( + reconstructed.ec.partners[0] + .ts_pull_token + .as_ref() + .map(Redacted::expose) + .map(String::as_str), + Some("resolved-partner-pull-token-32-bytes-ok") + ); + } + + #[test] + fn omitted_s3_secret_references_resolve_default_store_keys() { + let mut original = test_settings(); + let mut route = ProxyAssetRoute::new( + "/default-s3/", + "https://examplebucket.s3.us-east-1.amazonaws.com", + ); + route.auth = Some(AssetOriginAuth::S3SigV4( + toml::from_str("region = \"us-east-1\"").expect("should apply S3 secret defaults"), + )); + original.proxy.asset_routes.push(route); + + let reconstructed = settings_from_config_blob( + &envelope_json(&original), + &UnifiedSecretStore, + &StoreName::from("ts_secrets"), + ) + .expect("should resolve default S3 secret keys"); + + let AssetOriginAuth::S3SigV4(auth) = reconstructed.proxy.asset_routes[0] + .auth + .as_ref() + .expect("should preserve S3 auth"); + assert_eq!(auth.access_key_id.expose(), "AKIAIOSFODNN7EXAMPLE"); + assert_eq!( + auth.secret_access_key.expose(), + "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY" + ); + } + + #[test] + fn partner_without_api_token_loads_without_secret_resolution() { + let mut original = test_settings(); + let partner = serde_json::from_value(serde_json::json!({ + "name": "Example Partner", + "source_domain": "partner.example.com", + "bidstream_enabled": true, + })) + .expect("should build partner without API token"); + original.ec.partners.push(partner); + + let reconstructed = settings_from_config_blob( + &envelope_json(&original), + &UnifiedSecretStore, + &StoreName::from("ts_secrets"), + ) + .expect("should load partner without resolving an API token"); + + assert!( + reconstructed.ec.partners[0].api_token.is_none(), + "should preserve omitted API token" + ); + } + + #[test] + fn active_partner_pull_sync_fails_when_its_token_is_missing() { + let mut original = test_settings(); + original + .ec + .partners + .push(partner_with_pull_sync(true, "unused-partner-pull-token")); + + let error = settings_from_config_blob( + &envelope_json(&original), + &UnifiedSecretStore, + &StoreName::from("ts_secrets"), + ) + .expect_err("should reject a missing active pull-sync token"); + + assert!(error.to_string().contains("ec.partners[0].ts_pull_token")); + } + + #[test] + fn inactive_optional_features_do_not_resolve_stale_secret_references() { + let mut original = test_settings(); + original.tinybird.auction_token_secret = + Some(Redacted::new("unused-tinybird-key".to_string())); + original + .integrations + .insert_config( + "datadome", + &serde_json::json!({ + "enabled": true, + "enable_protection": false, + "server_side_key_secret_name": "unused-datadome-key", + "protection_test_bypass": { + "enabled": false, + "credential_secret_name": "unused-bypass-key", + }, + }), + ) + .expect("should configure inactive references"); + original + .ec + .partners + .push(partner_with_pull_sync(false, "unused-partner-pull-token")); + + let reconstructed = settings_from_config_blob( + &envelope_json(&original), + &UnifiedSecretStore, + &StoreName::from("ts_secrets"), + ) + .expect("should skip inactive optional feature references"); + + assert!(reconstructed.tinybird.auction_token_secret.is_none()); + assert!(reconstructed.ec.partners[0].ts_pull_token.is_none()); + let datadome = reconstructed + .integration_config::("datadome") + .expect("should parse inactive DataDome config") + .expect("client-side DataDome remains enabled"); + assert!(datadome.server_side_key_secret_name.is_none()); + assert!( + datadome + .protection_test_bypass + .as_ref() + .is_some_and(|bypass| bypass.credential_secret_name.is_none()) + ); + } + + #[test] + fn omitted_datadome_enabled_does_not_resolve_stale_protection_references() { + let mut original = test_settings(); + original + .integrations + .insert_config( + "datadome", + &serde_json::json!({ + "enable_protection": true, + "server_side_key_secret_name": "unused-datadome-key", + "protection_test_bypass": { + "enabled": true, + "credential_secret_name": "unused-bypass-key", + }, + }), + ) + .expect("should configure disabled DataDome references"); + + let reconstructed = settings_from_config_blob( + &envelope_json(&original), + &UnifiedSecretStore, + &StoreName::from("ts_secrets"), + ) + .expect("should skip stale DataDome protection references"); + + assert!( + reconstructed + .integration_config::("datadome") + .expect("should parse disabled DataDome config") + .is_none() + ); + } + #[test] fn legacy_blob_without_rewrite_creatives_preserves_rewriting() { let data = @@ -115,7 +586,7 @@ mod tests { let envelope_json = serde_json::to_string(&envelope).expect("should serialize envelope"); let reconstructed = - settings_from_config_blob(&envelope_json).expect("should reconstruct legacy settings"); + load_settings(&envelope_json).expect("should reconstruct legacy settings"); assert!( reconstructed.auction.rewrite_creatives, @@ -123,25 +594,12 @@ mod tests { ); } - #[test] - fn default_auction_payload_is_accepted_by_legacy_schema() { - let data = - serde_json::to_value(test_settings()).expect("should serialize settings to JSON"); - let auction = data - .get("auction") - .cloned() - .expect("should serialize auction settings"); - - serde_json::from_value::(auction) - .expect("should deserialize the default payload with the legacy schema"); - } - #[test] fn disabled_rewrite_creatives_survives_blob_round_trip() { let mut original = test_settings(); original.auction.rewrite_creatives = false; - let reconstructed = settings_from_config_blob(&envelope_json(&original)) + let reconstructed = load_settings(&envelope_json(&original)) .expect("should reconstruct disabled rewriting"); assert!( @@ -153,12 +611,13 @@ mod tests { #[test] fn strings_that_look_like_json_scalars_round_trip_as_strings() { let mut original = test_settings(); - original.publisher.proxy_secret = Redacted::new("1234567890".to_string()); + original.publisher.proxy_secret = + Redacted::new("12345678901234567890123456789012".to_string()); original.ec.passphrase = Redacted::new("12345678901234567890123456789012".to_string()); original.handlers[0].password = Redacted::new("true".to_string()); - let reconstructed = settings_from_config_blob(&envelope_json(&original)) - .expect("should reconstruct settings"); + let reconstructed = + load_settings(&envelope_json(&original)).expect("should reconstruct settings"); assert_eq!( reconstructed.publisher.proxy_secret.expose(), @@ -177,6 +636,85 @@ mod tests { ); } + #[test] + fn runtime_validation_accepts_short_resolved_proxy_secret() { + let mut settings = test_settings(); + settings.publisher.proxy_secret = Redacted::new("short_proxy".to_owned()); + + let reconstructed = load_settings(&envelope_json(&settings)) + .expect("should accept an existing short proxy secret"); + + assert_eq!( + reconstructed.publisher.proxy_secret.expose(), + "short_proxy", + "should preserve the resolved proxy secret" + ); + } + + #[test] + fn runtime_blob_rejects_enabled_browser_bidder_ownership_conflict() { + let original = settings_with_browser_bidder_overlap(true); + let error = load_settings(&envelope_json(&original)) + .expect_err("should reject enabled browser bidder ownership conflict"); + + assert!(error.to_string().contains("exampleBidder")); + assert!( + error + .to_string() + .contains("both client-side and server-side") + ); + } + + #[test] + fn runtime_validation_rejects_short_resolved_passphrase() { + let mut settings = test_settings(); + settings.ec.passphrase = Redacted::new("short_key".to_owned()); + + let err = load_settings(&envelope_json(&settings)) + .expect_err("should reject a short resolved passphrase"); + + assert!( + err.to_string().contains("short_passphrase") || err.to_string().contains("validation"), + "error should indicate runtime validation: {err:?}" + ); + assert!( + !err.to_string().contains("short_key"), + "error should not expose the secret value" + ); + } + + #[test] + fn placeholder_rejection_happens_after_secret_resolution() { + let mut settings = test_settings(); + settings.publisher.proxy_secret = Redacted::new("placeholder_proxy".to_owned()); + + let err = load_settings(&envelope_json(&settings)) + .expect_err("should reject a placeholder resolved from the secret store"); + + assert!( + err.to_string().contains("Insecure default"), + "error should identify the insecure default: {err:?}" + ); + assert!( + !err.to_string().contains("change-me-proxy-secret"), + "error should not expose the resolved secret value" + ); + } + + #[test] + fn runtime_blob_accepts_disabled_browser_bidder_ownership_overlap() { + let original = settings_with_browser_bidder_overlap(false); + let reconstructed = load_settings(&envelope_json(&original)) + .expect("should decode dormant conflicting runtime blob"); + let plan = Arc::new( + crate::auction::compile_auction_plan(&reconstructed) + .expect("should compile decoded disabled auction plan"), + ); + + IntegrationRegistry::with_plan(&reconstructed, plan) + .expect("runtime registry should accept disabled ownership overlap"); + } + #[test] fn tampered_blob_hash_is_rejected() { let mut envelope: BlobEnvelope = @@ -185,7 +723,7 @@ mod tests { let tampered = serde_json::to_string(&envelope).expect("should serialize tampered envelope"); - let err = settings_from_config_blob(&tampered).expect_err("should reject hash mismatch"); + let err = load_settings(&tampered).expect_err("should reject hash mismatch"); assert!( err.to_string().contains("integrity verification"), diff --git a/crates/trusted-server-core/src/constants.rs b/crates/trusted-server-core/src/constants.rs index e1152b1e7..1f85facb8 100644 --- a/crates/trusted-server-core/src/constants.rs +++ b/crates/trusted-server-core/src/constants.rs @@ -34,6 +34,8 @@ pub const HEADER_X_TS_ENV: HeaderName = HeaderName::from_static("x-ts-env"); // Fastly environment variables pub const ENV_FASTLY_SERVICE_VERSION: &str = "FASTLY_SERVICE_VERSION"; pub const ENV_FASTLY_IS_STAGING: &str = "FASTLY_IS_STAGING"; +pub const ENV_FASTLY_SERVICE_ID: &str = "FASTLY_SERVICE_ID"; +pub const ENV_FASTLY_POP: &str = "FASTLY_POP"; // Common standard header names used across modules pub const HEADER_USER_AGENT: HeaderName = HeaderName::from_static("user-agent"); diff --git a/crates/trusted-server-core/src/creative_opportunities.rs b/crates/trusted-server-core/src/creative_opportunities.rs index ab272e4da..4b1405359 100644 --- a/crates/trusted-server-core/src/creative_opportunities.rs +++ b/crates/trusted-server-core/src/creative_opportunities.rs @@ -173,8 +173,12 @@ fn sanitize_section(segment: &str) -> String { /// The path is used **raw** (not percent-decoded) so this stays consistent with /// how [`page_patterns`](CreativeOpportunitySlot::page_patterns) glob-match the /// same path — e.g. `/new%20s` yields `new_20s`, never the decoded `new_s`. +/// +/// Public so operator tooling that *infers* a `{section}` template from observed +/// ad-unit paths can check its inference against the exact derivation the +/// runtime will perform, rather than reimplementing the sanitization rules. #[must_use] -fn derive_section(path: &str, section_root: &str, section_segment: usize) -> String { +pub fn derive_section(path: &str, section_root: &str, section_segment: usize) -> String { match path .split('/') .filter(|segment| !segment.is_empty()) @@ -701,15 +705,7 @@ impl CreativeOpportunitySlot { // skip `compile_patterns`). Re-compiles on every call. self.page_patterns .iter() - .any(|pattern| match Pattern::new(pattern) { - Ok(p) => p.matches(path), - Err(_) => { - let normalised = pattern.replace("**", "*"); - Pattern::new(&normalised) - .map(|p| p.matches(path)) - .unwrap_or(false) - } - }) + .any(|pattern| compile_page_pattern(pattern).is_ok_and(|p| p.matches(path))) } /// Compile [`page_patterns`](Self::page_patterns) into the @@ -726,22 +722,20 @@ impl CreativeOpportunitySlot { self.compiled_patterns = self .page_patterns .iter() - .filter_map(|pattern| { - match Pattern::new(pattern).or_else(|_| Pattern::new(&pattern.replace("**", "*"))) { - Ok(compiled) => Some(compiled), - Err(_) => { - // Build-time validation only requires *one* valid pattern - // per slot, so a mixed valid/invalid set passes the build - // with the bad pattern silently dropped here. Warn so the - // operator can see the slot matches fewer pages than - // configured. - log::warn!( - "slot `{}`: dropping page pattern '{}' — it does not compile as a glob", - self.id, - pattern - ); - None - } + .filter_map(|pattern| match compile_page_pattern(pattern) { + Ok(compiled) => Some(compiled), + Err(error) => { + // Build-time validation only requires *one* valid pattern + // per slot, so a mixed valid/invalid set passes the build + // with the bad pattern silently dropped here. Warn so the + // operator can see the slot matches fewer pages than + // configured. + log::warn!( + "slot `{}`: dropping page pattern '{}': {error}", + self.id, + pattern + ); + None } }) .collect(); @@ -878,23 +872,16 @@ impl CreativeOpportunitySlot { /// Converts this slot into an [`AdSlot`] ready for use in an auction request. /// /// Prebid Server bidder params are wired into the `bidders` map keyed by - /// bidder name. Legacy APS slot params are accepted in configuration but - /// intentionally ignored by the APS `OpenRTB` provider. - /// - /// When [`PrebidSlotParams::bidders`] is empty, a `trustedServer` entry is - /// injected so [`PrebidAuctionProvider`] expands all `config.bidders` - /// automatically. The slot's `targeting.zone` value is forwarded as - /// `trustedServer.zone` so zone-aware bid-param override rules fire correctly. + /// bidder name. APS slot params are ignored by the generic APS profile. + /// When [`PrebidSlotParams::bidders`] is empty, a `trustedServer` marker is + /// inserted for stored-request compatibility and carries the optional zone. #[must_use] pub fn to_ad_slot(&self) -> AdSlot { let mut bidders: HashMap = HashMap::new(); if let Some(ref prebid) = self.providers.prebid { if prebid.bidders.is_empty() { - // No explicit per-bidder override: let the Prebid provider expand - // all config.bidders. The "trustedServer" key triggers - // expand_trusted_server_bidders in PrebidAuctionProvider, giving - // each bidder an empty params object that the override engine then - // fills with zone-aware rules. + // No explicit per-bidder params: preserve the stored-request + // marker and carry the zone for profile routing. let mut ts = serde_json::json!({ "bidderParams": {} }); if let Some(zone) = self.targeting.get("zone") { ts["zone"] = serde_json::Value::String(zone.clone()); @@ -984,17 +971,14 @@ pub struct ApsSlotParams { /// Inline Prebid Server bidder parameters for a slot. /// -/// When `bidders` is empty, `to_ad_slot` injects a `trustedServer` entry so -/// [`PrebidAuctionProvider`] expands all `config.bidders` automatically. -/// When `bidders` is non-empty the map is forwarded verbatim, bypassing -/// automatic expansion (useful for slots that need explicit per-bidder params). +/// When `bidders` is empty, `to_ad_slot` injects a `trustedServer` stored-request +/// marker. When non-empty, the bidder map is forwarded verbatim. #[derive(Debug, Clone, Deserialize, Serialize)] #[serde(deny_unknown_fields)] pub struct PrebidSlotParams { /// Per-bidder inline params map. Bidder name → params object. /// - /// Leave empty (or omit `bidders` in config) to auto-expand all - /// `config.bidders` with zone-aware param overrides. + /// Leave empty (or omit `bidders` in config) to use the stored-request path. /// /// Note: when this map is non-empty it is forwarded verbatim, so a slot's /// `targeting.zone` is **not** injected for these bidders (the `trustedServer` @@ -1004,6 +988,48 @@ pub struct PrebidSlotParams { pub bidders: HashMap, } +/// Compiles a [`page_patterns`](CreativeOpportunitySlot::page_patterns) entry +/// using the runtime's normalisation. +/// +/// This is the single definition of what the runtime accepts as a page glob: +/// a direct [`Pattern::new`], falling back to the `**`→`*` rewrite that +/// [`CreativeOpportunitySlot::compile_patterns`] and +/// [`matches_path`](CreativeOpportunitySlot::matches_path) apply. +/// +/// # Errors +/// +/// Returns an error string when the pattern compiles neither directly nor after +/// normalisation. +pub(crate) fn compile_page_pattern(pattern: &str) -> Result { + Pattern::new(pattern) + .or_else(|_| Pattern::new(&pattern.replace("**", "*"))) + .map_err(|error| format!("page pattern '{pattern}' is not a valid glob: {error}")) +} + +/// Validates a [`page_patterns`](CreativeOpportunitySlot::page_patterns) entry +/// using the runtime's normalisation. +/// +/// This exposes validation without leaking the runtime's `glob::Pattern` type +/// into the public API. +/// +/// # Errors +/// +/// Returns an error string when the pattern compiles neither directly nor after +/// the runtime's `**` to `*` normalisation. +/// +/// # Examples +/// +/// ``` +/// use trusted_server_core::creative_opportunities::validate_page_pattern; +/// +/// assert!(validate_page_pattern("/news/*").is_ok()); +/// assert!(validate_page_pattern("/20**").is_ok()); +/// assert!(validate_page_pattern("[").is_err()); +/// ``` +pub fn validate_page_pattern(pattern: &str) -> Result<(), String> { + compile_page_pattern(pattern).map(|_| ()) +} + /// Validates that a slot ID contains only safe characters. /// /// Allowed characters: ASCII alphanumerics, underscores (`_`), and hyphens (`-`). @@ -1036,9 +1062,318 @@ pub fn match_slots<'a>( slots.iter().filter(|s| s.matches_path(path)).collect() } +/// Three-state outcome of the server-side ad-stack gate. +/// +/// [`Yes`](RuntimeAdStackExpected::Yes) and [`No`](RuntimeAdStackExpected::No) +/// are decided purely from known inputs; [`Unknown`](RuntimeAdStackExpected::Unknown) +/// is reserved for callers (such as the operator CLI) that cannot prove the live +/// consent state and pass `None` for `consent_allows_auction`. +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +pub enum RuntimeAdStackExpected { + /// All known gates pass and consent is known to allow the auction. + Yes, + /// At least one known gate blocks the server-side ad stack. + No, + /// All known gates pass but consent is unproven. + Unknown, +} + +/// Identifies a single gate evaluated by [`evaluate_ad_stack_gate`]. +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +pub enum AdStackGateName { + /// Request method is `GET`. + MethodGet, + /// Request is a top-level navigation. + Navigation, + /// Request is not a prefetch. + NotPrefetch, + /// Request is not from a known bot. + NotBot, + /// At least one configured slot matches the request path. + MatchedSlots, + /// Consent is known to allow the auction. + ConsentAllowsAuction, + /// The global `[auction].enabled` kill switch is on. + AuctionEnabled, + /// The `[creative_opportunities].enabled` template switch is on. + AdTemplatesEnabled, +} + +impl AdStackGateName { + const ALL: [Self; 8] = [ + Self::MethodGet, + Self::Navigation, + Self::NotPrefetch, + Self::NotBot, + Self::MatchedSlots, + Self::ConsentAllowsAuction, + Self::AuctionEnabled, + Self::AdTemplatesEnabled, + ]; + + fn blocks(self, input: AdStackGateInput) -> bool { + match self { + Self::MethodGet => !input.method_get, + Self::Navigation => !input.navigation, + Self::NotPrefetch => input.prefetch, + Self::NotBot => input.bot, + Self::MatchedSlots => !input.matched_slots, + Self::ConsentAllowsAuction => input.consent_allows_auction == Some(false), + Self::AuctionEnabled => !input.auction_enabled, + Self::AdTemplatesEnabled => !input.ad_templates_enabled, + } + } +} + +/// Inputs to [`evaluate_ad_stack_gate`]. +/// +/// `consent_allows_auction` is tri-state: `Some(true)` allows, `Some(false)` +/// blocks, and `None` means the caller cannot prove the consent state. +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +pub struct AdStackGateInput { + /// Request method is `GET`. + pub method_get: bool, + /// Request is a top-level navigation. + pub navigation: bool, + /// Request advertises itself as a prefetch. + pub prefetch: bool, + /// Request is from a known bot. + pub bot: bool, + /// At least one configured slot matches the request path. + pub matched_slots: bool, + /// Whether consent allows the auction. + /// + /// `Some(true)` allows the auction, `Some(false)` blocks it, and `None` + /// means the caller cannot prove either state. Unknown consent is not a + /// denial: it produces [`RuntimeAdStackExpected::Unknown`] when every known + /// boolean gate passes. + pub consent_allows_auction: Option, + /// The global `[auction].enabled` kill switch. + pub auction_enabled: bool, + /// The `[creative_opportunities].enabled` template switch. + /// + /// `false` whenever creative opportunities are absent from the + /// configuration, so an unconfigured publisher blocks here as well. + pub ad_templates_enabled: bool, +} + +/// Result of [`evaluate_ad_stack_gate`]: the three-state expectation plus the +/// original inputs used to derive per-gate diagnostics on demand. +#[derive(Debug, Clone, Eq, PartialEq)] +pub struct AdStackGateResult { + /// The three-state ad-stack expectation. + pub expected: RuntimeAdStackExpected, + input: AdStackGateInput, +} + +impl AdStackGateResult { + /// Returns the gates that blocked the server-side ad stack. + pub fn blocking_gates(&self) -> impl Iterator + '_ { + AdStackGateName::ALL + .into_iter() + .filter(|gate| gate.blocks(self.input)) + } +} + +/// Evaluates whether the server-side ad stack should run for a request. +/// +/// Any known gate that fails sets [`No`](RuntimeAdStackExpected::No) and is +/// recorded in [`AdStackGateResult::blocking_gates`]. When no known gate blocks, +/// the result is [`Yes`](RuntimeAdStackExpected::Yes) if consent is known to +/// allow the auction, or [`Unknown`](RuntimeAdStackExpected::Unknown) when +/// `consent_allows_auction` is `None`. +/// +/// Gate polarity mirrors the runtime publisher path: `method_get`, `navigation`, +/// `matched_slots`, `auction_enabled`, and `ad_templates_enabled` block when +/// `false`; `prefetch` and `bot` block when `true`. +#[must_use] +pub fn evaluate_ad_stack_gate(input: AdStackGateInput) -> AdStackGateResult { + let known_gate_blocks = !input.method_get + || !input.navigation + || input.prefetch + || input.bot + || !input.matched_slots + || input.consent_allows_auction == Some(false) + || !input.auction_enabled + || !input.ad_templates_enabled; + let expected = if known_gate_blocks { + RuntimeAdStackExpected::No + } else if input.consent_allows_auction.is_none() { + RuntimeAdStackExpected::Unknown + } else { + RuntimeAdStackExpected::Yes + }; + + AdStackGateResult { expected, input } +} + #[cfg(test)] mod tests { + use std::collections::BTreeMap; + use std::str::FromStr as _; + + use edgezero_core::body::Body as EdgeBody; + use http::Request; + use super::*; + use crate::auction::plan::{ + AuctionPlan, AuctionPlanConfig, NotificationConfig, ProviderConfig, ProviderId, RoutingMode, + }; + use crate::auction::routing::route_auction; + use crate::auction::types::{AuctionRequest, PublisherInfo, UserInfo}; + + #[test] + fn ad_stack_gate_passes_for_eligible_navigation() { + let result = evaluate_ad_stack_gate(AdStackGateInput { + method_get: true, + navigation: true, + prefetch: false, + bot: false, + matched_slots: true, + consent_allows_auction: Some(true), + auction_enabled: true, + ad_templates_enabled: true, + }); + + assert_eq!(result.expected, RuntimeAdStackExpected::Yes); + assert_eq!(result.blocking_gates().count(), 0); + } + + #[test] + fn ad_stack_gate_blocks_known_kill_switch() { + let result = evaluate_ad_stack_gate(AdStackGateInput { + method_get: true, + navigation: true, + prefetch: false, + bot: false, + matched_slots: true, + consent_allows_auction: Some(true), + auction_enabled: false, + ad_templates_enabled: true, + }); + + assert_eq!(result.expected, RuntimeAdStackExpected::No); + assert!( + result + .blocking_gates() + .any(|gate| gate == AdStackGateName::AuctionEnabled) + ); + } + + #[test] + fn ad_stack_gate_blocks_disabled_ad_templates() { + let result = evaluate_ad_stack_gate(AdStackGateInput { + method_get: true, + navigation: true, + prefetch: false, + bot: false, + matched_slots: true, + consent_allows_auction: Some(true), + auction_enabled: true, + ad_templates_enabled: false, + }); + + assert_eq!( + result.expected, + RuntimeAdStackExpected::No, + "a disabled [creative_opportunities].enabled switch should block the ad stack" + ); + assert!( + result + .blocking_gates() + .any(|gate| gate == AdStackGateName::AdTemplatesEnabled), + "the template switch should be named as the blocking gate" + ); + } + + #[test] + fn ad_stack_gate_is_unknown_when_consent_is_unknown() { + let result = evaluate_ad_stack_gate(AdStackGateInput { + method_get: true, + navigation: true, + prefetch: false, + bot: false, + matched_slots: true, + consent_allows_auction: None, + auction_enabled: true, + ad_templates_enabled: true, + }); + + assert_eq!(result.expected, RuntimeAdStackExpected::Unknown); + } + + // Locks the spec §5.2 mirror invariant: with Some(consent) supplied for every + // input combination, `expected == Yes` must equal the legacy all-AND boolean. + #[test] + fn ad_stack_gate_with_known_consent_matches_legacy_boolean() { + for bits in 0u16..256 { + let input = AdStackGateInput { + method_get: bits & 1 != 0, + navigation: bits & 2 != 0, + prefetch: bits & 4 != 0, + bot: bits & 8 != 0, + matched_slots: bits & 16 != 0, + consent_allows_auction: Some(bits & 32 != 0), + auction_enabled: bits & 64 != 0, + ad_templates_enabled: bits & 128 != 0, + }; + // Legacy semantics: all positive gates true, both negative gates false. + let legacy = input.method_get + && input.navigation + && !input.prefetch + && !input.bot + && input.matched_slots + && input.consent_allows_auction == Some(true) + && input.auction_enabled + && input.ad_templates_enabled; + let got = evaluate_ad_stack_gate(input).expected == RuntimeAdStackExpected::Yes; + assert_eq!(got, legacy, "gate mismatch for bits={bits}"); + } + } + + #[test] + fn ad_stack_gate_with_unknown_consent_matches_known_boolean_gates() { + for bits in 0u8..128 { + let input = AdStackGateInput { + method_get: bits & 1 != 0, + navigation: bits & 2 != 0, + prefetch: bits & 4 != 0, + bot: bits & 8 != 0, + matched_slots: bits & 16 != 0, + consent_allows_auction: None, + auction_enabled: bits & 32 != 0, + ad_templates_enabled: bits & 64 != 0, + }; + let known_gates_pass = input.method_get + && input.navigation + && !input.prefetch + && !input.bot + && input.matched_slots + && input.auction_enabled + && input.ad_templates_enabled; + let expected = if known_gates_pass { + RuntimeAdStackExpected::Unknown + } else { + RuntimeAdStackExpected::No + }; + + assert_eq!( + evaluate_ad_stack_gate(input).expected, + expected, + "should match unknown-consent gate semantics for bits={bits}" + ); + } + } + + #[test] + fn validate_page_pattern_preserves_specific_compile_error() { + let error = validate_page_pattern("[").expect_err("should reject invalid glob"); + + assert!( + error.contains("page pattern '[' is not a valid glob"), + "should retain the invalid pattern in the error: {error}" + ); + } fn make_slot(id: &str, patterns: Vec<&str>) -> CreativeOpportunitySlot { CreativeOpportunitySlot { @@ -1956,6 +2291,69 @@ mod tests { ); } + #[test] + fn creative_opportunity_canonical_slot_feeds_shared_stored_router_with_zone() { + let mut slot = make_slot("header", vec!["/"]); + slot.targeting + .insert("zone".to_string(), "header".to_string()); + slot.providers.prebid = Some(PrebidSlotParams { + bidders: HashMap::new(), + }); + let plan = AuctionPlan::compile(AuctionPlanConfig { + timeout_ms: 900, + providers: BTreeMap::from([( + ProviderId::from_str("pbs-primary").expect("should parse provider ID"), + ProviderConfig { + protocol: "openrtb-2.6".to_string(), + profile: "prebid-server".to_string(), + endpoint: "https://pbs.example.test/openrtb".to_string(), + timeout_ms: None, + routing: RoutingMode::Explicit, + notifications: NotificationConfig::default(), + profile_config: serde_json::json!({}), + }, + )]), + bidders: BTreeMap::new(), + mediator: None, + request_signing: None, + }) + .expect("should compile plan"); + let auction_request = AuctionRequest { + id: "auction-1".to_string(), + slots: vec![slot.to_ad_slot()], + publisher: PublisherInfo { + domain: "publisher.example.test".to_string(), + page_url: None, + }, + user: UserInfo { + id: None, + consent: None, + eids: None, + }, + device: None, + site: None, + context: HashMap::new(), + }; + let inbound = Request::builder() + .uri("https://publisher.example.test/") + .body(EdgeBody::empty()) + .expect("should build request"); + let routed = route_auction(auction_request, &inbound, &plan, None); + + assert_eq!(routed.inputs().len(), 1); + assert!(routed.inputs()[0].slots()[0].has_trusted_stored_request()); + assert_eq!(routed.inputs()[0].slots()[0].prebid_zone(), Some("header")); + assert!( + routed.inputs()[0].slots().iter().all(|slot| { + !slot + .bidder_params() + .keys() + .any(|bidder| bidder.as_str() == "pbs-primary") + }), + "provider ID should not reach client-controlled bidder input" + ); + } + #[test] fn to_ad_slot_injects_trusted_server_without_zone_when_targeting_absent() { let mut slot = make_slot("no-zone", vec!["/"]); diff --git a/crates/trusted-server-core/src/ec/admin.rs b/crates/trusted-server-core/src/ec/admin.rs index 6219af7a9..e44708742 100644 --- a/crates/trusted-server-core/src/ec/admin.rs +++ b/crates/trusted-server-core/src/ec/admin.rs @@ -694,7 +694,7 @@ mod tests { source_domain: source_domain.to_owned(), openrtb_atype: EcPartner::default_openrtb_atype(), bidstream_enabled, - api_token: Redacted::new(format!("test-token-{source_domain:-<32}")), + api_token: Some(Redacted::new(format!("test-token-{source_domain:-<32}"))), batch_rate_limit: EcPartner::default_batch_rate_limit(), pull_sync_enabled: false, pull_sync_url: None, diff --git a/crates/trusted-server-core/src/ec/auth.rs b/crates/trusted-server-core/src/ec/auth.rs index 0e609f9a7..b7db6f1fa 100644 --- a/crates/trusted-server-core/src/ec/auth.rs +++ b/crates/trusted-server-core/src/ec/auth.rs @@ -52,7 +52,7 @@ mod tests { source_domain: source_domain.to_owned(), openrtb_atype: EcPartner::default_openrtb_atype(), bidstream_enabled: true, - api_token: Redacted::new(api_token.to_owned()), + api_token: Some(Redacted::new(api_token.to_owned())), batch_rate_limit: EcPartner::default_batch_rate_limit(), pull_sync_enabled: false, pull_sync_url: None, @@ -125,4 +125,23 @@ mod tests { "should return the matching partner" ); } + + #[test] + fn authenticate_bearer_rejects_token_for_partner_without_api_access() { + let mut partner = make_test_partner("ssp.example.com", VALID_API_TOKEN); + partner.api_token = None; + let registry = + PartnerRegistry::from_config(&[partner]).expect("should build registry without token"); + let req = Request::builder() + .method("GET") + .uri("https://edge.example.com/_ts/api/v1/identify") + .header("authorization", format!("Bearer {VALID_API_TOKEN}")) + .body(EdgeBody::empty()) + .expect("should build test request"); + + assert!( + authenticate_bearer(®istry, &req).is_none(), + "should reject authentication when API access is not configured" + ); + } } diff --git a/crates/trusted-server-core/src/ec/batch_sync.rs b/crates/trusted-server-core/src/ec/batch_sync.rs index 0e0f3b900..248e1dd27 100644 --- a/crates/trusted-server-core/src/ec/batch_sync.rs +++ b/crates/trusted-server-core/src/ec/batch_sync.rs @@ -341,7 +341,7 @@ mod tests { source_domain: source_domain.to_owned(), openrtb_atype: EcPartner::default_openrtb_atype(), bidstream_enabled: true, - api_token: Redacted::new(api_token.to_owned()), + api_token: Some(Redacted::new(api_token.to_owned())), batch_rate_limit: EcPartner::default_batch_rate_limit(), pull_sync_enabled: false, pull_sync_url: None, diff --git a/crates/trusted-server-core/src/ec/eids.rs b/crates/trusted-server-core/src/ec/eids.rs index 5dd5f3b05..1359f7509 100644 --- a/crates/trusted-server-core/src/ec/eids.rs +++ b/crates/trusted-server-core/src/ec/eids.rs @@ -154,7 +154,9 @@ mod tests { source_domain: source_domain.to_owned(), openrtb_atype: EcPartner::default_openrtb_atype(), bidstream_enabled: true, - api_token: Redacted::new(format!("token-{source_domain}-32-bytes-minimum-value")), + api_token: Some(Redacted::new(format!( + "token-{source_domain}-32-bytes-minimum-value" + ))), batch_rate_limit: EcPartner::default_batch_rate_limit(), pull_sync_enabled: false, pull_sync_url: None, diff --git a/crates/trusted-server-core/src/ec/finalize.rs b/crates/trusted-server-core/src/ec/finalize.rs index a553bb7a7..901f131b7 100644 --- a/crates/trusted-server-core/src/ec/finalize.rs +++ b/crates/trusted-server-core/src/ec/finalize.rs @@ -13,11 +13,12 @@ use crate::settings::Settings; use super::EcContext; use super::cookies::{expire_ec_cookie, set_ec_cookie}; -use super::generation::is_valid_ec_id; -use super::kv::KvIdentityGraph; -use super::log_id; -use super::prebid_eids::ingest_eid_cookies; +use super::generation::{generate_ec_id, is_valid_ec_id}; +use super::kv::{CreateIfAbsentOutcome, KvIdentityGraph, apply_partner_id_updates}; +use super::kv_types::KvEntry; +use super::prebid_eids::collect_eid_cookie_updates; use super::registry::PartnerRegistry; +use super::{EcKvSnapshot, current_timestamp, log_id}; /// TS-managed response headers tied to EC identity output. const EC_RESPONSE_HEADERS: &[&str] = &[ @@ -40,7 +41,7 @@ const EC_RESPONSE_HEADERS: &[&str] = &[ /// from the request *before* routing consumes it. pub fn ec_finalize_response( settings: &Settings, - ec_context: &EcContext, + ec_context: &mut EcContext, kv: Option<&KvIdentityGraph>, registry: &PartnerRegistry, eids_cookie: Option<&str>, @@ -69,12 +70,27 @@ pub fn ec_finalize_response( // for subsequent EC behavior. if let Some(graph) = kv { apply_withdrawal_tombstones(&ids_to_withdraw, |ec_id| { - if let Err(err) = graph.write_withdrawal_tombstone(ec_id) { - log::error!( - "Failed to write withdrawal tombstone for EC ID '{}': {err:?}", - log_id(ec_id), + let initial = if ec_context.kv_snapshot().belongs_to(ec_id) { + ec_context.kv_snapshot().clone() + } else { + EcKvSnapshot::NotRead + }; + let outcome = graph.tombstone_existing_from_snapshot(ec_id, initial); + // The browser cookie is already cleared, so a failed + // tombstone leaves a live row that server-side consumers + // still read as consented. Report every failure, including + // the non-active cookie ID whose outcome is not retained on + // the request context. + if matches!(outcome, EcKvSnapshot::Failed { .. }) { + log::warn!( + "EC withdrawal tombstone failed for '{}': the identity-graph row may \ + still be live with consent granted", + log_id(ec_id) ); } + if ec_context.ec_value() == Some(ec_id) { + ec_context.set_kv_snapshot(outcome); + } }); } } @@ -84,8 +100,21 @@ pub fn ec_finalize_response( // Returning user: consent is granted and EC came from request. if ec_context.ec_was_present() && !ec_context.ec_generated() && consent_allows_ec { - if let (Some(graph), Some(ec_id)) = (kv, ec_context.ec_value()) { - ingest_eid_cookies(eids_cookie, sharedid_cookie, ec_id, graph, registry); + if let (Some(graph), Some(ec_id)) = (kv, ec_context.ec_value().map(str::to_owned)) { + let updates = collect_eid_cookie_updates(eids_cookie, sharedid_cookie, registry); + let snapshot = graph.upsert_partner_ids_from_snapshot( + &ec_id, + &updates, + ec_context.kv_snapshot().clone(), + ); + ec_context.set_kv_snapshot(snapshot); + if matches!(ec_context.kv_snapshot(), EcKvSnapshot::Missing { .. }) + && ec_context.recovery_eligible() + { + confirm_then_recover_orphaned_ec( + settings, ec_context, graph, &ec_id, &updates, response, + ); + } } // Ordinary returning-user page views no longer refresh the browser @@ -97,13 +126,144 @@ pub fn ec_finalize_response( // there is no KV graph: that would mint a browser cookie with no backing // identity-graph row, producing a phantom ID on later requests. if ec_context.ec_generated() { - let (Some(graph), Some(ec_id)) = (kv, ec_context.ec_value()) else { + let (Some(graph), Some(ec_id)) = (kv, ec_context.ec_value().map(str::to_owned)) else { log::info!("Skipping generated EC response write because KV graph is unavailable"); return; }; - ingest_eid_cookies(eids_cookie, sharedid_cookie, ec_id, graph, registry); - set_ec_cookie_on_response(settings, ec_context, response); + let updates = collect_eid_cookie_updates(eids_cookie, sharedid_cookie, registry); + let snapshot = graph.upsert_partner_ids_from_snapshot( + &ec_id, + &updates, + ec_context.kv_snapshot().clone(), + ); + ec_context.set_kv_snapshot(snapshot); + if ec_context.kv_snapshot().entry_for(&ec_id).is_some() { + set_ec_cookie_on_response(settings, ec_context, response); + } else { + log::warn!("Skipping generated EC cookie because backing row is not authoritative"); + } + } +} + +fn recover_orphaned_ec( + settings: &Settings, + ec_context: &mut EcContext, + graph: &KvIdentityGraph, + updates: &[super::kv::PartnerIdUpdate], + response: &mut Response, +) { + // Snapshot the orphaned ID once so every fail-closed exit binds the failed + // snapshot to the same key. + let orphan_id = ec_context.ec_value().unwrap_or_default().to_owned(); + let Some(client_ip) = ec_context.client_ip().map(str::to_owned) else { + log::warn!("Orphan EC recovery skipped because client IP is unavailable"); + ec_context.set_kv_snapshot(EcKvSnapshot::Failed { + ec_id: orphan_id.clone(), + }); + return; + }; + + const MAX_RECOVERY_ATTEMPTS: usize = 5; + for _attempt in 0..MAX_RECOVERY_ATTEMPTS { + let ec_id = match generate_ec_id(settings, &client_ip) { + Ok(ec_id) => ec_id, + Err(err) => { + log::warn!("Orphan EC recovery ID generation failed: {err:?}"); + ec_context.set_kv_snapshot(EcKvSnapshot::Failed { + ec_id: orphan_id.clone(), + }); + return; + } + }; + let mut entry = KvEntry::new( + ec_context.consent(), + ec_context.geo_info(), + current_timestamp(), + &settings.publisher.domain, + ); + entry.device = ec_context + .device_signals() + .map(super::device::DeviceSignals::to_kv_device); + apply_partner_id_updates(&mut entry, updates); + + match graph.create_if_absent(&ec_id, &entry) { + Ok(CreateIfAbsentOutcome::Written) => { + let snapshot = EcKvSnapshot::Present { + ec_id: ec_id.clone(), + entry: Box::new(entry), + generation: None, + }; + ec_context.replace_with_generated(ec_id, snapshot); + set_ec_cookie_on_response(settings, ec_context, response); + return; + } + Ok(CreateIfAbsentOutcome::AlreadyExists) => continue, + Err(err) => { + log::warn!("Orphan EC recovery failed: {err:?}"); + ec_context.set_kv_snapshot(EcKvSnapshot::Failed { + ec_id: orphan_id.clone(), + }); + return; + } + } + } + + log::warn!("Orphan EC recovery exhausted collision retries"); + ec_context.set_kv_snapshot(EcKvSnapshot::Failed { + ec_id: orphan_id.clone(), + }); +} + +/// Proves an orphaned cookie is genuinely absent before rotating it. +/// +/// Rotation abandons a year-lived identity graph and its accumulated EIDs, so +/// it must never run on a stale read. The origin-overlapped preload reads the +/// row while the publisher origin is still in flight, and edge data stores are +/// eventually consistent: a recently created live key can read `Missing` at a +/// POP that has not converged. Two point reads do not fix that — both can be +/// stale — so absence has to be *proved*, not observed twice: +/// +/// - a row that became visible after the origin round trip is adopted, with any +/// pending updates merged, and is never rotated; +/// - a second miss is escalated to +/// [`key_exists_confirmed`](KvIdentityGraph::key_exists_confirmed), which +/// reads the primary data source. Only a proven-absent key rotates; +/// - a key the store still lists is left alone: the point reads were stale, so +/// the identity stays intact and recovery is retried on a later navigation; +/// - neither a read failure nor a failed existence check is a miss, and neither +/// rotates. +fn confirm_then_recover_orphaned_ec( + settings: &Settings, + ec_context: &mut EcContext, + graph: &KvIdentityGraph, + ec_id: &str, + updates: &[super::kv::PartnerIdUpdate], + response: &mut Response, +) { + let confirmed = graph.load_snapshot(ec_id); + match confirmed { + EcKvSnapshot::Present { .. } => { + // The row became visible after the origin round trip: adopt it and + // merge any pending updates rather than rotating a valid identity. + let merged = graph.upsert_partner_ids_from_snapshot(ec_id, updates, confirmed); + ec_context.set_kv_snapshot(merged); + } + EcKvSnapshot::Missing { .. } => match graph.key_exists_confirmed(ec_id) { + Ok(false) => recover_orphaned_ec(settings, ec_context, graph, updates, response), + Ok(true) => { + log::warn!( + "Orphan EC recovery skipped: both point reads missed a row the store still \ + lists; leaving the identity intact for a later navigation" + ); + } + Err(err) => { + log::warn!("Orphan EC recovery skipped: existence check failed: {err:?}"); + } + }, + // A failed or not-read confirmation is not an authoritative miss: leave + // the existing snapshot in place and do not rotate an unconfirmed miss. + EcKvSnapshot::Failed { .. } | EcKvSnapshot::NotRead => {} } } @@ -261,7 +421,9 @@ mod tests { source_domain: source_domain.to_owned(), openrtb_atype: EcPartner::default_openrtb_atype(), bidstream_enabled: true, - api_token: Redacted::new(format!("token-{source_domain}-32-bytes-minimum-value")), + api_token: Some(Redacted::new(format!( + "token-{source_domain}-32-bytes-minimum-value" + ))), batch_rate_limit: EcPartner::default_batch_rate_limit(), pull_sync_enabled: false, pull_sync_url: None, @@ -401,7 +563,7 @@ mod tests { source: ConsentSource::Cookie, ..Default::default() }; - let ec_context = + let mut ec_context = make_context_with_consent(Some(&ec_id), Some(&ec_id), true, false, consent); let mut response = empty_response(); set_header(&mut response, "x-ts-ec", "stale"); @@ -413,7 +575,7 @@ mod tests { let test_registry = PartnerRegistry::from_config(&partners).expect("should build registry"); ec_finalize_response( &settings, - &ec_context, + &mut ec_context, None, &test_registry, None, @@ -453,7 +615,7 @@ mod tests { let settings = create_test_settings(); let active_ec = sample_ec_id("activ1"); let cookie_ec = sample_ec_id("cook1e"); - let ec_context = make_context( + let mut ec_context = make_context( Some(&active_ec), Some(&cookie_ec), true, @@ -465,7 +627,7 @@ mod tests { let test_registry = PartnerRegistry::empty(); ec_finalize_response( &settings, - &ec_context, + &mut ec_context, None, &test_registry, None, @@ -487,7 +649,7 @@ mod tests { fn finalize_returning_user_sets_no_header_or_cookie() { let settings = create_test_settings(); let ec_id = sample_ec_id("mtch01"); - let ec_context = make_context( + let mut ec_context = make_context( Some(&ec_id), Some(&ec_id), true, @@ -499,7 +661,7 @@ mod tests { let test_registry = PartnerRegistry::empty(); ec_finalize_response( &settings, - &ec_context, + &mut ec_context, None, &test_registry, None, @@ -521,7 +683,7 @@ mod tests { fn finalize_generated_ec_without_kv_skips_cookie_and_header() { let settings = create_test_settings(); let generated_ec = sample_ec_id("gen123"); - let ec_context = make_context( + let mut ec_context = make_context( Some(&generated_ec), None, false, @@ -533,7 +695,7 @@ mod tests { let test_registry = PartnerRegistry::empty(); ec_finalize_response( &settings, - &ec_context, + &mut ec_context, None, &test_registry, None, @@ -551,16 +713,305 @@ mod tests { ); } + #[test] + fn finalize_rotates_orphaned_cookie_to_new_backed_ec() { + let settings = create_test_settings(); + let orphaned_ec = sample_ec_id("orphn1"); + let consent = ConsentContext { + jurisdiction: Jurisdiction::NonRegulated, + source: ConsentSource::Cookie, + ..Default::default() + }; + let mut ec_context = EcContext::new_for_test_with_ip( + Some(orphaned_ec.clone()), + consent, + Some("192.0.2.10".to_owned()), + ); + ec_context.set_recovery_eligible(true); + ec_context.set_kv_snapshot(EcKvSnapshot::Missing { + ec_id: orphaned_ec.clone(), + }); + let graph = KvIdentityGraph::in_memory("test_store"); + let mut response = empty_response(); + + ec_finalize_response( + &settings, + &mut ec_context, + Some(&graph), + &PartnerRegistry::empty(), + None, + None, + &mut response, + ); + + let replacement = ec_context.ec_value().expect("should rotate orphan"); + assert_ne!(replacement, orphaned_ec); + assert!( + graph + .get(replacement) + .expect("should read replacement") + .is_some(), + "replacement cookie should have a backing row" + ); + assert!( + get_header(&response, "set-cookie").is_some(), + "should emit replacement cookie after persistence" + ); + } + + #[test] + fn finalize_named_route_transient_miss_still_persists_eid_updates() { + // `/auction` and `/_ts/page-bids` save their first lookup into the + // context and are never recovery eligible, so a stale miss there has no + // later chance to retry. Finalization must revalidate before dropping + // the collected partner IDs. + let settings = create_test_settings(); + let ec_id = sample_ec_id("named1"); + let graph = KvIdentityGraph::in_memory("test_store"); + let live = KvEntry::new( + &granting_consent(), + None, + current_timestamp(), + &settings.publisher.domain, + ); + graph + .create(&ec_id, &live) + .expect("should seed the live row the endpoint lookup missed"); + let mut ec_context = returning_user_context( + &ec_id, + EcKvSnapshot::Missing { + ec_id: ec_id.clone(), + }, + false, + ); + let partners = vec![make_partner("sharedid.org")]; + let registry = PartnerRegistry::from_config(&partners).expect("should build registry"); + let mut response = empty_response(); + + ec_finalize_response( + &settings, + &mut ec_context, + Some(&graph), + ®istry, + None, + Some("shared-cookie-id"), + &mut response, + ); + + assert_did_not_rotate(&ec_context, &ec_id, &response); + let (stored, _) = graph + .get(&ec_id) + .expect("should read store") + .expect("row should remain"); + assert_eq!( + stored.ids.get("sharedid.org").map(|id| id.uid.as_str()), + Some("shared-cookie-id"), + "a stale endpoint miss must not suppress EID persistence" + ); + } + + #[test] + fn finalize_named_route_confirmed_miss_does_not_create_a_row() { + // The same path with a genuinely absent row must stay a no-op: a route + // without orphan recovery must never mint an identity-graph entry. + let settings = create_test_settings(); + let ec_id = sample_ec_id("named2"); + let graph = KvIdentityGraph::in_memory("test_store"); + let mut ec_context = returning_user_context( + &ec_id, + EcKvSnapshot::Missing { + ec_id: ec_id.clone(), + }, + false, + ); + let partners = vec![make_partner("sharedid.org")]; + let registry = PartnerRegistry::from_config(&partners).expect("should build registry"); + let mut response = empty_response(); + + ec_finalize_response( + &settings, + &mut ec_context, + Some(&graph), + ®istry, + None, + Some("shared-cookie-id"), + &mut response, + ); + + assert_did_not_rotate(&ec_context, &ec_id, &response); + assert!( + graph.get(&ec_id).expect("should read store").is_none(), + "a confirmed miss must not create a root entry" + ); + } + + #[test] + fn finalize_transient_missing_row_confirms_present_and_does_not_rotate() { + // The origin-overlapped preload transiently read `Missing` on an + // eventually-consistent store, but the row actually exists. The + // confirming re-read at finalize must adopt the live row instead of + // rotating a valid identity (transient Add -> Missing -> Present). + let settings = create_test_settings(); + let orphan = sample_ec_id("trans1"); + let graph = KvIdentityGraph::in_memory("test_store"); + let live = KvEntry::new( + &granting_consent(), + None, + current_timestamp(), + &settings.publisher.domain, + ); + graph + .create(&orphan, &live) + .expect("should seed the live row the preload missed"); + let mut ec_context = returning_user_context( + &orphan, + EcKvSnapshot::Missing { + ec_id: orphan.clone(), + }, + true, + ); + let mut response = empty_response(); + + ec_finalize_response( + &settings, + &mut ec_context, + Some(&graph), + &PartnerRegistry::empty(), + None, + None, + &mut response, + ); + + assert_did_not_rotate(&ec_context, &orphan, &response); + assert!( + matches!(ec_context.kv_snapshot(), EcKvSnapshot::Present { .. }), + "confirming read must adopt the now-visible row rather than rotating" + ); + } + + #[test] + fn finalize_two_missing_reads_do_not_rotate_a_row_the_store_still_lists() { + // Both the origin-overlapped preload and the confirming re-read missed, + // but the row is live — the point reads were stale. Two stale reads are + // not an absence proof, so the identity graph must be left intact and + // recovery retried on a later navigation rather than fragmented behind + // a replacement ID. + let settings = create_test_settings(); + let orphan = sample_ec_id("stale1"); + // One stale read: the preload miss is the `Missing` snapshot below, and + // this makes the confirming re-read miss too. + let graph = KvIdentityGraph::stale_lookup("test_store", 1); + let live = KvEntry::new( + &granting_consent(), + None, + current_timestamp(), + &settings.publisher.domain, + ); + graph + .create(&orphan, &live) + .expect("should seed the live row both point reads miss"); + let mut ec_context = returning_user_context( + &orphan, + EcKvSnapshot::Missing { + ec_id: orphan.clone(), + }, + true, + ); + let mut response = empty_response(); + + ec_finalize_response( + &settings, + &mut ec_context, + Some(&graph), + &PartnerRegistry::empty(), + None, + None, + &mut response, + ); + + assert_did_not_rotate(&ec_context, &orphan, &response); + assert_eq!( + graph + .get(&orphan) + .expect("should read store") + .map(|(entry, _)| entry.consent.ok), + Some(true), + "the original identity row must survive two stale point reads" + ); + } + + #[test] + fn finalize_does_not_rotate_when_the_existence_check_fails() { + // Absence is unprovable when the list itself errors. Rotation abandons a + // year-lived identity, so it must not run on an unproven miss. + let settings = create_test_settings(); + let orphan = sample_ec_id("nolist"); + let graph = KvIdentityGraph::unprovable_absence("test_store"); + let mut ec_context = returning_user_context( + &orphan, + EcKvSnapshot::Missing { + ec_id: orphan.clone(), + }, + true, + ); + let mut response = empty_response(); + + ec_finalize_response( + &settings, + &mut ec_context, + Some(&graph), + &PartnerRegistry::empty(), + None, + None, + &mut response, + ); + + assert_did_not_rotate(&ec_context, &orphan, &response); + } + + #[test] + fn finalize_generated_ec_does_not_emit_cookie_for_authoritative_missing_row() { + let settings = create_test_settings(); + let generated_ec = sample_ec_id("genmis"); + let mut ec_context = make_context( + Some(&generated_ec), + None, + false, + true, + Jurisdiction::NonRegulated, + ); + ec_context.set_kv_snapshot(EcKvSnapshot::Missing { + ec_id: generated_ec, + }); + let graph = KvIdentityGraph::in_memory("test_store"); + let mut response = empty_response(); + + ec_finalize_response( + &settings, + &mut ec_context, + Some(&graph), + &PartnerRegistry::empty(), + None, + None, + &mut response, + ); + + assert!( + get_header(&response, "set-cookie").is_none(), + "must not emit a cookie without an authoritative backing row" + ); + } + #[test] fn finalize_denied_without_cookie_is_noop() { let settings = create_test_settings(); - let ec_context = make_context(None, None, false, false, Jurisdiction::Unknown); + let mut ec_context = make_context(None, None, false, false, Jurisdiction::Unknown); let mut response = empty_response(); let test_registry = PartnerRegistry::empty(); ec_finalize_response( &settings, - &ec_context, + &mut ec_context, None, &test_registry, None, @@ -582,7 +1033,7 @@ mod tests { fn finalize_unknown_jurisdiction_strips_headers_without_expiring_cookie() { let settings = create_test_settings(); let ec_id = sample_ec_id("unk001"); - let ec_context = make_context( + let mut ec_context = make_context( Some(&ec_id), Some(&ec_id), true, @@ -596,7 +1047,7 @@ mod tests { let test_registry = PartnerRegistry::empty(); ec_finalize_response( &settings, - &ec_context, + &mut ec_context, None, &test_registry, None, @@ -617,4 +1068,202 @@ mod tests { "should not expire the cookie without an explicit withdrawal signal" ); } + + // ----------------------------------------------------------------------- + // Orphan-recovery gating and two-ID withdrawal + // ----------------------------------------------------------------------- + + fn granting_consent() -> ConsentContext { + ConsentContext { + jurisdiction: Jurisdiction::NonRegulated, + source: ConsentSource::Cookie, + ..Default::default() + } + } + + fn returning_user_context( + orphan: &str, + snapshot: EcKvSnapshot, + recovery_eligible: bool, + ) -> EcContext { + let mut ec = EcContext::new_for_test_with_ip( + Some(orphan.to_owned()), + granting_consent(), + Some("192.0.2.10".to_owned()), + ); + ec.set_recovery_eligible(recovery_eligible); + ec.set_kv_snapshot(snapshot); + ec + } + + fn assert_did_not_rotate(ec_context: &EcContext, orphan: &str, response: &Response) { + assert_eq!( + ec_context.ec_value(), + Some(orphan), + "must not rotate the active EC ID" + ); + assert!(!ec_context.ec_generated(), "must not mark a rotated EC"); + assert!( + get_header(response, "set-cookie").is_none(), + "must not emit a replacement cookie" + ); + } + + #[test] + fn finalize_not_read_snapshot_does_not_rotate() { + let settings = create_test_settings(); + let orphan = sample_ec_id("notrd1"); + let mut ec_context = returning_user_context(&orphan, EcKvSnapshot::NotRead, true); + let graph = KvIdentityGraph::in_memory("test_store"); + let mut response = empty_response(); + + ec_finalize_response( + &settings, + &mut ec_context, + Some(&graph), + &PartnerRegistry::empty(), + None, + None, + &mut response, + ); + + assert_did_not_rotate(&ec_context, &orphan, &response); + } + + #[test] + fn finalize_failed_snapshot_does_not_rotate() { + let settings = create_test_settings(); + let orphan = sample_ec_id("faild1"); + let mut ec_context = returning_user_context( + &orphan, + EcKvSnapshot::Failed { + ec_id: orphan.clone(), + }, + true, + ); + let graph = KvIdentityGraph::in_memory("test_store"); + let mut response = empty_response(); + + ec_finalize_response( + &settings, + &mut ec_context, + Some(&graph), + &PartnerRegistry::empty(), + None, + None, + &mut response, + ); + + assert_did_not_rotate(&ec_context, &orphan, &response); + } + + #[test] + fn finalize_tombstone_snapshot_does_not_rotate() { + let settings = create_test_settings(); + let orphan = sample_ec_id("tomb01"); + let tombstone = EcKvSnapshot::Present { + ec_id: orphan.clone(), + entry: Box::new(KvEntry::tombstone(current_timestamp())), + generation: Some(1), + }; + let mut ec_context = returning_user_context(&orphan, tombstone, true); + let graph = KvIdentityGraph::in_memory("test_store"); + let mut response = empty_response(); + + ec_finalize_response( + &settings, + &mut ec_context, + Some(&graph), + &PartnerRegistry::empty(), + None, + None, + &mut response, + ); + + assert_did_not_rotate(&ec_context, &orphan, &response); + } + + #[test] + fn finalize_subresource_missing_row_does_not_rotate() { + let settings = create_test_settings(); + let orphan = sample_ec_id("subrs1"); + // Missing row, but the request is not a recovery-eligible browser navigation. + let mut ec_context = returning_user_context( + &orphan, + EcKvSnapshot::Missing { + ec_id: orphan.clone(), + }, + false, + ); + let graph = KvIdentityGraph::in_memory("test_store"); + let mut response = empty_response(); + + ec_finalize_response( + &settings, + &mut ec_context, + Some(&graph), + &PartnerRegistry::empty(), + None, + None, + &mut response, + ); + + assert_did_not_rotate(&ec_context, &orphan, &response); + assert!( + graph.get(&orphan).expect("should read store").is_none(), + "a non-eligible request must not create the missing root" + ); + } + + #[test] + fn finalize_withdrawal_tombstones_present_id_and_skips_missing_other() { + let settings = create_test_settings(); + let active_ec = sample_ec_id("activ2"); + let cookie_ec = sample_ec_id("cook2e"); + let consent = ConsentContext { + jurisdiction: Jurisdiction::UsState("CA".to_owned()), + gpc: true, + source: ConsentSource::Cookie, + ..Default::default() + }; + let mut ec_context = + make_context_with_consent(Some(&active_ec), Some(&cookie_ec), true, false, consent); + // Carry a snapshot only for the active ID; the other ID must be looked up + // independently and never created if absent. + let graph = KvIdentityGraph::in_memory("test_store"); + graph + .create(&active_ec, &live_entry()) + .expect("should seed active row"); + ec_context.set_kv_snapshot(graph.load_snapshot(&active_ec)); + let mut response = empty_response(); + + ec_finalize_response( + &settings, + &mut ec_context, + Some(&graph), + &PartnerRegistry::empty(), + None, + None, + &mut response, + ); + + let (active_stored, _) = graph + .get(&active_ec) + .expect("should read active row") + .expect("active row should remain as a tombstone"); + assert!( + !active_stored.consent.ok, + "the present active ID should be tombstoned via its carried snapshot" + ); + assert!( + graph.get(&cookie_ec).expect("should read store").is_none(), + "a missing second ID must never be created by withdrawal" + ); + } + + fn live_entry() -> KvEntry { + let mut entry = KvEntry::tombstone(1000); + entry.consent.ok = true; + entry + } } diff --git a/crates/trusted-server-core/src/ec/identify.rs b/crates/trusted-server-core/src/ec/identify.rs index 6ca251905..94464d1a5 100644 --- a/crates/trusted-server-core/src/ec/identify.rs +++ b/crates/trusted-server-core/src/ec/identify.rs @@ -367,7 +367,7 @@ mod tests { source_domain: source_domain.to_owned(), openrtb_atype: EcPartner::default_openrtb_atype(), bidstream_enabled: true, - api_token: Redacted::new(api_token.to_owned()), + api_token: Some(Redacted::new(api_token.to_owned())), batch_rate_limit: EcPartner::default_batch_rate_limit(), pull_sync_enabled: false, pull_sync_url: None, diff --git a/crates/trusted-server-core/src/ec/kv.rs b/crates/trusted-server-core/src/ec/kv.rs index 3572581ce..f6807e738 100644 --- a/crates/trusted-server-core/src/ec/kv.rs +++ b/crates/trusted-server-core/src/ec/kv.rs @@ -23,7 +23,7 @@ use super::current_timestamp; use super::generation::ec_hash; use super::kv_backend::{EcKvLookup, EcKvStore, EcKvWrite, EcKvWriteMode, EcKvWriteOutcome}; use super::kv_types::{KvEntry, KvMetadata, KvNetwork}; -use super::log_id; +use super::{EcKvSnapshot, log_id}; /// Maximum number of CAS retry attempts before giving up. const MAX_CAS_RETRIES: u32 = 5; @@ -56,6 +56,15 @@ pub enum UpsertResult { Unchanged, } +/// Outcome of atomically creating an identity-graph root when absent. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum CreateIfAbsentOutcome { + /// The candidate entry was persisted. + Written, + /// A row already exists for the candidate key. + AlreadyExists, +} + /// Partner UID update to apply to a KV identity graph entry. #[derive(Debug, Clone, PartialEq, Eq)] pub(crate) struct PartnerIdUpdate { @@ -75,7 +84,7 @@ impl PartnerIdUpdate { } } -fn apply_partner_id_updates(entry: &mut KvEntry, updates: &[PartnerIdUpdate]) -> bool { +pub(crate) fn apply_partner_id_updates(entry: &mut KvEntry, updates: &[PartnerIdUpdate]) -> bool { let mut latest_updates = BTreeMap::new(); for update in updates { latest_updates.insert(update.partner_id.as_str(), update.uid.as_str()); @@ -205,6 +214,30 @@ impl KvIdentityGraph { Ok(Some((entry, lookup.generation))) } + /// Loads one request-scoped snapshot, preserving miss versus failure at the caller boundary. + #[must_use] + pub fn load_snapshot(&self, ec_id: &str) -> EcKvSnapshot { + match self.get(ec_id) { + Ok(Some((entry, generation))) => EcKvSnapshot::Present { + ec_id: ec_id.to_owned(), + entry: Box::new(entry), + generation: Some(generation), + }, + Ok(None) => EcKvSnapshot::Missing { + ec_id: ec_id.to_owned(), + }, + Err(err) => { + log::warn!( + "EC KV snapshot read failed for '{}': {err:?}", + log_id(ec_id) + ); + EcKvSnapshot::Failed { + ec_id: ec_id.to_owned(), + } + } + } + } + fn deserialize_entry( store_name: &str, ec_id: &str, @@ -273,6 +306,23 @@ impl KvIdentityGraph { } } + /// Atomically creates an entry while preserving collision as normal control flow. + /// + /// # Errors + /// + /// Returns [`TrustedServerError::KvStore`] when serialization or store I/O fails. + pub fn create_if_absent( + &self, + ec_id: &str, + entry: &KvEntry, + ) -> Result> { + let (body, meta_str) = Self::serialize_entry(entry, self.store_name())?; + match self.write_entry(ec_id, &body, &meta_str, ENTRY_TTL, EcKvWriteMode::Add)? { + EcKvWriteOutcome::Written => Ok(CreateIfAbsentOutcome::Written), + EcKvWriteOutcome::PreconditionFailed => Ok(CreateIfAbsentOutcome::AlreadyExists), + } + } + /// Low-level write with shared error context. fn write_entry( &self, @@ -464,6 +514,166 @@ impl KvIdentityGraph { ))) } + /// Merges partner IDs using request-scoped persisted state as the first CAS input. + /// + /// A caller-supplied `Present` snapshot is *proof* that the row exists — + /// either an `Add`-confirmed create from [`generate_if_needed`] or an + /// earlier authoritative read in the same request. Partner-ID enrichment is + /// best effort, so a refresh that reads absent or unreadable never + /// downgrades that proof: the update is skipped and logged, and the proven + /// snapshot is returned so cookie issuance continues from the confirmed + /// write. A failed *write* still reports [`EcKvSnapshot::Failed`] — that is + /// an operation failure, not an ambiguous read. + /// + /// A caller-supplied `Missing` snapshot is revalidated once before the + /// updates are dropped, because a point read on an eventually-consistent + /// store cannot prove absence and routes without orphan recovery have no + /// later chance to retry. + /// + /// [`generate_if_needed`]: super::generate_if_needed + pub(crate) fn upsert_partner_ids_from_snapshot( + &self, + ec_id: &str, + updates: &[PartnerIdUpdate], + snapshot: EcKvSnapshot, + ) -> EcKvSnapshot { + if updates.is_empty() { + return snapshot; + } + + // Existence proof carried by the incoming snapshot. Retained across + // every refresh so a best-effort enrichment read can never retract it. + let proven = match snapshot { + EcKvSnapshot::Present { + ec_id: ref snapshot_id, + .. + } if snapshot_id == ec_id => Some(snapshot.clone()), + _ => None, + }; + + // Resolve the initial usable snapshot without spending a CAS attempt. + // Every snapshot except a usable `Present` for this EC ID and a read + // that already failed is refreshed once. A `Missing` recorded earlier + // in the request is not proof of absence: edge KV point reads are + // eventually consistent, and named routes such as `/auction` and + // `/_ts/page-bids` never run orphan recovery, so short-circuiting on a + // stale miss there would drop the request's collected partner IDs + // outright. A refresh that still misses keeps the no-create behavior. + // A `Failed` read is returned as-is — the hot path never retries a + // lookup that already errored. Resolving here keeps all + // `MAX_CAS_RETRIES` iterations available for actual writes. + let mut current = match snapshot { + EcKvSnapshot::Present { + ec_id: ref snapshot_id, + generation: Some(_), + .. + } if snapshot_id == ec_id => snapshot, + EcKvSnapshot::Failed { + ec_id: ref snapshot_id, + } if snapshot_id == ec_id => return snapshot, + _ => self.load_snapshot(ec_id), + }; + + for _attempt in 0..MAX_CAS_RETRIES { + let (mut entry, generation) = match current { + EcKvSnapshot::Present { + ec_id: ref snapshot_id, + ref entry, + generation: Some(generation), + } if snapshot_id == ec_id => (entry.as_ref().clone(), generation), + // A refreshed read that is absent or unreadable is authoritative + // for this write: never create or overwrite a missing root. It + // is not authoritative for *existence* though, so a snapshot + // that already proved the row exists survives the refresh. + EcKvSnapshot::Missing { .. } | EcKvSnapshot::Failed { .. } => { + return Self::keep_proven(ec_id, current, proven.as_ref()); + } + // `load_snapshot` never yields `NotRead` or a generation-less + // `Present`; fail closed if that invariant is ever violated. + EcKvSnapshot::Present { .. } | EcKvSnapshot::NotRead => { + return EcKvSnapshot::Failed { + ec_id: ec_id.to_owned(), + }; + } + }; + + if !entry.consent.ok { + return current; + } + if !apply_partner_id_updates(&mut entry, updates) { + return current; + } + let Ok((body, meta_str)) = Self::serialize_entry(&entry, self.store_name()) else { + return EcKvSnapshot::Failed { + ec_id: ec_id.to_owned(), + }; + }; + match self.write_entry( + ec_id, + &body, + &meta_str, + ENTRY_TTL, + EcKvWriteMode::IfGenerationMatch(generation), + ) { + Ok(EcKvWriteOutcome::Written) => { + return EcKvSnapshot::Present { + ec_id: ec_id.to_owned(), + entry: Box::new(entry), + generation: None, + }; + } + Ok(EcKvWriteOutcome::PreconditionFailed) => { + current = self.load_snapshot(ec_id); + } + Err(err) => { + log::warn!( + "snapshot partner upsert failed for '{}': {err:?}", + log_id(ec_id) + ); + return EcKvSnapshot::Failed { + ec_id: ec_id.to_owned(), + }; + } + } + } + + log::warn!( + "snapshot partner upsert for '{}': CAS conflict after {MAX_CAS_RETRIES} retries; {} partner updates were not persisted", + log_id(ec_id), + updates.len(), + ); + EcKvSnapshot::Failed { + ec_id: ec_id.to_owned(), + } + } + + /// Returns `proven` instead of a read outcome that cannot disprove it. + /// + /// Partner-ID enrichment is best effort. When the caller already held proof + /// that the row exists — an `Add`-confirmed create or an earlier + /// authoritative read in the same request — a refresh that misses or fails + /// says nothing about existence, so the proof is kept and the skipped + /// update is logged. Write failures are *not* routed here: they are real + /// operation failures and stay [`EcKvSnapshot::Failed`] so callers can + /// report them. + fn keep_proven( + ec_id: &str, + downgraded: EcKvSnapshot, + proven: Option<&EcKvSnapshot>, + ) -> EcKvSnapshot { + match proven { + Some(proven) => { + log::warn!( + "snapshot partner upsert skipped for '{}': refresh was not authoritative; \ + keeping the confirmed row", + log_id(ec_id) + ); + proven.clone() + } + None => downgraded, + } + } + /// Atomically merges a partner ID into the existing entry. /// /// Uses CAS (generation markers) to avoid clobbering concurrent writes @@ -657,6 +867,177 @@ impl KvIdentityGraph { } } + /// Reports whether a row exists for `ec_id`, reading the primary data source. + /// + /// Point lookups on edge data stores are eventually consistent: a recently + /// created key can read absent at a POP that has not converged yet, so + /// `Ok(None)` from [`get`](Self::get) is *not* an absence proof. The list + /// API is the consistency-safe alternative — Fastly's KV list reads the + /// primary data source unless `eventual_consistency()` is requested — so an + /// empty prefix page proves absence where a stale point read cannot. + /// + /// EC IDs are fixed-width (`{64hex}.{6alnum}`), so listing with the full ID + /// as the prefix matches at most the key itself. A limit of 1 is enough: + /// only existence is in question, not the count. + /// + /// # Errors + /// + /// Returns [`TrustedServerError::KvStore`] on store open or list failure. + /// Callers must treat an error as "existence unknown" and fail closed — + /// never as absence. + pub fn key_exists_confirmed(&self, ec_id: &str) -> Result> { + Ok(self.store.count_keys_with_prefix(ec_id, 1)? > 0) + } + + /// Resolves a tombstone attempt whose point read reported the row absent. + /// + /// A proven-absent key is a no-op: there is nothing to withdraw, and a + /// forged cookie must not mint a row. A key that provably exists is + /// tombstoned unconditionally — no CAS generation is available after a + /// missed read, and a withdrawal must win over any concurrent write. An + /// existence check that itself fails leaves the withdrawal unresolved + /// rather than silently dropped. + fn tombstone_unproven_missing(&self, ec_id: &str, missing: EcKvSnapshot) -> EcKvSnapshot { + match self.key_exists_confirmed(ec_id) { + Ok(false) => missing, + Ok(true) => { + log::warn!( + "withdrawal tombstone for '{}': point read missed a row the store still \ + lists; writing an unconditional tombstone", + log_id(ec_id) + ); + let tombstone = KvEntry::tombstone(current_timestamp()); + match self.write_withdrawal_tombstone(ec_id) { + Ok(()) => EcKvSnapshot::Present { + ec_id: ec_id.to_owned(), + entry: Box::new(tombstone), + generation: None, + }, + Err(err) => { + log::warn!( + "unconditional withdrawal tombstone failed for '{}': {err:?}", + log_id(ec_id) + ); + EcKvSnapshot::Failed { + ec_id: ec_id.to_owned(), + } + } + } + } + Err(err) => { + log::warn!( + "withdrawal tombstone for '{}': existence check failed, cannot confirm \ + absence: {err:?}", + log_id(ec_id) + ); + EcKvSnapshot::Failed { + ec_id: ec_id.to_owned(), + } + } + } + } + + /// Writes a tombstone only when an existing row can be confirmed. + /// + /// Existing-key-only behavior is deliberate: a forged or expired `ts-ec` + /// cookie must not mint a row. But a *point read* cannot prove absence on + /// an eventually-consistent store, and dropping a withdrawal is worse than + /// a redundant read, so absence is established in two stages: + /// + /// 1. Any snapshot that is not a usable `Present` for this EC ID — a + /// publisher preload that read `Missing`, a read that `Failed`, or one + /// lacking a CAS generation — is re-read. On the publisher path that + /// re-read is separated from the preload by the full origin round trip, + /// which gives replication time to converge. + /// 2. A re-read that still reports the row absent is checked against + /// [`key_exists_confirmed`](Self::key_exists_confirmed), which reads the + /// primary data source. + /// + /// Resolving the initial snapshot happens outside the retry counter, so all + /// [`MAX_CAS_RETRIES`] iterations stay available for the tombstone write. + pub(crate) fn tombstone_existing_from_snapshot( + &self, + ec_id: &str, + snapshot: EcKvSnapshot, + ) -> EcKvSnapshot { + let mut current = match snapshot { + EcKvSnapshot::Present { + ec_id: ref snapshot_id, + generation: Some(_), + .. + } if snapshot_id == ec_id => snapshot, + _ => self.load_snapshot(ec_id), + }; + + for _attempt in 0..MAX_CAS_RETRIES { + let generation = match current { + EcKvSnapshot::Present { + ec_id: ref snapshot_id, + generation: Some(generation), + .. + } if snapshot_id == ec_id => generation, + // A missing row (including one that disappeared mid-retry) is + // only a no-op once absence is proven against the primary data + // source. + EcKvSnapshot::Missing { + ec_id: ref snapshot_id, + } if snapshot_id == ec_id => { + return self.tombstone_unproven_missing(ec_id, current); + } + // A refreshed read that failed (or any other unusable state) + // fails closed rather than silently dropping the withdrawal. + _ => { + return EcKvSnapshot::Failed { + ec_id: ec_id.to_owned(), + }; + } + }; + let tombstone = KvEntry::tombstone(current_timestamp()); + let Ok((body, meta_str)) = Self::serialize_entry(&tombstone, self.store_name()) else { + return EcKvSnapshot::Failed { + ec_id: ec_id.to_owned(), + }; + }; + match self.write_entry( + ec_id, + &body, + &meta_str, + TOMBSTONE_TTL, + EcKvWriteMode::IfGenerationMatch(generation), + ) { + Ok(EcKvWriteOutcome::Written) => { + return EcKvSnapshot::Present { + ec_id: ec_id.to_owned(), + entry: Box::new(tombstone), + generation: None, + }; + } + Ok(EcKvWriteOutcome::PreconditionFailed) => { + current = self.load_snapshot(ec_id); + } + Err(err) => { + log::warn!( + "conditional withdrawal tombstone failed for '{}': {err:?}", + log_id(ec_id) + ); + return EcKvSnapshot::Failed { + ec_id: ec_id.to_owned(), + }; + } + } + } + // Withdrawal enforcement lost every CAS race, so the row can still be + // live with consent granted while the browser cookie is cleared. That + // divergence is only visible to operators if it is logged here. + log::warn!( + "withdrawal tombstone for '{}': CAS conflict after {MAX_CAS_RETRIES} retries; the identity-graph row may still be live with consent granted", + log_id(ec_id) + ); + EcKvSnapshot::Failed { + ec_id: ec_id.to_owned(), + } + } + /// Counts the number of keys sharing the same EC hash prefix. /// /// Uses the platform KV list API with a prefix filter, limited to @@ -790,6 +1171,38 @@ impl KvIdentityGraph { store_name, )) } + + /// Test helper: a graph whose first `stale_lookups` point reads report the + /// key absent while the list API still sees it, mimicking an + /// eventually-consistent edge data store. + pub(crate) fn stale_lookup(store_name: impl Into, stale_lookups: u32) -> Self { + Self::new(super::kv_backend::test_support::StaleLookupEcKv::new( + store_name, + stale_lookups, + false, + )) + } + + /// Test helper: a graph that counts every point read through a shared + /// counter so tests can prove exactly how many reads a flow performs. + pub(crate) fn counting( + store_name: impl Into, + lookups: std::sync::Arc, + ) -> Self { + Self::new(super::kv_backend::test_support::CountingEcKv::new( + store_name, lookups, + )) + } + + /// Test helper: a graph whose point reads always miss and whose list API + /// errors, so absence can neither be observed nor proved. + pub(crate) fn unprovable_absence(store_name: impl Into) -> Self { + Self::new(super::kv_backend::test_support::StaleLookupEcKv::new( + store_name, + u32::MAX, + true, + )) + } } #[cfg(test)] @@ -810,6 +1223,28 @@ mod tests { assert!(ts > 0, "should return a nonzero timestamp"); } + #[test] + fn kv_span_accumulates_across_graph_operations() { + let timings = crate::request_timing::RequestTimings::new(); + let graph = KvIdentityGraph::new(crate::platform::TimedKvStore::new( + crate::ec::kv_backend::test_support::InMemoryEcKv::new("test-store"), + timings.clone(), + )); + + graph + .create("ec-1", &live_entry()) + .expect("should create entry through the timed store"); + graph + .get("ec-1") + .expect("should read the entry back through the timed store"); + + timings.mark_headers_ready(); + assert!( + timings.snapshot().kv_ms.is_some(), + "should accumulate Phase::EcKv across both graph operations, not just the last write" + ); + } + #[test] fn serialize_entry_produces_valid_json() { let entry = KvEntry::tombstone(1000); @@ -1200,6 +1635,34 @@ mod tests { ); } + #[test] + fn create_if_absent_reports_written_and_collision() { + let kv = KvIdentityGraph::in_memory("test_store"); + let ec_id = format!("{}.ABC123", "a".repeat(64)); + + assert_eq!( + kv.create_if_absent(&ec_id, &live_entry()) + .expect("should create absent entry"), + CreateIfAbsentOutcome::Written + ); + assert_eq!( + kv.create_if_absent(&ec_id, &live_entry()) + .expect("should report collision"), + CreateIfAbsentOutcome::AlreadyExists + ); + } + + #[test] + fn create_if_absent_propagates_store_error() { + let kv = KvIdentityGraph::failing("test_store"); + let ec_id = format!("{}.ABC123", "a".repeat(64)); + + assert!( + kv.create_if_absent(&ec_id, &live_entry()).is_err(), + "should preserve store failures instead of reporting a collision" + ); + } + #[test] fn create_or_revive_revives_tombstone() { let kv = KvIdentityGraph::in_memory("test_store"); @@ -1258,6 +1721,48 @@ mod tests { assert_eq!(result, UpsertResult::ConsentWithdrawn); } + #[test] + fn snapshot_bulk_upsert_returns_persisted_entry_without_stale_generation() { + let kv = KvIdentityGraph::in_memory("test_store"); + let ec_id = format!("{}.ABC123", "a".repeat(64)); + kv.create(&ec_id, &live_entry()).expect("should create"); + let snapshot = kv.load_snapshot(&ec_id); + let updates = [PartnerIdUpdate::new("ssp_x", "uid-1")]; + + let outcome = kv.upsert_partner_ids_from_snapshot(&ec_id, &updates, snapshot); + + let entry = outcome + .entry_for(&ec_id) + .expect("should retain persisted entry"); + assert_eq!( + entry.ids.get("ssp_x").map(|id| id.uid.as_str()), + Some("uid-1") + ); + assert_eq!( + outcome.generation_for(&ec_id), + None, + "backend does not return the post-write generation" + ); + } + + #[test] + fn snapshot_bulk_upsert_does_not_create_missing_root() { + let kv = KvIdentityGraph::in_memory("test_store"); + let ec_id = format!("{}.ABC123", "a".repeat(64)); + let updates = [PartnerIdUpdate::new("ssp_x", "uid-1")]; + + let outcome = kv.upsert_partner_ids_from_snapshot( + &ec_id, + &updates, + EcKvSnapshot::Missing { + ec_id: ec_id.clone(), + }, + ); + + assert!(matches!(outcome, EcKvSnapshot::Missing { .. })); + assert!(kv.get(&ec_id).expect("should read store").is_none()); + } + #[test] fn write_withdrawal_tombstone_overwrites_live_entry() { let kv = KvIdentityGraph::in_memory("test_store"); @@ -1273,4 +1778,730 @@ mod tests { .expect("should find tombstone entry"); assert!(!loaded.consent.ok, "should be withdrawn after tombstone"); } + + #[test] + fn tombstone_existing_from_snapshot_never_creates_missing_key() { + let kv = KvIdentityGraph::in_memory("test_store"); + let ec_id = format!("{}.ABC123", "a".repeat(64)); + let snapshot = EcKvSnapshot::Missing { + ec_id: ec_id.clone(), + }; + + let outcome = kv.tombstone_existing_from_snapshot(&ec_id, snapshot); + + assert!(matches!(outcome, EcKvSnapshot::Missing { .. })); + assert!( + kv.get(&ec_id).expect("should read store").is_none(), + "withdrawal must not create a tombstone for an absent key" + ); + } + + #[test] + fn tombstone_existing_from_snapshot_uses_existing_generation() { + let kv = KvIdentityGraph::in_memory("test_store"); + let ec_id = format!("{}.ABC123", "a".repeat(64)); + kv.create(&ec_id, &live_entry()).expect("should create"); + let snapshot = kv.load_snapshot(&ec_id); + + let outcome = kv.tombstone_existing_from_snapshot(&ec_id, snapshot); + + assert!( + outcome + .entry_for(&ec_id) + .is_some_and(|entry| !entry.consent.ok), + "should return the persisted tombstone" + ); + let (stored, _) = kv + .get(&ec_id) + .expect("should read store") + .expect("should preserve existing key"); + assert!(!stored.consent.ok, "should persist withdrawal state"); + } + + // ----------------------------------------------------------------------- + // Snapshot-aware mutation stores and tests + // ----------------------------------------------------------------------- + + /// [`EcKvStore`] whose reads succeed but every write fails, simulating a + /// store that becomes unwritable mid-request. + struct WriteFailingEcKv { + inner: InMemoryEcKv, + } + + impl WriteFailingEcKv { + fn new() -> Self { + Self { + inner: InMemoryEcKv::new("write-failing-store"), + } + } + } + + impl EcKvStore for WriteFailingEcKv { + fn store_name(&self) -> &str { + self.inner.store_name() + } + fn lookup(&self, key: &str) -> Result, Report> { + self.inner.lookup(key) + } + fn insert( + &self, + _key: &str, + _write: EcKvWrite<'_>, + ) -> Result> { + Err(Report::new(TrustedServerError::KvStore { + store_name: self.inner.store_name().to_owned(), + message: "write failing test store".to_owned(), + })) + } + fn count_keys_with_prefix( + &self, + prefix: &str, + limit: u32, + ) -> Result> { + self.inner.count_keys_with_prefix(prefix, limit) + } + fn delete(&self, key: &str) -> Result<(), Report> { + self.inner.delete(key) + } + } + + /// [`EcKvStore`] wrapper whose first CAS write both fails the precondition + /// and deletes the key, simulating a concurrent withdrawal that removes the + /// row between this writer's read and its write. + struct DisappearOnConflictEcKv { + inner: InMemoryEcKv, + conflicts_remaining: std::sync::Mutex, + } + + impl DisappearOnConflictEcKv { + fn new(conflicts: u32) -> Self { + Self { + inner: InMemoryEcKv::new("disappear-store"), + conflicts_remaining: std::sync::Mutex::new(conflicts), + } + } + fn seed_live(&self, ec_id: &str) { + let (body, meta) = + KvIdentityGraph::serialize_entry(&live_entry(), self.inner.store_name()) + .expect("should serialize seeded entry"); + self.inner + .insert( + ec_id, + EcKvWrite { + body: &body, + metadata: &meta, + ttl: ENTRY_TTL, + mode: EcKvWriteMode::Add, + }, + ) + .expect("should seed live entry"); + } + } + + impl EcKvStore for DisappearOnConflictEcKv { + fn store_name(&self) -> &str { + self.inner.store_name() + } + fn lookup(&self, key: &str) -> Result, Report> { + self.inner.lookup(key) + } + fn insert( + &self, + key: &str, + write: EcKvWrite<'_>, + ) -> Result> { + if matches!(write.mode, EcKvWriteMode::IfGenerationMatch(_)) { + let mut remaining = self + .conflicts_remaining + .lock() + .expect("should lock conflict counter"); + if *remaining > 0 { + *remaining -= 1; + self.inner.delete(key).expect("should delete on conflict"); + return Ok(EcKvWriteOutcome::PreconditionFailed); + } + } + self.inner.insert(key, write) + } + fn count_keys_with_prefix( + &self, + prefix: &str, + limit: u32, + ) -> Result> { + self.inner.count_keys_with_prefix(prefix, limit) + } + fn delete(&self, key: &str) -> Result<(), Report> { + self.inner.delete(key) + } + } + + fn snapshot_ec_id() -> String { + format!("{}.ABC123", "a".repeat(64)) + } + + #[test] + fn snapshot_upsert_with_generation_writes_without_reading() { + let lookups = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let graph = KvIdentityGraph::counting("counting-store", lookups.clone()); + let ec_id = snapshot_ec_id(); + graph.create(&ec_id, &live_entry()).expect("should seed"); + let snapshot = EcKvSnapshot::Present { + ec_id: ec_id.clone(), + entry: Box::new(live_entry()), + generation: Some(1), + }; + let updates = [PartnerIdUpdate::new("ssp_x", "uid-1")]; + + let outcome = graph.upsert_partner_ids_from_snapshot(&ec_id, &updates, snapshot); + + assert_eq!( + lookups.load(std::sync::atomic::Ordering::Relaxed), + 0, + "a usable generation must avoid the initial read" + ); + assert_eq!( + outcome + .entry_for(&ec_id) + .and_then(|entry| entry.ids.get("ssp_x")) + .map(|id| id.uid.as_str()), + Some("uid-1") + ); + assert_eq!(outcome.generation_for(&ec_id), None); + } + + #[test] + fn snapshot_upsert_unchanged_updates_preserve_generation() { + let kv = KvIdentityGraph::in_memory("test_store"); + let ec_id = snapshot_ec_id(); + let mut seeded = live_entry(); + apply_partner_id_updates(&mut seeded, &[PartnerIdUpdate::new("ssp_x", "uid-1")]); + kv.create(&ec_id, &seeded).expect("should seed"); + let snapshot = kv.load_snapshot(&ec_id); + assert_eq!(snapshot.generation_for(&ec_id), Some(1)); + + let outcome = kv.upsert_partner_ids_from_snapshot( + &ec_id, + &[PartnerIdUpdate::new("ssp_x", "uid-1")], + snapshot, + ); + + assert_eq!( + outcome.generation_for(&ec_id), + Some(1), + "an unchanged merge preserves the usable generation and performs no write" + ); + } + + #[test] + fn snapshot_upsert_refreshes_unavailable_generation_exactly_once() { + let lookups = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let graph = KvIdentityGraph::counting("counting-store", lookups.clone()); + let ec_id = snapshot_ec_id(); + graph.create(&ec_id, &live_entry()).expect("should seed"); + // Finalize-written style snapshot: entry known, generation unavailable. + let snapshot = EcKvSnapshot::Present { + ec_id: ec_id.clone(), + entry: Box::new(live_entry()), + generation: None, + }; + let updates = [PartnerIdUpdate::new("ssp_x", "uid-1")]; + + let outcome = graph.upsert_partner_ids_from_snapshot(&ec_id, &updates, snapshot); + + assert_eq!( + lookups.load(std::sync::atomic::Ordering::Relaxed), + 1, + "an unavailable generation refreshes exactly once before CAS" + ); + assert!( + outcome + .entry_for(&ec_id) + .is_some_and(|e| e.ids.contains_key("ssp_x")) + ); + } + + #[test] + fn snapshot_upsert_gen_unavailable_survives_four_conflicts_then_writes() { + // A generation-unavailable snapshot (finalize-written style) refreshes + // once to obtain a usable generation. That refresh must not consume a + // CAS attempt, so all five write attempts remain: four conflicts + // followed by a successful fifth write still persist the update. + let graph = KvIdentityGraph::new(ConflictInjectingEcKv::new(4, false)); + let ec_id = snapshot_ec_id(); + graph.create(&ec_id, &live_entry()).expect("should seed"); + let snapshot = EcKvSnapshot::Present { + ec_id: ec_id.clone(), + entry: Box::new(live_entry()), + generation: None, + }; + let updates = [PartnerIdUpdate::new("ssp_x", "uid-1")]; + + let outcome = graph.upsert_partner_ids_from_snapshot(&ec_id, &updates, snapshot); + + assert_eq!( + outcome + .entry_for(&ec_id) + .and_then(|entry| entry.ids.get("ssp_x")) + .map(|id| id.uid.as_str()), + Some("uid-1"), + "the fifth CAS attempt must still succeed after a refresh and four conflicts" + ); + } + + #[test] + fn snapshot_upsert_cas_conflict_remerges_concurrent_data() { + let graph = KvIdentityGraph::new(ConflictInjectingEcKv::new(1, true)); + let ec_id = snapshot_ec_id(); + graph.create(&ec_id, &live_entry()).expect("should seed"); + let snapshot = graph.load_snapshot(&ec_id); + let updates = [PartnerIdUpdate::new("ssp_x", "uid-1")]; + + let outcome = graph.upsert_partner_ids_from_snapshot(&ec_id, &updates, snapshot); + + let entry = outcome + .entry_for(&ec_id) + .expect("should persist re-merged entry"); + assert_eq!( + entry.ids.get("ssp_x").map(|id| id.uid.as_str()), + Some("uid-1"), + "conflict must re-merge our update onto the concurrently revived row" + ); + assert!(entry.consent.ok, "concurrent revive keeps the row live"); + } + + #[test] + fn snapshot_upsert_revalidates_transient_missing_and_persists() { + // An eventually-consistent point read earlier in the request missed a + // row that exists. Named routes such as `/auction` never run orphan + // recovery, so this refresh is the request's only chance to persist the + // collected partner IDs. + let kv = KvIdentityGraph::in_memory("test_store"); + let ec_id = snapshot_ec_id(); + kv.create(&ec_id, &live_entry()).expect("should seed live"); + let updates = [PartnerIdUpdate::new("ssp_x", "uid-1")]; + + let outcome = kv.upsert_partner_ids_from_snapshot( + &ec_id, + &updates, + EcKvSnapshot::Missing { + ec_id: ec_id.clone(), + }, + ); + + assert_eq!( + outcome + .entry_for(&ec_id) + .and_then(|entry| entry.ids.get("ssp_x").map(|id| id.uid.clone())), + Some("uid-1".to_owned()), + "a stale miss must be revalidated before the updates are dropped" + ); + let (stored, _) = kv + .get(&ec_id) + .expect("should read store") + .expect("row should remain"); + assert_eq!( + stored.ids.get("ssp_x").map(|id| id.uid.as_str()), + Some("uid-1"), + "the revalidated update must reach the store" + ); + } + + #[test] + fn snapshot_upsert_confirmed_missing_still_never_creates() { + // Revalidation only changes what a *stale* miss does. A row that is + // genuinely absent on the refresh must stay absent. + let kv = KvIdentityGraph::in_memory("test_store"); + let ec_id = snapshot_ec_id(); + let updates = [PartnerIdUpdate::new("ssp_x", "uid-1")]; + + let outcome = kv.upsert_partner_ids_from_snapshot( + &ec_id, + &updates, + EcKvSnapshot::Missing { + ec_id: ec_id.clone(), + }, + ); + + assert!( + matches!(outcome, EcKvSnapshot::Missing { .. }), + "a confirmed miss must stay missing" + ); + assert!( + kv.get(&ec_id).expect("should read store").is_none(), + "must not create a root entry for a missing key" + ); + } + + #[test] + fn snapshot_upsert_failed_snapshot_is_not_revalidated() { + // A lookup that already errored is not retried on the hot path, even + // though the row exists. + let kv = KvIdentityGraph::in_memory("test_store"); + let ec_id = snapshot_ec_id(); + kv.create(&ec_id, &live_entry()).expect("should seed live"); + let updates = [PartnerIdUpdate::new("ssp_x", "uid-1")]; + + let outcome = kv.upsert_partner_ids_from_snapshot( + &ec_id, + &updates, + EcKvSnapshot::Failed { + ec_id: ec_id.clone(), + }, + ); + + assert!( + matches!(outcome, EcKvSnapshot::Failed { .. }), + "a failed lookup must not be retried by partner enrichment" + ); + } + + #[test] + fn snapshot_upsert_rejects_tombstone() { + let kv = KvIdentityGraph::in_memory("test_store"); + let ec_id = snapshot_ec_id(); + kv.create(&ec_id, &KvEntry::tombstone(1000)) + .expect("should seed tombstone"); + let snapshot = kv.load_snapshot(&ec_id); + let updates = [PartnerIdUpdate::new("ssp_x", "uid-1")]; + + let outcome = kv.upsert_partner_ids_from_snapshot(&ec_id, &updates, snapshot); + + assert!( + outcome + .entry_for(&ec_id) + .is_some_and(|entry| entry.ids.is_empty()), + "a tombstone must reject partner enrichment" + ); + let (stored, _) = kv + .get(&ec_id) + .expect("should read store") + .expect("tombstone should remain"); + assert!(stored.ids.is_empty(), "no update should reach the store"); + } + + #[test] + fn snapshot_upsert_store_failure_returns_failed_not_request_local() { + let graph = KvIdentityGraph::new(WriteFailingEcKv::new()); + let ec_id = snapshot_ec_id(); + let snapshot = EcKvSnapshot::Present { + ec_id: ec_id.clone(), + entry: Box::new(live_entry()), + generation: Some(1), + }; + let updates = [PartnerIdUpdate::new("ssp_x", "uid-1")]; + + let outcome = graph.upsert_partner_ids_from_snapshot(&ec_id, &updates, snapshot); + + assert!( + matches!(outcome, EcKvSnapshot::Failed { .. }), + "a store write failure must not claim request-local IDs were persisted" + ); + } + + #[test] + fn tombstone_existing_from_snapshot_retries_cas_conflict() { + let graph = KvIdentityGraph::new(ConflictInjectingEcKv::new(1, false)); + let ec_id = snapshot_ec_id(); + graph.create(&ec_id, &live_entry()).expect("should seed"); + let snapshot = graph.load_snapshot(&ec_id); + + let outcome = graph.tombstone_existing_from_snapshot(&ec_id, snapshot); + + assert!( + outcome + .entry_for(&ec_id) + .is_some_and(|entry| !entry.consent.ok), + "should retry the conflict and persist the tombstone" + ); + } + + #[test] + fn tombstone_gen_unavailable_survives_four_conflicts_then_writes() { + // A generation-unavailable snapshot refreshes once before its CAS. That + // refresh must not spend a CAS attempt, so a withdrawal tombstone still + // persists after four conflicts and a successful fifth write. + let graph = KvIdentityGraph::new(ConflictInjectingEcKv::new(4, false)); + let ec_id = snapshot_ec_id(); + graph.create(&ec_id, &live_entry()).expect("should seed"); + let snapshot = EcKvSnapshot::Present { + ec_id: ec_id.clone(), + entry: Box::new(live_entry()), + generation: None, + }; + + let outcome = graph.tombstone_existing_from_snapshot(&ec_id, snapshot); + + assert!( + outcome + .entry_for(&ec_id) + .is_some_and(|entry| !entry.consent.ok), + "the fifth CAS attempt must persist the tombstone after a refresh and four conflicts" + ); + } + + #[test] + fn tombstone_existing_from_snapshot_returns_failed_after_cas_exhaustion() { + // Every CAS attempt loses its race, so the row stays live with consent + // granted while the browser cookie is already cleared. The caller must + // see a failure it can report rather than a silent no-op. + let graph = KvIdentityGraph::new(ConflictInjectingEcKv::new(MAX_CAS_RETRIES, false)); + let ec_id = snapshot_ec_id(); + graph.create(&ec_id, &live_entry()).expect("should seed"); + let snapshot = graph.load_snapshot(&ec_id); + + let outcome = graph.tombstone_existing_from_snapshot(&ec_id, snapshot); + + assert!( + matches!(outcome, EcKvSnapshot::Failed { .. }), + "CAS exhaustion must report a failed withdrawal" + ); + let (stored, _) = graph + .get(&ec_id) + .expect("should read store") + .expect("row should remain"); + assert!( + stored.consent.ok, + "the row is still live, which is exactly why the failure must be reported" + ); + } + + #[test] + fn tombstone_existing_from_snapshot_store_failure_returns_failed() { + let graph = KvIdentityGraph::new(WriteFailingEcKv::new()); + let ec_id = snapshot_ec_id(); + let snapshot = EcKvSnapshot::Present { + ec_id: ec_id.clone(), + entry: Box::new(live_entry()), + generation: Some(1), + }; + + let outcome = graph.tombstone_existing_from_snapshot(&ec_id, snapshot); + + assert!(matches!(outcome, EcKvSnapshot::Failed { .. })); + } + + #[test] + fn tombstone_existing_from_snapshot_noop_when_row_disappears_on_retry() { + let store = DisappearOnConflictEcKv::new(1); + store.seed_live(&snapshot_ec_id()); + let graph = KvIdentityGraph::new(store); + let ec_id = snapshot_ec_id(); + let snapshot = graph.load_snapshot(&ec_id); + + let outcome = graph.tombstone_existing_from_snapshot(&ec_id, snapshot); + + assert!( + matches!(outcome, EcKvSnapshot::Missing { .. }), + "a row that disappears during retry becomes a no-op" + ); + assert!( + graph.get(&ec_id).expect("should read store").is_none(), + "must not recreate the disappeared key" + ); + } + + #[test] + fn tombstone_existing_from_snapshot_reretries_failed_snapshot_read() { + // A prior request-scoped read failed, so the snapshot is `Failed`. A + // withdrawal must not silently drop consent removal: re-read the store + // and tombstone the row if it is authoritatively present. + let kv = KvIdentityGraph::in_memory("test_store"); + let ec_id = snapshot_ec_id(); + kv.create(&ec_id, &live_entry()).expect("should seed live"); + + let outcome = kv.tombstone_existing_from_snapshot( + &ec_id, + EcKvSnapshot::Failed { + ec_id: ec_id.clone(), + }, + ); + + assert!( + outcome + .entry_for(&ec_id) + .is_some_and(|entry| !entry.consent.ok), + "a failed snapshot must re-read and persist the tombstone" + ); + let (stored, _) = kv + .get(&ec_id) + .expect("should read store") + .expect("should preserve existing key"); + assert!(!stored.consent.ok, "withdrawal must reach the store"); + } + // ----------------------------------------------------------------------- + // Eventual-consistency guards + // ----------------------------------------------------------------------- + + #[test] + fn key_exists_confirmed_distinguishes_absence_from_a_stale_point_read() { + let graph = KvIdentityGraph::stale_lookup("stale-store", 1); + let ec_id = snapshot_ec_id(); + graph + .create(&ec_id, &live_entry()) + .expect("should seed live"); + + assert!( + graph.get(&ec_id).expect("should read store").is_none(), + "the first point read is stale by construction" + ); + assert!( + graph + .key_exists_confirmed(&ec_id) + .expect("should list the store"), + "the list API must still see a row the point read missed" + ); + + let absent = KvIdentityGraph::in_memory("empty-store"); + assert!( + !absent + .key_exists_confirmed(&ec_id) + .expect("should list the store"), + "an empty store must prove absence" + ); + } + + #[test] + fn snapshot_upsert_keeps_add_confirmed_present_when_refresh_misses() { + // `generate_if_needed` records a successful `Add` as `Present` without a + // generation. EID ingestion refreshes that snapshot to obtain one; on an + // eventually-consistent store the refresh can miss. Enrichment is best + // effort, so the miss must not retract the confirmed create — otherwise + // finalization suppresses the `ts-ec` cookie for a root that was written. + let graph = KvIdentityGraph::in_memory("empty-store"); + let ec_id = snapshot_ec_id(); + let add_confirmed = EcKvSnapshot::Present { + ec_id: ec_id.clone(), + entry: Box::new(live_entry()), + generation: None, + }; + let updates = [PartnerIdUpdate::new("ssp_x", "uid-1")]; + + let outcome = graph.upsert_partner_ids_from_snapshot(&ec_id, &updates, add_confirmed); + + assert!( + outcome.entry_for(&ec_id).is_some(), + "an Add-confirmed row must survive a non-authoritative refresh miss" + ); + assert!( + graph.get(&ec_id).expect("should read store").is_none(), + "a missed refresh must not create or overwrite a root" + ); + } + + #[test] + fn snapshot_upsert_keeps_add_confirmed_present_when_refresh_fails() { + let graph = KvIdentityGraph::failing("failing-store"); + let ec_id = snapshot_ec_id(); + let add_confirmed = EcKvSnapshot::Present { + ec_id: ec_id.clone(), + entry: Box::new(live_entry()), + generation: None, + }; + let updates = [PartnerIdUpdate::new("ssp_x", "uid-1")]; + + let outcome = graph.upsert_partner_ids_from_snapshot(&ec_id, &updates, add_confirmed); + + assert!( + outcome.entry_for(&ec_id).is_some(), + "a read failure is not evidence of absence and must not retract the create" + ); + } + + #[test] + fn snapshot_upsert_without_proof_still_reports_a_refresh_miss() { + // No prior proof of existence: a `NotRead` snapshot that refreshes into + // a miss must stay `Missing` so finalization can run orphan recovery. + let graph = KvIdentityGraph::in_memory("empty-store"); + let ec_id = snapshot_ec_id(); + let updates = [PartnerIdUpdate::new("ssp_x", "uid-1")]; + + let outcome = + graph.upsert_partner_ids_from_snapshot(&ec_id, &updates, EcKvSnapshot::NotRead); + + assert!( + matches!(outcome, EcKvSnapshot::Missing { .. }), + "an unproven refresh miss must remain a miss" + ); + } + + #[test] + fn tombstone_revalidates_preloaded_missing_and_writes_when_row_is_present() { + // The publisher preload read `Missing` at a POP that had not converged. + // Finalization runs after the origin round trip, so the re-read sees the + // row and the withdrawal must reach the store. + let kv = KvIdentityGraph::in_memory("test_store"); + let ec_id = snapshot_ec_id(); + kv.create(&ec_id, &live_entry()).expect("should seed live"); + + let outcome = kv.tombstone_existing_from_snapshot( + &ec_id, + EcKvSnapshot::Missing { + ec_id: ec_id.clone(), + }, + ); + + assert!( + outcome + .entry_for(&ec_id) + .is_some_and(|entry| !entry.consent.ok), + "a stale preloaded miss must not drop the withdrawal" + ); + let (stored, _) = kv + .get(&ec_id) + .expect("should read store") + .expect("should preserve existing key"); + assert!(!stored.consent.ok, "withdrawal must reach the store"); + } + + #[test] + fn tombstone_writes_unconditionally_when_both_point_reads_miss_a_listed_row() { + // Both the preload and the confirming re-read are stale. A point read + // cannot prove absence, so the list API decides: the row exists, and a + // withdrawal must win even without a CAS generation. + let kv = KvIdentityGraph::stale_lookup("stale-store", 1); + let ec_id = snapshot_ec_id(); + kv.create(&ec_id, &live_entry()).expect("should seed live"); + + let outcome = kv.tombstone_existing_from_snapshot( + &ec_id, + EcKvSnapshot::Missing { + ec_id: ec_id.clone(), + }, + ); + + assert!( + outcome + .entry_for(&ec_id) + .is_some_and(|entry| !entry.consent.ok), + "a listed row must be tombstoned even when point reads miss it" + ); + let (stored, _) = kv + .get(&ec_id) + .expect("should read store") + .expect("should preserve existing key"); + assert!( + !stored.consent.ok, + "the live row must not keep consent.ok after an explicit withdrawal" + ); + } + + #[test] + fn tombstone_fails_closed_when_the_existence_check_fails() { + // A forged-cookie no-op requires proof of absence. When the list itself + // fails, the withdrawal is left unresolved rather than silently dropped. + let kv = KvIdentityGraph::failing("failing-store"); + let ec_id = snapshot_ec_id(); + + let outcome = kv.tombstone_existing_from_snapshot( + &ec_id, + EcKvSnapshot::Missing { + ec_id: ec_id.clone(), + }, + ); + + assert!( + matches!(outcome, EcKvSnapshot::Failed { .. }), + "an unprovable absence must not report a completed withdrawal" + ); + } } diff --git a/crates/trusted-server-core/src/ec/kv_backend.rs b/crates/trusted-server-core/src/ec/kv_backend.rs index 60f938291..5a5c3766d 100644 --- a/crates/trusted-server-core/src/ec/kv_backend.rs +++ b/crates/trusted-server-core/src/ec/kv_backend.rs @@ -122,6 +122,126 @@ pub(crate) mod test_support { use super::*; + /// [`EcKvStore`] wrapper that counts `lookup` calls through a shared counter + /// so tests can prove exactly how many reads a flow performs. + pub(crate) struct CountingEcKv { + inner: InMemoryEcKv, + lookups: std::sync::Arc, + } + + impl CountingEcKv { + pub(crate) fn new( + name: impl Into, + lookups: std::sync::Arc, + ) -> Self { + Self { + inner: InMemoryEcKv::new(name), + lookups, + } + } + } + + impl EcKvStore for CountingEcKv { + fn store_name(&self) -> &str { + self.inner.store_name() + } + + fn lookup(&self, key: &str) -> Result, Report> { + self.lookups + .fetch_add(1, std::sync::atomic::Ordering::Relaxed); + self.inner.lookup(key) + } + + fn insert( + &self, + key: &str, + write: EcKvWrite<'_>, + ) -> Result> { + self.inner.insert(key, write) + } + + fn count_keys_with_prefix( + &self, + prefix: &str, + limit: u32, + ) -> Result> { + self.inner.count_keys_with_prefix(prefix, limit) + } + + fn delete(&self, key: &str) -> Result<(), Report> { + self.inner.delete(key) + } + } + + /// [`EcKvStore`] wrapper that models an eventually-consistent point read. + /// + /// The first `stale_lookups` calls to [`EcKvStore::lookup`] report the key + /// absent while [`EcKvStore::count_keys_with_prefix`] — the list API, which + /// reads the primary data source — still sees it. Writes reach the inner + /// store, so a test can assert what actually persisted. + /// + /// With `list_fails` set, the list API errors instead, modelling a store + /// that can neither find the key nor prove it absent. + pub(crate) struct StaleLookupEcKv { + inner: InMemoryEcKv, + stale_lookups_remaining: Mutex, + list_fails: bool, + } + + impl StaleLookupEcKv { + pub(crate) fn new(name: impl Into, stale_lookups: u32, list_fails: bool) -> Self { + Self { + inner: InMemoryEcKv::new(name), + stale_lookups_remaining: Mutex::new(stale_lookups), + list_fails, + } + } + } + + impl EcKvStore for StaleLookupEcKv { + fn store_name(&self) -> &str { + self.inner.store_name() + } + + fn lookup(&self, key: &str) -> Result, Report> { + let mut remaining = self + .stale_lookups_remaining + .lock() + .expect("should lock stale-lookup counter"); + if *remaining > 0 { + *remaining -= 1; + return Ok(None); + } + self.inner.lookup(key) + } + + fn insert( + &self, + key: &str, + write: EcKvWrite<'_>, + ) -> Result> { + self.inner.insert(key, write) + } + + fn count_keys_with_prefix( + &self, + prefix: &str, + limit: u32, + ) -> Result> { + if self.list_fails { + return Err(Report::new(TrustedServerError::KvStore { + store_name: self.inner.store_name().to_owned(), + message: "list unavailable".to_owned(), + })); + } + self.inner.count_keys_with_prefix(prefix, limit) + } + + fn delete(&self, key: &str) -> Result<(), Report> { + self.inner.delete(key) + } + } + /// In-memory [`EcKvStore`] with generation tracking for CAS tests. pub(crate) struct InMemoryEcKv { name: String, diff --git a/crates/trusted-server-core/src/ec/mod.rs b/crates/trusted-server-core/src/ec/mod.rs index 840ce90d3..6bb1b54d8 100644 --- a/crates/trusted-server-core/src/ec/mod.rs +++ b/crates/trusted-server-core/src/ec/mod.rs @@ -75,9 +75,71 @@ use crate::platform::RuntimeServices; use crate::settings::Settings; use device::DeviceSignals; -use self::kv::KvIdentityGraph; +use self::kv::{CreateIfAbsentOutcome, KvIdentityGraph}; use self::kv_types::KvEntry; +/// Request-scoped view of one EC identity-graph lookup. +/// +/// The state distinguishes an authoritative miss from a store failure and +/// binds persisted entry data to the EC ID that was actually read or written. +#[derive(Debug, Clone, Default, PartialEq)] +pub enum EcKvSnapshot { + /// No identity-graph lookup has been attempted for this request. + #[default] + NotRead, + /// The store authoritatively reported that this EC ID does not exist. + Missing { ec_id: String }, + /// Persisted entry data, optionally with a generation usable for CAS. + Present { + ec_id: String, + entry: Box, + generation: Option, + }, + /// The lookup failed, so absence is not authoritative. + Failed { ec_id: String }, +} + +impl EcKvSnapshot { + /// Returns whether this state was produced for `ec_id`. + #[must_use] + pub fn belongs_to(&self, ec_id: &str) -> bool { + match self { + Self::NotRead => false, + Self::Missing { ec_id: snapshot_id } + | Self::Present { + ec_id: snapshot_id, .. + } + | Self::Failed { ec_id: snapshot_id } => snapshot_id == ec_id, + } + } + + /// Returns the persisted entry only when the snapshot belongs to `ec_id`. + #[must_use] + pub fn entry_for(&self, ec_id: &str) -> Option<&KvEntry> { + match self { + Self::Present { + ec_id: snapshot_id, + entry, + .. + } if snapshot_id == ec_id => Some(entry.as_ref()), + _ => None, + } + } + + /// Returns a usable CAS generation only when the snapshot belongs to `ec_id`. + #[must_use] + pub fn generation_for(&self, ec_id: &str) -> Option { + match self { + Self::Present { + ec_id: snapshot_id, + generation, + .. + } if snapshot_id == ec_id => *generation, + _ => None, + } + } +} + pub use generation::{ ec_hash, generate_ec_id, is_valid_ec_hash, is_valid_ec_id, normalize_ec_id_for_kv, }; @@ -161,6 +223,10 @@ pub struct EcContext { /// Set via [`EcContext::set_device_signals`] before /// [`EcContext::generate_if_needed`] is called. device_signals: Option, + /// Request-scoped persisted identity-graph state for the active EC ID. + kv_snapshot: EcKvSnapshot, + /// Whether this request may rotate an orphaned EC identity. + recovery_eligible: bool, } impl EcContext { @@ -240,6 +306,8 @@ impl EcContext { client_ip, geo_info: geo_info.cloned(), device_signals: None, + kv_snapshot: EcKvSnapshot::NotRead, + recovery_eligible: false, }) } @@ -279,12 +347,10 @@ impl EcContext { }) })?; - let ec_id = generation::generate_ec_id(settings, client_ip)?; - log::info!("Generated new EC ID: {}", log_id(&ec_id)); - self.ec_value = Some(ec_id); - self.ec_generated = true; - - if let (Some(graph), Some(ec_value)) = (kv, self.ec_value.as_deref()) { + const MAX_CREATE_ATTEMPTS: usize = 5; + for attempt in 0..MAX_CREATE_ATTEMPTS { + let ec_id = generation::generate_ec_id(settings, client_ip)?; + log::info!("Generated new EC ID: {}", log_id(&ec_id)); let now = current_timestamp(); let mut entry = KvEntry::new( &self.consent, @@ -297,20 +363,45 @@ impl EcContext { .as_ref() .map(DeviceSignals::to_kv_device); - if let Err(err) = graph.create_or_revive(ec_value, &entry) { - log::error!( - "Failed to create or revive EC entry for id '{}' after generation: {err:?}", - log_id(ec_value), - ); - self.ec_value = None; - self.ec_generated = false; - return Err(err.change_context(TrustedServerError::EdgeCookie { - message: "Failed to persist generated EC ID to KV identity graph".to_string(), - })); + if let Some(graph) = kv { + match graph.create_if_absent(&ec_id, &entry) { + Ok(CreateIfAbsentOutcome::Written) => { + self.kv_snapshot = EcKvSnapshot::Present { + ec_id: ec_id.clone(), + entry: Box::new(entry), + generation: None, + }; + } + Ok(CreateIfAbsentOutcome::AlreadyExists) => { + log::warn!( + "Generated EC ID collision on attempt {}/{MAX_CREATE_ATTEMPTS}", + attempt + 1 + ); + continue; + } + Err(err) => { + log::error!( + "Failed to create EC entry for id '{}' after generation: {err:?}", + log_id(&ec_id), + ); + return Err(err.change_context(TrustedServerError::EdgeCookie { + message: "Failed to persist generated EC ID to KV identity graph" + .to_string(), + })); + } + } } + + self.ec_value = Some(ec_id); + self.ec_generated = true; + return Ok(()); } - Ok(()) + Err(Report::new(TrustedServerError::EdgeCookie { + message: format!( + "Failed to allocate a unique EC ID after {MAX_CREATE_ATTEMPTS} attempts" + ), + })) } /// Returns the EC ID value, if present (either from request or generated). @@ -383,6 +474,35 @@ impl EcContext { self.geo_info.as_ref() } + /// Returns the request-scoped identity-graph snapshot. + #[must_use] + pub fn kv_snapshot(&self) -> &EcKvSnapshot { + &self.kv_snapshot + } + + /// Replaces the request-scoped identity-graph snapshot. + pub fn set_kv_snapshot(&mut self, snapshot: EcKvSnapshot) { + self.kv_snapshot = snapshot; + } + + /// Marks a real-browser document navigation as eligible for orphan recovery. + pub fn set_recovery_eligible(&mut self, eligible: bool) { + self.recovery_eligible = eligible; + } + + /// Returns whether orphan recovery is allowed for this request. + #[must_use] + pub fn recovery_eligible(&self) -> bool { + self.recovery_eligible + } + + /// Replaces an orphaned active ID after its new backing row is persisted. + pub(crate) fn replace_with_generated(&mut self, ec_id: String, snapshot: EcKvSnapshot) { + self.ec_value = Some(ec_id); + self.ec_generated = true; + self.kv_snapshot = snapshot; + } + /// Returns whether EC creation is permitted by consent for this request. #[must_use] pub fn ec_allowed(&self) -> bool { @@ -428,6 +548,8 @@ impl EcContext { client_ip: None, geo_info: None, device_signals: None, + kv_snapshot: EcKvSnapshot::NotRead, + recovery_eligible: false, } } @@ -448,6 +570,8 @@ impl EcContext { client_ip, geo_info: None, device_signals: None, + kv_snapshot: EcKvSnapshot::NotRead, + recovery_eligible: false, } } @@ -471,6 +595,8 @@ impl EcContext { client_ip: None, geo_info: None, device_signals: None, + kv_snapshot: EcKvSnapshot::NotRead, + recovery_eligible: false, } } } @@ -494,9 +620,146 @@ pub(crate) fn current_timestamp() -> u64 { #[cfg(test)] mod tests { use super::*; + use crate::consent::jurisdiction::Jurisdiction; + use crate::consent::types::{ConsentContext, ConsentSource}; + use crate::ec::kv_backend::test_support::InMemoryEcKv; + use crate::ec::kv_backend::{ + EcKvLookup, EcKvStore, EcKvWrite, EcKvWriteMode, EcKvWriteOutcome, + }; use crate::platform::test_support::noop_services; use crate::test_support::tests::create_test_settings; + /// [`EcKvStore`] wrapper whose first `collisions` `Add` writes report a + /// precondition failure, forcing generation to retry with a fresh suffix. + struct AddCollidingEcKv { + inner: InMemoryEcKv, + collisions_remaining: std::sync::Mutex, + } + + impl AddCollidingEcKv { + fn new(collisions: u32) -> Self { + Self { + inner: InMemoryEcKv::new("add-colliding-store"), + collisions_remaining: std::sync::Mutex::new(collisions), + } + } + } + + impl EcKvStore for AddCollidingEcKv { + fn store_name(&self) -> &str { + self.inner.store_name() + } + fn lookup(&self, key: &str) -> Result, Report> { + self.inner.lookup(key) + } + fn insert( + &self, + key: &str, + write: EcKvWrite<'_>, + ) -> Result> { + if matches!(write.mode, EcKvWriteMode::Add) { + let mut remaining = self + .collisions_remaining + .lock() + .expect("should lock collision counter"); + if *remaining > 0 { + *remaining -= 1; + return Ok(EcKvWriteOutcome::PreconditionFailed); + } + } + self.inner.insert(key, write) + } + fn count_keys_with_prefix( + &self, + prefix: &str, + limit: u32, + ) -> Result> { + self.inner.count_keys_with_prefix(prefix, limit) + } + fn delete(&self, key: &str) -> Result<(), Report> { + self.inner.delete(key) + } + } + + fn granting_consent() -> ConsentContext { + ConsentContext { + jurisdiction: Jurisdiction::NonRegulated, + source: ConsentSource::Cookie, + ..Default::default() + } + } + + #[test] + fn generate_if_needed_retries_id_collision_then_persists() { + let settings = create_test_settings(); + let mut ec = + EcContext::new_for_test_with_ip(None, granting_consent(), Some("192.0.2.5".to_owned())); + let graph = KvIdentityGraph::new(AddCollidingEcKv::new(2)); + + ec.generate_if_needed(&settings, Some(&graph)) + .expect("should generate after bounded collisions"); + + assert!(ec.ec_value().is_some(), "should allocate a fresh EC ID"); + assert!(ec.ec_generated(), "should mark the EC as generated"); + assert!( + matches!(ec.kv_snapshot(), EcKvSnapshot::Present { .. }), + "generation should seed a present snapshot" + ); + } + + #[test] + fn generate_if_needed_errors_after_collision_exhaustion() { + let settings = create_test_settings(); + let mut ec = + EcContext::new_for_test_with_ip(None, granting_consent(), Some("192.0.2.6".to_owned())); + // Collide on every attempt so the bounded retry is exhausted. + let graph = KvIdentityGraph::new(AddCollidingEcKv::new(u32::MAX)); + + let result = ec.generate_if_needed(&settings, Some(&graph)); + + assert!(result.is_err(), "should fail after exhausting attempts"); + assert!( + ec.ec_value().is_none() && !ec.ec_generated(), + "must not activate an EC ID it could not persist" + ); + } + + #[test] + fn default_ec_context_is_recovery_ineligible_and_unread() { + let ec = EcContext::default(); + assert!( + !ec.recovery_eligible(), + "a default context must not authorize orphan recovery" + ); + assert!( + matches!(ec.kv_snapshot(), EcKvSnapshot::NotRead), + "a default context must carry no identity-graph state" + ); + } + + #[test] + fn read_from_request_does_not_authorize_recovery_from_navigation_headers() { + // Non-Fastly adapters build EC context through the shared read path and + // never call `set_recovery_eligible`. Navigation headers alone must not + // authorize orphan recovery or seed KV state. + let settings = create_test_settings(); + let ec_id = valid_ec_id("b", "CkEc01"); + let cookie = format!("ts-ec={ec_id}"); + let req = create_test_request(&[("cookie", &cookie), ("sec-fetch-dest", "document")]); + + let ec = EcContext::read_from_request(&settings, &req, &noop_services()) + .expect("should read EC context"); + + assert!( + !ec.recovery_eligible(), + "the shared read path must never authorize recovery from headers" + ); + assert!( + matches!(ec.kv_snapshot(), EcKvSnapshot::NotRead), + "the shared read path must leave the snapshot unread" + ); + } + fn create_test_request(headers: &[(&str, &str)]) -> Request { let mut builder = Request::builder().method("GET").uri("http://example.com"); for &(key, value) in headers { @@ -512,6 +775,55 @@ mod tests { format!("{}.{suffix}", prefix_char.repeat(64)) } + #[test] + fn kv_snapshot_distinguishes_non_present_states() { + assert!(EcKvSnapshot::NotRead.entry_for("ec-1").is_none()); + assert!( + EcKvSnapshot::Missing { + ec_id: "ec-1".to_owned() + } + .entry_for("ec-1") + .is_none() + ); + assert!( + EcKvSnapshot::Failed { + ec_id: "ec-1".to_owned() + } + .entry_for("ec-1") + .is_none() + ); + } + + #[test] + fn kv_snapshot_present_state_is_bound_to_ec_id() { + let consent = ConsentContext::default(); + let entry = KvEntry::new(&consent, None, 1_000, "example.com"); + let snapshot = EcKvSnapshot::Present { + ec_id: "ec-1".to_owned(), + entry: Box::new(entry.clone()), + generation: Some(7), + }; + + assert_eq!(snapshot.entry_for("ec-1"), Some(&entry)); + assert_eq!(snapshot.generation_for("ec-1"), Some(7)); + assert!(snapshot.entry_for("ec-2").is_none()); + assert_eq!(snapshot.generation_for("ec-2"), None); + } + + #[test] + fn kv_snapshot_retains_persisted_entry_without_generation() { + let consent = ConsentContext::default(); + let entry = KvEntry::new(&consent, None, 1_000, "example.com"); + let snapshot = EcKvSnapshot::Present { + ec_id: "ec-1".to_owned(), + entry: Box::new(entry.clone()), + generation: None, + }; + + assert_eq!(snapshot.entry_for("ec-1"), Some(&entry)); + assert_eq!(snapshot.generation_for("ec-1"), None); + } + #[test] fn read_from_request_ignores_header_ec() { let settings = create_test_settings(); diff --git a/crates/trusted-server-core/src/ec/prebid_eids.rs b/crates/trusted-server-core/src/ec/prebid_eids.rs index 4a1a8d156..3072473dd 100644 --- a/crates/trusted-server-core/src/ec/prebid_eids.rs +++ b/crates/trusted-server-core/src/ec/prebid_eids.rs @@ -165,6 +165,28 @@ pub fn ingest_eid_cookies( ingest_eid_cookies_with_writer(eids_cookie, sharedid_cookie, ec_id, kv, registry); } +/// Collects validated request-local partner updates without performing KV I/O. +pub(crate) fn collect_eid_cookie_updates( + eids_cookie: Option<&str>, + sharedid_cookie: Option<&str>, + registry: &PartnerRegistry, +) -> Vec { + if registry.is_empty() { + return Vec::new(); + } + + let mut updates = Vec::new(); + if let Some(cookie) = eids_cookie { + updates.extend(collect_prebid_eid_updates(cookie, registry)); + } + if let Some(cookie) = sharedid_cookie + && let Some(update) = collect_sharedid_update(cookie, registry) + { + updates.push(update); + } + dedupe_partner_updates(updates) +} + /// Parses a `ts-eids` cookie value and writes matched partner UIDs to KV. /// /// `cookie_value` is the raw base64-encoded cookie value, already extracted @@ -188,21 +210,7 @@ fn ingest_eid_cookies_with_writer( writer: &dyn PartnerIdBulkWriter, registry: &PartnerRegistry, ) { - if registry.is_empty() { - return; - } - - let mut updates = Vec::new(); - if let Some(cookie) = eids_cookie { - updates.extend(collect_prebid_eid_updates(cookie, registry)); - } - if let Some(cookie) = sharedid_cookie - && let Some(update) = collect_sharedid_update(cookie, registry) - { - updates.push(update); - } - - let updates = dedupe_partner_updates(updates); + let updates = collect_eid_cookie_updates(eids_cookie, sharedid_cookie, registry); if updates.is_empty() { return; } @@ -451,7 +459,9 @@ mod tests { source_domain: source_domain.to_owned(), openrtb_atype: EcPartner::default_openrtb_atype(), bidstream_enabled: true, - api_token: Redacted::new(format!("token-{source_domain}-32-bytes-minimum-value")), + api_token: Some(Redacted::new(format!( + "token-{source_domain}-32-bytes-minimum-value" + ))), batch_rate_limit: EcPartner::default_batch_rate_limit(), pull_sync_enabled: false, pull_sync_url: None, @@ -704,6 +714,39 @@ mod tests { ); } + #[test] + fn collect_eid_cookie_updates_merges_prebid_and_sharedid_without_kv() { + let registry = make_registry(vec![("id5", "id5-sync.com"), ("sharedid", "sharedid.org")]); + let eids_cookie = encode_json(&json!([ + {"source": "id5-sync.com", "uids": [{"id": "ID5_abc", "atype": 1}]} + ])); + + let updates = collect_eid_cookie_updates(Some(&eids_cookie), Some(" shared-1 "), ®istry); + + assert_eq!( + updates.len(), + 2, + "should collect prebid and sharedId matches" + ); + assert!(updates.contains(&PartnerIdUpdate::new("id5-sync.com", "ID5_abc"))); + assert!(updates.contains(&PartnerIdUpdate::new("sharedid.org", "shared-1"))); + } + + #[test] + fn collect_eid_cookie_updates_empty_registry_returns_no_updates() { + let registry = PartnerRegistry::empty(); + let eids_cookie = encode_json(&json!([ + {"source": "id5-sync.com", "uids": [{"id": "ID5_abc", "atype": 1}]} + ])); + + let updates = collect_eid_cookie_updates(Some(&eids_cookie), Some("shared-1"), ®istry); + + assert!( + updates.is_empty(), + "an empty registry matches no partners and touches no KV" + ); + } + #[test] fn dedupe_partner_updates_uses_last_partner_value() { let updates = vec![ diff --git a/crates/trusted-server-core/src/ec/pull_sync.rs b/crates/trusted-server-core/src/ec/pull_sync.rs index 546605f8e..e638258e6 100644 --- a/crates/trusted-server-core/src/ec/pull_sync.rs +++ b/crates/trusted-server-core/src/ec/pull_sync.rs @@ -20,19 +20,21 @@ use crate::platform::{ use crate::settings::Settings; use super::generation::{ec_hash, is_valid_ec_id}; -use super::kv::KvIdentityGraph; +use super::kv::{KvIdentityGraph, PartnerIdUpdate}; use super::kv_types::KvEntry; use super::rate_limiter::RateLimiter; use super::registry::{PartnerConfig, PartnerRegistry}; // `current_timestamp` is defined in the parent `ec` module. use super::EcContext; +use super::EcKvSnapshot; use super::current_timestamp; /// Inputs needed to dispatch pull sync after response flush. #[derive(Debug, Clone)] pub struct PullSyncContext { ec_id: String, + snapshot: EcKvSnapshot, } impl PullSyncContext { @@ -69,13 +71,26 @@ pub fn build_pull_sync_context(ec_context: &EcContext) -> Option entry.map(|(entry, _)| entry), - Err(err) => { - log::warn!( - "Pull sync: failed to read identity graph for '{}': {err:?}", - super::log_id(context.ec_id()) - ); - return; - } + let Some(request_entry) = context.snapshot.entry_for(context.ec_id()) else { + return; }; + if !request_entry.consent.ok { + return; + } let mut pull_partners = registry.pull_enabled_partners(); @@ -121,11 +132,40 @@ pub fn dispatch_pull_sync( let offset = (now / 3600) as usize % pull_partners.len(); pull_partners.rotate_left(offset); + // Drop partners the request snapshot already has a UID for before paying + // for the revalidation read: a dispatch with nothing to pull must not add a + // KV operation. + pull_partners.retain(|partner| is_partner_pull_eligible(partner, Some(request_entry))); + if pull_partners.is_empty() { + return; + } + + let live_snapshot = kv.load_snapshot(context.ec_id()); + let Some(live_entry) = live_snapshot.entry_for(context.ec_id()) else { + log::warn!( + "Pull sync: skipping dispatch for '{}' because the live identity row could not be \ + confirmed", + super::log_id(context.ec_id()) + ); + return; + }; + if !live_entry.consent.ok { + log::info!( + "Pull sync: skipping dispatch for '{}' because consent was withdrawn after the \ + request snapshot was captured", + super::log_id(context.ec_id()) + ); + return; + } + let max_concurrency = settings.ec.pull_sync_concurrency.max(1); let mut in_flight: Vec = Vec::new(); + let mut updates = Vec::new(); for partner in pull_partners { - if !is_partner_pull_eligible(partner, kv_entry.as_ref()) { + // Re-checked against the live row: a concurrent request may have filled + // this partner's UID since the request snapshot was captured. + if !is_partner_pull_eligible(partner, Some(live_entry)) { continue; } @@ -214,11 +254,24 @@ pub fn dispatch_pull_sync( }); if in_flight.len() >= max_concurrency { - drain_pull_batch(kv, context.ec_id(), &mut in_flight, services); + drain_pull_batch(&mut in_flight, services, &mut updates); } } - drain_pull_batch(kv, context.ec_id(), &mut in_flight, services); + drain_pull_batch(&mut in_flight, services, &mut updates); + if !updates.is_empty() { + // Write back from the revalidated snapshot, not the request one: its + // generation is current, so the first CAS attempt is not spent losing a + // conflict against the read that authorized this dispatch. + let outcome = + kv.upsert_partner_ids_from_snapshot(context.ec_id(), &updates, live_snapshot.clone()); + if matches!(outcome, EcKvSnapshot::Failed { .. }) { + log::warn!( + "Pull sync: failed to persist partner updates for '{}'", + super::log_id(context.ec_id()) + ); + } + } } fn is_partner_pull_eligible(partner: &PartnerConfig, kv_entry: Option<&KvEntry>) -> bool { @@ -287,10 +340,9 @@ fn pull_rate_limit_key(source_domain: &str, ec_id: &str) -> String { } fn drain_pull_batch( - kv: &KvIdentityGraph, - ec_id: &str, in_flight: &mut Vec, services: &RuntimeServices, + updates: &mut Vec, ) { for pending in in_flight.drain(..) { let source_domain = pending.source_domain; @@ -312,13 +364,7 @@ fn drain_pull_batch( continue; }; - if let Err(err) = kv.upsert_partner_id(ec_id, &source_domain, &uid) { - log::warn!( - "Pull sync: failed to upsert partner '{}' for ec_id '{}': {err:?}", - source_domain, - super::log_id(ec_id) - ); - } + updates.push(PartnerIdUpdate::new(source_domain, uid)); } } @@ -467,7 +513,7 @@ mod tests { fn pull_partner(ttl_sec: u64) -> PartnerConfig { PartnerConfig { name: "SSP X".to_owned(), - api_key_hash: "deadbeef".to_owned(), + api_key_hash: Some("deadbeef".to_owned()), bidstream_enabled: true, source_domain: "ssp.example.com".to_owned(), openrtb_atype: 3, @@ -711,4 +757,295 @@ mod tests { "hour 1 rotation should move beta to front" ); } + + // ----------------------------------------------------------------------- + // Snapshot-driven eligibility and request-wide aggregation + // ----------------------------------------------------------------------- + + use crate::error::TrustedServerError; + use crate::platform::test_support::{StubHttpClient, build_services_with_http_client}; + use crate::settings::EcPartner; + use crate::test_support::tests::create_test_settings; + use error_stack::Report; + use std::sync::Arc; + + struct AllowAllRateLimiter; + + impl RateLimiter for AllowAllRateLimiter { + fn exceeded( + &self, + _key: &str, + _hourly_limit: u32, + ) -> Result> { + Ok(false) + } + } + + fn pull_enabled_ec_partner(source_domain: &str) -> EcPartner { + EcPartner { + name: format!("Partner {source_domain}"), + source_domain: source_domain.to_owned(), + openrtb_atype: EcPartner::default_openrtb_atype(), + bidstream_enabled: true, + api_token: Some(Redacted::new(format!( + "{source_domain}-api-token-32-bytes-minimum" + ))), + batch_rate_limit: EcPartner::default_batch_rate_limit(), + pull_sync_enabled: true, + pull_sync_url: Some(format!("https://{source_domain}/sync")), + pull_sync_allowed_domains: vec![source_domain.to_owned()], + pull_sync_ttl_sec: 3600, + pull_sync_rate_limit: 100, + ts_pull_token: Some(Redacted::new("outbound-token".to_owned())), + } + } + + fn snapshot_ec_id() -> String { + format!("{}.ABC123", "a".repeat(64)) + } + + fn seed_present_snapshot(graph: &KvIdentityGraph, ec_id: &str) -> EcKvSnapshot { + let mut entry = KvEntry::tombstone(1000); + entry.consent.ok = true; + graph.create(ec_id, &entry).expect("should seed live entry"); + graph.load_snapshot(ec_id) + } + + #[test] + fn dispatch_pull_sync_aggregates_batches_into_one_bulk_write() { + let mut settings = create_test_settings(); + // Force one partner per concurrency batch so responses span batches. + settings.ec.pull_sync_concurrency = 1; + let registry = PartnerRegistry::from_config(&[ + pull_enabled_ec_partner("alpha.example.com"), + pull_enabled_ec_partner("beta.example.com"), + ]) + .expect("should build pull registry"); + + let graph = KvIdentityGraph::in_memory("pull_store"); + let ec_id = snapshot_ec_id(); + let snapshot = seed_present_snapshot(&graph, &ec_id); + + let stub = Arc::new(StubHttpClient::new()); + // One JSON response per partner, drained across two concurrency batches. + stub.push_response(200, br#"{"uid":"synced-uid"}"#.to_vec()); + stub.push_response(200, br#"{"uid":"synced-uid"}"#.to_vec()); + let services = build_services_with_http_client(stub.clone()); + + let context = PullSyncContext { + ec_id: ec_id.clone(), + snapshot, + }; + dispatch_pull_sync( + &settings, + &graph, + ®istry, + &AllowAllRateLimiter, + &context, + &services, + ); + + let (entry, generation) = graph + .get(&ec_id) + .expect("should read store") + .expect("entry should exist"); + assert_eq!( + entry.ids.get("alpha.example.com").map(|id| id.uid.as_str()), + Some("synced-uid"), + "first partner UID should persist" + ); + assert_eq!( + entry.ids.get("beta.example.com").map(|id| id.uid.as_str()), + Some("synced-uid"), + "second partner UID should persist" + ); + assert_eq!( + generation, 2, + "two partner responses across batches must persist in exactly one bulk write" + ); + } + + #[test] + fn dispatch_pull_sync_skips_non_present_snapshots() { + let mut settings = create_test_settings(); + settings.ec.pull_sync_concurrency = 4; + let registry = + PartnerRegistry::from_config(&[pull_enabled_ec_partner("alpha.example.com")]) + .expect("should build registry"); + let graph = KvIdentityGraph::in_memory("pull_store"); + let ec_id = snapshot_ec_id(); + let stub = Arc::new(StubHttpClient::new()); + let services = build_services_with_http_client(stub.clone()); + + for snapshot in [ + EcKvSnapshot::NotRead, + EcKvSnapshot::Missing { + ec_id: ec_id.clone(), + }, + EcKvSnapshot::Failed { + ec_id: ec_id.clone(), + }, + ] { + let context = PullSyncContext { + ec_id: ec_id.clone(), + snapshot, + }; + dispatch_pull_sync( + &settings, + &graph, + ®istry, + &AllowAllRateLimiter, + &context, + &services, + ); + } + + assert!( + stub.recorded_backend_names().is_empty(), + "not-read, missing, and failed snapshots must not dispatch pull sync" + ); + assert!( + graph.get(&ec_id).expect("should read store").is_none(), + "no snapshot state should create a missing root" + ); + } + + #[test] + fn dispatch_pull_sync_skips_tombstone_snapshot() { + let mut settings = create_test_settings(); + settings.ec.pull_sync_concurrency = 4; + let registry = + PartnerRegistry::from_config(&[pull_enabled_ec_partner("alpha.example.com")]) + .expect("should build registry"); + let graph = KvIdentityGraph::in_memory("pull_store"); + let ec_id = snapshot_ec_id(); + let snapshot = EcKvSnapshot::Present { + ec_id: ec_id.clone(), + entry: Box::new(KvEntry::tombstone(1000)), + generation: Some(1), + }; + let stub = Arc::new(StubHttpClient::new()); + let services = build_services_with_http_client(stub.clone()); + + let context = PullSyncContext { + ec_id: ec_id.clone(), + snapshot, + }; + dispatch_pull_sync( + &settings, + &graph, + ®istry, + &AllowAllRateLimiter, + &context, + &services, + ); + + assert!( + stub.recorded_backend_names().is_empty(), + "a tombstone snapshot must not dispatch pull sync" + ); + } + #[test] + fn dispatch_pull_sync_skips_dispatch_when_ec_is_tombstoned_after_snapshot_capture() { + // The request snapshot authorized pull sync, then a concurrent request + // completed a CMP withdrawal while the page response was in flight. + // Pull sync runs post-send and discloses the raw `ec_id` to partners, so + // it must revalidate the live row and cancel rather than leak an + // identity the user just withdrew. + let mut settings = create_test_settings(); + settings.ec.pull_sync_concurrency = 4; + let registry = + PartnerRegistry::from_config(&[pull_enabled_ec_partner("alpha.example.com")]) + .expect("should build registry"); + let graph = KvIdentityGraph::in_memory("pull_store"); + let ec_id = snapshot_ec_id(); + let snapshot = seed_present_snapshot(&graph, &ec_id); + + // Concurrent withdrawal lands after the snapshot was captured. + graph + .write_withdrawal_tombstone(&ec_id) + .expect("should tombstone the row"); + + let stub = Arc::new(StubHttpClient::new()); + stub.push_response(200, br#"{"uid":"leaked-uid"}"#.to_vec()); + let services = build_services_with_http_client(stub.clone()); + + let context = PullSyncContext { + ec_id: ec_id.clone(), + snapshot, + }; + dispatch_pull_sync( + &settings, + &graph, + ®istry, + &AllowAllRateLimiter, + &context, + &services, + ); + + assert!( + stub.recorded_backend_names().is_empty(), + "a withdrawal that lands after snapshot capture must cancel post-send pull sync" + ); + let (entry, _) = graph + .get(&ec_id) + .expect("should read store") + .expect("tombstone should remain"); + assert!( + entry.ids.is_empty(), + "no partner UID may be written back onto a tombstoned row" + ); + } + + #[test] + fn dispatch_pull_sync_skips_revalidation_read_when_no_partner_is_eligible() { + // Every pull-enabled partner already has a UID in the request snapshot, + // so there is nothing to dispatch. The revalidation read exists to + // authorize outbound calls; with no calls to authorize it must not cost + // a KV operation. + let mut settings = create_test_settings(); + settings.ec.pull_sync_concurrency = 4; + let registry = + PartnerRegistry::from_config(&[pull_enabled_ec_partner("alpha.example.com")]) + .expect("should build registry"); + let graph = KvIdentityGraph::in_memory("pull_store"); + let ec_id = snapshot_ec_id(); + let mut entry = KvEntry::tombstone(1000); + entry.consent.ok = true; + entry.ids.insert( + "alpha.example.com".to_owned(), + crate::ec::kv_types::KvPartnerId { + uid: "already-known".to_owned(), + }, + ); + graph + .create(&ec_id, &entry) + .expect("should seed live entry"); + let snapshot = EcKvSnapshot::Present { + ec_id: ec_id.clone(), + entry: Box::new(entry), + generation: Some(1), + }; + + let stub = Arc::new(StubHttpClient::new()); + let services = build_services_with_http_client(stub.clone()); + + let context = PullSyncContext { + ec_id: ec_id.clone(), + snapshot, + }; + dispatch_pull_sync( + &settings, + &graph, + ®istry, + &AllowAllRateLimiter, + &context, + &services, + ); + + assert!( + stub.recorded_backend_names().is_empty(), + "a fully synced entry must not dispatch pull sync" + ); + } } diff --git a/crates/trusted-server-core/src/ec/registry.rs b/crates/trusted-server-core/src/ec/registry.rs index 8532de03b..c4637b431 100644 --- a/crates/trusted-server-core/src/ec/registry.rs +++ b/crates/trusted-server-core/src/ec/registry.rs @@ -4,7 +4,7 @@ //! in-memory registry. `HashMap` indexes provide O(1) //! lookup by source domain and API key hash. -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; use error_stack::{Report, ResultExt as _}; @@ -28,8 +28,8 @@ pub struct PartnerConfig { pub openrtb_atype: i32, /// Whether this partner's UIDs appear in auction `user.eids`. pub bidstream_enabled: bool, - /// SHA-256 hex of the partner's API token (precomputed at startup). - pub api_key_hash: String, + /// SHA-256 hex of the partner's API token, when inbound API access is enabled. + pub api_key_hash: Option, /// Max batch sync API requests per partner per minute. pub batch_rate_limit: u32, /// Whether server-to-server pull sync is enabled. @@ -61,6 +61,78 @@ pub struct PartnerRegistry { } impl PartnerRegistry { + /// Validates partner structure without inspecting secret values. + /// + /// This is the push-time half of partner validation. API-token length, + /// placeholder, and collision checks remain in [`Self::from_config`], + /// after secret references have been resolved. + /// + /// # Errors + /// + /// Returns [`TrustedServerError::Configuration`] when non-secret partner + /// structure is invalid. + pub fn validate_config_for_deploy( + partners: &[EcPartner], + ) -> Result<(), Report> { + let mut source_domains = HashSet::with_capacity(partners.len()); + let mut api_token_key_references = HashMap::with_capacity(partners.len()); + + for partner in partners { + let normalized_source = normalize_partner_source_domain(&partner.source_domain) + .map_err(|msg| { + Report::new(TrustedServerError::Configuration { + message: format!("ec.partners: {msg}"), + }) + })?; + + if !source_domains.insert(normalized_source.clone()) { + return Err(Report::new(TrustedServerError::Configuration { + message: format!("ec.partners: duplicate source_domain '{normalized_source}'"), + })); + } + + if let Some(api_token) = &partner.api_token + && let Some(previous_source) = + api_token_key_references.insert(api_token.expose(), normalized_source.clone()) + { + return Err(Report::new(TrustedServerError::Configuration { + message: format!( + "ec.partners: API token key reference is shared by source_domain \ + '{previous_source}' and '{normalized_source}'" + ), + })); + } + + validate_rate_limits_values(partner.batch_rate_limit, partner.pull_sync_rate_limit) + .map_err(|error| { + Report::new(TrustedServerError::Configuration { + message: format!( + "ec.partners: invalid rate limits for '{normalized_source}': {error}" + ), + }) + })?; + + if partner.pull_sync_enabled { + validate_pull_sync_fields( + partner.pull_sync_url.as_deref(), + &partner.pull_sync_allowed_domains, + partner + .ts_pull_token + .as_ref() + .map(|token| token.expose().as_str()), + false, + ) + .change_context(TrustedServerError::Configuration { + message: format!( + "ec.partners: pull sync config invalid for '{normalized_source}'" + ), + })?; + } + } + + Ok(()) + } + /// Builds a registry from the config-defined partner list. /// /// # Errors @@ -86,20 +158,24 @@ impl PartnerRegistry { })); } - validate_api_token(&normalized_source, partner.api_token.expose())?; - - let api_key_hash = hash_api_key(partner.api_token.expose()); - - if by_api_key_hash.contains_key(&api_key_hash) { - return Err(Report::new(TrustedServerError::Configuration { - message: format!( - "ec.partners: source_domain '{normalized_source}' has an API token that collides \ - with another partner's token hash" - ), - })); - } - - let config = build_partner_config(partner, &normalized_source, &api_key_hash); + let api_key_hash = if let Some(api_token) = &partner.api_token { + validate_api_token(&normalized_source, api_token.expose())?; + + let api_key_hash = hash_api_key(api_token.expose()); + if by_api_key_hash.contains_key(&api_key_hash) { + return Err(Report::new(TrustedServerError::Configuration { + message: format!( + "ec.partners: source_domain '{normalized_source}' has an API token that collides \ + with another partner's token hash" + ), + })); + } + Some(api_key_hash) + } else { + None + }; + + let config = build_partner_config(partner, &normalized_source, api_key_hash.as_deref()); validate_rate_limits(&config).change_context(TrustedServerError::Configuration { message: format!( @@ -117,7 +193,9 @@ impl PartnerRegistry { })?; } - by_api_key_hash.insert(api_key_hash, normalized_source.clone()); + if let Some(api_key_hash) = api_key_hash { + by_api_key_hash.insert(api_key_hash, normalized_source.clone()); + } by_source_domain.insert(normalized_source, config); } @@ -212,14 +290,14 @@ fn validate_api_token( fn build_partner_config( partner: &EcPartner, normalized_source: &str, - api_key_hash: &str, + api_key_hash: Option<&str>, ) -> PartnerConfig { PartnerConfig { name: partner.name.clone(), source_domain: normalized_source.to_owned(), openrtb_atype: partner.openrtb_atype, bidstream_enabled: partner.bidstream_enabled, - api_key_hash: api_key_hash.to_owned(), + api_key_hash: api_key_hash.map(ToOwned::to_owned), batch_rate_limit: partner.batch_rate_limit, pull_sync_enabled: partner.pull_sync_enabled, pull_sync_url: partner.pull_sync_url.clone(), @@ -231,34 +309,56 @@ fn build_partner_config( } fn validate_rate_limits(config: &PartnerConfig) -> Result<(), Report> { - if config.batch_rate_limit == 0 { - return Err(Report::new(TrustedServerError::Configuration { - message: "batch_rate_limit must be greater than 0".to_owned(), - })); + validate_rate_limits_values(config.batch_rate_limit, config.pull_sync_rate_limit).map_err( + |message| { + Report::new(TrustedServerError::Configuration { + message: message.to_owned(), + }) + }, + ) +} + +fn validate_rate_limits_values( + batch_rate_limit: u32, + pull_sync_rate_limit: u32, +) -> Result<(), &'static str> { + if batch_rate_limit == 0 { + return Err("batch_rate_limit must be greater than 0"); } - if config.pull_sync_rate_limit == 0 { - return Err(Report::new(TrustedServerError::Configuration { - message: "pull_sync_rate_limit must be greater than 0".to_owned(), - })); + if pull_sync_rate_limit == 0 { + return Err("pull_sync_rate_limit must be greater than 0"); } Ok(()) } fn validate_pull_sync(config: &PartnerConfig) -> Result<(), Report> { - let url_str = config.pull_sync_url.as_deref().unwrap_or(""); + validate_pull_sync_fields( + config.pull_sync_url.as_deref(), + &config.pull_sync_allowed_domains, + config + .ts_pull_token + .as_ref() + .map(|token| token.expose().as_str()), + true, + ) +} + +fn validate_pull_sync_fields( + url: Option<&str>, + allowed_domains: &[String], + token_value: Option<&str>, + require_nonempty_token: bool, +) -> Result<(), Report> { + let url_str = url.unwrap_or(""); if url_str.is_empty() { return Err(Report::new(TrustedServerError::Configuration { message: "pull_sync_url is required when pull_sync_enabled is true".to_owned(), })); } - if config - .ts_pull_token - .as_ref() - .is_none_or(|token| token.expose().trim().is_empty()) - { + if token_value.is_none() { return Err(Report::new(TrustedServerError::Configuration { message: "ts_pull_token is required when pull_sync_enabled is true".to_owned(), })); @@ -289,7 +389,7 @@ fn validate_pull_sync(config: &PartnerConfig) -> Result<(), Report Result<(), Report { diff --git a/crates/trusted-server-core/src/html_processor.rs b/crates/trusted-server-core/src/html_processor.rs index 816f98167..b2b1c6e6c 100644 --- a/crates/trusted-server-core/src/html_processor.rs +++ b/crates/trusted-server-core/src/html_processor.rs @@ -519,6 +519,7 @@ pub fn create_html_processor(config: HtmlProcessorConfig) -> impl StreamProcesso move |el| { if let Some(mut href) = el.get_attribute("href") { let original_href = href.clone(); + let element_name = el.tag_name(); if let Some(rewritten) = patterns.rewrite_url_value(&href) { href = rewritten; } @@ -528,6 +529,7 @@ pub fn create_html_processor(config: HtmlProcessorConfig) -> impl StreamProcesso &href, &IntegrationAttributeContext { attribute_name: "href", + element_name: &element_name, request_host: &patterns.request_host, request_scheme: &patterns.request_scheme, origin_host: &patterns.origin_host, @@ -557,6 +559,7 @@ pub fn create_html_processor(config: HtmlProcessorConfig) -> impl StreamProcesso move |el| { if let Some(mut src) = el.get_attribute("src") { let original_src = src.clone(); + let element_name = el.tag_name(); if let Some(rewritten) = patterns.rewrite_url_value(&src) { src = rewritten; } @@ -565,6 +568,7 @@ pub fn create_html_processor(config: HtmlProcessorConfig) -> impl StreamProcesso &src, &IntegrationAttributeContext { attribute_name: "src", + element_name: &element_name, request_host: &patterns.request_host, request_scheme: &patterns.request_scheme, origin_host: &patterns.origin_host, @@ -594,6 +598,7 @@ pub fn create_html_processor(config: HtmlProcessorConfig) -> impl StreamProcesso move |el| { if let Some(mut action) = el.get_attribute("action") { let original_action = action.clone(); + let element_name = el.tag_name(); if let Some(rewritten) = patterns.rewrite_url_value(&action) { action = rewritten; } @@ -603,6 +608,7 @@ pub fn create_html_processor(config: HtmlProcessorConfig) -> impl StreamProcesso &action, &IntegrationAttributeContext { attribute_name: "action", + element_name: &element_name, request_host: &patterns.request_host, request_scheme: &patterns.request_scheme, origin_host: &patterns.origin_host, @@ -632,6 +638,7 @@ pub fn create_html_processor(config: HtmlProcessorConfig) -> impl StreamProcesso move |el| { if let Some(mut srcset) = el.get_attribute("srcset") { let original_srcset = srcset.clone(); + let element_name = el.tag_name(); let new_srcset = srcset .replace(&patterns.https_origin(), &patterns.replacement_url()) .replace(&patterns.http_origin(), &patterns.replacement_url()) @@ -649,6 +656,7 @@ pub fn create_html_processor(config: HtmlProcessorConfig) -> impl StreamProcesso &srcset, &IntegrationAttributeContext { attribute_name: "srcset", + element_name: &element_name, request_host: &patterns.request_host, request_scheme: &patterns.request_scheme, origin_host: &patterns.origin_host, @@ -678,6 +686,7 @@ pub fn create_html_processor(config: HtmlProcessorConfig) -> impl StreamProcesso move |el| { if let Some(mut imagesrcset) = el.get_attribute("imagesrcset") { let original_imagesrcset = imagesrcset.clone(); + let element_name = el.tag_name(); let new_imagesrcset = imagesrcset .replace(&patterns.https_origin(), &patterns.replacement_url()) .replace(&patterns.http_origin(), &patterns.replacement_url()) @@ -694,6 +703,7 @@ pub fn create_html_processor(config: HtmlProcessorConfig) -> impl StreamProcesso &imagesrcset, &IntegrationAttributeContext { attribute_name: "imagesrcset", + element_name: &element_name, request_host: &patterns.request_host, request_scheme: &patterns.request_scheme, origin_host: &patterns.origin_host, @@ -1050,8 +1060,14 @@ mod tests { crate::integrations::gpt_diagnostics::prepare_request(&settings, &mut request) .expect("should prepare diagnostics request"); let mut config = create_test_config(); - config.integrations = - IntegrationRegistry::new(&settings).expect("should build integration registry"); + config.integrations = IntegrationRegistry::with_plan( + &settings, + Arc::new( + crate::auction::compile_auction_plan(&settings) + .expect("should compile auction plan"), + ), + ) + .expect("should build integration registry"); config.gpt_diagnostics = Some(decision); let processor = create_html_processor(config); @@ -1147,7 +1163,14 @@ mod tests { #[test] fn test_html_processor_config_from_settings() { let settings = create_test_settings(); - let registry = IntegrationRegistry::new(&settings).expect("should create registry"); + let registry = IntegrationRegistry::with_plan( + &settings, + Arc::new( + crate::auction::compile_auction_plan(&settings) + .expect("should compile auction plan"), + ), + ) + .expect("should create registry"); let config = HtmlProcessorConfig::from_settings( &settings, ®istry, @@ -1315,7 +1338,14 @@ mod tests { ) .expect("should insert testlight config"); - let registry = IntegrationRegistry::new(&settings).expect("should create registry"); + let registry = IntegrationRegistry::with_plan( + &settings, + Arc::new( + crate::auction::compile_auction_plan(&settings) + .expect("should compile auction plan"), + ), + ) + .expect("should create registry"); let mut config = create_test_config(); config.integrations = registry; diff --git a/crates/trusted-server-core/src/http_util.rs b/crates/trusted-server-core/src/http_util.rs index 34656f63d..457e50356 100644 --- a/crates/trusted-server-core/src/http_util.rs +++ b/crates/trusted-server-core/src/http_util.rs @@ -272,6 +272,14 @@ fn detect_request_scheme( // 4. Check Fastly-SSL header. On the `EdgeZero` path this is injected from // authoritative Fastly TLS metadata after spoofable headers are stripped, // so it is reliable. On direct or legacy paths it can be spoofed by clients. + // + // Layering wart: this is a vendor-specific header name living in + // platform-neutral core. It is only a fallback — signal #1 above + // (`ClientInfo::tls_protocol`) is the neutral path adapters populate. The + // `fastly-ssl` fallback (plus its entry in `SPOOFABLE_FORWARDED_HEADERS` + // and the origin-forwarding strip in `publisher::rewrite_origin_request`) + // should be replaced by a platform-neutral scheme signal in a separate + // change, after confirming the legacy path is covered by `ClientInfo`. if let Some(ssl) = req.headers().get("fastly-ssl") && let Ok(ssl_str) = ssl.to_str() && (ssl_str == "1" || ssl_str.to_lowercase() == "true") diff --git a/crates/trusted-server-core/src/integrations/adserver_mock.rs b/crates/trusted-server-core/src/integrations/adserver_mock.rs index 1d6527934..d4fbd3231 100644 --- a/crates/trusted-server-core/src/integrations/adserver_mock.rs +++ b/crates/trusted-server-core/src/integrations/adserver_mock.rs @@ -313,6 +313,7 @@ impl AdServerMockProvider { width, height, bidder: restored_bidder, + returned_seat: original.and_then(|bid| bid.returned_seat.clone()), adomain: bid["adomain"].as_array().map(|arr| { arr.iter() .filter_map(|v| v.as_str().map(String::from)) @@ -399,7 +400,7 @@ impl AdServerMockProvider { #[async_trait(?Send)] impl AuctionProvider for AdServerMockProvider { - fn provider_name(&self) -> &'static str { + fn provider_name(&self) -> &str { "adserver_mock" } @@ -466,14 +467,14 @@ impl AuctionProvider for AdServerMockProvider { } } - // Uses context.timeout_ms (auction-scoped) rather than the 15 s fixed - // timeout in ensure_integration_backend, which is for proxy endpoints. - // Send async with auction-scoped timeout + // Uses the auction-scoped canonical transport timeout rather than the + // 15 s fixed timeout in ensure_integration_backend, which is for proxy + // endpoints. The exact logical budget remains in context.timeout_ms. let backend_name = ensure_integration_backend_with_timeout( context.services, &self.config.endpoint, "adserver_mock", - Duration::from_millis(u64::from(context.timeout_ms)), + Duration::from_millis(u64::from(context.transport_timeout_ms)), ) .change_context(TrustedServerError::Auction { message: format!( @@ -535,12 +536,16 @@ impl AuctionProvider for AdServerMockProvider { self.config.enabled } - fn backend_name(&self, services: &RuntimeServices, timeout_ms: u32) -> Option { + fn backend_name( + &self, + services: &RuntimeServices, + transport_timeout_ms: u32, + ) -> Option { predict_integration_backend_name( services, &self.config.endpoint, "adserver_mock", - Duration::from_millis(u64::from(timeout_ms)), + Duration::from_millis(u64::from(transport_timeout_ms)), ) .inspect_err(|e| { log::error!( @@ -638,6 +643,7 @@ mod tests { width: 728, height: 90, bidder: "aps".to_string(), + returned_seat: None, adomain: Some(vec!["advertiser.example".to_string()]), nurl: None, burl: None, @@ -687,6 +693,7 @@ mod tests { width: 728, height: 90, bidder: "aps".to_string(), + returned_seat: None, adomain: Some(vec!["advertiser.example".to_string()]), nurl: None, burl: None, @@ -713,6 +720,7 @@ mod tests { width: 728, height: 90, bidder: "test-bidder".to_string(), + returned_seat: None, adomain: None, nurl: Some("https://ssp.example/win?id=mock-bid-001".to_string()), burl: Some("https://ssp.example/bill?id=mock-bid-001".to_string()), @@ -796,6 +804,49 @@ mod tests { assert_eq!(bid.height, 90); } + #[test] + fn unmatched_mediator_seats_do_not_become_upstream_returned_seats() { + let provider = AdServerMockProvider::new(AdServerMockConfig::default()); + let mediation_response = json!({ + "seatbid": [ + { + "seat": "provider-instance", + "bid": [{ + "id": "bid-provider", + "impid": "slot-provider", + "price": 1.0, + "adm": "
Provider
", + "w": 300, + "h": 250, + "crid": "uncorrelated-provider-creative" + }] + }, + { + "seat": "unknown", + "bid": [{ + "id": "bid-unknown", + "impid": "slot-unknown", + "price": 2.0, + "adm": "
Unknown
", + "w": 728, + "h": 90, + "crid": "uncorrelated-unknown-creative" + }] + } + ] + }); + + let response = provider.parse_mediation_response(&mediation_response, 10, &BidIndex::new()); + + assert_eq!(response.bids.len(), 2); + assert_eq!(response.bids[0].bidder, "provider-instance"); + assert_eq!(response.bids[1].bidder, "unknown"); + assert!( + response.bids.iter().all(|bid| bid.returned_seat.is_none()), + "an unmatched mediator seat is provider correlation identity, not an upstream seat" + ); + } + #[test] fn parse_mediation_response_restores_original_bid_render_fields() { let provider = AdServerMockProvider::new(AdServerMockConfig::default()); @@ -834,6 +885,7 @@ mod tests { creative: Some("
Original Ad
".to_string()), adomain: Some(vec!["example.com".to_string()]), bidder: "mocktioneer".to_string(), + returned_seat: Some("upstream-seat".to_string()), width: 728, height: 90, nurl: Some("https://ssp.example/win".to_string()), @@ -906,6 +958,11 @@ mod tests { Some("/cache"), "should restore PBS cache path" ); + assert_eq!( + bid.returned_seat.as_deref(), + Some("upstream-seat"), + "should restore returned seat only from the matched original bid" + ); } #[test] @@ -942,6 +999,7 @@ mod tests { creative: Some("
Original Ad
".to_string()), adomain: None, bidder: "example-bidder".to_string(), + returned_seat: None, width: 728, height: 90, nurl: None, @@ -1103,6 +1161,7 @@ mod tests { width: 300, height: 250, bidder: "aps".to_string(), + returned_seat: None, adomain: Some(vec!["advertiser.example".to_string()]), nurl: None, burl: None, diff --git a/crates/trusted-server-core/src/integrations/aps.rs b/crates/trusted-server-core/src/integrations/aps.rs index 581e5c200..2d5eb3041 100644 --- a/crates/trusted-server-core/src/integrations/aps.rs +++ b/crates/trusted-server-core/src/integrations/aps.rs @@ -1,7 +1,8 @@ //! Amazon Publisher Services (APS/TAM) `OpenRTB` integration. -use std::collections::{BTreeMap, HashMap}; +use std::collections::{BTreeMap, HashMap, HashSet}; use std::sync::Arc; +#[cfg(test)] use std::time::Duration; use async_trait::async_trait; @@ -16,34 +17,53 @@ use serde_json::{Value as Json, json}; use url::Url; use validator::{Validate, ValidationError}; +use crate::auction::openrtb::ignored_bidder_params_count; +#[cfg(test)] +use crate::auction::plan::{AuctionPlanConfig, NotificationConfig, ProviderConfig, RoutingMode}; +use crate::auction::profile::ApsProfilePlan; +#[cfg(test)] use crate::auction::provider::{AuctionProvider, ProviderRequestOutcome}; +use crate::auction::routing::ProviderAuctionInput; +#[cfg(test)] +use crate::auction::types::{AdSlot, AuctionContext, AuctionRequest}; use crate::auction::types::{ - AdSlot, ApsRendererV1, ApsTagType, AuctionContext, AuctionRequest, AuctionResponse, Bid, - BidRenderer, MediaType, + ApsRendererV1, ApsTagType, AuctionResponse, Bid, BidRenderer, MediaType, }; use crate::error::TrustedServerError; use crate::integrations::{ IntegrationEndpoint, IntegrationHeadInjector, IntegrationHtmlContext, IntegrationProxy, IntegrationRegistration, UPSTREAM_RTB_MAX_RESPONSE_BYTES, collect_response_bounded, +}; +#[cfg(test)] +use crate::integrations::{ ensure_integration_backend_with_timeout, predict_integration_backend_name, }; +#[cfg(test)] +use crate::openrtb::ToExt; +#[cfg(test)] use crate::openrtb::{ - Banner, Device, Format, Geo, Imp, OpenRtbRequest, Publisher, Regs, RegsExt, Site, ToExt, User, + Banner, Device, Format, Geo, Imp, OpenRtbRequest, Publisher, Regs, RegsExt, Site, User, UserExt, to_openrtb_i32, }; -use crate::platform::{PlatformHttpRequest, PlatformResponse, RuntimeServices}; +#[cfg(test)] +use crate::platform::PlatformHttpRequest; +use crate::platform::{PlatformResponse, RuntimeServices}; use crate::settings::{IntegrationConfig, Settings}; const APS_INTEGRATION_ID: &str = "aps"; const APS_RENDERER_ROUTE: &str = "/integrations/aps/renderer"; const DEFAULT_CURRENCY: &str = "USD"; +#[cfg(test)] const APS_SDK_SOURCE: &str = "prebid"; +#[cfg(test)] const APS_SDK_VERSION: &str = "2.2.0"; const MAX_ACCOUNT_ID_BYTES: usize = 1024; const MAX_CREATIVE_ID_BYTES: usize = 1024; const MAX_DEBUG_RESPONSE_PREVIEW_BYTES: usize = 512; const MAX_CREATIVE_URL_BYTES: usize = 4096; +#[cfg(test)] const MAX_LANGUAGE_BYTES: usize = 8; +#[cfg(test)] const MAX_PAGE_URL_BYTES: usize = 8192; const MAX_RENDER_ENVELOPE_BYTES: usize = 256 * 1024; const APS_RENDERER_CSP: &str = "default-src 'none'; sandbox allow-forms allow-pointer-lock allow-popups allow-popups-to-escape-sandbox allow-scripts allow-top-navigation-by-user-activation; script-src 'unsafe-inline' https:; connect-src https:; frame-src https:; img-src https: data:; media-src https: blob:; style-src 'unsafe-inline' https:; font-src https: data:;"; @@ -57,42 +77,54 @@ const APS_RENDERER_DOCUMENT: &str = r#" var match=/^#tsaps=([A-Za-z0-9_-]{22,128})$/.exec(location.hash); var expected=match&&match[1]; try{history.replaceState(null,'',location.pathname+location.search);}catch(_error){} -if(!expected)return; +var reported=false; +function report(reason,nonce){ + if(reported)return; + reported=true; + try{parent.postMessage({message:'trusted-server/aps/renderer-failed',nonce:nonce,reason:reason},'*');}catch(_error){} +} +if(!expected){report('bad_hash');return;} function keys(value,expectedKeys){ if(!value||typeof value!=='object'||Array.isArray(value))return false; var actual=Object.keys(value).sort(); return actual.length===expectedKeys.length&&actual.every(function(key,index){return key===expectedKeys[index];}); } -function validRenderer(renderer){ +function rendererProblem(renderer){ if(!keys(renderer,['aaxResponse','accountId','bidId','creativeId','creativeUrl','height','tagType','type','version','width'])&& - !keys(renderer,['aaxResponse','accountId','bidId','creativeUrl','height','tagType','type','version','width']))return false; - if(renderer.type!=='aps'||renderer.version!==1||typeof renderer.accountId!=='string'||!renderer.accountId||new TextEncoder().encode(renderer.accountId).length>1024)return false; - if(typeof renderer.bidId!=='string'||!renderer.bidId||!Number.isInteger(renderer.width)||renderer.width<=0||!Number.isInteger(renderer.height)||renderer.height<=0)return false; - if(Object.prototype.hasOwnProperty.call(renderer,'creativeId')&&(typeof renderer.creativeId!=='string'||!renderer.creativeId||new TextEncoder().encode(renderer.creativeId).length>1024))return false; - if(renderer.tagType!=='iframe'&&renderer.tagType!=='script')return false; - if(typeof renderer.creativeUrl!=='string'||new TextEncoder().encode(renderer.creativeUrl).length>4096)return false; - if(typeof renderer.aaxResponse!=='string'||!renderer.aaxResponse||renderer.aaxResponse.length>349528)return false; + !keys(renderer,['aaxResponse','accountId','bidId','creativeUrl','height','tagType','type','version','width']))return 'descriptor_keys'; + if(renderer.type!=='aps'||renderer.version!==1||typeof renderer.accountId!=='string'||!renderer.accountId||new TextEncoder().encode(renderer.accountId).length>1024)return 'descriptor_fields'; + if(typeof renderer.bidId!=='string'||!renderer.bidId||!Number.isInteger(renderer.width)||renderer.width<=0||!Number.isInteger(renderer.height)||renderer.height<=0)return 'descriptor_fields'; + if(Object.prototype.hasOwnProperty.call(renderer,'creativeId')&&(typeof renderer.creativeId!=='string'||!renderer.creativeId||new TextEncoder().encode(renderer.creativeId).length>1024))return 'descriptor_fields'; + if(renderer.tagType!=='iframe'&&renderer.tagType!=='script')return 'descriptor_fields'; + if(typeof renderer.creativeUrl!=='string'||new TextEncoder().encode(renderer.creativeUrl).length>4096)return 'descriptor_fields'; + if(typeof renderer.aaxResponse!=='string'||!renderer.aaxResponse||renderer.aaxResponse.length>349528)return 'descriptor_fields'; try{ var url=new URL(renderer.creativeUrl); - if(url.protocol!=='https:'||url.username||url.password)return false; + if(url.protocol!=='https:'||url.username||url.password)return 'descriptor_envelope'; var binary=atob(renderer.aaxResponse); - if(binary.length>262144||btoa(binary)!==renderer.aaxResponse)return false; + if(binary.length>262144||btoa(binary)!==renderer.aaxResponse)return 'descriptor_envelope'; var bytes=Uint8Array.from(binary,function(character){return character.charCodeAt(0);}); var decoded=JSON.parse(new TextDecoder('utf-8',{fatal:true}).decode(bytes)); - if(!keys(decoded,['seatbid'])||!Array.isArray(decoded.seatbid)||decoded.seatbid.length!==1)return false; + if(!keys(decoded,['seatbid'])||!Array.isArray(decoded.seatbid)||decoded.seatbid.length!==1)return 'descriptor_envelope'; var seat=decoded.seatbid[0]; - if(!keys(seat,['bid'])||!Array.isArray(seat.bid)||seat.bid.length!==1)return false; + if(!keys(seat,['bid'])||!Array.isArray(seat.bid)||seat.bid.length!==1)return 'descriptor_envelope'; var bid=seat.bid[0]; - if(!keys(bid,['ext','h','id','price','w'])||!keys(bid.ext,['creativeurl','tagtype']))return false; - return bid.id===renderer.bidId&&bid.w===renderer.width&&bid.h===renderer.height&& + if(!keys(bid,['ext','h','id','price','w'])||!keys(bid.ext,['creativeurl','tagtype']))return 'descriptor_envelope'; + if(bid.id===renderer.bidId&&bid.w===renderer.width&&bid.h===renderer.height&& bid.ext.creativeurl===renderer.creativeUrl&&bid.ext.tagtype===renderer.tagType&& - typeof bid.price==='number'&&Number.isFinite(bid.price)&&bid.price>=0; - }catch(_error){return false;} + typeof bid.price==='number'&&Number.isFinite(bid.price)&&bid.price>=0)return undefined; + return 'descriptor_envelope'; + }catch(_error){return 'descriptor_envelope';} } function receive(event){ - if(event.source!==parent)return; var message=event.data; - if(!keys(message,['nonce','renderer'])||message.nonce!==expected||!validRenderer(message.renderer))return; + // Stay silent for traffic that is not shaped like the render handshake, so an + // unrelated sender cannot consume this frame's single report. + if(!keys(message,['nonce','renderer']))return; + if(event.source!==parent){report('source_mismatch');return;} + if(message.nonce!==expected){report('nonce_mismatch');return;} + var problem=rendererProblem(message.renderer); + if(problem){report(problem,message.nonce);return;} removeEventListener('message',receive); var acceptedNonce=expected; expected=''; @@ -107,7 +139,7 @@ function receive(event){ var script=document.createElement('script'); script.src='https://client.aps.amazon-adsystem.com/prebid-creative.js'; script.onload=function(){parent.postMessage({message:'trusted-server/aps/renderer-ready',nonce:acceptedNonce},'*');}; - script.onerror=function(){parent.postMessage({message:'trusted-server/aps/renderer-failed',nonce:acceptedNonce},'*');}; + script.onerror=function(){report('amazon_script_error',acceptedNonce);}; document.head.appendChild(script); } addEventListener('message',receive); @@ -127,9 +159,10 @@ pub enum ApsRenderingMode { } /// Configuration for the APS `OpenRTB` integration. +#[cfg(test)] #[derive(Debug, Clone, Deserialize, Serialize, Validate)] #[validate(schema(function = "validate_inventory_identity_override"))] -pub struct ApsConfig { +pub struct LegacyApsProviderConfig { /// Whether APS integration is enabled. #[serde(default = "default_enabled")] pub enabled: bool, @@ -219,6 +252,7 @@ where deserializer.deserialize_any(AccountIdVisitor) } +#[cfg(test)] fn validate_aps_endpoint(value: &str) -> Result<(), ValidationError> { let parsed = Url::parse(value).map_err(|_| ValidationError::new("invalid_aps_endpoint"))?; if parsed.scheme() != "https" @@ -280,12 +314,12 @@ fn validate_inventory_page_origin(value: &str) -> Result<(), ValidationError> { Ok(()) } -fn validate_inventory_identity_override(config: &ApsConfig) -> Result<(), ValidationError> { - let (Some(domain), Some(origin)) = ( - config.inventory_domain.as_deref(), - config.inventory_page_origin.as_deref(), - ) else { - if config.inventory_domain.is_none() && config.inventory_page_origin.is_none() { +fn validate_inventory_identity_override_values( + inventory_domain: Option<&str>, + inventory_page_origin: Option<&str>, +) -> Result<(), ValidationError> { + let (Some(domain), Some(origin)) = (inventory_domain, inventory_page_origin) else { + if inventory_domain.is_none() && inventory_page_origin.is_none() { return Ok(()); } return Err(ValidationError::new( @@ -309,19 +343,33 @@ fn validate_inventory_identity_override(config: &ApsConfig) -> Result<(), Valida Ok(()) } +#[cfg(test)] +fn validate_inventory_identity_override( + config: &LegacyApsProviderConfig, +) -> Result<(), ValidationError> { + validate_inventory_identity_override_values( + config.inventory_domain.as_deref(), + config.inventory_page_origin.as_deref(), + ) +} + +#[cfg(test)] fn default_enabled() -> bool { false } +#[cfg(test)] fn default_endpoint() -> String { "https://web.ads.aps.amazon-adsystem.com/e/pb/bid".to_string() } +#[cfg(test)] fn default_timeout_ms() -> u32 { 800 } -impl Default for ApsConfig { +#[cfg(test)] +impl Default for LegacyApsProviderConfig { fn default() -> Self { Self { enabled: false, @@ -337,20 +385,93 @@ impl Default for ApsConfig { } } +/// Browser integration toggle retained independently from APS server providers. +#[derive(Debug, Clone, Default, Deserialize, Serialize, Validate)] +#[serde(deny_unknown_fields)] +pub struct ApsConfig { + /// Whether browser-side APS integration behavior is enabled. + #[serde(default)] + pub enabled: bool, + /// Rendering owner for selected APS bids. + #[serde(default)] + pub rendering_mode: ApsRenderingMode, +} + +#[cfg(test)] +impl IntegrationConfig for LegacyApsProviderConfig { + fn is_enabled(&self) -> bool { + self.enabled + } +} + impl IntegrationConfig for ApsConfig { fn is_enabled(&self) -> bool { self.enabled } } +/// Typed server-side APS profile configuration used by the auction compiler. +#[derive(Debug, Clone, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct ApsProfileConfig { + #[serde(deserialize_with = "deserialize_account_id")] + pub(crate) account_id: String, + #[serde(default)] + pub(crate) debug: bool, + #[serde(default)] + pub(crate) allow_script_creatives: bool, + #[serde(default)] + pub(crate) inventory_domain: Option, + #[serde(default)] + pub(crate) inventory_page_origin: Option, +} + +/// Parse and validate server-owned APS profile fields without browser enablement. +pub(crate) fn compile_profile_config( + value: serde_json::Value, +) -> Result> { + let profile: ApsProfileConfig = serde_json::from_value(value).map_err(|error| { + Report::new(TrustedServerError::Configuration { + message: format!("invalid `aps` profile_config: {error}"), + }) + })?; + if let Some(domain) = profile.inventory_domain.as_deref() { + validate_inventory_domain(domain).map_err(|error| { + Report::new(TrustedServerError::Configuration { + message: format!("invalid `aps` profile_config inventory_domain: {error}"), + }) + })?; + } + if let Some(origin) = profile.inventory_page_origin.as_deref() { + validate_inventory_page_origin(origin).map_err(|error| { + Report::new(TrustedServerError::Configuration { + message: format!("invalid `aps` profile_config inventory_page_origin: {error}"), + }) + })?; + } + validate_inventory_identity_override_values( + profile.inventory_domain.as_deref(), + profile.inventory_page_origin.as_deref(), + ) + .map_err(|error| { + Report::new(TrustedServerError::Configuration { + message: format!("invalid `aps` profile_config inventory identity: {error}"), + }) + })?; + Ok(profile) +} + +#[cfg(test)] #[derive(Debug, Serialize)] struct ApsRequestExt<'a> { account: &'a str, sdk: ApsSdkExt, } +#[cfg(test)] impl ToExt for ApsRequestExt<'_> {} +#[cfg(test)] #[derive(Debug, Serialize)] struct ApsSdkExt { source: &'static str, @@ -367,21 +488,510 @@ struct ApsRendererInput<'a> { height: u32, } -#[derive(Clone)] -struct ApsDebugRequest { +#[derive(Debug, Clone)] +pub(crate) struct ApsDebugRequest { body: String, headers: BTreeMap>, } -/// APS `OpenRTB` auction provider. +impl ApsDebugRequest { + pub(crate) fn capture(body: &[u8], headers: &HeaderMap) -> Self { + Self { + body: String::from_utf8_lossy(body).into_owned(), + headers: aps_debug_headers(headers), + } + } +} + +struct PlannedApsResponsePolicy<'a> { + provider_id: &'a str, + endpoint: &'a str, + account_id: &'a str, + debug: bool, + allow_script_creatives: bool, + publisher_domain: &'a str, +} + +fn aps_debug_headers(headers: &HeaderMap) -> BTreeMap> { + // This metadata is client-visible. Keep the list fail-closed so upstream + // identity or authentication headers can never leak. + const ALLOWED_HEADERS: &[HeaderName] = &[header::CONTENT_TYPE]; + + let mut values = BTreeMap::>::new(); + for (name, value) in headers { + if !ALLOWED_HEADERS.contains(name) { + continue; + } + let Ok(value) = value.to_str() else { + continue; + }; + values + .entry(name.as_str().to_string()) + .or_default() + .push(value.to_string()); + } + values +} + +fn aps_debug_body_preview(body: &[u8]) -> String { + let preview_len = body.len().min(MAX_DEBUG_RESPONSE_PREVIEW_BYTES); + let mut preview = String::from_utf8_lossy(&body[..preview_len]).into_owned(); + if body.len() > preview_len { + preview.push_str(&format!("…(truncated {} bytes)", body.len() - preview_len)); + } + preview +} + +fn attach_planned_aps_metadata( + mut response: AuctionResponse, + policy: &PlannedApsResponsePolicy<'_>, + input: &ProviderAuctionInput, + debug_request: Option, + response_body: Option<&[u8]>, + response_headers: &BTreeMap>, + status: StatusCode, +) -> AuctionResponse { + response.metadata.insert( + "routing".to_string(), + json!({ + "unused_bidder_params_count": ignored_bidder_params_count(input) + }), + ); + if !policy.debug { + return response; + } + + let mut http_call = json!({ + "responseheaders": response_headers, + "status": status.as_u16(), + "uri": policy.endpoint, + }); + if let Some(http_call) = http_call.as_object_mut() { + if let Some(request) = debug_request { + http_call.insert("requestbody".to_string(), json!(request.body)); + http_call.insert("requestheaders".to_string(), json!(request.headers)); + } + if let Some(response_body) = response_body { + http_call.insert( + "responsebody".to_string(), + json!(aps_debug_body_preview(response_body)), + ); + } + } + response.with_metadata( + "debug", + json!({ + "httpcalls": { + (APS_INTEGRATION_ID): [http_call] + } + }), + ) +} + +fn planned_aps_renderer( + policy: &PlannedApsResponsePolicy<'_>, + input: ApsRendererInput<'_>, +) -> Option { + let tag_type_value = match input.tag_type { + ApsTagType::Iframe => "iframe", + ApsTagType::Script => "script", + }; + let envelope = json!({ + "seatbid": [{ + "bid": [{ + "id": input.bid_id, + "price": input.price, + "w": input.width, + "h": input.height, + "ext": { + "creativeurl": input.creative_url, + "tagtype": tag_type_value + } + }] + }] + }); + let serialized = serde_json::to_vec(&envelope).ok()?; + if serialized.len() > MAX_RENDER_ENVELOPE_BYTES { + return None; + } + Some(BidRenderer::Aps(ApsRendererV1 { + version: 1, + account_id: policy.account_id.to_string(), + bid_id: input.bid_id.to_string(), + creative_id: input.creative_id, + tag_type: input.tag_type, + creative_url: input.creative_url.to_string(), + aax_response: BASE64_STANDARD.encode(serialized), + width: input.width, + height: input.height, + })) +} + +fn planned_aps_valid_creative_url(value: &str, publisher_domain: &str) -> bool { + if value.len() > MAX_CREATIVE_URL_BYTES { + return false; + } + let Ok(parsed) = Url::parse(value) else { + return false; + }; + parsed.scheme() == "https" + && parsed + .host_str() + .is_some_and(|host| !host.eq_ignore_ascii_case(publisher_domain)) + && parsed.username().is_empty() + && parsed.password().is_none() +} + +fn planned_aps_parse_bid( + policy: &PlannedApsResponsePolicy<'_>, + value: &Json, + slots: &HashMap<&str, HashSet<(u32, u32)>>, + returned_seat: Option<&str>, +) -> Result { + let bid_id = value + .get("id") + .and_then(Json::as_str) + .filter(|value| !value.is_empty()) + .ok_or("missing_render_source")?; + let slot_id = value + .get("impid") + .and_then(Json::as_str) + .ok_or("unknown_impid")?; + let dimensions = slots.get(slot_id).ok_or("unknown_impid")?; + let price = value + .get("price") + .and_then(Json::as_f64) + .filter(|price| price.is_finite() && *price >= 0.0) + .ok_or("invalid_price")?; + if value + .get("mtype") + .is_some_and(|mtype| mtype.as_i64() != Some(1)) + { + return Err("unsupported_media_type"); + } + let width = value + .get("w") + .and_then(Json::as_u64) + .and_then(|value| u32::try_from(value).ok()) + .ok_or("invalid_dimensions")?; + let height = value + .get("h") + .and_then(Json::as_u64) + .and_then(|value| u32::try_from(value).ok()) + .ok_or("invalid_dimensions")?; + if width == 0 || height == 0 || !dimensions.contains(&(width, height)) { + return Err("invalid_dimensions"); + } + let ext = value + .get("ext") + .and_then(Json::as_object) + .ok_or("missing_render_source")?; + let creative_url = ext + .get("creativeurl") + .and_then(Json::as_str) + .ok_or("missing_render_source")?; + if !planned_aps_valid_creative_url(creative_url, policy.publisher_domain) { + return Err("invalid_creative_url"); + } + let tag_type = match ext.get("tagtype").and_then(Json::as_str) { + Some("iframe") => ApsTagType::Iframe, + Some("script") if policy.allow_script_creatives => ApsTagType::Script, + Some("script") => return Err("script_rendering_disabled"), + _ => return Err("unsupported_tagtype"), + }; + let creative_id = value + .get("crid") + .and_then(Json::as_str) + .filter(|creative_id| !creative_id.is_empty()) + .map(str::to_string); + if creative_id + .as_ref() + .is_some_and(|creative_id| creative_id.len() > MAX_CREATIVE_ID_BYTES) + { + return Err("creative_id_too_large"); + } + let renderer = planned_aps_renderer( + policy, + ApsRendererInput { + bid_id, + creative_id: creative_id.clone(), + tag_type, + creative_url, + price, + width, + height, + }, + ) + .ok_or("render_payload_too_large")?; + let adomain = value + .get("adomain") + .and_then(Json::as_array) + .map(|domains| { + domains + .iter() + .filter_map(Json::as_str) + .map(str::to_string) + .collect() + }); + + Ok(Bid { + slot_id: slot_id.to_string(), + price: Some(price), + currency: DEFAULT_CURRENCY.to_string(), + creative: None, + adomain, + bidder: APS_INTEGRATION_ID.to_string(), + returned_seat: returned_seat.map(str::to_string), + width, + height, + nurl: None, + burl: None, + bid_id: Some(bid_id.to_string()), + ad_id: value.get("adid").and_then(Json::as_str).map(str::to_string), + creative_id, + renderer: Some(renderer), + cache_id: None, + cache_host: None, + cache_path: None, + metadata: HashMap::new(), + }) +} + +fn increment_planned_aps_reason(reasons: &mut BTreeMap, reason: &'static str) { + *reasons.entry(reason.to_string()).or_default() += 1; +} + +fn parse_planned_aps_value( + value: &Json, + response_time_ms: u64, + input: &ProviderAuctionInput, + policy: &PlannedApsResponsePolicy<'_>, +) -> AuctionResponse { + if !value.is_object() + || value.get("contextual").is_some() + || value + .get("cur") + .is_some_and(|currency| !currency.is_string()) + || value + .get("seatbid") + .is_some_and(|seatbids| !seatbids.is_array()) + { + return AuctionResponse::error(policy.provider_id, response_time_ms) + .with_metadata("error_type", json!("parse_response")) + .with_metadata("drop_reasons", json!({"unexpected_response_shape": 1})); + } + if value + .get("cur") + .and_then(Json::as_str) + .is_some_and(|currency| !currency.eq_ignore_ascii_case(DEFAULT_CURRENCY)) + { + return AuctionResponse::no_bid(policy.provider_id, response_time_ms) + .with_metadata("drop_reasons", json!({"unsupported_currency": 1})); + } + + let slots = input + .slots() + .iter() + .map(|slot| { + let dimensions = slot + .slot() + .formats + .iter() + .filter(|format| format.media_type == MediaType::Banner) + .map(|format| (format.width, format.height)) + .collect::>(); + (slot.slot().id.as_str(), dimensions) + }) + .collect::>(); + let seatbids = value.get("seatbid").and_then(Json::as_array); + let seatbid_count = seatbids.map_or(0, Vec::len); + let mut reasons = BTreeMap::new(); + let mut selected: HashMap = HashMap::new(); + let mut dropped = 0_u64; + + for seatbid in seatbids.into_iter().flatten() { + let returned_seat = seatbid + .get("seat") + .and_then(Json::as_str) + .filter(|seat| !seat.is_empty()); + let Some(bids) = seatbid.get("bid").and_then(Json::as_array) else { + dropped += 1; + increment_planned_aps_reason(&mut reasons, "empty_seatbid_bids"); + continue; + }; + for value in bids { + match planned_aps_parse_bid(policy, value, &slots, returned_seat) { + Ok(candidate) => { + let replace = selected.get(&candidate.slot_id).is_none_or(|current| { + let candidate_price = candidate.price.unwrap_or_default(); + let current_price = current.price.unwrap_or_default(); + candidate_price > current_price + || (candidate_price == current_price + && candidate.bid_id.as_deref().unwrap_or_default() + < current.bid_id.as_deref().unwrap_or_default()) + }); + if replace { + if selected + .insert(candidate.slot_id.clone(), candidate) + .is_some() + { + dropped += 1; + increment_planned_aps_reason(&mut reasons, "lost_to_higher_bid"); + } + } else { + dropped += 1; + increment_planned_aps_reason(&mut reasons, "lost_to_higher_bid"); + } + } + Err(reason) => { + dropped += 1; + increment_planned_aps_reason(&mut reasons, reason); + } + } + } + } + + if seatbid_count == 0 { + increment_planned_aps_reason(&mut reasons, "empty_seatbid"); + } + let accepted = selected.len(); + let metadata = [ + ("seatbid_count".to_string(), json!(seatbid_count)), + ("accepted_bid_count".to_string(), json!(accepted)), + ("dropped_bid_count".to_string(), json!(dropped)), + ("drop_reasons".to_string(), json!(reasons)), + ]; + let mut response = if selected.is_empty() { + AuctionResponse::no_bid(policy.provider_id, response_time_ms) + } else { + AuctionResponse::success( + policy.provider_id, + selected.into_values().collect(), + response_time_ms, + ) + }; + response.metadata.extend(metadata); + response +} + +/// Parse one APS-profile response using only provider-local routed state. +pub(crate) async fn parse_planned_aps_response( + provider_id: &str, + profile: &ApsProfilePlan, + endpoint: &str, + input: &ProviderAuctionInput, + response: PlatformResponse, + response_time_ms: u64, + debug_request: Option, +) -> Result> { + let policy = PlannedApsResponsePolicy { + provider_id, + endpoint, + account_id: &profile.account_id, + debug: profile.debug, + allow_script_creatives: profile.allow_script_creatives, + publisher_domain: &input.common_request().publisher.domain, + }; + let response = response.response; + let status = response.status(); + let response_headers = if policy.debug { + aps_debug_headers(response.headers()) + } else { + BTreeMap::new() + }; + + if status == StatusCode::NO_CONTENT { + return Ok(attach_planned_aps_metadata( + AuctionResponse::no_bid(provider_id, response_time_ms), + &policy, + input, + debug_request, + Some(&[]), + &response_headers, + status, + )); + } + if !status.is_success() { + log::warn!("APS profile {provider_id} returns a non-success status"); + let body = if policy.debug { + match collect_response_bounded( + response.into_body(), + UPSTREAM_RTB_MAX_RESPONSE_BYTES, + APS_INTEGRATION_ID, + ) + .await + { + Ok(body) => Some(body), + Err(error) => { + log::warn!("Failed to read APS profile debug response body: {error:?}"); + None + } + } + } else { + None + }; + return Ok(attach_planned_aps_metadata( + AuctionResponse::error(provider_id, response_time_ms) + .with_metadata("error_type", json!("http_status")) + .with_metadata("http_status", json!(status.as_u16())), + &policy, + input, + debug_request, + body.as_deref(), + &response_headers, + status, + )); + } + let body = collect_response_bounded( + response.into_body(), + UPSTREAM_RTB_MAX_RESPONSE_BYTES, + APS_INTEGRATION_ID, + ) + .await + .change_context(TrustedServerError::Auction { + message: format!("Failed to read APS profile {provider_id} response body"), + })?; + let value: Json = match serde_json::from_slice(&body) { + Ok(value) => value, + Err(error) => { + log::warn!("Failed to parse APS profile {provider_id} response JSON: {error}"); + let parsed = AuctionResponse::error(provider_id, response_time_ms) + .with_metadata("error_type", json!("parse_response")) + .with_metadata("drop_reasons", json!({"unexpected_response_shape": 1})); + return Ok(attach_planned_aps_metadata( + parsed, + &policy, + input, + debug_request, + Some(&body), + &response_headers, + status, + )); + } + }; + let parsed = parse_planned_aps_value(&value, response_time_ms, input, &policy); + Ok(attach_planned_aps_metadata( + parsed, + &policy, + input, + debug_request, + Some(&body), + &response_headers, + status, + )) +} + +/// Legacy APS `OpenRTB` auction provider retained only for parity tests. +#[cfg(test)] pub struct ApsAuctionProvider { - config: ApsConfig, + config: LegacyApsProviderConfig, } +#[cfg(test)] impl ApsAuctionProvider { /// Create an APS provider from validated configuration. #[must_use] - pub fn new(config: ApsConfig) -> Self { + pub fn new(config: LegacyApsProviderConfig) -> Self { Self { config } } @@ -833,6 +1443,7 @@ impl ApsAuctionProvider { creative: None, adomain, bidder: APS_INTEGRATION_ID.to_string(), + returned_seat: None, width, height, nurl: None, @@ -1059,9 +1670,10 @@ impl ApsAuctionProvider { } } +#[cfg(test)] #[async_trait(?Send)] impl AuctionProvider for ApsAuctionProvider { - fn provider_name(&self) -> &'static str { + fn provider_name(&self) -> &str { APS_INTEGRATION_ID } @@ -1264,24 +1876,31 @@ impl IntegrationHeadInjector for ApsRendererIntegration { } } -/// Register the APS static renderer endpoint when APS is enabled. +/// Register renderer support when the auction plan contains an APS provider. +/// +/// Browser integration enablement does not control server-side APS rendering. +/// An absent or disabled browser block uses trusted-server rendering. An enabled +/// browser block may select publisher-native rendering. /// /// # Errors /// -/// Returns an error when enabled APS configuration is invalid. -pub fn register( +/// Returns an error when APS browser configuration is invalid. +pub fn register_for_plan( settings: &Settings, + plan: &crate::auction::AuctionPlan, ) -> Result, Report> { - let Some(config) = settings.integration_config::(APS_INTEGRATION_ID)? else { + if !plan.has_profile(APS_INTEGRATION_ID) { return Ok(None); - }; - let integration = Arc::new(ApsRendererIntegration { - rendering_mode: config.rendering_mode, - }); + } + let rendering_mode = settings + .integration_config::(APS_INTEGRATION_ID)? + .map(|config| config.rendering_mode) + .unwrap_or_default(); + let integration = Arc::new(ApsRendererIntegration { rendering_mode }); let registration = IntegrationRegistration::builder(APS_INTEGRATION_ID) .without_js() .with_head_injector(integration.clone()); - let registration = if config.rendering_mode == ApsRenderingMode::TrustedServer { + let registration = if rendering_mode == ApsRenderingMode::TrustedServer { registration.with_proxy(integration) } else { registration @@ -1294,10 +1913,54 @@ pub fn register( /// # Errors /// /// Returns an error when enabled APS configuration is invalid. +#[cfg(test)] +#[allow(clippy::missing_panics_doc)] +pub fn register( + settings: &Settings, +) -> Result, Report> { + let Some(config) = + settings.integration_config::(APS_INTEGRATION_ID)? + else { + return Ok(None); + }; + let mut browser_settings = settings.clone(); + browser_settings.integrations.insert_config( + APS_INTEGRATION_ID, + &ApsConfig { + enabled: true, + rendering_mode: config.rendering_mode, + }, + )?; + register_for_plan( + &browser_settings, + &crate::auction::AuctionPlan::compile(AuctionPlanConfig { + timeout_ms: 1000, + providers: BTreeMap::from([( + "aps".parse().expect("should parse APS provider ID"), + ProviderConfig { + protocol: "openrtb-2.6".to_string(), + profile: "aps".to_string(), + endpoint: default_endpoint(), + timeout_ms: None, + routing: RoutingMode::AllEligible, + notifications: NotificationConfig::default(), + profile_config: serde_json::json!({"account_id":"example-account"}), + }, + )]), + ..AuctionPlanConfig::default() + }) + .expect("should compile APS renderer test plan"), + ) +} + +#[cfg(test)] +#[allow(clippy::missing_errors_doc)] pub fn register_providers( settings: &Settings, ) -> Result>, Report> { - let Some(config) = settings.integration_config::(APS_INTEGRATION_ID)? else { + let Some(config) = + settings.integration_config::(APS_INTEGRATION_ID)? + else { return Ok(Vec::new()); }; log::info!("Registering APS OpenRTB provider"); @@ -1317,22 +1980,19 @@ pub fn register_providers( #[cfg(test)] mod tests { use super::*; + use crate::auction::test_support::canonical_parity_auction_request; use crate::auction::types::{ - AdFormat, AdSlot, AuctionContext, AuctionRequest, BidStatus, DeviceInfo, PublisherInfo, - UserInfo, + AdFormat, AdSlot, AuctionContext, AuctionRequest, BidStatus, PublisherInfo, UserInfo, }; - use crate::consent::ConsentContext; use crate::integrations::IntegrationDocumentState; - use crate::openrtb::{Eid, Uid}; - use crate::platform::GeoInfo; use crate::platform::test_support::{ StubHttpClient, build_services_with_http_client, noop_services, }; use crate::test_support::tests::create_test_settings; use serde_json::json; - fn config() -> ApsConfig { - ApsConfig { + fn config() -> LegacyApsProviderConfig { + LegacyApsProviderConfig { enabled: true, account_id: "example-account-id".to_string(), endpoint: default_endpoint(), @@ -1407,6 +2067,7 @@ mod tests { settings: &settings, request: &downstream, timeout_ms: 321, + transport_timeout_ms: 321, provider_responses: None, services: &services, }; @@ -1437,15 +2098,28 @@ mod tests { .expect("should parse APS response with context") } + #[test] + fn config_defaults_to_the_800ms_aps_budget() { + let parsed: LegacyApsProviderConfig = serde_json::from_value(json!({ + "account_id": "example-account" + })) + .expect("should parse APS defaults"); + + assert_eq!( + parsed.timeout_ms, 800, + "should preserve APS's 800ms default" + ); + } + #[test] fn config_accepts_canonical_alias_and_integer_ids() { - let canonical: ApsConfig = serde_json::from_value(json!({ + let canonical: LegacyApsProviderConfig = serde_json::from_value(json!({ "account_id": " example-account " })) .expect("should parse canonical account ID"); - let alias: ApsConfig = + let alias: LegacyApsProviderConfig = serde_json::from_value(json!({"pub_id": 1234})).expect("should parse legacy alias"); - let debug: ApsConfig = serde_json::from_value(json!({ + let debug: LegacyApsProviderConfig = serde_json::from_value(json!({ "account_id": "example-account", "debug": true })) @@ -1462,11 +2136,11 @@ mod tests { #[test] fn config_accepts_default_and_custom_openrtb_endpoints() { - let default = ApsConfig { + let default = LegacyApsProviderConfig { account_id: "example-account".to_string(), ..Default::default() }; - let custom: ApsConfig = serde_json::from_value(json!({ + let custom: LegacyApsProviderConfig = serde_json::from_value(json!({ "account_id": "example-account", "endpoint": "https://aps.example.com/custom/openrtb" })) @@ -1487,7 +2161,7 @@ mod tests { "https://aps.example.com/e/dtb/bid/", "https://aps.example.com/custom/e/dtb/bid", ] { - let parsed: ApsConfig = serde_json::from_value(json!({ + let parsed: LegacyApsProviderConfig = serde_json::from_value(json!({ "account_id": "example-account", "endpoint": endpoint })) @@ -1504,15 +2178,18 @@ mod tests { #[test] fn config_rejects_blank_duplicate_and_unsafe_endpoint() { - assert!(serde_json::from_value::(json!({"account_id": " "})).is_err()); assert!( - serde_json::from_value::( + serde_json::from_value::(json!({"account_id": " "})) + .is_err() + ); + assert!( + serde_json::from_value::( json!({"account_id": "x".repeat(MAX_ACCOUNT_ID_BYTES + 1)}) ) .is_err() ); assert!( - serde_json::from_value::(json!({ + serde_json::from_value::(json!({ "account_id": "one", "pub_id": "two" })) @@ -1520,7 +2197,7 @@ mod tests { ); assert!( serde_json::from_value::(json!({ - "account_id": "example-account", + "enabled": true, "rendering_mode": "unsupported" })) .is_err(), @@ -1531,7 +2208,7 @@ mod tests { "https://", "https://user:password@aps.example/e/pb/bid", ] { - let parsed: ApsConfig = serde_json::from_value(json!({ + let parsed: LegacyApsProviderConfig = serde_json::from_value(json!({ "account_id": "example-account", "endpoint": endpoint })) @@ -1572,7 +2249,7 @@ mod tests { "inventory_page_origin": "https://unrelated.example" }), ] { - let parsed: ApsConfig = + let parsed: LegacyApsProviderConfig = serde_json::from_value(value).expect("should deserialize before validation"); assert!( parsed.validate().is_err(), @@ -1583,7 +2260,7 @@ mod tests { #[test] fn inventory_identity_override_rewrites_site_and_preserves_page_path() { - let config: ApsConfig = serde_json::from_value(json!({ + let config: LegacyApsProviderConfig = serde_json::from_value(json!({ "enabled": true, "account_id": "example-account", "inventory_domain": "publisher.example", @@ -1612,6 +2289,7 @@ mod tests { settings: &settings, request: &downstream, timeout_ms: 321, + transport_timeout_ms: 321, provider_responses: None, services: &services, }; @@ -1641,54 +2319,7 @@ mod tests { #[test] fn builds_aps_openrtb_request_with_explicit_privacy_policy() { let provider = ApsAuctionProvider::new(config()); - let mut auction_request = request(); - auction_request.user.consent = Some(ConsentContext { - gdpr_applies: true, - raw_tc_string: Some("fictional-tcf".to_string()), - raw_us_privacy: Some("1YNN".to_string()), - raw_gpp_string: Some("fictional-gpp".to_string()), - gpp_section_ids: Some(vec![2, 6]), - ..Default::default() - }); - auction_request.user.eids = Some(vec![Eid { - source: "identity.example".to_string(), - uids: vec![Uid { - id: "fictional-uid".to_string(), - atype: Some(1), - ext: None, - }], - }]); - auction_request.slots[0].formats.extend([ - AdFormat { - media_type: MediaType::Video, - width: 640, - height: 480, - }, - AdFormat { - media_type: MediaType::Banner, - width: u32::MAX, - height: 90, - }, - AdFormat { - media_type: MediaType::Banner, - width: 728, - height: 90, - }, - ]); - auction_request.device = Some(DeviceInfo { - user_agent: Some("Fictional Browser".to_string()), - ip: Some("192.0.2.10".to_string()), - geo: Some(GeoInfo { - city: "Example City".to_string(), - country: "US".to_string(), - continent: "NA".to_string(), - latitude: 12.34, - longitude: 56.78, - metro_code: 501, - region: Some("CA".to_string()), - asn: None, - }), - }); + let auction_request = canonical_parity_auction_request(); let settings = create_test_settings(); let services = noop_services(); let downstream = http::Request::builder() @@ -1702,12 +2333,13 @@ mod tests { settings: &settings, request: &downstream, timeout_ms: 321, + transport_timeout_ms: 321, provider_responses: None, services: &services, }; let openrtb = provider.build_openrtb_request(&auction_request, &context); - let serialized = serde_json::to_value(openrtb).expect("should serialize request"); + let serialized = serde_json::to_value(&openrtb).expect("should serialize request"); assert_eq!(serialized["id"], "fictional-auction"); assert_eq!(serialized["tmax"], 321); @@ -1766,6 +2398,19 @@ mod tests { assert!(serialized["ext"].get("prebid").is_none()); assert!(serialized["ext"].get("trusted_server").is_none()); assert!(serialized["imp"][0].get("ext").is_none()); + assert!( + serialized["user"]["ext"].get("ConsentSettings").is_none(), + "should omit PBS-only Google Additional Consent placement" + ); + assert!( + serialized["imp"][0].get("tagid").is_none(), + "should ignore shared trustedServer bidder parameters" + ); + assert_eq!( + serde_json::to_string(&openrtb).expect("should serialize APS request"), + r#"{"id":"fictional-auction","imp":[{"id":"fictional-slot","banner":{"format":[{"w":300,"h":250},{"w":728,"h":90}],"w":300,"h":250,"topframe":0},"bidfloor":1.0,"bidfloorcur":"USD","secure":1}],"site":{"domain":"publisher.example","page":"https://publisher.example/article","publisher":{"domain":"publisher.example"}},"device":{"geo":{"type":2,"country":"US","region":"CA","metro":"501","city":"Example City"},"dnt":1,"ua":"Fictional Browser","ip":"192.0.2.10","language":"en"},"user":{"id":"fictional-user","consent":"fictional-tcf","ext":{"consent":"fictional-tcf","eids":[{"source":"identity.example","uids":[{"atype":1,"id":"fictional-uid"}]}]}},"tmax":321,"cur":["USD"],"regs":{"gdpr":1,"us_privacy":"1YNN","gpp":"fictional-gpp","gpp_sid":[2,6],"ext":{"gdpr":1,"gpp":"fictional-gpp","gpp_sid":[2,6],"us_privacy":"1YNN"}},"ext":{"account":"example-account-id","sdk":{"source":"prebid","version":"2.2.0"}}}"#, + "should preserve the complete APS wire shape without a signing extension" + ); } #[test] @@ -1786,6 +2431,7 @@ mod tests { settings: &settings, request: &downstream, timeout_ms: 321, + transport_timeout_ms: 321, provider_responses: None, services: &services, }; @@ -1802,13 +2448,17 @@ mod tests { fn parses_bid_and_builds_exact_minimized_envelope() { let provider = ApsAuctionProvider::new(config()); let response = provider.parse_aps_response( - &json!({"cur": "USD", "seatbid": [{"seat": 42, "bid": [bid("fictional-selected-bid-id", 1.23, "iframe")]}], "ext": {"userSyncs": []}}), + &json!({"cur": "USD", "seatbid": [{"seat": "fictional-upstream-seat", "bid": [bid("fictional-selected-bid-id", 1.23, "iframe")]}], "ext": {"userSyncs": []}}), 12, &request(), ); assert_eq!(response.bids.len(), 1); let parsed = &response.bids[0]; assert_eq!(parsed.bidder, "aps"); + assert!( + parsed.returned_seat.is_none(), + "legacy APS parsing must not attach planned telemetry identity" + ); assert_eq!(parsed.price, Some(1.23)); assert!(parsed.creative.is_none()); assert!(parsed.nurl.is_none()); @@ -1963,6 +2613,7 @@ mod tests { settings: &settings, request: &downstream, timeout_ms: 321, + transport_timeout_ms: 321, provider_responses: None, services: &services, }; @@ -2419,6 +3070,61 @@ mod tests { assert_eq!(response.status(), StatusCode::NOT_FOUND); } + fn plan_with_aps_profile() -> crate::auction::AuctionPlan { + crate::auction::AuctionPlan::compile(AuctionPlanConfig { + timeout_ms: 1_000, + providers: BTreeMap::from([( + "aps".parse().expect("should parse APS provider ID"), + ProviderConfig { + protocol: "openrtb-2.6".to_string(), + profile: "aps".to_string(), + endpoint: default_endpoint(), + timeout_ms: None, + routing: RoutingMode::AllEligible, + notifications: NotificationConfig::default(), + profile_config: serde_json::json!({"account_id":"example-account"}), + }, + )]), + ..AuctionPlanConfig::default() + }) + .expect("should compile APS plan") + } + + #[test] + fn aps_plan_registers_trusted_server_renderer_without_enabled_browser_config() { + for disabled_browser_config in [false, true] { + let mut settings = create_test_settings(); + if disabled_browser_config { + settings + .integrations + .insert_config( + APS_INTEGRATION_ID, + &json!({ + "enabled": false, + "rendering_mode": "publisher_native" + }), + ) + .expect("should insert disabled APS browser config"); + } + + let registration = register_for_plan(&settings, &plan_with_aps_profile()) + .expect("should register APS renderer support") + .expect("should return APS renderer registration"); + + assert_eq!( + registration.proxies.len(), + 1, + "APS plan should register trusted-server renderer route" + ); + assert!( + registration.head_injectors[0] + .tsjs_script_tag_attributes() + .is_empty(), + "disabled or absent browser config should not select publisher-native rendering" + ); + } + } + #[test] fn enabled_config_registers_renderer_proxy() { let mut settings = create_test_settings(); @@ -2613,4 +3319,41 @@ mod tests { assert!(APS_RENDERER_CSP.contains("sandbox allow-forms")); assert!(!APS_RENDERER_CSP.contains("allow-same-origin")); } + + #[test] + fn renderer_document_reports_a_reason_for_every_silent_guard() { + for reason in [ + "bad_hash", + "source_mismatch", + "nonce_mismatch", + "descriptor_keys", + "descriptor_fields", + "descriptor_envelope", + "amazon_script_error", + ] { + assert!( + APS_RENDERER_DOCUMENT.contains(reason), + "renderer document should report a `{reason}` reason instead of returning silently" + ); + } + + // Reasons travel on the existing failure message rather than a new channel. + assert!( + APS_RENDERER_DOCUMENT.contains("reason:reason"), + "should attach the reason to the failure message" + ); + + // A reason is a fixed category, never a copy of the rejected descriptor. + assert!(!APS_RENDERER_DOCUMENT.contains("JSON.stringify(renderer)")); + assert!(!APS_RENDERER_DOCUMENT.contains("reason:message")); + + // Reporting is one-shot so a hostile sender cannot flood the parent. + assert!( + APS_RENDERER_DOCUMENT.contains("if(reported)return"), + "should report at most one reason per frame" + ); + + // A foreign sender is answered through the parent, never the sender. + assert!(!APS_RENDERER_DOCUMENT.contains("event.source.postMessage")); + } } diff --git a/crates/trusted-server-core/src/integrations/datadome.rs b/crates/trusted-server-core/src/integrations/datadome.rs index d95ee35ee..dda4fbd6b 100644 --- a/crates/trusted-server-core/src/integrations/datadome.rs +++ b/crates/trusted-server-core/src/integrations/datadome.rs @@ -78,6 +78,7 @@ use crate::integrations::{ collect_body_bounded, collect_response_bounded, ensure_integration_backend, }; use crate::platform::{PlatformHttpRequest, RuntimeServices}; +use crate::redacted::Redacted; use crate::settings::{IntegrationConfig, Settings}; mod protection; @@ -90,6 +91,7 @@ pub use protection_scope::{ use protection_scope::ProtectionScope; pub(crate) const DATADOME_INTEGRATION_ID: &str = "datadome"; +pub(super) const MIN_TEST_BYPASS_CREDENTIAL_BYTES: usize = 32; /// Fixed request header used by the staging-only protection test bypass. pub(crate) const HEADER_DATADOME_TEST_BYPASS: &str = "x-ts-datadome-bypass"; @@ -133,13 +135,16 @@ pub struct ProtectionTestBypassConfig { #[serde(default)] pub enabled: bool, - /// Secret Store containing the temporary bypass credential. - #[serde(default = "default_protection_test_bypass_secret_store")] - pub credential_secret_store: String, + /// Deprecated feature-specific store selector accepted for migration only. + #[serde(default)] + pub credential_secret_store: Option, - /// Secret name containing at least 32 bytes of high-entropy bypass material. - #[serde(default = "default_protection_test_bypass_secret_name")] - pub credential_secret_name: String, + /// Secret reference containing the bypass credential. + /// + /// Holds the store key name in app config and the resolved credential at + /// runtime. Treat it as secret material after settings are built. + #[serde(default)] + pub credential_secret_name: Option>, } /// Configuration for `DataDome` integration. @@ -175,13 +180,16 @@ pub struct DataDomeConfig { #[serde(default)] pub enable_protection: bool, - /// Runtime secret store containing the `DataDome` server-side key. - #[serde(default = "default_server_side_key_secret_store")] - pub server_side_key_secret_store: String, + /// Deprecated feature-specific store selector accepted for migration only. + #[serde(default)] + pub server_side_key_secret_store: Option, - /// Secret name containing the `DataDome` server-side key. - #[serde(default = "default_server_side_key_secret_name")] - pub server_side_key_secret_name: String, + /// Secret reference containing the `DataDome` server-side key. + /// + /// Holds the store key name in app config and the resolved key at runtime. + /// Treat it as secret material after settings are built. + #[serde(default)] + pub server_side_key_secret_name: Option>, /// Base URL for the `DataDome` Protection API. #[serde(default = "default_protection_api_origin")] @@ -273,22 +281,6 @@ fn default_protection_api_origin() -> String { "https://api-fastly.datadome.co".to_string() } -fn default_server_side_key_secret_store() -> String { - "ts_secrets".to_string() -} - -fn default_server_side_key_secret_name() -> String { - "datadome_server_side_key".to_string() -} - -fn default_protection_test_bypass_secret_store() -> String { - "ts_secrets".to_string() -} - -fn default_protection_test_bypass_secret_name() -> String { - "datadome_test_bypass".to_string() -} - fn default_timeout_ms() -> u32 { 1500 } @@ -356,8 +348,8 @@ impl Default for DataDomeConfig { cache_ttl_seconds: default_cache_ttl(), rewrite_sdk: default_rewrite_sdk(), enable_protection: false, - server_side_key_secret_store: default_server_side_key_secret_store(), - server_side_key_secret_name: default_server_side_key_secret_name(), + server_side_key_secret_store: None, + server_side_key_secret_name: None, protection_api_origin: default_protection_api_origin(), timeout_ms: default_timeout_ms(), protection_excluded_methods: default_protection_excluded_methods(), @@ -395,22 +387,35 @@ impl DataDomeIntegration { } fn try_new(mut config: DataDomeConfig) -> Result, Report> { - config.server_side_key_secret_store = - config.server_side_key_secret_store.trim().to_string(); - config.server_side_key_secret_name = config.server_side_key_secret_name.trim().to_string(); + if config.server_side_key_secret_store.take().is_some() { + log::warn!( + "DataDome server_side_key_secret_store is deprecated and ignored; static credentials resolve through the default app-config secret store" + ); + } + config.server_side_key_secret_name = + config.server_side_key_secret_name.take().and_then(|value| { + let value = value.expose().trim().to_string(); + (!value.is_empty()).then(|| Redacted::new(value)) + }); config.protection_api_origin = config.protection_api_origin.trim().to_string(); config.client_side_tag_url = config.client_side_tag_url.trim().to_string(); if let Some(bypass) = &mut config.protection_test_bypass { - bypass.credential_secret_store = bypass.credential_secret_store.trim().to_string(); - bypass.credential_secret_name = bypass.credential_secret_name.trim().to_string(); + if bypass.credential_secret_store.take().is_some() { + log::warn!( + "DataDome credential_secret_store is deprecated and ignored; static credentials resolve through the default app-config secret store" + ); + } + bypass.credential_secret_name = + bypass.credential_secret_name.take().and_then(|value| { + let value = value.expose().trim().to_string(); + (!value.is_empty()).then(|| Redacted::new(value)) + }); } if config.enable_protection { - if config.server_side_key_secret_store.is_empty() - || config.server_side_key_secret_name.is_empty() - { + if config.server_side_key_secret_name.is_none() { return Err(Report::new(Self::error( - "server_side_key_secret_store and server_side_key_secret_name are required when enable_protection is true", + "server_side_key_secret_name is required when enable_protection is true", ))); } Self::validate_protection_api_origin(&config.protection_api_origin)?; @@ -477,6 +482,12 @@ impl DataDomeIntegration { Self::try_new(config).map(|_| ()) } + pub(crate) fn validate_config_for_deploy( + config: DataDomeConfig, + ) -> Result<(), Report> { + Self::try_new(config).map(|_| ()) + } + fn active_protection_test_bypass(&self) -> Option<&ProtectionTestBypassConfig> { if std::env::var(ENV_FASTLY_IS_STAGING).as_deref() != Ok("1") { return None; @@ -504,9 +515,9 @@ impl DataDomeIntegration { "protection_test_bypass requires enable_protection to be true", ))); } - if bypass.credential_secret_store.is_empty() || bypass.credential_secret_name.is_empty() { + if bypass.credential_secret_name.is_none() { return Err(Report::new(Self::error( - "protection_test_bypass credential_secret_store and credential_secret_name must not be empty when enabled", + "protection_test_bypass credential_secret_name is required when enabled", ))); } @@ -1013,6 +1024,7 @@ mod tests { api_origin: "https://api-js.datadome.co".to_string(), cache_ttl_seconds: 3600, rewrite_sdk: true, + server_side_key_secret_name: Some(Redacted::new("server-side-key".to_string())), ..DataDomeConfig::default() } } @@ -1200,14 +1212,11 @@ mod tests { } #[test] - fn protection_secret_defaults_match_sample_config() { + fn protection_secrets_are_absent_by_default() { let config = DataDomeConfig::default(); - assert_eq!(config.server_side_key_secret_store, "ts_secrets"); - assert_eq!( - config.server_side_key_secret_name, - "datadome_server_side_key" - ); + assert!(config.server_side_key_secret_store.is_none()); + assert!(config.server_side_key_secret_name.is_none()); assert!( config.protection_test_bypass.is_none(), "the temporary test bypass should be disabled by default" @@ -1234,33 +1243,39 @@ mod tests { assert!(bypass.enabled, "should retain the enabled flag"); assert_eq!( - bypass.credential_secret_store, "ts_secrets", - "should retain the configured credential Secret Store" + bypass.credential_secret_store.as_deref(), + Some("ts_secrets"), + "should accept the deprecated credential Secret Store" ); assert_eq!( - bypass.credential_secret_name, "datadome_test_bypass", - "should retain the configured credential secret name" + bypass + .credential_secret_name + .as_ref() + .map(Redacted::expose) + .map(String::as_str), + Some("datadome_test_bypass"), + "should retain the configured credential secret reference" ); } #[test] - fn protection_test_bypass_requires_protection_and_secret_references() { - for (enable_protection, store, name, expected_message) in [ + fn protection_test_bypass_requires_protection_and_credential_reference() { + for (enable_protection, credential, expected_message) in [ ( false, - "ts_secrets", - "datadome_test_bypass", + Some("test-bypass-credential"), "requires enable_protection", ), - (true, "", "datadome_test_bypass", "credential_secret_store"), - (true, "ts_secrets", "", "credential_secret_name"), + (true, None, "credential_secret_name"), ] { let mut config = test_config(); config.enable_protection = enable_protection; + config.server_side_key_secret_name = + Some(Redacted::new("resolved-server-key".to_string())); config.protection_test_bypass = Some(ProtectionTestBypassConfig { enabled: true, - credential_secret_store: store.to_string(), - credential_secret_name: name.to_string(), + credential_secret_store: None, + credential_secret_name: credential.map(|value| Redacted::new(value.to_string())), }); let err = match DataDomeIntegration::try_new(config) { @@ -1275,26 +1290,25 @@ mod tests { } #[test] - fn protection_enabled_requires_server_side_key_secret_store() { + fn protection_test_bypass_accepts_short_resolved_credential() { let mut config = test_config(); config.enable_protection = true; - config.server_side_key_secret_store = " ".to_string(); + config.server_side_key_secret_name = Some(Redacted::new("resolved-server-key".to_string())); + config.protection_test_bypass = Some(ProtectionTestBypassConfig { + enabled: true, + credential_secret_store: None, + credential_secret_name: Some(Redacted::new("short".to_string())), + }); - let err = match DataDomeIntegration::try_new(config) { - Ok(_) => panic!("should reject empty store"), - Err(err) => err, - }; - assert!( - format!("{err:?}").contains("server_side_key_secret_store"), - "should mention secret store config" - ); + DataDomeIntegration::try_new(config) + .expect("should defer bypass credential strength enforcement to requests"); } #[test] fn protection_enabled_requires_server_side_key_secret_name() { let mut config = test_config(); config.enable_protection = true; - config.server_side_key_secret_name = " ".to_string(); + config.server_side_key_secret_name = Some(Redacted::new(" ".to_string())); let err = match DataDomeIntegration::try_new(config) { Ok(_) => panic!("should reject empty name"), @@ -1523,6 +1537,7 @@ mod tests { let ctx = IntegrationAttributeContext { attribute_name: "src", + element_name: "script", request_host: "publisher.com", request_scheme: "https", origin_host: "origin.publisher.com", @@ -1557,6 +1572,7 @@ mod tests { let ctx = IntegrationAttributeContext { attribute_name: "src", + element_name: "script", request_host: "publisher.com", request_scheme: "https", origin_host: "origin.publisher.com", diff --git a/crates/trusted-server-core/src/integrations/datadome/protection.rs b/crates/trusted-server-core/src/integrations/datadome/protection.rs index 75de88afb..a7c55bf86 100644 --- a/crates/trusted-server-core/src/integrations/datadome/protection.rs +++ b/crates/trusted-server-core/src/integrations/datadome/protection.rs @@ -13,7 +13,7 @@ use crate::http_util::is_navigation_request; use crate::integrations::{ HeaderMutation, RequestFilterDecision, RequestFilterEffects, RequestFilterInput, }; -use crate::platform::{PlatformBackendSpec, PlatformHttpRequest, RuntimeServices, StoreName}; +use crate::platform::{PlatformBackendSpec, PlatformHttpRequest, RuntimeServices}; use crate::redacted::Redacted; use super::DataDomeIntegration; @@ -21,8 +21,6 @@ use super::protection_scope::{ ProtectionRequestFacts, ProtectionScopeDecision, ProtectionSkipReason, }; -const MIN_TEST_BYPASS_CREDENTIAL_BYTES: usize = 32; - const VALIDATE_REQUEST_PATH: &str = "/validate-request"; const REQUEST_MODULE_NAME: &str = "Trusted-Server-Rust"; const MODULE_VERSION: &str = env!("CARGO_PKG_VERSION"); @@ -43,8 +41,7 @@ impl DataDomeIntegration { &self, mut input: RequestFilterInput<'_>, ) -> RequestFilterDecision { - let test_bypass_matched = - self.take_protection_test_bypass_header(input.request, input.services); + let test_bypass_matched = self.take_protection_test_bypass_header(input.request); if test_bypass_matched { input .request @@ -87,9 +84,9 @@ impl DataDomeIntegration { .ensure_protection_backend(input.services, &api_url) .map_err(ProtectionRequestError::Setup)?; let server_side_key = self - .load_server_side_key(input.services) + .server_side_key() .map_err(ProtectionRequestError::Setup)?; - let payload = self.build_protection_payload(&input, &server_side_key); + let payload = self.build_protection_payload(&input, server_side_key); let encoded_body = form_encode(&payload.fields); let mut builder = request_builder() @@ -175,11 +172,7 @@ impl DataDomeIntegration { true } - fn take_protection_test_bypass_header( - &self, - req: &mut Request, - services: &RuntimeServices, - ) -> bool { + fn take_protection_test_bypass_header(&self, req: &mut Request) -> bool { let supplied_values = req .headers() .get_all(super::HEADER_DATADOME_TEST_BYPASS) @@ -200,28 +193,21 @@ impl DataDomeIntegration { return false; } - let store_name = StoreName::from(bypass.credential_secret_store.as_str()); - let credential = match services - .secret_store() - .get_string(&store_name, &bypass.credential_secret_name) - { - Ok(credential) if credential.len() >= MIN_TEST_BYPASS_CREDENTIAL_BYTES => credential, - Ok(_) => { - log::warn!( - "[datadome] DataDome test bypass credential does not meet security requirements; ignoring bypass header" - ); - return false; - } - Err(err) => { - log::warn!( - "[datadome] Failed to load DataDome test bypass credential; ignoring bypass header: {err:?}" - ); - return false; - } + let Some(credential) = bypass.credential_secret_name.as_ref() else { + log::warn!( + "[datadome] DataDome test bypass credential is unavailable; ignoring bypass header" + ); + return false; }; + if credential.expose().len() < super::MIN_TEST_BYPASS_CREDENTIAL_BYTES { + log::warn!( + "[datadome] DataDome test bypass credential does not meet security requirements; ignoring bypass header" + ); + return false; + } let actual = Sha256::digest(supplied_values[0].as_bytes()); - let expected = Sha256::digest(credential.as_bytes()); + let expected = Sha256::digest(credential.expose().as_bytes()); bool::from(actual.ct_eq(&expected)) } @@ -259,25 +245,15 @@ impl DataDomeIntegration { )) } - fn load_server_side_key( - &self, - services: &RuntimeServices, - ) -> Result, Report> { - let store_name = StoreName::from(self.config.server_side_key_secret_store.as_str()); - let key = services - .secret_store() - .get_string(&store_name, &self.config.server_side_key_secret_name) - .change_context(Self::error( - "Failed to read DataDome server-side key from secret store", - ))?; - let key = key.trim().to_string(); - if key.is_empty() { - return Err(Report::new(Self::error( - "DataDome server-side key secret must not be empty", - ))); - } - - Ok(Redacted::new(key)) + fn server_side_key(&self) -> Result<&Redacted, Report> { + self.config + .server_side_key_secret_name + .as_ref() + .ok_or_else(|| { + Report::new(Self::error( + "DataDome server-side key is unavailable after secret resolution", + )) + }) } fn build_protection_payload( @@ -854,13 +830,17 @@ mod tests { static FASTLY_IS_STAGING_ENV_LOCK: Mutex<()> = Mutex::new(()); - fn protection_integration() -> Arc { - let config = DataDomeConfig { + fn protection_config() -> DataDomeConfig { + DataDomeConfig { enabled: true, enable_protection: true, + server_side_key_secret_name: Some(Redacted::new("server-side-key".to_string())), ..DataDomeConfig::default() - }; - DataDomeIntegration::try_new(config).expect("should create integration") + } + } + + fn protection_integration() -> Arc { + DataDomeIntegration::try_new(protection_config()).expect("should create integration") } fn request_for_filter() -> Request { @@ -950,10 +930,12 @@ mod tests { enable_protection: true, protection_test_bypass: Some(ProtectionTestBypassConfig { enabled: true, - credential_secret_store: "ts_secrets".to_string(), - credential_secret_name: "datadome_test_bypass".to_string(), + credential_secret_store: None, + credential_secret_name: Some(Redacted::new( + "temporary-test-credential-32-bytes!".to_string(), + )), }), - ..DataDomeConfig::default() + ..protection_config() }; let integration = DataDomeIntegration::try_new(config).expect("should create integration"); let mut secrets = HashMap::new(); @@ -1002,15 +984,17 @@ mod tests { None, Some(ProtectionTestBypassConfig { enabled: false, - credential_secret_store: "ts_secrets".to_string(), - credential_secret_name: "datadome_test_bypass".to_string(), + credential_secret_store: None, + credential_secret_name: Some(Redacted::new( + "temporary-test-credential-32-bytes!".to_string(), + )), }), ] { let config = DataDomeConfig { enabled: true, enable_protection: true, protection_test_bypass, - ..DataDomeConfig::default() + ..protection_config() }; let integration = DataDomeIntegration::try_new(config).expect("should create integration"); @@ -1068,10 +1052,12 @@ mod tests { enable_protection: true, protection_test_bypass: Some(ProtectionTestBypassConfig { enabled: true, - credential_secret_store: "ts_secrets".to_string(), - credential_secret_name: "datadome_test_bypass".to_string(), + credential_secret_store: None, + credential_secret_name: Some(Redacted::new( + "temporary-test-credential-32-bytes!".to_string(), + )), }), - ..DataDomeConfig::default() + ..protection_config() }; let integration = DataDomeIntegration::try_new(config).expect("should create integration"); let mut secrets = HashMap::new(); @@ -1156,10 +1142,12 @@ mod tests { }], protection_test_bypass: Some(ProtectionTestBypassConfig { enabled: true, - credential_secret_store: "ts_secrets".to_string(), - credential_secret_name: "datadome_test_bypass".to_string(), + credential_secret_store: None, + credential_secret_name: Some(Redacted::new( + "temporary-test-credential-32-bytes!".to_string(), + )), }), - ..DataDomeConfig::default() + ..protection_config() }; let integration = DataDomeIntegration::try_new(config).expect("should create integration"); let mut secrets = HashMap::new(); @@ -1202,10 +1190,12 @@ mod tests { enable_protection: true, protection_test_bypass: Some(ProtectionTestBypassConfig { enabled: true, - credential_secret_store: "ts_secrets".to_string(), - credential_secret_name: "datadome_test_bypass".to_string(), + credential_secret_store: None, + credential_secret_name: Some(Redacted::new( + "temporary-test-credential-32-bytes!".to_string(), + )), }), - ..DataDomeConfig::default() + ..protection_config() }; let integration = DataDomeIntegration::try_new(config).expect("should create integration"); let mut secrets = HashMap::new(); @@ -1265,10 +1255,12 @@ mod tests { enable_protection: true, protection_test_bypass: Some(ProtectionTestBypassConfig { enabled: true, - credential_secret_store: "ts_secrets".to_string(), - credential_secret_name: "datadome_test_bypass".to_string(), + credential_secret_store: None, + credential_secret_name: Some(Redacted::new( + "temporary-test-credential-32-bytes!".to_string(), + )), }), - ..DataDomeConfig::default() + ..protection_config() }; let integration = DataDomeIntegration::try_new(config).expect("should create integration"); let mut secrets = HashMap::new(); @@ -1317,67 +1309,48 @@ mod tests { } #[test] - fn test_bypass_credential_requires_at_least_32_bytes() { - for (credential, should_match) in [ - (Some("1234567890123456789012345678901"), false), - (Some("12345678901234567890123456789012"), true), - (Some(""), false), - (None, false), - ] { - let config = DataDomeConfig { + fn short_test_bypass_credential_is_ignored_without_failing_startup() { + let config = DataDomeConfig { + protection_test_bypass: Some(ProtectionTestBypassConfig { enabled: true, - enable_protection: true, - protection_test_bypass: Some(ProtectionTestBypassConfig { - enabled: true, - credential_secret_store: "ts_secrets".to_string(), - credential_secret_name: "datadome_test_bypass".to_string(), - }), - ..DataDomeConfig::default() - }; - let integration = - DataDomeIntegration::try_new(config).expect("should create integration"); - let mut secrets = HashMap::new(); - secrets.insert( - "datadome_server_side_key".to_string(), - b"server-side-key".to_vec(), - ); - if let Some(credential) = credential { - secrets.insert( - "datadome_test_bypass".to_string(), - credential.as_bytes().to_vec(), - ); - } - let http_client = Arc::new(StubHttpClient::new()); - if !should_match { - http_client.push_response_with_headers( - 200, - Vec::new(), - vec![(HEADER_DATADOME_RESPONSE, "200")], - ); - } - let services = build_services_with_secret_and_http_client( - HashMapSecretStore::new(secrets), - http_client.clone(), - ); - let settings = Settings::default(); - let mut request = request_for_filter(); - let supplied = credential.unwrap_or("12345678901234567890123456789012"); - request.headers_mut().insert( - super::super::HEADER_DATADOME_TEST_BYPASS, - edgezero_core::http::HeaderValue::from_str(supplied) - .expect("should build bypass header"), - ); + credential_secret_store: None, + credential_secret_name: Some(Redacted::new("short".to_string())), + }), + ..protection_config() + }; + let integration = + DataDomeIntegration::try_new(config).expect("should accept short bypass credential"); + let http_client = Arc::new(StubHttpClient::new()); + http_client.push_response_with_headers( + 200, + Vec::new(), + vec![(HEADER_DATADOME_RESPONSE, "200")], + ); + let services = + build_services_with_secret_and_http_client(NoopSecretStore, http_client.clone()); + let settings = Settings::default(); + let mut request = request_for_filter(); + request.headers_mut().insert( + super::super::HEADER_DATADOME_TEST_BYPASS, + edgezero_core::http::HeaderValue::from_static("short"), + ); - let decision = filter_with_staging(&integration, &settings, &services, &mut request); + let decision = filter_with_staging(&integration, &settings, &services, &mut request); - assert!(matches!(decision, RequestFilterDecision::Continue(_))); - assert_eq!(has_client_tag_suppression_marker(&request), should_match); - assert_eq!( - http_client.recorded_backend_names().is_empty(), - should_match, - "only a credential meeting the minimum should skip the API" - ); - } + assert!(matches!(decision, RequestFilterDecision::Continue(_))); + assert!( + request + .headers() + .get(super::super::HEADER_DATADOME_TEST_BYPASS) + .is_none(), + "the invalid bypass credential should not reach the publisher origin" + ); + assert!(!has_client_tag_suppression_marker(&request)); + assert_eq!( + http_client.recorded_backend_names().len(), + 1, + "a short credential should not bypass the Protection API" + ); } #[test] @@ -1417,7 +1390,7 @@ mod tests { enabled: true, enable_protection: true, protection_excluded_ip_cidrs: vec!["192.0.2.0/24".to_string()], - ..DataDomeConfig::default() + ..protection_config() }; let inline_request = filter_marks_request(inline.clone(), &noop_services_with_client_ip(ip)); @@ -1456,7 +1429,7 @@ mod tests { cidrs: vec!["192.0.2.0/24".to_string()], }, }], - ..DataDomeConfig::default() + ..protection_config() }; let structured_request = filter_marks_request(structured_ip, &noop_services_with_client_ip(ip)); @@ -1477,7 +1450,7 @@ mod tests { key: "structured-source".to_string(), }, }], - ..DataDomeConfig::default() + ..protection_config() }; let mut structured_values = HashMap::new(); structured_values.insert("structured-source".to_string(), "192.0.2.0/24".to_string()); @@ -1534,7 +1507,7 @@ mod tests { methods: Vec::new(), matcher, }], - ..DataDomeConfig::default() + ..protection_config() }; let request = filter_marks_request_for_uri(config, &noop_services_with_client_ip(ip), None, uri); @@ -1569,7 +1542,7 @@ mod tests { }, }, ], - ..DataDomeConfig::default() + ..protection_config() }; let request = filter_marks_request(config, &noop_services_with_client_ip(ip)); @@ -1586,7 +1559,7 @@ mod tests { enabled: true, enable_protection: true, protection_excluded_asns: vec![64500], - ..DataDomeConfig::default() + ..protection_config() }; let geo_info = GeoInfo { city: String::new(), @@ -1615,7 +1588,7 @@ mod tests { enabled: true, enable_protection: true, protection_excluded_ip_cidrs: vec!["192.0.2.0/24".to_string()], - ..DataDomeConfig::default() + ..protection_config() }; let request = filter_marks_request( config, @@ -1628,39 +1601,27 @@ mod tests { } #[test] - fn load_server_side_key_reads_secret_store() { - let mut secrets = HashMap::new(); - secrets.insert( - "datadome_server_side_key".to_string(), - b"secret-from-store".to_vec(), - ); - let services = build_services_with_config_and_secret( - NoopConfigStore, - HashMapSecretStore::new(secrets), - ); + fn server_side_key_uses_resolved_config_value() { let integration = protection_integration(); let key = integration - .load_server_side_key(&services) - .expect("should load server-side key"); + .server_side_key() + .expect("should contain resolved server-side key"); - assert_eq!(key.expose(), "secret-from-store"); + assert_eq!(key.expose(), "server-side-key"); } #[test] - fn load_server_side_key_errors_when_secret_missing() { - let services = build_services_with_config_and_secret(NoopConfigStore, NoopSecretStore); + fn protection_startup_rejects_missing_resolved_server_side_key() { let config = DataDomeConfig { - enabled: true, - enable_protection: true, - server_side_key_secret_name: "missing_server_side_key".to_string(), - ..DataDomeConfig::default() + server_side_key_secret_name: None, + ..protection_config() }; - let integration = DataDomeIntegration::try_new(config).expect("should create integration"); - let result = integration.load_server_side_key(&services); - - assert!(result.is_err(), "should error when secret is missing"); + assert!( + DataDomeIntegration::try_new(config).is_err(), + "should reject a missing resolved server-side key" + ); } #[test] diff --git a/crates/trusted-server-core/src/integrations/didomi.rs b/crates/trusted-server-core/src/integrations/didomi.rs index f8472b796..4b0989183 100644 --- a/crates/trusted-server-core/src/integrations/didomi.rs +++ b/crates/trusted-server-core/src/integrations/didomi.rs @@ -412,7 +412,14 @@ mod tests { .insert_config(DIDOMI_INTEGRATION_ID, &config(true)) .expect("should insert config"); - let registry = IntegrationRegistry::new(&settings).expect("should create registry"); + let registry = IntegrationRegistry::with_plan( + &settings, + Arc::new( + crate::auction::compile_auction_plan(&settings) + .expect("should compile auction plan"), + ), + ) + .expect("should create registry"); assert!(registry.has_route(&Method::GET, "/integrations/didomi/consent/loader.js")); assert!(registry.has_route(&Method::POST, "/integrations/didomi/consent/api/events")); assert!(!registry.has_route(&Method::GET, "/other")); @@ -505,7 +512,14 @@ mod tests { .insert_config(DIDOMI_INTEGRATION_ID, &custom_config) .expect("should insert config"); - let registry = IntegrationRegistry::new(&settings).expect("should create registry"); + let registry = IntegrationRegistry::with_plan( + &settings, + Arc::new( + crate::auction::compile_auction_plan(&settings) + .expect("should compile auction plan"), + ), + ) + .expect("should create registry"); assert!(registry.has_route(&Method::GET, "/my-custom-consent/loader.js")); assert!(registry.has_route(&Method::POST, "/my-custom-consent/api/events")); assert!(!registry.has_route(&Method::GET, "/integrations/didomi/consent/loader.js")); diff --git a/crates/trusted-server-core/src/integrations/google_tag_manager.rs b/crates/trusted-server-core/src/integrations/google_tag_manager.rs index 162e9eb8c..2809f34fe 100644 --- a/crates/trusted-server-core/src/integrations/google_tag_manager.rs +++ b/crates/trusted-server-core/src/integrations/google_tag_manager.rs @@ -805,6 +805,7 @@ mod tests { let ctx = IntegrationAttributeContext { attribute_name: "src", + element_name: "script", request_host: "example.com", request_scheme: "https", origin_host: "origin.example.com", @@ -921,6 +922,7 @@ mod tests { let ctx = IntegrationAttributeContext { attribute_name: "href", + element_name: "a", request_host: "example.com", request_scheme: "https", origin_host: "origin.example.com", @@ -1638,7 +1640,14 @@ container_id = "GTM-DEFAULT" ) .expect("should update gtm config"); - let registry = IntegrationRegistry::new(&settings).expect("should create registry"); + let registry = IntegrationRegistry::with_plan( + &settings, + Arc::new( + crate::auction::compile_auction_plan(&settings) + .expect("should compile auction plan"), + ), + ) + .expect("should create registry"); let config = config_from_settings(&settings, ®istry); let processor = create_html_processor(config); let pipeline_config = PipelineConfig { @@ -1678,7 +1687,14 @@ container_id = "GTM-DEFAULT" .expect("should update gtm config"); // 2. Setup Pipeline - let registry = IntegrationRegistry::new(&settings).expect("should create registry"); + let registry = IntegrationRegistry::with_plan( + &settings, + Arc::new( + crate::auction::compile_auction_plan(&settings) + .expect("should compile auction plan"), + ), + ) + .expect("should create registry"); let config = config_from_settings(&settings, ®istry); let processor = create_html_processor(config); let pipeline_config = PipelineConfig { @@ -1744,7 +1760,14 @@ container_id = "GTM-DEFAULT" .expect("should update config"); // Inlined Pipeline Creation - let registry = IntegrationRegistry::new(&settings).expect("should create registry"); + let registry = IntegrationRegistry::with_plan( + &settings, + Arc::new( + crate::auction::compile_auction_plan(&settings) + .expect("should compile auction plan"), + ), + ) + .expect("should create registry"); let config = config_from_settings(&settings, ®istry); let processor = create_html_processor(config); let pipeline_config = PipelineConfig { @@ -2070,7 +2093,14 @@ container_id = "GTM-DEFAULT" ) .expect("should update config"); - let registry = IntegrationRegistry::new(&settings).expect("should create registry"); + let registry = IntegrationRegistry::with_plan( + &settings, + Arc::new( + crate::auction::compile_auction_plan(&settings) + .expect("should compile auction plan"), + ), + ) + .expect("should create registry"); let config = config_from_settings(&settings, ®istry); let processor = create_html_processor(config); @@ -2136,7 +2166,14 @@ container_id = "GTM-DEFAULT" ) .expect("should update nextjs config"); - let registry = IntegrationRegistry::new(&settings).expect("should create registry"); + let registry = IntegrationRegistry::with_plan( + &settings, + Arc::new( + crate::auction::compile_auction_plan(&settings) + .expect("should compile auction plan"), + ), + ) + .expect("should create registry"); let config = config_from_settings(&settings, ®istry); let processor = create_html_processor(config); @@ -2202,7 +2239,14 @@ container_id = "GTM-DEFAULT" ) .expect("should update nextjs config"); - let registry = IntegrationRegistry::new(&settings).expect("should create registry"); + let registry = IntegrationRegistry::with_plan( + &settings, + Arc::new( + crate::auction::compile_auction_plan(&settings) + .expect("should compile auction plan"), + ), + ) + .expect("should create registry"); let config = config_from_settings(&settings, ®istry); let processor = create_html_processor(config); diff --git a/crates/trusted-server-core/src/integrations/gpt.rs b/crates/trusted-server-core/src/integrations/gpt.rs index 84158c27e..3292cc5b3 100644 --- a/crates/trusted-server-core/src/integrations/gpt.rs +++ b/crates/trusted-server-core/src/integrations/gpt.rs @@ -579,6 +579,7 @@ mod tests { fn test_context() -> IntegrationAttributeContext<'static> { IntegrationAttributeContext { attribute_name: "src", + element_name: "script", request_host: "edge.example.com", request_scheme: "https", origin_host: "origin.example.com", @@ -1246,12 +1247,16 @@ mod tests { "should set ts_initial sentinel" ); assert!( - !combined.contains("addEventListener(\"slotRenderEnded\""), - "inline bootstrap cannot prove TS creative rendering from GPT slotRenderEnded" + combined.contains("addEventListener(\"slotRequested\""), + "should observe publisher GPT requests before delayed adInit" + ); + assert!( + combined.contains("addEventListener(\"slotRenderEnded\""), + "should observe publisher GPT renders before delayed adInit" ); assert!( !combined.contains("sendBeacon"), - "inline bootstrap must not fire win/billing beacons from GPT slotRenderEnded" + "inline bootstrap lifecycle ownership must not fire win/billing beacons" ); assert!( !combined.contains("getTargeting(\"hb_adid\")"), diff --git a/crates/trusted-server-core/src/integrations/gpt_bootstrap.js b/crates/trusted-server-core/src/integrations/gpt_bootstrap.js index 2475c5082..c7cceaa80 100644 --- a/crates/trusted-server-core/src/integrations/gpt_bootstrap.js +++ b/crates/trusted-server-core/src/integrations/gpt_bootstrap.js @@ -102,6 +102,179 @@ pubads.__tsInitialLoadHooked = true; }); + var FIRST_IMPRESSION_LEASE_MS = 5000; + var MAX_FIRST_IMPRESSION_SLOTS = 256; + + function firstImpressionState(now) { + var generation = ts.navGeneration || 0; + if ( + !ts.firstImpression || + ts.firstImpression.generation !== generation + ) { + ts.firstImpression = { + generation: generation, + nextToken: 0, + slots: {}, + fallbackSlots: {}, + }; + } + var state = ts.firstImpression; + state.slots = state.slots || {}; + state.fallbackSlots = state.fallbackSlots || {}; + Object.keys(state.slots).forEach(function (elementId) { + var claim = state.slots[elementId]; + if ( + claim.generation !== generation || + claim.slotElementId !== elementId || + claim.element.ownerDocument !== document || + claim.element !== document.getElementById(elementId) || + !claim.element.isConnected + ) { + delete state.slots[elementId]; + return; + } + var hasReservedFallback = + claim.owner === "publisher" && + (claim.phase === "auctioning" || claim.phase === "delivery_pending") && + state.fallbackSlots[elementId] === claim.element; + Object.keys(claim.publisherAuctions || {}).forEach(function (token) { + var auction = claim.publisherAuctions[token]; + if ( + auction.expiresAt <= now && + !hasReservedFallback && + !(claim.owner === "trusted_server" && auction.suppressDelivery) + ) { + delete claim.publisherAuctions[token]; + } + }); + if ( + claim.owner === "publisher" && + (claim.phase === "auctioning" || claim.phase === "delivery_pending") && + Object.keys(claim.publisherAuctions || {}).length === 0 && + claim.expiresAt <= now && + !hasReservedFallback + ) { + delete state.slots[elementId]; + } + }); + Object.keys(state.fallbackSlots).forEach(function (elementId) { + var element = state.fallbackSlots[elementId]; + if ( + !element.isConnected || + element.id !== elementId || + document.getElementById(elementId) !== element + ) { + delete state.fallbackSlots[elementId]; + } + }); + return state; + } + + function firstImpressionClaim(element) { + return firstImpressionState(Date.now()).slots[element.id]; + } + + function storeFirstImpressionClaim(state, claim) { + if ( + !state.slots[claim.slotElementId] && + Object.keys(state.slots).length >= MAX_FIRST_IMPRESSION_SLOTS + ) { + return false; + } + state.slots[claim.slotElementId] = claim; + return true; + } + + function claimFirstImpressionForTrustedServer(element) { + var now = Date.now(); + var state = firstImpressionState(now); + var existing = state.slots[element.id]; + if (existing) { + var canTransitionPublisherFallback = + existing.owner === "publisher" && + existing.phase !== "requested" && + existing.phase !== "rendered" && + existing.expiresAt <= now && + state.fallbackSlots[element.id] === element; + if (!canTransitionPublisherFallback) return null; + existing.owner = "trusted_server"; + existing.phase = "delivery_pending"; + existing.expiresAt = now + FIRST_IMPRESSION_LEASE_MS; + Object.keys(existing.publisherAuctions || {}).forEach(function (token) { + existing.publisherAuctions[token].suppressDelivery = true; + }); + return existing; + } + var claim = { + generation: state.generation, + slotElementId: element.id, + element: element, + owner: "trusted_server", + phase: "delivery_pending", + expiresAt: now + FIRST_IMPRESSION_LEASE_MS, + publisherAuctions: {}, + }; + return storeFirstImpressionClaim(state, claim) ? claim : null; + } + + function releaseTrustedServerFirstImpressionClaim(element, claim) { + var state = firstImpressionState(Date.now()); + if ( + state.slots[element.id] === claim && + claim.owner === "trusted_server" && + claim.phase === "delivery_pending" + ) { + delete state.slots[element.id]; + if (state.fallbackSlots[element.id] === element) { + delete state.fallbackSlots[element.id]; + } + } + } + + function installFirstImpressionListeners() { + if (ts.firstImpressionListenersInstalled) return; + tag.cmd.push(function () { + if (ts.firstImpressionListenersInstalled) return; + var pubads = window.googletag.pubads(); + if (!pubads || typeof pubads.addEventListener !== "function") return; + var observe = function (phase) { + return function (event) { + var elementId = + event.slot && event.slot.getSlotElementId + ? event.slot.getSlotElementId() + : ""; + var element = elementId && document.getElementById(elementId); + if (!element) return; + var state = firstImpressionState(Date.now()); + var claim = state.slots[elementId]; + if (!claim) { + storeFirstImpressionClaim(state, { + generation: state.generation, + slotElementId: elementId, + element: element, + owner: "publisher", + phase: phase, + expiresAt: Number.POSITIVE_INFINITY, + publisherAuctions: {}, + }); + return; + } + claim.phase = phase; + if (claim.owner === "publisher") { + claim.expiresAt = Number.POSITIVE_INFINITY; + } else { + claim.publisherRegistrationClosed = true; + } + }; + }; + pubads.addEventListener("slotRequested", observe("requested")); + pubads.addEventListener("slotRenderEnded", observe("rendered")); + ts.firstImpressionListenersInstalled = true; + }); + } + + installFirstImpressionListeners(); + // Minimal fallback for tsjs.scheduleInitialAdInit, mirroring the bundle's // hydration-safe scheduler in // crates/trusted-server-js/lib/src/integrations/gpt/index.ts: the @@ -412,10 +585,138 @@ installSlotHandoff(); + function bootstrapTargeting(slot, bid) { + var targeting = Object.assign({}, slot.targeting || {}); + ["hb_pb", "hb_bidder", "hb_adid", "hb_cache_host", "hb_cache_path"].forEach( + function (key) { + if (bid[key]) targeting[key] = String(bid[key]); + }, + ); + targeting.ts_initial = "1"; + return targeting; + } + + function scheduleFirstImpressionFallback(slot, bid, element, generation) { + var state = firstImpressionState(Date.now()); + if (state.fallbackSlots[element.id]) return; + state.fallbackSlots[element.id] = element; + + var retry = function () { + if ( + (ts.navGeneration || 0) !== generation || + !element.isConnected || + document.getElementById(element.id) !== element + ) { + return; + } + var claim = firstImpressionClaim(element); + if (claim) { + if ( + claim.owner !== "publisher" || + claim.phase === "requested" || + claim.phase === "rendered" + ) { + return; + } + var delay = Math.max(0, claim.expiresAt - Date.now()); + if (delay > 0) { + window.setTimeout(retry, delay + 1); + return; + } + } + + tag.cmd.push(function () { + if ( + (ts.navGeneration || 0) !== generation || + !element.isConnected || + document.getElementById(element.id) !== element + ) { + return; + } + var fallbackClaim = claimFirstImpressionForTrustedServer(element); + if (!fallbackClaim) return; + var pubads = window.googletag.pubads(); + var existingSlots = pubads.getSlots ? pubads.getSlots() : []; + var gptSlot = + existingSlots.find(function (candidate) { + return candidate.getSlotElementId() === element.id; + }) || null; + var tsOwned = false; + if (!gptSlot) { + gptSlot = runHandoffInternal(function () { + return window.googletag.defineSlot( + slot.gam_unit_path, + slot.formats, + element.id, + ); + }); + if (!gptSlot) { + releaseTrustedServerFirstImpressionClaim(element, fallbackClaim); + return; + } + gptSlot.addService(pubads); + tsOwned = true; + ts.gptSlotHandoffs = ts.gptSlotHandoffs || {}; + ts.gptSlotHandoffs[element.id] = { + gamUnitPath: slot.gam_unit_path, + formats: slot.formats, + divIdPrefix: slot.div_id, + slotElementId: element.id, + publisherClaimed: false, + suppressPublisherDisplay: false, + suppressPublisherRefresh: false, + }; + } + + var targeting = bootstrapTargeting(slot, bid); + Object.entries(targeting).forEach(function (entry) { + gptSlot.setTargeting(entry[0], entry[1]); + }); + fallbackClaim.targeting = targeting; + var slotElementId = gptSlot.getSlotElementId() || element.id; + ts.divToSlotId = ts.divToSlotId || {}; + ts.divToSlotId[element.id] = slot.id; + ts.divToSlotId[slotElementId] = slot.id; + ts.prevSlotTargetingKeys = ts.prevSlotTargetingKeys || {}; + var targetingKeys = Object.keys(slot.targeting || {}); + ts.prevSlotTargetingKeys[element.id] = targetingKeys; + ts.prevSlotTargetingKeys[slotElementId] = targetingKeys; + if (tsOwned) { + ts.prevGptSlots = ts.prevGptSlots || []; + ts.prevGptSlots.push(gptSlot); + } + if (!ts.servicesEnabled) { + pubads.enableSingleRequest(); + window.googletag.enableServices(); + ts.servicesEnabled = true; + } + if (tsOwned) { + runHandoffInternal(function () { + window.googletag.display(slotElementId); + }); + } + syncInitialLoadDisabled(window.googletag); + if (!tsOwned || ts.gptInitialLoadDisabled) { + ts.adInitRefreshInProgress = true; + try { + runHandoffInternal(function () { + pubads.refresh([gptSlot]); + }); + } finally { + ts.adInitRefreshInProgress = false; + } + } + }); + }; + + retry(); + } + ts.adInit = function () { var slots = ts.adSlots || []; var bids = ts.bids || {}; var divToSlotId = {}; + var nextSlotTargetingKeys = {}; // Generation this invocation belongs to. The slot work below is queued on // googletag.cmd, which drains only when GPT loads; recheck first inside // the queued callback so a navigation committed in the gap cancels the @@ -476,6 +777,14 @@ } var actualDivId = el.id; var b = bids[slot.id] || {}; + var tsClaim = claimFirstImpressionForTrustedServer(el); + if (!tsClaim) { + var currentClaim = firstImpressionClaim(el); + if (currentClaim && currentClaim.owner === "publisher") { + scheduleFirstImpressionFallback(slot, b, el, generation); + } + return; + } var existingSlots = googletag.pubads().getSlots(); var s = @@ -493,7 +802,10 @@ actualDivId, ); }); - if (!s) return; + if (!s) { + releaseTrustedServerFirstImpressionClaim(el, tsClaim); + return; + } s.addService(googletag.pubads()); tsOwned = true; ts.gptSlotHandoffs = ts.gptSlotHandoffs || {}; @@ -508,27 +820,21 @@ }; } - Object.entries(slot.targeting || {}).forEach(function (e) { - s.setTargeting(e[0], e[1]); - }); - [ - "hb_pb", - "hb_bidder", - "hb_adid", - "hb_cache_host", - "hb_cache_path", - ].forEach(function (k) { - if (b[k]) s.setTargeting(k, b[k]); + var targeting = bootstrapTargeting(slot, b); + Object.entries(targeting).forEach(function (entry) { + s.setTargeting(entry[0], entry[1]); }); - // Keep in sync with TS_INITIAL_TARGETING_KEY in index.ts - s.setTargeting("ts_initial", "1"); + tsClaim.targeting = targeting; // Map the resolved inner div to the slot ID. This bootstrap fires no // beacons and registers no slotRenderEnded listener; the map is consumed // by the bundle's render bridge (index.ts) once it loads. divToSlotId[actualDivId] = slot.id; var slotElementId = s.getSlotElementId(); + var targetingKeys = Object.keys(slot.targeting || {}); + nextSlotTargetingKeys[actualDivId] = targetingKeys; if (slotElementId && slotElementId !== actualDivId) { divToSlotId[slotElementId] = slot.id; + nextSlotTargetingKeys[slotElementId] = targetingKeys; } if (tsOwned) { newSlots.push(s); @@ -540,7 +846,10 @@ }); ts.prevGptSlots = newSlots; ts.divToSlotId = divToSlotId; - if (!ts.servicesEnabled) { + ts.prevSlotTargetingKeys = nextSlotTargetingKeys; + var hasRenderableWork = + slotsToDisplay.length > 0 || slotsToRefresh.length > 0; + if (!ts.servicesEnabled && hasRenderableWork) { googletag.pubads().enableSingleRequest(); googletag.enableServices(); ts.servicesEnabled = true; diff --git a/crates/trusted-server-core/src/integrations/gpt_diagnostics.rs b/crates/trusted-server-core/src/integrations/gpt_diagnostics.rs index b4a188f2a..1447a8358 100644 --- a/crates/trusted-server-core/src/integrations/gpt_diagnostics.rs +++ b/crates/trusted-server-core/src/integrations/gpt_diagnostics.rs @@ -467,7 +467,12 @@ mod tests { #[test] fn register_excludes_diagnostics_from_unified_and_deferred_bundles() { - let registry = IntegrationRegistry::new(&settings(true)).expect("should build registry"); + let settings = settings(true); + let plan = std::sync::Arc::new( + crate::auction::compile_auction_plan(&settings).expect("should compile auction plan"), + ); + let registry = + IntegrationRegistry::with_plan(&settings, plan).expect("should build registry"); assert!(registry.integration_enabled(GPT_DIAGNOSTICS_INTEGRATION_ID)); assert!( diff --git a/crates/trusted-server-core/src/integrations/js_asset_proxy.rs b/crates/trusted-server-core/src/integrations/js_asset_proxy.rs new file mode 100644 index 000000000..6338590e3 --- /dev/null +++ b/crates/trusted-server-core/src/integrations/js_asset_proxy.rs @@ -0,0 +1,1570 @@ +//! JavaScript asset proxy integration. +//! +//! This integration serves explicitly configured third-party JavaScript assets +//! from first-party paths. Each asset maps one exact publisher-facing path to +//! one exact HTTPS upstream URL and can independently enable proxying, disable +//! proxying, or block matching script tags from publisher HTML. + +use std::collections::HashSet; +use std::sync::Arc; + +use async_trait::async_trait; +use edgezero_core::body::Body as EdgeBody; +use error_stack::Report; +use http::{Method, Request, Response, StatusCode, header}; +use serde::{Deserialize, Serialize}; +use url::Url; +use validator::{Validate, ValidationError, ValidationErrors}; + +use crate::constants::{ + HEADER_ACCEPT, HEADER_ACCEPT_ENCODING, HEADER_ACCEPT_LANGUAGE, HEADER_USER_AGENT, +}; +use crate::error::TrustedServerError; +use crate::integrations::{ + AttributeRewriteAction, IntegrationAttributeContext, IntegrationAttributeRewriter, + IntegrationEndpoint, IntegrationProxy, IntegrationRegistration, +}; +use crate::platform::RuntimeServices; +use crate::proxy::{ProxyRequestConfig, proxy_request}; +use crate::settings::{IntegrationConfig, Settings}; + +pub(crate) const JS_ASSET_PROXY_INTEGRATION_ID: &str = "js_asset_proxy"; +const HEADER_X_TS_JS_ASSET_PROXY: &str = "X-TS-JS-Asset-Proxy"; +const HEADER_X_TS_ERROR: &str = "X-TS-Error"; +const JS_ASSET_CONTENT_TYPE: &str = "application/javascript; charset=utf-8"; +const X_CONTENT_TYPE_OPTIONS_NOSNIFF: &str = "nosniff"; +const ERROR_ORIGIN_UNREACHABLE: &str = "js-asset-origin-unreachable"; +const ERROR_ORIGIN_STATUS: &str = "js-asset-origin-status"; + +/// Configuration for the JavaScript asset proxy integration. +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct JsAssetProxyConfig { + /// Enables or disables the integration. + #[serde(default)] + pub enabled: bool, + /// Optional downstream cache TTL override for every asset. + #[serde(default)] + pub cache_ttl_seconds: Option, + /// JavaScript assets managed by this integration. + #[serde(default)] + pub assets: Vec, +} + +/// One configured JavaScript asset mapping. +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct JsAssetProxyAsset { + /// Exact first-party request path handled by Trusted Server. + pub path: String, + /// Exact upstream JavaScript URL to fetch and to match during HTML rewriting. + pub origin_url: String, + /// Per-asset proxy behavior. + #[serde(default)] + pub proxy: JsAssetProxyMode, + /// Optional downstream cache TTL override for this asset. + #[serde(default)] + pub cache_ttl_seconds: Option, +} + +/// Per-asset proxy behavior. +#[derive(Debug, Clone, Copy, Default, Deserialize, Eq, PartialEq, Serialize)] +#[serde(rename_all = "lowercase")] +pub enum JsAssetProxyMode { + /// Rewrite matching script URLs and serve the configured route. + #[default] + Enabled, + /// Keep the asset in configuration without rewriting or route registration. + Disabled, + /// Remove matching script elements without route registration. + Blocked, +} + +impl JsAssetProxyConfig { + fn normalize_origin_urls(&mut self) { + for asset in &mut self.assets { + if let Some(origin_url) = normalize_origin_url(&asset.origin_url) { + asset.origin_url = origin_url; + } + } + } +} + +impl IntegrationConfig for JsAssetProxyConfig { + fn is_enabled(&self) -> bool { + self.enabled + } +} + +impl Validate for JsAssetProxyConfig { + fn validate(&self) -> Result<(), ValidationErrors> { + let mut errors = ValidationErrors::new(); + errors.merge_self("assets", self.assets.validate()); + + if self.enabled && self.assets.is_empty() { + errors.add("assets", ValidationError::new("empty_assets")); + } + + let mut paths = HashSet::new(); + let mut origin_urls = HashSet::new(); + for asset in &self.assets { + if !paths.insert(asset.path.as_str()) { + errors.add("asset_path", ValidationError::new("duplicate_asset_path")); + } + let origin_url = + normalize_origin_url(&asset.origin_url).unwrap_or_else(|| asset.origin_url.clone()); + if !origin_urls.insert(origin_url) { + errors.add( + "asset_origin_url", + ValidationError::new("duplicate_asset_origin_url"), + ); + } + } + + if errors.is_empty() { + Ok(()) + } else { + Err(errors) + } + } +} + +impl Validate for JsAssetProxyAsset { + fn validate(&self) -> Result<(), ValidationErrors> { + let mut errors = ValidationErrors::new(); + + if !self.path.starts_with('/') { + errors.add("path", ValidationError::new("path_must_start_with_slash")); + } + if self.path == "/" { + errors.add("path", ValidationError::new("path_must_not_be_root")); + } + if self.path.starts_with("//") { + errors.add( + "path", + ValidationError::new("path_must_not_be_protocol_relative"), + ); + } + if self.path.contains('*') { + errors.add( + "path", + ValidationError::new("path_must_not_contain_wildcard"), + ); + } + if path_contains_parent_segment(&self.path) { + errors.add( + "path", + ValidationError::new("path_must_not_contain_parent_segment"), + ); + } + if self.path.contains(['{', '}']) { + errors.add("path", ValidationError::new("path_must_be_exact_route")); + } + if self.path.contains(['?', '#']) { + errors.add( + "path", + ValidationError::new("path_must_not_contain_query_or_fragment"), + ); + } + if self + .path + .chars() + .any(|ch| ch.is_whitespace() || ch.is_control()) + { + errors.add( + "path", + ValidationError::new("path_must_not_contain_whitespace_or_control"), + ); + } + + match Url::parse(&self.origin_url) { + Ok(url) => { + if url.scheme() != "https" { + errors.add( + "origin_url", + ValidationError::new("origin_url_must_be_https"), + ); + } + if url.host_str().is_none() { + errors.add( + "origin_url", + ValidationError::new("origin_url_must_have_host"), + ); + } + } + Err(_) => { + errors.add("origin_url", ValidationError::new("invalid_origin_url")); + } + } + + if errors.is_empty() { + Ok(()) + } else { + Err(errors) + } + } +} + +fn path_contains_parent_segment(path: &str) -> bool { + path.split('/').any(|segment| segment == "..") +} + +fn normalize_origin_url(origin_url: &str) -> Option { + let mut url = Url::parse(origin_url).ok()?; + let has_default_port = matches!( + (url.scheme(), url.port()), + ("http", Some(80)) | ("https", Some(443)) + ); + if has_default_port { + url.set_port(None).ok()?; + } + + Some(url.to_string()) +} + +fn normalize_script_src(script_src: &str, request_scheme: &str) -> Option { + let candidate = if script_src.starts_with("//") { + let request_scheme = request_scheme.to_ascii_lowercase(); + if !matches!(request_scheme.as_str(), "http" | "https") { + return None; + } + format!("{request_scheme}:{script_src}") + } else { + script_src.to_string() + }; + + normalize_origin_url(&candidate) +} + +/// JavaScript asset proxy integration implementation. +pub struct JsAssetProxyIntegration { + config: JsAssetProxyConfig, +} + +impl JsAssetProxyIntegration { + fn new(config: JsAssetProxyConfig) -> Arc { + Arc::new(Self { config }) + } + + fn error(message: impl Into) -> TrustedServerError { + TrustedServerError::Integration { + integration: JS_ASSET_PROXY_INTEGRATION_ID.to_string(), + message: message.into(), + } + } + + fn enabled_asset_for_path(&self, path: &str) -> Option<&JsAssetProxyAsset> { + self.config + .assets + .iter() + .find(|asset| asset.proxy == JsAssetProxyMode::Enabled && asset.path == path) + } + + fn asset_for_origin_url(&self, origin_url: &str) -> Option<&JsAssetProxyAsset> { + self.config + .assets + .iter() + .find(|asset| asset.origin_url == origin_url) + } + + fn asset_for_script_src( + &self, + script_src: &str, + ctx: &IntegrationAttributeContext<'_>, + ) -> Option<&JsAssetProxyAsset> { + self.asset_for_origin_url(script_src).or_else(|| { + let normalized_src = normalize_script_src(script_src, ctx.request_scheme)?; + self.asset_for_origin_url(&normalized_src) + }) + } + + fn build_proxy_config<'a>( + origin_url: &'a str, + req: &Request, + ) -> ProxyRequestConfig<'a> { + let mut config = ProxyRequestConfig::new(origin_url) + .with_streaming() + .with_stream_response() + .without_forward_headers(); + config.follow_redirects = false; + config.forward_ec_id = false; + + for header_name in [ + &HEADER_ACCEPT, + &HEADER_ACCEPT_LANGUAGE, + &HEADER_ACCEPT_ENCODING, + ] { + if let Some(value) = req.headers().get(header_name).cloned() { + config = config.with_header(header_name.clone(), value); + } + } + + config.with_header( + HEADER_USER_AGENT.clone(), + http::HeaderValue::from_static("TrustedServer/1.0"), + ) + } + + fn origin_host(origin_url: &str) -> String { + Url::parse(origin_url) + .ok() + .and_then(|url| url.host_str().map(str::to_string)) + .unwrap_or_else(|| "unknown".to_string()) + } + + fn origin_unreachable_response() -> Response { + Response::builder() + .status(StatusCode::BAD_GATEWAY) + .header(HEADER_X_TS_ERROR, ERROR_ORIGIN_UNREACHABLE) + .body(EdgeBody::empty()) + .expect("should build JS asset proxy unreachable response") + } + + fn origin_status_response() -> Response { + Response::builder() + .status(StatusCode::BAD_GATEWAY) + .header(HEADER_X_TS_ERROR, ERROR_ORIGIN_STATUS) + .body(EdgeBody::empty()) + .expect("should build JS asset proxy upstream status response") + } + + fn combined_header_values( + headers: &http::HeaderMap, + header_name: &http::header::HeaderName, + ) -> Option { + let values = headers + .get_all(header_name) + .iter() + .filter_map(|value| value.to_str().ok()) + .collect::>(); + + (!values.is_empty()).then(|| values.join(", ")) + } + + fn vary_with_accept_encoding(upstream_vary: Option<&str>) -> String { + match upstream_vary.map(str::trim) { + Some("*") => "*".to_string(), + Some(vary) if !vary.is_empty() => { + if vary + .split(',') + .any(|header_name| header_name.trim().eq_ignore_ascii_case("accept-encoding")) + { + vary.to_string() + } else { + format!("{vary}, Accept-Encoding") + } + } + _ => "Accept-Encoding".to_string(), + } + } + + fn resolved_cache_ttl_seconds(&self, asset: &JsAssetProxyAsset) -> Option { + asset.cache_ttl_seconds.or(self.config.cache_ttl_seconds) + } + + fn finalize_asset_response( + &self, + asset: &JsAssetProxyAsset, + response: Response, + ) -> Response { + let (parts, body) = response.into_parts(); + let status = parts.status; + let content_encoding = parts.headers.get(header::CONTENT_ENCODING).cloned(); + let etag = parts.headers.get(header::ETAG).cloned(); + let last_modified = parts.headers.get(header::LAST_MODIFIED).cloned(); + let upstream_vary = Self::combined_header_values(&parts.headers, &header::VARY); + let upstream_cache_control = + Self::combined_header_values(&parts.headers, &header::CACHE_CONTROL); + + let mut finalized = Response::new(body); + *finalized.status_mut() = status; + finalized.headers_mut().insert( + HEADER_X_TS_JS_ASSET_PROXY, + http::HeaderValue::from_static("true"), + ); + // Upstream bytes are served from the publisher origin, so the upstream + // cannot choose a document MIME type or opt into browser MIME sniffing. + finalized.headers_mut().insert( + header::CONTENT_TYPE, + http::HeaderValue::from_static(JS_ASSET_CONTENT_TYPE), + ); + finalized.headers_mut().insert( + header::X_CONTENT_TYPE_OPTIONS, + http::HeaderValue::from_static(X_CONTENT_TYPE_OPTIONS_NOSNIFF), + ); + + if let Some(content_encoding) = content_encoding { + finalized + .headers_mut() + .insert(header::CONTENT_ENCODING, content_encoding); + finalized.headers_mut().insert( + header::VARY, + http::HeaderValue::from_str(&Self::vary_with_accept_encoding( + upstream_vary.as_deref(), + )) + .expect("should build JS asset proxy Vary header"), + ); + } else if let Some(upstream_vary) = upstream_vary { + finalized.headers_mut().insert( + header::VARY, + http::HeaderValue::from_str(&upstream_vary) + .expect("should preserve JS asset proxy upstream Vary header"), + ); + } + if let Some(etag) = etag { + finalized.headers_mut().insert(header::ETAG, etag); + } + if let Some(last_modified) = last_modified { + finalized + .headers_mut() + .insert(header::LAST_MODIFIED, last_modified); + } + + if let Some(ttl) = self.resolved_cache_ttl_seconds(asset) { + finalized.headers_mut().insert( + header::CACHE_CONTROL, + http::HeaderValue::from_str(&format!("public, max-age={ttl}")) + .expect("should build JS asset proxy Cache-Control header"), + ); + } else if let Some(cache_control) = upstream_cache_control { + finalized.headers_mut().insert( + header::CACHE_CONTROL, + http::HeaderValue::from_str(&cache_control) + .expect("should preserve JS asset proxy upstream Cache-Control header"), + ); + } + + finalized + } +} + +fn build( + settings: &Settings, +) -> Result>, Report> { + let Some(mut config) = + settings.integration_config::(JS_ASSET_PROXY_INTEGRATION_ID)? + else { + return Ok(None); + }; + config.normalize_origin_urls(); + + Ok(Some(JsAssetProxyIntegration::new(config))) +} + +/// Register the JavaScript asset proxy integration. +/// +/// # Errors +/// +/// Returns an error when the integration is enabled with invalid configuration. +pub fn register( + settings: &Settings, +) -> Result, Report> { + let Some(integration) = build(settings)? else { + return Ok(None); + }; + + Ok(Some( + IntegrationRegistration::builder(JS_ASSET_PROXY_INTEGRATION_ID) + .with_proxy(integration.clone()) + .with_attribute_rewriter(integration) + .build(), + )) +} + +#[async_trait(?Send)] +impl IntegrationProxy for JsAssetProxyIntegration { + fn integration_name(&self) -> &'static str { + JS_ASSET_PROXY_INTEGRATION_ID + } + + fn routes(&self) -> Vec { + self.config + .assets + .iter() + .filter(|asset| asset.proxy == JsAssetProxyMode::Enabled) + .map(|asset| IntegrationEndpoint::new(Method::GET, asset.path.clone())) + .collect() + } + + async fn handle( + &self, + settings: &Settings, + services: &RuntimeServices, + req: Request, + ) -> Result, Report> { + let request_path = req.uri().path().to_string(); + let asset = self.enabled_asset_for_path(&request_path).ok_or_else(|| { + Report::new(Self::error(format!( + "Unknown JavaScript asset proxy route: {request_path}" + ))) + })?; + + let origin_host = Self::origin_host(&asset.origin_url); + let proxy_config = Self::build_proxy_config(&asset.origin_url, &req); + let response = match proxy_request(settings, req, proxy_config, services).await { + Ok(response) => response, + Err(error) => { + log::warn!( + "JS asset origin unreachable for path {} host {}: {:?}", + request_path, + origin_host, + error + ); + return Ok(Self::origin_unreachable_response()); + } + }; + + if !response.status().is_success() { + log::warn!( + "JS asset origin returned status {} for path {} host {}", + response.status(), + request_path, + origin_host + ); + return Ok(Self::origin_status_response()); + } + + Ok(self.finalize_asset_response(asset, response)) + } +} + +impl IntegrationAttributeRewriter for JsAssetProxyIntegration { + fn integration_id(&self) -> &'static str { + JS_ASSET_PROXY_INTEGRATION_ID + } + + fn handles_attribute(&self, attribute: &str) -> bool { + attribute == "src" + } + + fn rewrite( + &self, + attr_name: &str, + attr_value: &str, + ctx: &IntegrationAttributeContext<'_>, + ) -> AttributeRewriteAction { + if attr_name != "src" || !ctx.element_name.eq_ignore_ascii_case("script") { + return AttributeRewriteAction::keep(); + } + + let Some(asset) = self.asset_for_script_src(attr_value, ctx) else { + return AttributeRewriteAction::keep(); + }; + + match asset.proxy { + JsAssetProxyMode::Enabled => AttributeRewriteAction::replace(asset.path.clone()), + JsAssetProxyMode::Disabled => AttributeRewriteAction::keep(), + JsAssetProxyMode::Blocked => AttributeRewriteAction::remove_element(), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::io::Cursor; + use std::sync::Arc; + + use crate::constants::{HEADER_REFERER, HEADER_X_FORWARDED_FOR, HEADER_X_TS_EC}; + use crate::html_processor::{BodyCloseInjection, HtmlProcessorConfig, create_html_processor}; + use crate::integrations::{ + AttributeRewriteAction, IntegrationAttributeRewriter, IntegrationRegistry, + }; + use crate::platform::test_support::{StubHttpClient, build_services_with_http_client}; + use crate::streaming_processor::{Compression, PipelineConfig, StreamingPipeline}; + use crate::test_support::tests::create_test_settings; + use http::header; + use serde_json::json; + + fn build_http_request(method: Method, uri: &str) -> Request { + Request::builder() + .method(method) + .uri(uri) + .body(EdgeBody::empty()) + .expect("should build HTTP request") + } + + fn asset(path: &str, origin_url: &str, proxy: JsAssetProxyMode) -> JsAssetProxyAsset { + JsAssetProxyAsset { + path: path.to_string(), + origin_url: origin_url.to_string(), + proxy, + cache_ttl_seconds: None, + } + } + + fn config_with_assets(assets: Vec) -> JsAssetProxyConfig { + JsAssetProxyConfig { + enabled: true, + cache_ttl_seconds: None, + assets, + } + } + + fn rewrite_context() -> IntegrationAttributeContext<'static> { + IntegrationAttributeContext { + attribute_name: "src", + element_name: "script", + request_host: "publisher.example.com", + request_scheme: "https", + origin_host: "origin.example.com", + } + } + + fn process_html_with_integration( + html: &str, + integration: Arc, + ) -> String { + let rewriter: Arc = integration; + process_html_with_registry( + html, + IntegrationRegistry::from_rewriters(vec![rewriter], Vec::new()), + ) + } + + fn process_html_with_registry(html: &str, integrations: IntegrationRegistry) -> String { + let processor = create_html_processor(HtmlProcessorConfig { + csp_nonce_observed: None, + body_close: BodyCloseInjection::None, + origin_host: "origin.example.com".to_string(), + request_host: "publisher.example.com".to_string(), + request_scheme: "https".to_string(), + integrations, + ad_slots_script: None, + ad_bids_state: Arc::new(std::sync::Mutex::new(None)), + max_buffered_body_bytes: 16 * 1024 * 1024, + gpt_diagnostics: None, + suppress_datadome_client_side_tag: false, + }); + let pipeline_config = PipelineConfig { + input_compression: Compression::None, + output_compression: Compression::None, + chunk_size: 8192, + }; + let mut pipeline = StreamingPipeline::new(pipeline_config, processor); + + let mut output = Vec::new(); + pipeline + .process(Cursor::new(html.as_bytes()), &mut output) + .expect("should process HTML"); + String::from_utf8(output).expect("should produce UTF-8 HTML") + } + + #[test] + fn disabled_config_does_not_register_routes() { + let mut settings = create_test_settings(); + settings + .integrations + .insert_config( + JS_ASSET_PROXY_INTEGRATION_ID, + &json!({ + "enabled": false, + "assets": [{ + "path": "/assets/vendor.js", + "origin_url": "https://cdn.example.com/vendor.js" + }] + }), + ) + .expect("should insert integration config"); + + let registry = IntegrationRegistry::new(&settings).expect("should build registry"); + + assert!( + !registry.has_route(&Method::GET, "/assets/vendor.js"), + "disabled integration should not register asset route" + ); + } + + #[test] + fn enabled_config_requires_at_least_one_asset() { + let config = JsAssetProxyConfig { + enabled: true, + cache_ttl_seconds: None, + assets: Vec::new(), + }; + + assert!( + config.validate().is_err(), + "enabled config should reject empty assets" + ); + } + + #[test] + fn proxy_modes_control_routes_and_rewriting() { + let integration = JsAssetProxyIntegration::new(config_with_assets(vec![ + asset( + "/assets/enabled.js", + "https://cdn.example.com/enabled.js", + JsAssetProxyMode::Enabled, + ), + asset( + "/assets/disabled.js", + "https://cdn.example.com/disabled.js", + JsAssetProxyMode::Disabled, + ), + asset( + "/assets/blocked.js", + "https://cdn.example.com/blocked.js", + JsAssetProxyMode::Blocked, + ), + ])); + + let routes = integration.routes(); + assert_eq!( + routes.len(), + 1, + "only enabled assets should register routes" + ); + assert_eq!(routes[0].method, Method::GET); + assert_eq!(routes[0].path, "/assets/enabled.js"); + + let ctx = rewrite_context(); + assert!(matches!( + integration.rewrite("src", "https://cdn.example.com/enabled.js", &ctx), + AttributeRewriteAction::Replace(ref value) if value == "/assets/enabled.js" + )); + assert!(matches!( + integration.rewrite("src", "https://cdn.example.com/disabled.js", &ctx), + AttributeRewriteAction::Keep + )); + assert!(matches!( + integration.rewrite("src", "https://cdn.example.com/blocked.js", &ctx), + AttributeRewriteAction::RemoveElement + )); + } + + #[test] + fn non_exact_origin_url_matches_are_not_rewritten_or_blocked() { + let integration = JsAssetProxyIntegration::new(config_with_assets(vec![asset( + "/assets/vendor.js", + "https://cdn.example.com/vendor.js", + JsAssetProxyMode::Enabled, + )])); + let ctx = rewrite_context(); + + assert!(matches!( + integration.rewrite("src", "https://cdn.example.com/vendor.js?v=1", &ctx), + AttributeRewriteAction::Keep + )); + assert!(matches!( + integration.rewrite("src", "https://cdn.example.com/other.js", &ctx), + AttributeRewriteAction::Keep + )); + } + + #[test] + fn non_script_src_matches_are_not_rewritten_or_blocked() { + let integration = JsAssetProxyIntegration::new(config_with_assets(vec![ + asset( + "/assets/enabled.js", + "https://cdn.example.com/enabled.js", + JsAssetProxyMode::Enabled, + ), + asset( + "/assets/blocked.js", + "https://cdn.example.com/blocked.js", + JsAssetProxyMode::Blocked, + ), + ])); + let ctx = IntegrationAttributeContext { + attribute_name: "src", + element_name: "img", + request_host: "publisher.example.com", + request_scheme: "https", + origin_host: "origin.example.com", + }; + + assert!(matches!( + integration.rewrite("src", "https://cdn.example.com/enabled.js", &ctx), + AttributeRewriteAction::Keep + )); + assert!(matches!( + integration.rewrite("src", "https://cdn.example.com/blocked.js", &ctx), + AttributeRewriteAction::Keep + )); + } + + #[test] + fn html_rewriting_only_applies_to_script_src_elements() { + let integration = JsAssetProxyIntegration::new(config_with_assets(vec![ + asset( + "/assets/enabled.js", + "https://cdn.example.com/enabled.js", + JsAssetProxyMode::Enabled, + ), + asset( + "/assets/blocked.js", + "https://cdn.example.com/blocked.js", + JsAssetProxyMode::Blocked, + ), + ])); + let html = r#" + + + + + "#; + + let processed = process_html_with_integration(html, integration); + + assert!(processed.contains(r#""#)); + assert!(processed.contains(r#""#)); + assert!(!processed.contains("blocked()")); + assert!(processed.contains(r#""#)); + } + + #[test] + fn script_src_matching_normalizes_common_browser_url_forms() { + let integration = JsAssetProxyIntegration::new(config_with_assets(vec![asset( + "/assets/vendor.js", + "https://cdn.example.com/vendor.js", + JsAssetProxyMode::Enabled, + )])); + let ctx = rewrite_context(); + + for script_src in [ + "//cdn.example.com/vendor.js", + "HTTPS://CDN.EXAMPLE.COM/vendor.js", + "https://cdn.example.com:443/vendor.js", + ] { + assert!( + matches!( + integration.rewrite("src", script_src, &ctx), + AttributeRewriteAction::Replace(ref value) if value == "/assets/vendor.js" + ), + "script src {script_src} should normalize to the configured origin URL" + ); + } + } + + #[test] + fn js_asset_proxy_rewriter_takes_precedence_over_native_rewriters() { + let mut settings = create_test_settings(); + settings + .integrations + .insert_config("gpt", &json!({ "enabled": true })) + .expect("should insert GPT config"); + settings + .integrations + .insert_config( + JS_ASSET_PROXY_INTEGRATION_ID, + &json!({ + "enabled": true, + "assets": [{ + "path": "/assets/gpt.js", + "origin_url": "https://securepubads.g.doubleclick.net/tag/js/gpt.js", + "proxy": "enabled" + }] + }), + ) + .expect("should insert JS asset proxy config"); + let registry = IntegrationRegistry::new(&settings).expect("should build registry"); + let html = r#""#; + + let processed = process_html_with_registry(html, registry); + + assert!( + processed.contains(r#""#), + "JS asset proxy should rewrite before GPT native rewriter: {processed}" + ); + assert!( + !processed.contains("/integrations/gpt/script"), + "GPT native rewrite should not override JS asset proxy" + ); + } + + #[test] + fn js_asset_proxy_blocking_takes_precedence_over_native_rewriters() { + let mut settings = create_test_settings(); + settings + .integrations + .insert_config("gpt", &json!({ "enabled": true })) + .expect("should insert GPT config"); + settings + .integrations + .insert_config( + JS_ASSET_PROXY_INTEGRATION_ID, + &json!({ + "enabled": true, + "assets": [{ + "path": "/assets/gpt.js", + "origin_url": "https://securepubads.g.doubleclick.net/tag/js/gpt.js", + "proxy": "blocked" + }] + }), + ) + .expect("should insert JS asset proxy config"); + let registry = IntegrationRegistry::new(&settings).expect("should build registry"); + let html = r#""#; + + let processed = process_html_with_registry(html, registry); + + assert!( + !processed.contains("googletag.cmd"), + "blocked JS asset should remove the script element before GPT can rewrite it" + ); + assert!( + !processed.contains("/integrations/gpt/script"), + "GPT native rewrite should not keep a blocked script" + ); + } + + #[test] + fn disabled_js_asset_proxy_candidate_allows_native_rewriters() { + let mut settings = create_test_settings(); + settings + .integrations + .insert_config("gpt", &json!({ "enabled": true })) + .expect("should insert GPT config"); + settings + .integrations + .insert_config( + JS_ASSET_PROXY_INTEGRATION_ID, + &json!({ + "enabled": true, + "assets": [{ + "path": "/assets/gpt.js", + "origin_url": "https://securepubads.g.doubleclick.net/tag/js/gpt.js", + "proxy": "disabled" + }] + }), + ) + .expect("should insert JS asset proxy config"); + let registry = IntegrationRegistry::new(&settings).expect("should build registry"); + let html = r#""#; + + let processed = process_html_with_registry(html, registry); + + assert!( + processed.contains(r#""#), + "disabled JS asset proxy entries should not suppress native integration rewrites" + ); + } + + #[test] + fn rejects_duplicate_asset_paths() { + let config = config_with_assets(vec![ + asset( + "/assets/vendor.js", + "https://cdn.example.com/vendor-a.js", + JsAssetProxyMode::Enabled, + ), + asset( + "/assets/vendor.js", + "https://cdn.example.com/vendor-b.js", + JsAssetProxyMode::Enabled, + ), + ]); + + assert!( + config.validate().is_err(), + "duplicate asset paths should be rejected" + ); + } + + #[test] + fn rejects_duplicate_origin_urls() { + let config = config_with_assets(vec![ + asset( + "/assets/vendor-a.js", + "https://cdn.example.com/vendor.js", + JsAssetProxyMode::Enabled, + ), + asset( + "/assets/vendor-b.js", + "https://cdn.example.com/vendor.js", + JsAssetProxyMode::Enabled, + ), + ]); + + assert!( + config.validate().is_err(), + "duplicate origin URLs should be rejected" + ); + } + + #[test] + fn rejects_invalid_paths() { + for invalid_path in [ + "assets/vendor.js", + "//cdn.example.com/vendor.js", + "/assets/*.js", + "/assets/../vendor.js", + "/assets/{vendor}.js", + "/assets/vendor.js?v=1", + "/assets/vendor.js#v1", + "/", + "/assets/vendor js", + "/assets/vendor\n.js", + ] { + let config = config_with_assets(vec![asset( + invalid_path, + "https://cdn.example.com/vendor.js", + JsAssetProxyMode::Enabled, + )]); + + assert!( + config.validate().is_err(), + "path {invalid_path} should be rejected" + ); + } + } + + #[test] + fn rejects_non_https_origins() { + let config = config_with_assets(vec![asset( + "/assets/vendor.js", + "http://cdn.example.com/vendor.js", + JsAssetProxyMode::Enabled, + )]); + + assert!( + config.validate().is_err(), + "non-HTTPS origin should be rejected" + ); + } + + #[test] + fn rejects_unknown_proxy_mode() { + let toml = r#" + [[handlers]] + path = "^/secure" + username = "user" + password = "pass" + + [[handlers]] + path = "^/_ts/admin" + username = "admin" + password = "admin-pass" + + [publisher] + domain = "test-publisher.com" + cookie_domain = ".test-publisher.com" + origin_url = "https://origin.test-publisher.com" + proxy_secret = "unit-test-proxy-secret" + + [ec] + passphrase = "test-secret-key-32-bytes-minimum" + + [request_signing] + config_store_id = "test-config-store-id" + secret_store_id = "test-secret-store-id" + + [integrations.js_asset_proxy] + enabled = true + + [[integrations.js_asset_proxy.assets]] + path = "/assets/vendor.js" + origin_url = "https://cdn.example.com/vendor.js" + proxy = "passthrough" + "#; + let settings = Settings::from_toml(toml).expect("should parse settings TOML"); + + assert!( + settings + .integration_config::(JS_ASSET_PROXY_INTEGRATION_ID) + .is_err(), + "unknown proxy mode should fail deserialization" + ); + } + + #[test] + fn exact_configured_routes_are_registered() { + let mut settings = create_test_settings(); + settings + .integrations + .insert_config( + JS_ASSET_PROXY_INTEGRATION_ID, + &json!({ + "enabled": true, + "assets": [ + { + "path": "/assets/vendor.js", + "origin_url": "https://cdn.example.com/vendor.js" + }, + { + "path": "/assets/blocked.js", + "origin_url": "https://cdn.example.com/blocked.js", + "proxy": "blocked" + } + ] + }), + ) + .expect("should insert integration config"); + + let registry = IntegrationRegistry::new(&settings).expect("should build registry"); + + assert!(registry.has_route(&Method::GET, "/assets/vendor.js")); + assert!(!registry.has_route(&Method::GET, "/assets/vendor.js/extra")); + assert!(!registry.has_route(&Method::POST, "/assets/vendor.js")); + assert!(!registry.has_route(&Method::GET, "/assets/blocked.js")); + } + + #[test] + fn request_path_selects_the_correct_asset() { + let integration = JsAssetProxyIntegration::new(config_with_assets(vec![ + asset( + "/assets/a.js", + "https://cdn.example.com/a.js", + JsAssetProxyMode::Enabled, + ), + asset( + "/assets/b.js", + "https://cdn.example.com/b.js", + JsAssetProxyMode::Enabled, + ), + ])); + + let selected = integration + .enabled_asset_for_path("/assets/b.js") + .expect("should select configured asset"); + + assert_eq!(selected.origin_url, "https://cdn.example.com/b.js"); + } + + #[test] + fn successful_response_preserves_body_and_controls_expected_headers() { + let mut configured_asset = asset( + "/assets/vendor.js", + "https://cdn.example.com/vendor.js", + JsAssetProxyMode::Enabled, + ); + configured_asset.cache_ttl_seconds = Some(900); + let integration = + JsAssetProxyIntegration::new(config_with_assets(vec![configured_asset.clone()])); + let upstream = Response::builder() + .status(StatusCode::OK) + .header(header::CONTENT_TYPE, "text/html; charset=utf-8") + .header(header::CONTENT_ENCODING, "gzip") + .header(header::ETAG, "\"asset-etag\"") + .header(header::LAST_MODIFIED, "Tue, 10 Jun 2026 00:00:00 GMT") + .header(header::VARY, "Origin") + .header(header::CACHE_CONTROL, "private, max-age=1") + .header(header::SET_COOKIE, "session=1") + .body(EdgeBody::from("console.log('ok');")) + .expect("should build upstream JS asset response"); + + let response = integration.finalize_asset_response(&configured_asset, upstream); + + assert_eq!(response.status(), StatusCode::OK); + assert_eq!( + response + .headers() + .get(HEADER_X_TS_JS_ASSET_PROXY) + .and_then(|value| value.to_str().ok()), + Some("true") + ); + assert_eq!( + response + .headers() + .get(header::CONTENT_TYPE) + .and_then(|value| value.to_str().ok()), + Some(JS_ASSET_CONTENT_TYPE) + ); + assert_eq!( + response + .headers() + .get(header::X_CONTENT_TYPE_OPTIONS) + .and_then(|value| value.to_str().ok()), + Some(X_CONTENT_TYPE_OPTIONS_NOSNIFF) + ); + assert_eq!( + response + .headers() + .get(header::CONTENT_ENCODING) + .and_then(|value| value.to_str().ok()), + Some("gzip") + ); + assert_eq!( + response + .headers() + .get(header::ETAG) + .and_then(|value| value.to_str().ok()), + Some("\"asset-etag\"") + ); + assert_eq!( + response + .headers() + .get(header::LAST_MODIFIED) + .and_then(|value| value.to_str().ok()), + Some("Tue, 10 Jun 2026 00:00:00 GMT") + ); + assert_eq!( + response + .headers() + .get(header::VARY) + .and_then(|value| value.to_str().ok()), + Some("Origin, Accept-Encoding") + ); + assert_eq!( + response + .headers() + .get(header::CACHE_CONTROL) + .and_then(|value| value.to_str().ok()), + Some("public, max-age=900") + ); + assert!( + response.headers().get(header::SET_COOKIE).is_none(), + "Set-Cookie should not be forwarded" + ); + let body = futures::executor::block_on(response.into_body().into_bytes_bounded(1024)) + .expect("should read finalized JS asset body"); + assert_eq!( + body.to_vec(), + b"console.log('ok');".to_vec(), + "should preserve upstream body bytes" + ); + } + + #[test] + fn preserves_upstream_cache_control_without_ttl_override() { + let configured_asset = asset( + "/assets/vendor.js", + "https://cdn.example.com/vendor.js", + JsAssetProxyMode::Enabled, + ); + let integration = + JsAssetProxyIntegration::new(config_with_assets(vec![configured_asset.clone()])); + let upstream = Response::builder() + .status(StatusCode::OK) + .header(header::CACHE_CONTROL, "public, max-age=123") + .body(EdgeBody::from("body")) + .expect("should build upstream JS asset response"); + + let response = integration.finalize_asset_response(&configured_asset, upstream); + + assert_eq!( + response + .headers() + .get(header::CACHE_CONTROL) + .and_then(|value| value.to_str().ok()), + Some("public, max-age=123") + ); + } + + #[test] + fn integration_cache_ttl_overrides_upstream_cache_control() { + let configured_asset = asset( + "/assets/vendor.js", + "https://cdn.example.com/vendor.js", + JsAssetProxyMode::Enabled, + ); + let mut config = config_with_assets(vec![configured_asset.clone()]); + config.cache_ttl_seconds = Some(300); + let integration = JsAssetProxyIntegration::new(config); + let upstream = Response::builder() + .status(StatusCode::OK) + .header(header::CACHE_CONTROL, "private, max-age=1") + .body(EdgeBody::from("body")) + .expect("should build upstream JS asset response"); + + let response = integration.finalize_asset_response(&configured_asset, upstream); + + assert_eq!( + response + .headers() + .get(header::CACHE_CONTROL) + .and_then(|value| value.to_str().ok()), + Some("public, max-age=300") + ); + } + + #[test] + fn configured_origin_urls_are_canonicalized_for_matching_and_duplicates() { + let mut settings = create_test_settings(); + settings + .integrations + .insert_config( + JS_ASSET_PROXY_INTEGRATION_ID, + &json!({ + "enabled": true, + "assets": [{ + "path": "/assets/vendor.js", + "origin_url": "HTTPS://CDN.EXAMPLE.COM:443/vendor.js" + }] + }), + ) + .expect("should insert integration config"); + let registry = IntegrationRegistry::new(&settings).expect("should build registry"); + let processed = process_html_with_registry( + r#""#, + registry, + ); + + assert!(processed.contains(r#""#)); + + let config = config_with_assets(vec![ + asset( + "/assets/one.js", + "https://cdn.example.com/vendor.js", + JsAssetProxyMode::Enabled, + ), + asset( + "/assets/two.js", + "HTTPS://CDN.EXAMPLE.COM:443/vendor.js", + JsAssetProxyMode::Enabled, + ), + ]); + assert!( + config.validate().is_err(), + "canonical duplicate origin URLs should be rejected" + ); + } + + #[test] + fn finalize_asset_response_preserves_repeated_vary_and_cache_control_headers() { + let configured_asset = asset( + "/assets/vendor.js", + "https://cdn.example.com/vendor.js", + JsAssetProxyMode::Enabled, + ); + let integration = + JsAssetProxyIntegration::new(config_with_assets(vec![configured_asset.clone()])); + let upstream = Response::builder() + .status(StatusCode::OK) + .header(header::CONTENT_ENCODING, "gzip") + .header(header::VARY, "Origin") + .header(header::VARY, "User-Agent") + .header(header::CACHE_CONTROL, "public, max-age=60") + .header(header::CACHE_CONTROL, "immutable") + .body(EdgeBody::from("body")) + .expect("should build upstream JS asset response"); + + let response = integration.finalize_asset_response(&configured_asset, upstream); + + assert_eq!( + response + .headers() + .get(header::VARY) + .and_then(|value| value.to_str().ok()), + Some("Origin, User-Agent, Accept-Encoding") + ); + assert_eq!( + response + .headers() + .get(header::CACHE_CONTROL) + .and_then(|value| value.to_str().ok()), + Some("public, max-age=60, immutable") + ); + } + + #[test] + fn handle_maps_upstream_outcomes_and_respects_streaming_capability() { + futures::executor::block_on(async { + let settings = create_test_settings(); + let integration = JsAssetProxyIntegration::new(config_with_assets(vec![asset( + "/assets/vendor.js", + "https://cdn.example.com/vendor.js", + JsAssetProxyMode::Enabled, + )])); + + let buffered_stub = Arc::new(StubHttpClient::new()); + buffered_stub.set_streaming_responses_supported(false); + buffered_stub.push_response(200, b"ok".to_vec()); + let buffered_services = build_services_with_http_client( + Arc::clone(&buffered_stub) as Arc + ); + let success = integration + .handle( + &settings, + &buffered_services, + build_http_request( + Method::GET, + "https://publisher.example.com/assets/vendor.js", + ), + ) + .await + .expect("should proxy buffered asset response"); + assert_eq!(success.status(), StatusCode::OK); + assert_eq!(buffered_stub.recorded_stream_response_flags(), vec![false]); + + let streaming_stub = Arc::new(StubHttpClient::new()); + streaming_stub.set_streaming_responses_supported(true); + streaming_stub.push_response(200, b"ok".to_vec()); + let streaming_services = build_services_with_http_client( + Arc::clone(&streaming_stub) as Arc + ); + let success = integration + .handle( + &settings, + &streaming_services, + build_http_request( + Method::GET, + "https://publisher.example.com/assets/vendor.js", + ), + ) + .await + .expect("should proxy streaming asset response"); + assert_eq!(success.status(), StatusCode::OK); + assert_eq!(streaming_stub.recorded_stream_response_flags(), vec![true]); + + let unavailable_stub = Arc::new(StubHttpClient::new()); + let unavailable_services = build_services_with_http_client( + Arc::clone(&unavailable_stub) as Arc, + ); + let unavailable = integration + .handle( + &settings, + &unavailable_services, + build_http_request( + Method::GET, + "https://publisher.example.com/assets/vendor.js", + ), + ) + .await + .expect("should map unavailable origin response"); + assert_eq!(unavailable.status(), StatusCode::BAD_GATEWAY); + assert_eq!( + unavailable + .headers() + .get(HEADER_X_TS_ERROR) + .and_then(|value| value.to_str().ok()), + Some(ERROR_ORIGIN_UNREACHABLE) + ); + + let status_stub = Arc::new(StubHttpClient::new()); + status_stub.push_response(404, Vec::new()); + let status_services = build_services_with_http_client( + Arc::clone(&status_stub) as Arc + ); + let status = integration + .handle( + &settings, + &status_services, + build_http_request( + Method::GET, + "https://publisher.example.com/assets/vendor.js", + ), + ) + .await + .expect("should map non-success origin response"); + assert_eq!(status.status(), StatusCode::BAD_GATEWAY); + assert_eq!( + status + .headers() + .get(HEADER_X_TS_ERROR) + .and_then(|value| value.to_str().ok()), + Some(ERROR_ORIGIN_STATUS) + ); + }); + } + + #[test] + fn upstream_error_responses_have_expected_headers() { + let unreachable = JsAssetProxyIntegration::origin_unreachable_response(); + assert_eq!(unreachable.status(), StatusCode::BAD_GATEWAY); + assert_eq!( + unreachable + .headers() + .get(HEADER_X_TS_ERROR) + .and_then(|value| value.to_str().ok()), + Some(ERROR_ORIGIN_UNREACHABLE) + ); + + let origin_status = JsAssetProxyIntegration::origin_status_response(); + assert_eq!(origin_status.status(), StatusCode::BAD_GATEWAY); + assert_eq!( + origin_status + .headers() + .get(HEADER_X_TS_ERROR) + .and_then(|value| value.to_str().ok()), + Some(ERROR_ORIGIN_STATUS) + ); + } + + #[test] + fn build_proxy_config_forwards_only_asset_header_allowlist() { + let mut req = build_http_request( + Method::GET, + "https://publisher.example.com/assets/vendor.js", + ); + req.headers_mut().insert( + HEADER_ACCEPT.clone(), + http::HeaderValue::from_static("application/javascript"), + ); + req.headers_mut().insert( + HEADER_ACCEPT_LANGUAGE.clone(), + http::HeaderValue::from_static("en-US"), + ); + req.headers_mut().insert( + HEADER_ACCEPT_ENCODING.clone(), + http::HeaderValue::from_static("gzip, br"), + ); + req.headers_mut().insert( + HEADER_REFERER.clone(), + http::HeaderValue::from_static("https://publisher.example.com/page"), + ); + req.headers_mut().insert( + HEADER_X_FORWARDED_FOR.clone(), + http::HeaderValue::from_static("192.0.2.10"), + ); + req.headers_mut().insert( + HEADER_X_TS_EC.clone(), + http::HeaderValue::from_static("edge-cookie-id"), + ); + req.headers_mut() + .insert(header::COOKIE, http::HeaderValue::from_static("session=1")); + + let config = + JsAssetProxyIntegration::build_proxy_config("https://cdn.example.com/vendor.js", &req); + + assert!(!config.copy_request_headers); + assert!(!config.follow_redirects); + assert!(!config.forward_ec_id); + assert!(config.stream_passthrough); + assert!(config.stream_response); + + let forwarded: Vec<(String, String)> = config + .headers + .iter() + .map(|(name, value)| { + ( + name.as_str().to_string(), + value + .to_str() + .expect("should expose header value in test") + .to_string(), + ) + }) + .collect(); + + assert_eq!( + forwarded, + vec![ + ("accept".to_string(), "application/javascript".to_string()), + ("accept-language".to_string(), "en-US".to_string()), + ("accept-encoding".to_string(), "gzip, br".to_string()), + ("user-agent".to_string(), "TrustedServer/1.0".to_string()), + ] + ); + } + + #[test] + fn vary_with_accept_encoding_preserves_wildcard_and_existing_value() { + assert_eq!( + JsAssetProxyIntegration::vary_with_accept_encoding(Some("*")), + "*" + ); + assert_eq!( + JsAssetProxyIntegration::vary_with_accept_encoding(Some("Accept-Encoding")), + "Accept-Encoding" + ); + assert_eq!( + JsAssetProxyIntegration::vary_with_accept_encoding(Some("Origin")), + "Origin, Accept-Encoding" + ); + assert_eq!( + JsAssetProxyIntegration::vary_with_accept_encoding(None), + "Accept-Encoding" + ); + } + + #[test] + fn proxy_mode_defaults_to_enabled() { + let parsed: JsAssetProxyAsset = serde_json::from_value(json!({ + "path": "/assets/vendor.js", + "origin_url": "https://cdn.example.com/vendor.js" + })) + .expect("should deserialize asset"); + + assert_eq!(parsed.proxy, JsAssetProxyMode::Enabled); + } +} diff --git a/crates/trusted-server-core/src/integrations/lockr.rs b/crates/trusted-server-core/src/integrations/lockr.rs index 1f4f04b73..002182bd1 100644 --- a/crates/trusted-server-core/src/integrations/lockr.rs +++ b/crates/trusted-server-core/src/integrations/lockr.rs @@ -456,6 +456,7 @@ mod tests { fn test_context() -> IntegrationAttributeContext<'static> { IntegrationAttributeContext { attribute_name: "src", + element_name: "script", request_host: "edge.example.com", request_scheme: "https", origin_host: "origin.example.com", diff --git a/crates/trusted-server-core/src/integrations/mod.rs b/crates/trusted-server-core/src/integrations/mod.rs index 90d688693..c30b0e0c0 100644 --- a/crates/trusted-server-core/src/integrations/mod.rs +++ b/crates/trusted-server-core/src/integrations/mod.rs @@ -18,6 +18,7 @@ pub mod didomi; pub mod google_tag_manager; pub mod gpt; pub mod gpt_diagnostics; +pub mod js_asset_proxy; pub mod lockr; pub mod nextjs; pub mod osano; @@ -289,13 +290,10 @@ pub(crate) struct IntegrationBuilder { pub(crate) fn builders() -> &'static [IntegrationBuilder] { &[ + // This must remain first: attribute rewriters chain replacements and short-circuit removals. IntegrationBuilder { - id: "aps", - build: aps::register, - }, - IntegrationBuilder { - id: "prebid", - build: prebid::register, + id: js_asset_proxy::JS_ASSET_PROXY_INTEGRATION_ID, + build: js_asset_proxy::register, }, IntegrationBuilder { id: "testlight", diff --git a/crates/trusted-server-core/src/integrations/nextjs/mod.rs b/crates/trusted-server-core/src/integrations/nextjs/mod.rs index 5452260e7..015ba2475 100644 --- a/crates/trusted-server-core/src/integrations/nextjs/mod.rs +++ b/crates/trusted-server-core/src/integrations/nextjs/mod.rs @@ -160,7 +160,14 @@ mod tests { }), ) .expect("should update nextjs config"); - let registry = IntegrationRegistry::new(&settings).expect("should create registry"); + let registry = IntegrationRegistry::with_plan( + &settings, + Arc::new( + crate::auction::compile_auction_plan(&settings) + .expect("should compile auction plan"), + ), + ) + .expect("should create registry"); let config = config_from_settings(&settings, ®istry); let processor = create_html_processor(config); let pipeline_config = PipelineConfig { @@ -246,7 +253,14 @@ mod tests { }), ) .expect("should update nextjs config"); - let registry = IntegrationRegistry::new(&settings).expect("should create registry"); + let registry = IntegrationRegistry::with_plan( + &settings, + Arc::new( + crate::auction::compile_auction_plan(&settings) + .expect("should compile auction plan"), + ), + ) + .expect("should create registry"); let config = config_from_settings(&settings, ®istry); let processor = create_html_processor(config); let pipeline_config = PipelineConfig { @@ -316,7 +330,14 @@ mod tests { }), ) .expect("should update nextjs config"); - let registry = IntegrationRegistry::new(&settings).expect("should create registry"); + let registry = IntegrationRegistry::with_plan( + &settings, + Arc::new( + crate::auction::compile_auction_plan(&settings) + .expect("should compile auction plan"), + ), + ) + .expect("should create registry"); let config = config_from_settings(&settings, ®istry); let processor = create_html_processor(config); let pipeline_config = PipelineConfig { @@ -362,7 +383,14 @@ mod tests { }), ) .expect("should update nextjs config"); - let registry = IntegrationRegistry::new(&settings).expect("should create registry"); + let registry = IntegrationRegistry::with_plan( + &settings, + Arc::new( + crate::auction::compile_auction_plan(&settings) + .expect("should compile auction plan"), + ), + ) + .expect("should create registry"); let config = config_from_settings(&settings, ®istry); let processor = create_html_processor(config); let pipeline_config = PipelineConfig { @@ -411,7 +439,14 @@ mod tests { }), ) .expect("should update nextjs config"); - let registry = IntegrationRegistry::new(&settings).expect("should create registry"); + let registry = IntegrationRegistry::with_plan( + &settings, + Arc::new( + crate::auction::compile_auction_plan(&settings) + .expect("should compile auction plan"), + ), + ) + .expect("should create registry"); let config = config_from_settings(&settings, ®istry); let processor = create_html_processor(config); let pipeline_config = PipelineConfig { @@ -474,7 +509,14 @@ mod tests { ) .expect("should update nextjs config"); - let registry = IntegrationRegistry::new(&settings).expect("should create registry"); + let registry = IntegrationRegistry::with_plan( + &settings, + Arc::new( + crate::auction::compile_auction_plan(&settings) + .expect("should compile auction plan"), + ), + ) + .expect("should create registry"); let config = config_from_settings(&settings, ®istry); let processor = create_html_processor(config); let pipeline_config = PipelineConfig { @@ -543,7 +585,14 @@ mod tests { }), ) .expect("should update nextjs config"); - let registry = IntegrationRegistry::new(&settings).expect("should create registry"); + let registry = IntegrationRegistry::with_plan( + &settings, + Arc::new( + crate::auction::compile_auction_plan(&settings) + .expect("should compile auction plan"), + ), + ) + .expect("should create registry"); let config = config_from_settings(&settings, ®istry); let processor = create_html_processor(config); // Use small chunk size to force fragmentation @@ -604,7 +653,14 @@ mod tests { }), ) .expect("should update nextjs config"); - let registry = IntegrationRegistry::new(&settings).expect("should create registry"); + let registry = IntegrationRegistry::with_plan( + &settings, + Arc::new( + crate::auction::compile_auction_plan(&settings) + .expect("should compile auction plan"), + ), + ) + .expect("should create registry"); let config = config_from_settings(&settings, ®istry); let processor = create_html_processor(config); @@ -666,7 +722,14 @@ mod tests { }), ) .expect("should update nextjs config"); - let registry = IntegrationRegistry::new(&settings).expect("should create registry"); + let registry = IntegrationRegistry::with_plan( + &settings, + Arc::new( + crate::auction::compile_auction_plan(&settings) + .expect("should compile auction plan"), + ), + ) + .expect("should create registry"); let config = config_from_settings(&settings, ®istry); let processor = create_html_processor(config); diff --git a/crates/trusted-server-core/src/integrations/permutive.rs b/crates/trusted-server-core/src/integrations/permutive.rs index aa684c620..b59f4e382 100644 --- a/crates/trusted-server-core/src/integrations/permutive.rs +++ b/crates/trusted-server-core/src/integrations/permutive.rs @@ -541,6 +541,7 @@ mod tests { let ctx = IntegrationAttributeContext { attribute_name: "src", + element_name: "script", request_host: "edge.example.com", request_scheme: "https", origin_host: "origin.example.com", @@ -574,6 +575,7 @@ mod tests { let ctx = IntegrationAttributeContext { attribute_name: "src", + element_name: "script", request_host: "edge.example.com", request_scheme: "https", origin_host: "origin.example.com", diff --git a/crates/trusted-server-core/src/integrations/prebid.rs b/crates/trusted-server-core/src/integrations/prebid.rs index d0cf37275..b32939fc8 100644 --- a/crates/trusted-server-core/src/integrations/prebid.rs +++ b/crates/trusted-server-core/src/integrations/prebid.rs @@ -1,5 +1,8 @@ -use std::collections::{HashMap, HashSet}; +use std::collections::HashMap; +#[cfg(test)] +use std::collections::HashSet; use std::sync::{Arc, LazyLock}; +#[cfg(test)] use std::time::Duration; use async_trait::async_trait; @@ -19,29 +22,45 @@ use serde_json::Value as Json; use url::{Url, Url as ParsedUrl}; use validator::{Validate, ValidationError}; +use crate::auction::openrtb::{ + BidRejectionReason, ResponseAdmissionDiagnostics, parse_optional_bid_dimension, + resolve_bid_dimensions, +}; use crate::auction::orchestrator::ERROR_TYPE_HTTP_STATUS; +use crate::auction::plan::AuctionPlan; +use crate::auction::profile::PrebidProfilePlan; +#[cfg(test)] use crate::auction::provider::{AuctionProvider, ProviderRequestOutcome}; -use crate::auction::types::{ - AuctionContext, AuctionRequest, AuctionResponse, Bid as AuctionBid, MediaType, -}; +use crate::auction::routing::{PrebidTransportHeaders, ProviderAuctionInput}; +#[cfg(test)] +use crate::auction::types::{AuctionContext, AuctionRequest, MediaType}; +use crate::auction::types::{AuctionResponse, Bid as AuctionBid}; use crate::cache_policy::{CacheControlPolicy, EdgeCacheHeader}; use crate::consent_config::ConsentForwardingMode; use crate::cookies::{CONSENT_COOKIE_NAMES, strip_cookies}; use crate::error::TrustedServerError; +#[cfg(test)] use crate::http_util::RequestInfo; use crate::integrations::{ AttributeRewriteAction, IntegrationAttributeContext, IntegrationAttributeRewriter, IntegrationEndpoint, IntegrationHeadInjector, IntegrationHtmlContext, IntegrationProxy, IntegrationRegistration, UPSTREAM_RTB_MAX_RESPONSE_BYTES, collect_response_bounded, +}; +#[cfg(test)] +use crate::integrations::{ ensure_integration_backend_with_timeout, predict_integration_backend_name, }; +#[cfg(test)] use crate::openrtb::{ Banner, ConsentedProvidersSettings, Device, Format, Geo, Imp, ImpExt, ImpStoredRequest, OpenRtbRequest, PrebidExt, PrebidImpExt, Publisher, Regs, RegsExt, RequestExt, Site, ToExt, TrustedServerExt, User, UserExt, to_openrtb_i32, }; -use crate::platform::{PlatformHttpRequest, PlatformResponse, RuntimeServices}; +#[cfg(test)] +use crate::platform::PlatformHttpRequest; +use crate::platform::{PlatformResponse, RuntimeServices}; use crate::proxy::{ProxyRequestConfig, is_host_allowed, proxy_request}; +#[cfg(test)] use crate::request_signing::{RequestSigner, SIGNING_VERSION, SigningParams}; use crate::settings::{IntegrationConfig, Settings}; @@ -55,8 +74,11 @@ const PREBID_BUNDLE_ERROR_CACHE_CONTROL: &str = "no-store"; const PREBID_BUNDLE_ERROR_CONTENT_TYPE: &str = "text/plain; charset=utf-8"; const PREBID_BUNDLE_NOSNIFF_HEADER: &str = "x-content-type-options"; const PREBID_BUNDLE_NOSNIFF_VALUE: &str = "nosniff"; +#[cfg(test)] const TRUSTED_SERVER_BIDDER: &str = "trustedServer"; +#[cfg(test)] const BIDDER_PARAMS_KEY: &str = "bidderParams"; +#[cfg(test)] const ZONE_KEY: &str = "zone"; /// Default currency for `OpenRTB` bid floors and responses. @@ -200,8 +222,9 @@ fn extract_prebid_error_message( #[cfg(test)] const GPC_US_PRIVACY: &str = "1YYN"; +#[cfg(test)] #[derive(Debug, Clone, Deserialize, Serialize, Validate)] -pub struct PrebidIntegrationConfig { +pub struct LegacyPrebidServerConfig { #[serde(default = "default_enabled")] pub enabled: bool, #[validate(url)] @@ -343,13 +366,109 @@ pub struct PrebidIntegrationConfig { pub suppress_nurl_bidders: Vec, } +#[cfg(test)] +impl IntegrationConfig for LegacyPrebidServerConfig { + fn is_enabled(&self) -> bool { + self.enabled + } +} + +/// CLI build inputs retained in app config but ignored safely by the runtime. +#[derive(Debug, Clone, Default, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +pub struct PrebidBundleBuildConfig { + /// Prebid.js bidder adapters included by `ts prebid bundle`. + #[serde(default)] + pub adapters: Vec, + /// Optional Prebid.js user ID modules included by `ts prebid bundle`. + #[serde(default)] + pub user_id_modules: Option>, +} + +/// Browser-only Prebid integration settings. +#[derive(Debug, Clone, Deserialize, Serialize, Validate)] +#[serde(deny_unknown_fields)] +pub struct PrebidIntegrationConfig { + #[serde(default = "default_enabled")] + pub enabled: bool, + #[serde(default)] + pub account_id: Option, + #[serde(default = "default_timeout_ms")] + pub timeout_ms: u32, + #[serde(default)] + pub debug: bool, + #[serde( + default = "default_script_patterns", + deserialize_with = "crate::settings::vec_from_seq_or_map" + )] + pub script_patterns: Vec, + #[serde(default)] + #[validate(custom(function = "validate_external_bundle_url"))] + pub external_bundle_url: Option, + #[serde(default)] + #[validate(regex( + path = *EXTERNAL_BUNDLE_SHA256_PATTERN, + message = "external_bundle_sha256 must be a 64-character hex SHA-256" + ))] + pub external_bundle_sha256: Option, + #[serde(default)] + #[validate(custom(function = "validate_external_bundle_sri"))] + pub external_bundle_sri: Option, + #[serde(default, deserialize_with = "crate::settings::vec_from_seq_or_map")] + pub client_side_bidders: Vec, + #[serde(default, deserialize_with = "crate::settings::vec_from_seq_or_map")] + #[validate(custom(function = "validate_excluded_gam_ad_unit_path_suffixes"))] + pub excluded_gam_ad_unit_path_suffixes: Vec, + /// CLI-only external bundle build inputs; runtime registration ignores these fields. + #[serde(default)] + pub bundle: PrebidBundleBuildConfig, +} + +impl Default for PrebidIntegrationConfig { + fn default() -> Self { + Self { + enabled: default_enabled(), + account_id: None, + timeout_ms: default_timeout_ms(), + debug: false, + script_patterns: default_script_patterns(), + external_bundle_url: None, + external_bundle_sha256: None, + external_bundle_sri: None, + client_side_bidders: Vec::new(), + excluded_gam_ad_unit_path_suffixes: Vec::new(), + bundle: PrebidBundleBuildConfig::default(), + } + } +} + impl IntegrationConfig for PrebidIntegrationConfig { fn is_enabled(&self) -> bool { self.enabled } } -fn remove_aps_bidders(config: &mut PrebidIntegrationConfig) { +#[cfg(test)] +impl From<&LegacyPrebidServerConfig> for PrebidIntegrationConfig { + fn from(config: &LegacyPrebidServerConfig) -> Self { + Self { + enabled: config.enabled, + account_id: config.account_id.clone(), + timeout_ms: config.timeout_ms, + debug: config.debug, + script_patterns: config.script_patterns.clone(), + external_bundle_url: config.external_bundle_url.clone(), + external_bundle_sha256: config.external_bundle_sha256.clone(), + external_bundle_sri: config.external_bundle_sri.clone(), + client_side_bidders: config.client_side_bidders.clone(), + excluded_gam_ad_unit_path_suffixes: config.excluded_gam_ad_unit_path_suffixes.clone(), + bundle: PrebidBundleBuildConfig::default(), + } + } +} + +#[cfg(test)] +fn remove_aps_bidders(config: &mut LegacyPrebidServerConfig) { for (field, bidders) in [ ("bidders", &mut config.bidders), ("client_side_bidders", &mut config.client_side_bidders), @@ -406,7 +525,8 @@ fn validate_excluded_gam_ad_unit_path_suffixes(values: &[String]) -> Result<(), Ok(()) } -fn canonicalize_excluded_gam_ad_unit_path_suffixes(config: &mut PrebidIntegrationConfig) { +#[cfg(test)] +fn canonicalize_excluded_gam_ad_unit_path_suffixes(config: &mut LegacyPrebidServerConfig) { let mut canonical = Vec::with_capacity(config.excluded_gam_ad_unit_path_suffixes.len()); for suffix in std::mem::take(&mut config.excluded_gam_ad_unit_path_suffixes) { if !canonical.contains(&suffix) { @@ -416,11 +536,12 @@ fn canonicalize_excluded_gam_ad_unit_path_suffixes(config: &mut PrebidIntegratio config.excluded_gam_ad_unit_path_suffixes = canonical; } +#[cfg(test)] fn load_config( settings: &Settings, -) -> Result, Report> { +) -> Result, Report> { let Some(mut config) = - settings.integration_config::(PREBID_INTEGRATION_ID)? + settings.integration_config::(PREBID_INTEGRATION_ID)? else { return Ok(None); }; @@ -435,9 +556,10 @@ fn load_config( /// /// Returns a configuration error if enabled Prebid settings fail typed parsing, /// schema validation, or bidder-param override compilation. +#[cfg(test)] pub fn validate_config_for_startup( settings: &Settings, -) -> Result, Report> { +) -> Result, Report> { let Some(config) = load_config(settings)? else { return Ok(None); }; @@ -479,6 +601,7 @@ fn default_timeout_ms() -> u32 { 1000 } +#[cfg(test)] fn default_bidders() -> Vec { vec!["mocktioneer".to_string()] } @@ -609,11 +732,11 @@ fn validate_external_bundle_sri(value: &str) -> Result<(), ValidationError> { parse_external_bundle_sri(value) } -fn validate_external_bundle_config( - config: &PrebidIntegrationConfig, +fn validate_external_bundle_url_allowed( + external_bundle_url: Option<&str>, allowed_domains: &[String], ) -> Result<(), Report> { - let url = config.external_bundle_url.as_deref().ok_or_else(|| { + let url = external_bundle_url.ok_or_else(|| { Report::new(TrustedServerError::Configuration { message: "integrations.prebid.external_bundle_url is required when prebid is enabled" .to_string(), @@ -661,25 +784,96 @@ fn validate_external_bundle_config( Ok(()) } +pub(crate) fn validate_browser_config_for_startup( + config: &PrebidIntegrationConfig, + allowed_domains: &[String], +) -> Result<(), Report> { + validate_external_bundle_url_allowed(config.external_bundle_url.as_deref(), allowed_domains) +} + +pub(crate) fn validate_browser_bidder_ownership( + config: &PrebidIntegrationConfig, + plan: &AuctionPlan, +) -> Result<(), Report> { + if !plan.enabled() { + return Ok(()); + } + + let server_side = plan + .browser_bidder_codes() + .collect::>(); + let conflicts = config + .client_side_bidders + .iter() + .filter(|bidder| server_side.contains(bidder.as_str())) + .cloned() + .collect::>(); + if conflicts.is_empty() { + return Ok(()); + } + + Err(Report::new(TrustedServerError::Configuration { + message: format!( + "Prebid bidders must have exactly one browser owner; configured as both client-side and server-side: {}", + conflicts.into_iter().collect::>().join(", ") + ), + })) +} + +#[cfg(test)] +fn validate_external_bundle_config( + config: &LegacyPrebidServerConfig, + allowed_domains: &[String], +) -> Result<(), Report> { + validate_external_bundle_url_allowed(config.external_bundle_url.as_deref(), allowed_domains) +} + pub struct PrebidIntegration { config: PrebidIntegrationConfig, + planned_head_inserts: Option>, + #[cfg(test)] + legacy_config: Option, + #[cfg(test)] engine: Arc, } impl PrebidIntegration { - fn try_new(config: PrebidIntegrationConfig) -> Result, Report> { + #[cfg(test)] + fn try_new(config: LegacyPrebidServerConfig) -> Result, Report> { let engine = Arc::new(BidParamOverrideEngine::try_from_config(&config)?); - Ok(Arc::new(Self { config, engine })) + Ok(Arc::new(Self { + config: PrebidIntegrationConfig::from(&config), + planned_head_inserts: None, + legacy_config: Some(config), + engine, + })) } #[cfg(test)] - fn new(config: PrebidIntegrationConfig) -> Arc { + fn new(config: LegacyPrebidServerConfig) -> Arc { Self::try_new(config).expect("should compile prebid bid param overrides") } + fn for_browser_plan(config: &PrebidIntegrationConfig, plan: &AuctionPlan) -> Arc { + let mut integration = Self { + config: config.clone(), + planned_head_inserts: None, + #[cfg(test)] + legacy_config: None, + #[cfg(test)] + engine: Arc::new(BidParamOverrideEngine::default()), + }; + integration.planned_head_inserts = Some(integration.head_inserts_for_plan(config, plan)); + Arc::new(integration) + } + + #[cfg(test)] fn auction_provider(&self) -> PrebidAuctionProvider { PrebidAuctionProvider { - config: self.config.clone(), + config: self + .legacy_config + .clone() + .expect("should retain legacy config for legacy provider tests"), bid_param_override_engine: Arc::clone(&self.engine), } } @@ -767,23 +961,53 @@ impl PrebidIntegration { Ok(response) } - fn external_bundle_script_src(&self) -> String { - match self.config.external_bundle_sha256.as_deref() { - Some(sha256) => format!("{PREBID_BUNDLE_ROUTE}?v={sha256}"), - None => PREBID_BUNDLE_ROUTE.to_string(), - } + fn external_bundle_script_tag(&self) -> String { + external_bundle_script_tag( + self.config.external_bundle_sha256.as_deref(), + self.config.external_bundle_sri.as_deref(), + ) } - fn external_bundle_script_tag(&self) -> String { - let src = self.external_bundle_script_src(); - let integrity = self - .config - .external_bundle_sri - .as_deref() - .map(|value| format!(" integrity=\"{}\"", escape_html_attr(value))) - .unwrap_or_default(); + /// Build the prepared browser injection from browser settings and validated routes. + pub(crate) fn head_inserts_for_plan( + &self, + browser_config: &PrebidIntegrationConfig, + plan: &AuctionPlan, + ) -> Vec { + #[derive(Serialize)] + #[serde(rename_all = "camelCase")] + struct InjectedBrowserConfig<'a> { + account_id: &'a str, + timeout: u32, + debug: bool, + server_side_bidders: Vec<&'a str>, + #[serde(skip_serializing_if = "<[String]>::is_empty")] + client_side_bidders: &'a [String], + #[serde(skip_serializing_if = "<[String]>::is_empty")] + excluded_gam_ad_unit_path_suffixes: &'a [String], + } + + let payload = InjectedBrowserConfig { + account_id: browser_config.account_id.as_deref().unwrap_or_default(), + timeout: browser_config.timeout_ms, + debug: browser_config.debug, + server_side_bidders: if plan.enabled() { + plan.browser_bidder_codes().collect() + } else { + Vec::new() + }, + client_side_bidders: &browser_config.client_side_bidders, + excluded_gam_ad_unit_path_suffixes: &browser_config.excluded_gam_ad_unit_path_suffixes, + }; + let config_json = serialize_injected_prebid_config(&payload); - format!("") + vec![ + injected_prebid_config_script(&config_json), + external_bundle_script_tag( + browser_config.external_bundle_sha256.as_deref(), + browser_config.external_bundle_sri.as_deref(), + ), + ] } fn is_managed_external(&self) -> bool { @@ -952,6 +1176,23 @@ fn escape_html_attr(value: &str) -> String { .replace('>', ">") } +fn external_bundle_script_src(sha256: Option<&str>) -> String { + match sha256 { + Some(sha256) => format!("{PREBID_BUNDLE_ROUTE}?v={sha256}"), + None => PREBID_BUNDLE_ROUTE.to_string(), + } +} + +fn external_bundle_script_tag(sha256: Option<&str>, sri: Option<&str>) -> String { + let src = external_bundle_script_src(sha256); + let integrity = sri + .map(|value| format!(" integrity=\"{}\"", escape_html_attr(value))) + .unwrap_or_default(); + + format!("") +} + +#[cfg(test)] fn build( settings: &Settings, ) -> Result>, Report> { @@ -983,6 +1224,37 @@ fn build( /// /// Returns an error when the Prebid integration is enabled with invalid /// configuration. +pub fn register_for_plan( + settings: &Settings, + plan: &AuctionPlan, +) -> Result, Report> { + let Some(mut config) = + settings.integration_config::(PREBID_INTEGRATION_ID)? + else { + return Ok(None); + }; + let mut canonical = Vec::with_capacity(config.excluded_gam_ad_unit_path_suffixes.len()); + for suffix in std::mem::take(&mut config.excluded_gam_ad_unit_path_suffixes) { + if !canonical.contains(&suffix) { + canonical.push(suffix); + } + } + config.excluded_gam_ad_unit_path_suffixes = canonical; + validate_browser_config_for_startup(&config, &settings.proxy.allowed_domains)?; + validate_browser_bidder_ownership(&config, plan)?; + let integration = PrebidIntegration::for_browser_plan(&config, plan); + Ok(Some( + IntegrationRegistration::builder(PREBID_INTEGRATION_ID) + .with_proxy(integration.clone()) + .with_attribute_rewriter(integration.clone()) + .with_head_injector(integration) + .with_deferred_js() + .build(), + )) +} + +#[cfg(test)] +#[allow(clippy::missing_errors_doc)] pub fn register( settings: &Settings, ) -> Result, Report> { @@ -1072,12 +1344,31 @@ impl IntegrationAttributeRewriter for PrebidIntegration { } } +fn serialize_injected_prebid_config(payload: &impl Serialize) -> String { + // JSON appears in script raw-text, where every less-than sign must be escaped. + serde_json::to_string(payload) + .unwrap_or_else(|error| { + log::warn!("Prebid: failed to serialize client config: {error}"); + "{}".to_string() + }) + .replace('<', "\\u003c") +} + +fn injected_prebid_config_script(config_json: &str) -> String { + format!( + r#""# + ) +} + impl IntegrationHeadInjector for PrebidIntegration { fn integration_id(&self) -> &'static str { PREBID_INTEGRATION_ID } fn head_inserts(&self, _ctx: &IntegrationHtmlContext<'_>) -> Vec { + if let Some(inserts) = &self.planned_head_inserts { + return inserts.clone(); + } #[derive(Serialize)] #[serde(rename_all = "camelCase")] struct InjectedPrebidClientConfig<'a> { @@ -1095,22 +1386,24 @@ impl IntegrationHeadInjector for PrebidIntegration { account_id: self.config.account_id.as_deref().unwrap_or_default(), timeout: self.config.timeout_ms, debug: self.config.debug, - bidders: &self.config.bidders, + bidders: { + #[cfg(test)] + { + self.legacy_config + .as_ref() + .map_or(&[][..], |config| config.bidders.as_slice()) + } + #[cfg(not(test))] + { + &[] + } + }, client_side_bidders: &self.config.client_side_bidders, excluded_gam_ad_unit_path_suffixes: &self.config.excluded_gam_ad_unit_path_suffixes, }; - // Escape `window.pbjs=window.pbjs||{{}};window.pbjs.que=window.pbjs.que||[];window.pbjs.cmd=window.pbjs.cmd||[];window.__tsjs_prebid={config_json};"# - )]; + let config_json = serialize_injected_prebid_config(&payload); + let mut inserts = vec![injected_prebid_config_script(&config_json)]; inserts.push(self.external_bundle_script_tag()); @@ -1125,11 +1418,13 @@ impl IntegrationHeadInjector for PrebidIntegration { /// tell a fabricated empty from an explicitly supplied one — they are identical /// bytes on the wire. The merge uses this to stop an unusable value from /// clobbering real params, and the final pass uses it to drop whatever remains. +#[cfg(test)] fn is_unusable_bidder_params(params: &Json) -> bool { // `None` covers non-object values (e.g. `null`); an empty map covers `{}`. params.as_object().is_none_or(serde_json::Map::is_empty) } +#[cfg(test)] fn expand_trusted_server_bidders( configured_bidders: &[String], params: &Json, @@ -1187,7 +1482,8 @@ fn merge_bidder_param_object( // Generic bid-parameter override engine // ============================================================================ -fn warn_unconfigured_bidder(config: &PrebidIntegrationConfig, bidder: &str, field: &str) { +#[cfg(test)] +fn warn_unconfigured_bidder(config: &LegacyPrebidServerConfig, bidder: &str, field: &str) { if !config.bidders.iter().any(|b| b == bidder) { if config.client_side_bidders.iter().any(|b| b == bidder) { log::warn!( @@ -1204,7 +1500,7 @@ fn warn_unconfigured_bidder(config: &PrebidIntegrationConfig, bidder: &str, fiel } #[derive(Debug, Default, Clone)] -struct BidParamOverrideEngine { +pub(crate) struct BidParamOverrideEngine { rules: Vec, // Maps bidder name to the indices (into `rules`) of rules that constrain on that bidder. // Rules with no bidder constraint (zone-only or catch-all) are kept in `wildcard_indices`. @@ -1227,8 +1523,9 @@ struct BidParamOverrideFacts<'a> { } impl BidParamOverrideEngine { + #[cfg(test)] fn try_from_config( - config: &PrebidIntegrationConfig, + config: &LegacyPrebidServerConfig, ) -> Result> { let mut rules = Vec::new(); @@ -1284,6 +1581,45 @@ impl BidParamOverrideEngine { }) } + fn try_from_profile_config( + bid_param_zone_overrides: &std::collections::BTreeMap< + String, + std::collections::BTreeMap>, + >, + bid_param_overrides: &std::collections::BTreeMap>, + bid_param_override_rules: &[BidParamOverrideRule], + ) -> Result> { + let mut rules = Vec::new(); + for (bidder, set) in bid_param_overrides { + rules.push(CompiledBidParamOverrideRule::from_bidder_override( + bidder, set, + )?); + } + for (bidder, zone_override_sets) in bid_param_zone_overrides { + for (zone, set) in zone_override_sets { + rules.push(CompiledBidParamOverrideRule::from_zone_override( + bidder, zone, set, + )?); + } + } + for rule in bid_param_override_rules { + rules.push(CompiledBidParamOverrideRule::try_from(rule)?); + } + let mut bidder_index: HashMap> = HashMap::new(); + let mut wildcard_indices = Vec::new(); + for (index, rule) in rules.iter().enumerate() { + match &rule.bidder { + Some(bidder) => bidder_index.entry(bidder.clone()).or_default().push(index), + None => wildcard_indices.push(index), + } + } + Ok(Self { + rules, + bidder_index, + wildcard_indices, + }) + } + fn apply(&self, facts: BidParamOverrideFacts<'_>, params: &mut Json) { let bidder_indices = self.bidder_index.get(facts.bidder).map(Vec::as_slice); for idx in merged_rule_indices(&self.wildcard_indices, bidder_indices) { @@ -1306,6 +1642,30 @@ impl BidParamOverrideEngine { } } } + + /// Apply compiled profile rules to already centrally routed bidder params. + pub(crate) fn apply_routed(&self, bidder: &str, zone: Option<&str>, params: &mut Json) { + self.apply(BidParamOverrideFacts { bidder, zone }, params); + } +} + +/// Validate and compile server-side Prebid profile override fields. +/// +/// This narrow hook shares the existing override compiler without coupling +/// auction-profile availability to the browser integration's enablement. +pub(crate) fn compile_profile_override_rules( + bid_param_zone_overrides: &std::collections::BTreeMap< + String, + std::collections::BTreeMap>, + >, + bid_param_overrides: &std::collections::BTreeMap>, + bid_param_override_rules: &[BidParamOverrideRule], +) -> Result> { + BidParamOverrideEngine::try_from_profile_config( + bid_param_zone_overrides, + bid_param_overrides, + bid_param_override_rules, + ) } fn merged_rule_indices<'a>( @@ -1467,17 +1827,62 @@ fn non_empty_override_object( /// In [`ConsentForwardingMode::OpenrtbOnly`] mode, consent cookies are /// stripped from the `Cookie` header since consent travels exclusively /// through the `OpenRTB` body. +#[cfg(test)] fn copy_request_headers( from: &http::Request, to: &mut http::Request, consent_forwarding: ConsentForwardingMode, client_ip: Option, ) { - let headers_to_copy = [header::USER_AGENT, header::REFERER, header::ACCEPT_LANGUAGE]; + apply_prebid_header_values( + from.headers().get(header::COOKIE), + from.headers().get(header::USER_AGENT), + from.headers().get(header::REFERER), + from.headers().get(header::ACCEPT_LANGUAGE), + to, + consent_forwarding, + client_ip, + ); +} + +/// Apply the common raw-header transport policy for a planned PBS request. +pub(crate) fn apply_prebid_transport_headers( + from: &PrebidTransportHeaders, + to: &mut http::Request, + consent_forwarding: ConsentForwardingMode, + client_ip: Option, +) { + apply_prebid_header_values( + from.cookie(), + from.user_agent(), + from.referer(), + from.accept_language(), + to, + consent_forwarding, + client_ip, + ); +} - for header_name in &headers_to_copy { - if let Some(value) = from.headers().get(header_name) { - to.headers_mut().insert(header_name, value.clone()); +#[allow( + clippy::too_many_arguments, + reason = "the helper preserves four independently optional raw headers plus transport policy" +)] +fn apply_prebid_header_values( + cookie: Option<&HeaderValue>, + user_agent: Option<&HeaderValue>, + referer: Option<&HeaderValue>, + accept_language: Option<&HeaderValue>, + to: &mut http::Request, + consent_forwarding: ConsentForwardingMode, + client_ip: Option, +) { + for (name, value) in [ + (header::USER_AGENT, user_agent), + (header::REFERER, referer), + (header::ACCEPT_LANGUAGE, accept_language), + ] { + if let Some(value) = value { + to.headers_mut().insert(name, value.clone()); } } @@ -1488,24 +1893,20 @@ fn copy_request_headers( .insert(header::HeaderName::from_static("x-forwarded-for"), value); } - let Some(cookie_value) = from.headers().get(header::COOKIE) else { + let Some(cookie_value) = cookie else { return; }; - if !consent_forwarding.strips_consent_cookies() { to.headers_mut() .insert(header::COOKIE, cookie_value.clone()); return; } - match cookie_value.to_str() { Ok(value) => { let stripped = strip_cookies(value, CONSENT_COOKIE_NAMES); - if stripped.is_empty() { - return; - } - - if let Ok(cookie_header) = HeaderValue::from_str(&stripped) { + if !stripped.is_empty() + && let Ok(cookie_header) = HeaderValue::from_str(&stripped) + { to.headers_mut().insert(header::COOKIE, cookie_header); } } @@ -1518,6 +1919,7 @@ fn copy_request_headers( /// Appends query parameters to a URL, handling both URLs with and without existing query strings. /// Returns the original URL unchanged if params are empty or already present. +#[cfg(test)] fn append_query_params(url: &str, params: &str) -> String { if params.is_empty() || url.contains(params) { return url.to_string(); @@ -1529,30 +1931,315 @@ fn append_query_params(url: &str, params: &str) -> String { } } +/// Parse a planned PBS response with the configured profile behavior. +/// +/// This preserves the legacy PBS status, body, bid, cache, and metadata +/// semantics while allowing each planned provider to retain its own identity. +pub(crate) async fn parse_planned_prebid_response( + provider_id: &str, + profile: &PrebidProfilePlan, + input: &ProviderAuctionInput, + response: PlatformResponse, + response_time_ms: u64, + auction_id: &str, +) -> Result> { + let response = response.response; + let status = response.status(); + let content_type = response + .headers() + .get(header::CONTENT_TYPE) + .and_then(|value| value.to_str().ok()) + .map(str::to_owned); + let body_bytes = collect_response_bounded( + response.into_body(), + UPSTREAM_RTB_MAX_RESPONSE_BYTES, + "prebid", + ) + .await + .change_context(TrustedServerError::Prebid { + message: "Failed to read Prebid response body".to_string(), + }); + + if !status.is_success() { + log::warn!("Prebid auction {auction_id:?} returned non-success status: {status}"); + let body_bytes = match body_bytes { + Ok(body_bytes) => Some(body_bytes), + Err(error) => { + log::warn!( + "Prebid auction {auction_id:?} failed to read non-success response body: {error:?}" + ); + None + } + }; + if profile.debug + && let Some(body_bytes) = body_bytes.as_deref() + { + match prebid_body_preview(body_bytes) { + Some(preview) => { + let truncation = if preview.truncated { + " (truncated)" + } else { + "" + }; + log::warn!( + "Prebid auction {auction_id:?} error response body preview{truncation}: {}", + preview.text + ); + } + None => log::warn!( + "Prebid auction {auction_id:?} returned an empty error response body" + ), + } + } + + let status_code = status.as_u16(); + let mut parsed = AuctionResponse::error(provider_id, response_time_ms) + .with_metadata("error_type", serde_json::json!(ERROR_TYPE_HTTP_STATUS)) + .with_metadata("http_status", serde_json::json!(status_code)) + .with_metadata( + "message", + serde_json::json!(format!("Prebid Server returned HTTP {status_code}")), + ); + if profile.debug + && let Some(message) = body_bytes + .as_deref() + .and_then(|body| extract_prebid_error_message(body, content_type.as_deref())) + { + parsed.metadata.insert( + "upstream_message".to_string(), + serde_json::json!(message.text), + ); + parsed.metadata.insert( + "upstream_message_truncated".to_string(), + serde_json::json!(message.truncated), + ); + } + return Ok(parsed); + } + + let body_bytes = body_bytes?; + let response_json: Json = + serde_json::from_slice(&body_bytes).change_context(TrustedServerError::Prebid { + message: "Failed to parse Prebid response".to_string(), + })?; + if profile.debug && log::log_enabled!(log::Level::Trace) { + match serde_json::to_string_pretty(&response_json) { + Ok(json) => log::trace!("Prebid OpenRTB response:\n{json}"), + Err(error) => log::warn!("Prebid: failed to serialize response for logging: {error}"), + } + } + + let mut parsed = + parse_planned_prebid_openrtb(provider_id, input, &response_json, response_time_ms); + enrich_planned_prebid_metadata(profile, &response_json, &mut parsed); + log::info!( + "Prebid provider {provider_id} returned {} bids in {}ms", + parsed.bids.len(), + response_time_ms + ); + Ok(parsed) +} + +fn parse_planned_prebid_openrtb( + provider_id: &str, + input: &ProviderAuctionInput, + response_json: &Json, + response_time_ms: u64, +) -> AuctionResponse { + let Some(response) = response_json.as_object() else { + return AuctionResponse::error(provider_id, response_time_ms) + .with_metadata("error_type", serde_json::json!("parse_response")); + }; + match response.get("cur") { + None => {} + Some(Json::String(currency)) if currency.eq_ignore_ascii_case(DEFAULT_CURRENCY) => {} + Some(Json::String(currency)) => { + return AuctionResponse::no_bid(provider_id, response_time_ms) + .with_metadata("unsupported_currency", serde_json::json!(currency)); + } + Some(_) => { + return AuctionResponse::error(provider_id, response_time_ms) + .with_metadata("error_type", serde_json::json!("parse_response")); + } + } + + let mut diagnostics = ResponseAdmissionDiagnostics::default(); + let mut bids = Vec::new(); + if let Some(seatbids) = response_json.get("seatbid").and_then(Json::as_array) { + for seatbid in seatbids { + let returned_seat = seatbid + .get("seat") + .and_then(Json::as_str) + .filter(|seat| !seat.is_empty()); + let delivery_bidder = returned_seat.unwrap_or("unknown"); + if let Some(entries) = seatbid.get("bid").and_then(Json::as_array) { + for entry in entries { + match parse_planned_prebid_bid(entry, delivery_bidder, returned_seat, input) { + Ok(bid) => bids.push(bid), + Err(reason) => { + diagnostics.record(reason); + if reason == BidRejectionReason::InvalidBid { + let impression = entry + .get("impid") + .and_then(Json::as_str) + .unwrap_or(""); + log::warn!( + "Prebid: failed to parse bid from seat '{delivery_bidder}' for imp '{impression}'" + ); + } + } + } + } + } + } + } + let mut parsed = if bids.is_empty() { + AuctionResponse::no_bid(provider_id, response_time_ms) + } else { + AuctionResponse::success(provider_id, bids, response_time_ms) + }; + diagnostics.attach_to(&mut parsed); + parsed +} + +fn enrich_planned_prebid_metadata( + profile: &PrebidProfilePlan, + response_json: &Json, + parsed: &mut AuctionResponse, +) { + let ext = response_json.get("ext"); + for key in ["responsetimemillis", "errors", "warnings"] { + if let Some(value) = ext.and_then(|ext| ext.get(key)) { + parsed.metadata.insert(key.to_string(), value.clone()); + } + } + if profile.debug { + if let Some(value) = ext.and_then(|ext| ext.get("debug")) { + parsed.metadata.insert("debug".to_string(), value.clone()); + } + if let Some(value) = ext + .and_then(|ext| ext.get("prebid")) + .and_then(|prebid| prebid.get("bidstatus")) + { + parsed + .metadata + .insert("bidstatus".to_string(), value.clone()); + } + } +} + +fn parse_planned_prebid_bid( + bid: &Json, + delivery_bidder: &str, + returned_seat: Option<&str>, + input: &ProviderAuctionInput, +) -> Result { + let slot_id = bid + .get("impid") + .and_then(Json::as_str) + .filter(|slot_id| !slot_id.is_empty()) + .ok_or(BidRejectionReason::InvalidBid)? + .to_string(); + let width = parse_optional_bid_dimension(bid, "w")?; + let height = parse_optional_bid_dimension(bid, "h")?; + let (width, height) = resolve_bid_dimensions(input, &slot_id, width, height)?; + let price = bid + .get("price") + .and_then(Json::as_f64) + .filter(|price| price.is_finite() && *price >= 0.0) + .ok_or(BidRejectionReason::InvalidBid)?; + let creative = bid.get("adm").and_then(Json::as_str).map(String::from); + let cache_entry = bid + .get("ext") + .and_then(|ext| ext.get("prebid")) + .and_then(|prebid| prebid.get("cache")) + .and_then(|cache| cache.get("bids")); + let cache_id = cache_entry + .and_then(|cache| cache.get("cacheId")) + .and_then(Json::as_str) + .map(String::from); + let (cache_host, cache_path) = cache_entry + .and_then(|cache| cache.get("url")) + .and_then(Json::as_str) + .and_then(|value| { + ParsedUrl::parse(value) + .map_err(|error| log::debug!("PBS cache URL parse failed: {error}")) + .ok() + }) + .map(|url| { + let host = url.host_str().map(String::from); + let path = url.path().to_string(); + let path = (!path.is_empty() && path != "/").then_some(path); + (host, path) + }) + .unwrap_or((None, None)); + if cache_id.is_some() && cache_host.is_none() { + log::warn!( + "PBS bid has cache UUID but cache URL could not be parsed — creative will fail to render for slot '{slot_id}'" + ); + } + + Ok(AuctionBid { + slot_id, + price: Some(price), + currency: DEFAULT_CURRENCY.to_string(), + creative, + adomain: bid.get("adomain").and_then(Json::as_array).map(|domains| { + domains + .iter() + .filter_map(Json::as_str) + .map(String::from) + .collect() + }), + bidder: delivery_bidder.to_string(), + returned_seat: returned_seat.map(str::to_string), + width, + height, + nurl: bid.get("nurl").and_then(Json::as_str).map(String::from), + burl: bid.get("burl").and_then(Json::as_str).map(String::from), + bid_id: bid + .get("id") + .and_then(Json::as_str) + .filter(|value| !value.is_empty()) + .map(String::from), + ad_id: bid.get("adid").and_then(Json::as_str).map(String::from), + creative_id: bid.get("crid").and_then(Json::as_str).map(String::from), + renderer: None, + cache_id, + cache_host, + cache_path, + metadata: HashMap::new(), + }) +} + // ============================================================================ // Prebid Auction Provider // ============================================================================ -/// Prebid Server auction provider. +/// Legacy Prebid Server auction provider retained only for parity tests. +#[cfg(test)] pub struct PrebidAuctionProvider { - config: PrebidIntegrationConfig, + config: LegacyPrebidServerConfig, bid_param_override_engine: Arc, } +#[cfg(test)] #[derive(Default)] struct PrebidImpressionDisposition { aps_only: usize, invalid: usize, } +#[cfg(test)] struct PrebidRequestBuild { request: OpenRtbRequest, disposition: PrebidImpressionDisposition, } +#[cfg(test)] impl PrebidAuctionProvider { #[cfg(test)] - fn new(config: PrebidIntegrationConfig) -> Self { + fn new(config: LegacyPrebidServerConfig) -> Self { Self::try_new(config).expect("should compile prebid bid param overrides") } @@ -1561,7 +2248,7 @@ impl PrebidAuctionProvider { /// # Errors /// /// Returns an error when the configured bidder-param override rules are invalid. - pub fn try_new(config: PrebidIntegrationConfig) -> Result> { + pub fn try_new(config: LegacyPrebidServerConfig) -> Result> { Ok(Self { bid_param_override_engine: Arc::new(BidParamOverrideEngine::try_from_config(&config)?), config, @@ -2438,6 +3125,7 @@ impl PrebidAuctionProvider { creative, adomain, bidder: seat.to_string(), + returned_seat: None, width, height, nurl, @@ -2454,9 +3142,10 @@ impl PrebidAuctionProvider { } } +#[cfg(test)] #[async_trait(?Send)] impl AuctionProvider for PrebidAuctionProvider { - fn provider_name(&self) -> &'static str { + fn provider_name(&self) -> &str { PREBID_INTEGRATION_ID } @@ -2664,6 +3353,7 @@ impl AuctionProvider for PrebidAuctionProvider { /// /// Returns an error when the Prebid provider is enabled with invalid /// configuration. +#[cfg(test)] pub fn register_auction_provider( settings: &Settings, ) -> Result>, Report> { @@ -2672,10 +3362,12 @@ pub fn register_auction_provider( return Ok(Vec::new()); }; - log::info!( - "Registering Prebid auction provider (server_url={})", - integration.config.server_url - ); + if let Some(config) = integration.legacy_config.as_ref() { + log::info!( + "Registering Prebid auction provider (server_url={})", + config.server_url + ); + } if integration.config.debug { log::warn!( "Prebid debug mode is ON — debug data (httpcalls, resolvedrequest, \ @@ -2693,7 +3385,14 @@ mod tests { use super::*; use crate::auction::formats::convert_to_openrtb_response; use crate::auction::orchestrator::OrchestrationResult; - use crate::auction::test_support::create_test_auction_context as shared_test_auction_context; + use crate::auction::plan::{ + AuctionPlanConfig, BidderId, BidderRouteConfig, NotificationConfig, ProviderConfig, + ProviderId, RoutingMode, + }; + use crate::auction::test_support::{ + canonical_parity_auction_request, + create_test_auction_context as shared_test_auction_context, + }; use crate::auction::types::{ AdFormat, AdSlot, AuctionContext, AuctionRequest, DeviceInfo, PublisherInfo, UserInfo, }; @@ -2705,8 +3404,9 @@ mod tests { AttributeRewriteAction, IntegrationDocumentState, IntegrationRegistry, }; use crate::platform::test_support::{ - NoopConfigStore, NoopGeo, NoopHttpClient, NoopSecretStore, StubHttpClient, - build_services_with_http_client, + HashMapConfigStore, HashMapSecretStore, NoopConfigStore, NoopGeo, NoopHttpClient, + NoopSecretStore, StubHttpClient, build_services_with_config_secret_and_http_client, + build_services_with_http_client, build_services_with_http_client_and_client_ip, }; use crate::platform::{ ClientInfo, PlatformBackend, PlatformBackendSpec, PlatformError, RuntimeServices, @@ -2718,8 +3418,9 @@ mod tests { use bytes::Bytes; use http::Method; use serde_json::json; - use std::collections::HashMap; + use std::collections::{BTreeMap, HashMap}; use std::io::Cursor; + use std::str::FromStr as _; #[test] fn external_bundle_sha256_validation_matches_hex_pattern() { @@ -2727,7 +3428,6 @@ mod tests { let config = |sha: &str| -> PrebidIntegrationConfig { serde_json::from_value(serde_json::json!({ - "server_url": "https://prebid.example.com/openrtb2/auction", "external_bundle_sha256": sha, })) .expect("should deserialize prebid config") @@ -2758,8 +3458,26 @@ mod tests { create_test_settings() } - fn base_config() -> PrebidIntegrationConfig { - PrebidIntegrationConfig { + #[test] + fn injected_prebid_config_escapes_every_less_than_sign() { + let config_json = serialize_injected_prebid_config(&json!({ + "accountId": "x Compiler[Auction plan compiler] + Registry[Protocol and profile registry] --> Compiler + Compiler --> Plan[Immutable AuctionPlan] + Request[Canonical AuctionRequest] --> Router[Bidder and slot router] + Plan --> Router + Router --> Inputs[Per-provider ProviderAuctionInput] + Inputs --> Encoder[OpenRTB 2.6 driver and selected profile] + Encoder --> Transport[Existing platform HTTP transport] + Transport --> Decoder[OpenRTB decoder and selected profile] + Decoder --> Outcomes[Normalized provider outcomes] + Outcomes --> Decision[Existing ranking or mediation] + Decision --> Delivery[Existing creative delivery] +``` + +### Control plane + +The control plane parses configuration, registers available profiles, validates provider and bidder references, and compiles an immutable `AuctionPlan` during startup. + +### Runtime plane + +The runtime plane receives a canonical auction request, routes its bidder demand to provider plans, creates one provider-specific input per provider, executes the existing concurrent auction flow, and normalizes responses before the existing decision stage. + +Raw configuration is not repeatedly interpreted during auctions. + +## Configuration Schema + +### Auction configuration + +The provider blocks are the source of truth for bidder providers. A separate ordered bidder-provider name list is not required. + +The existing `[auction].mediator` reference may continue to select the current statically registered mock mediator. That mediator is not configured under `[auction.providers.*]`, is not bidder-routed, and is not compiled through the protocol and profile registry. This specification does not generalize mediation. + +```toml +[auction] +enabled = true +timeout_ms = 2000 + +[auction.providers.pbs-primary] +protocol = "openrtb-2.6" +profile = "prebid-server" +endpoint = "https://pbs.example/openrtb2/auction" +timeout_ms = 900 +routing = "explicit" + +[auction.providers.aps-primary] +protocol = "openrtb-2.6" +profile = "aps" +endpoint = "https://aps.example/bid" +timeout_ms = 700 +routing = "all_eligible" + +[auction.providers.aps-primary.profile_config] +account_id = "example-account" +allow_script_creatives = false + +[auction.providers.rubicon-direct] +protocol = "openrtb-2.6" +profile = "standard" +endpoint = "https://rubicon.example/bid" +timeout_ms = 650 +routing = "explicit" + +[auction.providers.rubicon-direct.profile_config] +request_ext = { account = "example-account" } +imp_ext = { placementGroup = "display" } +``` + +### Common provider fields + +| Field | Required | Default | Meaning | +| ------------ | -------- | --------------- | -------------------------------------------------------------------------- | +| `protocol` | Yes | None | Registered protocol identifier. First version supports only `openrtb-2.6`. | +| `profile` | No | `standard` | Registered OpenRTB profile identifier. | +| `endpoint` | Yes | None | Fixed operator-configured HTTPS endpoint. | +| `timeout_ms` | No | Profile default | Maximum provider timeout, capped by the remaining auction deadline. | +| `routing` | No | `explicit` | `explicit` or `all_eligible`. | + +Each provider may also define a `profile_config` table. The selected profile parses that table as typed configuration; an omitted table is treated as empty configuration. + +Standard OpenRTB notification suppression is common provider configuration: + +```toml +[auction.providers.pbs-primary.notifications] +suppress_all = false +suppress_seats = ["example-seat"] +``` + +- `suppress_all` removes `nurl` and `burl` from every normalized bid returned by the provider. +- `suppress_seats` removes those URLs only when the exact returned `seatbid.seat` value matches an entry. +- Seat suppression is independent of the bidder registry because returned seats may be aliases or originate from stored requests. +- Suppression entries must be nonempty, unique strings without ASCII control characters. A provider may configure at most 128 entries, and each entry may contain at most 128 UTF-8 bytes. +- Suppression is applied after response parsing and before bids reach ranking, mediation, or delivery. + +When `timeout_ms` is omitted, the compiler resolves it from the selected profile: `prebid-server` uses 1000 ms, `aps` uses 800 ms, and `standard` uses the auction timeout. An explicit provider value overrides that default. Runtime still caps the resolved timeout by the remaining auction deadline. + +Provider presence under `[auction.providers.*]` means the provider is configured for the enabled auction. The implementation may add a conventional enablement field only if required by the broader settings system; it must not reintroduce a separate provider inventory. + +### Bidder registry + +The central bidder registry maps each client-visible bidder to exactly one provider: + +```toml +[auction.bidders.rubicon] +provider = "rubicon-direct" + +[auction.bidders.pubmatic] +provider = "pbs-primary" + +[auction.bidders.appnexus] +provider = "pbs-primary" + +[auction.bidders.aps] +provider = "aps-primary" +``` + +The client requests bidders. It does not select providers or endpoints. + +### Browser integrations + +Browser-specific behavior remains separate: + +```toml +[integrations.prebid] +# Browser bundle, injection, adapter, and script behavior only. +``` + +Enabling or disabling a browser integration does not register, enable, or disable an auction provider. + +## Configuration Validation + +The auction plan compiler must reject configuration when: + +- A provider ID is duplicated or invalid. +- Two provider IDs collide after target-adapter backend-name encoding. +- A protocol is unknown. +- A profile is unknown. +- A profile configuration cannot be parsed or validated. +- A bidder references an unknown provider. +- A bidder has more than one provider route. +- A profile cannot support banner inventory. +- A provider endpoint is not an absolute HTTPS URL with a nonempty host, contains URL credentials or a fragment, or violates stricter selected-profile endpoint requirements. +- A provider's static extension configuration is not an object. +- Static extensions exceed bounded size or nesting limits. +- Static extensions collide with reserved fields owned by the OpenRTB driver, signing, or profile. +- Request signing is enabled but its structural configuration is missing or invalid. +- More than one active provider is configured for a platform adapter that cannot perform concurrent fan-out. This target-specific validation is conservative because one auction may request bidders routed to different providers, and any `all_eligible` provider may participate alongside them. + +Compilation has two explicit stages: + +1. Target-independent compilation parses settings, resolves profiles and defaults, validates routes and field ownership, canonicalizes endpoints, and produces the immutable plan. +2. Target validation receives that plan plus an adapter capability and shared pure backend-name prediction description. It validates fan-out support, deadline claims, and encoded backend-name uniqueness without rebuilding provider configuration or duplicating runtime naming algorithms. + +The same compiler, registry, and target validator must be used by deploy tooling and runtime startup. A target-aware deploy or push passes the selected adapter description and must reject target-specific failures before publication. A target-agnostic `config validate` command runs the complete first stage and clearly reports that target checks are deferred; adapter startup remains the final mandatory target check. Configuration schema or documentation generation may consume the same registry where supported. + +## Profile Registry + +### Registration + +Profiles are registered through ordinary Rust code compiled into Trusted Server. + +Conceptually: + +```text +auction core module → registers "standard" +prebid module → registers "prebid-server" +aps module → registers "aps" +``` + +A profile's availability does not depend on its corresponding browser integration being enabled. + +### Factory responsibility + +A profile factory: + +1. Parses its typed configuration. +2. Validates its configuration. +3. Reports supported media and creative representations. +4. Declares its fixed typed standard-field policy. +5. Compiles immutable runtime profile behavior. + +### Runtime responsibility + +A profile may: + +- Declare a fixed typed policy for standard OpenRTB fields. +- Augment a generic OpenRTB request within fields reserved to that profile. +- Interpret provider-specific response extensions. +- Apply provider-specific bid validation. +- Perform deterministic provider-local candidate reduction when required to preserve registered profile behavior. +- Produce the existing normalized creative or renderer representation. +- Extract provider-specific metadata required to preserve current behavior. + +A profile must not directly overwrite fields owned by the OpenRTB driver, central privacy enforcement, or signing. The common driver applies the profile's compiled standard-field policy while constructing those fields. Each profile also declares the request extensions and response fields it owns, and the compiler rejects ownership collisions. + +A profile may not: + +- Select its endpoint. +- Send HTTP requests. +- Register platform backends. +- Resolve secrets. +- Route other providers' bidders. +- Compare bids across providers, apply auction floors, or choose final auction winners. +- Invoke mediation. +- Override central privacy enforcement. +- Modify another provider plan. + +### Typed standard-field policy + +Central privacy enforcement defines the maximum data permitted for an auction. A compiled profile field policy may omit data from that approved view, but it cannot restore, derive, or request data that central enforcement removed. + +The common driver remains the only component that constructs standard OpenRTB fields. Each registered profile declares a fixed Rust policy for differences such as: + +- `imp.tagid`. +- Primary `banner.w` and `banner.h` fields. +- `banner.topframe`. +- `site.ref` forwarding. +- Precise latitude and longitude. + +These policies are registered, reviewed, and tested Rust behavior. They are not operator-configurable arbitrary field overrides. The `prebid-server` and `aps` profiles preserve their current field behavior through their respective policies. + +The `standard` profile uses the shared PBS and APS baseline. It includes request and impression IDs, banner formats, site domain and sanitized page URL, consent-approved user ID and EIDs, user agent, IP address, coarse geo, DNT, language, consent fields, floors, secure-impression requirements, effective timeout, and current USD currency assumptions. By default it omits `site.ref`, precise latitude and longitude, `imp.tagid`, primary `banner.w` and `banner.h`, and `banner.topframe`. + +Consent fields have a fixed wire policy rather than a profile-defined arbitrary map: + +- `standard` emits an admitted TCF string as `user.consent`; applies the current jurisdiction and applicability rules to `regs.gdpr` and `regs.ext.gdpr`; mirrors admitted USP, GPP, and GPP SID values in their current top-level `regs` and compatibility `regs.ext` placements; and omits `user.ext.ConsentedProvidersSettings`. +- `prebid-server` preserves that current OpenRTB policy plus its existing consent-forwarding mode and Google Additional Consent mapping in `user.ext.ConsentedProvidersSettings`. +- `aps` preserves its current OpenRTB consent placements and deliberately omits Google Additional Consent. + +Golden field-matrix tests are normative for absent consent, GDPR applicability and jurisdiction combinations, TCF, USP, GPP and section IDs, Google Additional Consent, and cookie-sourced versus KV- or policy-sourced consent. A common-driver extraction may not broaden one profile to fields currently exposed only by another. + +### Runtime inputs + +A profile receives only: + +- Its compiled profile configuration. +- The provider ID. +- The provider's routed slots and bidder parameters. +- Canonical publisher, user, device, consent, and context data already approved for the auction. +- The effective provider timeout. +- Request-local parse state where required. + +A profile does not receive the raw downstream HTTP request or unrestricted runtime services. Browser values needed by OpenRTB are normalized into canonical auction data before profile execution. + +Request admission also retains a transport-owned Prebid header snapshot containing exactly the first values selected by the current HTTP header API for `Cookie`, `User-Agent`, `Referer`, and `Accept-Language`. It preserves accepted header bytes without introducing a second truncation rule; the existing inbound request/header limit remains authoritative. The snapshot is not exposed to profiles or other unrestricted runtime code. Common Prebid transport forwards the current `User-Agent`, `Referer`, and `Accept-Language` values and synthesizes `X-Forwarded-For` only from the platform-attested client IP, never from a client-supplied `X-Forwarded-For` value. The Prebid field policy may also use the raw accepted `Referer` for its existing `site.ref` behavior; the canonical `site.page` remains sanitized separately. APS and `standard` do not receive these raw browser headers. + +To preserve current Prebid consent-forwarding behavior, the compiled Prebid profile selects `openrtb_only`, `cookies_only`, or `both`, and common transport applies the existing behavior to the snapshotted `Cookie` value: + +- `both` and `cookies_only` forward the complete selected `Cookie` header value unchanged. +- `openrtb_only` removes the existing allowlisted consent-cookie names and forwards the remaining cookies. +- When the header cannot be parsed by the existing stripping path, current fallback behavior is preserved, including forwarding the original non-UTF-8 value. +- If stripping removes every cookie, the upstream `Cookie` header is omitted. +- Consent originating from KV or policy state remains in the OpenRTB body when no browser consent cookie can carry it, including in `cookies_only` mode. + +No other first-version profile receives a browser `Cookie` header. Reducing Prebid forwarding to consent cookies only, changing malformed-header handling, or otherwise redesigning these modes requires a separate consent-focused specification. + +## Canonical Auction Model + +The canonical auction model remains independent of the OpenRTB wire format. + +Conceptually: + +```rust +AuctionRequest { + id, + slots, + publisher, + user, + device, + privacy, + context, +} +``` + +A slot separates requested demand from provider routing and provider-specific input: + +```rust +AuctionSlot { + id, + banner_formats, + floor, + bidder_params, + trusted_provider_routes, +} +``` + +- `bidder_params` is keyed by bidder ID and originates from client or server auction input. +- `trusted_provider_routes` is produced only by trusted server-side opportunity construction or admission normalization of a recognized integration envelope. +- Client input cannot choose provider IDs directly. + +## Routing Model + +### Prebid browser-envelope normalization + +The reserved `trustedServer` browser-adapter entry is an admission envelope, not a bidder ID. Before central routing, request admission unpacks its bounded `bidderParams` object into the canonical slot bidder map: + +```text +trustedServer.bidderParams.rubicon → bidder_params.rubicon +trustedServer.bidderParams.pubmatic → bidder_params.pubmatic +``` + +Each nested key becomes the client-requested bidder ID and is resolved only through `[auction.bidders]`. Nested values become that bidder's parameters. The envelope cannot name provider IDs or endpoints. Its optional `zone` value is preserved as a bounded Prebid slot-matching fact for existing override rules; it does not participate in provider routing. + +A usable `bidderParams` value is a bounded JSON object containing at least one nonempty bidder key whose value is an object accepted by the existing bidder-parameter admission rules. The fallback cases are exact: + +- Missing, `null`, or an empty `bidderParams` object creates stored-request routes to every configured `prebid-server` plan. +- A non-object `bidderParams`, an invalid bidder key or value, or a bounds violation is malformed input and does not trigger stored-request fan-out. +- An object containing a structurally valid but unregistered bidder is usable; it produces `unroutable_bidder` during central routing and does not trigger stored-request fallback. +- A partially malformed object is rejected rather than partially routed. + +A programmatic request may contain both a direct bidder entry and the same bidder inside the envelope. To preserve the existing Prebid merge rule, a usable direct object wins; an unusable direct value cannot overwrite a usable envelope value. Admission applies this rule deterministically before central routing. It never relies on map iteration order. + +When fallback applies, request admission derives a server-controlled stored-request route to each configured `prebid-server` provider plan. The client still does not select those provider IDs. Each routed Prebid plan receives the slot without inline bidder parameters and applies the existing stored-request fallback. This preserves initial and refresh auction behavior that currently uses an empty synthetic `trustedServer` bid. + +### Client-originated demand + +For each bidder requested on a slot: + +1. Look up the bidder in `[auction.bidders]`. +2. Resolve its single provider ID. +3. Add the slot and only that bidder's parameters to the provider's routed view. +4. Record an `unroutable_bidder` outcome when no route exists. +5. Continue the auction for other routable bidders and providers. + +Example client demand: + +```text +Slot: header +Bidders: rubicon, pubmatic, appnexus +``` + +Configured routes: + +```text +rubicon → rubicon-direct +pubmatic → pbs-primary +appnexus → pbs-primary +``` + +Resulting provider inputs: + +```text +rubicon-direct +└── header + └── rubicon parameters + +pbs-primary +└── header + ├── pubmatic parameters + └── appnexus parameters +``` + +### Trusted server-generated demand + +Server-generated opportunities that intentionally rely on stored requests may name trusted provider routes without supplying bidder parameters. + +For migration, existing creative-opportunity construction expresses empty Prebid stored-request intent without a provider ID; the trusted router expands it to every configured `prebid-server` plan, matching the browser-envelope rule. Explicit creative-opportunity bidder parameters continue through `[auction.bidders]`. Creative opportunities no longer hard-code an APS provider instance: an `aps` plan configured as `all_eligible` receives every compatible slot, while an explicitly routed APS plan participates only through a centrally routed bidder or a trusted server-generated route. + +This supports the existing Prebid stored-request and APS paths without allowing the browser to choose an endpoint. + +### Routing modes + +#### `explicit` + +The provider receives only slots routed through: + +- The central bidder registry. +- Trusted server-generated provider routes. + +This is the default. + +#### `all_eligible` + +The provider receives every banner-compatible slot, regardless of bidder routes. + +This mode must be explicitly configured. It is the migration-equivalent routing mode for the current APS provider, which receives every banner-compatible slot. Operators may deliberately choose `explicit` for narrower APS participation. + +### No eligible slots + +When a provider has no eligible slots: + +- No upstream request is sent. +- The provider is recorded as `skipped_no_eligible_slots`. +- This is distinct from no-bid because the provider was not called. + +## Provider Auction Input + +Routing produces one immutable `ProviderAuctionInput` per provider. + +It contains only: + +- Slots admitted by explicit routes or by that provider's `all_eligible` mode. +- Bidder parameters assigned to that provider through the central bidder registry. +- Privacy-approved canonical auction data. +- Provider identity. +- Effective timeout. + +`all_eligible` admits additional banner slots but does not grant access to bidder parameters assigned to another provider. A profile never receives another provider's bidder parameters and therefore does not need provider-specific exclusion logic. + +## OpenRTB 2.6 Driver + +The generic driver owns standard banner OpenRTB behavior and applies the selected profile's compiled standard-field policy. + +### Request responsibilities + +- Request and impression IDs. +- Banner formats. +- Site and publisher data currently supplied by Trusted Server. +- Device and user data currently supplied by Trusted Server. +- Existing consent and EID forwarding behavior. +- Floors and floor currency. +- `tmax` using the effective timeout. +- Current secure-impression requirements. +- Current auction currency assumptions. +- Common Trusted Server signing finalization when enabled, plus the documented PBS host/scheme-only behavior when disabled. + +### Response responsibilities + +- HTTP 204 and ordinary empty responses as no-bid where currently supported. +- Standard OpenRTB response decoding. +- Transport association between the dispatched provider request and its response. For parity, omission or mismatch of the OpenRTB response `id` alone does not reject a PBS or APS response in the first version; profiles may preserve stricter existing behavior where one already exists. +- `seatbid.seat` preservation independently from delivery bidder code. +- Standard bid ID, impression ID, price, dimensions, domains, creative markup, and notification URLs. +- Current banner compatibility checks. +- Existing response-size bounds. +- Existing error and outcome classifications where applicable. + +### Bidder parameters + +Client-supplied bidder parameters are profile input, not generic OpenRTB fields. + +The generic driver does not invent a location for them. + +- The `prebid-server` profile consumes them. +- Another profile may consume them in a provider-specific way. +- The `standard` profile does not forward nonempty bidder parameters by default. +- Unconsumed nonempty parameters produce bounded `unused_bidder_params` diagnostics. + +### Static extensions + +The standard profile may accept validated static objects for: + +- `request.ext` +- `imp.ext` + +Static extensions: + +- Cannot contain secrets. +- Cannot contain templates. +- Cannot read request data. +- Cannot use JSONPath or arbitrary expressions. +- Are bounded by size and nesting depth. +- Cannot overwrite fields reserved by signing, the OpenRTB driver, or another profile responsibility. + +### Ordinary field overrides + +The first version does not support operator-configured arbitrary overrides of fields such as `site.domain`, `device.ip`, `user.id`, or `imp.tagid`. + +A registered profile's fixed typed standard-field policy is not an arbitrary override. Typed operator configuration for additional standard fields should be added only when a concrete endpoint requires it. + +## Prebid Server Profile + +The `prebid-server` profile preserves required Prebid Server behavior while delegating standard fields to the OpenRTB driver. + +### Profile responsibilities + +- Construct `imp.ext.prebid.bidder` from routed bidder parameters. +- Preserve current deterministic bidder-parameter merging and validation semantics where still applicable. +- Support stored-request fallback for trusted server-generated provider routes and admission-generated empty `trustedServer` envelopes. +- Add Prebid-specific request extensions and test/debug fields. +- Preserve Prebid Cache coordinate extraction. +- Preserve Prebid response diagnostics required by current behavior. +- Preserve notification suppression behavior through the common provider outcome model. +- Preserve current request-local data needed to parse responses. + +### Responsibilities moved to common architecture + +- Endpoint and timeout configuration. +- Provider identity. +- Bidder routing. +- Standard OpenRTB request fields. +- Trusted Server signing invocation. +- Standard consent and EID forwarding. +- HTTP transport and backend correlation. +- Standard response parsing and validation. +- Winner selection and mediation. + +### Profile configuration parity + +The central bidder registry is the sole server-side bidder allowlist and route source. The Prebid profile does not define a second `bidders` list. + +The first version must preserve these server-side controls and defaults: + +| Current control | New owner | Default and validation | +| -------------------------- | ---------------------------------------------------------------- | ------------------------------------------------------------------------------------------ | +| `server_url` | Common provider `endpoint` | Required fixed HTTPS endpoint. | +| `timeout_ms` | Common provider `timeout_ms` | Defaults to 1000 ms for `prebid-server`; explicit values override it. | +| `bidders` | Central `[auction.bidders]` registry | Every bidder route is explicit and unique. | +| `debug` | `auction.providers..profile_config.debug` | `false`; preserves current request and response debug behavior. | +| `test_mode` | `auction.providers..profile_config.test_mode` | `false`; preserves the current OpenRTB test flag. | +| `debug_query_params` | `auction.providers..profile_config.debug_query_params` | Absent by default; preserves current page-URL behavior when configured. | +| `bid_param_zone_overrides` | `auction.providers..profile_config.bid_param_zone_overrides` | Empty by default; preserves current typed validation and merge behavior. | +| `bid_param_overrides` | `auction.providers..profile_config.bid_param_overrides` | Empty by default; preserves current typed validation and merge behavior. | +| `bid_param_override_rules` | `auction.providers..profile_config.bid_param_override_rules` | Empty by default; preserves current rule validation, ordering, and shallow-merge behavior. | +| `consent_forwarding` | `auction.providers..profile_config.consent_forwarding` | `both`; preserves the existing `openrtb_only`, `cookies_only`, and `both` behavior. | +| `suppress_nurl` | Common `auction.providers..notifications.suppress_all` | `false`; preserves global `nurl` and `burl` suppression. | +| `suppress_nurl_bidders` | Common `auction.providers..notifications.suppress_seats` | Empty; exact returned seat IDs, validated independently of bidder routes. | + +Stored-request fallback remains built-in Prebid profile behavior rather than another configuration switch. Existing browser-only fields, including bundle configuration, script patterns, client-side bidders, account injection, and excluded GAM ad-unit suffixes, remain under `[integrations.prebid]`. + +Parity tests must cover defaults and non-default values for every field in this table. + +### Browser integration separation + +The Prebid browser integration continues to own: + +- Browser bundle construction. +- JavaScript injection. +- Browser adapter behavior. +- Client-side bidder configuration. +- Script interception and rewriting. +- Browser `timeout_ms` and `debug`, with their current defaults of 1000 ms and `false`, for the injected global Prebid.js configuration. + +Browser `timeout_ms` and `debug` are independent from every server-side provider's common timeout and `profile_config.debug`. No value is selected from multiple provider plans for browser injection. The browser integration does not own the server-side Prebid provider endpoint or bidder route map. + +## APS Profile + +The `aps` profile preserves APS-specific OpenRTB and rendering behavior. + +### Profile responsibilities + +- Add APS account and SDK request extensions. +- Preserve APS inventory identity behavior. +- Interpret the APS response shape and extension fields. +- Preserve APS-specific bid validation. +- Preserve the current highest-price-per-impression candidate reduction and bid-ID tie-breaker, including displaced-bid diagnostics. +- Extract creative URL and tag type. +- Produce the existing typed APS renderer descriptor. +- Preserve script-creative opt-in behavior. +- Preserve APS diagnostics required by current behavior. +- Preserve a valid returned `seatbid.seat` separately while continuing to mark accepted APS bids with delivery bidder code `aps` for the existing browser renderer. + +### Responsibilities moved to common architecture + +- Endpoint and timeout configuration. +- Provider identity. +- Bidder routing or explicit `all_eligible` routing. +- Standard OpenRTB request fields. +- Trusted Server signing invocation. +- Standard consent and EID forwarding. +- HTTP transport and backend correlation. +- Winner selection and mediation. + +### Profile configuration parity + +The first version must preserve these APS controls and defaults: + +| Current control | New owner | Default and validation | +| ------------------------ | -------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------ | +| `endpoint` | Common provider `endpoint` | Required fixed HTTPS endpoint; legacy unsupported endpoint forms remain rejected. | +| `timeout_ms` | Common provider `timeout_ms` | Defaults to 800 ms for `aps`; explicit values override it. | +| `account_id` | `auction.providers..profile_config.account_id` | Required, nonempty, and subject to the current size and input validation. | +| `debug` | `auction.providers..profile_config.debug` | `false`; preserves current debug behavior. | +| `allow_script_creatives` | `auction.providers..profile_config.allow_script_creatives` | `false`; preserves the existing explicit script opt-in. | +| `inventory_domain` | `auction.providers..profile_config.inventory_domain` | Absent by default; preserves current domain validation. | +| `inventory_page_origin` | `auction.providers..profile_config.inventory_page_origin` | Absent by default; must be configured with `inventory_domain` and preserve current origin/domain validation. | + +Parity tests must cover defaults, `all_eligible` routing as the current-behavior migration path, optional narrower `explicit` routing, debug behavior, inventory override validation, iframe creatives, permitted script creatives, and rejected script creatives. + +Using the APS profile must activate any server-side renderer support it requires independently of browser integration enablement. + +## Request Signing + +Request signing remains controlled by the existing optional global configuration. When global signing is enabled, signing is an auction-wide requirement and every OpenRTB provider request contains the complete version 1.1 Trusted Server signing extension. + +When global signing is disabled, no provider request contains signature-bearing fields (`version`, `signature`, `kid`, or `ts`). To preserve current Prebid wire behavior, the `prebid-server` profile may still emit `ext.trusted_server` containing only its existing `request_host` and `request_scheme` fields. APS and `standard` emit no `ext.trusted_server` object while signing is disabled. Static extension configuration cannot claim this reserved object. + +Applying the complete extension to APS and configuration-defined standard providers is an intentional coverage expansion from the current PBS-only behavior. It is not request-body signing and is not described as wire-parity with the current APS request. + +The project will: + +- Reuse the existing signing implementation and version 1.1 wire contract. +- Avoid cryptographic or protocol redesign. +- Keep signing configuration global rather than repeated under providers. +- Compile only the enabled signing policy into the immutable auction plan; loaded key material is never stored in the plan. +- Load the current signer once during auction admission, before provider routing and dispatch, when global signing is enabled. +- Fail the auction before any provider request is dispatched when the current signer cannot be loaded. +- Reuse that auction-local signer for every provider request in the fan-out. +- Preserve live key rotation by loading the current key for each admitted auction rather than only at process startup. + +Signing version 1.1 authenticates only its existing canonical payload: version, key ID, publisher host, publisher scheme, OpenRTB request ID, and timestamp. It does not authenticate the serialized OpenRTB body, provider ID, endpoint, bidder parameters, or profile extensions. Body or provider binding requires a future signing-protocol version and is outside this specification. + +Profiles augment their owned request fields before common request finalization. The common finalizer then inserts the signing extension and freezes the request ID and signing-owned fields. This ordering prevents profiles or static extensions from overwriting signing fields; it does not imply that version 1.1 authenticates the augmented body. + +Parity and compatibility tests must prove that Prebid Server preserves the existing signing contract and that APS accepts requests containing the extension. The standard-profile compatibility endpoint is a fictional local mock that exists only in automated tests. It implements the documented standard OpenRTB subset, requires no authentication or additional typed fields, and must not become a runtime endpoint, built-in provider, production-support claim, or new profile. + +## Transport and Execution + +The existing platform transport abstractions remain responsible for: + +- Backend registration and naming. +- Asynchronous request dispatch. +- Response correlation. +- Existing request and response bounds. +- Existing timeout behavior. +- Existing platform-specific fan-out capability checks. + +The compiler canonicalizes each provider endpoint once. The endpoint must be an absolute HTTPS URL with a nonempty host and no embedded username, password, or fragment. The same canonical endpoint supplies both the outbound request URI and the platform backend specification. Automatic redirects are not followed; a different destination requires a configuration change and recompilation. Existing TLS certificate and hostname verification remain enabled. Registered profiles may impose stricter endpoint validation, including the APS legacy-endpoint rejection. + +Every provider backend specification uses the validated provider ID as its discriminator. A profile ID is never sufficient for backend discrimination because multiple provider instances may use the same profile, endpoint, and timeout. Target-specific validation must reject provider IDs that collide after any adapter-specific backend-name encoding; lossy normalization may not silently merge them. + +Dispatch state associates the resulting backend identity with exactly one provider ID and compiled profile within an auction. A collision must fail before either mapping can overwrite the other. This first-version contract preserves the existing backend-name correlation mechanism without introducing a new cross-adapter per-request token system. + +At most one outbound request is sent per provider per auction. Exactly one request is sent for each provider with eligible slots, containing every slot admitted by its routing mode. + +The first version does not split one provider's slots across multiple requests and does not send one request per slot. + +The runtime always computes the logical provider budget as: + +```text +min(provider timeout, auction time remaining) +``` + +That exact logical budget controls whether a provider may launch, the OpenRTB `tmax` value, and whether later upstream or mediator network work may launch. It is distinct from a transport timeout used for backend construction and from a hard transport deadline. An adapter may canonicalize or quantize its transport timeout for stable backend identity, but that derived value must not replace the exact logical budget or shorten `tmax`. + +A hard network deadline is enforced only when the target adapter exposes an abortable total-request deadline. On such an adapter, a completion after the enforced deadline is discarded and classified as a provider timeout. + +Fastly currently provides first-byte and between-byte backend timeout controls, not an absolute total-request deadline. Axum has broader task/client cancellation behavior, and Cloudflare and Spin may use eager or broader platform HTTP execution, but no current adapter claims an enforceable provider-wide total-request deadline for this capability. To preserve current behavior, an already-launched call may therefore complete after its logical budget and its completed response remains eligible for local ranking and delivery. No additional provider or mediator network work may launch once the auction has no remaining logical budget, but local decision and delivery still complete. This rule applies equally when split dispatch/collect observes a completed response after the logical deadline. Documentation and tests must state that such an auction can exceed the configured wall-clock budget. + +The plan compiler and target-specific validator use an adapter capability description that distinguishes: + +- Concurrent fan-out support. +- Enforceable total-request transport deadlines. + +Adapters without concurrent fan-out continue to reject configurations with more than one active bidder provider. An adapter without an enforceable outbound deadline may still run one provider, preserving current behavior, but its documentation and tests must identify the transport limitation. + +Provider failures remain isolated from other provider outcomes. + +The automated standard-profile endpoint fixture is unauthenticated and test-only. This specification does not define bearer-token, custom-header, secret-store, or other endpoint-authentication schemas. + +## Response Normalization + +Every provider response is normalized into the existing shared auction response and bid model, or its clean architectural equivalent. + +The normalized result must preserve: + +- Provider ID. +- Valid returned seat, when present. +- Delivery bidder code. +- Slot/impression ID. +- Bid ID. +- Decoded price and existing currency assumptions. +- Banner dimensions. +- Standard creative markup or existing typed renderer. +- Existing notification URL behavior. +- Provider-specific metadata required for current diagnostics. + +Profile-specific response interpretation occurs before bids reach auction ranking or mediation. A registered profile may deterministically reduce its own provider response when required for parity, but it cannot compare bids across providers, apply auction floors, select final winners, or invoke mediation. + +Identity normalization is explicit: + +- A valid string `seatbid.seat` becomes `returned_seat`. +- A missing or non-string seat becomes no returned seat and cannot match `notifications.suppress_seats`. +- Prebid Server uses the valid returned seat as its delivery bidder code and otherwise preserves the current `unknown` fallback. +- APS always uses `aps` as its delivery bidder code, independently of its returned seat. +- Provider ID remains the only backend, health, and transport-correlation identity. + +One provider's malformed response does not fail another provider. Existing behavior for whether an invalid individual bid or full response is dropped should be preserved unless the common driver can enforce an equivalent stricter check without changing externally visible behavior. + +## Decision, Mock Mediation, and Delivery + +This project does not redesign the decision or delivery stages and does not introduce a generic mediator type. The existing `[auction].mediator` reference and statically registered mock mediator remain outside the compiled bidder-provider plan. Provider profiles cannot invoke mediation. + +After normalization, the existing system continues to: + +- Select the highest decoded-price bid per slot when no mediator is configured. +- Apply existing floors. +- Use existing USD assumptions. +- Invoke the existing mediator path when configured. +- Fall back according to existing mediation behavior. +- Sanitize and rewrite creatives according to existing settings. +- Serialize standard creatives and APS renderer descriptors according to existing contracts. + +Provider profiles do not perform cross-provider ranking or choose final auction winners. Provider-local candidate reduction remains part of response normalization where explicitly registered for parity. + +## Telemetry and Diagnostics + +Existing auction telemetry and provider result reporting remain in scope for parity. + +The new architecture must preserve the ability to report: + +- Provider instance ID. +- Provider outcome. +- Response time. +- Bid count. +- Returned seats. +- Existing error classifications. +- Winner status. +- Existing profile-specific diagnostics when enabled. + +The redesign may centralize how diagnostics are carried, but it must not introduce a new telemetry product or schema as part of this work. The existing telemetry seat carrier uses `returned_seat` when present and otherwise falls back to the delivery bidder code to preserve missing-seat behavior. If mock mediation reconstructs a bid, it restores the original provider bid's returned seat rather than deriving it from the mediator or delivery alias. + +New routing outcomes should be distinguishable: + +- `unroutable_bidder` +- `skipped_no_eligible_slots` +- `unused_bidder_params` + +They use a fixed `routing` object in existing metadata maps rather than new response or telemetry fields: + +- Auction-level `OrchestrationResult.metadata["routing"]` carries `unroutable_bidder_count` for internal diagnostics and bounded structured logging. +- A skipped provider produces its ordinary provider result with `AuctionResponse.metadata["routing"].skipped_no_eligible_slots = true`, so the existing `ProviderSummary.metadata` carrier remains usable. +- A called provider that receives but does not consume routed parameter objects records `AuctionResponse.metadata["routing"].unused_bidder_params_count`. + +Only booleans and saturating counts are carried. Bidder parameter values and bidder-ID lists are never included. Existing telemetry may consume these existing metadata carriers, but this work adds no telemetry columns or new client response fields. + +## Runtime Flow + +```text +1. Receive or construct canonical banner auction request. +2. Apply existing consent, identity, and privacy enforcement. +3. Resolve each client bidder through the central bidder registry. +4. Add trusted provider routes for server-generated opportunities. +5. Build one filtered ProviderAuctionInput per provider. +6. Skip providers with no eligible slots. +7. Use the OpenRTB 2.6 driver to construct the standard request. +8. Invoke the selected profile to augment the request. +9. Apply the existing Trusted Server request signature. +10. Dispatch one request per provider through existing transport. +11. Decode the standard OpenRTB response. +12. Invoke the selected profile for provider-specific normalization. +13. Produce normalized provider outcomes. +14. Run existing local ranking or mediation. +15. Run existing creative delivery and telemetry. +``` + +## Failure Behavior + +The system must preserve partial-auction behavior. + +| Condition | Required outcome | +| ----------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------ | +| Unknown bidder in runtime request | Record `unroutable_bidder`; continue other demand. | +| Provider has no eligible slots after applying its routing mode and banner filtering | Record `skipped_no_eligible_slots`; do not call provider. | +| Profile configuration invalid | Reject configuration at deploy/startup. | +| Enabled auction signing cannot load the current signer | Fail the auction before any provider request is dispatched. | +| Provider cannot build a valid request | Provider-local launch/build failure. | +| Provider transport fails | Provider-local transport failure. | +| Provider times out | Provider-local timeout. | +| Provider returns valid no-bid | Provider no-bid. | +| Provider response cannot be decoded | Provider-local parse failure. | +| Individual bid is invalid | Preserve current profile/common validation behavior; valid sibling bids remain eligible where currently supported. | +| No providers produce valid bids | Existing auction no-winner behavior. | + +No request with zero eligible impressions should be sent upstream. + +## Security Requirements + +- Provider endpoints are fixed operator configuration, never client-derived. +- Clients cannot select provider IDs or endpoint URLs. +- Each bidder has one server-controlled provider route. +- Provider endpoints are canonical absolute HTTPS URLs with nonempty hosts and no URL credentials or fragments. +- The same canonical endpoint is used for the request URI and backend registration, redirects are not followed automatically, and existing TLS certificate and hostname verification remain enabled. +- Static extensions cannot contain secrets or request templates. +- Profiles cannot bypass central privacy enforcement. +- Profiles cannot access raw browser requests or unrestricted runtime services. +- Provider parameters must not be logged or returned in unbounded diagnostics. +- Existing response-body limits remain enforced. +- Existing creative sanitization remains enforced after winner selection. + +## Acceptance Criteria + +### Configuration and compilation + +- [ ] Provider instances are configured under `[auction.providers.*]`, with the selected profile's typed settings under `profile_config`. +- [ ] The first version recognizes only `openrtb-2.6`. +- [ ] `standard`, `prebid-server`, and `aps` profiles are registered through Rust profile factories. +- [ ] Profile availability is independent of browser integration enablement. +- [ ] Provider and profile configuration is compiled once at startup. +- [ ] Omitted provider timeouts resolve to 1000 ms for `prebid-server`, 800 ms for `aps`, and the auction timeout for `standard`; explicit values override those defaults. +- [ ] Deploy validation and runtime startup use the same two-stage provider compiler, profile registry, adapter capability descriptions, and shared backend-name prediction algorithms; target-agnostic validation reports deferred target checks. +- [ ] Provider IDs enforce the documented lowercase ASCII grammar and length; duplicate IDs, target-encoded backend-name collisions, unknown profiles, invalid endpoints, and invalid bidder routes fail validation. +- [ ] Endpoint tests require canonical absolute HTTPS URLs, reject missing hosts, credentials, and fragments, use the same URL for request and backend construction, and prove redirects are not followed automatically. +- [ ] Every backend specification uses the provider ID rather than the profile ID as its discriminator. +- [ ] Two provider IDs using the same profile, endpoint, and timeout dispatch and correlate independently. +- [ ] Target-specific validation rejects more than one active provider on adapters without concurrent fan-out support. +- [ ] Adapter capabilities distinguish concurrent fan-out from enforceable total-request transport deadlines. +- [ ] Invalid enabled signing configuration fails deploy/startup validation. +- [ ] When signing is enabled, inability to load the current signer fails auction admission before any provider dispatch. + +### Routing + +- [ ] Clients submit bidder identities and parameters without selecting providers. +- [ ] Request admission unfolds the reserved `trustedServer.bidderParams` envelope into canonical bidder IDs before routing. +- [ ] The reserved `trustedServer` envelope cannot name provider IDs or endpoints, and its `zone` value cannot influence routing. +- [ ] Missing, null, and empty recognized `trustedServer.bidderParams` create server-controlled stored-request routes to configured `prebid-server` plans; malformed, unknown-bidder, partial-validity, bounds, and direct/envelope collision cases follow the documented deterministic rules. +- [ ] Initial and refresh auction tests cover mixed PBS, APS, direct server-side, and client-side bidders, proving that only server-side entries are unfolded and routed. +- [ ] `[auction.bidders]` routes each bidder to exactly one provider. +- [ ] Unknown runtime bidders are recorded as `unroutable_bidder` without failing other demand. +- [ ] `explicit` is the default provider routing mode. +- [ ] `all_eligible` is available only through explicit provider configuration and is documented as the current-behavior migration mode for APS. +- [ ] Trusted server-generated slots may route directly to providers without inline bidder parameters. +- [ ] Provider inputs contain only slots admitted by the provider's routing mode and only bidder parameters assigned to that provider. +- [ ] `all_eligible` does not expose bidder parameters assigned to another provider. +- [ ] Providers with no eligible slots are skipped without an upstream request. + +### OpenRTB and profiles + +- [ ] The common OpenRTB driver constructs current standard banner request fields by applying the selected profile's fixed typed field policy. +- [ ] Central privacy enforcement defines the maximum permitted data, and profile policies may only omit from that approved view. +- [ ] The `prebid-server` and `aps` field policies preserve their current request differences, including the complete consent-field matrix. +- [ ] The `standard` profile uses the documented shared PBS and APS baseline, consent placements, and omissions. +- [ ] Prebid transport preserves current `User-Agent`, raw `Referer`, `Accept-Language`, platform-attested `X-Forwarded-For`, and exact selected `Cookie` header behavior without exposing the raw request to profiles. +- [ ] The common driver preserves existing consent, identity, floor, timeout, currency, and signing wire semantics while intentionally expanding enabled signing coverage to every OpenRTB provider. +- [ ] Tests prove that standard, Prebid Server, and APS requests receive the version 1.1 signing extension after profile request augmentation when global signing is enabled. +- [ ] Tests prove that global signing disabled removes all signature-bearing fields, preserves the existing PBS-only host/scheme object, and omits `ext.trusted_server` from APS and `standard`. +- [ ] Signing tests document that version 1.1 does not bind the request body, provider ID, endpoint, bidder parameters, or profile extensions. +- [ ] APS and the automated, fictional standard-profile mock endpoint have signed-request compatibility fixtures; the latter exists only in tests and requires no runtime feature or authentication. +- [ ] Standard OpenRTB endpoints can be configured without adding a provider implementation. +- [ ] Static `request.ext` and `imp.ext` objects are bounded and validated. +- [ ] Client bidder parameters are not assigned an invented generic wire location. +- [ ] The Prebid profile preserves current bidder parameters, stored requests, cache handling, diagnostics, and notification behavior. +- [ ] Common notification suppression preserves `nurl` and `burl` by default, supports provider-wide suppression, and matches per-seat suppression against exact returned seat IDs independently of bidder routes. +- [ ] Prebid parity tests cover the exact existing `openrtb_only`, `cookies_only`, and `both` Cookie-header behavior, including KV/policy-sourced body-consent fallback. +- [ ] The APS profile preserves current account extensions, inventory identity, response validation, renderer, script policy, diagnostics, and deterministic highest-price-per-impression reduction with its bid-ID tie-breaker. +- [ ] Prebid and APS no longer register singleton auction-provider instances. + +### Runtime behavior + +- [ ] At most one outbound request is sent per provider per auction, and none is sent when the provider has no eligible slots. +- [ ] The exact logical provider budget always controls launch eligibility and OpenRTB `tmax`; adapter-specific transport-timeout canonicalization does not replace or shorten it. +- [ ] Hard total-request deadlines discard late completions where supported and are not claimed for any current adapter without an abortable absolute deadline. +- [ ] On adapters without enforceable provider deadlines, completed late responses remain eligible, no new network work launches after logical budget exhaustion, and synchronous plus split execution tests document the possible wall-clock overrun. +- [ ] Existing adapter-specific concurrent fan-out and timeout behavior remains intact. +- [ ] Provider failures remain isolated. +- [ ] Provider ID, returned seat, and delivery bidder code remain distinct through direct and mediated outcomes, browser delivery, and the existing telemetry seat carrier; PBS and APS preserve their documented delivery aliases. +- [ ] Existing winner selection, floors, mock mediation, creative delivery, and telemetry continue to behave as before. +- [ ] Banner behavior has parity with the existing Prebid and APS paths. +- [ ] Non-banner formats are excluded before routing and are never emitted upstream. +- [ ] A slot with no banner formats is skipped. +- [ ] Video and native are not introduced by this work. + +## Deferred Design Areas + +The following require separate requirements before implementation: + +- Additional protocols. +- Video and native support. +- Multiple provider routes for one bidder. +- Provider groups and label/rule-based routing. +- Request splitting for large auctions. +- New money or currency-conversion models. +- New signing protocol versions. +- New privacy or data-sharing controls. +- Consent-cookie forwarding minimization or changes to existing malformed-header behavior. +- New diagnostics and telemetry schemas. +- General endpoint authentication configuration. +- Arbitrary standard-field mappings. +- Runtime or sandboxed profile extensions. +- Generic mediator types, mediator profiles, and configuration-first mediation. +- New abortable outbound-timeout implementations for adapters that do not currently provide them. +- Broader outbound-network policy for IP literals, private or reserved networks, custom ports, and DNS rebinding. + +## Open Questions + +No unresolved product decisions currently block this specification. + +Implementation planning must still identify: + +- The exact Rust profile-factory and compiled-profile interfaces. +- The minimal changes required to separate the current Prebid and APS logic into common OpenRTB and profile-owned behavior. +- The internal organization of the unauthenticated, automated standard-profile mock endpoint and its fixtures. +- The complete parity test fixture set for Prebid, APS, routing, signing, and split dispatch/collect execution. +- The exact configuration representation needed by existing environment override and app-config tooling. + +These are implementation-planning questions and must not expand the product scope defined above. + +## Related Designs + +- [Auction Orchestration Flow](./2026-03-19-auction-orchestration-flow-design.md) +- [Prebid Generic Bid Parameter Override Rules](./2026-04-08-prebid-generic-bid-param-override-rules-design.md) +- [APS OpenRTB First-Class Integration](./2026-07-15-aps-openrtb-first-class-integration-design.md) diff --git a/docs/superpowers/specs/2026-08-18-contiguous-generated-slot-tables-design.md b/docs/superpowers/specs/2026-08-18-contiguous-generated-slot-tables-design.md new file mode 100644 index 000000000..200a8abbe --- /dev/null +++ b/docs/superpowers/specs/2026-08-18-contiguous-generated-slot-tables-design.md @@ -0,0 +1,15 @@ +# Contiguous Generated Slot Tables Design + +## Problem + +`splice_creative_slots` parses rendered slots in a temporary `toml_edit::DocumentMut` and moves its `ArrayOfTables` into the target document. Parsed tables retain document-local numeric positions. Those positions collide with positions in the target document, so serialization can interleave generated slot and provider tables with unrelated top-level tables even though the resulting TOML remains semantically valid. + +## Design + +Before insertion, assign the generated slot tables and all nested provider tables the target `[creative_opportunities]` table's document position. `toml_edit` performs a stable position sort, so equal positions retain traversal order: the creative table, each slot, and that slot's provider tables remain contiguous. For a newly created creative section, allocate an anchor after the greatest existing parsed-table position. + +The update continues to preserve unrelated values, comments, line endings, and semantic table ownership. It does not reformat existing operator-authored content or modify slot inference. + +## Testing + +Add a regression fixture with a late `[creative_opportunities]` section and unrelated tables whose positions overlap those from the temporary generated document. Assert that the parent, generated slots, and provider subtables serialize contiguously before the next unrelated table. Retain the existing semantic-preservation and CRLF tests, then run the CLI test suite, formatting, and native CLI clippy. diff --git a/docs/superpowers/specs/2026-08-18-kv-snapshot-main-integration-design.md b/docs/superpowers/specs/2026-08-18-kv-snapshot-main-integration-design.md new file mode 100644 index 000000000..336c6f21e --- /dev/null +++ b/docs/superpowers/specs/2026-08-18-kv-snapshot-main-integration-design.md @@ -0,0 +1,134 @@ +# KV Snapshot Main Integration Design + +**Date:** 2026-08-18 +**Status:** Approved for implementation + +## Goal + +Bring PR #885 onto current `main` without regressing the true-streaming SSAT +publisher path, and verify that its request-scoped EC KV snapshot and orphaned +cookie recovery remain complete after the intervening auction, telemetry, +privacy, and adapter changes. + +## Findings + +PR #885 remains necessary. Current `main` still performs a synchronous +identity-graph read while decorating an SSAT auction, repeats EC graph reads in +later request phases, and does not recover a valid `ts-ec` cookie whose KV root +expired. + +The branch predates true publisher streaming. Its origin-first scheduling uses +`PlatformHttpClient::send_async`, while current Fastly code rejects +`stream_response` on that path. A mechanical merge would therefore either +buffer publisher responses again or restore the KV-before-origin latency. + +PR #1013 also retains the synchronous KV read before its C2 lookup. A warm ESI +template hit therefore has an additional ordering concern that cannot be fixed +inside a `main`-based PR without importing #1013. This integration will verify +the combined branches and report that concern separately. + +## Design + +### Preserve current `main` + +Merge `origin/main` into the published feature branch. Resolve conflicts by +retaining current streaming response conversion, auction telemetry, configured +publisher-domain attribution, inline creative handling, conditional/range +stripping, cache bypass, DataDome suppression, and final response privacy. + +The EC additions remain request-scoped: + +1. Build or receive the active EC ID. +2. Start the publisher origin request before reading the identity graph when + the platform can provide concurrent fan-out without sacrificing response + streaming. +3. Load one `EcKvSnapshot` and use it to decorate the auction. +4. Dispatch the auction and await the already-running origin request. +5. Carry the updated snapshot through response finalization and post-send pull + sync. + +### Async origin response streaming + +Extend the platform HTTP contract narrowly so a single pending origin request +can preserve a streamed response when it is awaited. Fastly will support this +path; adapters that cannot combine pending sends and streamed responses retain +the eager fallback. + +The capability must be explicit. The publisher handler may use origin-first +scheduling only when the client advertises concurrent fan-out and pending +streamed-response support. Otherwise it loads the snapshot, dispatches the +auction, and uses the existing streamed `send` path. + +Fastly's auction fan-out continues using the existing multi-request `select` +behavior. A stream-preserving pending request carries the response-stream flag +and original request method through the opaque pending handle. Fastly overrides +`wait` to complete that handle directly with `fastly::PendingRequest::wait()` +and passes both values to the existing response converter, which preserves +HEAD and bodiless-status framing while returning `EdgeBody::Stream` only when +a response body is allowed. + +A stream-preserving pending request must never enter multi-request `select`: +Fastly rejects that combination defensively. Regular auction pendings retain +the current buffered `select` path and its backend-correlation behavior. This +single-wait boundary prevents stream metadata from becoming ambiguously +associated when Fastly reorders remaining pending handles. + +### Recovery and privacy + +Orphan recovery remains limited to a consent-granted, real-browser publisher +navigation after the origin request has successfully started. A missing row is +confirmed with a second read before rotation; failed reads do not rotate. + +Generated IDs are persisted with add-only semantics before a cookie is emitted. +Consent withdrawal tombstones existing roots only, including both the incoming +cookie ID and a different active ID when applicable. Finalization updates the +request snapshot, and pull sync performs one request-wide bulk CAS from that +state. + +## Error Handling + +- A concurrent pending-origin start failure returns the existing proxy error + and performs no EC KV preload or auction dispatch. +- A pending-origin wait failure occurs after auction dispatch. It emits exactly + one existing `origin_proxy_error` abandonment event and returns the existing + proxy error. +- On the eager fallback, origin send failure likewise happens after snapshot + preload and auction dispatch and retains current `origin_proxy_error` + abandonment behavior. +- KV read failure becomes a non-authoritative failed snapshot; auctions proceed + without server-side EIDs and recovery does not rotate. +- Pending streamed-response setup failure follows the existing proxy error path. +- Platforms without combined pending/streaming support use the eager fallback. +- All failures continue using `error-stack` and existing logging conventions. + +## Testing + +Tests will be written before production changes and must prove: + +- a streaming-capable concurrent client starts origin before the EC KV read; +- the pending origin response remains an `EdgeBody::Stream`; +- Fastly's direct single-request `wait` does not call `select`, while ordinary + auction fan-out remains on the buffered `select` path; +- stream-marked pendings are rejected by multi-request `select`; +- HEAD plus `1xx`, `204`, `205`, and `304` responses remain bodiless with their + existing content-length/framing semantics on the pending-stream path; +- cache bypass, request rewriting, conditional/range removal, and DataDome + behavior survive the reordered path; +- a pending-origin start failure performs no KV read, auction dispatch, or + abandonment emission; +- a pending-origin wait failure after dispatch emits exactly one + `origin_proxy_error` abandonment and returns the existing proxy error; +- fallback clients preload before auction dispatch, use `send`, request a + streamed response exactly when `supports_streaming_responses` is true, and + otherwise retain their existing buffered behavior; +- the snapshot is reused by auction, finalize, cookie ingestion, withdrawal, + and pull sync; +- missing, failed, tombstoned, newly created, and transiently missing snapshots + retain the existing fail-closed behavior; +- current SSAT telemetry and publisher-domain attribution remain intact, + including the configured publisher domain in both the `AuctionRequest` and + emitted observation rows on the reordered path. + +After focused red/green tests, run repository formatting, all adapter test +aliases, all target-matched Clippy aliases, and a temporary merge check against +`1009-esi-cacheable-root-spec`. No #1013 code will be committed to PR #885. diff --git a/docs/superpowers/specs/2026-08-18-pr-823-review-resolution-design.md b/docs/superpowers/specs/2026-08-18-pr-823-review-resolution-design.md new file mode 100644 index 000000000..5ff4d0707 --- /dev/null +++ b/docs/superpowers/specs/2026-08-18-pr-823-review-resolution-design.md @@ -0,0 +1,254 @@ +# PR 823 Review Resolution Design + +## Goal + +Resolve the actionable findings in review `4958563121` on PR 823 without +unrelated refactoring, verify the complete branch, publish the fixes, and reply +to every inline review thread with concrete resolution evidence. + +## Scope + +The implementation covers all 28 inline threads and all actionable items in the +review summary. The summary's explicitly out-of-scope pre-existing +partially-invalid `page_patterns` behavior is not expanded into this PR unless a +fix is required by another in-scope change. The PR description's stale legacy +alias sentence is corrected after the branch changes are published. + +Each reviewer suggestion is verified against the current code. A suggestion is +implemented when it is correct for this repository. Where repository evidence +contradicts a suggestion, the implementation retains the correct behavior and +the review response explains the evidence. + +## Design Principles + +- Preserve operator-authored configuration, comments, ordering, and unrelated + sections byte-for-byte wherever possible. +- Never print secrets or whole effective configuration documents as diagnostic + output. +- Never turn uncertain crawl evidence into a runnable fabricated ad-unit path. +- Treat browser navigation as a session, not a sequence of isolated launches. +- Keep `generate`, `verify`, static CLI commands, and runtime matching on shared + domain rules instead of parallel reimplementations. +- Bound all page-controlled data and browser operations. +- Use test-first changes for behavior corrections and minimal annotations for + code-quality-only corrections. + +## Component Design + +### 1. Configuration integrity and command output + +`slot_toml` will replace the line-oriented slot-boundary heuristic with a +TOML-aware edit strategy. The resulting document must preserve every top-level +item outside the managed creative-opportunity fields and preserve comments +adjacent to or between operator sections. Non-contiguous slot declarations, +multiline values, arrays whose continuation lines begin with `[`, trailing +comments, CRLF input, and inline-slot conversion receive regression coverage. +The updater will reject a candidate if preservation cannot be proven. + +Generation will re-read the source config immediately before the atomic write +and refuse to overwrite a concurrently edited file. `--dry-run` will emit only +the managed creative-opportunities change, never the complete config. Notes and +rollback warnings go to stderr so machine-readable stdout remains clean. Tests +will prove that dry-run leaves the source file byte-identical and does not expose +unrelated secret-bearing keys. + +Merge behavior remains add-only for operator-authored data: existing templated +unit paths are retained, newly observed formats are unioned, and multiple +discovered placements absorbed by one broad configured div prefix produce an +operator note. + +### 2. Crawl evidence and inference + +Inference will preserve evidence instead of silently collapsing it: + +- Non-ASCII shared-prefix computation uses UTF-8 byte boundaries. +- Same-page normalization collisions retain distinct raw placements and emit a + diagnostic rather than silently dropping formats. Numeric-only stable tokens + are not classified as hexadecimal hash noise. +- Multi-slot SRA request fallbacks are ignored when `dids` names more than one + slot. +- A page is considered empty only when no audited profile found slots there. +- Fragment detection requires stronger evidence: a useful shared prefix, or at + least three disjoint fragments. Ambiguous two-slot groups are retained with a + note. +- Locale landing paths are emitted literally when they are shorter than the + inferred section depth, and literal path segments are escaped before being + interpolated into globs. +- Refused template decisions are omitted from generated slots and surfaced with + their reasons. The documentation and tests will consistently describe these + cases as refusal, not literal fallback. +- The redundant witness rule is removed or made independently meaningful. The + actual crawler will support the section depth that inference can produce; + locale-prefixed behavior will not exist only in hand-built evidence tests. +- Dropped-section diagnostics are capped, percent-encoded paths are normalized + before filtering, and page-like extensions are classified consistently. + +The root page and section pages for a device profile are collected in one +browser session. Page analysis that parses full HTML is moved off the +current-thread CDP event pump. Each page/tab is closed on every success and +error path. + +### 3. Shared browser behavior + +The browser collectors will share executable discovery and launch/session +configuration. Browser options exposed to operators will have one meaning in +`page`, `verify`, and `generate`: Chrome override, settling, headful/headless +mode, device profile/viewport, proxy, consent assumption, cookies, and TLS +policy. + +`verify` will reuse one browser/runtime/profile across its URLs so clearance and +session state survive. The generic/legacy generator will default to the same +consent assumption as ad-template generation and expose the opt-out rather than +depending on `derive(Default)`. + +Cookie parameters are explicitly host-only with `Path=/`. A same-host +`http`-to-`https` upgrade is accepted with a redirect note; host changes, +downgrades, and unexpected port changes remain cross-origin refusals. Failure to +read or parse the final browser URL fails closed instead of substituting the +requested URL. + +Every post-navigation evaluation is time-bounded. The collector enlarges the +resource timing buffer before navigation, waits for an interactive or complete +document before accruing quiet time, honors sub-poll quiet windows, validates +`quiet <= max`, and reports saturation. Navigation load-event timeout is a +warning after a successful `goto`; it does not discard readable page evidence. +Evidence payload bytes and captured string lengths are capped before expensive +decode/allocation. + +Init-script and page-evaluation failures become explicit warnings or errors +rather than empty evidence. Promise-returning sitemap evaluation awaits its +result. Main-frame-only collection is disclosed when frames are skipped. + +The injected collector will be behavior-preserving: size pairs enforce the +`u32` range, the `googletag` setter is total, the unused non-variadic `cmd.push` +wrapper is removed, wrapping markers are closure-local/non-enumerable, and +page-derived warning text is terminal-safe. + +### 4. Runtime and static-command parity + +Expected-slot projection uses the runtime's renderability rule. Slots the +runtime omits for a path do not count as matched verification slots; diagnostics +state that the runtime omits the slot on that path rather than claiming the +whole config is rejected. + +Configured media type remains a typed `MediaType` through comparison and is +rendered to a string only at the output boundary. Slots that the phase-one +checker cannot confirm (video/native-only) are represented as unconfirmable and +do not fail `--strict`; genuinely partial or missing confirmable slots still +fail, including a live out-of-page slot with no sizes matched against +banner-configured formats, which is partial. Slot phase is absent when no +evidence exists. +The server-side APS compatibility field no longer creates unconditional +client-side `fetchBids` warnings. + +Collector warnings are included in page results. Human output includes the +runtime expectation, gate summary, matched count, extra evidence, and warnings +already present in JSON. Output escaping covers Unicode bidi controls and all +config-derived strings. + +`explain` reports exactly the shared runtime gate result. Provider configuration +is a separate advisory. The unsupported `--edgezero-enabled` model and stale +legacy-fallback claim are removed because no runtime condition backs them. +Gate diagnostics consume the shared gate result instead of rebuilding lists by +hand. The hot runtime gate avoids heap allocation, the seven-boolean wrapper is +removed, and the consent tri-state is documented and exhaustively tested. + +`compile_page_pattern` becomes crate-private and a public validation-only API is +used by the CLI. `lint` explicitly reports every configured page pattern the +runtime would drop, while the broader pre-existing runtime acceptance policy +remains out of scope. Specific compile failures are retained in logs. HTTP +methods use `http::Method` parsing so CLI semantics match the runtime. + +Full URLs and bare path inputs pass through the same URL normalization rules: +percent-encoding, dot-segment resolution, query/fragment removal, and leading +slash behavior must be identical. Scheme detection is anchored to the path +portion before `?`, so an absolute URL inside a query value does not cause a +bare path to be parsed as a full URL. + +### 5. CLI contracts, documentation, and CI + +Clap owns argument validation: URL parsing happens at the value parser, the +audit namespace uses help-on-missing-subcommand, `check` uses an argument group +and conflicts, and settle bounds are rejected during parsing. Parser tests cover +the visible command shapes and legacy restrictions. + +CI-oriented assertion failures exit 1; tool/configuration/navigation failures +exit 2. Assertion text is written directly and cannot disappear behind a log +filter. The guide documents all four `ts config ad-templates` commands, all +flags, shared config-loading flags, browser flags, consent/profile behavior, +dry-run output, and exit codes. + +Browser fixture CI either installs/resolves Chrome and requires the tests to +execute, or explicitly opts into a mode that fails when Chrome is unavailable; +it may not report success after silently skipping every browser assertion. + +All real-looking customer identifiers and names introduced by this PR are +replaced with fictional values in tests, comments, and documentation. Stale +module-level lint suppressions, inaccurate docs, assertion messages, enum +ordering, dead query matching, and orphaned comments are corrected without +unrelated cleanup. + +## Inline Review Traceability + +| Thread | Resolution area | +| -------------------------- | ------------------------------------------------------------------ | +| `3802056460`, `3802056470` | TOML-aware splice and comment/value preservation | +| `3802056474` | Secret-safe dry-run and stderr diagnostics | +| `3802056481` | Omit and explain refused slots | +| `3802056488` | UTF-8-safe div prefix calculation | +| `3802056494` | Same-page normalized-div collisions | +| `3802056497` | Locale landing-page patterns | +| `3802056502` | Multi-profile empty-page accounting | +| `3802056508` | Close every browser tab | +| `3802056513` | Enforce JavaScript-to-Rust `u32` bounds | +| `3802056521`, `3802056529` | Total GPT hook and removal of behavior-changing `cmd.push` wrapper | +| `3802056539` | Shared faithful browser launch configuration | +| `3802056549`, `3802056555` | Correct settling and load-timeout handling | +| `3802056559` | Preserve injected collector warnings | +| `3802056564`, `3802056571` | Runtime renderability parity and accurate diagnostics | +| `3802056580`, `3802056584` | Unconfirmable status and removal of false APS warning | +| `3802056586` | Identical URL and bare-path normalization | +| `3802056593` | Fictional committed examples | +| `3802056599` | Browser fixture CI must execute or fail loudly | +| `3802056605` | Add-only merge of formats with broad-prefix diagnostics | +| `3802056614` | Consent parity for generic and legacy generation | +| `3802056623` | Refusal behavior, tests, and documentation agree | +| `3802056628` | Safe same-host HTTP-to-HTTPS redirect handling | +| `3802056638` | Remove ungrounded EdgeZero fallback model | + +## Error Handling and Compatibility + +All new Rust fallible paths use the repository's existing `CliResult` / +`error-stack` conventions. Browser failures identify the operation and URL but +do not include cookies, configuration values, or page payloads. Best-effort +cleanup must not replace an earlier collection error. + +JSON compatibility is preserved where possible. New distinctions are additive +or correct semantically invalid fields: unconfirmable status is explicit, and +phase may be omitted when there was no evidence. Documentation is updated with +the exact wire behavior. + +## Verification Strategy + +Each behavioral issue follows red-green-refactor: + +1. Add the smallest unit, parser, orchestration, or fixture test reproducing the + review finding. +2. Run the narrow test and confirm the expected failure. +3. Implement the minimal correction. +4. Re-run the narrow test and the affected crate suite. + +Final verification runs the repository-required commands relevant to the +changed surface: CLI tests through `scripts/test-cli.sh`, target-matched Rust +tests, JS tests when the collector script changes, `cargo fmt --all -- --check`, +all target-matched clippy aliases, documentation formatting, and browser fixture +tests with an available Chrome. Any environment-dependent test that cannot run +is reported explicitly and is not described as passing. + +## Review Replies and Publication + +Changes are grouped into reviewable commits by component, then pushed to the PR +branch after final verification. Each inline reply is posted in its existing +thread and states the concrete change, relevant test, or evidence-backed reason +for retaining behavior. Replies avoid generic acknowledgements. Threads are not +replied to as fixed until the corresponding commit is visible on GitHub. diff --git a/docs/superpowers/specs/2026-08-18-pre-navigation-cookie-install-design.md b/docs/superpowers/specs/2026-08-18-pre-navigation-cookie-install-design.md new file mode 100644 index 000000000..e9025174d --- /dev/null +++ b/docs/superpowers/specs/2026-08-18-pre-navigation-cookie-install-design.md @@ -0,0 +1,15 @@ +# Pre-navigation Cookie Installation Design + +## Problem + +The audit collectors open `about:blank` so initialization scripts can be installed before publisher code runs. Cookies are explicitly scoped by domain and `/`, but `chromiumoxide::Page::set_cookie` rejects cookies without a URL while the page is still `about:blank`. Consequently, any audit using `--cookie` fails before navigation; audits without cookies are unaffected. + +## Design + +Build the same host-only, root-scoped `CookieParam` values, then install them through `Browser::set_cookies` before creating the page. Browser-level installation sends the explicit domain/path cookie directly to Chrome without deriving scope from the current page URL. Both verification and generation collectors use one shared helper so their behavior cannot drift. + +Cookie-installation errors remain fatal and identify the affected cookie without logging its value. Page initialization, first-request authentication, browser-session reuse, and cookie scope remain unchanged. + +## Testing + +Add a Chrome-backed regression test that starts with `about:blank`, installs a cookie through the shared browser helper, navigates to a local HTTP fixture, and verifies the cookie is visible on the first loaded document. Run the focused CLI tests, formatting, and lint checks required for the touched crate. diff --git a/docs/superpowers/specs/2026-08-19-ad-template-generation-progress-design.md b/docs/superpowers/specs/2026-08-19-ad-template-generation-progress-design.md new file mode 100644 index 000000000..8f75d0dcf --- /dev/null +++ b/docs/superpowers/specs/2026-08-19-ad-template-generation-progress-design.md @@ -0,0 +1,59 @@ +# Ad-template generation progress design + +## Problem + +`ts audit ad-templates generate` audits up to the configured page budget for +each selected device profile (17 pages by default). Navigation and page settling +are intentionally bounded but can still take tens of seconds per page. The +browser collector buffers page results until the browser session closes, so the +command currently emits no output during most of that work and appears stuck. + +## Design + +Emit line-oriented progress on stderr while collection is running. Progress +must identify the device profile, current page, known total, and safe page +location. It must also identify non-page phases where a noticeable pause can +occur: launching the browser, planning the crawl after the root page, and +finalizing the browser session. + +Progress is an explicit collector callback rather than direct terminal output +inside the browser implementation. This keeps output policy in the command +layer, makes the behavior testable with in-memory writers, and lets non-browser +collectors preserve the same contract. Each line is flushed immediately. + +The first profile's root navigation has no final total because follow-up pages +are planned from the rendered root. It is reported as `1/?`; once planning +finishes, subsequent pages use a stable `current/total` count. Later profiles +receive the complete target list and report the root as `1/total`. Totals include +the root, and every attempted page advances the current count even if collection +fails. + +Progress never prints a full URL. It renders only the origin-free path, omitting +userinfo, query, and fragment data, then applies the CLI's existing terminal-text +sanitizer. An empty path is rendered as `/`. + +Stdout remains reserved for the generated diff or success summary. This keeps +`--dry-run` and shell redirection stable. Progress is intentionally plain text, +not an animated spinner, so it remains useful in logs and does not add a terminal +UI dependency. + +## Error handling + +Failure to write or flush progress is returned as a normal CLI output error. A +callback failure during a browser session stops further collection but does not +skip finalization, browser close, or process wait. An earlier collection or +planning error takes precedence over a later progress error; either takes +precedence over teardown errors. Close and wait are still attempted +independently. No cookie values, URL credentials, query values, fragments, or +browser credentials are included in progress. + +## Tests + +Unit tests will verify that progress is emitted before collection completes, +contains the specified profile-aware page counts, keeps stdout unchanged, +redacts URL credentials/query/fragment data, sanitizes paths, and reports +finalization. Writer tests will cover write failure, flush failure, and explicit +flush invocation. Collector tests will verify teardown still runs after progress +failure and that collection/planning errors, progress errors, and teardown errors +retain the stated precedence. The existing CLI and Chrome-backed suites will +verify the collector behavior and browser lifecycle remain intact. diff --git a/docs/superpowers/specs/2026-08-19-refuse-volatile-div-collisions-design.md b/docs/superpowers/specs/2026-08-19-refuse-volatile-div-collisions-design.md new file mode 100644 index 000000000..bdc9c5b9c --- /dev/null +++ b/docs/superpowers/specs/2026-08-19-refuse-volatile-div-collisions-design.md @@ -0,0 +1,95 @@ +# Refuse Volatile Div-ID Collisions + +## Problem + +GPT discovery normalizes per-render div IDs such as +`ad-in_content--in_content-0` to the stable prefix `ad-in_content`. +When several live elements on the same page normalize to that prefix, the +runtime cannot represent them safely: one prefix resolves at most one element, +while each exact raw ID changes on a later render. The current collision path +preserves the raw IDs, causing `--replace` to write unusable literal slots. + +## Design + +Treat a source-local normalized collision as ambiguous and refuse the entire +group. The first observation remains tentatively accepted. When a second raw div +ID that describes a _different element_ normalizes to the same prefix, remove +the first slot, record the group as ambiguous, and suppress every later member. +Emit one diagnostic when the group first becomes ambiguous, naming the +normalized prefix and explaining that neither a single prefix nor volatile exact +IDs are safe. Tell the operator to expose distinct stable div IDs or prefixes in +publisher markup before configuring the placements. + +Two raw IDs sharing a stem are not by themselves two elements. One element +re-rendered under a fresh framework token produces exactly that shape, and +absorbing it is what normalization is for: a React publisher reports +`ad-header-0-_R_3f_` from the server render and `ad-header-0-_r_0_` from the +client one, and refusing that pair would generate no slots at all. The two cases +are separated by comparing what the ephemeral markers did _not_ cover — the +marker spans are excised and the remaining parts compared, so identical +residues mean one element observed twice, while `-in_content-0` against +`-in_content-1` means two siblings and is refused. + +The verdict is site-wide, not page-local. Article pages carry several in-content +units and refuse the shared prefix while a landing page carries one, so a +page-local refusal would let crawl sampling decide whether the ambiguous prefix +reaches the config. `DiscoveredSlots` therefore carries the refused stems, +`EvidenceTable` unions them across pages, and the slot iterator the writer reads +suppresses them regardless of which page contributed them. + +Registry and request-derived evidence retain separate collision maps, matching +the current source precedence: even an ambiguous registry stem continues to +suppress request fallback for that stem. Network-ID discovery is unaffected. + +`DiscoveredSlots` records whether any otherwise usable GPT slot evidence was +seen independently of how many safe slots remain. `EvidenceTable::fold_page` +uses that signal when classifying empty pages, so a collision-only page is not +mistaken for a bot challenge. Cross-page slot inference, merging, and +`--replace` otherwise remain unchanged because ambiguous slots never enter +those stages. + +Some ad stacks build IDs as `__`, where the +render token — at least ten leading digits followed by more alphanumerics, +that is, a millisecond timestamp plus entropy — sits _before_ the part that +distinguishes one placement from the next. Such an ID can be written neither +literally nor as a prefix: the only stable prefix stops at the token and reaches +every placement in the family at once. Discovery refuses a single otherwise +usable registry or request observation of that shape, preserves the page/network +evidence, and emits one diagnostic naming the family prefix. The shape decides +rather than a vendor name, so any stack with this layout is covered without a +code change, and every placement after the token is covered rather than an +enumerated few. A token in trailing position is _not_ this case — everything +before it still identifies the element — and is left to normalization and the +collision check. + +## Safety and Output + +The generator prefers omission over a configuration that cannot match future +renders. For an observed desktop crawl of a site with this mix, replacement +output should therefore contain the stable `ad-header-0` and `ad-fixed_bottom-0` +slots, while the in-content collision group and the volatile-token family are +explained in notes. + +## Tests + +- A two-element same-page normalization collision yields no slots and one + diagnostic containing the prefix, both unsafe alternatives, and operator + action. +- Two renders of one element (identical residues either side of the marker, + including a React server/client pair) collapse to one slot with no diagnostic. +- Repeats of the first and second IDs plus a third distinct ID after a collision + remain suppressed and do not create additional diagnostics. +- Request-derived collisions follow the same policy. +- An ambiguous registry stem still suppresses request fallback, and network-ID + discovery survives when every collided slot is omitted. +- A stem refused on one page stays refused after a later page contributes a + single member of the group. +- A collision-only page is recorded as having evidence rather than as an empty + challenge page. +- Single registry- and request-derived render-token observations are omitted + while retaining evidence and any parseable network ID, for every placement + suffix after the token. +- IDs with no render token, with a bare digit run, or with a trailing token stay + eligible. +- Existing normalization, request fallback, fragment detection, and full CLI + tests remain green. diff --git a/docs/superpowers/specs/2026-08-21-pr-823-round-5-review-resolution-design.md b/docs/superpowers/specs/2026-08-21-pr-823-round-5-review-resolution-design.md new file mode 100644 index 000000000..8962f4862 --- /dev/null +++ b/docs/superpowers/specs/2026-08-21-pr-823-round-5-review-resolution-design.md @@ -0,0 +1,98 @@ +# PR 823 Round-5 Review Resolution + +## Goal + +Resolve review `4989897698` on PR 823 without weakening the generator's safety +rules, silently changing existing CLI defaults, or expanding the change beyond +the audit CLI and its documentation. + +## Browser and CLI Compatibility + +The hidden `ts audit ` compatibility form keeps accepting the same browser +flags as `ts audit generate `, but those flags must remain hidden and must +require the legacy URL positional. A dedicated `LegacyBrowserOpts` mirrors the +seven generation browser fields and converts into `GenerateBrowserOpts` when the +legacy command is dispatched. Consequently, flags placed before a real audit +subcommand are rejected instead of parsed and ignored. + +Generation retains its established 750 ms quiet period and 12-second maximum +settle wait. Generation defaults have one source of truth shared by clap, +`GenerateBrowserOpts::default`, and `BrowserAuditCollector::default`; applying +parsed options must not silently shorten the collector's maximum. The generic +page/verification collector keeps its existing independent 10-second default. + +Redirect notes show the origin and path for both requested and final URLs. This +makes scheme and host changes visible without exposing URL userinfo, queries, or +fragments. + +## Root-Less Template Safety + +Template inference records which slot stems borrowed the config-level +`section_root` because those slots were never witnessed on a path without the +configured section segment. Such a template is safe only while its page patterns +are derived from the paths where the slot was observed. + +Operator-supplied `--page-pattern` values replace those derived patterns for +every slot. If inference contains any borrowed-root slot and explicit patterns +were supplied, generation fails before rendering or writing a candidate config. +The error identifies the affected slots, explains that explicit patterns cannot +prove the borrowed-root invariant, and directs the operator to remove +`--page-pattern`. Failing the command is preferable to silently omitting real +inventory or attempting an unsound glob intersection. + +When no config-level section policy can be inferred because every otherwise +templatable slot lacks a root witness, each affected slot's refusal reason names +that crawl gap rather than claiming that its paths failed to generalize. + +## Merge Policy + +An explicitly configured `section_segment` is operator intent even when +`section_root` is currently unset. If preserved `{section}` slots exist and an +inferred policy would change that configured segment, merge fails and requires +`--replace` for the migration. If the configured segment matches, or is unset, +the inferred `section_root` may be adopted so the previously incomplete config +becomes loadable. + +## Diagnostics and Early Validation + +Warnings produced while folding a collected page include the device-profile +label as well as the path. Identical warnings from desktop and mobile therefore +remain distinguishable. The consent-stub warning remains a single unscoped +run-level note, and site-wide discovery warnings remain deduplicated. + +The existing config is parsed as TOML before Chrome starts. A whole-document +syntax error is returned immediately; a valid document with settings unknown to +the CLI still permits extraction of `[creative_opportunities]`; and a present +but unreadable creative section remains an error. + +The volatile div-id token recognizer requires at least ten leading digits plus +an alphanumeric suffix. This continues to recognize timestamp-like generated +tokens while preventing an eight-digit calendar date followed by a stable +letter from causing a single-observation family refusal. + +## Consistency Corrections + +Tests pin the Rust evidence cap to the embedded JavaScript collector constant. +The terminal-escaping test claims only controls it can actually inject; URL's +own percent-encoding is covered by an exact final-URL assertion rather than +presented as evidence for terminal escaping. Existing code escaping the final +URL remains as defense in depth. + +The affected guide, prior volatile-collision spec and plan, documentation +comments, `expect` message, and method spacing are corrected to describe the +implemented behavior exactly. The root-less templating behavior and this review +resolution are documented by this design and its paired implementation plan. + +## Testing and Delivery + +Every behavioral correction starts with a focused regression test that fails on +the current branch. Tests cover hidden legacy flags, the 12-second generation +default, complete redirect notes, borrowed-root rejection with explicit +patterns, configured-segment preservation, profile-specific warnings, +whole-document TOML failure, the evidence-cap invariant, and the calendar-date +token control. + +After focused tests pass, verification runs the host-target CLI suite and +audit/generate tests, CLI clippy with warnings denied, Rust formatting, docs +formatting, and `git diff --check`. No GitHub replies or push are part of this +change unless separately requested. diff --git a/docs/superpowers/specs/2026-08-24-ad-template-div-id-reconciliation-design.md b/docs/superpowers/specs/2026-08-24-ad-template-div-id-reconciliation-design.md new file mode 100644 index 000000000..9290e6e5c --- /dev/null +++ b/docs/superpowers/specs/2026-08-24-ad-template-div-id-reconciliation-design.md @@ -0,0 +1,118 @@ +# Ad-template div-ID reconciliation design + +## Goal + +Prevent `ts audit ad-templates generate` from losing numeric sibling creative +opportunities during merge or persisting a singleton div ID whose middle token +is demonstrably per-render. + +This follows a live validation crawl. The crawl observed +`ad-sidebar-1`, `ad-sidebar-10`, and other siblings, but the merge treated the +configured literal `ad-sidebar-1` as a prefix and absorbed the longer IDs. It +also proposed one `vendor-tag_12345678AbCdEfGhIjKl_slot_overlay_1`-shaped slot +because the volatile-token classifier recognizes ten leading digits but this +token has eight. + +## Scope + +The change is limited to div-ID identity and volatility classification during +generation: + +- Preserve every distinct normalized, usable div identity retained by the + evidence table when an existing configured div ID was itself observed + exactly. +- Preserve intentional configured prefix behavior when that prefix was not + observed as a literal element ID. +- Refuse singleton IDs with a conservative eight-digit-plus-long-suffix token + shape in a non-trailing segment. +- Keep the existing warning and refusal behavior for ambiguous and fragmented + placements. + +This does not implement the broader cross-page-type preservation requested by +GitHub issue #1059, change crawl planning, or change runtime slot resolution. + +## Exact versus prefix reconciliation + +The generator already carries the div identities from `EvidenceTable::slots()` +into the TOML merge. This is intentionally not collector-level raw DOM input: +the identities have passed per-page normalization and usability checks, while +slots later rejected by template inference or cross-page fragmentation remain +present. Page-local volatile and ambiguous identities already refused by GPT +discovery do not re-enter reconciliation. + +The merge will classify a configured or newly appended slot as an observed +literal when its resolved div identity appears exactly in that normalized +evidence set. + +Matching proceeds in this order: + +1. Prefer an exact stable-key match. +2. Otherwise consider configured-prefix matches whose prefix was not observed + as a literal normalized div identity during this crawl. +3. Choose the longest remaining prefix, retaining configuration order for + equal-length ties. +4. Append the discovered slot when neither exact nor eligible prefix matching + succeeds. + +Consequently, `ad-sidebar-1` matches itself but cannot claim +`ad-sidebar-10`. A hand-authored broad prefix such as `ad-`, absent as a literal +DOM ID, retains its existing merge behavior. Newly appended discovered slots +are also protected because the decision is based on the normalized evidence +set, not only the original configuration indexes. + +The same reconciliation rules will drive observed/unobserved diagnostics so a +slot cannot be merged one way and classified for staleness another way. + +## Volatile token classification + +The existing vendor-neutral classifier refuses a div ID when a non-trailing +segment contains a per-render token before the placement suffix. It currently +recognizes a segment with at least ten leading digits followed by alphanumerics. + +Retain that rule and add a narrower alternative for shorter counters: + +- at least eight leading ASCII digits; and +- at least eight trailing ASCII alphanumeric characters in the same segment. + +The token must still occur before another div-ID segment. This catches the +`12345678AbCdEfGhIjKl` shape without claiming: + +- bare numeric placement IDs; +- seven-digit counters with long suffixes; +- eight-digit values with fewer than eight trailing characters, including + calendar-like `20260820a`; or +- trailing tokens whose preceding prefix can still identify the element. + +The warning remains vendor-neutral and names the stable family prefix. The slot +continues to count as evidence of an ad stack but is not rendered into config. + +## Diagnostics and failure behavior + +No new command failure is introduced. Unsafe singleton volatile slots are +skipped with the existing volatile-family note. Literal numeric siblings are +written separately and no longer produce the broad-prefix collision note. +Truly intentional broad prefixes can still produce that note when they claim +multiple observed divs. + +Normal merge continues to preserve configured slots. `--replace` retains its +existing replacement semantics. + +## Testing + +Use test-driven development with focused regressions: + +- A merge containing configured `ad-sidebar-1` and normalized observations for + `ad-sidebar-1`, `ad-sidebar-10`, and `ad-sidebar-11` must produce three slots. +- A configured `ad-` prefix that was not observed literally must continue to + merge multiple matching discovered divs and emit its collision note. +- Newly appended observed literals must not absorb later numeric siblings. +- A framework-bearing DOM ID normalized to a stable stem must classify the + matching configured stem as literal; identities refused during per-page GPT + discovery must not be reintroduced solely for merge classification. +- Registry and request evidence containing a singleton shorter high-entropy token + must be refused with the volatile-family warning. +- Boundary tests cover seven leading digits, eight digits with a seven-character + suffix, eight digits with an eight-character suffix, bare digits, and the + existing calendar-shaped example. +- Run the complete CLI suite, including the real-Chrome scrolling fixture, plus + formatting and the repository's target-specific verification gates. diff --git a/docs/superpowers/specs/2026-08-24-ad-template-generate-scroll-staleness-design.md b/docs/superpowers/specs/2026-08-24-ad-template-generate-scroll-staleness-design.md new file mode 100644 index 000000000..8709f6330 --- /dev/null +++ b/docs/superpowers/specs/2026-08-24-ad-template-generate-scroll-staleness-design.md @@ -0,0 +1,98 @@ +# Ad-template generation scroll and staleness diagnostics design + +## Problem + +`ts audit ad-templates generate` currently collects each page only after its +initial settle. Unlike `ts audit page` and `ts audit ad-templates verify`, it +cannot request the deterministic scroll pass that triggers lazy ad inventory. +On a lazy-loading publisher site this produced fewer observable frames than a +scrolled page audit of the same page. + +Generation also merges by default, deliberately preserving configured slots +that the current crawl did not rediscover. That safety behavior is correct, but +it is silent: stale slots look as though the latest crawl confirmed them. + +## Scope + +Add opt-in scrolling to `ts audit ad-templates generate` and report configured +slots that a merge preserved without observing during the current crawl. + +This change does not prune slots automatically, enable scrolling by default, +alter crawl planning or budgets, change volatile-div refusal, or implement +GitHub issue #1059. `--replace` remains the only intentional pruning mode. + +## Command behavior + +`ts audit ad-templates generate` accepts a boolean `--scroll` option. Its +default is false, preserving current crawl cost and side effects. When enabled, +every page on every selected device profile performs the same deterministic +stepped scroll used by the existing page audit: scroll to 33%, 66%, and 100% of +the document, pause between steps, return to the top, then wait for the page to +settle again before reading HTML, GPT registry entries, and network evidence. + +The browser collector carries the option as session configuration so root, +planned section, desktop, and mobile page loads all behave consistently. Scroll +evaluation failures are best-effort page warnings; they do not discard evidence +that was already available after the initial settle. + +The implementation will share the deterministic scroll primitive with the +existing browser audit rather than maintain a second sequence of scroll steps. +Verifier-only evidence-phase bookkeeping remains in the verifier call path. + +## Merge diagnostics + +During a normal merge, generation tracks which pre-existing configured slots +matched at least one discovered slot. After processing all discovered slots, it +reports every unmatched pre-existing slot in configuration order. Those slots +remain unchanged in the output. + +The diagnostic is explicit about the limits of negative crawl evidence. Its +human-readable form for a non-scrolling run is equivalent to: + +```text +note: preserved 2 configured slot(s) not observed during this crawl: ad-header-0, ad-fixed_bottom-0. Re-run with broader coverage or --scroll; `--replace` prunes them but also discards every hand-written field on the slots the run did rediscover. +``` + +When the current run already used `--scroll`, the follow-up omits that redundant +suggestion and recommends broader page/profile coverage before intentional +pruning. + +No staleness diagnostic is emitted when all configured slots were rediscovered, +when there were no existing slots, or under `--replace`, because that mode does +not preserve unmatched slots. Matching uses the same reconciliation logic as +the merge itself, avoiding a second definition of slot identity. + +Diagnostics go to stderr through the existing generation-note path. Stdout +remains limited to the dry-run diff or successful write summary, so redirection +and machine comparison remain stable. + +## Safety and compatibility + +The default command behavior, merge result, and generated TOML remain unchanged +unless `--scroll` discovers additional evidence. The warning never mutates or +deletes operator configuration. It names only configured slot IDs and does not +include cookies, URL credentials, query strings, or fragments. + +Scrolling can trigger additional ad requests and publisher behavior, which is +why it remains explicit. Existing page-delay, settle-window, browser-proxy, +certificate, cookie, and device-profile behavior applies unchanged. + +## Tests + +CLI parsing tests cover `--scroll` and its false default. Browser-collector tests +use a deterministic local page that defines a GPT slot only after scrolling and +prove that generation captures it with the option enabled but not without it. +Existing browser lifecycle and settle tests continue to cover teardown and +timeouts. + +Merge unit tests cover multiple unmatched configured slots, stable diagnostic +ordering, partial rediscovery, full rediscovery, an empty existing config, and +`--replace`. Command-level tests verify that the warning reaches stderr while +stdout and the preserved generated configuration retain their existing +contracts. + +Verification will run the host CLI test suite and relevant Chrome-backed CLI +tests, followed by the repository-required formatting and CLI lint gates. A +manual dry run against a live publisher site may be used when a fresh +bot-protection cookie and proxy are available, but network-dependent behavior +is not a required CI test. diff --git a/docs/superpowers/specs/2026-08-24-request-phase-timing-design.md b/docs/superpowers/specs/2026-08-24-request-phase-timing-design.md new file mode 100644 index 000000000..e3d9d6e50 --- /dev/null +++ b/docs/superpowers/specs/2026-08-24-request-phase-timing-design.md @@ -0,0 +1,673 @@ +# Request phase timing: Server-Timing subtimings and access telemetry + +**Date:** 2026-08-24 +**Status:** Approved design, revised for review rounds 1 and 2, pending implementation +plan. +**Scope:** `trusted-server-core`, Fastly and Axum adapters, `tinybird/` schema, +performance dashboard (separate repo). + +--- + +## 1. Problem + +On 2026-08-21 a production deployment (publisher redacted, `prospect-a.example`) showed +an episodic stall: for a window of roughly 40 minutes, every request that reached the +application path carried a uniform extra ~600 ms of Fastly `time-elapsed`, and then +recovered to 20-50 ms with no deploy or config change we could observe. `/health` +(2-4 ms, short-circuits before app construction) and `/_ts/debug/ja4` (6-9 ms, settings +load only) stayed fast throughout, so the stall lived between app construction and +response send. + +Attributing that window required a live probing session: route-by-route bisection, +cookie-deletion experiments, and an eight-agent code trace. The trace found no +unconditional await on the path that could cost 570 ms, and exactly two +config-conditional candidates (the pre-route request filter's synchronous verification +POST, and EC identity KV writes before send), plus one dependency shared by every +application route (two geo hostcalls per request). We could not tell which one stalled, +because nothing in the response says where server time went. + +The Compute CPU budget is ~50 ms per request, so a large `time-elapsed` strongly +suggests wall-clock time outside active guest CPU: dependency awaits are the leading +explanation, with platform scheduling and hostcall queueing as the residual ones. The +comparison figure here is the fronting delivery layer's `time-elapsed` Server-Timing +entry, observed at its deliver phase. Either way, these are exactly the numbers a +response can carry about itself. + +## 2. Goals + +1. Every normal application response attributes its own server time by phase in a + standard header. Browsers expose the values to same-origin JavaScript via + `PerformanceResourceTiming.serverTiming`, so RUM tooling that reads that API can + surface the breakdown. Whether a given vendor or the publisher's own monitoring + extension actually collects it is verified separately in rollout; the publisher + extension needs a small change to render it. +2. The same numbers flow to Tinybird so we hold p50/p95/p99 per phase, per route class, + per PoP, per deployed version, and a future stall window self-diagnoses in one query. +3. No additional awaited I/O before first byte. The pre-send cost is a handful of + monotonic clock reads, one small allocation at entry, and rendering one header; + telemetry emission happens strictly after the last body byte. + +Scope note: phases cover the application lifecycle after T0. The `/health` and +`/_ts/debug/ja4` short-circuits, config-store open failures, and request-conversion +failures bypass the lifecycle and emit nothing. Requests served entirely by the +fronting cache never reach the guest and produce neither header entries nor rows. + +## 3. Non-goals + +- No trailer-based Server-Timing for body-phase spans (browsers do not expose trailer + values to JavaScript). +- No per-filter naming in any emitted surface. The request-filter span is `ts-filter` + regardless of which filter runs; vendor identity stays out of headers and telemetry. +- No Cloudflare or Spin emission wiring in v1. Core collection is adapter-neutral; those + adapters can wire emission later without core changes. +- No Tinybird endpoint pipe and no rollup materialized views in v1. Grafana queries the + datasource through the ClickHouse connector, matching the auction dashboards; rollups + only if panel latency demands them. +- No sampling of the header. The header is all-traffic when enabled; only Tinybird rows + sample. +- No cross-request circuit breaker for telemetry emission. Compute runs one isolate per + request; there is no shared mutable state to hold breaker state. The controls are the + bounded per-request cost and the `access_sample_rate` lever (section 10). + +## 4. Design overview + +``` +adapter entry (T0) + | RequestTimings::new() -> shared handle + v +app construction ................ ts-appbuild (adapter) +pre-route request filters ....... ts-filter (adapter wrapper) +geo lookup (single, deduped) .... ts-geo (adapter; result carried forward) +template cache lookup ........... ts-template-cache (core: publisher.rs) +origin fetch to resp headers .... ts-origin (core: publisher.rs) +EC identity KV, pre-send ........ ts-kv (core: KV abstraction) +auction wait, buffered mode ..... auction_wait_ms (row only; pre-header in this mode) + | +send_edgezero_response, immediately before into_parts(): + mark_headers_ready() snapshot (unconditional) + build AccessTelemetrySnapshot (unconditional) + append Server-Timing header (flag-gated, only on conclusively private responses) + | +headers committed; body streams + auction hold at seam .......... auction_wait_ms (row only; in-stream in this mode) + stream duration, bytes ........ stream_ms, resp_bytes (row only) + | +post-send (adapter main): + request_elapsed snapshot, then existing pull-sync, then: + sample gate -> one NDJSON row -> Tinybird Events API + bounded response await, 2xx validated +``` + +Collection is always-on and flag-free, including the `mark_headers_ready()` snapshot. +Two independent flags gate emission: the header (`observability.server_timing_enabled`) +and the telemetry row (`tinybird.access_enabled`). + +## 5. `RequestTimings` (core) + +New module `crates/trusted-server-core/src/request_timing.rs`. + +- `Phase`: a closed enum: `AppBuild`, `Filter`, `Geo`, `EcKv`, `Origin`, + `TemplateCacheLookup`, `AuctionWait`, `Stream`. Header rendering covers the first six + plus the stored total; the last two are row-only. +- Inner state: one fixed-size array of `Option` slots indexed by phase, + `t0: Instant`, `headers_ready_total: Option`, + `auction_wait_placement: Option` (`PreHeader` or `InStream`), + and `resp_bytes: Option`. Phases that repeat within a request (geo, KV) + accumulate by saturating addition into the same slot. +- `mark_headers_ready()`: stores `t0.elapsed()` once at the response-commit boundary, + unconditionally, before either emission flag is consulted. The header renders this + stored value as `ts-total`; the telemetry row reads the same stored value as + `time_elapsed_ms`. The two surfaces cannot disagree, and the row stays correct when + the header flag is off. Full request duration is captured separately as + `request_elapsed_ms`, snapshotted immediately after the body-stream drive returns and + before any other post-send work, so pull-sync and telemetry emission are never + included in it. +- Sharing: `RequestTimings` is a cheap-clone handle, `Arc>`. It crosses + three boundaries: adapter entry to core handlers, the streaming body closure (records + body-phase spans after the response object has been handed off), and the adapter's + post-send emission read. Access is exclusively `try_lock()`: a contended or + poisoned lock drops the sample immediately rather than waiting, so recording can + never delay a request. +- Recording API: `timings.record(Phase::Geo, dur)` and a scope guard + `timings.span(Phase::Origin)` that records on drop. Guards use saturating duration + math; a non-monotonic reading records zero rather than panicking. The auction-wait + recorder takes the placement explicitly so the two modes cannot be conflated. +- Rendering: `server_timing_value(&self) -> Option` produces + `ts-total;dur=41.2, ts-appbuild;dur=18.4, ts-filter;dur=9.1` with durations in + milliseconds at one decimal. Phases never recorded are omitted. Returns `None` when + `mark_headers_ready()` has not run. + +`Instant` is already used freely in the guest (`publisher.rs`, `auction/telemetry.rs`), +so no new clock abstraction is needed. + +## 6. Span taxonomy and recording sites + +| Entry | Measures | Site | +| ------------------- | ------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------- | +| `ts-total` | T0 to `mark_headers_ready()` at the response-commit boundary | stored snapshot | +| `ts-appbuild` | config-store open, Settings parse, orchestrator + registry + router build | Fastly `main.rs` around `open_trusted_server_config_store()` + `build_app_with_state()` | +| `ts-filter` | pre-route request filters, end to end (backend ensure, secret read, POST) | around `run_pre_route_filters` (`app.rs:751`) | +| `ts-geo` | geo hostcall (single after dedupe; accumulates if any path still repeats) | `build_ec_request_state` (`app.rs:410`) and timed finalizer fallback lookups | +| `ts-kv` | EC identity KV operations before response send (see enumeration below) | the shared KV abstraction | +| `ts-origin` | publisher backend send to response headers available (read-through cache hit or miss) | `publisher.rs` around the origin `send` | +| `ts-template-cache` | template cache `lookup_or_reserve`, including the hit-path full-body read | `publisher.rs` around the lookup | + +Naming follows the completed template-cache terminology migration (`x-ts-template-cache` +is the emitted header on `main`; `c2` naming is retired). + +`ts-kv` is instrumented by a timing decorator implementing `PlatformKvStore` that +wraps the store handed to request-scoped consumers, because no single existing +abstraction covers the taxonomy: EC graph operations go through `KvIdentityGraph` +while consent persistence uses `PlatformKvStore` directly, and graphs are constructed +independently in request setup, identify, admin lookup, batch sync, and finalization. +Every request-path graph construction receives the timed store; pull-sync explicitly +constructs its graph from an untimed store. Consent-store reads pass through the same +decorator and are timed like any other store call. Included pre-send operations: EC +generation `create_or_revive`, identify-path graph reads and evaluation, finalize-path +`ingest_eid_cookies`/`upsert_partner_ids` and withdrawal tombstones, consent-store +reads on consent routes, and batch-sync graph access when it runs before send. +Explicitly excluded: pull-sync work, which runs strictly after `send_to_client` and is +invisible to both surfaces. The decorator measures store-call latency only: no value +passing through it is read, parsed, or recorded, and the emitted surfaces carry no +consent or identity payloads. This feature therefore needs no consent gate; it is the +site measuring its own infrastructure, not processing user data. The timings handle +reaches `ec_finalize_response` inside the graph it already receives; that function +keeps the repository maximum of seven arguments and does not gain an eighth. + +Row-only fields: + +| Field | Measures | Site | +| -------------------- | --------------------------------------------------------- | ------------------------------------------------ | +| `auction_wait_ms` | wait on the dispatched auction (placement varies by mode) | seam hold (streaming) or buffered finalizer wait | +| `body_mode` | `streamed` or `buffered` response assembly | set where the response body is built | +| `stream_ms` | headers committed to last body byte | adapter around the body-stream drive | +| `resp_bytes` | bytes written to the client body | same | +| `request_elapsed_ms` | T0 to immediately after the body-stream drive returns | post-send snapshot, before pull-sync | + +Auction-wait placement is not universal. On the ordinary streaming path the wait +happens at the `` seam inside the body stream and nests inside `stream_ms`. On +buffered paths (the Fastly shared-template authorized miss, which buffers the full +transform and auction before returning a response, and every Axum response) the wait +completes before headers commit. The row therefore carries `body_mode` plus +`auction_wait_placement` (`pre_header` or `in_stream`), and derivations are +conditional: + +- `in_stream`: `stream_other_ms = greatest(coalesce(stream_ms, 0) - coalesce(auction_wait_ms, 0), 0)`. +- `pre_header`: `auction_wait_ms` joins the pre-header phase set, and `stream_other_ms = coalesce(stream_ms, 0)`. + +`unattributed_ms = greatest(coalesce(time_elapsed_ms, 0) - (coalesce(appbuild_ms, 0) + +coalesce(filter_ms, 0) + coalesce(geo_ms, 0) + coalesce(kv_ms, 0) + +coalesce(origin_ms, 0) + coalesce(template_cache_ms, 0) + pre-header auction wait), 0)`. +Every phase column is nullable, so every query-time formula wraps each term in +`coalesce(column, 0)` and every subtraction in `greatest(..., 0)`; query tests cover +sparse phase combinations. + +## 7. Freeze point and header emission + +The freeze-and-emit point is `send_edgezero_response` (Fastly `main.rs`), immediately +before `response.into_parts()`. This is the single choke point every send path shares, +and it runs after everything that can still mutate the response: the router middleware, +entry-point finalize (`apply_finalize_headers`, asset-policy reapplication), EC +finalization and its KV work, and terminal filter/privacy effects. +`apply_finalize_headers` itself does not emit; the `HEADER_X_TS_FINALIZED` sentinel +marks middleware finalization, not header commitment, and must not be treated as the +timing boundary. + +At the freeze point, in order: `mark_headers_ready()` (unconditional), the +`AccessTelemetrySnapshot` build (unconditional, section 10), then, gated on +`observability.server_timing_enabled`, append one `Server-Timing` header from +`server_timing_value()`. Append semantics, never insert: an origin-supplied +Server-Timing survives, and the fronting delivery layer's own entries (`time-elapsed`, +`hit-state`) are additive per the header's list semantics. + +Header emission is conservative: it happens only when the response is conclusively +non-storable by any shared cache, meaning `Cache-Control` contains `private` or +`no-store` (the existing `cache_control_headers_are_private_or_no_store` predicate). +Anything else, including bare `max-age`, `s-maxage` without `private`, +heuristically-cacheable responses with no cache header at all, and anything a fronting +cache override might store, emits no header, because a stored object would replay one +request's timings for its full lifetime. The long-lived immutable `tsjs` asset route +is the concrete excluded case. The snapshot and the telemetry row are unaffected by +this skip, so excluded routes still report through Tinybird. + +The Axum adapter applies the same emission rule at its terminal point before response +serialization, with adapter-specific phase semantics (section 8a). + +## 8. Geo lookup dedupe (rider) + +Today every dispatched request pays two geo hostcalls for one answer: request-phase in +`build_ec_request_state` (`app.rs:410`) and response-phase in +`FinalizeResponseMiddleware` (`middleware.rs:83`, alternate site `main.rs:285`). + +Plain request extensions cannot carry the result out: the middleware moves the request +context into `next.run(ctx)` and holds only the response afterward. The resolved geo +travels on a dedicated `GeoLookupState` response extension, attached on every exit +path that attempted a lookup, including the asset fallback, which runs +`build_ec_request_state` and then returns without `EcFinalizeState` (which is why +`EcFinalizeState` is not an acceptable carrier). States: `NotAttempted`, +`Attempted(None)` (lookup ran and failed, do not retry), and `Resolved(GeoInfo)`. The +finalize path consumes the carried value and performs a live lookup only in the +`NotAttempted` state; those legitimate fallback lookups (admin, batch, error paths) +are themselves timed into `ts-geo` so degraded geo cannot hide inside +`unattributed_ms`. The 401 rule (`resolve_geo_for_response` skips lookup for +unauthorized responses) is preserved. + +## 8a. Adapter phase semantics + +The Fastly adapter is the reference implementation of the taxonomy. Axum differs +structurally and its emissions are defined accordingly rather than pretending parity: + +- `ts-appbuild` is absent: Axum builds application state once at startup. +- `body_mode` is always `buffered`: the Axum HTTP client buffers upstream bodies, so + `stream_ms` measures buffered-body write-out and `auction_wait_placement` is always + `pre_header`. +- The freeze point is an outer service wrapper around the `RouterService` inside + `AxumDevServer`, not router middleware: router-generated 404/405 responses bypass + router middleware, and middleware returns before Axum serializes the body. The + wrapper sees every response including router-generated ones; `/health` is excluded + by path match inside the wrapper. +- Axum emits the header only; no Tinybird rows in v1 (unchanged). + +Cloudflare and Spin: collection compiles, no emission wiring in v1 (unchanged). + +## 9. Access telemetry row + +Extends the reserved `tinybird/datasources/access_logs_raw.datasource`. + +Kept columns: `event_ts`, `method`, `status`, `time_elapsed_ms` (defined as the +`mark_headers_ready()` snapshot; nullable because a contended lock drop can lose the +snapshot), `sample_rate`, 30-day TTL. (`event_date` was later dropped for the +`toDate(event_ts)` sorting-key expression; see section 9's schema note.) + +Removed: raw `path`. Route identifiers like `/_ts/admin/ec/{id}` would otherwise put +EC identifiers into a 30-day dataset, and publisher paths carry unbounded cardinality +and user-generated content (search terms, usernames, emails in slugs). Replaced by +`route_template`: + +- Named routes: the matched route-table pattern verbatim, parameters left as + placeholders. +- Publisher fallback: a coarse fixed template, `/` plus the first path segment + restricted to a bounded allowlisted charset, plus `/*` when deeper (for example + `/news/*`). The auction-telemetry normalizer is explicitly not sufficient here: it + redacts long tokens but preserves short identifiers and arbitrary slugs. +- Rejection is whole-segment, never truncation: a segment is dropped to `/other/*` + when it fails the charset allowlist, exceeds 32 characters, carries more than 7 + ASCII digits, or is the only segment in the path. Depth is what makes a first + segment a section name: single-segment paths are documents (WordPress + `/%postname%/` puts every article at depth 1), so they reject wholesale, root + landing pages included. The character allowlist alone does not bound identity (`[a-z0-9_-]` + is exactly the alphabet of UUIDs, hex ids, and reset tokens), and a truncated + prefix of any of those is still identifying, so the length and digit bounds reject + the segment outright. +- Tests are adversarial, not just the happy path: a literal EC identifier on the admin + route, an email address in a path segment, search-term-shaped segments, overlong + segments, UUIDs, hex ids, reset tokens, and full article slugs must all normalize + to bounded, content-free templates. + +Added columns (all dimension columns non-nullable with an `unknown` sentinel, because +ClickHouse sorting keys cannot contain nullable columns): + +``` +`service_id` LowCardinality(String), -- FASTLY_SERVICE_ID; immutable deployment identity +`publisher_domain` LowCardinality(String), -- matches auction schema +`env` LowCardinality(String), -- adapter-derived: production | staging | unknown +`route_class` LowCardinality(String), -- publisher_html | tsjs | integration_proxy | ec | auction_api | other +`route_template` String, -- bounded, normalized; replaces path +`body_mode` LowCardinality(String), -- streamed | buffered +`auction_wait_placement` LowCardinality(String), -- pre_header | in_stream | none +`appbuild_ms` Nullable(UInt32), +`filter_ms` Nullable(UInt32), +`geo_ms` Nullable(UInt32), +`kv_ms` Nullable(UInt32), +`origin_ms` Nullable(UInt32), +`template_cache_ms` Nullable(UInt32), +`auction_wait_ms` Nullable(UInt32), +`stream_ms` Nullable(UInt32), +`request_elapsed_ms` Nullable(UInt32), +`resp_bytes` Nullable(UInt64), +`template_cache_state` LowCardinality(String), -- from the typed response extension, not the public header +`country` LowCardinality(String), +`ts_version` LowCardinality(String), +`pop` LowCardinality(String) -- FASTLY_POP, 'unknown' when absent +``` + +The matched route pattern does not survive dispatch today, so a typed +`RouteMetadata` response extension carries `route_class` and `route_template`: each +named-route handler wrapper attaches its route-table pattern verbatim (handlers +serving multiple patterns attach the one that matched), and the fallback and tsjs +handlers attach their class plus the coarse template. The freeze point consumes the +extension; nothing reconstructs routes from a handler enum or path regex. + +Typed sources only: `env` is adapter-owned, derived from the same Fastly +`FASTLY_IS_STAGING` input that drives `x-ts-env` (`Settings` has no environment +field and does not gain one). `template_cache_state` comes from a typed response +extension, not the `x-ts-template-cache` header (operator-configured response +headers can override managed headers): the currently private +`TemplateCacheResponseState` in `publisher.rs` becomes a typed response extension, +and every state transition sets the managed header and the extension together so +the two can never drift. `service_id` and `pop` come from the Fastly environment. `cache_state` +from the reserved schema is dropped: the guest cannot observe the fronting cache, and +guest-visible cache behavior is already carried by `template_cache_state` and +`origin_ms`. Rows exist only for guest-handled requests; fronting-cache hits are +invisible by construction and the dashboard documentation says so. + +Sorting key: `(toDate(event_ts), service_id, publisher_domain, env, route_class, +pop, status)`. Every column carries a `json:$.` path (the Events API rejects +NDJSON into a datasource without JSONPaths, discovered live); `event_date` was +dropped in favor of the sorting-key expression because a DEFAULT column cannot +carry a JSONPath the producer never sends. Grafana time filtering uses `$__timeFilter(event_ts)` and every panel query +also carries a `toDate(event_ts)` predicate so the primary index prunes; rollout validates +the panel queries with `EXPLAIN` before the dashboard is committed. This replaces the +reserved key `(event_date, path, status, method)`. Rollout step 4 verifies whether the +reserved datasource was ever deployed to the remote workspace; if it was, this schema +ships as a versioned replacement datasource with a cutover, not an in-place edit. + +## 10. Emission mechanics + +- `AccessTelemetrySnapshot`: built unconditionally at the freeze point, before + `into_parts()` consumes the response. It captures method, status, route metadata + (from the `RouteMetadata` extension), and typed dimension states (`env`, + `template_cache_state`, geo country). It exists because nothing else survives to + post-send on every path: the request is consumed by dispatch, the response by + `into_parts()`, and `EcFinalizeState` is absent on asset, admin, and error paths. +- The emitter's transport context is adapter-owned and route-independent: the + Events API target (backend spec, secret store name, dataset, token secret, sample + rate) derives from settings once at entry in `main.rs`, and the HTTP client is the + adapter's stateless platform client. Asset, admin, and error responses therefore + emit without `RuntimeServices` or `EcFinalizeState`. +- `send_edgezero_response` returns a delivery outcome instead of `()`, with + per-mode semantics because the two body paths observe different things. Streamed + bodies: a counting writer reports bytes written and distinguishes complete, + partial (truncated), and error outcomes. Buffered bodies: `send_to_client()` + returns no delivery result, so the byte count is captured from the body length + before the send and the outcome is complete-on-return with no partial detection; + `body_mode` in the row keeps the two regimes distinguishable in analysis. +- Ordering after the body-stream drive returns: snapshot `request_elapsed_ms` first, + run the existing pull-sync dispatch unchanged, then telemetry emission last, so + pull-sync is never delayed behind the ingest await and never included in + `request_elapsed_ms`. +- Sampling: uniform per-request decision against `tinybird.access_sample_rate`. No + client stickiness. Sampled-out requests are silent; every other drop (row build + failure, send failure, non-2xx) logs one warning naming the reason. There is no + cross-request warning suppression (per-request isolates hold no shared state); the + overload controls are the 2 s bounded await, the single-warning-per-request cap, and + `access_sample_rate` pushed down by config as the operational abort lever. Ingest + health is monitored from the Tinybird side via ingestion freshness on the + datasource, which catches quarantine and schema rejection that per-request warnings + cannot. +- Transport: one NDJSON row to the Tinybird Events API: same `api_host`, reserved + `access_dataset` and `access_token_secret`, 2 s first-byte and between-bytes + timeouts, `max_body_bytes` guard, no retry. +- Delivery confirmation: unlike the auction sink, which starts `send_async` and drops + the pending response (it runs before delivery completes and cannot afford to wait), + the access emitter runs after the client has the full response and therefore awaits + the bounded ingest response and validates 2xx. A non-2xx or timeout logs a warning + with the status. +- Budget: at `access_sample_rate = 1.0` this adds one backend request per request to + the service, after delivery; during a Tinybird outage each such request holds its + sandbox for up to the bounded timeout. The sample rate is the budget control; 1.0 is + a diagnosis setting, not a steady state, and rollout treats sustained emission + warnings as the signal to dial it down. +- Axum adapter: emits the header only; no Tinybird rows in v1. + +## 11. Dashboard and query model + +No endpoint pipe in v1. Grafana queries `access_logs_raw` directly through the +ClickHouse connector with `$__timeFilter(event_ts)` plus a `toDate(event_ts)` +predicate, matching the auction dashboards. + +Dashboard: a new standalone `grafana/dashboards/edge-performance.json` in the +telemetry repo (`trusted-server-tinybird`), performance only, no panels shared with +the revenue and auction dashboards. Panels: + +- Phase percentiles (p50/p95/p99) by `route_class`, per phase column. +- Stacked phase breakdown over time using the non-overlapping set: `appbuild_ms`, + `filter_ms`, `geo_ms`, `kv_ms`, `origin_ms`, `template_cache_ms`, pre-header + auction wait (where `auction_wait_placement = 'pre_header'`), and derived + `unattributed_ms`. In-stream auction wait and derived `stream_other_ms` chart in a + separate body-phase panel and never stack with pre-header phases. +- PoP split, `ts_version` overlay, template-cache state rates. +- Stall panel: rows with `request_elapsed_ms > 500` (post-body total, so body-only + stalls are caught) grouped by dominant phase, where `unattributed_ms` competes as a + phase so the panel cannot confidently blame a small measured span while most time is + uninstrumented. + +All derivations use the `coalesce`/`greatest` forms from section 6; query tests cover +sparse phase combinations and both `auction_wait_placement` modes. + +Sampling semantics for every aggregate: `sample_rate` must be operationally stable +within any queried window. Quantile panels filter strictly to a single `sample_rate` +value. Volume panels weight each row by `1.0 / sample_rate` (the inverse-probability +estimator is `sum(1.0 / sample_rate)` over emitted rows; `count() / rate` is valid +only when the query is already filtered to one rate). Pooled unweighted quantiles +across a rate change are documented as invalid. + +## 12. Config surface + +```toml +[observability] +# Append TS phase timings to the Server-Timing response header. +server_timing_enabled = false # example default +``` + +New `ObservabilitySettings` struct with the single boolean, default off, standard +environment override (`TRUSTED_SERVER__OBSERVABILITY__SERVER_TIMING_ENABLED`). +Collection has no flag: the flags gate the two emission surfaces independently. + +Tinybird flag structure: `tinybird.enabled` today arms the auction sink by itself, so +"enable Tinybird for access telemetry" would silently enable auction emission too. The +master flag is demoted to transport-only (host, store, credentials), and each emitter +gets its own switch: a new `tinybird.auction_enabled` defaulting to `true` (preserving +current behavior for existing configs) and the reserved `tinybird.access_enabled` +defaulting to `false`. A settings test locks the decoupling in both directions. + +Validation when `access_enabled = true`: `tinybird.enabled`, non-empty `api_host`, +non-empty `secret_store`, `access_dataset`, and `access_token_secret`, a positive +`max_body_bytes`, and `access_sample_rate > 0`. An armed-but-silent configuration +(`access_enabled = true`, `access_sample_rate = 0`) is a configuration error, not a +valid state; disabling is done with the flag, not the rate. + +Rollback and compatibility, because `Settings` is `deny_unknown_fields`: + +- Deployment order is binary first, config second. Rollback order is config first + (remove the `[observability]` table and any new tinybird keys), binary second. A + config containing the new fields must never be pushed while a pre-observability + binary can still run. +- Config serialization omits the table when it equals the default, so round-tripping a + config through tooling does not inject a field an older binary rejects. A + compatibility test asserts the serialized default config parses under the previous + schema. +- The environment-variable overlay cannot create a missing leaf, so the key ships + present-but-false in the base operator TOML (the same pattern the GPT integration + documents in `trusted-server.example.toml`) and is flipped by config push. + +## 13. Error handling + +- Recording is infallible: saturating math, lock-failure drops the sample, no panics. +- Header rendering failure (defensive `HeaderValue::from_str` error) logs and skips + the header. +- Row emission failure logs one warning naming the reason and drops the row. The + response has already been delivered; there is nothing to degrade. + +## 14. Testing + +- Core unit tests: phase accumulation, saturating math, `mark_headers_ready()` + idempotence and both-surface consistency, render format (one decimal, omission of + unrecorded phases), row serialization shape, auction-wait placement recording. +- Adapter tests (Fastly via Viceroy, Axum native): header present and well-formed on a + conclusively-private publisher route with the flag on; absent with the flag off; + absent on the shared-cacheable tsjs route and on a bare `max-age` response with the + flag on; exactly one TS-owned metric set (a single `ts-total`) with every + pre-existing Server-Timing value preserved, across all send paths; `ts-kv` captures + EC finalize work (proving the freeze point sits after it). +- Body-mode tests: ordinary streaming (in-stream wait nested in `stream_ms`), + Fastly shared-template authorized miss (buffered, pre-header wait), and Axum + (always buffered), each asserting placement and non-negative derivations. +- Geo dedupe: finalize consumes `Resolved`; no retry on `Attempted(None)`; live + lookup only on `NotAttempted`; fallback lookups timed into `ts-geo`; asset-fallback + path carries `GeoLookupState` without `EcFinalizeState`; 401 skip preserved. +- Route template: adversarial normalization tests (literal EC identifier on the admin + route, email address in a segment, search-term segments, overlong segments) all + producing bounded content-free templates. +- Settings: the access validation matrix including the armed-but-silent rejection; + auction/access flag decoupling in both directions; the former rejection test becomes + the wiring test; the serialized-default-config compatibility test against the + previous schema. +- Sink tests: `RecordingHttpClient` pattern; assert URI, NDJSON body shape, token + header, 2xx validation and warning on non-2xx, skip when sampled out, ordering after + pull-sync. +- Query tests: derivation formulas against sparse rows and both placements. + +## 15. Rollout and verification + +1. Land collection + freeze point + header emission behind the flag, off everywhere. + Full CI gate. +2. Staging deploy with the flag on. Delivery-layer verification is two-sided: a + pass-through request confirming the appended Server-Timing survives the fronting + VCL, and a MISS-then-HIT replay against a cacheable route confirming no stale + timing header is ever served from cache. Fallback if the VCL clobbers the header: a + one-line VCL change on the delivery service, or mirroring the value to + `x-ts-timing` while that lands. +3. Production flag on. Confirm + `performance.getEntriesByType('navigation')[0].serverTiming` shows `ts-*` entries + in a real browser session, and separately confirm what the publisher's RUM tooling + actually collects; the publisher monitoring extension renders it only after a small + change on their side. +4. Verify whether `access_logs_raw` exists in the remote Tinybird workspace. If yes, + ship the schema as a versioned replacement with cutover; if no, edit in place. + Validate the dashboard panel queries with `EXPLAIN` against the sorting key. Then + land the row schema, sink, and settings changes; sample at 1.0 during stall + diagnosis with ingestion-freshness monitoring on the datasource; then the + dashboard. +5. Success criterion: the next stall window is attributable from one response header + or one dashboard query, with no live probing session. + +## 16. Overhead + +Roughly ten monotonic clock reads, two stored snapshots, and one ~130-byte header per +request; one sampled HTTP POST with a bounded await after the response has fully +streamed. No allocation in the hot path beyond the one `Arc` at entry, the +`AccessTelemetrySnapshot` at the freeze point, and the rendered header string. + +## 17. Decisions and open questions + +- **Public exposure is a decision, not an open question.** The header is all-traffic + when enabled. Rationale: values are durations only; the delivery layer already + exposes `hit-state` and `time-elapsed` publicly on every response; filter vendor + identity is masked; emission is restricted to conclusively-private responses so no + cache can replay stale timings. Revisit (quantization or gating) only if a concrete + abuse surfaces. +- The fronting delivery layer's Server-Timing pass-through is unverified until the + first staging deploy (step 2). This is the only known external dependency. +- Body-phase capture threads the timings handle into the streaming closure in + `publisher.rs`; the exact seam is an implementation-plan detail, with the constraint + that a dropped handle (error paths, early client disconnect) must still yield a + valid row with null body-phase fields and a recorded delivery outcome. +- The stall window itself remains unattributed until this ships. If it recurs first, + the bisection runbook from 2026-08-21 (cookie-free curl UA request, static-asset + path versus HTML path) is the fallback. + +## 18. Auction timeline offsets (follow-up increment) + +Status: spec amendment for a follow-up PR; not part of the initial implementation +(#1074). Builds only on machinery that spec sections 5, 9, and 10 already define. + +### Problem + +The pipeline has two clocks that never meet. The auction dataset +(`auction_events_raw`, PR #813) measures the auction internally: `total_time_ms` +from auction start to terminal, `provider_response_time_ms` per bidder call. Its +clock starts when the auction observation is created, so nothing places those +numbers on the request timeline. The access row is T0-anchored but records only +`auction_wait_ms`: time the handler was blocked at collect, deliberately not the +auction's own timeline. + +That leaves three questions unanswerable today: + +1. At what request-relative time did the auction start (dispatch leave the edge)? +2. At what request-relative time did the auction resolve (final bid or timeout)? +3. At what request-relative time were the results committed toward GAM? + +These are the overlap-proof questions. A client-side wrapper cannot dispatch until +the browser boots (t≈3000ms on measured prospect pages); the server-side auction +dispatches while the origin fetch is in flight. Proving that requires all +milestones on one clock. + +### Design + +Three first-call-wins marks on `RequestTimings`, in the style of +`mark_headers_ready()`, each storing `Option` since T0: + +| Mark | Recorded at | Meaning | +| --------------------------- | --------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------ | +| `mark_auction_dispatched()` | immediately before `orchestrator.dispatch_auction` returns control to the caller (`publisher.rs` dispatch site) | bid requests have left the edge | +| `mark_auction_resolved()` | immediately after `collect_dispatched_auction` returns, both collect sites | final bid returned or auction timed out; terminal either way | +| `mark_auction_committed()` | immediately after `write_bids_to_state` returns, both call sites | winning bids are in page state, available to the response pipeline | + +Notes on the definitions: + +- "Committed toward GAM" is defined as `write_bids_to_state` returning: the common + point in buffered and streaming modes where targeting becomes part of the + response. TS never calls GAM server-side; the browser's GPT call carries the + targeting, and that half of the timeline belongs to client-side measurement. + The edge proves when targeting was available; the client proves when GAM saw it. +- First-call-wins on all three marks. A request produces at most one publisher-path + auction today; if a second auction ever occurs in one request, the row describes + the first and the auction dataset still carries both in full. +- Same locking and failure model as every other `RequestTimings` write: `try_lock`, + drop on contention, saturating conversion at serialization. + +### Row changes + +Four additive columns on `access_logs_raw`, all populated from the +`TimingSnapshot` at the existing freeze/emission points (no new emission path): + +``` +`auction_dispatched_ms` Nullable(UInt32), `json:$.auction_dispatched_ms` +`auction_resolved_ms` Nullable(UInt32), `json:$.auction_resolved_ms` +`auction_committed_ms` Nullable(UInt32), `json:$.auction_committed_ms` +`auction_id` String, `json:$.auction_id` +``` + +- The three offsets are null when no auction ran (the common case: assets, EC + endpoints, auction-disabled deployments). Null means "no auction", never "zero". +- `auction_id` is the telemetry auction UUID already present on every + `auction_events_raw` row, carried onto the access row as the join key between + the T0 timeline and per-bidder detail. Sentinel `none` when no auction ran, + matching the non-nullable-dimension convention of section 9. It is a random + UUID, not identity-bearing; unbounded cardinality is accepted for the same + reason it is accepted in the auction dataset. +- Schema evolution is additive with JSONPaths on every new column and + `FORWARD_QUERY` carrying the existing columns, per the deployed datasource's + established evolution path. Verified with `tb --cloud deploy --check` before + deploy. + +### Interpretation model + +Combined with existing columns, one access row now reads as a timeline: + +``` +t=0 ......... request entry +t=D ......... auction_dispatched_ms (bids out; origin fetch typically in flight) +t=R ......... auction_resolved_ms (R - D ~ auction duration; join auction_id + for the per-bidder long pole) +t=C ......... auction_committed_ms (targeting in page state) +t=H ......... time_elapsed_ms (headers committed) +``` + +Derivations the dashboard can add without schema help: auction duration on the +request clock (`R - D`), commit latency (`C - R`), and overlap ratio (share of +`R - D` that ran concurrently with `ts-origin`). `auction_wait_ms` keeps its +existing meaning (blocked time only) and is now interpretable next to the +timeline: `R - D` minus `auction_wait_ms` approximates how much of the auction +was absorbed by work the request needed anyway. + +### Scope + +- Fastly emits; Axum, Cloudflare, and Spin collect the marks but do not emit, + matching section 8a adapter semantics. +- No header emission for any of these values: they are post-hoc analysis fields, + and two of the three are typically unknown at the header freeze point in + streaming mode. +- No config surface: the marks are always-on collection like every other phase, + gated at emission by the existing `tinybird.access_enabled`. diff --git a/docs/superpowers/specs/2026-08-27-pr-1079-review-remediation-design.md b/docs/superpowers/specs/2026-08-27-pr-1079-review-remediation-design.md new file mode 100644 index 000000000..f3603e767 --- /dev/null +++ b/docs/superpowers/specs/2026-08-27-pr-1079-review-remediation-design.md @@ -0,0 +1,80 @@ +# PR 1079 Review Remediation Design + +## Goal + +Make the first-impression ownership and APS creative bridge safe under overlapping +publisher auctions, late callbacks, SPA navigation, mixed GPT refresh lists, and +nested 1x1 GAM shells. Preserve PR 1079's first-claimant policy: Trusted Server may +win an untouched physical slot, but must neither overwrite a publisher impression +nor let a stale response affect a later navigation. + +## Ownership model + +First-impression state remains keyed by navigation generation and exact physical +element identity. Each publisher auction gets an independent token whose +suppression decision is fixed when the auction is registered. When Trusted Server +commits its request, registration closes for new losing publisher auctions, while +already-registered losing tokens remain suppressible. Those tokens remain as +tombstones for the lifetime of the same navigation and exact physical element. +Unresolved suppressing tombstones are never evicted or removed by timeout or +auction failure; only navigation change or physical element replacement removes +them. The existing per-slot registration limit bounds the set before registration +closes, so an arbitrarily late correlated callback cannot become unrelated. + +Prebid's pending bid/code correlation records carry the navigation generation and +physical element identity captured at registration. A record is usable only while +both still match, and consuming one exact ad-ID delivery removes only its auction's +registration. A code-only delivery consumes a record only when exactly one current +registration matches. Ambiguous ordinary code-only deliveries run an independent +auction rather than guessing; ambiguous TS-owned suppressing deliveries fail closed +without deleting their tombstones. Scoped `requestBids({ adUnitCodes })` calls +inspect, mutate, claim, and correlate only those requested global ad units. + +## Refresh suppression + +The Prebid delivery wrapper is the owner of first-impression delivery suppression. +When it suppresses a GPT slot, it also consumes any equivalent late-handoff +one-shot flag so the inner GPT wrapper cannot suppress the next legitimate +refresh. When it delegates a permitted GPT request, it consumes that flag at the +delegation boundary so the inner wrapper cannot silently drop the request. Mixed +refresh calls always forward the already-filtered slot list, including the path +where every remaining slot is excluded from a Prebid auction. That all-excluded +path performs the same ownership registration and consumption synchronously +before delegating. A bare refresh delayed by an auction becomes an explicit list +at callback time, preventing slots added after the snapshot from joining it. + +A publisher-triggered GPT refresh that starts a synthetic Prebid auction registers +its own per-slot first-impression tokens before waiting for the asynchronous +callback. A publisher-first token reserves the slot so TS cannot claim it while +the auction is pending. A token registered against an earlier TS claim is consumed +at callback time, filtering that slot from the eventual GPT request. When TS emits +its first GPT request, registration closes for new losing publisher tokens so +ordinary later publisher refreshes continue normally. Mixed callbacks forward +only their unsuppressed slots and scope Prebid targeting to the same filtered set. +The callback also revalidates the captured navigation generation and exact +physical element, dropping stale work rather than refreshing a replacement slot. + +## Creative bridge + +Every asynchronous renderer/cache result is revalidated before posting a creative +response or recording successful response/billing evidence. A stale result may be +recorded as safe failure telemetry, but is never recorded as a response or win. +Validation covers navigation generation, winning bid identity, authenticated +source iframe identity, DOM connectivity, and containment in the authenticated +slot root. When a configured prefix matches several roots, the requesting frame +may disambiguate them only when exactly one candidate root owns that source. + +After a valid response is posted, a collapsed 1x1 source iframe is expanded to the +winning creative size. The bridge walks all collapsed ancestors through the +authenticated slot root and expands each clipping shell. It refuses all resizing +for fixed/sticky, anchor, vignette, interstitial, detached, oversized, or +otherwise unauthenticated shells. + +## Verification + +Regression tests cover all seven review findings, including wrapper composition, +scoped ad-unit requests, mixed excluded refreshes, stale SPA callbacks, +overlapping auctions, stale cache responses with no successful response/billing +evidence, and two nested +collapsed ancestors. Existing JS unit/browser suites, formatting, lint, build, +and repository Rust verification remain the completion gates. diff --git a/fastly.toml b/fastly.toml index 56002bc5a..78ad6fac7 100644 --- a/fastly.toml +++ b/fastly.toml @@ -57,11 +57,20 @@ build = """ key = "tinybird_auction_append_token" data = "test-tinybird-auction-append-token" + # App-config references use logical `trusted_server_secrets`; the + # edgezero_runtime_env mapping below resolves it to physical `ts_secrets`. [[local_server.secret_stores.ts_secrets]] - key = "tinybird_access_append_token" - data = "test-tinybird-access-append-token" + key = "placeholder" + data = "placeholder" [local_server.config_stores] + [local_server.config_stores.edgezero_runtime_env] + format = "inline-toml" + [local_server.config_stores.edgezero_runtime_env.contents] + # Viceroy reports this fixed synthetic service id. The pinned EdgeZero + # feature branch scopes Fastly runtime mappings by service id. + EDGEZERO__SERVICES__0000000000000000000000__STORES__SECRETS__TRUSTED_SERVER_SECRETS__NAME = "ts_secrets" + [local_server.config_stores.trusted_server_config] format = "inline-toml" [local_server.config_stores.trusted_server_config.contents] diff --git a/scripts/template-cache-local-test.sh b/scripts/template-cache-local-test.sh index ad9939d0a..d6627e383 100755 --- a/scripts/template-cache-local-test.sh +++ b/scripts/template-cache-local-test.sh @@ -24,6 +24,7 @@ esac REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" WORK="$(mktemp -d)" ORIGIN_PORT="${ORIGIN_PORT:-9099}" +BID_PORT="${BID_PORT:-9100}" TS_PORT="${TS_PORT:-7788}" HOST_TRIPLE="$(rustc -vV | sed -n 's/^host: //p')" @@ -31,6 +32,7 @@ HOST_TRIPLE="$(rustc -vV | sed -n 's/^host: //p')" # observable in the timings: with an instant auction, buffered and streaming # assembly are indistinguishable. BID_DELAY="${BID_DELAY:-1.5}" +REQUEST_TIMEOUT_SECONDS="${REQUEST_TIMEOUT_SECONDS:-30}" PASS=0 FAIL=0 @@ -45,8 +47,16 @@ check() { # check cleanup() { local status=$? - [ -n "${VICEROY_PID:-}" ] && kill "$VICEROY_PID" 2>/dev/null || true - [ -n "${ORIGIN_PID:-}" ] && kill "$ORIGIN_PID" 2>/dev/null || true + if [ -n "${VICEROY_PID:-}" ]; then + kill "$VICEROY_PID" 2>/dev/null || true + wait "$VICEROY_PID" 2>/dev/null || true + fi + if [ -n "${ORIGIN_PID:-}" ]; then + # macOS may launch the framework Python process as a child of the shim. + pkill -TERM -P "$ORIGIN_PID" 2>/dev/null || true + kill "$ORIGIN_PID" 2>/dev/null || true + wait "$ORIGIN_PID" 2>/dev/null || true + fi rm -rf "$WORK" exit $status } @@ -60,13 +70,17 @@ command -v node >/dev/null || { echo "node not found. The harness executes the real GPT bundle to verify slot setup." >&2 exit 1 } +command -v openssl >/dev/null || { + echo "openssl not found. The harness needs it for the local HTTPS bid endpoint." >&2 + exit 1 +} # A port already in use means requests would go to something else entirely — most # likely a leftover run, whose warm cache and stale config would read as a result. -for port in "$ORIGIN_PORT" "$TS_PORT"; do +for port in "$ORIGIN_PORT" "$BID_PORT" "$TS_PORT"; do if lsof -nP -iTCP:"$port" -sTCP:LISTEN >/dev/null 2>&1; then echo "Port $port is already in use. Stop the process, or set" >&2 - echo "ORIGIN_PORT / TS_PORT to something free." >&2 + echo "ORIGIN_PORT / BID_PORT / TS_PORT to something free." >&2 lsof -nP -iTCP:"$port" -sTCP:LISTEN >&2 exit 1 fi @@ -78,19 +92,35 @@ cargo build -p trusted-server-cli --target "$HOST_TRIPLE" >/dev/null WASM="$REPO_ROOT/target/wasm32-wasip1/debug/trusted-server-adapter-fastly.wasm" TS="$REPO_ROOT/target/$HOST_TRIPLE/debug/ts" -info "Starting stub origin on :$ORIGIN_PORT" +info "Generating a local CA and HTTPS bid certificate" +openssl req -x509 -newkey rsa:2048 -sha256 -days 1 -nodes \ + -subj "/CN=Trusted Server local harness CA" \ + -keyout "$WORK/ca-key.pem" -out "$WORK/ca-cert.pem" >/dev/null 2>&1 +openssl req -newkey rsa:2048 -sha256 -nodes -subj "/CN=localhost" \ + -keyout "$WORK/server-key.pem" -out "$WORK/server.csr" >/dev/null 2>&1 +cat > "$WORK/server.ext" <<'EOF' +basicConstraints=CA:FALSE +keyUsage=digitalSignature,keyEncipherment +extendedKeyUsage=serverAuth +subjectAltName=DNS:localhost,IP:127.0.0.1 +EOF +openssl x509 -req -sha256 -days 1 -in "$WORK/server.csr" \ + -CA "$WORK/ca-cert.pem" -CAkey "$WORK/ca-key.pem" -CAcreateserial \ + -extfile "$WORK/server.ext" -out "$WORK/server-cert.pem" >/dev/null 2>&1 + +info "Starting stub origin on :$ORIGIN_PORT and HTTPS bidder on :$BID_PORT" cat > "$WORK/origin.py" < "$WORK/origin.log" 2>&1 & ORIGIN_PID=$! sleep 1 info "Generating stub config (mode: $MODE)" -python3 - "$REPO_ROOT/trusted-server.example.toml" "$WORK/app.toml" "$MODE" "$ORIGIN_PORT" <<'PYEOF' -import sys, re -src, out, mode, port = sys.argv[1:5] +python3 - "$REPO_ROOT/trusted-server.example.toml" "$WORK/app.toml" "$MODE" \ + "$ORIGIN_PORT" "$BID_PORT" <<'PYEOF' +import sys + +src, out, mode, origin_port, bid_port = sys.argv[1:6] s = open(src).read() -s = s.replace('origin_url = "https://origin.example.com"', f'origin_url = "http://127.0.0.1:{port}"', 1) -# The example config ships placeholders that validation rejects outright, -# including the reserved publisher domain/cookie_domain. -s = s.replace('domain = "example.com"', 'domain = "local-harness.example"', 1) -s = s.replace('cookie_domain = ".example.com"', 'cookie_domain = ".local-harness.example"', 1) -s = s.replace('password = "replace-with-admin-password-32-bytes"', - 'password = "local-harness-admin-password-not-a-real-one"', 1) -s = s.replace('proxy_secret = "change-me-proxy-secret"', - 'proxy_secret = "local-harness-proxy-secret-not-a-real-one"', 1) -s = re.sub(r'passphrase = "[^"]*"', - 'passphrase = "local-harness-ec-passphrase-not-a-real-one"', s, count=1) - -# A real auction, pointed at the stub's slow endpoint, so the timings mean something. -s = s.replace('[integrations.prebid]\nenabled = false\nserver_url = "https://prebid.example.com/openrtb2/auction"', - f'[integrations.prebid]\nenabled = true\nserver_url = "http://127.0.0.1:{port}/bid"\n' - 'external_bundle_url = "https://assets.example.com/prebid/trusted-prebid-stub.js"', 1) -s = s.replace('providers = []', 'providers = ["prebid"]', 1) -s = s.replace('\n[proxy]\n', '\n[proxy]\nallowed_domains = ["assets.example.com", "127.0.0.1"]\n', 1) -s = s.replace('[auction]\nenabled = false', '[auction]\nenabled = true', 1) -s = s.replace('auction_timeout_ms = 500', 'auction_timeout_ms = 5000', 1) -# Both the overall auction and provider transport budgets must exceed the -# harness's deliberate three-second bid delay. -s = s.replace('timeout_ms = 2000', 'timeout_ms = 5000', 1) -s = s.replace('timeout_ms = 1000', 'timeout_ms = 5000', 1) + +def replace_once(content, old, new, description): + if content.count(old) != 1: + raise SystemExit(f"expected one {description} replacement target") + return content.replace(old, new, 1) + + +s = replace_once( + s, + 'origin_url = "https://origin.example.com"', + f'origin_url = "http://127.0.0.1:{origin_port}"', + "publisher origin", +) +# The example publisher domains are reserved placeholders that validation rejects. +s = replace_once( + s, + 'domain = "example.com"', + 'domain = "local-harness.example"', + "publisher domain", +) +s = replace_once( + s, + 'cookie_domain = ".example.com"', + 'cookie_domain = ".local-harness.example"', + "publisher cookie domain", +) + +# A real auction points at the slow HTTPS stub so the timings mean something. +s = replace_once( + s, + '[integrations.prebid]\nenabled = false', + '[integrations.prebid]\nenabled = true\n' + 'external_bundle_url = "https://assets.example.com/prebid/trusted-prebid-stub.js"', + "Prebid integration", +) +s = replace_once( + s, + 'endpoint = "https://prebid.example.com/openrtb2/auction"', + f'endpoint = "https://localhost:{bid_port}/bid"\ntimeout_ms = 5000', + "Prebid provider endpoint", +) +s = replace_once( + s, + '\n[proxy]\n', + '\n[proxy]\nallowed_domains = ["assets.example.com", "127.0.0.1"]\n', + "proxy table", +) +s = replace_once( + s, + '[auction]\n# Keep disabled until provider endpoints, routes, and profile values below are\n' + '# replaced with deployment-specific settings.\nenabled = false', + '[auction]\n# Keep disabled until provider endpoints, routes, and profile values below are\n' + '# replaced with deployment-specific settings.\nenabled = true', + "auction enablement", +) +s = replace_once( + s, + 'sanitize_creatives = false\ntimeout_ms = 2000', + 'sanitize_creatives = false\ntimeout_ms = 10000', + "auction timeout", +) +s = replace_once( + s, + 'auction_timeout_ms = 500', + 'auction_timeout_ms = 10000', + "creative opportunity auction timeout", +) # The template-cache keys go directly under the table header. The slot is a table of its own # and must go at the end: inserted here it would swallow every scalar key that @@ -233,6 +322,69 @@ info "Seeding an isolated config store (tracked fastly.toml remains untouched)" # pointed at this checkout without copying the workspace. cp "$REPO_ROOT/edgezero.toml" "$WORK/edgezero.toml" cp "$REPO_ROOT/fastly.toml" "$WORK/fastly.toml" +python3 - "$WORK/fastly.toml" <<'PYEOF' +import sys + +with open(sys.argv[1], "a") as manifest: + manifest.write(''' +[[local_server.secret_stores.ts_secrets]] +key = "publisher_proxy_secret" +data = "fictional-local-publisher-proxy-secret-value" + +[[local_server.secret_stores.ts_secrets]] +key = "ec_passphrase" +data = "fictional-local-ec-passphrase-secret-value" + +[[local_server.secret_stores.ts_secrets]] +key = "handler_password" +data = "fictional-local-handler-password-secret-value" +''') +PYEOF + +# The application registers provider backends dynamically. Pre-register the exact +# deterministic name so Viceroy reuses a local backend that trusts the temporary CA. +python3 - "$WORK/fastly.toml" "$WORK/ca-cert.pem" "$BID_PORT" <<'PYEOF' +import hashlib +import json +import sys + +manifest, ca_certificate, port = sys.argv[1:4] +provider_id = "pbs-main" +timeout_ms = "5000" + + +def field(value): + return f"{len(value)}:{value}" + + +canonical = "".join([ + field("https"), + field("localhost"), + field(port), + field("1"), + "n", + "s", + field(provider_id), + field(timeout_ms), + field(timeout_ms), +]) +digest = hashlib.sha256(canonical.encode()).hexdigest()[:32] +readable = f"https_localhost_{port}_p_{provider_id}_fb{timeout_ms}_bb{timeout_ms}" +backend_name = f"backend_{readable}_{digest}" +backend = f'''[local_server.backends.{backend_name}] +url = "https://localhost:{port}" +cert_host = "localhost" +ca_certificate.file = {json.dumps(ca_certificate)} +''' + +content = open(manifest).read() +marker = "[local_server.backends]\n\n" +if content.count(marker) != 1: + raise SystemExit("expected one local backend insertion target") +content = content.replace(marker, f"{marker}{backend}\n", 1) +open(manifest, "w").write(content) +PYEOF + ln -s "$REPO_ROOT/crates" "$WORK/crates" (cd "$WORK" && "$TS" config push --adapter fastly --local \ --manifest "$WORK/edgezero.toml" --app-config "$WORK/app.toml" \ @@ -259,7 +411,7 @@ fi req() { # req [extra curl args...] local out="$1"; shift - curl -sS -D "$out.headers" -o "$out" \ + curl -sS --max-time "$REQUEST_TIMEOUT_SECONDS" -D "$out.headers" -o "$out" \ -w '%{time_starttransfer} %{time_total} %{http_code}' \ -H "Host: ts.example.com" \ -H "Accept-Encoding: gzip" \ @@ -329,7 +481,8 @@ assembly_state() { # Shared by the ESI assertions below. check_hit_is_private() { local hdrs - hdrs=$(curl -s -D- -o /dev/null -H "Host: ts.example.com" \ + hdrs=$(curl -sS --max-time "$REQUEST_TIMEOUT_SECONDS" -D- -o /dev/null \ + -H "Host: ts.example.com" \ -H "Accept-Encoding: gzip" \ -H "sec-fetch-dest: document" -H "sec-fetch-mode: navigate" \ "http://127.0.0.1:$TS_PORT/article") @@ -340,7 +493,8 @@ check_hit_is_private() { check_post_reaches_origin() { local before before=$(grep -cF "origin: received POST /article" "$WORK/origin.log" || true) - curl -s -o /dev/null -X POST -d 'x=1' -H "Host: ts.example.com" \ + curl -sS --max-time "$REQUEST_TIMEOUT_SECONDS" -o /dev/null \ + -X POST -d 'x=1' -H "Host: ts.example.com" \ -H "Accept-Encoding: gzip" \ "http://127.0.0.1:$TS_PORT/article" check "a POST still reaches the origin" \ @@ -526,16 +680,17 @@ first chunk looks identical to one that does not. import socket, sys, time host, port, path = sys.argv[1], int(sys.argv[2]), sys.argv[3] -extra = sys.argv[4] if len(sys.argv) > 4 else "" +timeout = float(sys.argv[4]) req = ( f"GET {path} HTTP/1.1\r\nHost: ts.example.com\r\n" "sec-fetch-dest: document\r\nsec-fetch-mode: navigate\r\n" "accept-encoding: gzip\r\n" - f"{extra}Connection: close\r\n\r\n" + "Connection: close\r\n\r\n" ).encode() -s = socket.create_connection((host, port)) +s = socket.create_connection((host, port), timeout=timeout) +s.settimeout(timeout) t0 = time.time() s.sendall(req) @@ -572,7 +727,9 @@ cat <<'EOF' one that does not. EOF echo -probe_body_ms() { python3 "$WORK/probe.py" 127.0.0.1 "$TS_PORT" /article; } +probe_body_ms() { + python3 "$WORK/probe.py" 127.0.0.1 "$TS_PORT" /article "$REQUEST_TIMEOUT_SECONDS" +} echo " request A: $(probe_body_ms)" B_LINE="$(probe_body_ms)" echo " request B: $B_LINE" diff --git a/scripts/test-cli.sh b/scripts/test-cli.sh index eef9e2f7d..7b562c96e 100755 --- a/scripts/test-cli.sh +++ b/scripts/test-cli.sh @@ -19,3 +19,20 @@ if ! rustup target list --installed | awk -v target="$HOST_TARGET" '$0 == target fi cargo test --package trusted-server-cli --target "$HOST_TARGET" +export TS_AUDIT_BROWSER_TESTS=1 +AUDIT_BROWSER_TEST_FILTERS=( + "commands::audit::browser::tests::" + "commands::audit::generate::browser_collector::tests::" +) +for AUDIT_BROWSER_TEST_FILTER in "${AUDIT_BROWSER_TEST_FILTERS[@]}"; do + AUDIT_BROWSER_TEST_COUNT="$({ + cargo test --package trusted-server-cli --target "$HOST_TARGET" \ + "$AUDIT_BROWSER_TEST_FILTER" -- --ignored --list + } | awk '/: test$/ { count += 1 } END { print count + 0 }')" + if [ "$AUDIT_BROWSER_TEST_COUNT" -eq 0 ]; then + echo "No ignored browser audit fixtures matched $AUDIT_BROWSER_TEST_FILTER" >&2 + exit 1 + fi + cargo test --package trusted-server-cli --target "$HOST_TARGET" \ + "$AUDIT_BROWSER_TEST_FILTER" -- --ignored --test-threads=1 +done diff --git a/tinybird/datasources/access_logs_raw.datasource b/tinybird/datasources/access_logs_raw.datasource index 42f214e07..4441de384 100644 --- a/tinybird/datasources/access_logs_raw.datasource +++ b/tinybird/datasources/access_logs_raw.datasource @@ -1,19 +1,43 @@ DESCRIPTION > - Optional sampled Trusted Server access telemetry rows. Disabled by default in Fastly config. + Per-request phase-timing telemetry rows, sampled and emitted post-send by the edge service. SCHEMA > - `event_ts` DateTime64(3), - `method` LowCardinality(String), - `path` String, - `status` UInt16, - `time_elapsed_ms` UInt32, - `cache_state` LowCardinality(Nullable(String)), - `country` LowCardinality(String), - `sample_rate` Float64, - `event_date` Date DEFAULT toDate(event_ts) + `event_ts` DateTime64(3) `json:$.event_ts`, + `method` LowCardinality(String) `json:$.method`, + `status` UInt16 `json:$.status`, + `time_elapsed_ms` Nullable(UInt32) `json:$.time_elapsed_ms`, + `sample_rate` Float64 `json:$.sample_rate`, + `service_id` LowCardinality(String) `json:$.service_id`, + `publisher_domain` LowCardinality(String) `json:$.publisher_domain`, + `env` LowCardinality(String) `json:$.env`, + `route_class` LowCardinality(String) `json:$.route_class`, + `route_template` String `json:$.route_template`, + `body_mode` LowCardinality(String) `json:$.body_mode`, + `auction_wait_placement` LowCardinality(String) `json:$.auction_wait_placement`, + `appbuild_ms` Nullable(UInt32) `json:$.appbuild_ms`, + `filter_ms` Nullable(UInt32) `json:$.filter_ms`, + `geo_ms` Nullable(UInt32) `json:$.geo_ms`, + `kv_ms` Nullable(UInt32) `json:$.kv_ms`, + `origin_ms` Nullable(UInt32) `json:$.origin_ms`, + `template_cache_ms` Nullable(UInt32) `json:$.template_cache_ms`, + `auction_wait_ms` Nullable(UInt32) `json:$.auction_wait_ms`, + `stream_ms` Nullable(UInt32) `json:$.stream_ms`, + `request_elapsed_ms` Nullable(UInt32) `json:$.request_elapsed_ms`, + `resp_bytes` Nullable(UInt64) `json:$.resp_bytes`, + `template_cache_state` LowCardinality(String) `json:$.template_cache_state`, + `country` LowCardinality(String) `json:$.country`, + `ts_version` LowCardinality(String) `json:$.ts_version`, + `pop` LowCardinality(String) `json:$.pop`, + `auction_dispatched_ms` Nullable(UInt32) `json:$.auction_dispatched_ms`, + `auction_resolved_ms` Nullable(UInt32) `json:$.auction_resolved_ms`, + `auction_committed_ms` Nullable(UInt32) `json:$.auction_committed_ms`, + `auction_id` String `json:$.auction_id` ENGINE "MergeTree" -ENGINE_SORTING_KEY "event_date, path, status, method" -TTL "event_date + INTERVAL 30 DAY" +ENGINE_SORTING_KEY "toDate(event_ts), service_id, publisher_domain, env, route_class, pop, status" +TTL "toDate(event_ts) + INTERVAL 30 DAY" + +FORWARD_QUERY > + SELECT event_ts, method, status, time_elapsed_ms, sample_rate, service_id, publisher_domain, env, route_class, route_template, body_mode, auction_wait_placement, appbuild_ms, filter_ms, geo_ms, kv_ms, origin_ms, template_cache_ms, auction_wait_ms, stream_ms, request_elapsed_ms, resp_bytes, template_cache_state, country, ts_version, pop, CAST(NULL AS Nullable(UInt32)) AS auction_dispatched_ms, CAST(NULL AS Nullable(UInt32)) AS auction_resolved_ms, CAST(NULL AS Nullable(UInt32)) AS auction_committed_ms, 'none' AS auction_id TOKEN ts_access_ingest APPEND diff --git a/tinybird/datasources/auction_events_raw.datasource b/tinybird/datasources/auction_events_raw.datasource index d62f8ae5d..97e8251bb 100644 --- a/tinybird/datasources/auction_events_raw.datasource +++ b/tinybird/datasources/auction_events_raw.datasource @@ -12,6 +12,7 @@ SCHEMA > `region` Nullable(String), `is_mobile` UInt8, `is_known_browser` UInt8, + `user_agent` Nullable(String), `gdpr_applies` UInt8, `consent_present` UInt8, `terminal_status` LowCardinality(Nullable(String)), diff --git a/tinybird/fixtures/access_logs_raw.ndjson b/tinybird/fixtures/access_logs_raw.ndjson new file mode 100644 index 000000000..3c82c5ca6 --- /dev/null +++ b/tinybird/fixtures/access_logs_raw.ndjson @@ -0,0 +1 @@ +{"event_ts":"2026-06-23 12:00:00.000","method":"GET","status":200,"time_elapsed_ms":145,"sample_rate":0.1,"service_id":"abc123","publisher_domain":"test-publisher.com","env":"production","route_class":"publisher_html","route_template":"/news/*","body_mode":"streamed","auction_wait_placement":"in_stream","appbuild_ms":12,"filter_ms":5,"geo_ms":3,"kv_ms":8,"origin_ms":25,"template_cache_ms":10,"auction_wait_ms":45,"stream_ms":18,"request_elapsed_ms":145,"resp_bytes":8192,"template_cache_state":"hit","country":"US","ts_version":"v1.2.3","pop":"SFO"} diff --git a/trusted-server.example.toml b/trusted-server.example.toml index b0e359cb4..d74efface 100644 --- a/trusted-server.example.toml +++ b/trusted-server.example.toml @@ -38,7 +38,7 @@ # Regex matched against the request path. This one guards the admin surface. path = "^/_ts/admin" username = "admin" -password = "replace-with-admin-password-32-bytes" +password = "handler_password" # You can add more handlers to basic-auth-protect other path prefixes. The # sample password below is a known placeholder that deploy validation rejects, @@ -59,8 +59,8 @@ domain = "example.com" cookie_domain = ".example.com" # Upstream origin to proxy publisher content from. No trailing slash. origin_url = "https://origin.example.com" -# HMAC secret for signing first-party proxy URLs. Replace before deploying. -proxy_secret = "change-me-proxy-secret" +# Secret Store key for the HMAC secret used to sign first-party proxy URLs. +proxy_secret = "publisher_proxy_secret" # Optional: override the outbound Host header sent to origin_url. # origin_host_header_override = "www.example.com" # Optional: max bytes buffered when a response is post-processed in full (HTML @@ -73,9 +73,8 @@ proxy_secret = "change-me-proxy-secret" # REQUIRED — Edge Cookie (EC) identity # ----------------------------------------------------------------------------- [ec] -# Secret used to derive EC identifiers. Must be >= 32 chars and non-placeholder -# in production (deploy validation rejects known placeholders). -passphrase = "trusted-server-placeholder-secret" +# Secret Store key for the secret used to derive EC identifiers. +passphrase = "ec_passphrase" # KV store that persists EC identity state. This is the physical store name # bound per adapter (e.g. `ec_identity_store` in fastly.toml); edgezero.toml's # logical KV id is `trusted_server_kv`. @@ -86,9 +85,8 @@ pull_sync_concurrency = 3 # cluster_trust_threshold = 10 # entries with cluster_size <= this are individual users # cluster_recheck_secs = 3600 # re-evaluate cluster_size after this many seconds -# Optional identity partners (SSP/DSP/identity vendors). Each needs a real, -# non-placeholder api_token (>= 32 bytes) at deploy. Configure real partners via -# private config, not this template. +# Example partner configuration. Provision referenced keys in +# trusted_server_secrets before validating/pushing. # [[ec.partners]] # name = "Example Partner" # source_domain = "partner.example.com" @@ -97,7 +95,10 @@ pull_sync_concurrency = 3 # openrtb_atype = 3 # include this partner's UIDs in auction user.eids # bidstream_enabled = true -# api_token = "replace-with-partner-api-token-32-bytes-minimum" +# Only for inbound identify or batch-sync API access: +# api_token = "partner_api_token" +# Required when pull sync is enabled: +# ts_pull_token = "partner_ts_pull_token" # batch_rate_limit = 60 # max batch-sync requests/min (default 60) # pull_sync_enabled = false # default false @@ -162,6 +163,15 @@ pull_sync_concurrency = 3 # mode = "restrictive" # "restrictive" | "newest" | "permissive" # freshness_threshold_days = 30 +[integrations.js_asset_proxy] +enabled = false +cache_ttl_seconds = 3600 + +[[integrations.js_asset_proxy.assets]] +path = "/assets/example-vendor-loader.js" +origin_url = "https://cdn.example.com/vendor-loader.js" +proxy = "enabled" + # Proxy behavior and first-party asset routing. Kept active with defaults. [proxy] # Verify TLS certs when proxying to HTTPS origins (default true; false only for @@ -238,10 +248,12 @@ pull_sync_concurrency = 3 # edge_ttl_seconds = 31536000 # immutable = true -# Server-side auction. Provider/mediator names must match enabled integrations. -# Kept active with the creative-processing leaves present so the EdgeZero -# environment override can apply to them. +# Server-side auction. Providers are declared below; mediator names must match +# enabled integrations. Kept active with the creative-processing leaves present +# so the EdgeZero environment override can apply to them. [auction] +# Keep disabled until provider endpoints, routes, and profile values below are +# replaced with deployment-specific settings. enabled = false # Rewrite winning-bid creative HTML to first-party endpoints (default true). Set # false to skip proxy/click-URL conversion and creative TSJS injection. @@ -259,13 +271,54 @@ rewrite_creatives = true # server's iframe), since it removes script-based creatives entirely and would # blank slots on a script-heavy demand stack. sanitize_creatives = false -providers = [] timeout_ms = 2000 # mediator = "adserver_mock" # optional mediator integration # Context keys the JS client may forward into auction requests (allowlist; # empty blocks all). allowed_context_keys = [] +# Example server-side provider declarations. Remove or customize before enabling auctions. +[auction.providers.pbs-main] +protocol = "openrtb-2.6" +profile = "prebid-server" +endpoint = "https://prebid.example.com/openrtb2/auction" +# `prebid-server` defaults to 1000 ms when omitted. +routing = "explicit" + +[auction.providers.pbs-main.profile_config] +debug = false +test_mode = false +consent_forwarding = "both" + +[auction.providers.pbs-main.notifications] +suppress_all = false +suppress_seats = [] + +[auction.bidders.example-bidder] +provider = "pbs-main" + +# APS server behavior is also provider/profile-owned. Use routing = "all_eligible" +# when every banner-compatible slot should be eligible. The APS profile timeout +# defaults to 800 ms; debug and script creatives default to false. +# [auction.providers.aps-main] +# protocol = "openrtb-2.6" +# profile = "aps" +# endpoint = "https://aps.example.com/e/pb/bid" +# routing = "all_eligible" +# +# [auction.providers.aps-main.profile_config] +# account_id = "example-aps-account-id" +# debug = false +# allow_script_creatives = false +# inventory_domain = "publisher.example" +# inventory_page_origin = "https://www.publisher.example" + +[observability] +# Keep this leaf present so the environment override can apply; the overlay +# cannot create a missing configuration leaf. The Server-Timing header stays +# off until enabled. +server_timing_enabled = false + # Server-side ad slot templates + creative-opportunity auction. Kept active. [creative_opportunities] # Set false to disable server-side ad templates while keeping slot definitions @@ -356,9 +409,14 @@ auction_timeout_ms = 500 # [tinybird] # enabled = true # api_host = "api.us-east.tinybird.example" # required when enabled; host only -# secret_store = "ts_secrets" # Secret Store holding the append token -# auction_dataset = "auction_events" # Events API datasource name -# auction_token_secret = "tinybird_auction_append_token" # Secret Store key for the token +# auction_enabled = true # emit auction telemetry +# auction_dataset = "auction_events_raw" # auction Events API datasource +# auction_token_secret = "tinybird_auction_append_token" # Key in trusted_server_secrets +# access_enabled = false # emit sampled access telemetry +# access_dataset = "access_logs_raw" # access Events API datasource +# access_token_secret = "tinybird_access_append_token" # Key in trusted_server_secrets +# access_sample_rate = 0.0 # fraction from 0.0 through 1.0 +# max_body_bytes = 1048576 # maximum NDJSON request body # Debug endpoints (all default false — never enable in production). # [debug] @@ -399,14 +457,13 @@ auction_timeout_ms = 500 # their section is commented out. Required fields are noted per block. # ============================================================================= -# Prebid Server-side auction + first-party Prebid.js bundle. -# When enabled: `server_url` is required, and `external_bundle_url` is required -# (its host must be listed in [proxy].allowed_domains). Kept active but disabled. +# Browser-side Prebid.js integration. Server-side bidder routing belongs under +# [auction.providers] and [auction.bidders]. When enabled, +# `external_bundle_url` is required and its host must be listed in +# [proxy].allowed_domains. Kept active but disabled. [integrations.prebid] enabled = false -server_url = "https://prebid.example.com/openrtb2/auction" timeout_ms = 1000 -bidders = [] debug = false client_side_bidders = [] # bidders running via native Prebid.js adapters # Keep selected GAM inventory out of Trusted Server's Prebid refresh auctions. @@ -416,12 +473,6 @@ client_side_bidders = [] # bidders running via native Prebid.js adapter # external_bundle_url = "https://assets.example.com/prebid/trusted-prebid-.js" # external_bundle_sha256 = "" # external_bundle_sri = "" -# Per-bidder / per-zone param overrides (canonical rule form): -# [[integrations.prebid.bid_param_override_rules]] -# when.bidder = "examplebidder" -# when.zone = "header" -# set = { placementId = "_abc" } -# # Bundle build inputs consumed by the `ts prebid bundle` CLI (not the runtime): # [integrations.prebid.bundle] # adapters = ["rubicon"] @@ -490,7 +541,8 @@ enabled = false # rewrite_sdk = true # Server-side Protection API validation (fails open on timeout/error): # enable_protection = false -# server_side_key_secret_store = "ts_secrets" +# Required when enable_protection = true. The value is a key in +# trusted_server_secrets, not the DataDome credential itself. # server_side_key_secret_name = "datadome_server_side_key" # protection_api_origin = "https://api.example.com" # timeout_ms = 1500 @@ -527,24 +579,13 @@ gam_attribution_enabled = false # [integrations.gpt_diagnostics] # enabled = true -# Amazon Publisher Services (APS/TAM) OpenRTB. `account_id` required when -# enabled (`pub_id` is accepted as a deserialization alias only). +# APS browser renderer ownership. Server-side APS behavior belongs under an +# [auction.providers] entry with profile = "aps". # [integrations.aps] # enabled = true -# account_id = "example-aps-account-id" # required (non-empty); your APS account -# endpoint = "https://aps.example.com/e/pb/bid" -# timeout_ms = 1000 -# Include raw APS request/response data in /auction metadata on test sites only. -# debug = false -# Script creatives require separate security validation before opt-in. -# allow_script_creatives = false -# Winning-bid renderer. Default `trusted_server` uses TS's opaque static renderer -# route; set `publisher_native` only for the controlled publisher-origin -# friendly-frame experiment. +# Default `trusted_server` uses TS's opaque static renderer route. Set +# `publisher_native` only for a controlled publisher-origin friendly-frame cohort. # rendering_mode = "trusted_server" -# Set both when the deployment hostname differs from APS-authorized inventory. -# inventory_domain = "publisher.example" -# inventory_page_origin = "https://www.publisher.example" # Google Tag Manager first-party proxy. Kept active but disabled so `ts audit` # can fill container_id and flip `enabled` when GTM is detected. `container_id`