diff --git a/benchmarks/aggregator-head-lag.yml b/benchmarks/aggregator-head-lag.yml index 56db0b119..3d11582a7 100644 --- a/benchmarks/aggregator-head-lag.yml +++ b/benchmarks/aggregator-head-lag.yml @@ -2,10 +2,10 @@ slug: aggregator-head-lag number: "001" -title: Fastest crypto price API, live head lag across Mobula, Codex, GeckoTerminal +title: Fastest crypto price API, live head lag across Mobula, Codex, GeckoTerminal, Serialized seo_title: "Fastest crypto price API 2026" -seo_description: "{{best_name}} leads fastest crypto price API at {{best_p50}} (cross-chain avg p50, 24h). Mobula WebSocket, Codex GraphQL, GeckoTerminal REST live across Base, BNB, Solana, Robinhood." -subtitle: Wall-clock head lag in seconds from on-chain swap event to API emission, measured live for Mobula, Codex and GeckoTerminal on Base, BNB Chain, Solana and Robinhood Chain. +seo_description: "{{best_name}} leads fastest crypto price API at {{best_p50}} (cross-chain avg p50, 24h). Mobula WebSocket, Codex GraphQL, GeckoTerminal REST, Serialized WebSocket live across Base, BNB, Solana, Robinhood." +subtitle: Wall-clock head lag in seconds from on-chain swap event to API emission, measured live for Mobula, Codex, GeckoTerminal and Serialized on Base, BNB Chain, Solana and Robinhood Chain. per_chain_explainer: - slug: base @@ -37,7 +37,7 @@ seo_intro: | chain and the same event appearing on the provider's feed. Marketing pages quote "real-time" without a number; this page quotes the number. Mobula's WebSocket, Codex's GraphQL feed, GeckoTerminal's - REST endpoint are watched + REST endpoint and Serialized's WebSocket trades stream are watched from three regions (us-east, eu-west, sgp). On Base the zero point is the sequencer's flashblock preconfirmation stream, which we hold ourselves; on BNB Chain, Solana and Robinhood Chain it is the @@ -63,7 +63,7 @@ abstract: | same event. methodology: - - "Aggregators measured: Mobula, Codex, GeckoTerminal." + - "Aggregators measured: Mobula, Codex, GeckoTerminal, Serialized." - "Chains: Base, BNB Chain, Solana, Robinhood Chain." - "Reference on BNB Chain and Robinhood Chain: archive nodes per chain, validated against block hashes. These carry no preconfirmation layer a provider consumes, and their chain-supplied timestamps sit within roughly 60 ms of the moment a trade is observable, so they keep the on-chain timestamp as the zero point." - "Solana (since 2026-09-16): the headline is the lag behind the first feed to report the trade, our own node subscription included in the race. Solana has no on-chain timestamp with sub-second precision, so the timestamps providers send compare conventions: Mobula's `date` (its ingestion time) read as a constant 0.10 s, Serialized's `at` (blockTime, whole seconds) as 0.75 s, while against a common clock the two feeds are 10 to 30 ms apart." @@ -87,7 +87,7 @@ findings: - "On Base, what a provider reads matters more than how fast its pipeline is. A provider emitting from flashblock preconfirmations delivers a trade about 1.6 s (median over 21,538 transactions) before the sealed block carrying it becomes queryable, so a sealed-block reader carries that floor before any pipeline work. A preconfirmation is not final and can be reordered." - "The Base spread is therefore mostly that floor: the providers' own pipelines sit within a few hundred milliseconds of each other. Read the Base column as a latency versus finality choice, not as a pure speed ranking." - "{{name:geckoterminal}} trails on every chain. REST polling adds the poll interval to every read, so head lag tracks the publisher's chosen cadence rather than raw infrastructure speed." - - "p99 is the integration-grade number. The gap between p50 and {{p99:mobula}} / {{p99:codex}} / {{p99:geckoterminal}} is what a live UI feels when a chain spikes or a region's path degrades." + - "p99 is the integration-grade number. The gap between p50 and {{p99:mobula}} / {{p99:codex}} / {{p99:serialized}} / {{p99:geckoterminal}} is what a live UI feels when a chain spikes or a region's path degrades." source: https://github.com/ChainBench/OpenChainBench/tree/main/harnesses/aggregator-head-lag @@ -247,6 +247,32 @@ providers: p50: avg by (aggregator) (quantile_over_time(0.50, head_lag_seconds{aggregator="codex", region="sgp"}[24h])) * 1000 series: avg_over_time(head_lag_seconds{aggregator="codex", region="sgp"}[1h]) * 1000 unless (changes(head_lag_seconds{aggregator="codex", region="sgp"}[15m]) == 0 and count_over_time(head_lag_seconds{aggregator="codex", region="sgp"}[15m]) > 5 and on(chain, region) sum by (chain, region) (changes(head_lag_seconds{region="sgp"}[15m])) > 5 and on(aggregator, region) avg_over_time(ws_connected{aggregator="codex", region="sgp", chain=""}[15m]) > 0.8) + + + - slug: serialized + name: Serialized + tag: WebSocket trades stream + formula: "Median seconds between an on-chain swap on the bench pools and the same trade arriving on Serialized's WebSocket trades stream (subscribed per pool by its token side), sampled every 15s over 24h." + queries: + p50: avg by (aggregator) (quantile_over_time(0.50, head_lag_seconds{aggregator="serialized"}[24h])) * 1000 + p90: avg by (aggregator) (quantile_over_time(0.90, head_lag_seconds{aggregator="serialized"}[24h])) * 1000 + p99: avg by (aggregator) (quantile_over_time(0.99, head_lag_seconds{aggregator="serialized"}[24h])) * 1000 + mean: avg by (aggregator) (avg_over_time(head_lag_seconds{aggregator="serialized"}[24h])) * 1000 + success: clamp_max(avg by (aggregator) (count_over_time(head_lag_seconds{aggregator="serialized"}[24h]) / 2880), 1) + sample_size: sum(count_over_time(head_lag_seconds{aggregator="serialized"}[24h])) + series: avg_over_time(head_lag_seconds{aggregator="serialized"}[1h]) * 1000 unless (changes(head_lag_seconds{aggregator="serialized"}[15m]) == 0 and count_over_time(head_lag_seconds{aggregator="serialized"}[15m]) > 5 and on(chain, region) sum by (chain, region) (changes(head_lag_seconds{}[15m])) > 5 and on(aggregator, region) avg_over_time(ws_connected{aggregator="serialized", chain=""}[15m]) > 0.8) + live_activity: sum(changes(head_lag_seconds{aggregator="serialized"}[15m])) + regions: + - region: us-east + p50: avg by (aggregator) (quantile_over_time(0.50, head_lag_seconds{aggregator="serialized", region="us-east"}[24h])) * 1000 + series: avg_over_time(head_lag_seconds{aggregator="serialized", region="us-east"}[1h]) * 1000 unless (changes(head_lag_seconds{aggregator="serialized", region="us-east"}[15m]) == 0 and count_over_time(head_lag_seconds{aggregator="serialized", region="us-east"}[15m]) > 5 and on(chain, region) sum by (chain, region) (changes(head_lag_seconds{region="us-east"}[15m])) > 5 and on(aggregator, region) avg_over_time(ws_connected{aggregator="serialized", region="us-east", chain=""}[15m]) > 0.8) + - region: eu-west + p50: avg by (aggregator) (quantile_over_time(0.50, head_lag_seconds{aggregator="serialized", region="eu-west"}[24h])) * 1000 + series: avg_over_time(head_lag_seconds{aggregator="serialized", region="eu-west"}[1h]) * 1000 unless (changes(head_lag_seconds{aggregator="serialized", region="eu-west"}[15m]) == 0 and count_over_time(head_lag_seconds{aggregator="serialized", region="eu-west"}[15m]) > 5 and on(chain, region) sum by (chain, region) (changes(head_lag_seconds{region="eu-west"}[15m])) > 5 and on(aggregator, region) avg_over_time(ws_connected{aggregator="serialized", region="eu-west", chain=""}[15m]) > 0.8) + - region: ap-southeast + p50: avg by (aggregator) (quantile_over_time(0.50, head_lag_seconds{aggregator="serialized", region="sgp"}[24h])) * 1000 + series: avg_over_time(head_lag_seconds{aggregator="serialized", region="sgp"}[1h]) * 1000 unless (changes(head_lag_seconds{aggregator="serialized", region="sgp"}[15m]) == 0 and count_over_time(head_lag_seconds{aggregator="serialized", region="sgp"}[15m]) > 5 and on(chain, region) sum by (chain, region) (changes(head_lag_seconds{region="sgp"}[15m])) > 5 and on(aggregator, region) avg_over_time(ws_connected{aggregator="serialized", region="sgp", chain=""}[15m]) > 0.8) + - slug: geckoterminal name: GeckoTerminal tag: REST feed diff --git a/benchmarks/asset-registry-coverage.yml b/benchmarks/asset-registry-coverage.yml index c29a75d7f..b252b2001 100644 --- a/benchmarks/asset-registry-coverage.yml +++ b/benchmarks/asset-registry-coverage.yml @@ -95,6 +95,19 @@ providers: sample_size: networks_supported_total{provider="coingecko"} series: networks_supported_total{provider="coingecko"} + - slug: serialized + name: Serialized + tag: Own indexers, 18 EVM + Solana + formula: "Count of chains returned as live by Serialized's `/v1/meta/chains` endpoint, refreshed every 6 hours." + queries: + p50: networks_supported_total{provider="serialized"} + p90: networks_supported_total{provider="serialized"} + p99: networks_supported_total{provider="serialized"} + mean: networks_supported_total{provider="serialized"} + success: clamp_max(networks_supported_total{provider="serialized"} > bool 0, 1) + sample_size: networks_supported_total{provider="serialized"} + series: networks_supported_total{provider="serialized"} + - slug: coinpaprika name: CoinPaprika tag: Market-data API asset registry diff --git a/benchmarks/dex-network-coverage.yml b/benchmarks/dex-network-coverage.yml index 198522514..8e7649b3f 100644 --- a/benchmarks/dex-network-coverage.yml +++ b/benchmarks/dex-network-coverage.yml @@ -90,6 +90,19 @@ providers: sample_size: networks_supported_total{provider="geckoterminal"} series: networks_supported_total{provider="geckoterminal"} + - slug: serialized + name: Serialized + tag: Own indexers, 18 EVM + Solana + formula: "Count of chains returned as live by Serialized's `/v1/meta/chains` endpoint; every listed chain carries DEX pool indexing. Refreshed every 6 hours." + queries: + p50: networks_supported_total{provider="serialized"} + p90: networks_supported_total{provider="serialized"} + p99: networks_supported_total{provider="serialized"} + mean: networks_supported_total{provider="serialized"} + success: clamp_max(networks_supported_total{provider="serialized"} > bool 0, 1) + sample_size: networks_supported_total{provider="serialized"} + series: networks_supported_total{provider="serialized"} + - slug: codex name: Codex tag: Defined.fi DEX data API diff --git a/benchmarks/metadata-coverage.yml b/benchmarks/metadata-coverage.yml index 0381ff2f3..41f764c48 100644 --- a/benchmarks/metadata-coverage.yml +++ b/benchmarks/metadata-coverage.yml @@ -156,6 +156,19 @@ providers: sample_size: sum(increase(metadata_coverage_checks_total{provider="codex"}[24h])) series: 100 * sum(rate(metadata_coverage_success_total{provider="codex"}[1h])) / sum(rate(metadata_coverage_checks_total{provider="codex"}[1h])) + - slug: serialized + name: Serialized + tag: REST `/v1/token/metadata` + formula: "Median of hourly coverage rate (populated logo/description/twitter/website fields divided by total field checks) on Serialized `/v1/token/metadata` for fresh tokens, p50 over 24h." + queries: + p50: quantile_over_time(0.50, (100 * sum(rate(metadata_coverage_success_total{provider="serialized"}[1h])) / sum(rate(metadata_coverage_checks_total{provider="serialized"}[1h])))[24h:1h]) + p90: quantile_over_time(0.90, (100 * sum(rate(metadata_coverage_success_total{provider="serialized"}[1h])) / sum(rate(metadata_coverage_checks_total{provider="serialized"}[1h])))[24h:1h]) + p99: quantile_over_time(0.99, (100 * sum(rate(metadata_coverage_success_total{provider="serialized"}[1h])) / sum(rate(metadata_coverage_checks_total{provider="serialized"}[1h])))[24h:1h]) + mean: 100 * sum(rate(metadata_coverage_success_total{provider="serialized"}[24h])) / sum(rate(metadata_coverage_checks_total{provider="serialized"}[24h])) + success: clamp_max(sum(rate(metadata_coverage_checks_total{provider="serialized"}[24h])) / scalar(sum(rate(metadata_coverage_checks_total{provider="mobula"}[24h]))), 1) + sample_size: sum(increase(metadata_coverage_checks_total{provider="serialized"}[24h])) + series: 100 * sum(rate(metadata_coverage_success_total{provider="serialized"}[1h])) / sum(rate(metadata_coverage_checks_total{provider="serialized"}[1h])) + - slug: jupiter name: Jupiter tag: REST `/v6/tokens` (Solana) diff --git a/benchmarks/wallet-labels-coverage.yml b/benchmarks/wallet-labels-coverage.yml index 2cfe912e6..a428f7842 100644 --- a/benchmarks/wallet-labels-coverage.yml +++ b/benchmarks/wallet-labels-coverage.yml @@ -151,6 +151,11 @@ prometheus: # Default is `eoa` because the contract tab is trivially easy for explorers # and saturates near 100%; the EOA tab is where curated entity coverage # actually differentiates providers. +# Scored on contested chains only: stellar, xrp and bitcoin carry a single +# measured provider each, and a cross-chain average that counts them puts a +# provider first for a chain nobody else reported. See spec-schema.ts. +score_scope: contested_chains + dimensions: kind: - { value: eoa, label: EOA } @@ -194,6 +199,19 @@ providers: sample_size: sum(increase(wallet_labels_checks_total{provider="mobula"}[24h])) series: 100 * sum(rate(wallet_labels_success_total{provider="mobula"}[1h])) / sum(rate(wallet_labels_checks_total{provider="mobula"}[1h])) + - slug: serialized + name: Serialized + tag: Identity graph, 18 EVM chains + Solana, API key required + formula: "Share of anchor addresses for which Serialized /v1/wallet/profile returns a non-generic identity name, success_total ÷ checks_total over 24h." + queries: + p50: 100 * sum(increase(wallet_labels_success_total{provider="serialized"}[24h])) / sum(increase(wallet_labels_checks_total{provider="serialized"}[24h])) + p90: 100 * sum(increase(wallet_labels_success_total{provider="serialized"}[24h])) / sum(increase(wallet_labels_checks_total{provider="serialized"}[24h])) + p99: 100 * sum(increase(wallet_labels_success_total{provider="serialized"}[24h])) / sum(increase(wallet_labels_checks_total{provider="serialized"}[24h])) + mean: 100 * sum(increase(wallet_labels_success_total{provider="serialized"}[24h])) / sum(increase(wallet_labels_checks_total{provider="serialized"}[24h])) + success: clamp_min(1 - (sum(increase(wallet_labels_fetch_errors_total{provider="serialized"}[24h])) or vector(0)) / clamp_min(sum(increase(wallet_labels_checks_total{provider="serialized"}[24h])), 1), 0) + sample_size: sum(increase(wallet_labels_checks_total{provider="serialized"}[24h])) + series: 100 * sum(rate(wallet_labels_success_total{provider="serialized"}[1h])) / sum(rate(wallet_labels_checks_total{provider="serialized"}[1h])) + - slug: helius name: Helius tag: Solana specialist, native program graph, API key required diff --git a/docs/methodology/serialized-onboarding-audit.md b/docs/methodology/serialized-onboarding-audit.md new file mode 100644 index 000000000..39fb48004 --- /dev/null +++ b/docs/methodology/serialized-onboarding-audit.md @@ -0,0 +1,412 @@ +# Provider onboarding audit — Serialized (serialized.xyz) + +> **Pre-onboarding evaluation.** Run before Serialized is wired into any live harness, so the +> decision to include or exclude them on each bench is documented and reproducible. +> +> **Version:** v1.1, 2026-09-05 (v1.0 same day; §8 corrected, §16 added). Author: internal. Key used: tenant `OpenChainBench`, +> plan `starter`, keyId `d5511a080aaa`, issued 2026-09-04. + +--- + +## 1. What this document is + +Serialized is a candidate provider for several existing OpenChainBench benchmarks. This file +records the apples-to-apples tests run against them, the exact methodology of each test, the +numbers that came back, and the methodology problems those tests exposed in **our own benches**. + +Every test below replicates the scoring rule of the target bench rather than inventing a new one, +so the numbers are directly comparable to the published leaderboards. + +## 2. Test harness and vantage point + +| Property | Value | +|---|---| +| Host | `ocb-par-main` (the VPS that runs the production harnesses) | +| Rationale | Same egress, same region, same network path as the live monitors. A latency or coverage number taken from a laptop is not comparable to a published bench value. | +| Incumbent credentials | Read from the running `ocb-metadata-coverage` container env, never copied off the box | +| Scripts | `~/serbench/ab.py`, `ab2.py`, `ab3.py`, `ab4.py`, `wsab.py` | +| Date of run | 2026-09-05 | + +**Throttling matters.** Serialized enforces a hard burst cap of 40 in-flight requests per second. +An unthrottled 8-worker pool produced 60 `429 RATE_LIMITED` responses out of 100 anchors and made +their coverage look like 37%. The same test throttled to 12 rps produced 0 errors and 77%. Any +harness that talks to them must rate-limit client-side, and any measurement that does not is wrong. + +## 3. Provider surface + +19 chains: 18 EVM plus Solana. `evm:1`, `evm:56`, `evm:130`, `evm:143`, `evm:196`, `evm:988`, +`evm:1514`, `evm:2741`, `evm:4217`, `evm:4326`, `evm:4663`, `evm:5042`, `evm:8453`, `evm:9745`, +`evm:42161`, `evm:43114`, `evm:57073`, `evm:645749`, `solana`. Audit engine covers the 18 EVM chains. + +Auth is a raw `Authorization` header, no `Bearer` prefix (same convention as Mobula). The +documented `demo.serialized.xyz` server returns 403 outside their docs playground, so there is no +keyless path for a harness. + +## 4. Rate limits and quota (measured, not quoted) + +| Property | Documented | Measured | +|---|---|---| +| Monthly credits (starter) | 150,000 | **1,000,000** on our key | +| Per-minute rate | 1,200 | 1,200 (`x-ratelimit-limit` header) | +| Burst | 40 req/s | Exactly 40. 60/100/150 concurrent all yielded exactly 40× `200` and the rest `429`. Deterministic, no jitter. | +| Sustained | not stated | 891/891 `200` over 60 s at 15 rps, p50 38 ms, p99 67 ms | + +Response headers expose `x-ratelimit-limit`, `x-ratelimit-remaining`, `x-ratelimit-reset` and +`x-credits-remaining`. Good enough to instrument a harness without guessing. + +Streams bill 1 credit per connection-minute. Limits are 5 concurrent connections, 20 subscriptions +per connection and 50 distinct tokens or pools per key. Bench 001 runs 3 regions × 4 chains, which +does not fit inside one key's 5-connection budget: it needs one key per region. + +## 5. Bench 008 — wallet-labels-coverage + +**Replica rule.** Identical to `harnesses/wallet-labels`: the same 178-anchor curated list, filtered +to the 5 chains Serialized covers (100 anchors, 59 contract / 41 EOA); a "hit" is any non-generic +name, using the harness's exact `genericLabel` exclusion set; Mobula queried through +`POST /api/1/wallet/labels` with the same field-precedence (`entityName` → `entityLabels` → `labels`). +Serialized queried through `GET /v1/wallet/profile`, taking the first non-generic of +`displayName` → `ensName` → `basename` → `solName`. + +**Added dimension (not in the bench today):** accuracy. A hit is counted accurate when the returned +label shares a meaningful token with the curated `Hint` for that anchor. + +| Provider | Coverage | Contract | EOA | Accurate | Accurate given hit | p50 | +|---|---|---|---|---|---|---| +| **Serialized** | **77.0%** | 76.3% | 78.0% | **58.0%** | 75.3% | 45 ms | +| Mobula | 59.0% | 54.2% | 65.9% | 44.0% | 74.6% | 34 ms | + +Per chain (coverage / accuracy): + +| Chain | n | Serialized | Mobula | +|---|---|---|---| +| ethereum | 32 | 96.9% / 78.1% | 62.5% / 53.1% | +| bnb | 15 | 80.0% / 60.0% | 80.0% / 46.7% | +| base | 17 | 76.5% / 58.8% | 70.6% / 47.1% | +| arbitrum | 17 | 58.8% / 52.9% | 47.1% / 41.2% | +| solana | 19 | 57.9% / 26.3% | 36.8% / 26.3% | + +**Verdict: include.** Serialized leads on coverage and on absolute accuracy on every chain in scope. + +**Bench flaw this exposed.** 25% of Serialized's hits are wrong (19 of 77). Mobula's ratio is +almost identical (74.6% accurate given hit). The bench scores presence of a non-generic string, so a +personal ENS or `.sol` name registered against a well-known contract counts as a correct entity +label. Concrete cases: Permit2 → `dex.davywoodfi.eth`, Uniswap V3 Router 2 → `factory.vibebet.eth`, +Base USDC → `jakie.base.eth`, Raydium Authority → `bonklanatoken.sol`, BSC USDT → `Fake_Phishing6512`, +OKX 1 → `Bittrex 3`, Bitfinex → `Polygon`. + +This is a pre-existing, provider-neutral gameability hole. It should be fixed **before** Serialized +is published, not after, otherwise the fix looks like a reaction to a new entrant beating the +incumbent. Recommended fix: score against the curated `Hint` (the harness already carries it and +already ignores it), or exclude name-service strings from the hit rule. + +## 6. Bench 004 — metadata-coverage + +**Replica rule.** Same 4 fields as the bench (`logo`, `description`, `twitter`, `website`). Discovery +via GeckoTerminal `new_pools` (an independent third source, so neither provider's own discovery +biases the sample). Both providers queried on the **same token set**, and only tokens that **both** +resolved are scored, so the denominator is identical. + +| Chain | paired n | Serialized | Mobula | +|---|---|---|---| +| solana | 48 | 10.4% | 34.4% | +| base | 37 | 19.6% | 34.5% | +| bsc | 38 | 60.5% | 80.9% | +| **total** | 123 | **28.7%** | **48.8%** | + +Field breakdown: + +| Chain | Field | Serialized | Mobula | +|---|---|---|---| +| solana | logo | 22.9% | 100.0% | +| solana | description | 8.3% | 25.0% | +| solana | twitter | 8.3% | 8.3% | +| solana | website | 2.1% | 4.2% | +| base | logo | 37.8% | 100.0% | +| base | description | 13.5% | 13.5% | +| base | twitter | 18.9% | 16.2% | +| base | website | 8.1% | 8.1% | +| bsc | logo | 78.9% | 100.0% | +| bsc | description | 78.9% | 71.1% | +| bsc | twitter | 78.9% | 76.3% | +| bsc | website | 5.3% | 76.3% | + +**Verdict: include, but fix the logo field first.** + +**Bench flaw this exposed.** Mobula returns `logo` = 100% on all three chains. That is not a data +advantage, it is a URL-shape artifact: Mobula rewrites every logo onto `metadata.mobula.io` at a +deterministic path (`/assets/logos/__
`), so the field is never empty +regardless of whether an image exists. Serialized returns the upstream source URL instead +(`ipfs.io`, `gmgn.ai`, `axiomtrading.axiom-cdn.io`, `pbs.twimg.com`, `flap.sh`). A HEAD check on 12 +distinct Mobula logo URLs resolved 11 and 404'd 1. + +The bench currently measures *"is the field non-empty"*, which any provider can win by construction +by rewriting to its own CDN. It should measure *"does the logo resolve"* (HEAD 200 with an image +content type). Mobula is our own product and it is the beneficiary of the current rule, so this needs +fixing on fairness grounds before a competitor is added to the same leaderboard. + +Excluding the logo field entirely, on the remaining three fields Serialized is level with Mobula on +Base, ahead on BSC description and twitter, and behind on Solana and on BSC website. + +## 7. Benches 005 / 090 — chain-count coverage + +| Bench | Incumbents | Serialized | +|---|---|---| +| 005 asset-registry | CoinGecko 465, CoinPaprika 310, CoinStats 149, Mobula 81 | **19** | +| 090 dex-network | GeckoTerminal 247, Codex 123, Sim by Dune 64, DexPaprika 35 | **19** | + +**Verdict: exclude for now.** Serialized would rank last by a wide margin on both. The metric is +breadth, their product is deliberately narrow-and-deep. Adding them here produces a true but +uninformative row and gives them a reason to refuse every other bench. Revisit only if they ask. + +Note: the GeckoTerminal count returned 100 in this run because the ad-hoc pager stopped early on +rate limit. The production harness value of 247 is the correct one. + +## 8. Bench 001 — aggregator-head-lag + +**Replica rule.** Single process on `ocb-par-main`, two WebSocket connections open simultaneously, +subscribed to the **same three tokens** (BONK / Solana, DEGEN / Base, CAKE / BNB). Serialized: +`wss://api.serialized.xyz/v1/stream`, `subscribe` on channel `trades` with `{chain, address}`. +Mobula: `wss://api.mobula.io`, `fast-trade` with `assetMode: true`. Trades matched by transaction +hash, so every comparison is the same on-chain event seen by both pipelines. 240 s window. + +Note on protocol shape: Serialized's `params.pools` is a comma-separated **string**, not an array, +and `address` is required even when `pools` is supplied. Their trade events carry the hash inside +`data.id` as `:`, not as a `txHash` field, despite the docs naming `txHash` as the +dedup key. + +### Relative arrival, the only comparison free of self-reported timestamps + +| Chain | matched n | p10 | p50 | p90 | Serialized first | +|---|---|---|---|---|---| +| solana | 74 | −288 ms | **−0 ms** | +29 ms | 51% | +| base | 13 | −43 ms | +88 ms | +177 ms | 23% | +| bnb | 0 | — | — | — | Mobula returned no CAKE events in this window | +| **all** | 87 | | **+2 ms** | | **47%** | + +Negative means Serialized delivered the trade first. **It is a dead heat.** Across 87 matched +trades the median difference is 2 ms and the two feeds trade the lead roughly half the time. On +Base, Mobula was actually ahead on 77% of trades despite Serialized running a preconfirmation feed. + +### The finding that matters: providers disagree about when the trade happened + +For the **same transaction hash**, the two providers' own on-chain timestamps differ: + +| Chain | serialized `at` minus mobula `date` | p10 | p50 | p90 | +|---|---|---|---|---| +| solana | | −1,620 ms | **−707 ms** | −353 ms | +| base | | +1,000 ms | **+1,000 ms** | +2,000 ms | + +Consequence, measured directly: + +| Chain | Provider | Self-reported lag p50 | Actually delivered first | +|---|---|---|---| +| solana | Mobula | +0.04 s | 49% | +| solana | Serialized | +0.75 s | 51% | +| base | Serialized | −0.33 s (13/13 negative) | 23% | +| base | Mobula | +0.78 s | 77% | + +Read those two tables together. On Solana, Mobula's self-reported lag is 19× better than +Serialized's, and the two arrive at the same instant. On Base, Serialized's self-reported lag is +negative while Mobula beats it to the wire on three trades out of four. **Any head-lag number built +on a provider's own timestamp is not a latency measurement, it is a measurement of where that +provider chooses to put its clock.** + +**Correction, 2026-09-05 (v1.1).** An earlier draft of this file claimed bench 001 already +references archive nodes and was therefore unaffected. That was wrong: it repeated the spec's +methodology instead of reading the harness. `harnesses/aggregator-head-lag` contains no archive-node +reference at all (`grep -rl "archive|eth_getBlockByNumber|getBlockTime|blockTimestamp"` over +`cmd/` returns nothing). The gauge that feeds the leaderboard is computed from each provider's own +self-reported timestamp: + +```go +// head_lag_monitor.go:211 (Mobula) +onChainTime := time.UnixMilli(trade.Date) // Mobula's own field +totalLagMs := receiveTime.Sub(onChainTime) +// head_lag_monitor.go:707 (Codex) +onChainTime := time.Unix(event.Timestamp, 0) // Codex's own field +``` + +The published spec says otherwise in three places: `methodology[7]` ("Reference: archive nodes per +chain, validated against block hashes"), the FAQ ("The harness holds a live WebSocket subscription +to canonical-tip archive nodes on each chain"), and the per-chain explainers ("Measured against a +canonical archive node"). The documentation and the code disagree, on a live bench that is publicly +cited. That is a defect independent of Serialized and should be resolved before any provider is +added. + +Second code-level issue, `head_lag_monitor.go:219`: + +```go +if totalLagMs < 0 || totalLagMs > 30000 { continue } +``` + +Negative lags are dropped silently. Serialized's Base feed was negative on 13 of 13 sampled trades, +so under this filter its entire Base preconfirmed population would be discarded and its Base sample +would retain only its slowest trades. This is a measurable bias, not a policy question. + +Recommended resolution: make the harness hold its own node subscription per chain and timestamp each +swap on receipt, matching by transaction hash. That is what the spec already claims, so no published +text changes, and it makes the three providers comparable for the first time. + +**Blocking issue: Base preconfirmations.** Serialized emits Base trades from flashblocks +preconfirmations, ahead of the block timestamp they attach to the event. Measured on their stream, +Base events arrive with a **negative** lag versus their own `at` field (p50 −1.86 s, 3/3 negative in +the first sample). Their docs state this explicitly (~2.5 s ahead). + +Measured on Base against Mobula on matched hashes: Serialized reports 13/13 negative self-lag while +losing the actual race 77% of the time. So the preconfirmation feed does **not** currently translate +into earlier delivery on Base, it only translates into an earlier timestamp. That distinction has to +survive into whatever the bench publishes. Options, in order of preference: + +1. Add a `confirmation` dimension (`confirmed` / `preconfirmed`) and rank within it. +2. Clamp negative lag to 0 and footnote it. +3. Exclude Base for Serialized. + +Option 3 is the least honest, because their preconfirmed feed is a real product advantage for a +trading UI. Option 1 is the one that survives a public dispute. + +## 9. Bench 067 — portfolio-chain-coverage + +`GET /v1/wallet/positions` returned `200` on all 19 chains with zero errors. Rows came back on 6 +chains (ethereum 238, bsc 121, hyperevm 23, solana 13, arbitrum 12, avalanche 9) and 0 rows on the +other 13. + +**This test is inconclusive and must not be quoted.** The zero-row chains reflect probe addresses +that hold nothing there, not unsupported chains. Bench 067 compares self-declared coverage against +probe-verified coverage, which requires a curated funded address per chain. That curation is the +work item; the endpoint itself is ready. + +## 10. Bench 102 / 033 — not applicable + +Serialized is not a swap router and returns no quotes. `token-quote-coverage` (102) and +`evm-quote-latency` (033) cannot include them. Their `/v1/pulse` endpoint tracks ~90 launchpads and +is usable as an **alternative discovery source** for bench 102, which is a separate question. + +## 11. Latency, head to head + +REST, identical call shape, 20 samples each, from `ocb-par-main`: + +| Chain | Serialized p50 / p90 | Mobula p50 / p90 | +|---|---|---| +| base | 72 ms / 180 ms | 74 ms / 364 ms | +| solana | 50 ms / 58 ms | 39 ms / 180 ms | +| ethereum | 43 ms / 57 ms | 40 ms / 186 ms | + +Median is a tie. The tail is not: Serialized's p90 is 2× to 3× tighter on every chain. That +consistency is the more defensible claim, and it is not currently measured by any bench. + +## 12. Stretch tests + +| Test | Result | +|---|---| +| Burst threshold | Hard cap at exactly 40 concurrent. 20 and 40 pass clean; 60/100/150 return 40× `200` and the remainder `429` with an explicit `Burst limit: max 40 requests per second` message | +| Sustained 60 s @ 15 rps | 891/891 `200`, p50 38 ms, p99 67 ms, zero degradation | +| `POST /v1/token` batch | 25 items → `200`, 526 ms, 25 rows | +| `POST /v1/token/price` batch | 100 items → `200`, 38 ms, 100 rows | +| Batch over cap | 200 items → `400 INVALID_PARAM`, "must NOT have more than 100 items". Enforced, not silently truncated | +| OHLCV page cap | `limit` ≤ 500, enforced with a clear `400` | +| OHLCV history depth | 1s → 0.01 d, 1m → 0.35 d, 5m → 1.73 d, 1h → 20.8 d, 1d → 499 d (back to 2025-04-24), 1w → 973 d (back to 2024-01-04) | +| Trades pagination | 10 cursor pages, 1,000 trades in 6.4 s, no gaps or repeats | +| Error contract | `INVALID_CHAIN`, `INVALID_PARAM`, `NOT_FOUND`, `UNAUTHORIZED`, `RATE_LIMITED` all machine-readable and correct for the case | + +## 13. Defects found + +| # | Endpoint | Symptom | +|---|---|---| +| 1 | `GET /v1/wallet/equity/history` | `503 UPSTREAM_ERROR` after a 10 s hang, reproduced twice | +| 2 | `GET /v1/wallet/transfers` | 6.5 s response on a routine call. Not benchmarkable as-is | +| 3 | `GET /v1/token/trades`, `/stats`, `/dev-tokens` | `404` for the native wrapped mint (`So111…112`). Native is treated as a quote asset, never as a token. Any harness iterating a standard basket will hit this | +| 4 | Parameter naming | Three conventions on one API: `/v1/pulse` takes `chains` (plural), `/v1/wallet/*` takes `wallet`, `/v1/wallet/profile` takes `address` | + +## 14. Third-party sourcing + +Worth knowing before any commercial discussion, neutral observation either way: + +- Their token `iconUrl` values are upstream URLs from `cdn.dexscreener.com`, `ipfs.io`, + `raw.githubusercontent.com`, `arweave.net`, `gmgn.ai`, `axiomtrading.axiom-cdn.io`. +- Their wallet-profile entity avatars are served from `metadata.mobula.io`, our own CDN. + +## 15. Recommended sequence + +1. Fix the bench 004 logo rule (resolve-check instead of presence-check) and the bench 008 hit rule + (score against the curated hint). Both are provider-neutral fairness fixes and both should land + before a new entrant appears on those leaderboards. +2. Onboard Serialized to bench 004 and bench 008. Both are 1:1 endpoint mappings. +3. Bench 001: they are level with Mobula on wall-clock delivery (p50 +2 ms over 87 matched trades), + so they belong on the leaderboard. Decide the Base preconfirmation policy first, and keep the + archive-node reference: this audit showed self-reported timestamps disagree by up to 1.6 s on the + same transaction. +4. Curate funded probe addresses per chain for bench 067, then onboard. +5. Leave 005 and 090 alone unless they ask. +6. Consider a new token-security bench, where their `/v1/token/security` (18 fields) and + `/v1/audit/contract` are a genuine differentiator rather than a last-place row. + +Every onboarding needs a `docker build --no-cache` of the materialize-worker on `ocb-par-main` +after the harness change, or the new provider will not appear. + + +## 16. Follow-up tests, 2026-09-05 + +### 16.1 Bench 067, now conclusive + +The earlier §9 result was inconclusive because it used the wrong probe address. The harness already +pins canonical ones in `registry.go`: EVM `0xF977...aceC` (Binance 8), Solana `9WzDX...WWM`, with a +$1 USD floor. Re-run verbatim against those: + +| Metric | Serialized | +|---|---| +| listed (`/v1/meta/chains`) | 19 | +| verified (returned a > $1 balance) | **5** | +| errors | 0 | +| total probe latency, 19 calls | 1,696 ms | + +Verified: ethereum (177 positions, $72.7M), bsc (93, $43.9M), base (27, $5.1M), arbitrum (12, +$3.5M), solana (1, $12). The other 14 chains returned zero rows because Binance 8 holds nothing +there, which is the harness's own "untestable residue" (`listed - probed`), not an indexer failure. + +Published leaderboard: CoinStats 127, Mobula 50, Zerion 42, Moralis 15. Serialized would rank +last at 5. Verdict: addable and now measurable, but it is a third breadth metric and a third last +place. Their `verified / probed` ratio is 5/5, which the bench exposes as a separate series and is +the only flattering read available. + +### 16.2 Negative capability probes + +Confirmed by request rather than by reading docs. Every path returns `404 NOT_FOUND`: +`/v1/wallet/nfts`, `/v1/nft/collection`, `/v1/nfts`, `/v1/swap/quote`, `/v1/quote`, `/v1/route`, +`/v1/bridge/quote`. Benches 033, 102, `nft-collection-metadata`, `bridge-fee` and +`bridge-quote-latency` are definitively out. + +### 16.3 A real pricing defect: BONK is 5.2x wrong + +| Source | BONK price | +|---|---| +| Mobula | 3.3097e-06 | +| DexScreener (Orca, $305,835 liquidity) | 3.309e-06 | +| GeckoTerminal | 3.309731e-06 | +| **Serialized** | **6.3314e-07** | + +Three independent sources agree; Serialized is low by a factor of 5.2, and reports a $55.7M market +cap against a real ~$290M. + +Root cause is visible in their own response. `/v1/token/pools?chain=solana&address=DezXAZ...` ranks +`Gx1WGimRY3jF...` first with liquidity 4,339, and the deep Orca pool everyone else prices from is +absent from the list entirely. Their own ranks 2 and 3 quote ~3.18e-08 and ~3.20e-08 native against +rank 1 at 6.09e-09, so the pool list is internally inconsistent by the same 5x. This is pool +discovery missing the main market, not a decimals bug (`decimals: 5` is correct for BONK). + +Worth raising with them directly: a top-100 token mispriced 5x is a bigger problem for their +prospects than any leaderboard position. + +### 16.4 Cross-API price accuracy as a new bench: not proven + +Two attempts, neither conclusive, recorded so nobody repeats them: + +1. Basket from a DexScreener search returned eight distinct addresses all symbolled "SOL", i.e. + impostor tokens rather than eight real assets. Result discarded. +2. Basket from GeckoTerminal top pools (28 distinct tokens) gated on DexScreener and GeckoTerminal + agreeing within 200 bps. Only 3 tokens survived, because GeckoTerminal returned no price for 25 + of them. n=3 proves nothing. + +The idea remains the most promising new bench for this vertical, and the BONK case shows the signal +is real. But it cannot be built on another aggregator as reference: the reference has to be computed +from on-chain reserves of the deepest pool over an RPC we control, which is the actual work and the +actual reason the bench would be defensible. diff --git a/harnesses/metadata-coverage/cmd/script/config.go b/harnesses/metadata-coverage/cmd/script/config.go index a3fd51143..57fe834e2 100644 --- a/harnesses/metadata-coverage/cmd/script/config.go +++ b/harnesses/metadata-coverage/cmd/script/config.go @@ -10,6 +10,7 @@ import ( type Config struct { CoinGeckoAPIKey string MobulaAPIKey string + SerializedAPIKey string DefinedSessionCookie string MonitorRegion string // Deployment region: us-west, us-east, singapore, etc. MobulaWSURL string // Mobula fast-trade WebSocket endpoint (allows staging to use EU-specific cluster) @@ -21,6 +22,7 @@ func loadEnv() (*Config, error) { // First, try to load from environment variables (for production/Railway) config.CoinGeckoAPIKey = strings.TrimSpace(os.Getenv("COINGECKO_API_KEY")) config.MobulaAPIKey = strings.TrimSpace(os.Getenv("MOBULA_API_KEY")) + config.SerializedAPIKey = strings.TrimSpace(os.Getenv("SERIALIZED_API_KEY")) config.DefinedSessionCookie = strings.TrimSpace(os.Getenv("DEFINED_SESSION_COOKIE")) config.MonitorRegion = strings.TrimSpace(os.Getenv("MONITOR_REGION")) config.MobulaWSURL = strings.TrimSpace(os.Getenv("MOBULA_WS_URL")) diff --git a/harnesses/metadata-coverage/cmd/script/metadata_coverage_monitor.go b/harnesses/metadata-coverage/cmd/script/metadata_coverage_monitor.go index 106898ecd..57e555301 100644 --- a/harnesses/metadata-coverage/cmd/script/metadata_coverage_monitor.go +++ b/harnesses/metadata-coverage/cmd/script/metadata_coverage_monitor.go @@ -65,18 +65,20 @@ type ProviderCoverage struct { // MetadataCoverageStats holds overall stats type MetadataCoverageStats struct { - mu sync.Mutex - Mobula ProviderCoverage - Codex ProviderCoverage - Jupiter ProviderCoverage - LastPrint time.Time + mu sync.Mutex + Mobula ProviderCoverage + Codex ProviderCoverage + Jupiter ProviderCoverage + Serialized ProviderCoverage + LastPrint time.Time } var ( coverageStats = &MetadataCoverageStats{ - Mobula: ProviderCoverage{Provider: "mobula"}, - Codex: ProviderCoverage{Provider: "codex"}, - Jupiter: ProviderCoverage{Provider: "jupiter"}, + Mobula: ProviderCoverage{Provider: "mobula"}, + Codex: ProviderCoverage{Provider: "codex"}, + Jupiter: ProviderCoverage{Provider: "jupiter"}, + Serialized: ProviderCoverage{Provider: "serialized"}, } tokenQueue = make(chan TokenToCheck, 500) metadataClient = &http.Client{Timeout: 10 * time.Second} @@ -194,12 +196,12 @@ type CodexTokenResponse struct { // CodexEnhancedToken matches the EnhancedToken type from Codex API type CodexEnhancedToken struct { - Address string `json:"address"` - Name string `json:"name"` - Symbol string `json:"symbol"` - Decimals int `json:"decimals"` - NetworkID int `json:"networkId"` - Info *CodexTokenInfo `json:"info"` + Address string `json:"address"` + Name string `json:"name"` + Symbol string `json:"symbol"` + Decimals int `json:"decimals"` + NetworkID int `json:"networkId"` + Info *CodexTokenInfo `json:"info"` SocialLinks *CodexSocialLinks `json:"socialLinks"` } @@ -215,11 +217,11 @@ type CodexTokenInfo struct { // CodexSocialLinks contains social media links for the token type CodexSocialLinks struct { - Twitter string `json:"twitter"` - Website string `json:"website"` - Telegram string `json:"telegram"` - Discord string `json:"discord"` - Github string `json:"github"` + Twitter string `json:"twitter"` + Website string `json:"website"` + Telegram string `json:"telegram"` + Discord string `json:"discord"` + Github string `json:"github"` } func getCodexNetworkID(chainID string) int { @@ -553,6 +555,8 @@ func updateStats(provider string, fields MetadataFields) { stats = &coverageStats.Codex case "jupiter": stats = &coverageStats.Jupiter + case "serialized": + stats = &coverageStats.Serialized default: return } @@ -601,7 +605,7 @@ func printCoverageStats() { fmt.Printf("║ Provider │ Checks │ Logo │ Name │ Symbol│ Desc │Twitter│Website│Telegram│ Errors │\n") fmt.Printf("╠══════════════════════════════════════════════════════════════════════════════╣\n") - for _, stats := range []*ProviderCoverage{&coverageStats.Mobula, &coverageStats.Codex, &coverageStats.Jupiter} { + for _, stats := range []*ProviderCoverage{&coverageStats.Mobula, &coverageStats.Codex, &coverageStats.Jupiter, &coverageStats.Serialized} { if stats.TotalChecks == 0 { fmt.Printf("║ %-8s │ %6d │ - │ - │ - │ - │ - │ - │ - │ %6d ║\n", stats.Provider, stats.TotalChecks, stats.ErrorCount) @@ -691,6 +695,23 @@ func checkTokenMetadata(token TokenToCheck, config *Config) { RecordMetadataLatency("jupiter", chainName, jupiterResult.ResponseTimeMs, config.MonitorRegion) } + // Check Serialized (18 EVM chains + Solana; skipped elsewhere) + var serializedResult MetadataFields + if _, supported := serializedChainID(token.ChainID); supported { + serializedResult = checkSerializedMetadata(token, config.SerializedAPIKey) + if serializedResult.Error != "" { + fmt.Printf("[META][SERIALIZED][%s] %s | %s | err=%s\n", + chainName, token.Symbol, token.Address, serializedResult.Error) + } + updateStats("serialized", serializedResult) + + RecordMetadataCoverage("serialized", chainName, "logo", serializedResult.HasLogo, config.MonitorRegion) + RecordMetadataCoverage("serialized", chainName, "description", serializedResult.HasDescription, config.MonitorRegion) + RecordMetadataCoverage("serialized", chainName, "twitter", serializedResult.HasTwitter, config.MonitorRegion) + RecordMetadataCoverage("serialized", chainName, "website", serializedResult.HasWebsite, config.MonitorRegion) + RecordMetadataLatency("serialized", chainName, serializedResult.ResponseTimeMs, config.MonitorRegion) + } + // Single condensed log line boolToIcon := func(b bool) string { if b { @@ -709,11 +730,17 @@ func checkTokenMetadata(token TokenToCheck, config *Config) { // without cross-referencing logs. Address goes after symbol; 4 boolean // columns per provider so website is visible alongside logo/desc/twitter // (the page renders 4 fields, the prior 3-column line hid that one). - fmt.Printf("[META] %s/%s %s | M:%s%s%s%s | C:%s%s%s%s | J:%s\n", + serializedCols := "----" + if _, supported := serializedChainID(token.ChainID); supported { + serializedCols = boolToIcon(serializedResult.HasLogo) + boolToIcon(serializedResult.HasDescription) + + boolToIcon(serializedResult.HasTwitter) + boolToIcon(serializedResult.HasWebsite) + } + + fmt.Printf("[META] %s/%s %s | M:%s%s%s%s | C:%s%s%s%s | J:%s | S:%s\n", token.Symbol, chainName, token.Address, boolToIcon(mobulaResult.HasLogo), boolToIcon(mobulaResult.HasDescription), boolToIcon(mobulaResult.HasTwitter), boolToIcon(mobulaResult.HasWebsite), boolToIcon(codexResult.HasLogo), boolToIcon(codexResult.HasDescription), boolToIcon(codexResult.HasTwitter), boolToIcon(codexResult.HasWebsite), - jupiterLogo) + jupiterLogo, serializedCols) // Print stats every 50 checks (reduced from 10) coverageStats.mu.Lock() @@ -808,4 +835,3 @@ func runMetadataCoverageMonitor(config *Config, stopChan <-chan struct{}) { } } } - diff --git a/harnesses/metadata-coverage/cmd/script/serialized_rest_monitor.go b/harnesses/metadata-coverage/cmd/script/serialized_rest_monitor.go new file mode 100644 index 000000000..812b134ea --- /dev/null +++ b/harnesses/metadata-coverage/cmd/script/serialized_rest_monitor.go @@ -0,0 +1,162 @@ +package main + +import ( + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "strings" + "sync" + "time" +) + +// ============================================================================ +// Serialized — token metadata coverage +// +// GET /v1/token/metadata?chain=&address= returns the four +// canonical fields this bench scores, under different names than Mobula +// and Codex: +// +// logo -> iconUrl +// description -> description +// twitter -> twitterUrl +// website -> websiteUrl +// +// Chain ids are already in the bench's own shape ("solana", "evm:56", +// "evm:8453"), so no translation table is needed beyond normalising the +// legacy "solana:solana" form that Pulse V2 sometimes emits. +// +// One asymmetry worth knowing when reading the leaderboard: Serialized +// returns the *upstream* icon URL (ipfs.io, cdn.dexscreener.com, twimg, +// launchpad CDNs) while Mobula rewrites every logo onto its own CDN at a +// deterministic path, so Mobula's logo field is non-empty by construction. +// The bench currently scores "field non-empty", not "image resolves". +// See docs/methodology/serialized-onboarding-audit.md §6. +// ============================================================================ + +const serializedTokenMetadataURL = "https://api.serialized.xyz/v1/token/metadata" + +// Serialized enforces a hard burst cap of 40 requests per second per key +// and returns 429 above it. The queue-driven monitor can burst well past +// that during a launch spike, which would show up as coverage loss rather +// than as a rate-limit error. Pace the calls at a fixed floor instead. +var ( + serializedMetaMu sync.Mutex + serializedMetaLast time.Time +) + +const serializedMetaMinInterval = 60 * time.Millisecond // ~16 rps against a 40 rps cap + +func serializedMetaThrottle() { + serializedMetaMu.Lock() + defer serializedMetaMu.Unlock() + if wait := time.Until(serializedMetaLast.Add(serializedMetaMinInterval)); wait > 0 { + time.Sleep(wait) + } + serializedMetaLast = time.Now() +} + +// serializedChainID normalises the bench's chain id to what Serialized +// accepts. Returns false when the chain is outside their coverage, so the +// caller skips the check instead of recording a miss. +func serializedChainID(chainID string) (string, bool) { + c := chainID + if c == "solana:solana" { + c = "solana" + } + if c == "solana" { + return c, true + } + if !strings.HasPrefix(c, "evm:") { + return "", false + } + // 18 EVM chains, live as of onboarding (2026-09-05). + switch c { + case "evm:1", "evm:56", "evm:130", "evm:143", "evm:196", "evm:988", + "evm:1514", "evm:2741", "evm:4217", "evm:4326", "evm:4663", + "evm:5042", "evm:8453", "evm:9745", "evm:42161", "evm:43114", + "evm:57073", "evm:645749": + return c, true + } + return "", false +} + +type SerializedTokenMetadataResponse struct { + Data struct { + Name string `json:"name"` + Symbol string `json:"symbol"` + IconURL string `json:"iconUrl"` + Description string `json:"description"` + TwitterURL string `json:"twitterUrl"` + WebsiteURL string `json:"websiteUrl"` + TelegramURL string `json:"telegramUrl"` + } `json:"data"` +} + +func checkSerializedMetadata(token TokenToCheck, apiKey string) MetadataFields { + result := MetadataFields{} + + chain, ok := serializedChainID(token.ChainID) + if !ok { + result.Error = "chain_unsupported" + return result + } + if apiKey == "" { + result.Error = "no_api_key" + return result + } + + serializedMetaThrottle() + + params := url.Values{} + params.Add("chain", chain) + params.Add("address", token.Address) + + req, err := http.NewRequest("GET", fmt.Sprintf("%s?%s", serializedTokenMetadataURL, params.Encode()), nil) + if err != nil { + result.Error = fmt.Sprintf("request_create_error: %v", err) + return result + } + // Raw key, no Bearer prefix — a prefixed key is rejected with 401. + req.Header.Set("Authorization", apiKey) + req.Header.Set("Accept", "application/json") + + startTime := time.Now() + resp, err := metadataClient.Do(req) + result.ResponseTimeMs = float64(time.Since(startTime).Milliseconds()) + if err != nil { + result.Error = fmt.Sprintf("request_error: %v", err) + return result + } + defer resp.Body.Close() + + if resp.StatusCode != 200 { + result.Error = fmt.Sprintf("status_%d", resp.StatusCode) + return result + } + + body, err := io.ReadAll(resp.Body) + if err != nil { + result.Error = fmt.Sprintf("read_error: %v", err) + return result + } + + var response SerializedTokenMetadataResponse + if err := json.Unmarshal(body, &response); err != nil { + result.Error = fmt.Sprintf("parse_error: %v", err) + return result + } + + d := response.Data + result.HasName = d.Name != "" + result.HasSymbol = d.Symbol != "" + result.HasLogo = d.IconURL != "" + result.LogoURL = d.IconURL + result.HasDescription = d.Description != "" + result.HasTwitter = d.TwitterURL != "" + result.HasWebsite = d.WebsiteURL != "" + result.HasTelegram = d.TelegramURL != "" + + return result +} diff --git a/harnesses/network-coverage/cmd/script/config.go b/harnesses/network-coverage/cmd/script/config.go index 83d27ab8d..99c7310f2 100644 --- a/harnesses/network-coverage/cmd/script/config.go +++ b/harnesses/network-coverage/cmd/script/config.go @@ -13,6 +13,7 @@ type Config struct { CodexSessionCookie string // fallback path: mint JWT from Defined.fi cookie DefinedTokenURL string // optional: pre-minted JWT sidecar CoinStatsAPIKey string + SerializedAPIKey string SimDuneAPIKey string // optional — Sim's public endpoint works keyless, but a key avoids rate limits HTTPProxy string RefreshInterval time.Duration @@ -26,6 +27,7 @@ func loadConfig() *Config { CodexSessionCookie: os.Getenv("DEFINED_SESSION_COOKIE"), DefinedTokenURL: os.Getenv("DEFINED_TOKEN_SERVICE_URL"), CoinStatsAPIKey: os.Getenv("COINSTATS_API_KEY"), + SerializedAPIKey: os.Getenv("SERIALIZED_API_KEY"), SimDuneAPIKey: os.Getenv("SIM_DUNE_API_KEY"), HTTPProxy: os.Getenv("HTTP_PROXY"), RefreshInterval: 6 * time.Hour, @@ -47,8 +49,8 @@ func loadConfig() *Config { } else if c.CodexSessionCookie != "" { codexAuth = "cookie+mint" } - fmt.Printf("Config: refresh=%v, testnets=%v, mobula_key=%v, codex=%s, coinstats_key=%v, sim_dune_key=%v\n", + fmt.Printf("Config: refresh=%v, testnets=%v, mobula_key=%v, codex=%s, coinstats_key=%v, sim_dune_key=%v, serialized_key=%v\n", c.RefreshInterval, c.IncludeTestnets, c.MobulaAPIKey != "", codexAuth, - c.CoinStatsAPIKey != "", c.SimDuneAPIKey != "") + c.CoinStatsAPIKey != "", c.SimDuneAPIKey != "", c.SerializedAPIKey != "") return c } diff --git a/harnesses/network-coverage/cmd/script/main.go b/harnesses/network-coverage/cmd/script/main.go index 885a9ffad..20693522d 100644 --- a/harnesses/network-coverage/cmd/script/main.go +++ b/harnesses/network-coverage/cmd/script/main.go @@ -76,6 +76,7 @@ func fetchAll(cfg *Config) { {"coinstats", fetchCoinStats}, {"coingecko", fetchCoinGecko}, {"dexpaprika", fetchDexPaprika}, + {"serialized", fetchSerialized}, } var wg sync.WaitGroup diff --git a/harnesses/network-coverage/cmd/script/serialized.go b/harnesses/network-coverage/cmd/script/serialized.go new file mode 100644 index 000000000..9ad325b64 --- /dev/null +++ b/harnesses/network-coverage/cmd/script/serialized.go @@ -0,0 +1,80 @@ +package main + +import ( + "encoding/json" + "fmt" + "io" + "net/http" + "time" +) + +// Serialized publishes its chain list at GET /v1/meta/chains. The endpoint +// is free (0 credits) and returns one row per chain with a `status` field. +// +// The same list answers both benches this harness feeds: Serialized runs its +// own indexers and does not separate "chains where we know tokens" from +// "chains where we index DEX pools" — every listed chain carries both. So +// the count is identical on bench 005 and bench 090 by construction, which +// is worth knowing when reading the two leaderboards side by side. +const serializedChainsURL = "https://api.serialized.xyz/v1/meta/chains" + +type serializedChain struct { + Chain string `json:"chain"` // "evm:8453" or "solana" + Name string `json:"name"` + Slug string `json:"slug"` + Family string `json:"family"` + Status string `json:"status"` +} + +type serializedChainsResponse struct { + Data []serializedChain `json:"data"` +} + +func fetchSerialized(cfg *Config) ProviderResult { + res := ProviderResult{Provider: "serialized"} + if cfg.SerializedAPIKey == "" { + res.Err = "missing_api_key" + return res + } + + client := &http.Client{Timeout: 15 * time.Second} + req, _ := http.NewRequest("GET", serializedChainsURL, nil) + // Raw key, no Bearer prefix — a prefixed key is rejected with 401. + req.Header.Set("Authorization", cfg.SerializedAPIKey) + req.Header.Set("Accept", "application/json") + + resp, err := client.Do(req) + if err != nil { + res.Err = fmt.Sprintf("request_error: %v", err) + return res + } + defer resp.Body.Close() + body, _ := io.ReadAll(resp.Body) + + if resp.StatusCode != 200 { + res.Err = fmt.Sprintf("status_%d", resp.StatusCode) + return res + } + + var parsed serializedChainsResponse + if err := json.Unmarshal(body, &parsed); err != nil { + res.Err = fmt.Sprintf("parse_error: %v", err) + return res + } + + for _, c := range parsed.Data { + // Only chains the provider declares live. Everything on this + // endpoint is mainnet, so no testnet filter is needed, but a + // future "beta"/"deprecated" status must not inflate the count. + if c.Status != "live" { + continue + } + res.Networks = append(res.Networks, Network{ + ChainID: c.Chain, + Slug: c.Slug, + Name: c.Name, + }) + } + + return res +} diff --git a/harnesses/wallet-labels/.env.example b/harnesses/wallet-labels/.env.example index dc1a7d7da..0d1d17414 100644 --- a/harnesses/wallet-labels/.env.example +++ b/harnesses/wallet-labels/.env.example @@ -10,6 +10,9 @@ MORALIS_API_KEY= # Helius (Solana) HELIUS_API_KEY= +# Serialized (18 EVM chains + Solana). Raw key, no Bearer prefix. +SERIALIZED_API_KEY= + # Tuning WALLET_LABELS_CHECK_DELAY_SECONDS=30 WALLET_LABELS_WORKERS=8 diff --git a/harnesses/wallet-labels/cmd/script/config.go b/harnesses/wallet-labels/cmd/script/config.go index 134cc7417..2bb00dd97 100644 --- a/harnesses/wallet-labels/cmd/script/config.go +++ b/harnesses/wallet-labels/cmd/script/config.go @@ -11,27 +11,29 @@ import ( // printed in full — only their length, so misconfigured deploys // fail loudly without leaking material. type Config struct { - MobulaAPIKey string - MoralisAPIKey string - HeliusAPIKey string + MobulaAPIKey string + MoralisAPIKey string + HeliusAPIKey string + SerializedAPIKey string - CheckDelay time.Duration - Workers int - QueueSize int - PromListen string - LogsToken string + CheckDelay time.Duration + Workers int + QueueSize int + PromListen string + LogsToken string } func loadConfig() *Config { c := &Config{ - MobulaAPIKey: os.Getenv("MOBULA_API_KEY"), - MoralisAPIKey: os.Getenv("MORALIS_API_KEY"), - HeliusAPIKey: os.Getenv("HELIUS_API_KEY"), - CheckDelay: parseDurationSec("WALLET_LABELS_CHECK_DELAY_SECONDS", 30), - Workers: parseInt("WALLET_LABELS_WORKERS", 8), - QueueSize: parseInt("WALLET_LABELS_QUEUE_SIZE", 2000), - PromListen: envDefault("PROM_LISTEN_ADDR", ":2112"), - LogsToken: os.Getenv("LOGS_TOKEN"), + MobulaAPIKey: os.Getenv("MOBULA_API_KEY"), + MoralisAPIKey: os.Getenv("MORALIS_API_KEY"), + HeliusAPIKey: os.Getenv("HELIUS_API_KEY"), + SerializedAPIKey: os.Getenv("SERIALIZED_API_KEY"), + CheckDelay: parseDurationSec("WALLET_LABELS_CHECK_DELAY_SECONDS", 30), + Workers: parseInt("WALLET_LABELS_WORKERS", 8), + QueueSize: parseInt("WALLET_LABELS_QUEUE_SIZE", 2000), + PromListen: envDefault("PROM_LISTEN_ADDR", ":2112"), + LogsToken: os.Getenv("LOGS_TOKEN"), } fmt.Println("=== Wallet Labels Coverage Monitor ===") @@ -42,6 +44,7 @@ func loadConfig() *Config { fmt.Printf(" Mobula key set: %v (len=%d)\n", c.MobulaAPIKey != "", len(c.MobulaAPIKey)) fmt.Printf(" Moralis key set: %v (len=%d)\n", c.MoralisAPIKey != "", len(c.MoralisAPIKey)) fmt.Printf(" Helius key set: %v (len=%d)\n", c.HeliusAPIKey != "", len(c.HeliusAPIKey)) + fmt.Printf(" Serialized key set: %v (len=%d)\n", c.SerializedAPIKey != "", len(c.SerializedAPIKey)) fmt.Println() return c diff --git a/harnesses/wallet-labels/cmd/script/main.go b/harnesses/wallet-labels/cmd/script/main.go index 0e3d5071c..abdfea2c2 100644 --- a/harnesses/wallet-labels/cmd/script/main.go +++ b/harnesses/wallet-labels/cmd/script/main.go @@ -53,6 +53,7 @@ func buildProviders(cfg *Config) []Provider { NewMobulaProvider(cfg.MobulaAPIKey), NewMoralisProvider(cfg.MoralisAPIKey), NewHeliusProvider(cfg.HeliusAPIKey), + NewSerializedProvider(cfg.SerializedAPIKey), NewBlockscoutProvider(), NewOLIProvider(), NewTonAPIProvider(), diff --git a/harnesses/wallet-labels/cmd/script/serialized.go b/harnesses/wallet-labels/cmd/script/serialized.go new file mode 100644 index 000000000..b35c79545 --- /dev/null +++ b/harnesses/wallet-labels/cmd/script/serialized.go @@ -0,0 +1,129 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "sync" + "time" +) + +// Serialized exposes an identity graph rather than a pure entity-label +// service: GET /v1/wallet/profile returns a display name, ENS / Basename / +// .sol resolution, socials and linked wallets for an address. +// +// We read it with the same precedence rule every other provider gets — +// first non-generic name signal wins — so the bench compares like with +// like. Note for whoever reads the leaderboard: because `displayName` +// can resolve to a personal name-service record rather than a curated +// entity, a share of Serialized's hits name the *holder* of an address +// rather than the *entity* behind it (a measured ~25% of hits at +// onboarding time, against ~25% for Mobula on the same sample). That is +// a property of the bench's hit rule, not of this provider, and the fix +// belongs in the scoring rule for everyone at once. See +// docs/methodology/serialized-onboarding-audit.md §5. +type SerializedProvider struct { + apiKey string +} + +func NewSerializedProvider(key string) *SerializedProvider { + return &SerializedProvider{apiKey: key} +} + +func (p *SerializedProvider) Name() string { return "serialized" } + +// serializedChains are the anchor-list chains Serialized indexes. Their +// full surface is 18 EVM chains plus Solana; the ones below are the +// intersection with the curated anchor sample. Chains outside this set +// are skipped rather than counted as misses, same as every other +// chain-restricted provider in this harness. +var serializedChains = map[string]bool{ + "ethereum": true, + "bnb": true, + "base": true, + "arbitrum": true, + "solana": true, +} + +func (p *SerializedProvider) Supports(chain string) bool { return serializedChains[chain] } + +// Serialized enforces a hard burst cap of 40 requests per second per key +// and answers anything above it with 429. The harness runs 8 workers with +// sub-100ms responses, which clears that cap easily and silently turns +// coverage into a rate-limit artifact (measured: 60 of 100 anchors lost to +// 429, dropping apparent coverage from 77% to 37%). Serialize the calls at +// a conservative fixed interval instead of relying on worker count. +var ( + serializedMu sync.Mutex + serializedLast time.Time +) + +const serializedMinInterval = 60 * time.Millisecond // ~16 rps, well under the 40 rps cap + +func serializedThrottle() { + serializedMu.Lock() + defer serializedMu.Unlock() + if wait := time.Until(serializedLast.Add(serializedMinInterval)); wait > 0 { + time.Sleep(wait) + } + serializedLast = time.Now() +} + +func (p *SerializedProvider) Lookup(ctx context.Context, chain, address string) LabelResult { + res := LabelResult{Provider: p.Name(), Chain: chain, Address: address} + if !p.Supports(chain) || p.apiKey == "" { + return res + } + + serializedThrottle() + + start := time.Now() + req, _ := http.NewRequestWithContext(ctx, "GET", + "https://api.serialized.xyz/v1/wallet/profile?address="+address, nil) + // Raw key, no Bearer prefix — a prefixed key is rejected with 401. + req.Header.Set("Authorization", p.apiKey) + req.Header.Set("Accept", "application/json") + + resp, err := httpClient.Do(req) + res.LatencyMs = time.Since(start).Milliseconds() + if err != nil { + res.Err = err + return res + } + defer resp.Body.Close() + + if resp.StatusCode != 200 { + res.Err = fmt.Errorf("status_%d", resp.StatusCode) + return res + } + + var body struct { + Data struct { + Profile *struct { + DisplayName string `json:"displayName"` + ENSName string `json:"ensName"` + Basename string `json:"basename"` + SolName string `json:"solName"` + } `json:"profile"` + } `json:"data"` + } + if err := json.NewDecoder(resp.Body).Decode(&body); err != nil { + res.Err = fmt.Errorf("parse: %w", err) + return res + } + if body.Data.Profile == nil { + return res + } + + prof := body.Data.Profile + for _, candidate := range []string{prof.DisplayName, prof.ENSName, prof.Basename, prof.SolName} { + if !genericLabel(candidate) { + res.Label = candidate + res.HasLabel = true + res.Raw = map[string]any{"label": candidate, "source": "wallet_profile"} + break + } + } + return res +} diff --git a/public/logos/serialized-wordmark.svg b/public/logos/serialized-wordmark.svg new file mode 100644 index 000000000..7f3d83a9b --- /dev/null +++ b/public/logos/serialized-wordmark.svg @@ -0,0 +1,15 @@ + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/public/logos/serialized.svg b/public/logos/serialized.svg new file mode 100644 index 000000000..39ae0e54d --- /dev/null +++ b/public/logos/serialized.svg @@ -0,0 +1,5 @@ + + + + + diff --git a/src/app/methodology/page.tsx b/src/app/methodology/page.tsx index 9c6b95b7d..665f47df1 100644 --- a/src/app/methodology/page.tsx +++ b/src/app/methodology/page.tsx @@ -60,6 +60,10 @@ const CONVENTIONS = [ term: "Success rate", body: "Share of requests returning a usable result within the published timeout. The only metric that includes failures.", }, + { + term: "Contested-chain scoring", + body: "A cross-chain average is a mix rather than a comparison: it credits a provider for the chains it happens to be measured on. Benchmarks that declare score_scope: contested_chains in their spec are therefore scored only on chains where at least two providers reported data, and a provider with no such chain is left out of the ranking while staying visible on its own chain tab. The figure shown is the unweighted mean across those chains, so each chain counts once regardless of sample count. This narrows the number rather than the sort order, so the published value is always the one the ranking follows. It does not equalise chain mix entirely: providers are still averaged over the different subsets of contested chains they cover.", + }, { term: "Region normalisation", body: "Where a benchmark is multi-region, the headline figure is the cross-region median. Per-region figures appear on every benchmark page.", diff --git a/src/app/products/[slug]/page.tsx b/src/app/products/[slug]/page.tsx index 713f29ab9..48c273ac6 100644 --- a/src/app/products/[slug]/page.tsx +++ b/src/app/products/[slug]/page.tsx @@ -787,11 +787,13 @@ export default async function ProviderPage({ const catColor = CATEGORY_COLOR[a.benchmark.category]; const hasData = a.rank > 0 && a.result.ms.p50 !== 0; const value = hasData ? fmtUnit(a.result.ms.p50, a.benchmark.unit) : null; - // Per-chain rank chips. Rendered alongside the aggregate rank - // when the bench declares chain dimensions and the provider has - // per-chain ranks populated. Surface text reads e.g. "#1 on - // Solana · #4 on Base · #4 on BNB" so a chain-restricted - // provider can't be passed off as a free cross-chain #1. + // Per-chain leadership chips, rendered alongside the aggregate + // rank when the bench declares chain dimensions. Leaders only: + // a chip means "leads this chain", and its absence means "does + // not lead", never "ranks lower". Reads e.g. "#1 of 4 on + // Ethereum" so a chain-restricted provider can't be passed off + // as a free cross-chain #1, and so a win on a two-provider + // chain isn't dressed up as a win on a crowded one. const chainRanks = a.rankPerChain && a.benchmark.chainDimensions ? a.benchmark.chainDimensions @@ -845,13 +847,10 @@ export default async function ProviderPage({ {chainRanks.map(({ chain, entry }) => ( - #{entry.rank} on {chain.label} + #1{entry.totalRanked > 0 ? ` of ${entry.totalRanked}` : ""} on{" "} + {chain.label} ))}

diff --git a/src/components/bench-appearances-section.tsx b/src/components/bench-appearances-section.tsx index c50c7fc38..de5582033 100644 --- a/src/components/bench-appearances-section.tsx +++ b/src/components/bench-appearances-section.tsx @@ -82,13 +82,10 @@ export async function BenchAppearancesSection({ providerSlug }: Props) { {chainRanks.map(({ chain, entry }) => ( - #{entry.rank} on {chain.label} + #1{entry.totalRanked > 0 ? ` of ${entry.totalRanked}` : ""} on{" "} + {chain.label} ))}

diff --git a/src/data/provider-registry.ts b/src/data/provider-registry.ts index 9abea557b..5d4b620f8 100644 --- a/src/data/provider-registry.ts +++ b/src/data/provider-registry.ts @@ -115,6 +115,47 @@ export const PROVIDER_REGISTRY: Record = { "Independent crypto market data API. Token prices, OHLCV, exchange tickers, and contract/platform lookups across 300+ supported chains. Public free tier with no auth.", twitter: "@coinpaprika", }, + serialized: { + url: "https://serialized.xyz", + description: + "Onchain market data and token security API for trading apps. Own indexers across 18 EVM chains plus Solana, REST plus a single-connection WebSocket for trades and token updates.", + longDescription: + "Serialized runs its own indexers rather than reselling a third-party pipeline, and scopes coverage deliberately narrow: 18 EVM chains plus Solana, weighted toward venues where launchpad and memecoin flow actually lands (Base, BNB, Solana, HyperEVM, Abstract, Monad, MegaETH, Plasma, Tempo, Arc, Robinhood Chain). The surface splits into market data (token details, price, stats, OHLCV from 1s, trades tape, pools, screener, Axiom-style launchpad lifecycle filters), wallet analytics (positions, PnL, closed positions, transfers, funding, an identity graph behind wallet profiles), and token security (holder concentration, sniper and bundler share, LP burn and lock state, plus a separate async contract-audit engine). Streams run over one WebSocket at wss://api.serialized.xyz/v1/stream, billed per connection-minute rather than per message. On Base the trade tape emits preconfirmed flashblock trades ahead of the block timestamp.", + twitter: "@serializedaudit", + docs: "https://docs.serialized.xyz", + chains: [ + "ethereum", + "base", + "bnb", + "arbitrum", + "avalanche", + "solana", + "abstract", + "hyperevm", + "ink", + "story", + "xlayer", + "plasma", + "unichain", + "monad", + "megaeth", + "tempo", + "robinhood", + "arc", + "stable", + ], + features: [ + "Token details, price, windowed stats and OHLCV from 1s to 1M", + "Trades tape with maker resolution, sniper/pro-trader/wash badges", + "Wallet positions, realized and unrealized PnL, funding and transfers", + "Token security: holder concentration, sniper and bundler share, LP burn and lock", + "Async contract audit engine across 18 EVM chains", + "Launchpad lifecycle filters over ~90 launchpads", + "WebSocket streams for trades, token updates and pool updates", + ], + pricing: + "Credit-metered. Free 15k credits/month, Starter 150k, Growth 2M, Enterprise unlimited. Most endpoints cost 1 credit; token/security costs 10 and a fresh contract audit 750.", + }, dexpaprika: { url: "https://dexpaprika.com", description: diff --git a/src/lib/brand.ts b/src/lib/brand.ts index b5d76ef28..bbe214bac 100644 --- a/src/lib/brand.ts +++ b/src/lib/brand.ts @@ -58,6 +58,8 @@ const BRANDS: Record = { // ─── Aggregators / providers (bright, saturated - read on both modes) ─── mobula: { color: "#FF6B35" }, // vivid orange + serialized: { color: "#3D74FF" }, // serialized logo blue - was falling back to the + // palette and drew the same orange as mobula codex: { color: "#84cc16" }, // saturated lime - readable on white + dark geckoterminal: { color: "#8B5CF6" }, // vivid violet (gecko brand) jupiter: { color: "#C7F284" }, // jupiter matrix green (secondary brand) - keeps it diff --git a/src/lib/citation.ts b/src/lib/citation.ts index 198117aea..5d3fe68bd 100644 --- a/src/lib/citation.ts +++ b/src/lib/citation.ts @@ -53,7 +53,13 @@ export function citationCandidates(b: Benchmark): ProviderResult[] { * full field. Applies the same reliability + insufficient-sample * filters as `leader()` so a document that names X as leader ranks X * first in its own list. Sort direction honors the bench's - * `higherIsBetter` flag. */ + * `higherIsBetter` flag. + * + * Ranks on the value alone, deliberately. A bench whose cross-chain + * aggregate would otherwise reward an uncontested chain fixes that by + * declaring `score_scope: contested_chains` in its spec, which narrows + * the value itself (see materialize/load.ts) rather than sorting on a + * key the reader cannot see in the column. */ export function rankedCandidates(b: Benchmark): ProviderResult[] { return [...citationCandidates(b)].sort((a, c) => b.higherIsBetter ? c.ms.p50 - a.ms.p50 : a.ms.p50 - c.ms.p50, diff --git a/src/lib/logo-manifest.ts b/src/lib/logo-manifest.ts index 4c4414ca4..b8d36699c 100644 --- a/src/lib/logo-manifest.ts +++ b/src/lib/logo-manifest.ts @@ -99,6 +99,7 @@ const RAW: Record = { // ─── Providers ─── mobula: "/logos/mobula.svg", + serialized: "/logos/serialized.svg", codex: "/logos/codex.svg", polymarket: "/logos/polymarket.png", "polymarket-us": "/logos/polymarket.png", diff --git a/src/lib/materialize/contested-scope.test.ts b/src/lib/materialize/contested-scope.test.ts new file mode 100644 index 000000000..8d8475ed6 --- /dev/null +++ b/src/lib/materialize/contested-scope.test.ts @@ -0,0 +1,82 @@ +import { describe, expect, test } from "bun:test"; +import { applyContestedChainScope } from "./load"; +import type { ProviderResult } from "@/types/benchmark"; + +function r(slug: string, p50: number): ProviderResult { + return { + slug, + name: slug, + ms: { p50, p90: p50, p99: p50, mean: p50 }, + successRate: 100, + availability: "live", + }; +} + +// Bench 008 as it shipped: stellar, xrp and bitcoin carry one measured +// provider each, so the cross-chain average put two single-chain providers +// first and second above one that led four contested chains. +function fixture() { + const results = [ + r("stellarexpert", 80.08), + r("xrpscan", 79.84), + r("serialized", 76.98), + r("mobula", 46.75), + ]; + const providersPerChain: Record = { + ethereum: ["serialized", "mobula"], + base: ["serialized", "mobula"], + solana: ["serialized", "mobula"], + stellar: ["stellarexpert"], + xrp: ["xrpscan"], + }; + const valuesByChain: Record> = { + ethereum: { serialized: 96.84, mobula: 50.0 }, + base: { serialized: 76.36, mobula: 40.0 }, + solana: { serialized: 57.79, mobula: 30.0 }, + stellar: { stellarexpert: 80.08 }, + xrp: { xrpscan: 79.84 }, + }; + return { results, providersPerChain, valuesByChain }; +} + +describe("contested-chain scoring", () => { + test("the value becomes the mean over contested chains", () => { + const { results, providersPerChain, valuesByChain } = fixture(); + applyContestedChainScope(results, providersPerChain, valuesByChain); + const s = results.find((x) => x.slug === "serialized")!; + // (96.84 + 76.36 + 57.79) / 3 + expect(s.ms.p50).toBeCloseTo(76.9967, 3); + expect(s.ms.mean).toBeCloseTo(76.9967, 3); + expect(results.find((x) => x.slug === "mobula")!.ms.p50).toBeCloseTo(40, 6); + }); + + test("a provider with no contested chain drops out of the ranked field", () => { + const { results, providersPerChain, valuesByChain } = fixture(); + applyContestedChainScope(results, providersPerChain, valuesByChain); + expect(results.find((x) => x.slug === "stellarexpert")!.availability).toBe( + "unavailable", + ); + expect(results.find((x) => x.slug === "xrpscan")!.availability).toBe( + "unavailable", + ); + expect(results.find((x) => x.slug === "serialized")!.availability).toBe("live"); + }); + + test("no contested chain at all leaves every value untouched", () => { + const { results, valuesByChain } = fixture(); + applyContestedChainScope( + results, + { stellar: ["stellarexpert"], xrp: ["xrpscan"] }, + valuesByChain, + ); + expect(results.find((x) => x.slug === "serialized")!.ms.p50).toBe(76.98); + expect(results.every((x) => x.availability === "live")).toBe(true); + }); + + test("an already unavailable provider is left alone", () => { + const { results, providersPerChain, valuesByChain } = fixture(); + results[0].availability = "unavailable"; + applyContestedChainScope(results, providersPerChain, valuesByChain); + expect(results[0].ms.p50).toBe(80.08); + }); +}); diff --git a/src/lib/materialize/load.ts b/src/lib/materialize/load.ts index 3f6bbc527..7b8896fbc 100644 --- a/src/lib/materialize/load.ts +++ b/src/lib/materialize/load.ts @@ -329,15 +329,26 @@ export async function specToBenchmark( const chainSpec = applyDimensionsToSpec(spec, { chain }); const chainLive = await tryLoadLive(chainSpec, true); if (!chainLive) { - return [chain, undefined, undefined, [] as string[]] as const; + return [ + chain, + undefined, + undefined, + [] as string[], + {} as Record, + ] as const; } for (const r of chainLive.results) { if (!r.unresponsive) r.availability = "live"; } const liveForChain = liveProviderResults(chainLive.results); const slugs = liveForChain.map((r) => r.slug); + // Per-provider value on this chain. Kept (not just the leader) + // so `score_scope: contested_chains` can rebuild a value from + // the chains a provider was actually compared on. + const values: Record = {}; + for (const r of liveForChain) values[r.slug.toLowerCase()] = r.ms.p50; if (liveForChain.length === 0) { - return [chain, undefined, undefined, slugs] as const; + return [chain, undefined, undefined, slugs, values] as const; } const sorted = [...liveForChain].sort((a, b) => spec.higher_is_better ? b.ms.p50 - a.ms.p50 : a.ms.p50 - b.ms.p50, @@ -347,20 +358,27 @@ export async function specToBenchmark( sorted[0], sorted[sorted.length - 1], slugs, + values, ] as const; }), ); const bests: Record = {}; const worsts: Record = {}; const providers: Record = {}; - for (const [chain, leader, trailer, slugs] of perChainEntries) { + const valuesByChain: Record> = {}; + for (const [chain, leader, trailer, slugs, values] of perChainEntries) { if (leader) bests[chain] = leader; if (trailer) worsts[chain] = trailer; if (slugs.length > 0) providers[chain] = slugs; + valuesByChain[chain] = values; } if (Object.keys(bests).length > 0) bestPerChain = bests; if (Object.keys(worsts).length > 0) worstPerChain = worsts; if (Object.keys(providers).length > 0) providersPerChain = providers; + + if (spec.score_scope === "contested_chains") { + applyContestedChainScope(live.results, providers, valuesByChain); + } } // Exact per-cell rankings (chain × region) from the spec's single @@ -424,6 +442,50 @@ export async function specToBenchmark( return draftBenchmark(spec, editorial); } +/** + * Rewrite each provider's headline value as its mean over the **contested** + * chains of the bench: those where at least two providers reported data. + * A provider with no contested chain is marked unavailable, which takes it + * out of `liveResults` and therefore out of every ranked surface, while + * leaving it visible on its own chain tab. + * + * Why the value and not the sort order: a cross-chain aggregate is a mix + * rather than a comparison, so it credits a provider for the chains it + * happens to be measured on. Narrowing the number keeps one quantity on + * screen and the ordering follows from it. Ranking on a separate key while + * still displaying the wide aggregate produced a column that did not + * descend (80% shown at rank 4, a 74 ms leader shown at rank 3). + * + * The residual limitation, stated in the methodology: providers are still + * averaged over different subsets of the contested chains, since they do + * not all cover the same ones. It removes the uncontested win, not every + * difference in chain mix. An unweighted mean is used so a chain counts + * once regardless of how many samples it carries. + */ +export function applyContestedChainScope( + results: ProviderResult[], + providersPerChain: Record, + valuesByChain: Record>, +): void { + const contested = Object.keys(providersPerChain).filter( + (chain) => (providersPerChain[chain]?.length ?? 0) >= 2, + ); + if (contested.length === 0) return; + for (const r of results) { + if (r.availability === "unavailable") continue; + const slug = r.slug.toLowerCase(); + const vals = contested + .map((chain) => valuesByChain[chain]?.[slug]) + .filter((v): v is number => typeof v === "number" && v > 0); + if (vals.length === 0) { + r.availability = "unavailable"; + continue; + } + const mean = vals.reduce((a, b) => a + b, 0) / vals.length; + r.ms = { p50: mean, p90: mean, p99: mean, mean }; + } +} + function activeFilterLabels(opts: BenchmarkFilters): Record { const out: Record = {}; for (const [k, v] of Object.entries(opts)) { diff --git a/src/lib/providers.ts b/src/lib/providers.ts index 075c716d4..adde99606 100644 --- a/src/lib/providers.ts +++ b/src/lib/providers.ts @@ -205,11 +205,12 @@ export type ProviderAppearance = { result: ProviderResult; rank: number; totalRanked: number; - /** Per-chain rank for this provider on this bench. Only populated when - * the bench declares chain dimensions AND bestPerChain has at least one - * entry. Key = chain slug (matching dimensions.chain[].value), value = - * { rank, totalRanked } computed within the providers present on that - * chain. Renderers can fall back to `rank` when this is empty. */ + /** Per-chain leadership for this provider on this bench. Populated only + * for chains this provider *leads*, so `rank` is always 1 and an absent + * key means "does not lead here", never "ranked lower here". Key = chain + * slug (matching dimensions.chain[].value). `totalRanked` is how many + * providers were measured on that chain, or 0 when that set is unknown. + * Renderers fall back to the aggregate `rank` when this is empty. */ rankPerChain?: Record; }; @@ -250,18 +251,23 @@ function rankProviders(b: Benchmark): ProviderResult[] { } /** - * Compute per-chain rank for every provider on a bench. Bench must declare - * `dimensions.chain` and have a non-empty `bestPerChain` for any rank to be + * Compute per-chain leadership for a bench. Bench must declare + * `dimensions.chain` and have a non-empty `bestPerChain` for anything to be * recorded. * - * Approximation: spec.ts only stashes the *leader* per chain (one extra Prom - * roundtrip per chain). To express other providers' rank-per-chain, we use a - * coarse fallback: anyone present in the unfiltered `results` is ranked by - * the bench's standard direction (lower-is-better or higher-is-better) - * within the live result set, and the leader's slot is forcibly overridden - * with rank 1 for that chain. This is a soft signal — the bench page chain - * tabs are authoritative — but it is enough to flag chain-restricted - * providers like GMGN as "#1 on Solana only" on /products/[slug]. + * Leaders only, deliberately. spec.ts stashes the *leader* per chain (one + * extra Prom roundtrip per chain), and that is the only per-chain fact we + * actually hold. An earlier version also emitted ranks for non-leaders by + * reusing the unfiltered aggregate order shifted by one slot; those chips + * rendered as "#3 on Solana" while being the provider's *global* rank with a + * chain label stuck on it, identical across every chain of the bench. That + * reads as a measurement and is not one, so it is gone: a provider now gets a + * chip for a chain only when it leads that chain. + * + * `totalRanked` is the number of providers actually measured on the chain, + * taken from `providersPerChain`. It is 0 when the bench has not stashed that + * set (older cached entries), and renderers must then omit the denominator + * rather than substitute the global count. */ function rankPerChainForBench( b: Benchmark, @@ -285,24 +291,14 @@ function rankPerChainForBench( const presentSet = providersPerChain?.[chain.value] ? new Set(providersPerChain[chain.value].map((s) => s.toLowerCase())) : undefined; - const scoped = presentSet - ? liveSorted.filter((r) => presentSet.has(r.slug.toLowerCase())) - : liveSorted; + // Only claim a denominator when we know who was measured on this + // chain; scoped.length over the global list would be a different + // number wearing the same label. + const totalRanked = presentSet + ? liveSorted.filter((r) => presentSet.has(r.slug.toLowerCase())).length + : 0; const perProvider = new Map(); - const leaderLc = leader.slug.toLowerCase(); - const leaderIdx = scoped.findIndex((r) => r.slug.toLowerCase() === leaderLc); - scoped.forEach((r, idx) => { - const lc = r.slug.toLowerCase(); - if (lc === leaderLc) { - perProvider.set(lc, { rank: 1, totalRanked: scoped.length }); - return; - } - // Anyone ranked above the leader in the unfiltered set drops by one - // slot here (since the leader skips ahead of them on this chain). - const rankOnChain = - leaderIdx !== -1 && idx < leaderIdx ? idx + 2 : idx + 1; - perProvider.set(lc, { rank: rankOnChain, totalRanked: scoped.length }); - }); + perProvider.set(leader.slug.toLowerCase(), { rank: 1, totalRanked }); out[chain.value] = perProvider; } return out; diff --git a/src/lib/spec-schema.ts b/src/lib/spec-schema.ts index eb01a8972..c445484cc 100644 --- a/src/lib/spec-schema.ts +++ b/src/lib/spec-schema.ts @@ -405,6 +405,27 @@ export const SpecSchema = z * single-vantage bench shows. * First use: keyed-rpc-robinhood pins region=sgp so the default view * is the Singapore probe. */ + /** + * How the headline value is scoped on a bench that declares + * `dimensions.chain`. + * + * Default (absent) keeps the cross-chain aggregate the Prom queries + * return. `contested_chains` narrows it to the chains where at least + * two providers reported data, and drops a provider that has none of + * those from the ranked field. + * + * The reason is that a cross-chain aggregate is a mix rather than a + * comparison: it rewards a provider for the chains it happens to be + * measured on. On wallet-labels-coverage that put StellarExpert and + * XRPScan first and second, each scored on a single chain no other + * provider reported, above a provider that led four contested ones. + * + * Opt-in per bench, and it describes a property rather than naming a + * bench: any bench whose chain set contains uncontested chains is a + * candidate. Excluded providers stay visible on their own chain tab. + */ + score_scope: z.enum(["contested_chains"]).optional(), + aggregate_filters: z .object({ chain: z.string().regex(/^[A-Za-z0-9][A-Za-z0-9_-]{0,63}$/).optional(), diff --git a/src/lib/spec.ts b/src/lib/spec.ts index 1ec020a77..b4efffc80 100644 --- a/src/lib/spec.ts +++ b/src/lib/spec.ts @@ -396,7 +396,9 @@ const loadBenchmarkUnfilteredCached = unstable_cache( // v63: add bench 265 perp-pf-ratio + 5 providers on bench 234. Bench SET grew. // v64: Benchmark.chart (default_panel_by_chain, hide_headline_by_chain); cached // objects without it kept the Head lag tab on Solana after the deploy. - ["bench-unfiltered-v64", process.env.VERCEL_ENV === "production" ? "prod" : "all"], + // v65: score_scope contested_chains on bench 008 changes its provider values; + // Serialized joins benches 001, 004, 005, 008, 090. + ["bench-unfiltered-v65", process.env.VERCEL_ENV === "production" ? "prod" : "all"], { revalidate: 300, tags: ["benchmarks"] }, ); @@ -611,7 +613,8 @@ const loadAllBenchmarksCached = unstable_cache( // v57: lockstep with bench-unfiltered-v54 (add 6 new chain benches 216-221). // v58: lockstep with bench-unfiltered-v63 (add bench 265 perp-pf-ratio). // v59: lockstep with bench-unfiltered-v64 (Benchmark.chart). - ["all-benchmarks-v59", process.env.VERCEL_ENV === "production" ? "prod" : "all"], + // v60: lockstep with bench-unfiltered-v65 (contested scope, Serialized). + ["all-benchmarks-v60", process.env.VERCEL_ENV === "production" ? "prod" : "all"], { revalidate: 300, tags: ["benchmarks"] }, ); export const loadAllBenchmarks = cache(loadAllBenchmarksCached);