From 6c4576e68939992ef180d599bcb75b1d72467559 Mon Sep 17 00:00:00 2001 From: Flotapponnier <160007691+Flotapponnier@users.noreply.github.com> Date: Tue, 8 Sep 2026 16:28:19 +0200 Subject: [PATCH 01/10] fix(products): per-chain chips show real leadership only, with the denominator (#2279) Two problems on /products/[slug], both visible on Serialized. rankPerChainForBench only holds one real per-chain fact: the leader, from bestPerChain. For every other provider it reused the unfiltered aggregate order shifted by one slot, so a chip reading "#3 on Solana" was the global rank with a chain label on it, repeated identically across every chain of the bench. That reads as a measurement and is not one. Non-leader chips are gone: a chip now means "leads this chain", and its absence means "does not lead", not "ranks lower". The chips also hid how many providers were measured on the chain. "#1 on Solana" was #1 of 2 on bench 008 while sitting next to "#3 of 8" for the aggregate. They now read "#1 of 4 on Ethereum", with the denominator taken from providersPerChain and omitted when that set is unknown rather than substituting the global count. Wins accounting is unchanged: it already counted rank === 1 entries only. Co-authored-by: Claude Opus 5 (1M context) (cherry picked from commit eda2da8c42a47150aa06c96b626c943ce3a99683) --- src/app/products/[slug]/page.tsx | 21 ++++--- src/components/bench-appearances-section.tsx | 9 +-- src/lib/providers.ts | 60 +++++++++----------- 3 files changed, 41 insertions(+), 49 deletions(-) diff --git a/src/app/products/[slug]/page.tsx b/src/app/products/[slug]/page.tsx index 713f29ab9..48c273ac6 100644 --- a/src/app/products/[slug]/page.tsx +++ b/src/app/products/[slug]/page.tsx @@ -787,11 +787,13 @@ export default async function ProviderPage({ const catColor = CATEGORY_COLOR[a.benchmark.category]; const hasData = a.rank > 0 && a.result.ms.p50 !== 0; const value = hasData ? fmtUnit(a.result.ms.p50, a.benchmark.unit) : null; - // Per-chain rank chips. Rendered alongside the aggregate rank - // when the bench declares chain dimensions and the provider has - // per-chain ranks populated. Surface text reads e.g. "#1 on - // Solana · #4 on Base · #4 on BNB" so a chain-restricted - // provider can't be passed off as a free cross-chain #1. + // Per-chain leadership chips, rendered alongside the aggregate + // rank when the bench declares chain dimensions. Leaders only: + // a chip means "leads this chain", and its absence means "does + // not lead", never "ranks lower". Reads e.g. "#1 of 4 on + // Ethereum" so a chain-restricted provider can't be passed off + // as a free cross-chain #1, and so a win on a two-provider + // chain isn't dressed up as a win on a crowded one. const chainRanks = a.rankPerChain && a.benchmark.chainDimensions ? a.benchmark.chainDimensions @@ -845,13 +847,10 @@ export default async function ProviderPage({ {chainRanks.map(({ chain, entry }) => ( - #{entry.rank} on {chain.label} + #1{entry.totalRanked > 0 ? ` of ${entry.totalRanked}` : ""} on{" "} + {chain.label} ))}

diff --git a/src/components/bench-appearances-section.tsx b/src/components/bench-appearances-section.tsx index c50c7fc38..de5582033 100644 --- a/src/components/bench-appearances-section.tsx +++ b/src/components/bench-appearances-section.tsx @@ -82,13 +82,10 @@ export async function BenchAppearancesSection({ providerSlug }: Props) { {chainRanks.map(({ chain, entry }) => ( - #{entry.rank} on {chain.label} + #1{entry.totalRanked > 0 ? ` of ${entry.totalRanked}` : ""} on{" "} + {chain.label} ))}

diff --git a/src/lib/providers.ts b/src/lib/providers.ts index 075c716d4..adde99606 100644 --- a/src/lib/providers.ts +++ b/src/lib/providers.ts @@ -205,11 +205,12 @@ export type ProviderAppearance = { result: ProviderResult; rank: number; totalRanked: number; - /** Per-chain rank for this provider on this bench. Only populated when - * the bench declares chain dimensions AND bestPerChain has at least one - * entry. Key = chain slug (matching dimensions.chain[].value), value = - * { rank, totalRanked } computed within the providers present on that - * chain. Renderers can fall back to `rank` when this is empty. */ + /** Per-chain leadership for this provider on this bench. Populated only + * for chains this provider *leads*, so `rank` is always 1 and an absent + * key means "does not lead here", never "ranked lower here". Key = chain + * slug (matching dimensions.chain[].value). `totalRanked` is how many + * providers were measured on that chain, or 0 when that set is unknown. + * Renderers fall back to the aggregate `rank` when this is empty. */ rankPerChain?: Record; }; @@ -250,18 +251,23 @@ function rankProviders(b: Benchmark): ProviderResult[] { } /** - * Compute per-chain rank for every provider on a bench. Bench must declare - * `dimensions.chain` and have a non-empty `bestPerChain` for any rank to be + * Compute per-chain leadership for a bench. Bench must declare + * `dimensions.chain` and have a non-empty `bestPerChain` for anything to be * recorded. * - * Approximation: spec.ts only stashes the *leader* per chain (one extra Prom - * roundtrip per chain). To express other providers' rank-per-chain, we use a - * coarse fallback: anyone present in the unfiltered `results` is ranked by - * the bench's standard direction (lower-is-better or higher-is-better) - * within the live result set, and the leader's slot is forcibly overridden - * with rank 1 for that chain. This is a soft signal — the bench page chain - * tabs are authoritative — but it is enough to flag chain-restricted - * providers like GMGN as "#1 on Solana only" on /products/[slug]. + * Leaders only, deliberately. spec.ts stashes the *leader* per chain (one + * extra Prom roundtrip per chain), and that is the only per-chain fact we + * actually hold. An earlier version also emitted ranks for non-leaders by + * reusing the unfiltered aggregate order shifted by one slot; those chips + * rendered as "#3 on Solana" while being the provider's *global* rank with a + * chain label stuck on it, identical across every chain of the bench. That + * reads as a measurement and is not one, so it is gone: a provider now gets a + * chip for a chain only when it leads that chain. + * + * `totalRanked` is the number of providers actually measured on the chain, + * taken from `providersPerChain`. It is 0 when the bench has not stashed that + * set (older cached entries), and renderers must then omit the denominator + * rather than substitute the global count. */ function rankPerChainForBench( b: Benchmark, @@ -285,24 +291,14 @@ function rankPerChainForBench( const presentSet = providersPerChain?.[chain.value] ? new Set(providersPerChain[chain.value].map((s) => s.toLowerCase())) : undefined; - const scoped = presentSet - ? liveSorted.filter((r) => presentSet.has(r.slug.toLowerCase())) - : liveSorted; + // Only claim a denominator when we know who was measured on this + // chain; scoped.length over the global list would be a different + // number wearing the same label. + const totalRanked = presentSet + ? liveSorted.filter((r) => presentSet.has(r.slug.toLowerCase())).length + : 0; const perProvider = new Map(); - const leaderLc = leader.slug.toLowerCase(); - const leaderIdx = scoped.findIndex((r) => r.slug.toLowerCase() === leaderLc); - scoped.forEach((r, idx) => { - const lc = r.slug.toLowerCase(); - if (lc === leaderLc) { - perProvider.set(lc, { rank: 1, totalRanked: scoped.length }); - return; - } - // Anyone ranked above the leader in the unfiltered set drops by one - // slot here (since the leader skips ahead of them on this chain). - const rankOnChain = - leaderIdx !== -1 && idx < leaderIdx ? idx + 2 : idx + 1; - perProvider.set(lc, { rank: rankOnChain, totalRanked: scoped.length }); - }); + perProvider.set(leader.slug.toLowerCase(), { rank: 1, totalRanked }); out[chain.value] = perProvider; } return out; From 90a869009c0a1be7e35a5a781f07e98687672b3b Mon Sep 17 00:00:00 2001 From: Flotapponnier <160007691+Flotapponnier@users.noreply.github.com> Date: Tue, 8 Sep 2026 22:22:04 +0200 Subject: [PATCH 02/10] fix(ranking): rank multi-chain benches by contested-chain wins, not the mix (#2280) On a bench with chain dimensions the cross-chain aggregate is a mix, not a comparison. Ranking on it alone let a provider measured on one chain that nobody else reported finish above a provider that led several contested ones. Five live benches shipped that way: rpc-capabilities Binance 1st on 1 chain, PublicNode led 6 wallet-labels-coverage XRPScan 1st on 1 chain, Serialized led 4 token-quote-coverage Jupiter 1st on 1 chain, Mobula led 2 bridge-fee Squid Router 1st on 1 chain perp-liq-rate Lighter 1st on 1 chain rankedCandidates now sorts by contested-chain wins first and uses the aggregate value only to break ties. A chain counts only when at least two providers reported on it, so an uncontested chain awards nothing: you do not win a race you ran alone. Guarded by the per-chain stashes, which materialize/load.ts populates only on the unfiltered view. A chain-filtered variant has none, so ?chain=bnb keeps ranking by value as before. providers.ts reuses the same ordering. The two surfaces disagreeing is what put "#3 of 8" next to five chain-leadership chips on the same bench row. Known and accepted: a provider with one contested win now ranks above one with none and a higher aggregate figure (TonAPI over XRPScan on 008). That is what ranking on head-to-head record means; the win count is on the row. Rule documented in methodology, section II. Co-authored-by: Claude Opus 5 (1M context) (cherry picked from commit b6ba6b05e6b47c16985c3d4b3318e4f61559206f) --- src/app/methodology/page.tsx | 4 +++ src/lib/citation.test.ts | 70 +++++++++++++++++++++++++++++++++++- src/lib/citation.ts | 54 +++++++++++++++++++++++++--- src/lib/providers.ts | 18 +++++++--- 4 files changed, 136 insertions(+), 10 deletions(-) diff --git a/src/app/methodology/page.tsx b/src/app/methodology/page.tsx index 9c6b95b7d..46acf0fe0 100644 --- a/src/app/methodology/page.tsx +++ b/src/app/methodology/page.tsx @@ -60,6 +60,10 @@ const CONVENTIONS = [ term: "Success rate", body: "Share of requests returning a usable result within the published timeout. The only metric that includes failures.", }, + { + term: "Ranking on multi-chain benchmarks", + body: "When a benchmark measures several chains, providers are ranked first by the number of chains they lead, and only then by their cross-chain figure. A chain counts toward that total only when at least two providers reported data on it. A cross-chain average is a mix rather than a comparison, so ranking on it alone let a provider measured on one uncontested chain finish above a provider that led several contested ones.", + }, { term: "Region normalisation", body: "Where a benchmark is multi-region, the headline figure is the cross-region median. Per-region figures appear on every benchmark page.", diff --git a/src/lib/citation.test.ts b/src/lib/citation.test.ts index 376995f8c..0e51f3833 100644 --- a/src/lib/citation.test.ts +++ b/src/lib/citation.test.ts @@ -1,5 +1,5 @@ import { describe, expect, test } from "bun:test"; -import { leader, fieldValue, rankedCandidates } from "./citation"; +import { chainWins, leader, fieldValue, rankedCandidates } from "./citation"; import type { Benchmark, ProviderResult } from "@/types/benchmark"; function r( @@ -111,3 +111,71 @@ describe("citation reliability threshold", () => { expect(top?.value).toBe(ranks[0].ms.p50); }); }); + +describe("contested-chain wins drive the ranking", () => { + // Bench 008 as it actually shipped: XRPScan and StellarExpert sat 1st + // and 2nd on the cross-chain average, each measured on a single chain + // nobody else reported, while Serialized led four contested ones. + const b008 = (): Benchmark => ({ + ...bench([ + r("stellarexpert", "StellarExpert", 80.08), + r("xrpscan", "XRPScan", 79.84), + r("serialized", "Serialized", 76.98), + r("mobula", "Mobula", 46.75), + ]), + higherIsBetter: true, + bestPerChain: { + ethereum: r("serialized", "Serialized", 96.84), + base: r("serialized", "Serialized", 76.36), + solana: r("serialized", "Serialized", 57.79), + arbitrum: r("serialized", "Serialized", 70.0), + bnb: r("mobula", "Mobula", 79.78), + xrp: r("xrpscan", "XRPScan", 79.84), + stellar: r("stellarexpert", "StellarExpert", 80.08), + }, + providersPerChain: { + ethereum: ["serialized", "mobula", "oli", "blockscout"], + base: ["serialized", "mobula", "oli", "blockscout"], + solana: ["serialized", "mobula"], + arbitrum: ["serialized", "mobula", "oli"], + bnb: ["mobula", "serialized", "oli"], + xrp: ["xrpscan"], + stellar: ["stellarexpert"], + }, + }); + + test("the provider leading the most contested chains ranks first", () => { + expect(rankedCandidates(b008()).map((r) => r.slug)).toEqual([ + "serialized", + "mobula", + "stellarexpert", + "xrpscan", + ]); + expect(leader(b008())?.slug).toBe("serialized"); + }); + + test("a chain with one measured provider awards no win", () => { + const wins = chainWins(b008()); + expect(wins?.get("xrpscan")).toBeUndefined(); + expect(wins?.get("stellarexpert")).toBeUndefined(); + expect(wins?.get("serialized")).toBe(4); + expect(wins?.get("mobula")).toBe(1); + }); + + test("providers with equal wins fall back to the aggregate value", () => { + const b = b008(); + // Strip every contested win so the whole field ties at zero. + b.providersPerChain = { ethereum: ["serialized"], bnb: ["mobula"] }; + expect(rankedCandidates(b).map((r) => r.slug)).toEqual([ + "stellarexpert", + "xrpscan", + "serialized", + "mobula", + ]); + }); + + test("a bench without per-chain stashes ranks by value alone", () => { + const b = { ...b008(), bestPerChain: undefined, providersPerChain: undefined }; + expect(rankedCandidates(b).map((r) => r.slug)[0]).toBe("stellarexpert"); + }); +}); diff --git a/src/lib/citation.ts b/src/lib/citation.ts index 198117aea..b3136a374 100644 --- a/src/lib/citation.ts +++ b/src/lib/citation.ts @@ -48,16 +48,60 @@ export function citationCandidates(b: Benchmark): ProviderResult[] { return pool.filter((r) => r.dataConfidence !== "insufficient"); } +/** + * Chains each provider leads, counting **contested** chains only: a chain + * where at least two providers reported data. + * + * The exclusion is the whole point. On a chain-dimensioned bench the + * cross-chain aggregate is a mix, not a comparison, and a provider + * measured on exactly one easy chain with no competitor on it can top the + * board without ever beating anyone. Bench 008 shipped that way: + * StellarExpert and XRPScan sat 1st and 2nd, each measured on a single + * uncontested chain, above Serialized which led four contested ones. Four + * other live benches had the same shape, `rpc-capabilities` worst of all + * (Binance 1st on one chain while PublicNode led six). + * + * Returns null when the bench cannot support the count — no chain + * dimensions, or the per-chain stashes absent. Those stashes are only + * populated on the unfiltered view (see materialize/load.ts), which is + * also the guard that keeps a chain-filtered variant from being ranked by + * cross-chain wins: on `?chain=bnb` there is nothing to count. + */ +export function chainWins(b: Benchmark): Map | null { + const best = b.bestPerChain; + const present = b.providersPerChain; + if (!best || !present) return null; + const wins = new Map(); + for (const [chain, chainLeader] of Object.entries(best)) { + if ((present[chain]?.length ?? 0) < 2) continue; + const slug = chainLeader.slug.toLowerCase(); + wins.set(slug, (wins.get(slug) ?? 0) + 1); + } + return wins.size > 0 ? wins : null; +} + /** Sorted candidate pool for the machine-readable `rankings` array on * `/api/stat`, MCP, llm-context and any downstream that ranks the * full field. Applies the same reliability + insufficient-sample * filters as `leader()` so a document that names X as leader ranks X - * first in its own list. Sort direction honors the bench's - * `higherIsBetter` flag. */ + * first in its own list. + * + * On a bench that can count contested-chain wins, those wins are the + * primary key and the aggregate value only breaks ties: head-to-head + * record first, chain-mix average second. Everywhere else (no chain + * dimensions, filtered variants) it is the aggregate value alone, sorted + * in the direction the bench's `higherIsBetter` flag asks for. */ export function rankedCandidates(b: Benchmark): ProviderResult[] { - return [...citationCandidates(b)].sort((a, c) => - b.higherIsBetter ? c.ms.p50 - a.ms.p50 : a.ms.p50 - c.ms.p50, - ); + const byValue = (a: ProviderResult, c: ProviderResult) => + b.higherIsBetter ? c.ms.p50 - a.ms.p50 : a.ms.p50 - c.ms.p50; + const pool = [...citationCandidates(b)]; + const wins = chainWins(b); + if (!wins) return pool.sort(byValue); + return pool.sort((a, c) => { + const delta = + (wins.get(c.slug.toLowerCase()) ?? 0) - (wins.get(a.slug.toLowerCase()) ?? 0); + return delta !== 0 ? delta : byValue(a, c); + }); } /** Timestamp of the last real measurement, or null when the bench has diff --git a/src/lib/providers.ts b/src/lib/providers.ts index adde99606..84a0f3c63 100644 --- a/src/lib/providers.ts +++ b/src/lib/providers.ts @@ -13,7 +13,7 @@ import { unstable_cache } from "next/cache"; import { getBenchmarksSafe } from "@/data/benchmarks"; import { loadProvidersFromBlob } from "@/lib/bench-blob"; import { liveResults } from "@/lib/provider-filters"; -import { citationCandidates } from "@/lib/citation"; +import { chainWins, citationCandidates } from "@/lib/citation"; import { readBestPerChain } from "@/lib/per-chain-contract"; import type { Benchmark, ProviderResult } from "@/types/benchmark"; @@ -245,9 +245,19 @@ function rankProviders(b: Benchmark): ProviderResult[] { // a best-of-bad-options ranking. const pool = citationCandidates(b); const live = pool.length > 0 ? pool : liveResults(b.results); - return [...live].sort((a, c) => - b.higherIsBetter ? c.ms.p50 - a.ms.p50 : a.ms.p50 - c.ms.p50, - ); + // Same ordering as the bench page: contested-chain wins first, aggregate + // value as the tiebreak (see rankedCandidates). Sorting these two + // surfaces differently is what let /products show "#3 of 8" beside five + // chain-leadership chips on the same bench. + const byValue = (a: ProviderResult, c: ProviderResult) => + b.higherIsBetter ? c.ms.p50 - a.ms.p50 : a.ms.p50 - c.ms.p50; + const wins = chainWins(b); + if (!wins) return [...live].sort(byValue); + return [...live].sort((a, c) => { + const delta = + (wins.get(c.slug.toLowerCase()) ?? 0) - (wins.get(a.slug.toLowerCase()) ?? 0); + return delta !== 0 ? delta : byValue(a, c); + }); } /** From 63510a0c5aa230059439bddca68c038b8a64c4be Mon Sep 17 00:00:00 2001 From: Flotapponnier <160007691+Flotapponnier@users.noreply.github.com> Date: Wed, 9 Sep 2026 16:47:19 +0200 Subject: [PATCH 03/10] fix(scoring): scope the value to contested chains, opt-in per bench (#2281) Replaces the global chain-wins sort from #2280, which fixed the ordering but broke the reading of it. Ranking on a key the reader cannot see in the column produced tables that no longer descend: 79.88% at rank 4 on 008, and a 74 ms leader at rank 3 on rpc-capabilities where lower is better. The defect was never the sort key. It is that the aggregate includes chains where a provider had no competitor, so the fix belongs on the value: with `score_scope: contested_chains` a bench is scored only on chains carrying at least two measured providers, and a provider with none of those leaves the ranked field (still visible on its own chain tab). One quantity on screen, ordering follows from it. Opt-in per bench and named for the property, not the bench, so any bench whose chain set contains uncontested chains is a candidate and the rule is readable in the public YAML. Enabled on wallet-labels-coverage only: stellar, xrp and bitcoin carry one measured provider each there. load.ts already fetched every provider's per-chain value inside its per-chain loop and discarded all but the leader; it now keeps them. Four other live benches qualify and are deliberately left untouched pending review: rpc-capabilities, token-quote-coverage, bridge-fee, perp-liq-rate. Residual limitation documented in methodology: providers are still averaged over the different subsets of contested chains they cover. Co-authored-by: Claude Opus 5 (1M context) (cherry picked from commit 401d0a709466e00dd260a754aaadc98386c7d430) --- benchmarks/wallet-labels-coverage.yml | 5 ++ src/app/methodology/page.tsx | 4 +- src/lib/citation.test.ts | 70 +----------------- src/lib/citation.ts | 58 +++------------ src/lib/materialize/contested-scope.test.ts | 82 +++++++++++++++++++++ src/lib/materialize/load.ts | 68 ++++++++++++++++- src/lib/providers.ts | 18 +---- src/lib/spec-schema.ts | 21 ++++++ 8 files changed, 190 insertions(+), 136 deletions(-) create mode 100644 src/lib/materialize/contested-scope.test.ts diff --git a/benchmarks/wallet-labels-coverage.yml b/benchmarks/wallet-labels-coverage.yml index 2cfe912e6..8f08fa327 100644 --- a/benchmarks/wallet-labels-coverage.yml +++ b/benchmarks/wallet-labels-coverage.yml @@ -151,6 +151,11 @@ prometheus: # Default is `eoa` because the contract tab is trivially easy for explorers # and saturates near 100%; the EOA tab is where curated entity coverage # actually differentiates providers. +# Scored on contested chains only: stellar, xrp and bitcoin carry a single +# measured provider each, and a cross-chain average that counts them puts a +# provider first for a chain nobody else reported. See spec-schema.ts. +score_scope: contested_chains + dimensions: kind: - { value: eoa, label: EOA } diff --git a/src/app/methodology/page.tsx b/src/app/methodology/page.tsx index 46acf0fe0..665f47df1 100644 --- a/src/app/methodology/page.tsx +++ b/src/app/methodology/page.tsx @@ -61,8 +61,8 @@ const CONVENTIONS = [ body: "Share of requests returning a usable result within the published timeout. The only metric that includes failures.", }, { - term: "Ranking on multi-chain benchmarks", - body: "When a benchmark measures several chains, providers are ranked first by the number of chains they lead, and only then by their cross-chain figure. A chain counts toward that total only when at least two providers reported data on it. A cross-chain average is a mix rather than a comparison, so ranking on it alone let a provider measured on one uncontested chain finish above a provider that led several contested ones.", + term: "Contested-chain scoring", + body: "A cross-chain average is a mix rather than a comparison: it credits a provider for the chains it happens to be measured on. Benchmarks that declare score_scope: contested_chains in their spec are therefore scored only on chains where at least two providers reported data, and a provider with no such chain is left out of the ranking while staying visible on its own chain tab. The figure shown is the unweighted mean across those chains, so each chain counts once regardless of sample count. This narrows the number rather than the sort order, so the published value is always the one the ranking follows. It does not equalise chain mix entirely: providers are still averaged over the different subsets of contested chains they cover.", }, { term: "Region normalisation", diff --git a/src/lib/citation.test.ts b/src/lib/citation.test.ts index 0e51f3833..376995f8c 100644 --- a/src/lib/citation.test.ts +++ b/src/lib/citation.test.ts @@ -1,5 +1,5 @@ import { describe, expect, test } from "bun:test"; -import { chainWins, leader, fieldValue, rankedCandidates } from "./citation"; +import { leader, fieldValue, rankedCandidates } from "./citation"; import type { Benchmark, ProviderResult } from "@/types/benchmark"; function r( @@ -111,71 +111,3 @@ describe("citation reliability threshold", () => { expect(top?.value).toBe(ranks[0].ms.p50); }); }); - -describe("contested-chain wins drive the ranking", () => { - // Bench 008 as it actually shipped: XRPScan and StellarExpert sat 1st - // and 2nd on the cross-chain average, each measured on a single chain - // nobody else reported, while Serialized led four contested ones. - const b008 = (): Benchmark => ({ - ...bench([ - r("stellarexpert", "StellarExpert", 80.08), - r("xrpscan", "XRPScan", 79.84), - r("serialized", "Serialized", 76.98), - r("mobula", "Mobula", 46.75), - ]), - higherIsBetter: true, - bestPerChain: { - ethereum: r("serialized", "Serialized", 96.84), - base: r("serialized", "Serialized", 76.36), - solana: r("serialized", "Serialized", 57.79), - arbitrum: r("serialized", "Serialized", 70.0), - bnb: r("mobula", "Mobula", 79.78), - xrp: r("xrpscan", "XRPScan", 79.84), - stellar: r("stellarexpert", "StellarExpert", 80.08), - }, - providersPerChain: { - ethereum: ["serialized", "mobula", "oli", "blockscout"], - base: ["serialized", "mobula", "oli", "blockscout"], - solana: ["serialized", "mobula"], - arbitrum: ["serialized", "mobula", "oli"], - bnb: ["mobula", "serialized", "oli"], - xrp: ["xrpscan"], - stellar: ["stellarexpert"], - }, - }); - - test("the provider leading the most contested chains ranks first", () => { - expect(rankedCandidates(b008()).map((r) => r.slug)).toEqual([ - "serialized", - "mobula", - "stellarexpert", - "xrpscan", - ]); - expect(leader(b008())?.slug).toBe("serialized"); - }); - - test("a chain with one measured provider awards no win", () => { - const wins = chainWins(b008()); - expect(wins?.get("xrpscan")).toBeUndefined(); - expect(wins?.get("stellarexpert")).toBeUndefined(); - expect(wins?.get("serialized")).toBe(4); - expect(wins?.get("mobula")).toBe(1); - }); - - test("providers with equal wins fall back to the aggregate value", () => { - const b = b008(); - // Strip every contested win so the whole field ties at zero. - b.providersPerChain = { ethereum: ["serialized"], bnb: ["mobula"] }; - expect(rankedCandidates(b).map((r) => r.slug)).toEqual([ - "stellarexpert", - "xrpscan", - "serialized", - "mobula", - ]); - }); - - test("a bench without per-chain stashes ranks by value alone", () => { - const b = { ...b008(), bestPerChain: undefined, providersPerChain: undefined }; - expect(rankedCandidates(b).map((r) => r.slug)[0]).toBe("stellarexpert"); - }); -}); diff --git a/src/lib/citation.ts b/src/lib/citation.ts index b3136a374..5d3fe68bd 100644 --- a/src/lib/citation.ts +++ b/src/lib/citation.ts @@ -48,60 +48,22 @@ export function citationCandidates(b: Benchmark): ProviderResult[] { return pool.filter((r) => r.dataConfidence !== "insufficient"); } -/** - * Chains each provider leads, counting **contested** chains only: a chain - * where at least two providers reported data. - * - * The exclusion is the whole point. On a chain-dimensioned bench the - * cross-chain aggregate is a mix, not a comparison, and a provider - * measured on exactly one easy chain with no competitor on it can top the - * board without ever beating anyone. Bench 008 shipped that way: - * StellarExpert and XRPScan sat 1st and 2nd, each measured on a single - * uncontested chain, above Serialized which led four contested ones. Four - * other live benches had the same shape, `rpc-capabilities` worst of all - * (Binance 1st on one chain while PublicNode led six). - * - * Returns null when the bench cannot support the count — no chain - * dimensions, or the per-chain stashes absent. Those stashes are only - * populated on the unfiltered view (see materialize/load.ts), which is - * also the guard that keeps a chain-filtered variant from being ranked by - * cross-chain wins: on `?chain=bnb` there is nothing to count. - */ -export function chainWins(b: Benchmark): Map | null { - const best = b.bestPerChain; - const present = b.providersPerChain; - if (!best || !present) return null; - const wins = new Map(); - for (const [chain, chainLeader] of Object.entries(best)) { - if ((present[chain]?.length ?? 0) < 2) continue; - const slug = chainLeader.slug.toLowerCase(); - wins.set(slug, (wins.get(slug) ?? 0) + 1); - } - return wins.size > 0 ? wins : null; -} - /** Sorted candidate pool for the machine-readable `rankings` array on * `/api/stat`, MCP, llm-context and any downstream that ranks the * full field. Applies the same reliability + insufficient-sample * filters as `leader()` so a document that names X as leader ranks X - * first in its own list. + * first in its own list. Sort direction honors the bench's + * `higherIsBetter` flag. * - * On a bench that can count contested-chain wins, those wins are the - * primary key and the aggregate value only breaks ties: head-to-head - * record first, chain-mix average second. Everywhere else (no chain - * dimensions, filtered variants) it is the aggregate value alone, sorted - * in the direction the bench's `higherIsBetter` flag asks for. */ + * Ranks on the value alone, deliberately. A bench whose cross-chain + * aggregate would otherwise reward an uncontested chain fixes that by + * declaring `score_scope: contested_chains` in its spec, which narrows + * the value itself (see materialize/load.ts) rather than sorting on a + * key the reader cannot see in the column. */ export function rankedCandidates(b: Benchmark): ProviderResult[] { - const byValue = (a: ProviderResult, c: ProviderResult) => - b.higherIsBetter ? c.ms.p50 - a.ms.p50 : a.ms.p50 - c.ms.p50; - const pool = [...citationCandidates(b)]; - const wins = chainWins(b); - if (!wins) return pool.sort(byValue); - return pool.sort((a, c) => { - const delta = - (wins.get(c.slug.toLowerCase()) ?? 0) - (wins.get(a.slug.toLowerCase()) ?? 0); - return delta !== 0 ? delta : byValue(a, c); - }); + return [...citationCandidates(b)].sort((a, c) => + b.higherIsBetter ? c.ms.p50 - a.ms.p50 : a.ms.p50 - c.ms.p50, + ); } /** Timestamp of the last real measurement, or null when the bench has diff --git a/src/lib/materialize/contested-scope.test.ts b/src/lib/materialize/contested-scope.test.ts new file mode 100644 index 000000000..8d8475ed6 --- /dev/null +++ b/src/lib/materialize/contested-scope.test.ts @@ -0,0 +1,82 @@ +import { describe, expect, test } from "bun:test"; +import { applyContestedChainScope } from "./load"; +import type { ProviderResult } from "@/types/benchmark"; + +function r(slug: string, p50: number): ProviderResult { + return { + slug, + name: slug, + ms: { p50, p90: p50, p99: p50, mean: p50 }, + successRate: 100, + availability: "live", + }; +} + +// Bench 008 as it shipped: stellar, xrp and bitcoin carry one measured +// provider each, so the cross-chain average put two single-chain providers +// first and second above one that led four contested chains. +function fixture() { + const results = [ + r("stellarexpert", 80.08), + r("xrpscan", 79.84), + r("serialized", 76.98), + r("mobula", 46.75), + ]; + const providersPerChain: Record = { + ethereum: ["serialized", "mobula"], + base: ["serialized", "mobula"], + solana: ["serialized", "mobula"], + stellar: ["stellarexpert"], + xrp: ["xrpscan"], + }; + const valuesByChain: Record> = { + ethereum: { serialized: 96.84, mobula: 50.0 }, + base: { serialized: 76.36, mobula: 40.0 }, + solana: { serialized: 57.79, mobula: 30.0 }, + stellar: { stellarexpert: 80.08 }, + xrp: { xrpscan: 79.84 }, + }; + return { results, providersPerChain, valuesByChain }; +} + +describe("contested-chain scoring", () => { + test("the value becomes the mean over contested chains", () => { + const { results, providersPerChain, valuesByChain } = fixture(); + applyContestedChainScope(results, providersPerChain, valuesByChain); + const s = results.find((x) => x.slug === "serialized")!; + // (96.84 + 76.36 + 57.79) / 3 + expect(s.ms.p50).toBeCloseTo(76.9967, 3); + expect(s.ms.mean).toBeCloseTo(76.9967, 3); + expect(results.find((x) => x.slug === "mobula")!.ms.p50).toBeCloseTo(40, 6); + }); + + test("a provider with no contested chain drops out of the ranked field", () => { + const { results, providersPerChain, valuesByChain } = fixture(); + applyContestedChainScope(results, providersPerChain, valuesByChain); + expect(results.find((x) => x.slug === "stellarexpert")!.availability).toBe( + "unavailable", + ); + expect(results.find((x) => x.slug === "xrpscan")!.availability).toBe( + "unavailable", + ); + expect(results.find((x) => x.slug === "serialized")!.availability).toBe("live"); + }); + + test("no contested chain at all leaves every value untouched", () => { + const { results, valuesByChain } = fixture(); + applyContestedChainScope( + results, + { stellar: ["stellarexpert"], xrp: ["xrpscan"] }, + valuesByChain, + ); + expect(results.find((x) => x.slug === "serialized")!.ms.p50).toBe(76.98); + expect(results.every((x) => x.availability === "live")).toBe(true); + }); + + test("an already unavailable provider is left alone", () => { + const { results, providersPerChain, valuesByChain } = fixture(); + results[0].availability = "unavailable"; + applyContestedChainScope(results, providersPerChain, valuesByChain); + expect(results[0].ms.p50).toBe(80.08); + }); +}); diff --git a/src/lib/materialize/load.ts b/src/lib/materialize/load.ts index 3f6bbc527..7b8896fbc 100644 --- a/src/lib/materialize/load.ts +++ b/src/lib/materialize/load.ts @@ -329,15 +329,26 @@ export async function specToBenchmark( const chainSpec = applyDimensionsToSpec(spec, { chain }); const chainLive = await tryLoadLive(chainSpec, true); if (!chainLive) { - return [chain, undefined, undefined, [] as string[]] as const; + return [ + chain, + undefined, + undefined, + [] as string[], + {} as Record, + ] as const; } for (const r of chainLive.results) { if (!r.unresponsive) r.availability = "live"; } const liveForChain = liveProviderResults(chainLive.results); const slugs = liveForChain.map((r) => r.slug); + // Per-provider value on this chain. Kept (not just the leader) + // so `score_scope: contested_chains` can rebuild a value from + // the chains a provider was actually compared on. + const values: Record = {}; + for (const r of liveForChain) values[r.slug.toLowerCase()] = r.ms.p50; if (liveForChain.length === 0) { - return [chain, undefined, undefined, slugs] as const; + return [chain, undefined, undefined, slugs, values] as const; } const sorted = [...liveForChain].sort((a, b) => spec.higher_is_better ? b.ms.p50 - a.ms.p50 : a.ms.p50 - b.ms.p50, @@ -347,20 +358,27 @@ export async function specToBenchmark( sorted[0], sorted[sorted.length - 1], slugs, + values, ] as const; }), ); const bests: Record = {}; const worsts: Record = {}; const providers: Record = {}; - for (const [chain, leader, trailer, slugs] of perChainEntries) { + const valuesByChain: Record> = {}; + for (const [chain, leader, trailer, slugs, values] of perChainEntries) { if (leader) bests[chain] = leader; if (trailer) worsts[chain] = trailer; if (slugs.length > 0) providers[chain] = slugs; + valuesByChain[chain] = values; } if (Object.keys(bests).length > 0) bestPerChain = bests; if (Object.keys(worsts).length > 0) worstPerChain = worsts; if (Object.keys(providers).length > 0) providersPerChain = providers; + + if (spec.score_scope === "contested_chains") { + applyContestedChainScope(live.results, providers, valuesByChain); + } } // Exact per-cell rankings (chain × region) from the spec's single @@ -424,6 +442,50 @@ export async function specToBenchmark( return draftBenchmark(spec, editorial); } +/** + * Rewrite each provider's headline value as its mean over the **contested** + * chains of the bench: those where at least two providers reported data. + * A provider with no contested chain is marked unavailable, which takes it + * out of `liveResults` and therefore out of every ranked surface, while + * leaving it visible on its own chain tab. + * + * Why the value and not the sort order: a cross-chain aggregate is a mix + * rather than a comparison, so it credits a provider for the chains it + * happens to be measured on. Narrowing the number keeps one quantity on + * screen and the ordering follows from it. Ranking on a separate key while + * still displaying the wide aggregate produced a column that did not + * descend (80% shown at rank 4, a 74 ms leader shown at rank 3). + * + * The residual limitation, stated in the methodology: providers are still + * averaged over different subsets of the contested chains, since they do + * not all cover the same ones. It removes the uncontested win, not every + * difference in chain mix. An unweighted mean is used so a chain counts + * once regardless of how many samples it carries. + */ +export function applyContestedChainScope( + results: ProviderResult[], + providersPerChain: Record, + valuesByChain: Record>, +): void { + const contested = Object.keys(providersPerChain).filter( + (chain) => (providersPerChain[chain]?.length ?? 0) >= 2, + ); + if (contested.length === 0) return; + for (const r of results) { + if (r.availability === "unavailable") continue; + const slug = r.slug.toLowerCase(); + const vals = contested + .map((chain) => valuesByChain[chain]?.[slug]) + .filter((v): v is number => typeof v === "number" && v > 0); + if (vals.length === 0) { + r.availability = "unavailable"; + continue; + } + const mean = vals.reduce((a, b) => a + b, 0) / vals.length; + r.ms = { p50: mean, p90: mean, p99: mean, mean }; + } +} + function activeFilterLabels(opts: BenchmarkFilters): Record { const out: Record = {}; for (const [k, v] of Object.entries(opts)) { diff --git a/src/lib/providers.ts b/src/lib/providers.ts index 84a0f3c63..adde99606 100644 --- a/src/lib/providers.ts +++ b/src/lib/providers.ts @@ -13,7 +13,7 @@ import { unstable_cache } from "next/cache"; import { getBenchmarksSafe } from "@/data/benchmarks"; import { loadProvidersFromBlob } from "@/lib/bench-blob"; import { liveResults } from "@/lib/provider-filters"; -import { chainWins, citationCandidates } from "@/lib/citation"; +import { citationCandidates } from "@/lib/citation"; import { readBestPerChain } from "@/lib/per-chain-contract"; import type { Benchmark, ProviderResult } from "@/types/benchmark"; @@ -245,19 +245,9 @@ function rankProviders(b: Benchmark): ProviderResult[] { // a best-of-bad-options ranking. const pool = citationCandidates(b); const live = pool.length > 0 ? pool : liveResults(b.results); - // Same ordering as the bench page: contested-chain wins first, aggregate - // value as the tiebreak (see rankedCandidates). Sorting these two - // surfaces differently is what let /products show "#3 of 8" beside five - // chain-leadership chips on the same bench. - const byValue = (a: ProviderResult, c: ProviderResult) => - b.higherIsBetter ? c.ms.p50 - a.ms.p50 : a.ms.p50 - c.ms.p50; - const wins = chainWins(b); - if (!wins) return [...live].sort(byValue); - return [...live].sort((a, c) => { - const delta = - (wins.get(c.slug.toLowerCase()) ?? 0) - (wins.get(a.slug.toLowerCase()) ?? 0); - return delta !== 0 ? delta : byValue(a, c); - }); + return [...live].sort((a, c) => + b.higherIsBetter ? c.ms.p50 - a.ms.p50 : a.ms.p50 - c.ms.p50, + ); } /** diff --git a/src/lib/spec-schema.ts b/src/lib/spec-schema.ts index eb01a8972..c445484cc 100644 --- a/src/lib/spec-schema.ts +++ b/src/lib/spec-schema.ts @@ -405,6 +405,27 @@ export const SpecSchema = z * single-vantage bench shows. * First use: keyed-rpc-robinhood pins region=sgp so the default view * is the Singapore probe. */ + /** + * How the headline value is scoped on a bench that declares + * `dimensions.chain`. + * + * Default (absent) keeps the cross-chain aggregate the Prom queries + * return. `contested_chains` narrows it to the chains where at least + * two providers reported data, and drops a provider that has none of + * those from the ranked field. + * + * The reason is that a cross-chain aggregate is a mix rather than a + * comparison: it rewards a provider for the chains it happens to be + * measured on. On wallet-labels-coverage that put StellarExpert and + * XRPScan first and second, each scored on a single chain no other + * provider reported, above a provider that led four contested ones. + * + * Opt-in per bench, and it describes a property rather than naming a + * bench: any bench whose chain set contains uncontested chains is a + * candidate. Excluded providers stay visible on their own chain tab. + */ + score_scope: z.enum(["contested_chains"]).optional(), + aggregate_filters: z .object({ chain: z.string().regex(/^[A-Za-z0-9][A-Za-z0-9_-]{0,63}$/).optional(), From eb248f0aa39e31466be8992a87066cd6ce8fb6cb Mon Sep 17 00:00:00 2001 From: Flotapponnier <160007691+Flotapponnier@users.noreply.github.com> Date: Wed, 9 Sep 2026 16:52:50 +0200 Subject: [PATCH 04/10] chore(cache): bump bench-unfiltered and all-benchmarks for contested-chain scope (#2282) v62 entries still carry the old cross-chain aggregate, so staging kept serving StellarExpert first. Both keys bumped in lockstep as usual. Co-authored-by: Claude Opus 5 (1M context) (cherry picked from commit 5ef21d8adf84e6a80e6ad8864b710f4451072f1a) --- src/lib/spec.ts | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/lib/spec.ts b/src/lib/spec.ts index 1ec020a77..b4efffc80 100644 --- a/src/lib/spec.ts +++ b/src/lib/spec.ts @@ -396,7 +396,9 @@ const loadBenchmarkUnfilteredCached = unstable_cache( // v63: add bench 265 perp-pf-ratio + 5 providers on bench 234. Bench SET grew. // v64: Benchmark.chart (default_panel_by_chain, hide_headline_by_chain); cached // objects without it kept the Head lag tab on Solana after the deploy. - ["bench-unfiltered-v64", process.env.VERCEL_ENV === "production" ? "prod" : "all"], + // v65: score_scope contested_chains on bench 008 changes its provider values; + // Serialized joins benches 001, 004, 005, 008, 090. + ["bench-unfiltered-v65", process.env.VERCEL_ENV === "production" ? "prod" : "all"], { revalidate: 300, tags: ["benchmarks"] }, ); @@ -611,7 +613,8 @@ const loadAllBenchmarksCached = unstable_cache( // v57: lockstep with bench-unfiltered-v54 (add 6 new chain benches 216-221). // v58: lockstep with bench-unfiltered-v63 (add bench 265 perp-pf-ratio). // v59: lockstep with bench-unfiltered-v64 (Benchmark.chart). - ["all-benchmarks-v59", process.env.VERCEL_ENV === "production" ? "prod" : "all"], + // v60: lockstep with bench-unfiltered-v65 (contested scope, Serialized). + ["all-benchmarks-v60", process.env.VERCEL_ENV === "production" ? "prod" : "all"], { revalidate: 300, tags: ["benchmarks"] }, ); export const loadAllBenchmarks = cache(loadAllBenchmarksCached); From 828851ebf04af38224e5a70aff03195119ab098e Mon Sep 17 00:00:00 2001 From: Flotapponnier <160007691+Flotapponnier@users.noreply.github.com> Date: Sat, 5 Sep 2026 18:58:48 +0200 Subject: [PATCH 05/10] feat: add Serialized as provider on benches 004 and 008 (#2261) Wires serialized.xyz into metadata-coverage (004) and wallet-labels (008), the two benches where their endpoints map 1:1 to the existing scoring rule. Both providers throttle client-side at ~16 rps: Serialized enforces a hard 40 req/s burst cap and an unthrottled worker pool turns coverage into a rate-limit artifact (measured 77% -> 37%). Adds the registry entry, logo and a full onboarding audit documenting the apples-to-apples numbers and two scoring flaws the tests exposed in our own benches (logo presence vs resolution on 004, name-service strings counted as entity labels on 008). Claude-Session: https://claude.ai/code/session_01CpArutAtXuBb1BVNUDXoYA Co-authored-by: Claude Opus 5 (1M context) (cherry picked from commit 5dc8d214aa06a60de91bbf05d17197d6e2fd84f6) --- .../serialized-onboarding-audit.md | 313 ++++++++++++++++++ .../metadata-coverage/cmd/script/config.go | 2 + .../cmd/script/metadata_coverage_monitor.go | 72 ++-- .../cmd/script/serialized_rest_monitor.go | 162 +++++++++ harnesses/wallet-labels/.env.example | 3 + harnesses/wallet-labels/cmd/script/config.go | 35 +- harnesses/wallet-labels/cmd/script/main.go | 1 + .../wallet-labels/cmd/script/serialized.go | 129 ++++++++ public/logos/serialized-wordmark.svg | 15 + public/logos/serialized.svg | 5 + src/data/provider-registry.ts | 41 +++ src/lib/logo-manifest.ts | 1 + 12 files changed, 740 insertions(+), 39 deletions(-) create mode 100644 docs/methodology/serialized-onboarding-audit.md create mode 100644 harnesses/metadata-coverage/cmd/script/serialized_rest_monitor.go create mode 100644 harnesses/wallet-labels/cmd/script/serialized.go create mode 100644 public/logos/serialized-wordmark.svg create mode 100644 public/logos/serialized.svg diff --git a/docs/methodology/serialized-onboarding-audit.md b/docs/methodology/serialized-onboarding-audit.md new file mode 100644 index 000000000..a34244c51 --- /dev/null +++ b/docs/methodology/serialized-onboarding-audit.md @@ -0,0 +1,313 @@ +# Provider onboarding audit — Serialized (serialized.xyz) + +> **Pre-onboarding evaluation.** Run before Serialized is wired into any live harness, so the +> decision to include or exclude them on each bench is documented and reproducible. +> +> **Version:** v1.0, first commit 2026-09-05. Author: internal. Key used: tenant `OpenChainBench`, +> plan `starter`, keyId `d5511a080aaa`, issued 2026-09-04. + +--- + +## 1. What this document is + +Serialized is a candidate provider for several existing OpenChainBench benchmarks. This file +records the apples-to-apples tests run against them, the exact methodology of each test, the +numbers that came back, and the methodology problems those tests exposed in **our own benches**. + +Every test below replicates the scoring rule of the target bench rather than inventing a new one, +so the numbers are directly comparable to the published leaderboards. + +## 2. Test harness and vantage point + +| Property | Value | +|---|---| +| Host | `ocb-par-main` (the VPS that runs the production harnesses) | +| Rationale | Same egress, same region, same network path as the live monitors. A latency or coverage number taken from a laptop is not comparable to a published bench value. | +| Incumbent credentials | Read from the running `ocb-metadata-coverage` container env, never copied off the box | +| Scripts | `~/serbench/ab.py`, `ab2.py`, `ab3.py`, `ab4.py`, `wsab.py` | +| Date of run | 2026-09-05 | + +**Throttling matters.** Serialized enforces a hard burst cap of 40 in-flight requests per second. +An unthrottled 8-worker pool produced 60 `429 RATE_LIMITED` responses out of 100 anchors and made +their coverage look like 37%. The same test throttled to 12 rps produced 0 errors and 77%. Any +harness that talks to them must rate-limit client-side, and any measurement that does not is wrong. + +## 3. Provider surface + +19 chains: 18 EVM plus Solana. `evm:1`, `evm:56`, `evm:130`, `evm:143`, `evm:196`, `evm:988`, +`evm:1514`, `evm:2741`, `evm:4217`, `evm:4326`, `evm:4663`, `evm:5042`, `evm:8453`, `evm:9745`, +`evm:42161`, `evm:43114`, `evm:57073`, `evm:645749`, `solana`. Audit engine covers the 18 EVM chains. + +Auth is a raw `Authorization` header, no `Bearer` prefix (same convention as Mobula). The +documented `demo.serialized.xyz` server returns 403 outside their docs playground, so there is no +keyless path for a harness. + +## 4. Rate limits and quota (measured, not quoted) + +| Property | Documented | Measured | +|---|---|---| +| Monthly credits (starter) | 150,000 | **1,000,000** on our key | +| Per-minute rate | 1,200 | 1,200 (`x-ratelimit-limit` header) | +| Burst | 40 req/s | Exactly 40. 60/100/150 concurrent all yielded exactly 40× `200` and the rest `429`. Deterministic, no jitter. | +| Sustained | not stated | 891/891 `200` over 60 s at 15 rps, p50 38 ms, p99 67 ms | + +Response headers expose `x-ratelimit-limit`, `x-ratelimit-remaining`, `x-ratelimit-reset` and +`x-credits-remaining`. Good enough to instrument a harness without guessing. + +Streams bill 1 credit per connection-minute. Limits are 5 concurrent connections, 20 subscriptions +per connection and 50 distinct tokens or pools per key. Bench 001 runs 3 regions × 4 chains, which +does not fit inside one key's 5-connection budget: it needs one key per region. + +## 5. Bench 008 — wallet-labels-coverage + +**Replica rule.** Identical to `harnesses/wallet-labels`: the same 178-anchor curated list, filtered +to the 5 chains Serialized covers (100 anchors, 59 contract / 41 EOA); a "hit" is any non-generic +name, using the harness's exact `genericLabel` exclusion set; Mobula queried through +`POST /api/1/wallet/labels` with the same field-precedence (`entityName` → `entityLabels` → `labels`). +Serialized queried through `GET /v1/wallet/profile`, taking the first non-generic of +`displayName` → `ensName` → `basename` → `solName`. + +**Added dimension (not in the bench today):** accuracy. A hit is counted accurate when the returned +label shares a meaningful token with the curated `Hint` for that anchor. + +| Provider | Coverage | Contract | EOA | Accurate | Accurate given hit | p50 | +|---|---|---|---|---|---|---| +| **Serialized** | **77.0%** | 76.3% | 78.0% | **58.0%** | 75.3% | 45 ms | +| Mobula | 59.0% | 54.2% | 65.9% | 44.0% | 74.6% | 34 ms | + +Per chain (coverage / accuracy): + +| Chain | n | Serialized | Mobula | +|---|---|---|---| +| ethereum | 32 | 96.9% / 78.1% | 62.5% / 53.1% | +| bnb | 15 | 80.0% / 60.0% | 80.0% / 46.7% | +| base | 17 | 76.5% / 58.8% | 70.6% / 47.1% | +| arbitrum | 17 | 58.8% / 52.9% | 47.1% / 41.2% | +| solana | 19 | 57.9% / 26.3% | 36.8% / 26.3% | + +**Verdict: include.** Serialized leads on coverage and on absolute accuracy on every chain in scope. + +**Bench flaw this exposed.** 25% of Serialized's hits are wrong (19 of 77). Mobula's ratio is +almost identical (74.6% accurate given hit). The bench scores presence of a non-generic string, so a +personal ENS or `.sol` name registered against a well-known contract counts as a correct entity +label. Concrete cases: Permit2 → `dex.davywoodfi.eth`, Uniswap V3 Router 2 → `factory.vibebet.eth`, +Base USDC → `jakie.base.eth`, Raydium Authority → `bonklanatoken.sol`, BSC USDT → `Fake_Phishing6512`, +OKX 1 → `Bittrex 3`, Bitfinex → `Polygon`. + +This is a pre-existing, provider-neutral gameability hole. It should be fixed **before** Serialized +is published, not after, otherwise the fix looks like a reaction to a new entrant beating the +incumbent. Recommended fix: score against the curated `Hint` (the harness already carries it and +already ignores it), or exclude name-service strings from the hit rule. + +## 6. Bench 004 — metadata-coverage + +**Replica rule.** Same 4 fields as the bench (`logo`, `description`, `twitter`, `website`). Discovery +via GeckoTerminal `new_pools` (an independent third source, so neither provider's own discovery +biases the sample). Both providers queried on the **same token set**, and only tokens that **both** +resolved are scored, so the denominator is identical. + +| Chain | paired n | Serialized | Mobula | +|---|---|---|---| +| solana | 48 | 10.4% | 34.4% | +| base | 37 | 19.6% | 34.5% | +| bsc | 38 | 60.5% | 80.9% | +| **total** | 123 | **28.7%** | **48.8%** | + +Field breakdown: + +| Chain | Field | Serialized | Mobula | +|---|---|---|---| +| solana | logo | 22.9% | 100.0% | +| solana | description | 8.3% | 25.0% | +| solana | twitter | 8.3% | 8.3% | +| solana | website | 2.1% | 4.2% | +| base | logo | 37.8% | 100.0% | +| base | description | 13.5% | 13.5% | +| base | twitter | 18.9% | 16.2% | +| base | website | 8.1% | 8.1% | +| bsc | logo | 78.9% | 100.0% | +| bsc | description | 78.9% | 71.1% | +| bsc | twitter | 78.9% | 76.3% | +| bsc | website | 5.3% | 76.3% | + +**Verdict: include, but fix the logo field first.** + +**Bench flaw this exposed.** Mobula returns `logo` = 100% on all three chains. That is not a data +advantage, it is a URL-shape artifact: Mobula rewrites every logo onto `metadata.mobula.io` at a +deterministic path (`/assets/logos/__
`), so the field is never empty +regardless of whether an image exists. Serialized returns the upstream source URL instead +(`ipfs.io`, `gmgn.ai`, `axiomtrading.axiom-cdn.io`, `pbs.twimg.com`, `flap.sh`). A HEAD check on 12 +distinct Mobula logo URLs resolved 11 and 404'd 1. + +The bench currently measures *"is the field non-empty"*, which any provider can win by construction +by rewriting to its own CDN. It should measure *"does the logo resolve"* (HEAD 200 with an image +content type). Mobula is our own product and it is the beneficiary of the current rule, so this needs +fixing on fairness grounds before a competitor is added to the same leaderboard. + +Excluding the logo field entirely, on the remaining three fields Serialized is level with Mobula on +Base, ahead on BSC description and twitter, and behind on Solana and on BSC website. + +## 7. Benches 005 / 090 — chain-count coverage + +| Bench | Incumbents | Serialized | +|---|---|---| +| 005 asset-registry | CoinGecko 465, CoinPaprika 310, CoinStats 149, Mobula 81 | **19** | +| 090 dex-network | GeckoTerminal 247, Codex 123, Sim by Dune 64, DexPaprika 35 | **19** | + +**Verdict: exclude for now.** Serialized would rank last by a wide margin on both. The metric is +breadth, their product is deliberately narrow-and-deep. Adding them here produces a true but +uninformative row and gives them a reason to refuse every other bench. Revisit only if they ask. + +Note: the GeckoTerminal count returned 100 in this run because the ad-hoc pager stopped early on +rate limit. The production harness value of 247 is the correct one. + +## 8. Bench 001 — aggregator-head-lag + +**Replica rule.** Single process on `ocb-par-main`, two WebSocket connections open simultaneously, +subscribed to the **same three tokens** (BONK / Solana, DEGEN / Base, CAKE / BNB). Serialized: +`wss://api.serialized.xyz/v1/stream`, `subscribe` on channel `trades` with `{chain, address}`. +Mobula: `wss://api.mobula.io`, `fast-trade` with `assetMode: true`. Trades matched by transaction +hash, so every comparison is the same on-chain event seen by both pipelines. 240 s window. + +Note on protocol shape: Serialized's `params.pools` is a comma-separated **string**, not an array, +and `address` is required even when `pools` is supplied. Their trade events carry the hash inside +`data.id` as `:`, not as a `txHash` field, despite the docs naming `txHash` as the +dedup key. + +### Relative arrival, the only comparison free of self-reported timestamps + +| Chain | matched n | p10 | p50 | p90 | Serialized first | +|---|---|---|---|---|---| +| solana | 74 | −288 ms | **−0 ms** | +29 ms | 51% | +| base | 13 | −43 ms | +88 ms | +177 ms | 23% | +| bnb | 0 | — | — | — | Mobula returned no CAKE events in this window | +| **all** | 87 | | **+2 ms** | | **47%** | + +Negative means Serialized delivered the trade first. **It is a dead heat.** Across 87 matched +trades the median difference is 2 ms and the two feeds trade the lead roughly half the time. On +Base, Mobula was actually ahead on 77% of trades despite Serialized running a preconfirmation feed. + +### The finding that matters: providers disagree about when the trade happened + +For the **same transaction hash**, the two providers' own on-chain timestamps differ: + +| Chain | serialized `at` minus mobula `date` | p10 | p50 | p90 | +|---|---|---|---|---| +| solana | | −1,620 ms | **−707 ms** | −353 ms | +| base | | +1,000 ms | **+1,000 ms** | +2,000 ms | + +Consequence, measured directly: + +| Chain | Provider | Self-reported lag p50 | Actually delivered first | +|---|---|---|---| +| solana | Mobula | +0.04 s | 49% | +| solana | Serialized | +0.75 s | 51% | +| base | Serialized | −0.33 s (13/13 negative) | 23% | +| base | Mobula | +0.78 s | 77% | + +Read those two tables together. On Solana, Mobula's self-reported lag is 19× better than +Serialized's, and the two arrive at the same instant. On Base, Serialized's self-reported lag is +negative while Mobula beats it to the wire on three trades out of four. **Any head-lag number built +on a provider's own timestamp is not a latency measurement, it is a measurement of where that +provider chooses to put its clock.** + +Bench 001 already does the right thing by referencing archive nodes and validating against block +hashes, so the published leaderboard is not affected by this. It does mean two things going forward: +the archive-node reference is load-bearing and must never be relaxed to a self-reported field, and +Serialized cannot be onboarded through a shortcut that trusts their `at`. + +**Blocking issue: Base preconfirmations.** Serialized emits Base trades from flashblocks +preconfirmations, ahead of the block timestamp they attach to the event. Measured on their stream, +Base events arrive with a **negative** lag versus their own `at` field (p50 −1.86 s, 3/3 negative in +the first sample). Their docs state this explicitly (~2.5 s ahead). + +Measured on Base against Mobula on matched hashes: Serialized reports 13/13 negative self-lag while +losing the actual race 77% of the time. So the preconfirmation feed does **not** currently translate +into earlier delivery on Base, it only translates into an earlier timestamp. That distinction has to +survive into whatever the bench publishes. Options, in order of preference: + +1. Add a `confirmation` dimension (`confirmed` / `preconfirmed`) and rank within it. +2. Clamp negative lag to 0 and footnote it. +3. Exclude Base for Serialized. + +Option 3 is the least honest, because their preconfirmed feed is a real product advantage for a +trading UI. Option 1 is the one that survives a public dispute. + +## 9. Bench 067 — portfolio-chain-coverage + +`GET /v1/wallet/positions` returned `200` on all 19 chains with zero errors. Rows came back on 6 +chains (ethereum 238, bsc 121, hyperevm 23, solana 13, arbitrum 12, avalanche 9) and 0 rows on the +other 13. + +**This test is inconclusive and must not be quoted.** The zero-row chains reflect probe addresses +that hold nothing there, not unsupported chains. Bench 067 compares self-declared coverage against +probe-verified coverage, which requires a curated funded address per chain. That curation is the +work item; the endpoint itself is ready. + +## 10. Bench 102 / 033 — not applicable + +Serialized is not a swap router and returns no quotes. `token-quote-coverage` (102) and +`evm-quote-latency` (033) cannot include them. Their `/v1/pulse` endpoint tracks ~90 launchpads and +is usable as an **alternative discovery source** for bench 102, which is a separate question. + +## 11. Latency, head to head + +REST, identical call shape, 20 samples each, from `ocb-par-main`: + +| Chain | Serialized p50 / p90 | Mobula p50 / p90 | +|---|---|---| +| base | 72 ms / 180 ms | 74 ms / 364 ms | +| solana | 50 ms / 58 ms | 39 ms / 180 ms | +| ethereum | 43 ms / 57 ms | 40 ms / 186 ms | + +Median is a tie. The tail is not: Serialized's p90 is 2× to 3× tighter on every chain. That +consistency is the more defensible claim, and it is not currently measured by any bench. + +## 12. Stretch tests + +| Test | Result | +|---|---| +| Burst threshold | Hard cap at exactly 40 concurrent. 20 and 40 pass clean; 60/100/150 return 40× `200` and the remainder `429` with an explicit `Burst limit: max 40 requests per second` message | +| Sustained 60 s @ 15 rps | 891/891 `200`, p50 38 ms, p99 67 ms, zero degradation | +| `POST /v1/token` batch | 25 items → `200`, 526 ms, 25 rows | +| `POST /v1/token/price` batch | 100 items → `200`, 38 ms, 100 rows | +| Batch over cap | 200 items → `400 INVALID_PARAM`, "must NOT have more than 100 items". Enforced, not silently truncated | +| OHLCV page cap | `limit` ≤ 500, enforced with a clear `400` | +| OHLCV history depth | 1s → 0.01 d, 1m → 0.35 d, 5m → 1.73 d, 1h → 20.8 d, 1d → 499 d (back to 2025-04-24), 1w → 973 d (back to 2024-01-04) | +| Trades pagination | 10 cursor pages, 1,000 trades in 6.4 s, no gaps or repeats | +| Error contract | `INVALID_CHAIN`, `INVALID_PARAM`, `NOT_FOUND`, `UNAUTHORIZED`, `RATE_LIMITED` all machine-readable and correct for the case | + +## 13. Defects found + +| # | Endpoint | Symptom | +|---|---|---| +| 1 | `GET /v1/wallet/equity/history` | `503 UPSTREAM_ERROR` after a 10 s hang, reproduced twice | +| 2 | `GET /v1/wallet/transfers` | 6.5 s response on a routine call. Not benchmarkable as-is | +| 3 | `GET /v1/token/trades`, `/stats`, `/dev-tokens` | `404` for the native wrapped mint (`So111…112`). Native is treated as a quote asset, never as a token. Any harness iterating a standard basket will hit this | +| 4 | Parameter naming | Three conventions on one API: `/v1/pulse` takes `chains` (plural), `/v1/wallet/*` takes `wallet`, `/v1/wallet/profile` takes `address` | + +## 14. Third-party sourcing + +Worth knowing before any commercial discussion, neutral observation either way: + +- Their token `iconUrl` values are upstream URLs from `cdn.dexscreener.com`, `ipfs.io`, + `raw.githubusercontent.com`, `arweave.net`, `gmgn.ai`, `axiomtrading.axiom-cdn.io`. +- Their wallet-profile entity avatars are served from `metadata.mobula.io`, our own CDN. + +## 15. Recommended sequence + +1. Fix the bench 004 logo rule (resolve-check instead of presence-check) and the bench 008 hit rule + (score against the curated hint). Both are provider-neutral fairness fixes and both should land + before a new entrant appears on those leaderboards. +2. Onboard Serialized to bench 004 and bench 008. Both are 1:1 endpoint mappings. +3. Bench 001: they are level with Mobula on wall-clock delivery (p50 +2 ms over 87 matched trades), + so they belong on the leaderboard. Decide the Base preconfirmation policy first, and keep the + archive-node reference: this audit showed self-reported timestamps disagree by up to 1.6 s on the + same transaction. +4. Curate funded probe addresses per chain for bench 067, then onboard. +5. Leave 005 and 090 alone unless they ask. +6. Consider a new token-security bench, where their `/v1/token/security` (18 fields) and + `/v1/audit/contract` are a genuine differentiator rather than a last-place row. + +Every onboarding needs a `docker build --no-cache` of the materialize-worker on `ocb-par-main` +after the harness change, or the new provider will not appear. diff --git a/harnesses/metadata-coverage/cmd/script/config.go b/harnesses/metadata-coverage/cmd/script/config.go index a3fd51143..57fe834e2 100644 --- a/harnesses/metadata-coverage/cmd/script/config.go +++ b/harnesses/metadata-coverage/cmd/script/config.go @@ -10,6 +10,7 @@ import ( type Config struct { CoinGeckoAPIKey string MobulaAPIKey string + SerializedAPIKey string DefinedSessionCookie string MonitorRegion string // Deployment region: us-west, us-east, singapore, etc. MobulaWSURL string // Mobula fast-trade WebSocket endpoint (allows staging to use EU-specific cluster) @@ -21,6 +22,7 @@ func loadEnv() (*Config, error) { // First, try to load from environment variables (for production/Railway) config.CoinGeckoAPIKey = strings.TrimSpace(os.Getenv("COINGECKO_API_KEY")) config.MobulaAPIKey = strings.TrimSpace(os.Getenv("MOBULA_API_KEY")) + config.SerializedAPIKey = strings.TrimSpace(os.Getenv("SERIALIZED_API_KEY")) config.DefinedSessionCookie = strings.TrimSpace(os.Getenv("DEFINED_SESSION_COOKIE")) config.MonitorRegion = strings.TrimSpace(os.Getenv("MONITOR_REGION")) config.MobulaWSURL = strings.TrimSpace(os.Getenv("MOBULA_WS_URL")) diff --git a/harnesses/metadata-coverage/cmd/script/metadata_coverage_monitor.go b/harnesses/metadata-coverage/cmd/script/metadata_coverage_monitor.go index 106898ecd..57e555301 100644 --- a/harnesses/metadata-coverage/cmd/script/metadata_coverage_monitor.go +++ b/harnesses/metadata-coverage/cmd/script/metadata_coverage_monitor.go @@ -65,18 +65,20 @@ type ProviderCoverage struct { // MetadataCoverageStats holds overall stats type MetadataCoverageStats struct { - mu sync.Mutex - Mobula ProviderCoverage - Codex ProviderCoverage - Jupiter ProviderCoverage - LastPrint time.Time + mu sync.Mutex + Mobula ProviderCoverage + Codex ProviderCoverage + Jupiter ProviderCoverage + Serialized ProviderCoverage + LastPrint time.Time } var ( coverageStats = &MetadataCoverageStats{ - Mobula: ProviderCoverage{Provider: "mobula"}, - Codex: ProviderCoverage{Provider: "codex"}, - Jupiter: ProviderCoverage{Provider: "jupiter"}, + Mobula: ProviderCoverage{Provider: "mobula"}, + Codex: ProviderCoverage{Provider: "codex"}, + Jupiter: ProviderCoverage{Provider: "jupiter"}, + Serialized: ProviderCoverage{Provider: "serialized"}, } tokenQueue = make(chan TokenToCheck, 500) metadataClient = &http.Client{Timeout: 10 * time.Second} @@ -194,12 +196,12 @@ type CodexTokenResponse struct { // CodexEnhancedToken matches the EnhancedToken type from Codex API type CodexEnhancedToken struct { - Address string `json:"address"` - Name string `json:"name"` - Symbol string `json:"symbol"` - Decimals int `json:"decimals"` - NetworkID int `json:"networkId"` - Info *CodexTokenInfo `json:"info"` + Address string `json:"address"` + Name string `json:"name"` + Symbol string `json:"symbol"` + Decimals int `json:"decimals"` + NetworkID int `json:"networkId"` + Info *CodexTokenInfo `json:"info"` SocialLinks *CodexSocialLinks `json:"socialLinks"` } @@ -215,11 +217,11 @@ type CodexTokenInfo struct { // CodexSocialLinks contains social media links for the token type CodexSocialLinks struct { - Twitter string `json:"twitter"` - Website string `json:"website"` - Telegram string `json:"telegram"` - Discord string `json:"discord"` - Github string `json:"github"` + Twitter string `json:"twitter"` + Website string `json:"website"` + Telegram string `json:"telegram"` + Discord string `json:"discord"` + Github string `json:"github"` } func getCodexNetworkID(chainID string) int { @@ -553,6 +555,8 @@ func updateStats(provider string, fields MetadataFields) { stats = &coverageStats.Codex case "jupiter": stats = &coverageStats.Jupiter + case "serialized": + stats = &coverageStats.Serialized default: return } @@ -601,7 +605,7 @@ func printCoverageStats() { fmt.Printf("║ Provider │ Checks │ Logo │ Name │ Symbol│ Desc │Twitter│Website│Telegram│ Errors │\n") fmt.Printf("╠══════════════════════════════════════════════════════════════════════════════╣\n") - for _, stats := range []*ProviderCoverage{&coverageStats.Mobula, &coverageStats.Codex, &coverageStats.Jupiter} { + for _, stats := range []*ProviderCoverage{&coverageStats.Mobula, &coverageStats.Codex, &coverageStats.Jupiter, &coverageStats.Serialized} { if stats.TotalChecks == 0 { fmt.Printf("║ %-8s │ %6d │ - │ - │ - │ - │ - │ - │ - │ %6d ║\n", stats.Provider, stats.TotalChecks, stats.ErrorCount) @@ -691,6 +695,23 @@ func checkTokenMetadata(token TokenToCheck, config *Config) { RecordMetadataLatency("jupiter", chainName, jupiterResult.ResponseTimeMs, config.MonitorRegion) } + // Check Serialized (18 EVM chains + Solana; skipped elsewhere) + var serializedResult MetadataFields + if _, supported := serializedChainID(token.ChainID); supported { + serializedResult = checkSerializedMetadata(token, config.SerializedAPIKey) + if serializedResult.Error != "" { + fmt.Printf("[META][SERIALIZED][%s] %s | %s | err=%s\n", + chainName, token.Symbol, token.Address, serializedResult.Error) + } + updateStats("serialized", serializedResult) + + RecordMetadataCoverage("serialized", chainName, "logo", serializedResult.HasLogo, config.MonitorRegion) + RecordMetadataCoverage("serialized", chainName, "description", serializedResult.HasDescription, config.MonitorRegion) + RecordMetadataCoverage("serialized", chainName, "twitter", serializedResult.HasTwitter, config.MonitorRegion) + RecordMetadataCoverage("serialized", chainName, "website", serializedResult.HasWebsite, config.MonitorRegion) + RecordMetadataLatency("serialized", chainName, serializedResult.ResponseTimeMs, config.MonitorRegion) + } + // Single condensed log line boolToIcon := func(b bool) string { if b { @@ -709,11 +730,17 @@ func checkTokenMetadata(token TokenToCheck, config *Config) { // without cross-referencing logs. Address goes after symbol; 4 boolean // columns per provider so website is visible alongside logo/desc/twitter // (the page renders 4 fields, the prior 3-column line hid that one). - fmt.Printf("[META] %s/%s %s | M:%s%s%s%s | C:%s%s%s%s | J:%s\n", + serializedCols := "----" + if _, supported := serializedChainID(token.ChainID); supported { + serializedCols = boolToIcon(serializedResult.HasLogo) + boolToIcon(serializedResult.HasDescription) + + boolToIcon(serializedResult.HasTwitter) + boolToIcon(serializedResult.HasWebsite) + } + + fmt.Printf("[META] %s/%s %s | M:%s%s%s%s | C:%s%s%s%s | J:%s | S:%s\n", token.Symbol, chainName, token.Address, boolToIcon(mobulaResult.HasLogo), boolToIcon(mobulaResult.HasDescription), boolToIcon(mobulaResult.HasTwitter), boolToIcon(mobulaResult.HasWebsite), boolToIcon(codexResult.HasLogo), boolToIcon(codexResult.HasDescription), boolToIcon(codexResult.HasTwitter), boolToIcon(codexResult.HasWebsite), - jupiterLogo) + jupiterLogo, serializedCols) // Print stats every 50 checks (reduced from 10) coverageStats.mu.Lock() @@ -808,4 +835,3 @@ func runMetadataCoverageMonitor(config *Config, stopChan <-chan struct{}) { } } } - diff --git a/harnesses/metadata-coverage/cmd/script/serialized_rest_monitor.go b/harnesses/metadata-coverage/cmd/script/serialized_rest_monitor.go new file mode 100644 index 000000000..812b134ea --- /dev/null +++ b/harnesses/metadata-coverage/cmd/script/serialized_rest_monitor.go @@ -0,0 +1,162 @@ +package main + +import ( + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "strings" + "sync" + "time" +) + +// ============================================================================ +// Serialized — token metadata coverage +// +// GET /v1/token/metadata?chain=&address= returns the four +// canonical fields this bench scores, under different names than Mobula +// and Codex: +// +// logo -> iconUrl +// description -> description +// twitter -> twitterUrl +// website -> websiteUrl +// +// Chain ids are already in the bench's own shape ("solana", "evm:56", +// "evm:8453"), so no translation table is needed beyond normalising the +// legacy "solana:solana" form that Pulse V2 sometimes emits. +// +// One asymmetry worth knowing when reading the leaderboard: Serialized +// returns the *upstream* icon URL (ipfs.io, cdn.dexscreener.com, twimg, +// launchpad CDNs) while Mobula rewrites every logo onto its own CDN at a +// deterministic path, so Mobula's logo field is non-empty by construction. +// The bench currently scores "field non-empty", not "image resolves". +// See docs/methodology/serialized-onboarding-audit.md §6. +// ============================================================================ + +const serializedTokenMetadataURL = "https://api.serialized.xyz/v1/token/metadata" + +// Serialized enforces a hard burst cap of 40 requests per second per key +// and returns 429 above it. The queue-driven monitor can burst well past +// that during a launch spike, which would show up as coverage loss rather +// than as a rate-limit error. Pace the calls at a fixed floor instead. +var ( + serializedMetaMu sync.Mutex + serializedMetaLast time.Time +) + +const serializedMetaMinInterval = 60 * time.Millisecond // ~16 rps against a 40 rps cap + +func serializedMetaThrottle() { + serializedMetaMu.Lock() + defer serializedMetaMu.Unlock() + if wait := time.Until(serializedMetaLast.Add(serializedMetaMinInterval)); wait > 0 { + time.Sleep(wait) + } + serializedMetaLast = time.Now() +} + +// serializedChainID normalises the bench's chain id to what Serialized +// accepts. Returns false when the chain is outside their coverage, so the +// caller skips the check instead of recording a miss. +func serializedChainID(chainID string) (string, bool) { + c := chainID + if c == "solana:solana" { + c = "solana" + } + if c == "solana" { + return c, true + } + if !strings.HasPrefix(c, "evm:") { + return "", false + } + // 18 EVM chains, live as of onboarding (2026-09-05). + switch c { + case "evm:1", "evm:56", "evm:130", "evm:143", "evm:196", "evm:988", + "evm:1514", "evm:2741", "evm:4217", "evm:4326", "evm:4663", + "evm:5042", "evm:8453", "evm:9745", "evm:42161", "evm:43114", + "evm:57073", "evm:645749": + return c, true + } + return "", false +} + +type SerializedTokenMetadataResponse struct { + Data struct { + Name string `json:"name"` + Symbol string `json:"symbol"` + IconURL string `json:"iconUrl"` + Description string `json:"description"` + TwitterURL string `json:"twitterUrl"` + WebsiteURL string `json:"websiteUrl"` + TelegramURL string `json:"telegramUrl"` + } `json:"data"` +} + +func checkSerializedMetadata(token TokenToCheck, apiKey string) MetadataFields { + result := MetadataFields{} + + chain, ok := serializedChainID(token.ChainID) + if !ok { + result.Error = "chain_unsupported" + return result + } + if apiKey == "" { + result.Error = "no_api_key" + return result + } + + serializedMetaThrottle() + + params := url.Values{} + params.Add("chain", chain) + params.Add("address", token.Address) + + req, err := http.NewRequest("GET", fmt.Sprintf("%s?%s", serializedTokenMetadataURL, params.Encode()), nil) + if err != nil { + result.Error = fmt.Sprintf("request_create_error: %v", err) + return result + } + // Raw key, no Bearer prefix — a prefixed key is rejected with 401. + req.Header.Set("Authorization", apiKey) + req.Header.Set("Accept", "application/json") + + startTime := time.Now() + resp, err := metadataClient.Do(req) + result.ResponseTimeMs = float64(time.Since(startTime).Milliseconds()) + if err != nil { + result.Error = fmt.Sprintf("request_error: %v", err) + return result + } + defer resp.Body.Close() + + if resp.StatusCode != 200 { + result.Error = fmt.Sprintf("status_%d", resp.StatusCode) + return result + } + + body, err := io.ReadAll(resp.Body) + if err != nil { + result.Error = fmt.Sprintf("read_error: %v", err) + return result + } + + var response SerializedTokenMetadataResponse + if err := json.Unmarshal(body, &response); err != nil { + result.Error = fmt.Sprintf("parse_error: %v", err) + return result + } + + d := response.Data + result.HasName = d.Name != "" + result.HasSymbol = d.Symbol != "" + result.HasLogo = d.IconURL != "" + result.LogoURL = d.IconURL + result.HasDescription = d.Description != "" + result.HasTwitter = d.TwitterURL != "" + result.HasWebsite = d.WebsiteURL != "" + result.HasTelegram = d.TelegramURL != "" + + return result +} diff --git a/harnesses/wallet-labels/.env.example b/harnesses/wallet-labels/.env.example index dc1a7d7da..0d1d17414 100644 --- a/harnesses/wallet-labels/.env.example +++ b/harnesses/wallet-labels/.env.example @@ -10,6 +10,9 @@ MORALIS_API_KEY= # Helius (Solana) HELIUS_API_KEY= +# Serialized (18 EVM chains + Solana). Raw key, no Bearer prefix. +SERIALIZED_API_KEY= + # Tuning WALLET_LABELS_CHECK_DELAY_SECONDS=30 WALLET_LABELS_WORKERS=8 diff --git a/harnesses/wallet-labels/cmd/script/config.go b/harnesses/wallet-labels/cmd/script/config.go index 134cc7417..2bb00dd97 100644 --- a/harnesses/wallet-labels/cmd/script/config.go +++ b/harnesses/wallet-labels/cmd/script/config.go @@ -11,27 +11,29 @@ import ( // printed in full — only their length, so misconfigured deploys // fail loudly without leaking material. type Config struct { - MobulaAPIKey string - MoralisAPIKey string - HeliusAPIKey string + MobulaAPIKey string + MoralisAPIKey string + HeliusAPIKey string + SerializedAPIKey string - CheckDelay time.Duration - Workers int - QueueSize int - PromListen string - LogsToken string + CheckDelay time.Duration + Workers int + QueueSize int + PromListen string + LogsToken string } func loadConfig() *Config { c := &Config{ - MobulaAPIKey: os.Getenv("MOBULA_API_KEY"), - MoralisAPIKey: os.Getenv("MORALIS_API_KEY"), - HeliusAPIKey: os.Getenv("HELIUS_API_KEY"), - CheckDelay: parseDurationSec("WALLET_LABELS_CHECK_DELAY_SECONDS", 30), - Workers: parseInt("WALLET_LABELS_WORKERS", 8), - QueueSize: parseInt("WALLET_LABELS_QUEUE_SIZE", 2000), - PromListen: envDefault("PROM_LISTEN_ADDR", ":2112"), - LogsToken: os.Getenv("LOGS_TOKEN"), + MobulaAPIKey: os.Getenv("MOBULA_API_KEY"), + MoralisAPIKey: os.Getenv("MORALIS_API_KEY"), + HeliusAPIKey: os.Getenv("HELIUS_API_KEY"), + SerializedAPIKey: os.Getenv("SERIALIZED_API_KEY"), + CheckDelay: parseDurationSec("WALLET_LABELS_CHECK_DELAY_SECONDS", 30), + Workers: parseInt("WALLET_LABELS_WORKERS", 8), + QueueSize: parseInt("WALLET_LABELS_QUEUE_SIZE", 2000), + PromListen: envDefault("PROM_LISTEN_ADDR", ":2112"), + LogsToken: os.Getenv("LOGS_TOKEN"), } fmt.Println("=== Wallet Labels Coverage Monitor ===") @@ -42,6 +44,7 @@ func loadConfig() *Config { fmt.Printf(" Mobula key set: %v (len=%d)\n", c.MobulaAPIKey != "", len(c.MobulaAPIKey)) fmt.Printf(" Moralis key set: %v (len=%d)\n", c.MoralisAPIKey != "", len(c.MoralisAPIKey)) fmt.Printf(" Helius key set: %v (len=%d)\n", c.HeliusAPIKey != "", len(c.HeliusAPIKey)) + fmt.Printf(" Serialized key set: %v (len=%d)\n", c.SerializedAPIKey != "", len(c.SerializedAPIKey)) fmt.Println() return c diff --git a/harnesses/wallet-labels/cmd/script/main.go b/harnesses/wallet-labels/cmd/script/main.go index 0e3d5071c..abdfea2c2 100644 --- a/harnesses/wallet-labels/cmd/script/main.go +++ b/harnesses/wallet-labels/cmd/script/main.go @@ -53,6 +53,7 @@ func buildProviders(cfg *Config) []Provider { NewMobulaProvider(cfg.MobulaAPIKey), NewMoralisProvider(cfg.MoralisAPIKey), NewHeliusProvider(cfg.HeliusAPIKey), + NewSerializedProvider(cfg.SerializedAPIKey), NewBlockscoutProvider(), NewOLIProvider(), NewTonAPIProvider(), diff --git a/harnesses/wallet-labels/cmd/script/serialized.go b/harnesses/wallet-labels/cmd/script/serialized.go new file mode 100644 index 000000000..b35c79545 --- /dev/null +++ b/harnesses/wallet-labels/cmd/script/serialized.go @@ -0,0 +1,129 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "sync" + "time" +) + +// Serialized exposes an identity graph rather than a pure entity-label +// service: GET /v1/wallet/profile returns a display name, ENS / Basename / +// .sol resolution, socials and linked wallets for an address. +// +// We read it with the same precedence rule every other provider gets — +// first non-generic name signal wins — so the bench compares like with +// like. Note for whoever reads the leaderboard: because `displayName` +// can resolve to a personal name-service record rather than a curated +// entity, a share of Serialized's hits name the *holder* of an address +// rather than the *entity* behind it (a measured ~25% of hits at +// onboarding time, against ~25% for Mobula on the same sample). That is +// a property of the bench's hit rule, not of this provider, and the fix +// belongs in the scoring rule for everyone at once. See +// docs/methodology/serialized-onboarding-audit.md §5. +type SerializedProvider struct { + apiKey string +} + +func NewSerializedProvider(key string) *SerializedProvider { + return &SerializedProvider{apiKey: key} +} + +func (p *SerializedProvider) Name() string { return "serialized" } + +// serializedChains are the anchor-list chains Serialized indexes. Their +// full surface is 18 EVM chains plus Solana; the ones below are the +// intersection with the curated anchor sample. Chains outside this set +// are skipped rather than counted as misses, same as every other +// chain-restricted provider in this harness. +var serializedChains = map[string]bool{ + "ethereum": true, + "bnb": true, + "base": true, + "arbitrum": true, + "solana": true, +} + +func (p *SerializedProvider) Supports(chain string) bool { return serializedChains[chain] } + +// Serialized enforces a hard burst cap of 40 requests per second per key +// and answers anything above it with 429. The harness runs 8 workers with +// sub-100ms responses, which clears that cap easily and silently turns +// coverage into a rate-limit artifact (measured: 60 of 100 anchors lost to +// 429, dropping apparent coverage from 77% to 37%). Serialize the calls at +// a conservative fixed interval instead of relying on worker count. +var ( + serializedMu sync.Mutex + serializedLast time.Time +) + +const serializedMinInterval = 60 * time.Millisecond // ~16 rps, well under the 40 rps cap + +func serializedThrottle() { + serializedMu.Lock() + defer serializedMu.Unlock() + if wait := time.Until(serializedLast.Add(serializedMinInterval)); wait > 0 { + time.Sleep(wait) + } + serializedLast = time.Now() +} + +func (p *SerializedProvider) Lookup(ctx context.Context, chain, address string) LabelResult { + res := LabelResult{Provider: p.Name(), Chain: chain, Address: address} + if !p.Supports(chain) || p.apiKey == "" { + return res + } + + serializedThrottle() + + start := time.Now() + req, _ := http.NewRequestWithContext(ctx, "GET", + "https://api.serialized.xyz/v1/wallet/profile?address="+address, nil) + // Raw key, no Bearer prefix — a prefixed key is rejected with 401. + req.Header.Set("Authorization", p.apiKey) + req.Header.Set("Accept", "application/json") + + resp, err := httpClient.Do(req) + res.LatencyMs = time.Since(start).Milliseconds() + if err != nil { + res.Err = err + return res + } + defer resp.Body.Close() + + if resp.StatusCode != 200 { + res.Err = fmt.Errorf("status_%d", resp.StatusCode) + return res + } + + var body struct { + Data struct { + Profile *struct { + DisplayName string `json:"displayName"` + ENSName string `json:"ensName"` + Basename string `json:"basename"` + SolName string `json:"solName"` + } `json:"profile"` + } `json:"data"` + } + if err := json.NewDecoder(resp.Body).Decode(&body); err != nil { + res.Err = fmt.Errorf("parse: %w", err) + return res + } + if body.Data.Profile == nil { + return res + } + + prof := body.Data.Profile + for _, candidate := range []string{prof.DisplayName, prof.ENSName, prof.Basename, prof.SolName} { + if !genericLabel(candidate) { + res.Label = candidate + res.HasLabel = true + res.Raw = map[string]any{"label": candidate, "source": "wallet_profile"} + break + } + } + return res +} diff --git a/public/logos/serialized-wordmark.svg b/public/logos/serialized-wordmark.svg new file mode 100644 index 000000000..7f3d83a9b --- /dev/null +++ b/public/logos/serialized-wordmark.svg @@ -0,0 +1,15 @@ + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/public/logos/serialized.svg b/public/logos/serialized.svg new file mode 100644 index 000000000..3198acb25 --- /dev/null +++ b/public/logos/serialized.svg @@ -0,0 +1,5 @@ + + + + + diff --git a/src/data/provider-registry.ts b/src/data/provider-registry.ts index 9abea557b..5d4b620f8 100644 --- a/src/data/provider-registry.ts +++ b/src/data/provider-registry.ts @@ -115,6 +115,47 @@ export const PROVIDER_REGISTRY: Record = { "Independent crypto market data API. Token prices, OHLCV, exchange tickers, and contract/platform lookups across 300+ supported chains. Public free tier with no auth.", twitter: "@coinpaprika", }, + serialized: { + url: "https://serialized.xyz", + description: + "Onchain market data and token security API for trading apps. Own indexers across 18 EVM chains plus Solana, REST plus a single-connection WebSocket for trades and token updates.", + longDescription: + "Serialized runs its own indexers rather than reselling a third-party pipeline, and scopes coverage deliberately narrow: 18 EVM chains plus Solana, weighted toward venues where launchpad and memecoin flow actually lands (Base, BNB, Solana, HyperEVM, Abstract, Monad, MegaETH, Plasma, Tempo, Arc, Robinhood Chain). The surface splits into market data (token details, price, stats, OHLCV from 1s, trades tape, pools, screener, Axiom-style launchpad lifecycle filters), wallet analytics (positions, PnL, closed positions, transfers, funding, an identity graph behind wallet profiles), and token security (holder concentration, sniper and bundler share, LP burn and lock state, plus a separate async contract-audit engine). Streams run over one WebSocket at wss://api.serialized.xyz/v1/stream, billed per connection-minute rather than per message. On Base the trade tape emits preconfirmed flashblock trades ahead of the block timestamp.", + twitter: "@serializedaudit", + docs: "https://docs.serialized.xyz", + chains: [ + "ethereum", + "base", + "bnb", + "arbitrum", + "avalanche", + "solana", + "abstract", + "hyperevm", + "ink", + "story", + "xlayer", + "plasma", + "unichain", + "monad", + "megaeth", + "tempo", + "robinhood", + "arc", + "stable", + ], + features: [ + "Token details, price, windowed stats and OHLCV from 1s to 1M", + "Trades tape with maker resolution, sniper/pro-trader/wash badges", + "Wallet positions, realized and unrealized PnL, funding and transfers", + "Token security: holder concentration, sniper and bundler share, LP burn and lock", + "Async contract audit engine across 18 EVM chains", + "Launchpad lifecycle filters over ~90 launchpads", + "WebSocket streams for trades, token updates and pool updates", + ], + pricing: + "Credit-metered. Free 15k credits/month, Starter 150k, Growth 2M, Enterprise unlimited. Most endpoints cost 1 credit; token/security costs 10 and a fresh contract audit 750.", + }, dexpaprika: { url: "https://dexpaprika.com", description: diff --git a/src/lib/logo-manifest.ts b/src/lib/logo-manifest.ts index 4c4414ca4..b8d36699c 100644 --- a/src/lib/logo-manifest.ts +++ b/src/lib/logo-manifest.ts @@ -99,6 +99,7 @@ const RAW: Record = { // ─── Providers ─── mobula: "/logos/mobula.svg", + serialized: "/logos/serialized.svg", codex: "/logos/codex.svg", polymarket: "/logos/polymarket.png", "polymarket-us": "/logos/polymarket.png", From 45a7c0803882ddeac34747351402603ddb28d2f2 Mon Sep 17 00:00:00 2001 From: Flotapponnier <160007691+Flotapponnier@users.noreply.github.com> Date: Sat, 5 Sep 2026 19:14:17 +0200 Subject: [PATCH 06/10] feat: declare Serialized in the 004 and 008 bench specs (#2262) The harnesses emit provider="serialized" but the leaderboards are driven by the explicit providers block in each spec, so the series never rendered. Adds the block for both benches, same query shape as the incumbents. Validated with SpecSchema.safeParse across all 218 specs; every formula stays under the 240-char cap that silently drops a spec. Claude-Session: https://claude.ai/code/session_01CpArutAtXuBb1BVNUDXoYA Co-authored-by: Claude Opus 5 (1M context) (cherry picked from commit 0730b30e4632ac60df80f4bc3a54d12c2e02367c) --- benchmarks/metadata-coverage.yml | 13 +++++++++++++ benchmarks/wallet-labels-coverage.yml | 13 +++++++++++++ 2 files changed, 26 insertions(+) diff --git a/benchmarks/metadata-coverage.yml b/benchmarks/metadata-coverage.yml index 0381ff2f3..41f764c48 100644 --- a/benchmarks/metadata-coverage.yml +++ b/benchmarks/metadata-coverage.yml @@ -156,6 +156,19 @@ providers: sample_size: sum(increase(metadata_coverage_checks_total{provider="codex"}[24h])) series: 100 * sum(rate(metadata_coverage_success_total{provider="codex"}[1h])) / sum(rate(metadata_coverage_checks_total{provider="codex"}[1h])) + - slug: serialized + name: Serialized + tag: REST `/v1/token/metadata` + formula: "Median of hourly coverage rate (populated logo/description/twitter/website fields divided by total field checks) on Serialized `/v1/token/metadata` for fresh tokens, p50 over 24h." + queries: + p50: quantile_over_time(0.50, (100 * sum(rate(metadata_coverage_success_total{provider="serialized"}[1h])) / sum(rate(metadata_coverage_checks_total{provider="serialized"}[1h])))[24h:1h]) + p90: quantile_over_time(0.90, (100 * sum(rate(metadata_coverage_success_total{provider="serialized"}[1h])) / sum(rate(metadata_coverage_checks_total{provider="serialized"}[1h])))[24h:1h]) + p99: quantile_over_time(0.99, (100 * sum(rate(metadata_coverage_success_total{provider="serialized"}[1h])) / sum(rate(metadata_coverage_checks_total{provider="serialized"}[1h])))[24h:1h]) + mean: 100 * sum(rate(metadata_coverage_success_total{provider="serialized"}[24h])) / sum(rate(metadata_coverage_checks_total{provider="serialized"}[24h])) + success: clamp_max(sum(rate(metadata_coverage_checks_total{provider="serialized"}[24h])) / scalar(sum(rate(metadata_coverage_checks_total{provider="mobula"}[24h]))), 1) + sample_size: sum(increase(metadata_coverage_checks_total{provider="serialized"}[24h])) + series: 100 * sum(rate(metadata_coverage_success_total{provider="serialized"}[1h])) / sum(rate(metadata_coverage_checks_total{provider="serialized"}[1h])) + - slug: jupiter name: Jupiter tag: REST `/v6/tokens` (Solana) diff --git a/benchmarks/wallet-labels-coverage.yml b/benchmarks/wallet-labels-coverage.yml index 8f08fa327..a428f7842 100644 --- a/benchmarks/wallet-labels-coverage.yml +++ b/benchmarks/wallet-labels-coverage.yml @@ -199,6 +199,19 @@ providers: sample_size: sum(increase(wallet_labels_checks_total{provider="mobula"}[24h])) series: 100 * sum(rate(wallet_labels_success_total{provider="mobula"}[1h])) / sum(rate(wallet_labels_checks_total{provider="mobula"}[1h])) + - slug: serialized + name: Serialized + tag: Identity graph, 18 EVM chains + Solana, API key required + formula: "Share of anchor addresses for which Serialized /v1/wallet/profile returns a non-generic identity name, success_total ÷ checks_total over 24h." + queries: + p50: 100 * sum(increase(wallet_labels_success_total{provider="serialized"}[24h])) / sum(increase(wallet_labels_checks_total{provider="serialized"}[24h])) + p90: 100 * sum(increase(wallet_labels_success_total{provider="serialized"}[24h])) / sum(increase(wallet_labels_checks_total{provider="serialized"}[24h])) + p99: 100 * sum(increase(wallet_labels_success_total{provider="serialized"}[24h])) / sum(increase(wallet_labels_checks_total{provider="serialized"}[24h])) + mean: 100 * sum(increase(wallet_labels_success_total{provider="serialized"}[24h])) / sum(increase(wallet_labels_checks_total{provider="serialized"}[24h])) + success: clamp_min(1 - (sum(increase(wallet_labels_fetch_errors_total{provider="serialized"}[24h])) or vector(0)) / clamp_min(sum(increase(wallet_labels_checks_total{provider="serialized"}[24h])), 1), 0) + sample_size: sum(increase(wallet_labels_checks_total{provider="serialized"}[24h])) + series: 100 * sum(rate(wallet_labels_success_total{provider="serialized"}[1h])) / sum(rate(wallet_labels_checks_total{provider="serialized"}[1h])) + - slug: helius name: Helius tag: Solana specialist, native program graph, API key required From e222dcf600310feec057b840f33403200ada1494 Mon Sep 17 00:00:00 2001 From: Flotapponnier <160007691+Flotapponnier@users.noreply.github.com> Date: Sat, 5 Sep 2026 19:42:26 +0200 Subject: [PATCH 07/10] feat: add Serialized to benches 005 and 090 (chain coverage) (#2263) Serialized publishes its chain list at the free GET /v1/meta/chains. The same list answers both benches: they run their own indexers and do not separate asset-registry coverage from DEX-pool coverage, so the count is identical on 005 and 090 by construction. Only chains the endpoint marks status=live are counted, so a future beta or deprecated status cannot inflate the number. Claude-Session: https://claude.ai/code/session_01CpArutAtXuBb1BVNUDXoYA Co-authored-by: Claude Opus 5 (1M context) (cherry picked from commit 8ae50fe8e44b3fb4f5e9b806cd540bab052e4d3d) --- benchmarks/asset-registry-coverage.yml | 13 +++ benchmarks/dex-network-coverage.yml | 13 +++ .../network-coverage/cmd/script/config.go | 6 +- harnesses/network-coverage/cmd/script/main.go | 1 + .../network-coverage/cmd/script/serialized.go | 80 +++++++++++++++++++ 5 files changed, 111 insertions(+), 2 deletions(-) create mode 100644 harnesses/network-coverage/cmd/script/serialized.go diff --git a/benchmarks/asset-registry-coverage.yml b/benchmarks/asset-registry-coverage.yml index c29a75d7f..b252b2001 100644 --- a/benchmarks/asset-registry-coverage.yml +++ b/benchmarks/asset-registry-coverage.yml @@ -95,6 +95,19 @@ providers: sample_size: networks_supported_total{provider="coingecko"} series: networks_supported_total{provider="coingecko"} + - slug: serialized + name: Serialized + tag: Own indexers, 18 EVM + Solana + formula: "Count of chains returned as live by Serialized's `/v1/meta/chains` endpoint, refreshed every 6 hours." + queries: + p50: networks_supported_total{provider="serialized"} + p90: networks_supported_total{provider="serialized"} + p99: networks_supported_total{provider="serialized"} + mean: networks_supported_total{provider="serialized"} + success: clamp_max(networks_supported_total{provider="serialized"} > bool 0, 1) + sample_size: networks_supported_total{provider="serialized"} + series: networks_supported_total{provider="serialized"} + - slug: coinpaprika name: CoinPaprika tag: Market-data API asset registry diff --git a/benchmarks/dex-network-coverage.yml b/benchmarks/dex-network-coverage.yml index 198522514..8e7649b3f 100644 --- a/benchmarks/dex-network-coverage.yml +++ b/benchmarks/dex-network-coverage.yml @@ -90,6 +90,19 @@ providers: sample_size: networks_supported_total{provider="geckoterminal"} series: networks_supported_total{provider="geckoterminal"} + - slug: serialized + name: Serialized + tag: Own indexers, 18 EVM + Solana + formula: "Count of chains returned as live by Serialized's `/v1/meta/chains` endpoint; every listed chain carries DEX pool indexing. Refreshed every 6 hours." + queries: + p50: networks_supported_total{provider="serialized"} + p90: networks_supported_total{provider="serialized"} + p99: networks_supported_total{provider="serialized"} + mean: networks_supported_total{provider="serialized"} + success: clamp_max(networks_supported_total{provider="serialized"} > bool 0, 1) + sample_size: networks_supported_total{provider="serialized"} + series: networks_supported_total{provider="serialized"} + - slug: codex name: Codex tag: Defined.fi DEX data API diff --git a/harnesses/network-coverage/cmd/script/config.go b/harnesses/network-coverage/cmd/script/config.go index 83d27ab8d..99c7310f2 100644 --- a/harnesses/network-coverage/cmd/script/config.go +++ b/harnesses/network-coverage/cmd/script/config.go @@ -13,6 +13,7 @@ type Config struct { CodexSessionCookie string // fallback path: mint JWT from Defined.fi cookie DefinedTokenURL string // optional: pre-minted JWT sidecar CoinStatsAPIKey string + SerializedAPIKey string SimDuneAPIKey string // optional — Sim's public endpoint works keyless, but a key avoids rate limits HTTPProxy string RefreshInterval time.Duration @@ -26,6 +27,7 @@ func loadConfig() *Config { CodexSessionCookie: os.Getenv("DEFINED_SESSION_COOKIE"), DefinedTokenURL: os.Getenv("DEFINED_TOKEN_SERVICE_URL"), CoinStatsAPIKey: os.Getenv("COINSTATS_API_KEY"), + SerializedAPIKey: os.Getenv("SERIALIZED_API_KEY"), SimDuneAPIKey: os.Getenv("SIM_DUNE_API_KEY"), HTTPProxy: os.Getenv("HTTP_PROXY"), RefreshInterval: 6 * time.Hour, @@ -47,8 +49,8 @@ func loadConfig() *Config { } else if c.CodexSessionCookie != "" { codexAuth = "cookie+mint" } - fmt.Printf("Config: refresh=%v, testnets=%v, mobula_key=%v, codex=%s, coinstats_key=%v, sim_dune_key=%v\n", + fmt.Printf("Config: refresh=%v, testnets=%v, mobula_key=%v, codex=%s, coinstats_key=%v, sim_dune_key=%v, serialized_key=%v\n", c.RefreshInterval, c.IncludeTestnets, c.MobulaAPIKey != "", codexAuth, - c.CoinStatsAPIKey != "", c.SimDuneAPIKey != "") + c.CoinStatsAPIKey != "", c.SimDuneAPIKey != "", c.SerializedAPIKey != "") return c } diff --git a/harnesses/network-coverage/cmd/script/main.go b/harnesses/network-coverage/cmd/script/main.go index 885a9ffad..20693522d 100644 --- a/harnesses/network-coverage/cmd/script/main.go +++ b/harnesses/network-coverage/cmd/script/main.go @@ -76,6 +76,7 @@ func fetchAll(cfg *Config) { {"coinstats", fetchCoinStats}, {"coingecko", fetchCoinGecko}, {"dexpaprika", fetchDexPaprika}, + {"serialized", fetchSerialized}, } var wg sync.WaitGroup diff --git a/harnesses/network-coverage/cmd/script/serialized.go b/harnesses/network-coverage/cmd/script/serialized.go new file mode 100644 index 000000000..9ad325b64 --- /dev/null +++ b/harnesses/network-coverage/cmd/script/serialized.go @@ -0,0 +1,80 @@ +package main + +import ( + "encoding/json" + "fmt" + "io" + "net/http" + "time" +) + +// Serialized publishes its chain list at GET /v1/meta/chains. The endpoint +// is free (0 credits) and returns one row per chain with a `status` field. +// +// The same list answers both benches this harness feeds: Serialized runs its +// own indexers and does not separate "chains where we know tokens" from +// "chains where we index DEX pools" — every listed chain carries both. So +// the count is identical on bench 005 and bench 090 by construction, which +// is worth knowing when reading the two leaderboards side by side. +const serializedChainsURL = "https://api.serialized.xyz/v1/meta/chains" + +type serializedChain struct { + Chain string `json:"chain"` // "evm:8453" or "solana" + Name string `json:"name"` + Slug string `json:"slug"` + Family string `json:"family"` + Status string `json:"status"` +} + +type serializedChainsResponse struct { + Data []serializedChain `json:"data"` +} + +func fetchSerialized(cfg *Config) ProviderResult { + res := ProviderResult{Provider: "serialized"} + if cfg.SerializedAPIKey == "" { + res.Err = "missing_api_key" + return res + } + + client := &http.Client{Timeout: 15 * time.Second} + req, _ := http.NewRequest("GET", serializedChainsURL, nil) + // Raw key, no Bearer prefix — a prefixed key is rejected with 401. + req.Header.Set("Authorization", cfg.SerializedAPIKey) + req.Header.Set("Accept", "application/json") + + resp, err := client.Do(req) + if err != nil { + res.Err = fmt.Sprintf("request_error: %v", err) + return res + } + defer resp.Body.Close() + body, _ := io.ReadAll(resp.Body) + + if resp.StatusCode != 200 { + res.Err = fmt.Sprintf("status_%d", resp.StatusCode) + return res + } + + var parsed serializedChainsResponse + if err := json.Unmarshal(body, &parsed); err != nil { + res.Err = fmt.Sprintf("parse_error: %v", err) + return res + } + + for _, c := range parsed.Data { + // Only chains the provider declares live. Everything on this + // endpoint is mainnet, so no testnet filter is needed, but a + // future "beta"/"deprecated" status must not inflate the count. + if c.Status != "live" { + continue + } + res.Networks = append(res.Networks, Network{ + ChainID: c.Chain, + Slug: c.Slug, + Name: c.Name, + }) + } + + return res +} From cb05621a9374bd4bc5292a2cc3fc4d7285fff8e8 Mon Sep 17 00:00:00 2001 From: Flotapponnier <160007691+Flotapponnier@users.noreply.github.com> Date: Sat, 5 Sep 2026 20:28:34 +0200 Subject: [PATCH 08/10] docs: correct the head-lag claim in the Serialized audit, add follow-ups (#2264) v1.0 claimed bench 001 references archive nodes and was therefore unaffected by provider clock disagreement. That repeated the spec instead of reading the harness: there is no archive-node reference in aggregator-head-lag at all, and the leaderboard gauge is computed from each provider's own timestamp. The spec says the opposite in three places. Also records the now-conclusive bench 067 result (verified 5 of 19 chains), probe-confirmed negative capabilities, and a verified 5.2x BONK mispricing traced to pool discovery missing the main market. Claude-Session: https://claude.ai/code/session_01CpArutAtXuBb1BVNUDXoYA Co-authored-by: Claude Opus 5 (1M context) (cherry picked from commit dcfe0803fb956c8e63bc8451623e998263fe81ee) --- .../serialized-onboarding-audit.md | 109 +++++++++++++++++- 1 file changed, 104 insertions(+), 5 deletions(-) diff --git a/docs/methodology/serialized-onboarding-audit.md b/docs/methodology/serialized-onboarding-audit.md index a34244c51..39fb48004 100644 --- a/docs/methodology/serialized-onboarding-audit.md +++ b/docs/methodology/serialized-onboarding-audit.md @@ -3,7 +3,7 @@ > **Pre-onboarding evaluation.** Run before Serialized is wired into any live harness, so the > decision to include or exclude them on each bench is documented and reproducible. > -> **Version:** v1.0, first commit 2026-09-05. Author: internal. Key used: tenant `OpenChainBench`, +> **Version:** v1.1, 2026-09-05 (v1.0 same day; §8 corrected, §16 added). Author: internal. Key used: tenant `OpenChainBench`, > plan `starter`, keyId `d5511a080aaa`, issued 2026-09-04. --- @@ -211,10 +211,41 @@ negative while Mobula beats it to the wire on three trades out of four. **Any he on a provider's own timestamp is not a latency measurement, it is a measurement of where that provider chooses to put its clock.** -Bench 001 already does the right thing by referencing archive nodes and validating against block -hashes, so the published leaderboard is not affected by this. It does mean two things going forward: -the archive-node reference is load-bearing and must never be relaxed to a self-reported field, and -Serialized cannot be onboarded through a shortcut that trusts their `at`. +**Correction, 2026-09-05 (v1.1).** An earlier draft of this file claimed bench 001 already +references archive nodes and was therefore unaffected. That was wrong: it repeated the spec's +methodology instead of reading the harness. `harnesses/aggregator-head-lag` contains no archive-node +reference at all (`grep -rl "archive|eth_getBlockByNumber|getBlockTime|blockTimestamp"` over +`cmd/` returns nothing). The gauge that feeds the leaderboard is computed from each provider's own +self-reported timestamp: + +```go +// head_lag_monitor.go:211 (Mobula) +onChainTime := time.UnixMilli(trade.Date) // Mobula's own field +totalLagMs := receiveTime.Sub(onChainTime) +// head_lag_monitor.go:707 (Codex) +onChainTime := time.Unix(event.Timestamp, 0) // Codex's own field +``` + +The published spec says otherwise in three places: `methodology[7]` ("Reference: archive nodes per +chain, validated against block hashes"), the FAQ ("The harness holds a live WebSocket subscription +to canonical-tip archive nodes on each chain"), and the per-chain explainers ("Measured against a +canonical archive node"). The documentation and the code disagree, on a live bench that is publicly +cited. That is a defect independent of Serialized and should be resolved before any provider is +added. + +Second code-level issue, `head_lag_monitor.go:219`: + +```go +if totalLagMs < 0 || totalLagMs > 30000 { continue } +``` + +Negative lags are dropped silently. Serialized's Base feed was negative on 13 of 13 sampled trades, +so under this filter its entire Base preconfirmed population would be discarded and its Base sample +would retain only its slowest trades. This is a measurable bias, not a policy question. + +Recommended resolution: make the harness hold its own node subscription per chain and timestamp each +swap on receipt, matching by transaction hash. That is what the spec already claims, so no published +text changes, and it makes the three providers comparable for the first time. **Blocking issue: Base preconfirmations.** Serialized emits Base trades from flashblocks preconfirmations, ahead of the block timestamp they attach to the event. Measured on their stream, @@ -311,3 +342,71 @@ Worth knowing before any commercial discussion, neutral observation either way: Every onboarding needs a `docker build --no-cache` of the materialize-worker on `ocb-par-main` after the harness change, or the new provider will not appear. + + +## 16. Follow-up tests, 2026-09-05 + +### 16.1 Bench 067, now conclusive + +The earlier §9 result was inconclusive because it used the wrong probe address. The harness already +pins canonical ones in `registry.go`: EVM `0xF977...aceC` (Binance 8), Solana `9WzDX...WWM`, with a +$1 USD floor. Re-run verbatim against those: + +| Metric | Serialized | +|---|---| +| listed (`/v1/meta/chains`) | 19 | +| verified (returned a > $1 balance) | **5** | +| errors | 0 | +| total probe latency, 19 calls | 1,696 ms | + +Verified: ethereum (177 positions, $72.7M), bsc (93, $43.9M), base (27, $5.1M), arbitrum (12, +$3.5M), solana (1, $12). The other 14 chains returned zero rows because Binance 8 holds nothing +there, which is the harness's own "untestable residue" (`listed - probed`), not an indexer failure. + +Published leaderboard: CoinStats 127, Mobula 50, Zerion 42, Moralis 15. Serialized would rank +last at 5. Verdict: addable and now measurable, but it is a third breadth metric and a third last +place. Their `verified / probed` ratio is 5/5, which the bench exposes as a separate series and is +the only flattering read available. + +### 16.2 Negative capability probes + +Confirmed by request rather than by reading docs. Every path returns `404 NOT_FOUND`: +`/v1/wallet/nfts`, `/v1/nft/collection`, `/v1/nfts`, `/v1/swap/quote`, `/v1/quote`, `/v1/route`, +`/v1/bridge/quote`. Benches 033, 102, `nft-collection-metadata`, `bridge-fee` and +`bridge-quote-latency` are definitively out. + +### 16.3 A real pricing defect: BONK is 5.2x wrong + +| Source | BONK price | +|---|---| +| Mobula | 3.3097e-06 | +| DexScreener (Orca, $305,835 liquidity) | 3.309e-06 | +| GeckoTerminal | 3.309731e-06 | +| **Serialized** | **6.3314e-07** | + +Three independent sources agree; Serialized is low by a factor of 5.2, and reports a $55.7M market +cap against a real ~$290M. + +Root cause is visible in their own response. `/v1/token/pools?chain=solana&address=DezXAZ...` ranks +`Gx1WGimRY3jF...` first with liquidity 4,339, and the deep Orca pool everyone else prices from is +absent from the list entirely. Their own ranks 2 and 3 quote ~3.18e-08 and ~3.20e-08 native against +rank 1 at 6.09e-09, so the pool list is internally inconsistent by the same 5x. This is pool +discovery missing the main market, not a decimals bug (`decimals: 5` is correct for BONK). + +Worth raising with them directly: a top-100 token mispriced 5x is a bigger problem for their +prospects than any leaderboard position. + +### 16.4 Cross-API price accuracy as a new bench: not proven + +Two attempts, neither conclusive, recorded so nobody repeats them: + +1. Basket from a DexScreener search returned eight distinct addresses all symbolled "SOL", i.e. + impostor tokens rather than eight real assets. Result discarded. +2. Basket from GeckoTerminal top pools (28 distinct tokens) gated on DexScreener and GeckoTerminal + agreeing within 200 bps. Only 3 tokens survived, because GeckoTerminal returned no price for 25 + of them. n=3 proves nothing. + +The idea remains the most promising new bench for this vertical, and the BONK case shows the signal +is real. But it cannot be built on another aggregator as reference: the reference has to be computed +from on-chain reserves of the deepest pool over an RPC we control, which is the actual work and the +actual reason the bench would be defensible. From 2730b0b1d1f7979cfcdcd6829477d582cc8713ce Mon Sep 17 00:00:00 2001 From: Flotapponnier <160007691+Flotapponnier@users.noreply.github.com> Date: Thu, 10 Sep 2026 01:01:58 +0200 Subject: [PATCH 09/10] feat(001): measure Base against the flashblock reference, re-add Serialized (#2297) Base was the only chain whose zero point was wrong, and measurably so: a block becomes queryable ~0.36 s after the timestamp it carries (median over 25 consecutive blocks) while the sequencer publishes flashblock preconfirmations every 200 ms inside the 2 s interval. Measuring against the block timestamp therefore described the chain's stamping convention, not the provider's pipeline, and charged anything reading preconfirmations with a negative lag: 99% of Serialized's Base emissions, 140 of 147. base_flashblock_ref.go feeds the existing reference clock from that stream. The clock already keeps the first observation, so the preconfirmation wins over the sealed-block logs subscription without further change. The endpoint is public and anycast: 1.1 ms RTT from both the Paris and Singapore boxes, the same distance as the provider endpoints, so no region is handicapped against the feeds it measures. headlineLag applies the substitution in one place, so it lands on all four providers at once. Re-basing one provider and leaving the others on the old ruler is the asymmetry this change exists to remove. A Base emission with no flashblock match is dropped, not measured against a different ruler: a dead reference must show up as missing data, never as quietly different numbers. Verified from two vantage points before shipping. Serialized on Base moves from -1.016 to +0.239 (Paris) and -1.081 to +0.244 (laptop), stable, with 4 negatives out of 125 left as network jitter. Mobula's old-reference figure from Paris (+1.070) matches production (0.951), which is what makes the vantage trustworthy. Only Base changes. Solana, BNB and Robinhood carry no preconfirmation layer a provider consumes and their chain-supplied timestamps sit within ~60 ms of the observable moment, so they keep the on-chain timestamp. Serialized is re-added with the pinned Robinhood token address corrected: the old value acked and delivered nothing, which is what produced the "they publish nothing on Robinhood" report. With the address /v1/pool returns, they deliver 98 events in 5 minutes at p50 +0.596 s. Co-authored-by: Claude Opus 5 (1M context) (cherry picked from commit 9a354bc5f6965949aae66ff66917a1034bd21458) --- benchmarks/aggregator-head-lag.yml | 38 +++++++++++++++++++++++++----- 1 file changed, 32 insertions(+), 6 deletions(-) diff --git a/benchmarks/aggregator-head-lag.yml b/benchmarks/aggregator-head-lag.yml index 56db0b119..3d11582a7 100644 --- a/benchmarks/aggregator-head-lag.yml +++ b/benchmarks/aggregator-head-lag.yml @@ -2,10 +2,10 @@ slug: aggregator-head-lag number: "001" -title: Fastest crypto price API, live head lag across Mobula, Codex, GeckoTerminal +title: Fastest crypto price API, live head lag across Mobula, Codex, GeckoTerminal, Serialized seo_title: "Fastest crypto price API 2026" -seo_description: "{{best_name}} leads fastest crypto price API at {{best_p50}} (cross-chain avg p50, 24h). Mobula WebSocket, Codex GraphQL, GeckoTerminal REST live across Base, BNB, Solana, Robinhood." -subtitle: Wall-clock head lag in seconds from on-chain swap event to API emission, measured live for Mobula, Codex and GeckoTerminal on Base, BNB Chain, Solana and Robinhood Chain. +seo_description: "{{best_name}} leads fastest crypto price API at {{best_p50}} (cross-chain avg p50, 24h). Mobula WebSocket, Codex GraphQL, GeckoTerminal REST, Serialized WebSocket live across Base, BNB, Solana, Robinhood." +subtitle: Wall-clock head lag in seconds from on-chain swap event to API emission, measured live for Mobula, Codex, GeckoTerminal and Serialized on Base, BNB Chain, Solana and Robinhood Chain. per_chain_explainer: - slug: base @@ -37,7 +37,7 @@ seo_intro: | chain and the same event appearing on the provider's feed. Marketing pages quote "real-time" without a number; this page quotes the number. Mobula's WebSocket, Codex's GraphQL feed, GeckoTerminal's - REST endpoint are watched + REST endpoint and Serialized's WebSocket trades stream are watched from three regions (us-east, eu-west, sgp). On Base the zero point is the sequencer's flashblock preconfirmation stream, which we hold ourselves; on BNB Chain, Solana and Robinhood Chain it is the @@ -63,7 +63,7 @@ abstract: | same event. methodology: - - "Aggregators measured: Mobula, Codex, GeckoTerminal." + - "Aggregators measured: Mobula, Codex, GeckoTerminal, Serialized." - "Chains: Base, BNB Chain, Solana, Robinhood Chain." - "Reference on BNB Chain and Robinhood Chain: archive nodes per chain, validated against block hashes. These carry no preconfirmation layer a provider consumes, and their chain-supplied timestamps sit within roughly 60 ms of the moment a trade is observable, so they keep the on-chain timestamp as the zero point." - "Solana (since 2026-09-16): the headline is the lag behind the first feed to report the trade, our own node subscription included in the race. Solana has no on-chain timestamp with sub-second precision, so the timestamps providers send compare conventions: Mobula's `date` (its ingestion time) read as a constant 0.10 s, Serialized's `at` (blockTime, whole seconds) as 0.75 s, while against a common clock the two feeds are 10 to 30 ms apart." @@ -87,7 +87,7 @@ findings: - "On Base, what a provider reads matters more than how fast its pipeline is. A provider emitting from flashblock preconfirmations delivers a trade about 1.6 s (median over 21,538 transactions) before the sealed block carrying it becomes queryable, so a sealed-block reader carries that floor before any pipeline work. A preconfirmation is not final and can be reordered." - "The Base spread is therefore mostly that floor: the providers' own pipelines sit within a few hundred milliseconds of each other. Read the Base column as a latency versus finality choice, not as a pure speed ranking." - "{{name:geckoterminal}} trails on every chain. REST polling adds the poll interval to every read, so head lag tracks the publisher's chosen cadence rather than raw infrastructure speed." - - "p99 is the integration-grade number. The gap between p50 and {{p99:mobula}} / {{p99:codex}} / {{p99:geckoterminal}} is what a live UI feels when a chain spikes or a region's path degrades." + - "p99 is the integration-grade number. The gap between p50 and {{p99:mobula}} / {{p99:codex}} / {{p99:serialized}} / {{p99:geckoterminal}} is what a live UI feels when a chain spikes or a region's path degrades." source: https://github.com/ChainBench/OpenChainBench/tree/main/harnesses/aggregator-head-lag @@ -247,6 +247,32 @@ providers: p50: avg by (aggregator) (quantile_over_time(0.50, head_lag_seconds{aggregator="codex", region="sgp"}[24h])) * 1000 series: avg_over_time(head_lag_seconds{aggregator="codex", region="sgp"}[1h]) * 1000 unless (changes(head_lag_seconds{aggregator="codex", region="sgp"}[15m]) == 0 and count_over_time(head_lag_seconds{aggregator="codex", region="sgp"}[15m]) > 5 and on(chain, region) sum by (chain, region) (changes(head_lag_seconds{region="sgp"}[15m])) > 5 and on(aggregator, region) avg_over_time(ws_connected{aggregator="codex", region="sgp", chain=""}[15m]) > 0.8) + + + - slug: serialized + name: Serialized + tag: WebSocket trades stream + formula: "Median seconds between an on-chain swap on the bench pools and the same trade arriving on Serialized's WebSocket trades stream (subscribed per pool by its token side), sampled every 15s over 24h." + queries: + p50: avg by (aggregator) (quantile_over_time(0.50, head_lag_seconds{aggregator="serialized"}[24h])) * 1000 + p90: avg by (aggregator) (quantile_over_time(0.90, head_lag_seconds{aggregator="serialized"}[24h])) * 1000 + p99: avg by (aggregator) (quantile_over_time(0.99, head_lag_seconds{aggregator="serialized"}[24h])) * 1000 + mean: avg by (aggregator) (avg_over_time(head_lag_seconds{aggregator="serialized"}[24h])) * 1000 + success: clamp_max(avg by (aggregator) (count_over_time(head_lag_seconds{aggregator="serialized"}[24h]) / 2880), 1) + sample_size: sum(count_over_time(head_lag_seconds{aggregator="serialized"}[24h])) + series: avg_over_time(head_lag_seconds{aggregator="serialized"}[1h]) * 1000 unless (changes(head_lag_seconds{aggregator="serialized"}[15m]) == 0 and count_over_time(head_lag_seconds{aggregator="serialized"}[15m]) > 5 and on(chain, region) sum by (chain, region) (changes(head_lag_seconds{}[15m])) > 5 and on(aggregator, region) avg_over_time(ws_connected{aggregator="serialized", chain=""}[15m]) > 0.8) + live_activity: sum(changes(head_lag_seconds{aggregator="serialized"}[15m])) + regions: + - region: us-east + p50: avg by (aggregator) (quantile_over_time(0.50, head_lag_seconds{aggregator="serialized", region="us-east"}[24h])) * 1000 + series: avg_over_time(head_lag_seconds{aggregator="serialized", region="us-east"}[1h]) * 1000 unless (changes(head_lag_seconds{aggregator="serialized", region="us-east"}[15m]) == 0 and count_over_time(head_lag_seconds{aggregator="serialized", region="us-east"}[15m]) > 5 and on(chain, region) sum by (chain, region) (changes(head_lag_seconds{region="us-east"}[15m])) > 5 and on(aggregator, region) avg_over_time(ws_connected{aggregator="serialized", region="us-east", chain=""}[15m]) > 0.8) + - region: eu-west + p50: avg by (aggregator) (quantile_over_time(0.50, head_lag_seconds{aggregator="serialized", region="eu-west"}[24h])) * 1000 + series: avg_over_time(head_lag_seconds{aggregator="serialized", region="eu-west"}[1h]) * 1000 unless (changes(head_lag_seconds{aggregator="serialized", region="eu-west"}[15m]) == 0 and count_over_time(head_lag_seconds{aggregator="serialized", region="eu-west"}[15m]) > 5 and on(chain, region) sum by (chain, region) (changes(head_lag_seconds{region="eu-west"}[15m])) > 5 and on(aggregator, region) avg_over_time(ws_connected{aggregator="serialized", region="eu-west", chain=""}[15m]) > 0.8) + - region: ap-southeast + p50: avg by (aggregator) (quantile_over_time(0.50, head_lag_seconds{aggregator="serialized", region="sgp"}[24h])) * 1000 + series: avg_over_time(head_lag_seconds{aggregator="serialized", region="sgp"}[1h]) * 1000 unless (changes(head_lag_seconds{aggregator="serialized", region="sgp"}[15m]) == 0 and count_over_time(head_lag_seconds{aggregator="serialized", region="sgp"}[15m]) > 5 and on(chain, region) sum by (chain, region) (changes(head_lag_seconds{region="sgp"}[15m])) > 5 and on(aggregator, region) avg_over_time(ws_connected{aggregator="serialized", region="sgp", chain=""}[15m]) > 0.8) + - slug: geckoterminal name: GeckoTerminal tag: REST feed From f89aa09f1bd33ad18fa6989189e9bd93fd34acad Mon Sep 17 00:00:00 2001 From: Flotapponnier <160007691+Flotapponnier@users.noreply.github.com> Date: Wed, 16 Sep 2026 15:23:08 +0200 Subject: [PATCH 10/10] Serialized: brand blue (was the palette orange next to Mobula), logo mark centred in its box (#2400) Claude-Session: https://claude.ai/code/session_01HJgbZCqjR4nvCfcJSzofbw Co-authored-by: Flotapponnier Co-authored-by: Claude Opus 5 (1M context) (cherry picked from commit eb3591f723672619072a5c41b9ac4d16f5d25e44) --- public/logos/serialized.svg | 2 +- src/lib/brand.ts | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/public/logos/serialized.svg b/public/logos/serialized.svg index 3198acb25..39ae0e54d 100644 --- a/public/logos/serialized.svg +++ b/public/logos/serialized.svg @@ -1,4 +1,4 @@ - + diff --git a/src/lib/brand.ts b/src/lib/brand.ts index b5d76ef28..bbe214bac 100644 --- a/src/lib/brand.ts +++ b/src/lib/brand.ts @@ -58,6 +58,8 @@ const BRANDS: Record = { // ─── Aggregators / providers (bright, saturated - read on both modes) ─── mobula: { color: "#FF6B35" }, // vivid orange + serialized: { color: "#3D74FF" }, // serialized logo blue - was falling back to the + // palette and drew the same orange as mobula codex: { color: "#84cc16" }, // saturated lime - readable on white + dark geckoterminal: { color: "#8B5CF6" }, // vivid violet (gecko brand) jupiter: { color: "#C7F284" }, // jupiter matrix green (secondary brand) - keeps it