diff --git a/app/snapshots/SnapshotsClient.tsx b/app/snapshots/SnapshotsClient.tsx index 99de84a4..6d965765 100644 --- a/app/snapshots/SnapshotsClient.tsx +++ b/app/snapshots/SnapshotsClient.tsx @@ -21,6 +21,7 @@ import { formatBytes, formatDate, formatNumber, + presetSize, PresetName, PRESETS, Snapshot, @@ -393,9 +394,7 @@ export function SnapshotsClient({ snapshots }: SnapshotsClientProps) { className="grid grid-cols-1 gap-3 lg:grid-cols-3" > {PRESETS.map((p) => { - const size = activeSnapshot.components - .filter((c) => p.components.includes(c.name)) - .reduce((sum, c) => sum + c.size, 0); + const size = presetSize(activeSnapshot.components, p); const selected = preset === p.name; const includedComponents = displayComponents.filter((c) => { if (c.name === 'state_history') return p.components.includes('account_changesets'); diff --git a/app/snapshots/data.test.ts b/app/snapshots/data.test.ts new file mode 100644 index 00000000..35c7226b --- /dev/null +++ b/app/snapshots/data.test.ts @@ -0,0 +1,49 @@ +import { PRESETS, presetSize, type SnapshotComponent } from './data'; + +const component = (name: string, size: number, fullSize?: number): SnapshotComponent => ({ + name, + displayName: name, + description: name, + size, + ...(fullSize === undefined ? {} : { fullSize }), +}); + +describe('presetSize', () => { + const components = [ + component('state', 100), + component('headers', 20, 3), + component('transactions', 80, 8), + component('transaction_senders', 10, 1), + component('receipts', 70, 7), + component('account_changesets', 60, 6), + component('storage_changesets', 50, 5), + component('rocksdb_indices', 40), + ]; + + it('uses all state and headers plus the history window of full static-file components', () => { + const full = PRESETS.find((preset) => preset.name === 'full')!; + + expect(presetSize(components, full)).toBe(100 + 20 + 8 + 7 + 6 + 5); + }); + + it('excludes senders and RocksDB from full', () => { + const full = PRESETS.find((preset) => preset.name === 'full')!; + + expect(full.components).not.toContain('transaction_senders'); + expect(full.components).not.toContain('rocksdb_indices'); + }); + + it('continues to use complete component sizes for archive and minimal', () => { + const archive = PRESETS.find((preset) => preset.name === 'archive')!; + const minimal = PRESETS.find((preset) => preset.name === 'minimal')!; + + expect(presetSize(components, archive)).toBe(430); + expect(presetSize(components, minimal)).toBe(120); + }); + + it('falls back to the complete size when older API data has no tail size', () => { + const full = PRESETS.find((preset) => preset.name === 'full')!; + + expect(presetSize([component('transactions', 80)], full)).toBe(80); + }); +}); diff --git a/app/snapshots/data.ts b/app/snapshots/data.ts index 54e63021..61a26ad1 100644 --- a/app/snapshots/data.ts +++ b/app/snapshots/data.ts @@ -6,6 +6,7 @@ export type SnapshotComponent = { displayName: string; description: string; size: number; // bytes + fullSize?: number; // bytes downloaded for this component by the Full preset }; export type Snapshot = { @@ -71,6 +72,26 @@ export const PRESETS: Preset[] = [ }, ]; +const FULL_HISTORY_COMPONENTS = new Set([ + 'transactions', + 'receipts', + 'account_changesets', + 'storage_changesets', +]); + +export function presetSize(components: SnapshotComponent[], preset: Preset): number { + return components + .filter((component) => preset.components.includes(component.name)) + .reduce( + (sum, component) => + sum + + (preset.name === 'full' && FULL_HISTORY_COMPONENTS.has(component.name) + ? (component.fullSize ?? component.size) + : component.size), + 0, + ); +} + export const CHAIN_NAME_BY_NETWORK: Record = { mainnet: 'base', sepolia: 'base-sepolia', @@ -122,11 +143,15 @@ export const COMPONENT_META: Record): SnapshotComponent[] { +function buildComponents( + sizesGB: Record, + fullSizesGB: Record, +): SnapshotComponent[] { return COMPONENT_ORDER.map((name) => ({ name, ...COMPONENT_META[name], size: (sizesGB[name] ?? 0) * GB, + ...(fullSizesGB[name] === undefined ? {} : { fullSize: fullSizesGB[name] * GB }), })); } @@ -136,8 +161,9 @@ function sampleSnapshot( chainId: string, block: number, sizesGB: Record, + fullSizesGB: Record, ): Snapshot { - const components = buildComponents(sizesGB); + const components = buildComponents(sizesGB, fullSizesGB); return { chainId, chainName, @@ -156,36 +182,72 @@ function sampleSnapshot( } export const SAMPLE_SNAPSHOTS: Snapshot[] = [ - sampleSnapshot('mainnet', 'Base Mainnet', '8453', 34200000, { - state: 900, - headers: 9, - transactions: 420, - transaction_senders: 55, - receipts: 520, - account_changesets: 310, - storage_changesets: 680, - rocksdb_indices: 240, - }), - sampleSnapshot('sepolia', 'Base Sepolia', '84532', 19800000, { - state: 140, - headers: 3, - transactions: 60, - transaction_senders: 9, - receipts: 70, - account_changesets: 40, - storage_changesets: 90, - rocksdb_indices: 35, - }), + sampleSnapshot( + 'mainnet', + 'Base Mainnet', + '8453', + 34200000, + { + state: 900, + headers: 9, + transactions: 420, + transaction_senders: 55, + receipts: 520, + account_changesets: 310, + storage_changesets: 680, + rocksdb_indices: 240, + }, + { + transactions: 42, + receipts: 21, + account_changesets: 4, + storage_changesets: 21, + }, + ), + sampleSnapshot( + 'sepolia', + 'Base Sepolia', + '84532', + 19800000, + { + state: 140, + headers: 3, + transactions: 60, + transaction_senders: 9, + receipts: 70, + account_changesets: 40, + storage_changesets: 90, + rocksdb_indices: 35, + }, + { + transactions: 9, + receipts: 10, + account_changesets: 1, + storage_changesets: 8, + }, + ), // Hidden from the page (see isNetworkVisibleInUi) but still served by the API, // so the dev fallback mirrors what /api/snapshots returns. - sampleSnapshot('zeronet', 'Base Zeronet', '84530', 512000, { - state: 12, - headers: 1, - transactions: 3, - transaction_senders: 1, - receipts: 4, - account_changesets: 2, - storage_changesets: 5, - rocksdb_indices: 2, - }), + sampleSnapshot( + 'zeronet', + 'Base Zeronet', + '84530', + 512000, + { + state: 12, + headers: 1, + transactions: 3, + transaction_senders: 1, + receipts: 4, + account_changesets: 2, + storage_changesets: 5, + rocksdb_indices: 2, + }, + { + transactions: 1, + receipts: 1, + account_changesets: 1, + storage_changesets: 2, + }, + ), ]; diff --git a/app/snapshots/r2.test.ts b/app/snapshots/r2.test.ts index 66ee94ea..aa9b2a7b 100644 --- a/app/snapshots/r2.test.ts +++ b/app/snapshots/r2.test.ts @@ -1,4 +1,4 @@ -import { decodeXml, isNetworkVisibleInUi, NETWORK_IDS } from './r2'; +import { decodeXml, historyChunkSize, isNetworkVisibleInUi, NETWORK_IDS } from './r2'; describe('network visibility', () => { // Zeronet was removed outright in "Cobalt and fixes" (#14) to hide it from the @@ -50,3 +50,30 @@ describe('decodeXml', () => { expect(decodeXml('a&b<c>d')).toBe('a&bd'); }); }); + +describe('historyChunkSize', () => { + const chunkFiles = [ + 'static_files/transactions-48500000-48999999.tar.zst', + 'static_files/transactions-49000000-49499999.tar.zst', + 'static_files/transactions-49500000-49999999.tar.zst', + 'static_files/transactions-50000000-50499999.tar.zst', + 'snapshot/transactions-50500000-50999999.tar.zst', + ]; + const chunkSizes = [10, 20, 30, 40, 50]; + + it('includes four files when a partial tail chunk puts the 1,339,200-block cutoff in the fourth file', () => { + expect(historyChunkSize(50_800_000, chunkSizes, chunkFiles)).toBe(20 + 30 + 40 + 50); + }); + + it('includes three files when they cover the entire 1,339,200-block window', () => { + expect(historyChunkSize(50_900_000, chunkSizes, chunkFiles)).toBe(30 + 40 + 50); + }); + + it('is absent for components that are not chunked', () => { + expect(historyChunkSize(50_758_927, undefined, undefined)).toBeUndefined(); + }); + + it('is absent when chunk files cannot be matched safely to their sizes', () => { + expect(historyChunkSize(50_758_927, [10], ['transactions.tar.zst'])).toBeUndefined(); + }); +}); diff --git a/app/snapshots/r2.ts b/app/snapshots/r2.ts index 2828b953..87bf726f 100644 --- a/app/snapshots/r2.ts +++ b/app/snapshots/r2.ts @@ -45,6 +45,7 @@ class R2RequestError extends Error { type R2ManifestComponent = { size?: number; chunk_sizes?: number[]; + chunk_files?: string[]; output_files?: { size?: number }[]; chunk_output_files?: { size?: number }[][]; }; @@ -329,7 +330,7 @@ function signR2Request(url: URL, r2Config: R2Config): Headers { function buildSnapshot(network: NetworkConfig, prefix: string, manifest: R2Manifest): Snapshot { const components = Object.entries(manifest.components) - .map(([name, component]) => buildComponent(name, component)) + .map(([name, component]) => buildComponent(name, component, manifest.block)) .sort((a, b) => componentSortIndex(a.name) - componentSortIndex(b.name)); const size = components.reduce((sum, component) => sum + component.size, 0); const timestamp = String(manifest.timestamp); @@ -357,9 +358,19 @@ function buildSnapshot(network: NetworkConfig, prefix: string, manifest: R2Manif }; } -function buildComponent(name: string, component: R2ManifestComponent): SnapshotComponent { +function buildComponent( + name: string, + component: R2ManifestComponent, + snapshotBlock: number, +): SnapshotComponent { const metadata = COMPONENT_META[name] ?? { displayName: titleize(name), description: titleize(name) }; - return { name, ...metadata, size: componentSize(component) }; + const fullSize = historyChunkSize(snapshotBlock, component.chunk_sizes, component.chunk_files); + return { + name, + ...metadata, + size: componentSize(component), + ...(fullSize === undefined ? {} : { fullSize }), + }; } function componentSize(component: R2ManifestComponent): number { @@ -371,6 +382,31 @@ function componentSize(component: R2ManifestComponent): number { return 0; } +const FULL_HISTORY_BLOCKS = 1_339_200; + +export function historyChunkSize( + snapshotBlock: number, + chunkSizes: number[] | undefined, + chunkFiles: string[] | undefined, +): number | undefined { + if (!chunkSizes || !chunkFiles || chunkSizes.length !== chunkFiles.length) return undefined; + + const ranges = chunkFiles.map((file) => file.match(/-(\d+)-(\d+)\.tar\.zst$/)); + if (ranges.some((range) => !range)) return undefined; + + const earliestBlock = Math.max(0, snapshotBlock - FULL_HISTORY_BLOCKS); + const includedSizes = chunkSizes.filter((_, index) => { + const range = ranges[index]!; + const startBlock = Number(range[1]); + const endBlock = Number(range[2]); + return startBlock <= snapshotBlock && endBlock >= earliestBlock; + }); + + return includedSizes.length > 0 + ? includedSizes.reduce((sum, size) => sum + size, 0) + : undefined; +} + function componentSortIndex(name: string): number { const index = COMPONENT_ORDER.indexOf(name); return index === -1 ? COMPONENT_ORDER.length : index;