Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 2 additions & 3 deletions app/snapshots/SnapshotsClient.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ import {
formatBytes,
formatDate,
formatNumber,
presetSize,
PresetName,
PRESETS,
Snapshot,
Expand Down Expand Up @@ -393,9 +394,7 @@ export function SnapshotsClient({ snapshots }: SnapshotsClientProps) {
className="grid grid-cols-1 gap-3 lg:grid-cols-3"
>
{PRESETS.map((p) => {
const size = activeSnapshot.components
.filter((c) => p.components.includes(c.name))
.reduce((sum, c) => sum + c.size, 0);
const size = presetSize(activeSnapshot.components, p);
const selected = preset === p.name;
const includedComponents = displayComponents.filter((c) => {
if (c.name === 'state_history') return p.components.includes('account_changesets');
Expand Down
49 changes: 49 additions & 0 deletions app/snapshots/data.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
import { PRESETS, presetSize, type SnapshotComponent } from './data';

const component = (name: string, size: number, fullSize?: number): SnapshotComponent => ({
name,
displayName: name,
description: name,
size,
...(fullSize === undefined ? {} : { fullSize }),
});

describe('presetSize', () => {
const components = [
component('state', 100),
component('headers', 20, 3),
component('transactions', 80, 8),
component('transaction_senders', 10, 1),
component('receipts', 70, 7),
component('account_changesets', 60, 6),
component('storage_changesets', 50, 5),
component('rocksdb_indices', 40),
];

it('uses all state and headers plus the history window of full static-file components', () => {
const full = PRESETS.find((preset) => preset.name === 'full')!;

expect(presetSize(components, full)).toBe(100 + 20 + 8 + 7 + 6 + 5);
});

it('excludes senders and RocksDB from full', () => {
const full = PRESETS.find((preset) => preset.name === 'full')!;

expect(full.components).not.toContain('transaction_senders');
expect(full.components).not.toContain('rocksdb_indices');
});

it('continues to use complete component sizes for archive and minimal', () => {
const archive = PRESETS.find((preset) => preset.name === 'archive')!;
const minimal = PRESETS.find((preset) => preset.name === 'minimal')!;

expect(presetSize(components, archive)).toBe(430);
expect(presetSize(components, minimal)).toBe(120);
});

it('falls back to the complete size when older API data has no tail size', () => {
const full = PRESETS.find((preset) => preset.name === 'full')!;

expect(presetSize([component('transactions', 80)], full)).toBe(80);
});
});
126 changes: 94 additions & 32 deletions app/snapshots/data.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ export type SnapshotComponent = {
displayName: string;
description: string;
size: number; // bytes
fullSize?: number; // bytes downloaded for this component by the Full preset
};

export type Snapshot = {
Expand Down Expand Up @@ -71,6 +72,26 @@ export const PRESETS: Preset[] = [
},
];

const FULL_HISTORY_COMPONENTS = new Set([
'transactions',
'receipts',
'account_changesets',
'storage_changesets',
]);

export function presetSize(components: SnapshotComponent[], preset: Preset): number {
return components
.filter((component) => preset.components.includes(component.name))
.reduce(
(sum, component) =>
sum +
(preset.name === 'full' && FULL_HISTORY_COMPONENTS.has(component.name)
? (component.fullSize ?? component.size)
: component.size),
0,
);
}

export const CHAIN_NAME_BY_NETWORK: Record<string, string> = {
mainnet: 'base',
sepolia: 'base-sepolia',
Expand Down Expand Up @@ -122,11 +143,15 @@ export const COMPONENT_META: Record<string, { displayName: string; description:

export const COMPONENT_ORDER = Object.keys(COMPONENT_META);

function buildComponents(sizesGB: Record<string, number>): SnapshotComponent[] {
function buildComponents(
sizesGB: Record<string, number>,
fullSizesGB: Record<string, number>,
): SnapshotComponent[] {
return COMPONENT_ORDER.map((name) => ({
name,
...COMPONENT_META[name],
size: (sizesGB[name] ?? 0) * GB,
...(fullSizesGB[name] === undefined ? {} : { fullSize: fullSizesGB[name] * GB }),
}));
}

Expand All @@ -136,8 +161,9 @@ function sampleSnapshot(
chainId: string,
block: number,
sizesGB: Record<string, number>,
fullSizesGB: Record<string, number>,
): Snapshot {
const components = buildComponents(sizesGB);
const components = buildComponents(sizesGB, fullSizesGB);
return {
chainId,
chainName,
Expand All @@ -156,36 +182,72 @@ function sampleSnapshot(
}

export const SAMPLE_SNAPSHOTS: Snapshot[] = [
sampleSnapshot('mainnet', 'Base Mainnet', '8453', 34200000, {
state: 900,
headers: 9,
transactions: 420,
transaction_senders: 55,
receipts: 520,
account_changesets: 310,
storage_changesets: 680,
rocksdb_indices: 240,
}),
sampleSnapshot('sepolia', 'Base Sepolia', '84532', 19800000, {
state: 140,
headers: 3,
transactions: 60,
transaction_senders: 9,
receipts: 70,
account_changesets: 40,
storage_changesets: 90,
rocksdb_indices: 35,
}),
sampleSnapshot(
'mainnet',
'Base Mainnet',
'8453',
34200000,
{
state: 900,
headers: 9,
transactions: 420,
transaction_senders: 55,
receipts: 520,
account_changesets: 310,
storage_changesets: 680,
rocksdb_indices: 240,
},
{
transactions: 42,
receipts: 21,
account_changesets: 4,
storage_changesets: 21,
},
),
sampleSnapshot(
'sepolia',
'Base Sepolia',
'84532',
19800000,
{
state: 140,
headers: 3,
transactions: 60,
transaction_senders: 9,
receipts: 70,
account_changesets: 40,
storage_changesets: 90,
rocksdb_indices: 35,
},
{
transactions: 9,
receipts: 10,
account_changesets: 1,
storage_changesets: 8,
},
),
// Hidden from the page (see isNetworkVisibleInUi) but still served by the API,
// so the dev fallback mirrors what /api/snapshots returns.
sampleSnapshot('zeronet', 'Base Zeronet', '84530', 512000, {
state: 12,
headers: 1,
transactions: 3,
transaction_senders: 1,
receipts: 4,
account_changesets: 2,
storage_changesets: 5,
rocksdb_indices: 2,
}),
sampleSnapshot(
'zeronet',
'Base Zeronet',
'84530',
512000,
{
state: 12,
headers: 1,
transactions: 3,
transaction_senders: 1,
receipts: 4,
account_changesets: 2,
storage_changesets: 5,
rocksdb_indices: 2,
},
{
transactions: 1,
receipts: 1,
account_changesets: 1,
storage_changesets: 2,
},
),
];
29 changes: 28 additions & 1 deletion app/snapshots/r2.test.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import { decodeXml, isNetworkVisibleInUi, NETWORK_IDS } from './r2';
import { decodeXml, historyChunkSize, isNetworkVisibleInUi, NETWORK_IDS } from './r2';

describe('network visibility', () => {
// Zeronet was removed outright in "Cobalt and fixes" (#14) to hide it from the
Expand Down Expand Up @@ -50,3 +50,30 @@ describe('decodeXml', () => {
expect(decodeXml('a&amp;b&lt;c&gt;d')).toBe('a&b<c>d');
});
});

describe('historyChunkSize', () => {
const chunkFiles = [
'static_files/transactions-48500000-48999999.tar.zst',
'static_files/transactions-49000000-49499999.tar.zst',
'static_files/transactions-49500000-49999999.tar.zst',
'static_files/transactions-50000000-50499999.tar.zst',
'snapshot/transactions-50500000-50999999.tar.zst',
];
const chunkSizes = [10, 20, 30, 40, 50];

it('includes four files when a partial tail chunk puts the 1,339,200-block cutoff in the fourth file', () => {
expect(historyChunkSize(50_800_000, chunkSizes, chunkFiles)).toBe(20 + 30 + 40 + 50);
});

it('includes three files when they cover the entire 1,339,200-block window', () => {
expect(historyChunkSize(50_900_000, chunkSizes, chunkFiles)).toBe(30 + 40 + 50);
});

it('is absent for components that are not chunked', () => {
expect(historyChunkSize(50_758_927, undefined, undefined)).toBeUndefined();
});

it('is absent when chunk files cannot be matched safely to their sizes', () => {
expect(historyChunkSize(50_758_927, [10], ['transactions.tar.zst'])).toBeUndefined();
});
});
42 changes: 39 additions & 3 deletions app/snapshots/r2.ts
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,7 @@ class R2RequestError extends Error {
type R2ManifestComponent = {
size?: number;
chunk_sizes?: number[];
chunk_files?: string[];
output_files?: { size?: number }[];
chunk_output_files?: { size?: number }[][];
};
Expand Down Expand Up @@ -329,7 +330,7 @@ function signR2Request(url: URL, r2Config: R2Config): Headers {

function buildSnapshot(network: NetworkConfig, prefix: string, manifest: R2Manifest): Snapshot {
const components = Object.entries(manifest.components)
.map(([name, component]) => buildComponent(name, component))
.map(([name, component]) => buildComponent(name, component, manifest.block))
.sort((a, b) => componentSortIndex(a.name) - componentSortIndex(b.name));
const size = components.reduce((sum, component) => sum + component.size, 0);
const timestamp = String(manifest.timestamp);
Expand Down Expand Up @@ -357,9 +358,19 @@ function buildSnapshot(network: NetworkConfig, prefix: string, manifest: R2Manif
};
}

function buildComponent(name: string, component: R2ManifestComponent): SnapshotComponent {
function buildComponent(
name: string,
component: R2ManifestComponent,
snapshotBlock: number,
): SnapshotComponent {
const metadata = COMPONENT_META[name] ?? { displayName: titleize(name), description: titleize(name) };
return { name, ...metadata, size: componentSize(component) };
const fullSize = historyChunkSize(snapshotBlock, component.chunk_sizes, component.chunk_files);
return {
name,
...metadata,
size: componentSize(component),
...(fullSize === undefined ? {} : { fullSize }),
};
}

function componentSize(component: R2ManifestComponent): number {
Expand All @@ -371,6 +382,31 @@ function componentSize(component: R2ManifestComponent): number {
return 0;
}

const FULL_HISTORY_BLOCKS = 1_339_200;

export function historyChunkSize(
snapshotBlock: number,
chunkSizes: number[] | undefined,
chunkFiles: string[] | undefined,
): number | undefined {
if (!chunkSizes || !chunkFiles || chunkSizes.length !== chunkFiles.length) return undefined;

const ranges = chunkFiles.map((file) => file.match(/-(\d+)-(\d+)\.tar\.zst$/));
if (ranges.some((range) => !range)) return undefined;

const earliestBlock = Math.max(0, snapshotBlock - FULL_HISTORY_BLOCKS);
const includedSizes = chunkSizes.filter((_, index) => {
const range = ranges[index]!;
const startBlock = Number(range[1]);
const endBlock = Number(range[2]);
return startBlock <= snapshotBlock && endBlock >= earliestBlock;
});

return includedSizes.length > 0
? includedSizes.reduce((sum, size) => sum + size, 0)
: undefined;
}

function componentSortIndex(name: string): number {
const index = COMPONENT_ORDER.indexOf(name);
return index === -1 ? COMPONENT_ORDER.length : index;
Expand Down
Loading