From e68fc6bd1a8414065d2cfa5658fb55a8b97a9e20 Mon Sep 17 00:00:00 2001 From: Santiago Palladino Date: Fri, 21 Aug 2026 10:21:05 +0100 Subject: [PATCH] fix(archiver): avoid loading full blocks during checkpoint ingestion Checkpoint ingestion validated chaining by fetching the previous checkpoint's last block as a full L2Block inside the write transaction, deserializing every tx effect for four header fields. Read the lightweight block data instead, and hoist the per-tx/per-block hash serialization out of the log indexing loop. Cuts per-checkpoint ingestion time by 49%/42% at 36/120 txs per block. --- .../archiver/src/store/block_store.ts | 21 ++++++++++++++----- yarn-project/archiver/src/store/log_store.ts | 4 ++-- .../src/store/log_store_codec.test.ts | 11 +++++++--- .../archiver/src/store/log_store_codec.ts | 19 ++++++++++++++--- 4 files changed, 42 insertions(+), 13 deletions(-) diff --git a/yarn-project/archiver/src/store/block_store.ts b/yarn-project/archiver/src/store/block_store.ts index 1e887246d0f..fa8de5e82ea 100644 --- a/yarn-project/archiver/src/store/block_store.ts +++ b/yarn-project/archiver/src/store/block_store.ts @@ -70,6 +70,12 @@ type BlockStorage = { indexWithinCheckpoint: number; }; +/** + * The subset of {@link L2Block} needed to check that a block chains onto the one before it, so validation can run + * against either a full block or the lightweight metadata read back from the store. + */ +type PreviousBlockInfo = Pick; + /** Reason a checkpoint was rejected during sync. */ export type RejectedCheckpointReason = 'invalid-attestations' | 'descends-from-invalid-attestations'; @@ -443,10 +449,10 @@ export class BlockStore { } /** - * Gets the last block of the checkpoint before the given one. + * Gets the chaining info for the last block of the checkpoint before the given one. * Returns undefined if there is no previous checkpoint (i.e. genesis). */ - private async getPreviousCheckpointBlock(checkpointNumber: CheckpointNumber): Promise { + private async getPreviousCheckpointBlock(checkpointNumber: CheckpointNumber): Promise { const previousCheckpointNumber = CheckpointNumber(checkpointNumber - 1); if (previousCheckpointNumber === INITIAL_CHECKPOINT_NUMBER - 1) { return undefined; @@ -462,11 +468,16 @@ export class BlockStore { } const previousBlockNumber = BlockNumber(predecessor.startBlock + predecessor.blockCount - 1); - const previousBlock = await this.getBlock({ number: previousBlockNumber }); + const previousBlock = await this.getBlockData({ number: previousBlockNumber }); if (previousBlock === undefined) { throw new BlockNotFoundError(previousBlockNumber); } - return previousBlock; + return { + number: previousBlock.header.globalVariables.blockNumber, + checkpointNumber: previousBlock.checkpointNumber, + indexWithinCheckpoint: previousBlock.indexWithinCheckpoint, + archive: previousBlock.archive, + }; } /** @@ -474,7 +485,7 @@ export class BlockStore { * This is the same validation used for both confirmed checkpoints (addCheckpoints) and * proposed checkpoints (addProposedCheckpoint). */ - private validateCheckpointBlocks(blocks: L2Block[], previousBlock: L2Block | undefined): void { + private validateCheckpointBlocks(blocks: L2Block[], previousBlock: PreviousBlockInfo | undefined): void { for (const block of blocks) { if (previousBlock) { if (previousBlock.number !== block.number - 1) { diff --git a/yarn-project/archiver/src/store/log_store.ts b/yarn-project/archiver/src/store/log_store.ts index 9c03b504a1e..92083c183c7 100644 --- a/yarn-project/archiver/src/store/log_store.ts +++ b/yarn-project/archiver/src/store/log_store.ts @@ -83,7 +83,7 @@ export class LogStore { addLogs(blocks: L2Block[]): Promise { return this.db.transactionAsync(async () => { for (const block of blocks) { - const blockHash = await block.hash(); + const blockHash = (await block.hash()).toBuffer(); const blockNumber = block.number; const blockTimestamp = block.timestamp; @@ -94,7 +94,7 @@ export class LogStore { for (let txIndexWithinBlock = 0; txIndexWithinBlock < block.body.txEffects.length; txIndexWithinBlock++) { const txEffect = block.body.txEffects[txIndexWithinBlock]; - const txHash = txEffect.txHash; + const txHash = txEffect.txHash.toBuffer(); // Private and public log indices are counted independently per tx, each starting at 0. let privateLogIndexWithinTx = 0; diff --git a/yarn-project/archiver/src/store/log_store_codec.test.ts b/yarn-project/archiver/src/store/log_store_codec.test.ts index 25ee762ee10..70601c91034 100644 --- a/yarn-project/archiver/src/store/log_store_codec.test.ts +++ b/yarn-project/archiver/src/store/log_store_codec.test.ts @@ -200,7 +200,7 @@ describe('log_store_codec', () => { const txHash = TxHash.random(); const blockHash = BlockHash.random(); const blockTimestamp = 1234567890n; - const original = { txHash, blockHash, blockTimestamp, logData: [] }; + const original = { txHash: txHash.toBuffer(), blockHash: blockHash.toBuffer(), blockTimestamp, logData: [] }; const buf = encodeValue(original); const decoded = decodeValue(buf); expect(decoded.txHash.equals(txHash)).toBe(true); @@ -214,7 +214,7 @@ describe('log_store_codec', () => { const blockHash = BlockHash.random(); const blockTimestamp = 9999999999n; const logData = [Fr.random(), Fr.random(), Fr.random()]; - const original = { txHash, blockHash, blockTimestamp, logData }; + const original = { txHash: txHash.toBuffer(), blockHash: blockHash.toBuffer(), blockTimestamp, logData }; const buf = encodeValue(original); const decoded = decodeValue(buf); expect(decoded.txHash.equals(txHash)).toBe(true); @@ -230,7 +230,12 @@ describe('log_store_codec', () => { const txHash = TxHash.random(); const blockHash = BlockHash.random(); const blockTimestamp = 0n; - const original = { txHash, blockHash, blockTimestamp, logData: [Fr.random()] }; + const original = { + txHash: txHash.toBuffer(), + blockHash: blockHash.toBuffer(), + blockTimestamp, + logData: [Fr.random()], + }; const buf = encodeValue(original); // Convert to Uint8Array (not a Buffer). const uint8 = new Uint8Array(buf.buffer, buf.byteOffset, buf.byteLength); diff --git a/yarn-project/archiver/src/store/log_store_codec.ts b/yarn-project/archiver/src/store/log_store_codec.ts index 6af7a898ea6..8067baea358 100644 --- a/yarn-project/archiver/src/store/log_store_codec.ts +++ b/yarn-project/archiver/src/store/log_store_codec.ts @@ -31,6 +31,19 @@ export type StoredLogValue = { logData: Fr[]; }; +/** + * Input to {@link encodeValue}. The hashes come in already serialized because a whole block's logs + * share one `blockHash` and every log of a tx shares its `txHash`, so callers serialize each once. + */ +export type EncodeLogValueInput = { + /** 32-byte serialized tx hash. */ + txHash: Buffer; + /** 32-byte serialized block hash. */ + blockHash: Buffer; + blockTimestamp: bigint; + logData: Fr[]; +}; + /** Returns the 64-char lowercase hex representation of a field, stripping the `0x` prefix. */ export function fieldHex(value: Fr | { toString: () => string }): string { // Fr.toString() and AztecAddress.toString() both return `0x` + 64 lowercase hex chars. @@ -108,10 +121,10 @@ export function incKey(key: string): string { return key + HEX_SENTINEL; } -export function encodeValue(value: StoredLogValue): Buffer { +export function encodeValue(value: EncodeLogValueInput): Buffer { const head = Buffer.allocUnsafe(32 + 32 + 8 + 4); - value.txHash.toBuffer().copy(head, 0); - value.blockHash.toBuffer().copy(head, 32); + value.txHash.copy(head, 0); + value.blockHash.copy(head, 32); head.writeBigUInt64BE(value.blockTimestamp, 64); head.writeUInt32BE(value.logData.length, 72); const fieldBufs = value.logData.map(f => f.toBuffer());