From b93f07e2a418e95d860efe010e0f62b6717fe7e5 Mon Sep 17 00:00:00 2001 From: clearloop Date: Thu, 21 May 2026 07:35:29 +0800 Subject: [PATCH 01/18] perf(crypto): swap blake2 to blake2b_simd --- Cargo.lock | 22 ++++++++++++++++++-- Cargo.toml | 2 +- crates/core/service/Cargo.toml | 5 ++--- crates/core/service/src/lib.rs | 13 +++++++----- crates/crypto/Cargo.toml | 10 +++++++-- crates/crypto/benches/blake2b.rs | 35 ++++++++++++++++++++++++++++++++ crates/crypto/src/lib.rs | 13 +++++++----- 7 files changed, 82 insertions(+), 18 deletions(-) create mode 100644 crates/crypto/benches/blake2b.rs diff --git a/Cargo.lock b/Cargo.lock index cf319ad4..e16a72f6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -568,6 +568,17 @@ dependencies = [ "digest", ] +[[package]] +name = "blake2b_simd" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b79834656f71332577234b50bfc009996f7449e0c056884e6a02492ded0ca2f3" +dependencies = [ + "arrayref", + "arrayvec", + "constant_time_eq 0.4.2", +] + [[package]] name = "blake3" version = "1.8.2" @@ -578,7 +589,7 @@ dependencies = [ "arrayvec", "cc", "cfg-if", - "constant_time_eq", + "constant_time_eq 0.3.1", ] [[package]] @@ -778,6 +789,12 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7c74b8349d32d297c9134b8c88677813a227df8f779daa29bfc29c183fe3dca6" +[[package]] +name = "constant_time_eq" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d52eff69cd5e647efe296129160853a42795992097e8af39800e1060caeea9b" + [[package]] name = "core-foundation" version = "0.10.1" @@ -3515,6 +3532,7 @@ dependencies = [ "ark-serialize 0.5.0", "ark-vrf", "blake2", + "blake2b_simd", "blake3", "criterion", "ed25519-zebra", @@ -3620,7 +3638,7 @@ name = "spacejam-service" version = "0.1.2-pre.1" dependencies = [ "anyhow", - "blake2", + "blake2b_simd", "hex", "serde", "serde-jam", diff --git a/Cargo.toml b/Cargo.toml index cbcb13d4..0599db68 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -77,7 +77,7 @@ ark-serialize = { version = "0.5", default-features = false } ark-vrf = { version = "0.1.0", features = ["bandersnatch", "ring", "parallel"] } async-trait = "0.1.89" bindgen = "0.72" -blake2 = { version = "0.10.6", default-features = false } +blake2b_simd = "1.0.4" blake3 = "1.8.2" cc = "1.2" clap = "4.5.47" diff --git a/crates/core/service/Cargo.toml b/crates/core/service/Cargo.toml index 03cec4fa..f41d252e 100644 --- a/crates/core/service/Cargo.toml +++ b/crates/core/service/Cargo.toml @@ -17,7 +17,7 @@ serde.workspace = true spec.workspace = true # optional dependencies -blake2 = { workspace = true, optional = true } +blake2b_simd = { workspace = true, optional = true } hex = { workspace = true, optional = true } serde_json = { workspace = true, features = ["alloc"], optional = true } spacejson = { workspace = true, optional = true } @@ -26,14 +26,13 @@ spacejson = { workspace = true, optional = true } default = ["std", "tiny"] std = [ "anyhow/std", - "blake2?/std", "codec/std", "hex/std", "serde/std", "serde_json/std", "spacejson?/std", ] -blake2 = ["dep:blake2"] +blake2 = ["dep:blake2b_simd"] json = ["hex", "serde_json", "spacejson", "spacejson/codec"] tiny = ["spec/tiny"] full = ["spec/full"] diff --git a/crates/core/service/src/lib.rs b/crates/core/service/src/lib.rs index 4ddf1d73..7a218dad 100644 --- a/crates/core/service/src/lib.rs +++ b/crates/core/service/src/lib.rs @@ -75,9 +75,12 @@ pub type BlsPublic = [u8; 144]; #[cfg(feature = "blake2")] /// Compute the BLAKE2b 256-bit hash of a given input. pub fn blake2b(input: &[u8]) -> [u8; 32] { - use blake2::{Blake2b, Digest, digest::consts::U32}; - - let mut hasher = Blake2b::::new(); - hasher.update(input); - hasher.finalize().into() + let hash = blake2b_simd::Params::new() + .hash_length(32) + .to_state() + .update(input) + .finalize(); + let mut out = [0u8; 32]; + out.copy_from_slice(hash.as_bytes()); + out } diff --git a/crates/crypto/Cargo.toml b/crates/crypto/Cargo.toml index 65e92972..15d2322a 100644 --- a/crates/crypto/Cargo.toml +++ b/crates/crypto/Cargo.toml @@ -9,7 +9,7 @@ ark-ec = { workspace = true, optional = true } ark-ed-on-bls12-381-bandersnatch = { workspace = true, optional = true } ark-serialize = { workspace = true, optional = true } ark-vrf = { workspace = true, optional = true } -blake2 = { workspace = true, optional = true } +blake2b_simd = { workspace = true, optional = true } blake3 = { workspace = true, optional = true } ed25519-zebra = { workspace = true, optional = true } once_cell = { workspace = true, optional = true } @@ -24,7 +24,7 @@ tracing.workspace = true [features] default = [] full = [] -blake2 = ["dep:blake2"] +blake2 = ["dep:blake2b_simd"] blake3 = ["dep:blake3"] bls = ["w3f-bls"] ed25519 = ["anyhow", "ed25519-zebra", "rand", "rayon"] @@ -44,6 +44,7 @@ vrf = [ ] [dev-dependencies] +blake2 = "0.10.6" criterion = { version = "0.5", default-features = false } hex = { workspace = true, features = ["std"] } serde.workspace = true @@ -54,6 +55,11 @@ name = "ed25519" harness = false required-features = ["ed25519"] +[[bench]] +name = "blake2b" +harness = false +required-features = ["blake2"] + [build-dependencies] ark-serialize.workspace = true ark-vrf.workspace = true diff --git a/crates/crypto/benches/blake2b.rs b/crates/crypto/benches/blake2b.rs new file mode 100644 index 00000000..ab3d0f56 --- /dev/null +++ b/crates/crypto/benches/blake2b.rs @@ -0,0 +1,35 @@ +//! Microbench: BLAKE2b-256 across input sizes representative of the hot paths +//! (trie31 leaves/branches at 32–64B, header/preimage at 100–1KiB, work bundles +//! at 8–64KiB). Compares the in-tree blake2b_simd impl against the RustCrypto +//! `blake2` crate side by side. + +use criterion::{BenchmarkId, Criterion, Throughput, black_box, criterion_group, criterion_main}; + +fn simd(input: &[u8]) -> [u8; 32] { + spacejam_crypto::blake2b(input) +} + +fn rustcrypto(input: &[u8]) -> [u8; 32] { + use blake2::{Blake2b, Digest, digest::consts::U32}; + let mut hasher = Blake2b::::new(); + hasher.update(input); + hasher.finalize().into() +} + +fn bench(c: &mut Criterion) { + let mut group = c.benchmark_group("blake2b"); + for &n in &[32usize, 64, 128, 512, 1024, 8192, 65536] { + let input = vec![0xa5u8; n]; + group.throughput(Throughput::Bytes(n as u64)); + group.bench_with_input(BenchmarkId::new("simd", n), &input, |b, input| { + b.iter(|| simd(black_box(input))); + }); + group.bench_with_input(BenchmarkId::new("rustcrypto", n), &input, |b, input| { + b.iter(|| rustcrypto(black_box(input))); + }); + } + group.finish(); +} + +criterion_group!(benches, bench); +criterion_main!(benches); diff --git a/crates/crypto/src/lib.rs b/crates/crypto/src/lib.rs index cc670f28..b985a649 100644 --- a/crates/crypto/src/lib.rs +++ b/crates/crypto/src/lib.rs @@ -10,11 +10,14 @@ pub mod vrf; #[cfg(feature = "blake2")] /// Compute the BLAKE2b 256-bit hash of a given input. pub fn blake2b(input: &[u8]) -> [u8; 32] { - use blake2::{digest::consts::U32, Blake2b, Digest}; - - let mut hasher = Blake2b::::new(); - hasher.update(input); - hasher.finalize().into() + let hash = blake2b_simd::Params::new() + .hash_length(32) + .to_state() + .update(input) + .finalize(); + let mut out = [0u8; 32]; + out.copy_from_slice(hash.as_bytes()); + out } #[cfg(feature = "blake3")] From 548eaa7536bf057e2418bc97af4c8b8d9e6c72d8 Mon Sep 17 00:00:00 2001 From: clearloop Date: Thu, 21 May 2026 07:47:10 +0800 Subject: [PATCH 02/18] perf(runtime): key MemoryDb by TrieKey --- crates/runtime/src/storage/kv.rs | 24 +++++++++++------------ crates/runtime/src/tx/block/mod.rs | 31 +++++++++++++++--------------- 2 files changed, 28 insertions(+), 27 deletions(-) diff --git a/crates/runtime/src/storage/kv.rs b/crates/runtime/src/storage/kv.rs index b5bec88a..ad56b720 100644 --- a/crates/runtime/src/storage/kv.rs +++ b/crates/runtime/src/storage/kv.rs @@ -2,7 +2,7 @@ use crate::storage::{Column, Commit}; use anyhow::Result; -use score::TrieKey; +use score::{TrieKey, state::StateKeyLike}; use std::{ collections::HashMap, sync::{Arc, RwLock}, @@ -46,7 +46,7 @@ pub trait KVStorage: Send + Sync + 'static { /// It's useful for testing and for situations where persistence isn't required. #[derive(Default)] pub struct MemoryDb { - data: Arc, Vec>>>, + data: Arc>>>, } impl MemoryDb { @@ -54,7 +54,7 @@ impl MemoryDb { /// holding the read lock for the duration. pub fn with_data(&self, f: F) -> Result where - F: FnOnce(&HashMap, Vec>) -> R, + F: FnOnce(&HashMap>) -> R, { let data = self .data @@ -64,7 +64,7 @@ impl MemoryDb { } /// Reset the memory database - pub fn reset(&self, data: HashMap, Vec>) { + pub fn reset(&self, data: HashMap>) { let mut curr = self.data.write().unwrap(); *curr = data; } @@ -78,11 +78,11 @@ impl KVStorage for MemoryDb { .map_err(|_| anyhow::anyhow!("RwLock poisoned"))?; for (key, value) in commit.iset() { - data.insert(key.to_vec(), value.clone()); + data.insert(*key, value.clone()); } for key in commit.iremoval() { - data.remove(key.as_ref()); + data.remove(key); } Ok(()) @@ -94,7 +94,7 @@ impl KVStorage for MemoryDb { .write() .map_err(|_| anyhow::anyhow!("RwLock poisoned"))?; - data.insert(key.as_ref().to_vec(), value.as_ref().to_vec()); + data.insert(key.as_ref().as_state_key(), value.as_ref().to_vec()); Ok(()) } @@ -103,7 +103,7 @@ impl KVStorage for MemoryDb { .data .read() .map_err(|_| anyhow::anyhow!("RwLock poisoned"))?; - Ok(data.get(key.as_ref()).cloned()) + Ok(data.get(&key.as_ref().as_state_key()).cloned()) } fn iter(&self, _column: Column) -> Result, Vec)>>> { @@ -114,7 +114,7 @@ impl KVStorage for MemoryDb { // Clone all entries to avoid holding the lock during iteration let entries: Vec<(Vec, Vec)> = - data.iter().map(|(k, v)| (k.clone(), v.clone())).collect(); + data.iter().map(|(k, v)| (k.to_vec(), v.clone())).collect(); Ok(entries.into_iter().map(Ok)) } @@ -128,13 +128,13 @@ impl KVStorage for MemoryDb { .data .read() .map_err(|_| anyhow::anyhow!("RwLock poisoned"))?; - let prefix_bytes = prefix.as_ref().to_vec(); + let prefix_bytes = prefix.as_ref(); // Clone all matching entries to avoid holding the lock during iteration let matches: Vec<(Vec, Vec)> = data .iter() - .filter(|(k, _)| k.starts_with(&prefix_bytes)) - .map(|(k, v)| (k.clone(), v.clone())) + .filter(|(k, _)| k.starts_with(prefix_bytes)) + .map(|(k, v)| (k.to_vec(), v.clone())) .collect(); Ok(matches.into_iter().map(Ok)) diff --git a/crates/runtime/src/tx/block/mod.rs b/crates/runtime/src/tx/block/mod.rs index 29a28319..812ed767 100644 --- a/crates/runtime/src/tx/block/mod.rs +++ b/crates/runtime/src/tx/block/mod.rs @@ -8,7 +8,7 @@ use crate::{ use anyhow::Result; use crypto::merkle; use pvm::Pvm; -use score::{Block, OpaqueHash, state::StateKeyLike}; +use score::{Block, OpaqueHash, TrieKey, state::StateKeyLike}; use std::{collections::HashMap, sync::Arc}; pub mod header; @@ -120,6 +120,10 @@ impl TestChain { /// Initialize the chain with the given block. pub fn init(&mut self, state: HashMap, Vec>) -> anyhow::Result { + let state: HashMap> = state + .into_iter() + .map(|(k, v)| (k.as_slice().as_state_key(), v)) + .collect(); self.data.reset(state); let head = self .data @@ -145,11 +149,9 @@ impl Default for TestChain { /// Compute the state root and cache it for the given header hash. /// /// NOTE: this method overrides the StateStorage::root for zero-copy. -fn handle_root(head: OpaqueHash, data: &HashMap, Vec>) -> OpaqueHash { - let mut kvs: Vec<([u8; 31], &[u8])> = data - .iter() - .map(|(k, v)| (k.as_slice().as_state_key(), v.as_slice())) - .collect(); +fn handle_root(head: OpaqueHash, data: &HashMap>) -> OpaqueHash { + let mut kvs: Vec<(TrieKey, &[u8])> = + data.iter().map(|(k, v)| (*k, v.as_slice())).collect(); kvs.sort_by_key(|a| a.0); let state_root = merkle::trie31(&kvs); root::set(head, state_root); @@ -161,21 +163,20 @@ fn handle_root(head: OpaqueHash, data: &HashMap, Vec>) -> OpaqueHash /// NOTE: this method overrides the StateStorage::root for zero-copy. fn handle_root_with_diff( head: OpaqueHash, - base: &HashMap, Vec>, - diff: &crate::storage::Commit>, + base: &HashMap>, + diff: &crate::storage::Commit>, ) -> OpaqueHash { - let mut kvs: Vec<([u8; 31], &[u8])> = Vec::with_capacity(base.len() + diff.update.len()); + let mut kvs: Vec<(TrieKey, &[u8])> = Vec::with_capacity(base.len() + diff.update.len()); // Add base entries, applying overlay for (k, v) in base.iter() { - let trie_key = k.as_slice().as_state_key(); - if diff.removal.contains(&trie_key) { + if diff.removal.contains(k) { continue; } - if let Some(updated) = diff.update.get(&trie_key) { - kvs.push((trie_key, updated.as_slice())); + if let Some(updated) = diff.update.get(k) { + kvs.push((*k, updated.as_slice())); } else { - kvs.push((trie_key, v.as_slice())); + kvs.push((*k, v.as_slice())); } } @@ -184,7 +185,7 @@ fn handle_root_with_diff( if diff.removal.contains(trie_key) { continue; } - if !base.contains_key(trie_key.as_ref()) { + if !base.contains_key(trie_key) { kvs.push((*trie_key, v.as_slice())); } } From 2fc5b01866e3b57d80b0b776aff8202317545b2f Mon Sep 17 00:00:00 2001 From: clearloop Date: Thu, 21 May 2026 07:55:28 +0800 Subject: [PATCH 03/18] perf(runtime): drop state-root sort with sorted base + merge-walk --- Cargo.lock | 38 +++++++++--------- Cargo.toml | 2 +- crates/runtime/src/storage/kv.rs | 8 ++-- crates/runtime/src/tx/block/mod.rs | 62 ++++++++++++++++++------------ 4 files changed, 61 insertions(+), 49 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e16a72f6..fe5ed219 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1015,7 +1015,7 @@ dependencies = [ [[package]] name = "cranelift-pvm" -version = "0.1.2-pre.1" +version = "0.1.2-pre.2" dependencies = [ "anyhow", "cranelift", @@ -2757,7 +2757,7 @@ dependencies = [ [[package]] name = "pvmc" -version = "0.1.2-pre.1" +version = "0.1.2-pre.2" dependencies = [ "anyhow", "bindgen", @@ -3292,7 +3292,7 @@ dependencies = [ [[package]] name = "serde-jam" -version = "0.1.2-pre.1" +version = "0.1.2-pre.2" dependencies = [ "anyhow", "serde", @@ -3460,7 +3460,7 @@ dependencies = [ [[package]] name = "spacejam" -version = "0.1.2-pre.1" +version = "0.1.2-pre.2" dependencies = [ "anyhow", "async-trait", @@ -3497,7 +3497,7 @@ dependencies = [ [[package]] name = "spacejam-account" -version = "0.1.2-pre.1" +version = "0.1.2-pre.2" dependencies = [ "anyhow", "serde-jam", @@ -3508,7 +3508,7 @@ dependencies = [ [[package]] name = "spacejam-core" -version = "0.1.2-pre.1" +version = "0.1.2-pre.2" dependencies = [ "anyhow", "erased-serde", @@ -3563,7 +3563,7 @@ dependencies = [ [[package]] name = "spacejam-network" -version = "0.1.2-pre.1" +version = "0.1.2-pre.2" dependencies = [ "anyhow", "clap", @@ -3586,7 +3586,7 @@ dependencies = [ [[package]] name = "spacejam-offchain" -version = "0.1.2-pre.1" +version = "0.1.2-pre.2" dependencies = [ "anyhow", "async-trait", @@ -3602,7 +3602,7 @@ dependencies = [ [[package]] name = "spacejam-rpc" -version = "0.1.2-pre.1" +version = "0.1.2-pre.2" dependencies = [ "anyhow", "jsonrpsee", @@ -3614,7 +3614,7 @@ dependencies = [ [[package]] name = "spacejam-runtime" -version = "0.1.2-pre.1" +version = "0.1.2-pre.2" dependencies = [ "anyhow", "hex", @@ -3635,7 +3635,7 @@ dependencies = [ [[package]] name = "spacejam-service" -version = "0.1.2-pre.1" +version = "0.1.2-pre.2" dependencies = [ "anyhow", "blake2b_simd", @@ -3649,14 +3649,14 @@ dependencies = [ [[package]] name = "spacejam-spec" -version = "0.1.2-pre.1" +version = "0.1.2-pre.2" dependencies = [ "serde", ] [[package]] name = "spacejam-testing" -version = "0.1.2-pre.1" +version = "0.1.2-pre.2" dependencies = [ "anyhow", "hex", @@ -3686,7 +3686,7 @@ dependencies = [ [[package]] name = "spacejam-testnet" -version = "0.1.2-pre.1" +version = "0.1.2-pre.2" dependencies = [ "anyhow", "clap", @@ -3697,7 +3697,7 @@ dependencies = [ [[package]] name = "spacejam-worker" -version = "0.1.2-pre.1" +version = "0.1.2-pre.2" dependencies = [ "anyhow", "pvm", @@ -3713,7 +3713,7 @@ dependencies = [ [[package]] name = "spacejson" -version = "0.1.2-pre.1" +version = "0.1.2-pre.2" dependencies = [ "anyhow", "hex", @@ -3724,7 +3724,7 @@ dependencies = [ [[package]] name = "spacejson-derive" -version = "0.1.2-pre.1" +version = "0.1.2-pre.2" dependencies = [ "proc-macro2", "quote", @@ -3733,7 +3733,7 @@ dependencies = [ [[package]] name = "spacevm" -version = "0.1.2-pre.1" +version = "0.1.2-pre.2" dependencies = [ "anyhow", "hex", @@ -3747,7 +3747,7 @@ dependencies = [ [[package]] name = "spacevm-export" -version = "0.1.2-pre.1" +version = "0.1.2-pre.2" dependencies = [ "pvm", "pvmc", diff --git a/Cargo.toml b/Cargo.toml index 0599db68..dd51f22b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -21,7 +21,7 @@ resolver = "1" [workspace.package] edition = "2024" -version = "0.1.2-pre.1" +version = "0.1.2-pre.2" homepage = "https://spacejam.app" repository = "https://github.com/spacejamapp/jade" license = "GPL-3.0" diff --git a/crates/runtime/src/storage/kv.rs b/crates/runtime/src/storage/kv.rs index ad56b720..cf68aeba 100644 --- a/crates/runtime/src/storage/kv.rs +++ b/crates/runtime/src/storage/kv.rs @@ -4,7 +4,7 @@ use crate::storage::{Column, Commit}; use anyhow::Result; use score::{TrieKey, state::StateKeyLike}; use std::{ - collections::HashMap, + collections::BTreeMap, sync::{Arc, RwLock}, }; @@ -46,7 +46,7 @@ pub trait KVStorage: Send + Sync + 'static { /// It's useful for testing and for situations where persistence isn't required. #[derive(Default)] pub struct MemoryDb { - data: Arc>>>, + data: Arc>>>, } impl MemoryDb { @@ -54,7 +54,7 @@ impl MemoryDb { /// holding the read lock for the duration. pub fn with_data(&self, f: F) -> Result where - F: FnOnce(&HashMap>) -> R, + F: FnOnce(&BTreeMap>) -> R, { let data = self .data @@ -64,7 +64,7 @@ impl MemoryDb { } /// Reset the memory database - pub fn reset(&self, data: HashMap>) { + pub fn reset(&self, data: BTreeMap>) { let mut curr = self.data.write().unwrap(); *curr = data; } diff --git a/crates/runtime/src/tx/block/mod.rs b/crates/runtime/src/tx/block/mod.rs index 812ed767..bb190635 100644 --- a/crates/runtime/src/tx/block/mod.rs +++ b/crates/runtime/src/tx/block/mod.rs @@ -9,7 +9,10 @@ use anyhow::Result; use crypto::merkle; use pvm::Pvm; use score::{Block, OpaqueHash, TrieKey, state::StateKeyLike}; -use std::{collections::HashMap, sync::Arc}; +use std::{ + collections::{BTreeMap, HashMap}, + sync::Arc, +}; pub mod header; pub mod history; @@ -120,7 +123,7 @@ impl TestChain { /// Initialize the chain with the given block. pub fn init(&mut self, state: HashMap, Vec>) -> anyhow::Result { - let state: HashMap> = state + let state: BTreeMap> = state .into_iter() .map(|(k, v)| (k.as_slice().as_state_key(), v)) .collect(); @@ -148,11 +151,11 @@ impl Default for TestChain { /// Compute the state root and cache it for the given header hash. /// +/// Base is sorted (BTreeMap), so iteration already yields keys in trie order. +/// /// NOTE: this method overrides the StateStorage::root for zero-copy. -fn handle_root(head: OpaqueHash, data: &HashMap>) -> OpaqueHash { - let mut kvs: Vec<(TrieKey, &[u8])> = - data.iter().map(|(k, v)| (*k, v.as_slice())).collect(); - kvs.sort_by_key(|a| a.0); +fn handle_root(head: OpaqueHash, data: &BTreeMap>) -> OpaqueHash { + let kvs: Vec<(TrieKey, &[u8])> = data.iter().map(|(k, v)| (*k, v.as_slice())).collect(); let state_root = merkle::trie31(&kvs); root::set(head, state_root); state_root @@ -160,37 +163,46 @@ fn handle_root(head: OpaqueHash, data: &HashMap>) -> OpaqueHash /// Compute the state root from base data + overlay diff (no base clone). /// +/// Both base and diff.update are sorted by TrieKey, so we merge-walk them in +/// O(N+M) and skip the final sort. On ties diff wins (it's the newer write); +/// keys present in diff.removal are dropped after selection so removals take +/// precedence over updates of the same key. +/// /// NOTE: this method overrides the StateStorage::root for zero-copy. fn handle_root_with_diff( head: OpaqueHash, - base: &HashMap>, + base: &BTreeMap>, diff: &crate::storage::Commit>, ) -> OpaqueHash { let mut kvs: Vec<(TrieKey, &[u8])> = Vec::with_capacity(base.len() + diff.update.len()); - - // Add base entries, applying overlay - for (k, v) in base.iter() { - if diff.removal.contains(k) { - continue; + let mut base_iter = base.iter(); + let mut diff_iter = diff.update.iter(); + let mut b = base_iter.next(); + let mut d = diff_iter.next(); + + while let Some((key, value)) = match (b, d) { + (Some((bk, _)), Some((dk, dv))) if dk <= bk => { + if dk == bk { + b = base_iter.next(); + } + d = diff_iter.next(); + Some((dk, dv.as_slice())) } - if let Some(updated) = diff.update.get(k) { - kvs.push((*k, updated.as_slice())); - } else { - kvs.push((*k, v.as_slice())); + (Some((bk, bv)), _) => { + b = base_iter.next(); + Some((bk, bv.as_slice())) } - } - - // Add new keys from overlay (not in base) - for (trie_key, v) in diff.update.iter() { - if diff.removal.contains(trie_key) { - continue; + (None, Some((dk, dv))) => { + d = diff_iter.next(); + Some((dk, dv.as_slice())) } - if !base.contains_key(trie_key) { - kvs.push((*trie_key, v.as_slice())); + (None, None) => None, + } { + if !diff.removal.contains(key) { + kvs.push((*key, value)); } } - kvs.sort_by_key(|a| a.0); let state_root = merkle::trie31(&kvs); root::set(head, state_root); state_root From 548fbdb12d52144ef698baa85047c4dd63293d02 Mon Sep 17 00:00:00 2001 From: clearloop Date: Thu, 21 May 2026 09:20:29 +0800 Subject: [PATCH 04/18] perf(runtime): incremental state root via parity-db multitree --- Cargo.lock | 6 +- Cargo.toml | 2 +- crates/crypto/src/merkle/trie31.rs | 44 +++++- crates/runtime/src/storage/branch.rs | 30 +++- crates/runtime/src/storage/kv.rs | 143 +++++++++++++++++- crates/runtime/src/storage/mod.rs | 9 +- crates/runtime/src/storage/multitree.rs | 192 ++++++++++++++++++++++++ crates/runtime/src/tx/block/mod.rs | 57 +++++-- crates/runtime/tests/trie.rs | 129 ++++++++++++++++ crates/spacejam/src/fuzz/env.rs | 2 +- crates/spacejam/src/storage/parity.rs | 67 ++++++++- 11 files changed, 649 insertions(+), 32 deletions(-) create mode 100644 crates/runtime/src/storage/multitree.rs create mode 100644 crates/runtime/tests/trie.rs diff --git a/Cargo.lock b/Cargo.lock index fe5ed219..e80a5dab 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -24,6 +24,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" dependencies = [ "cfg-if", + "getrandom 0.3.3", "once_cell", "version_check", "zerocopy", @@ -2544,10 +2545,11 @@ checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" [[package]] name = "parity-db" -version = "0.5.4" +version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6985a45b0597d68448dac9db2907f9f72bbaf63fe3383d4ba15f99096c87212f" +checksum = "8b63063d738c6e39a9e29e0821fc7293f1d862e75bac6e39bcea131b44c03ade" dependencies = [ + "ahash", "blake2", "crc32fast", "fs2", diff --git a/Cargo.toml b/Cargo.toml index dd51f22b..382d5047 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -102,7 +102,7 @@ libc = "0.2" log = "0.4.28" once_cell = "1.21.3" paste = "1.0.15" -parity-db = "0.5.4" +parity-db = "0.5.5" postcard = "1.1.3" proc-macro2 = "1.0.101" prometheus = { version = "0.23.1", package = "prometheus-client" } diff --git a/crates/crypto/src/merkle/trie31.rs b/crates/crypto/src/merkle/trie31.rs index 0b0986c0..ea778373 100644 --- a/crates/crypto/src/merkle/trie31.rs +++ b/crates/crypto/src/merkle/trie31.rs @@ -34,9 +34,36 @@ fn merkle(kvs: &mut [([u8; 31], &[u8])], depth: usize) -> [u8; 32] { blake2b(&branch(l_hash, r_hash)) } +/// Find the partition index of a slice of keys already sorted such that +/// `bit=0` entries come before `bit=1` entries at `depth`. Linear scan — use +/// this when the slice's relative order is known to be preserved. +pub fn split_at_bit(keys: &[[u8; 31]], depth: usize) -> usize { + keys.iter().position(|k| bit(k, depth)).unwrap_or(keys.len()) +} + +/// Recover `(left, right)` child slots for a branch given its encoded payload +/// and stored child addresses (which omit empty subtrees). `None` if the +/// payload's empty-side encoding doesn't match the address count. Generic +/// over the address type so storage backends can keep their own +/// representation. +pub fn split_branch_children( + data: &[u8], + children: &[T], +) -> Option<(Option, Option)> { + let l_empty = data[0] == 0 && data[1..32].iter().all(|&b| b == 0); + let r_empty = data[32..64].iter().all(|&b| b == 0); + match (l_empty, r_empty, children.len()) { + (false, false, 2) => Some((Some(children[0]), Some(children[1]))), + (true, false, 1) => Some((None, Some(children[0]))), + (false, true, 1) => Some((Some(children[0]), None)), + (true, true, 0) => Some((None, None)), + _ => None, + } +} + /// Partition `kvs` in-place so that entries with bit 0 at `depth` come first. /// Returns the index of the first "right" (bit=1) entry. -fn partition(kvs: &mut [([u8; 31], &[u8])], depth: usize) -> usize { +pub fn partition(kvs: &mut [([u8; 31], &[u8])], depth: usize) -> usize { let mut left_end = 0; for i in 0..kvs.len() { if !bit(&kvs[i].0, depth) { @@ -47,8 +74,13 @@ fn partition(kvs: &mut [([u8; 31], &[u8])], depth: usize) -> usize { left_end } -/// Branch encoding -fn branch(l: [u8; 32], r: [u8; 32]) -> [u8; 64] { +/// Whether a node's encoded payload is a leaf (high bit of byte 0 set). +pub fn is_leaf(data: &[u8]) -> bool { + data.first().is_some_and(|b| b & 0x80 != 0) +} + +/// Branch encoding (graypaper D.6). +pub fn branch(l: [u8; 32], r: [u8; 32]) -> [u8; 64] { let mut encoded = [0u8; 64]; encoded[0] = l[0] & 0x7F; // 0b01111111 encoded[1..32].copy_from_slice(&l[1..]); @@ -56,8 +88,8 @@ fn branch(l: [u8; 32], r: [u8; 32]) -> [u8; 64] { encoded } -/// Leaf encoding -fn leaf(k: [u8; 31], v: &[u8]) -> [u8; 64] { +/// Leaf encoding (graypaper D.6). +pub fn leaf(k: [u8; 31], v: &[u8]) -> [u8; 64] { let mut encoded = [0u8; 64]; if v.len() <= 32 { @@ -80,7 +112,7 @@ fn leaf(k: [u8; 31], v: &[u8]) -> [u8; 64] { /// Get the bit at the specified position in the key /// For 31-byte keys, we have 248 bits (31*8) -fn bit(k: &[u8; 31], depth: usize) -> bool { +pub fn bit(k: &[u8; 31], depth: usize) -> bool { let byte_idx = depth / 8; let bit_idx = 7 - (depth % 8); (k[byte_idx] & (1 << bit_idx)) != 0 diff --git a/crates/runtime/src/storage/branch.rs b/crates/runtime/src/storage/branch.rs index eefdd9ae..3a956e48 100644 --- a/crates/runtime/src/storage/branch.rs +++ b/crates/runtime/src/storage/branch.rs @@ -2,10 +2,10 @@ use crate::{ Storage, - storage::{Column, Commit, KVStorage, StateStorage}, + storage::{Column, Commit, KVStorage, MultiTreeStore, NewNode, NodeAddress, StateStorage}, }; use anyhow::Result; -use score::{TrieKey, state::StateKeyLike}; +use score::{OpaqueHash, TrieKey, state::StateKeyLike}; use std::{ collections::{BTreeMap, btree_map::IntoIter}, mem, @@ -117,6 +117,32 @@ impl KVStorage for Branch { } } +impl MultiTreeStore for Branch { + fn insert_tree(&self, column: Column, key: OpaqueHash, root: NewNode) -> Result<()> { + self.state.insert_tree(column, key, root) + } + + fn dereference_tree(&self, column: Column, key: OpaqueHash) -> Result<()> { + self.state.dereference_tree(column, key) + } + + fn get_root( + &self, + column: Column, + key: OpaqueHash, + ) -> Result, Vec)>> { + self.state.get_root(column, key) + } + + fn get_node( + &self, + column: Column, + address: NodeAddress, + ) -> Result, Vec)>> { + self.state.get_node(column, address) + } +} + impl Clone for Branch { fn clone(&self) -> Self { Self { diff --git a/crates/runtime/src/storage/kv.rs b/crates/runtime/src/storage/kv.rs index cf68aeba..db6846da 100644 --- a/crates/runtime/src/storage/kv.rs +++ b/crates/runtime/src/storage/kv.rs @@ -1,10 +1,10 @@ //! Key-value storage abstraction -use crate::storage::{Column, Commit}; +use crate::storage::{Column, Commit, MultiTreeStore, NewNode, NodeAddress, NodeRef}; use anyhow::Result; -use score::{TrieKey, state::StateKeyLike}; +use score::{OpaqueHash, TrieKey, state::StateKeyLike}; use std::{ - collections::BTreeMap, + collections::{BTreeMap, HashMap}, sync::{Arc, RwLock}, }; @@ -47,6 +47,81 @@ pub trait KVStorage: Send + Sync + 'static { #[derive(Default)] pub struct MemoryDb { data: Arc>>>, + tries: Arc>, +} + +/// Internal ref-counted node store backing [`MultiTreeStore`]. +#[derive(Default)] +struct TrieStore { + nodes: HashMap, + roots: HashMap, + next_addr: NodeAddress, +} + +struct NodeEntry { + data: Vec, + children: Vec, + refcount: u32, +} + +impl TrieStore { + fn alloc(&mut self) -> NodeAddress { + let addr = self.next_addr; + self.next_addr += 1; + addr + } + + /// Recursively materialize a `NodeRef` into the store, returning the + /// address of its root and bumping refcounts as we go. + fn insert(&mut self, node_ref: NodeRef) -> NodeAddress { + match node_ref { + NodeRef::Existing(addr) => { + let entry = self + .nodes + .get_mut(&addr) + .unwrap_or_else(|| panic!("NodeRef::Existing({addr}) but node is gone")); + entry.refcount += 1; + addr + } + NodeRef::New(NewNode { data, children }) => { + let child_addrs: Vec<_> = children.into_iter().map(|c| self.insert(c)).collect(); + let addr = self.alloc(); + self.nodes.insert( + addr, + NodeEntry { + data, + children: child_addrs, + refcount: 1, + }, + ); + addr + } + } + } + + /// Drop a reference to `addr`, recursively GC'ing descendants that hit 0. + fn dereference(&mut self, addr: NodeAddress) { + let drop_children = match self.nodes.get_mut(&addr) { + Some(entry) => { + entry.refcount = entry + .refcount + .checked_sub(1) + .unwrap_or_else(|| panic!("refcount underflow on addr {addr}")); + if entry.refcount == 0 { + Some(std::mem::take(&mut entry.children)) + } else { + None + } + } + None => None, + }; + if let Some(children) = drop_children { + self.nodes.remove(&addr); + for child in children { + self.dereference(child); + } + } + } } impl MemoryDb { @@ -140,3 +215,65 @@ impl KVStorage for MemoryDb { Ok(matches.into_iter().map(Ok)) } } + +impl MultiTreeStore for MemoryDb { + fn insert_tree(&self, _column: Column, key: OpaqueHash, root: NewNode) -> Result<()> { + let mut tries = self + .tries + .write() + .map_err(|_| anyhow::anyhow!("RwLock poisoned"))?; + if tries.roots.contains_key(&key) { + anyhow::bail!( + "insert_tree: tree already present at key 0x{} — caller must dereference_tree first", + hex::encode(key) + ); + } + let addr = tries.insert(NodeRef::New(root)); + tries.roots.insert(key, addr); + Ok(()) + } + + fn dereference_tree(&self, _column: Column, key: OpaqueHash) -> Result<()> { + let mut tries = self + .tries + .write() + .map_err(|_| anyhow::anyhow!("RwLock poisoned"))?; + if let Some(addr) = tries.roots.remove(&key) { + tries.dereference(addr); + } + Ok(()) + } + + fn get_root( + &self, + _column: Column, + key: OpaqueHash, + ) -> Result, Vec)>> { + let tries = self + .tries + .read() + .map_err(|_| anyhow::anyhow!("RwLock poisoned"))?; + let Some(&addr) = tries.roots.get(&key) else { + return Ok(None); + }; + Ok(tries + .nodes + .get(&addr) + .map(|n| (n.data.clone(), n.children.clone()))) + } + + fn get_node( + &self, + _column: Column, + address: NodeAddress, + ) -> Result, Vec)>> { + let tries = self + .tries + .read() + .map_err(|_| anyhow::anyhow!("RwLock poisoned"))?; + Ok(tries + .nodes + .get(&address) + .map(|n| (n.data.clone(), n.children.clone()))) + } +} diff --git a/crates/runtime/src/storage/mod.rs b/crates/runtime/src/storage/mod.rs index 9f01baaf..830c038c 100644 --- a/crates/runtime/src/storage/mod.rs +++ b/crates/runtime/src/storage/mod.rs @@ -7,6 +7,7 @@ pub use { branch::Branch, commit::{Commit, Operation}, kv::{KVStorage, MemoryDb}, + multitree::{MultiTreeStore, NewNode, NodeAddress, NodeRef}, state::StateStorage, sync::SyncStorage, }; @@ -15,6 +16,7 @@ mod archive; mod branch; mod commit; mod kv; +mod multitree; pub mod root; mod state; pub mod sync; @@ -23,14 +25,17 @@ pub mod sync; #[repr(u8)] #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] pub enum Column { - /// The column for the sync + /// The column for the state State = 0, - /// The column for the state + /// The column for the sync Sync = 1, /// The column for the archive Archive = 2, + + /// The column for incremental trie nodes (parity-db multitree). + TrieNodes = 3, } /// The storage of SpaceJam diff --git a/crates/runtime/src/storage/multitree.rs b/crates/runtime/src/storage/multitree.rs new file mode 100644 index 00000000..87a74b24 --- /dev/null +++ b/crates/runtime/src/storage/multitree.rs @@ -0,0 +1,192 @@ +//! Multi-tree (ref-counted Merkle node) storage abstraction. +//! +//! Mirrors parity-db's multitree API so the runtime can drive incremental trie +//! updates against either parity-db or the in-memory shim in [`MemoryDb`], +//! without depending on parity-db directly. + +use crate::storage::Column; +use anyhow::{Result, anyhow}; +use crypto::merkle::trie31; +use score::{OpaqueHash, TrieKey}; + +const ZERO_HASH: OpaqueHash = [0u8; 32]; + +/// Address of a node already persisted in the store. +pub type NodeAddress = u64; + +/// A reference used when building a tree to commit: either a brand-new subtree +/// that must be written, or a pointer to an existing node that the new tree +/// should reuse (and which the store will ref-count automatically). +#[derive(Debug, Clone)] +pub enum NodeRef { + /// A fresh subtree. + New(NewNode), + /// A node already in the store, retained as-is. + Existing(NodeAddress), +} + +/// A newly-built node with its child references. +#[derive(Debug, Clone)] +pub struct NewNode { + /// Encoded node payload (leaf or branch). + pub data: Vec, + /// Child references, in canonical order. + pub children: Vec, +} + +/// Ref-counted Merkle tree storage. +pub trait MultiTreeStore: Send + Sync + 'static { + /// Commit a new tree rooted at `key`. Nodes referenced via + /// [`NodeRef::Existing`] have their refcount bumped automatically. + fn insert_tree(&self, column: Column, key: OpaqueHash, root: NewNode) -> Result<()>; + + /// Drop the tree at `key`. Nodes whose refcount falls to zero are GC'd. + fn dereference_tree(&self, column: Column, key: OpaqueHash) -> Result<()>; + + /// Get the root node (encoded data + immediate children) for the tree at + /// `key`, if present. + fn get_root( + &self, + column: Column, + key: OpaqueHash, + ) -> Result, Vec)>>; + + /// Get a node by its address. + fn get_node( + &self, + column: Column, + address: NodeAddress, + ) -> Result, Vec)>>; + + /// Compute the new state root from `new_state`, reusing unchanged subtrees + /// of the tree at `prev_root`. Writes the new tree into the store and + /// dereferences the old one if it differs. + /// + /// - The tree shape matches `crypto::merkle::trie31` (graypaper D.6), so + /// the returned root is bit-identical to the non-incremental reference. + /// - `new_state` is the full sorted post-diff state. + /// - `dirty_keys` is the sorted set of keys touched by the diff (inserts, + /// updates, removals). Subtrees whose dirty slice is empty are reused + /// via [`NodeRef::Existing`]. + fn apply( + &self, + column: Column, + prev_root: Option, + new_state: &[(TrieKey, &[u8])], + dirty_keys: &[TrieKey], + ) -> Result { + let prev_root_node = match prev_root { + Some(h) if h != ZERO_HASH => self.get_root(column, h)?, + _ => None, + }; + + let (root_ref, new_root) = self.build(column, prev_root_node, new_state, dirty_keys, 0)?; + + // If the tree didn't change there's nothing to write or release; the + // existing tree at `new_root` already holds every node the rebuild + // would have referenced. + if matches!(prev_root, Some(p) if p == new_root) { + return Ok(new_root); + } + + if let Some(NodeRef::New(root_node)) = root_ref { + self.insert_tree(column, new_root, root_node)?; + } + + if let Some(prev) = prev_root { + if prev != ZERO_HASH && prev != new_root { + self.dereference_tree(column, prev)?; + } + } + + Ok(new_root) + } + + /// Build the subtree at this position, with `prev_node` being the matching + /// node from the previous tree (if any). Internal helper for [`apply`]. + fn build( + &self, + column: Column, + prev_node: Option<(Vec, Vec)>, + new_keys: &[(TrieKey, &[u8])], + new_dirty: &[TrieKey], + depth: usize, + ) -> Result<(Option, OpaqueHash)> { + if new_keys.is_empty() { + return Ok((None, ZERO_HASH)); + } + + if new_keys.len() == 1 { + let (k, v) = new_keys[0]; + let data = trie31::leaf(k, v).to_vec(); + let hash = crypto::blake2b(&data); + return Ok((Some(NodeRef::New(NewNode { data, children: vec![] })), hash)); + } + + // Branch case: partition both keys and dirty by bit at `depth`. + let mut buf = new_keys.to_vec(); + let key_mid = trie31::partition(&mut buf, depth); + let (left_keys, right_keys) = buf.split_at(key_mid); + let dirty_mid = trie31::split_at_bit(new_dirty, depth); + let (left_dirty, right_dirty) = new_dirty.split_at(dirty_mid); + + let (prev_left, prev_right) = match &prev_node { + Some((data, children)) if !trie31::is_leaf(data) => trie31::split_branch_children( + data, children, + ) + .ok_or_else(|| { + anyhow!( + "trie node shape mismatch at depth {depth}: children.len()={}", + children.len() + ) + })?, + _ => (None, None), + }; + + let (l_ref, l_hash) = self.descend(column, prev_left, left_keys, left_dirty, depth + 1)?; + let (r_ref, r_hash) = self.descend(column, prev_right, right_keys, right_dirty, depth + 1)?; + + let data = trie31::branch(l_hash, r_hash).to_vec(); + let hash = crypto::blake2b(&data); + let mut children = Vec::with_capacity(2); + if let Some(l) = l_ref { + children.push(l); + } + if let Some(r) = r_ref { + children.push(r); + } + Ok((Some(NodeRef::New(NewNode { data, children })), hash)) + } + + /// Recurse into a child of the current branch. Reuses the subtree as-is + /// when no dirty key touches it. Internal helper for [`apply`]. + fn descend( + &self, + column: Column, + prev_addr: Option, + new_keys: &[(TrieKey, &[u8])], + new_dirty: &[TrieKey], + depth: usize, + ) -> Result<(Option, OpaqueHash)> { + if new_dirty.is_empty() { + if let Some(addr) = prev_addr { + let (data, _children) = self + .get_node(column, addr)? + .ok_or_else(|| anyhow!("missing prev node at address {addr}"))?; + let hash = crypto::blake2b(&data); + return Ok((Some(NodeRef::Existing(addr)), hash)); + } + if new_keys.is_empty() { + return Ok((None, ZERO_HASH)); + } + // Fall through: build fresh when caller's dirty tracking is + // over-conservative. + } + + let prev_node = match prev_addr { + Some(addr) => self.get_node(column, addr)?, + None => None, + }; + self.build(column, prev_node, new_keys, new_dirty, depth) + } +} diff --git a/crates/runtime/src/tx/block/mod.rs b/crates/runtime/src/tx/block/mod.rs index bb190635..2e477df3 100644 --- a/crates/runtime/src/tx/block/mod.rs +++ b/crates/runtime/src/tx/block/mod.rs @@ -2,7 +2,7 @@ use crate::{ Storage, - storage::{Branch, Column, KVStorage, MemoryDb, StateStorage, root}, + storage::{Branch, Column, Commit, KVStorage, MemoryDb, MultiTreeStore, StateStorage, root}, tx, }; use anyhow::Result; @@ -14,6 +14,9 @@ use std::{ sync::Arc, }; +/// Zero hash sentinel — `current_root` before [`TestChain::init`]. +const EMPTY_ROOT: OpaqueHash = [0; 32]; + pub mod header; pub mod history; @@ -49,6 +52,10 @@ pub struct TestChain { /// The forks and their states (diff only). pub forks: HashMap, + + /// The state root corresponding to `data`. Tracked incrementally via the + /// multitree column so we can skip the O(N log N) full retrie per block. + current_root: OpaqueHash, } impl TestChain { @@ -65,7 +72,20 @@ impl TestChain { .read() .map_err(|_| anyhow::anyhow!("lock poisoned"))? .clone(); + let dirty = collect_dirty_keys(&commit); self.data.commit(Column::State, commit)?; + + // Incrementally rebuild the trie against the new `data` and prime + // the root cache so the next block's `state()` hits it. + let prev = (self.current_root != EMPTY_ROOT).then_some(self.current_root); + let new_root = self.data.with_data(|data| { + let kvs: Vec<(TrieKey, &[u8])> = + data.iter().map(|(k, v)| (*k, v.as_slice())).collect(); + self.data.apply(Column::TrieNodes, prev, &kvs, &dirty) + })??; + root::set(parent, new_root); + self.current_root = new_root; + self.finalized = parent; self.forks.clear(); } @@ -135,7 +155,18 @@ impl TestChain { .ok_or(anyhow::anyhow!("no recent blocks"))? .header_hash; self.finalized = head; - self.data.with_data(|data| handle_root(head, data)) + + // Bootstrap the incremental trie. Every key is dirty on the first + // build; the algorithm degenerates to a full retrie. + let new_root = self.data.with_data(|data| { + let kvs: Vec<(TrieKey, &[u8])> = + data.iter().map(|(k, v)| (*k, v.as_slice())).collect(); + let dirty: Vec = data.keys().copied().collect(); + self.data.apply(Column::TrieNodes, None, &kvs, &dirty) + })??; + root::set(head, new_root); + self.current_root = new_root; + Ok(new_root) } } @@ -145,20 +176,22 @@ impl Default for TestChain { finalized: Default::default(), data: Arc::new(MemoryDb::default()), forks: HashMap::new(), + current_root: EMPTY_ROOT, } } } -/// Compute the state root and cache it for the given header hash. -/// -/// Base is sorted (BTreeMap), so iteration already yields keys in trie order. -/// -/// NOTE: this method overrides the StateStorage::root for zero-copy. -fn handle_root(head: OpaqueHash, data: &BTreeMap>) -> OpaqueHash { - let kvs: Vec<(TrieKey, &[u8])> = data.iter().map(|(k, v)| (*k, v.as_slice())).collect(); - let state_root = merkle::trie31(&kvs); - root::set(head, state_root); - state_root +/// Sorted, deduplicated union of the keys an overlay commit touches. +fn collect_dirty_keys(commit: &Commit>) -> Vec { + let mut keys: Vec = commit + .update + .keys() + .copied() + .chain(commit.removal.iter().copied()) + .collect(); + keys.sort_unstable(); + keys.dedup(); + keys } /// Compute the state root from base data + overlay diff (no base clone). diff --git a/crates/runtime/tests/trie.rs b/crates/runtime/tests/trie.rs new file mode 100644 index 00000000..1dd1f812 --- /dev/null +++ b/crates/runtime/tests/trie.rs @@ -0,0 +1,129 @@ +//! Cross-checks the incremental [`MultiTreeStore::apply`] root against the +//! reference [`crypto::merkle::trie31`] root. + +use crypto::{blake2b, merkle}; +use spacejam_runtime::storage::{Column, MemoryDb, MultiTreeStore}; +use score::{OpaqueHash, TrieKey}; +use std::collections::BTreeMap; + +fn key(seed: u64) -> TrieKey { + let mut k = [0u8; 31]; + let bytes = blake2b(&seed.to_le_bytes()); + k.copy_from_slice(&bytes[..31]); + k +} + +fn value(seed: u64, len: usize) -> Vec { + let mut v = Vec::with_capacity(len); + let mut s = seed; + while v.len() < len { + s = s.wrapping_mul(6364136223846793005).wrapping_add(1); + v.extend_from_slice(&s.to_le_bytes()); + } + v.truncate(len); + v +} + +fn reference_root(state: &BTreeMap>) -> OpaqueHash { + let kvs: Vec<(TrieKey, &[u8])> = state.iter().map(|(k, v)| (*k, v.as_slice())).collect(); + merkle::trie31(&kvs) +} + +fn apply( + db: &MemoryDb, + prev: Option, + state: &BTreeMap>, + dirty: &[TrieKey], +) -> OpaqueHash { + let kvs: Vec<(TrieKey, &[u8])> = state.iter().map(|(k, v)| (*k, v.as_slice())).collect(); + db.apply(Column::TrieNodes, prev, &kvs, dirty).expect("apply") +} + +#[test] +fn empty_state() { + let db = MemoryDb::default(); + let root = apply(&db, None, &BTreeMap::new(), &[]); + assert_eq!(root, [0u8; 32]); +} + +#[test] +fn single_leaf() { + let db = MemoryDb::default(); + let mut state = BTreeMap::new(); + state.insert(key(1), value(1, 10)); + let dirty: Vec<_> = state.keys().copied().collect(); + let root = apply(&db, None, &state, &dirty); + assert_eq!(root, reference_root(&state)); +} + +#[test] +fn bootstrap_matches_trie31() { + let db = MemoryDb::default(); + let mut state = BTreeMap::new(); + for i in 0..500u64 { + state.insert(key(i), value(i, 40)); + } + let dirty: Vec<_> = state.keys().copied().collect(); + let root = apply(&db, None, &state, &dirty); + assert_eq!(root, reference_root(&state)); +} + +#[test] +fn incremental_update_matches_trie31() { + let db = MemoryDb::default(); + let mut state = BTreeMap::new(); + for i in 0..500u64 { + state.insert(key(i), value(i, 40)); + } + let dirty0: Vec<_> = state.keys().copied().collect(); + let root0 = apply(&db, None, &state, &dirty0); + assert_eq!(root0, reference_root(&state)); + + let mut dirty = Vec::new(); + for i in 0..5u64 { + let k = key(i); + state.insert(k, value(i + 1000, 50)); + dirty.push(k); + } + for i in 500..503u64 { + let k = key(i); + state.insert(k, value(i, 60)); + dirty.push(k); + } + for i in 100..102u64 { + let k = key(i); + state.remove(&k); + dirty.push(k); + } + dirty.sort(); + + let root1 = apply(&db, Some(root0), &state, &dirty); + assert_eq!(root1, reference_root(&state)); +} + +#[test] +fn many_sequential_diffs_match_trie31() { + let db = MemoryDb::default(); + let mut state = BTreeMap::new(); + for i in 0..200u64 { + state.insert(key(i), value(i, 32)); + } + let dirty: Vec<_> = state.keys().copied().collect(); + let mut root = apply(&db, None, &state, &dirty); + assert_eq!(root, reference_root(&state)); + + for round in 1..=20u64 { + let mut dirty = Vec::new(); + for j in 0..10u64 { + let k = key((round * 7 + j) % 200); + // Offset by 1_000_000 so we never accidentally write back a + // bootstrap value and leave the root unchanged. + state.insert(k, value(round * 100 + j + 1_000_000, 32)); + dirty.push(k); + } + dirty.sort(); + dirty.dedup(); + root = apply(&db, Some(root), &state, &dirty); + assert_eq!(root, reference_root(&state), "round {round}"); + } +} diff --git a/crates/spacejam/src/fuzz/env.rs b/crates/spacejam/src/fuzz/env.rs index f8ac3e90..c4dde299 100644 --- a/crates/spacejam/src/fuzz/env.rs +++ b/crates/spacejam/src/fuzz/env.rs @@ -117,7 +117,7 @@ fn require_env(name: &str) -> Result { fn init_logger(level: Option<&str>) { let Some(level) = level else { return }; let _ = tracing_subscriber::fmt() - .with_env_filter(EnvFilter::new(level)) + .with_env_filter(EnvFilter::new(format!("spacejam={level}"))) .with_target(false) .with_ansi(false) .try_init(); diff --git a/crates/spacejam/src/storage/parity.rs b/crates/spacejam/src/storage/parity.rs index ae88fe56..7252f0af 100644 --- a/crates/spacejam/src/storage/parity.rs +++ b/crates/spacejam/src/storage/parity.rs @@ -1,9 +1,14 @@ //! The parity database storage use anyhow::Result; -use parity_db::{BTreeIterator, ColumnOptions, Db, Operation as Op, Options}; -use runtime::storage::{Column, Commit, KVStorage, Operation}; -use score::TrieKey; +use parity_db::{ + BTreeIterator, ColumnOptions, Db, NewNode as PdNewNode, NodeRef as PdNodeRef, + Operation as Op, Options, +}; +use runtime::storage::{ + Column, Commit, KVStorage, MultiTreeStore, NewNode, NodeAddress, NodeRef, Operation, +}; +use score::{OpaqueHash, TrieKey}; use std::path::PathBuf; /// The parity database storage @@ -46,6 +51,52 @@ impl KVStorage for Parity { } } +impl MultiTreeStore for Parity { + fn insert_tree(&self, column: Column, key: OpaqueHash, root: NewNode) -> Result<()> { + self.0.commit_changes([( + column as u8, + Op::InsertTree(key.to_vec(), to_pd_newnode(root)), + )])?; + Ok(()) + } + + fn dereference_tree(&self, column: Column, key: OpaqueHash) -> Result<()> { + self.0 + .commit_changes([(column as u8, Op::DereferenceTree(key.to_vec()))])?; + Ok(()) + } + + fn get_root( + &self, + column: Column, + key: OpaqueHash, + ) -> Result, Vec)>> { + Ok(self.0.get_root(column as u8, key.as_ref())?) + } + + fn get_node( + &self, + column: Column, + address: NodeAddress, + ) -> Result, Vec)>> { + Ok(self.0.get_node(column as u8, address)?) + } +} + +fn to_pd_newnode(n: NewNode) -> PdNewNode { + PdNewNode { + data: n.data, + children: n.children.into_iter().map(to_pd_noderef).collect(), + } +} + +fn to_pd_noderef(r: NodeRef) -> PdNodeRef { + match r { + NodeRef::New(n) => PdNodeRef::New(to_pd_newnode(n)), + NodeRef::Existing(addr) => PdNodeRef::Existing(addr), + } +} + /// The iterator wrapper pub struct ParityIter<'a>(BTreeIterator<'a>); @@ -64,18 +115,28 @@ impl TryFrom for Parity { let options = Options { path, columns: vec![ + // Column::State ColumnOptions { btree_index: true, ..Default::default() }, + // Column::Sync ColumnOptions { btree_index: true, ..Default::default() }, + // Column::Archive ColumnOptions { btree_index: true, ..Default::default() }, + // Column::TrieNodes (multitree, content-addressed trie nodes) + ColumnOptions { + multitree: true, + preimage: true, + allow_direct_node_access: true, + ..Default::default() + }, ], sync_wal: true, sync_data: true, From 2454fac3980bf7c639d0f9a1c1f7db4981010dca Mon Sep 17 00:00:00 2001 From: clearloop Date: Thu, 21 May 2026 09:30:32 +0800 Subject: [PATCH 05/18] chore(crypto): bump ark-vrf to 0.2.1 for ring batch verify api --- Cargo.lock | 17 ++++++++--------- Cargo.toml | 3 +-- crates/crypto/Cargo.toml | 2 -- crates/crypto/tests/vrf.rs | 2 +- 4 files changed, 10 insertions(+), 14 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e80a5dab..553a54f7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -400,9 +400,9 @@ dependencies = [ [[package]] name = "ark-vrf" -version = "0.1.0" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9501da18569b2afe0eb934fb7afd5a247d238b94116155af4dd068f319adfe6d" +checksum = "b722df2194fd06cd069bb3cb29cd4bfce8e023b6cace34d314ee2cf4bd4df9b5" dependencies = [ "ark-bls12-381 0.5.0", "ark-ec 0.5.0", @@ -3548,7 +3548,6 @@ dependencies = [ "tiny-keccak", "tracing", "w3f-bls", - "w3f-ring-proof", ] [[package]] @@ -4328,9 +4327,9 @@ dependencies = [ [[package]] name = "w3f-pcs" -version = "0.0.2" +version = "0.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fbe7a8d5c914b69392ab3b267f679a2e546fe29afaddce47981772ac71bd02e1" +checksum = "3ea1046a1deb6d26c34ba2d1f1bab4222d695d126502ee765f80b021753cb674" dependencies = [ "ark-ec 0.5.0", "ark-ff 0.5.0", @@ -4343,9 +4342,9 @@ dependencies = [ [[package]] name = "w3f-plonk-common" -version = "0.0.2" +version = "0.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1aca389e494fe08c5c108b512e2328309036ee1c0bc7bdfdb743fef54d448c8c" +checksum = "077db25196f87773d7f0784c0ea5b11f18f38d336fa25c24bc67d7936af05d7a" dependencies = [ "ark-ec 0.5.0", "ark-ff 0.5.0", @@ -4360,9 +4359,9 @@ dependencies = [ [[package]] name = "w3f-ring-proof" -version = "0.0.2" +version = "0.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a639379402ad51504575dbd258740383291ac8147d3b15859bdf1ea48c677de" +checksum = "a3afac5d485a6eed7c1762be4b7b70a6f9b346bd6eebe485f48b7d909a6773f5" dependencies = [ "ark-ec 0.5.0", "ark-ff 0.5.0", diff --git a/Cargo.toml b/Cargo.toml index 382d5047..bf38760c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -74,7 +74,7 @@ ark-ed-on-bls12-381-bandersnatch = { version = "0.5", default-features = false } ark-ff = { version = "0.5", default-features = false } ark-std = { version = "0.5", default-features = false } ark-serialize = { version = "0.5", default-features = false } -ark-vrf = { version = "0.1.0", features = ["bandersnatch", "ring", "parallel"] } +ark-vrf = { version = "=0.2.1", features = ["bandersnatch", "ring", "parallel"] } async-trait = "0.1.89" bindgen = "0.72" blake2b_simd = "1.0.4" @@ -112,7 +112,6 @@ rand = "0.8.5" rayon = "1.11.0" rcgen = { version = "0.13.2", default-features = false, features = ["crypto", "pem", "ring"] } reed-solomon = { package = "reed-solomon-simd", version = "3.0.1" } -ring = { package = "w3f-ring-proof", version = "0.0.2", default-features = false } rustls = { version = "0.23.31", default-features = false, features = ["std", "logging", "tls12", "ring"] } rustls-webpki = { version = "0.102.8", default-features = false, features = ["std", "ring"] } serde = { version = "1.0.219", features = ["derive"], default-features = false } diff --git a/crates/crypto/Cargo.toml b/crates/crypto/Cargo.toml index 15d2322a..9b3766e4 100644 --- a/crates/crypto/Cargo.toml +++ b/crates/crypto/Cargo.toml @@ -16,7 +16,6 @@ once_cell = { workspace = true, optional = true } rand = { workspace = true, optional = true } rayon = { workspace = true, optional = true } rcgen = { workspace = true, optional = true } -ring = { workspace = true, optional = true } tiny-keccak = { workspace = true, optional = true } w3f-bls = { workspace = true, optional = true } tracing.workspace = true @@ -40,7 +39,6 @@ vrf = [ "ark-ed-on-bls12-381-bandersnatch", "ark-serialize", "once_cell", - "ring", ] [dev-dependencies] diff --git a/crates/crypto/tests/vrf.rs b/crates/crypto/tests/vrf.rs index 0a36ea8f..43e062bb 100644 --- a/crates/crypto/tests/vrf.rs +++ b/crates/crypto/tests/vrf.rs @@ -24,7 +24,7 @@ fn test_vrf() -> anyhow::Result<()> { let prover_key_index = 3; // NOTE: any key can be replaced with the padding point - let padding_point = Public::from(RingProofParams::padding_point()); + let padding_point = Public(RingProofParams::padding_point()); ring[2] = padding_point; ring[5] = padding_point; From 4562635125ebb8e735914480bfda62a92df8b0c1 Mon Sep 17 00:00:00 2001 From: clearloop Date: Thu, 21 May 2026 09:46:36 +0800 Subject: [PATCH 06/18] perf(runtime): batch-verify ring vrf for ticket extrinsic --- Cargo.lock | 38 ++++++------- Cargo.toml | 2 +- crates/crypto/Cargo.toml | 5 ++ crates/crypto/benches/ring_vrf.rs | 62 ++++++++++++++++++++ crates/crypto/src/vrf.rs | 30 ++++++++++ crates/crypto/tests/vrf.rs | 37 ++++++++++++ crates/runtime/src/tx/assurance/mod.rs | 4 +- crates/runtime/src/tx/block/mod.rs | 3 +- crates/runtime/src/tx/mod.rs | 7 ++- crates/runtime/src/tx/ticket/lazy.rs | 37 +++--------- crates/runtime/src/tx/ticket/verify.rs | 78 +++++++++++--------------- crates/spacejam/src/fuzz/target.rs | 6 +- 12 files changed, 207 insertions(+), 102 deletions(-) create mode 100644 crates/crypto/benches/ring_vrf.rs diff --git a/Cargo.lock b/Cargo.lock index 553a54f7..6400fb12 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1016,7 +1016,7 @@ dependencies = [ [[package]] name = "cranelift-pvm" -version = "0.1.2-pre.2" +version = "0.1.2-pre.4" dependencies = [ "anyhow", "cranelift", @@ -2759,7 +2759,7 @@ dependencies = [ [[package]] name = "pvmc" -version = "0.1.2-pre.2" +version = "0.1.2-pre.4" dependencies = [ "anyhow", "bindgen", @@ -3294,7 +3294,7 @@ dependencies = [ [[package]] name = "serde-jam" -version = "0.1.2-pre.2" +version = "0.1.2-pre.4" dependencies = [ "anyhow", "serde", @@ -3462,7 +3462,7 @@ dependencies = [ [[package]] name = "spacejam" -version = "0.1.2-pre.2" +version = "0.1.2-pre.4" dependencies = [ "anyhow", "async-trait", @@ -3499,7 +3499,7 @@ dependencies = [ [[package]] name = "spacejam-account" -version = "0.1.2-pre.2" +version = "0.1.2-pre.4" dependencies = [ "anyhow", "serde-jam", @@ -3510,7 +3510,7 @@ dependencies = [ [[package]] name = "spacejam-core" -version = "0.1.2-pre.2" +version = "0.1.2-pre.4" dependencies = [ "anyhow", "erased-serde", @@ -3564,7 +3564,7 @@ dependencies = [ [[package]] name = "spacejam-network" -version = "0.1.2-pre.2" +version = "0.1.2-pre.4" dependencies = [ "anyhow", "clap", @@ -3587,7 +3587,7 @@ dependencies = [ [[package]] name = "spacejam-offchain" -version = "0.1.2-pre.2" +version = "0.1.2-pre.4" dependencies = [ "anyhow", "async-trait", @@ -3603,7 +3603,7 @@ dependencies = [ [[package]] name = "spacejam-rpc" -version = "0.1.2-pre.2" +version = "0.1.2-pre.4" dependencies = [ "anyhow", "jsonrpsee", @@ -3615,7 +3615,7 @@ dependencies = [ [[package]] name = "spacejam-runtime" -version = "0.1.2-pre.2" +version = "0.1.2-pre.4" dependencies = [ "anyhow", "hex", @@ -3636,7 +3636,7 @@ dependencies = [ [[package]] name = "spacejam-service" -version = "0.1.2-pre.2" +version = "0.1.2-pre.4" dependencies = [ "anyhow", "blake2b_simd", @@ -3650,14 +3650,14 @@ dependencies = [ [[package]] name = "spacejam-spec" -version = "0.1.2-pre.2" +version = "0.1.2-pre.4" dependencies = [ "serde", ] [[package]] name = "spacejam-testing" -version = "0.1.2-pre.2" +version = "0.1.2-pre.4" dependencies = [ "anyhow", "hex", @@ -3687,7 +3687,7 @@ dependencies = [ [[package]] name = "spacejam-testnet" -version = "0.1.2-pre.2" +version = "0.1.2-pre.4" dependencies = [ "anyhow", "clap", @@ -3698,7 +3698,7 @@ dependencies = [ [[package]] name = "spacejam-worker" -version = "0.1.2-pre.2" +version = "0.1.2-pre.4" dependencies = [ "anyhow", "pvm", @@ -3714,7 +3714,7 @@ dependencies = [ [[package]] name = "spacejson" -version = "0.1.2-pre.2" +version = "0.1.2-pre.4" dependencies = [ "anyhow", "hex", @@ -3725,7 +3725,7 @@ dependencies = [ [[package]] name = "spacejson-derive" -version = "0.1.2-pre.2" +version = "0.1.2-pre.4" dependencies = [ "proc-macro2", "quote", @@ -3734,7 +3734,7 @@ dependencies = [ [[package]] name = "spacevm" -version = "0.1.2-pre.2" +version = "0.1.2-pre.4" dependencies = [ "anyhow", "hex", @@ -3748,7 +3748,7 @@ dependencies = [ [[package]] name = "spacevm-export" -version = "0.1.2-pre.2" +version = "0.1.2-pre.4" dependencies = [ "pvm", "pvmc", diff --git a/Cargo.toml b/Cargo.toml index bf38760c..5386c210 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -21,7 +21,7 @@ resolver = "1" [workspace.package] edition = "2024" -version = "0.1.2-pre.2" +version = "0.1.2-pre.4" homepage = "https://spacejam.app" repository = "https://github.com/spacejamapp/jade" license = "GPL-3.0" diff --git a/crates/crypto/Cargo.toml b/crates/crypto/Cargo.toml index 9b3766e4..8571e9ab 100644 --- a/crates/crypto/Cargo.toml +++ b/crates/crypto/Cargo.toml @@ -58,6 +58,11 @@ name = "blake2b" harness = false required-features = ["blake2"] +[[bench]] +name = "ring_vrf" +harness = false +required-features = ["vrf"] + [build-dependencies] ark-serialize.workspace = true ark-vrf.workspace = true diff --git a/crates/crypto/benches/ring_vrf.rs b/crates/crypto/benches/ring_vrf.rs new file mode 100644 index 00000000..373e0e64 --- /dev/null +++ b/crates/crypto/benches/ring_vrf.rs @@ -0,0 +1,62 @@ +//! Microbench: single ring-vrf verify vs batched verify, swept across batch sizes. + +use criterion::{BenchmarkId, Criterion, Throughput, black_box, criterion_group, criterion_main}; +use spacejam_crypto::{ + ring::RING_SIZE, + vrf::{KeyPair, Verifier}, +}; + +/// Generate `n` deterministic (message, signature) triples signed under distinct ring positions. +fn gen_items(n: usize) -> (Verifier, Vec<(Vec, [u8; 784])>) { + assert!(n <= RING_SIZE, "batch size {n} exceeds RING_SIZE {RING_SIZE}"); + let ring: Vec = (0..RING_SIZE).map(|i| KeyPair::from([i as u8; 32])).collect(); + let pks = ring + .iter() + .map(|k| k.public().unwrap()) + .collect::>(); + let pkeys = ring.iter().map(|k| k.public).collect::>(); + let verifier = Verifier::new(pkeys); + + let items: Vec<(Vec, [u8; 784])> = (0..n) + .map(|i| { + let msg = format!("benchmark ring-vrf message {i:08}").into_bytes(); + let sig = ring[i].ring_sign(pks.clone(), &msg, &[]).unwrap(); + (msg, sig) + }) + .collect(); + + (verifier, items) +} + +fn bench(c: &mut Criterion) { + let mut group = c.benchmark_group("ring_vrf"); + group.sample_size(20); + + for &n in [1usize, 3, 8, 16, 32, 64].iter().filter(|&&n| n <= RING_SIZE) { + let (verifier, items) = gen_items(n); + group.throughput(Throughput::Elements(n as u64)); + + group.bench_with_input(BenchmarkId::new("single", n), &items, |b, items| { + b.iter(|| { + for (m, s) in items { + verifier.ring_vrf_verify(black_box(m), &[], s).unwrap(); + } + }) + }); + + group.bench_with_input(BenchmarkId::new("batch", n), &items, |b, items| { + b.iter(|| { + verifier + .ring_vrf_verify_batch( + items.iter().map(|(m, s)| (m.as_slice(), [].as_slice(), s.as_slice())), + ) + .unwrap() + }) + }); + } + + group.finish(); +} + +criterion_group!(benches, bench); +criterion_main!(benches); diff --git a/crates/crypto/src/vrf.rs b/crates/crypto/src/vrf.rs index 96fefdf7..1b16116c 100644 --- a/crates/crypto/src/vrf.rs +++ b/crates/crypto/src/vrf.rs @@ -298,6 +298,36 @@ impl Verifier { Ok(vrf_output_hash) } + /// Batched anonymous VRF signature verification. + /// + /// Collapses N independent ring verifies into one randomized pairing check, + /// amortizing the dominant cost across the batch. On success, returns the + /// per-item VRF output hashes in iteration order. On failure, the entire + /// batch is rejected without per-item attribution. + pub fn ring_vrf_verify_batch<'a>( + &self, + items: impl IntoIterator, + ) -> anyhow::Result> { + let verifier_key = RING_CTX.verifier_key_from_commitment(self.commitment.clone()); + let ring_verifier = RING_CTX.verifier(verifier_key); + let mut batch = ark_vrf::ring::BatchVerifier::new(ring_verifier); + + let mut outputs = Vec::new(); + for (vrf_input_data, aux_data, signature) in items { + let signature = RingVrfSignature::deserialize_compressed_unchecked(signature)?; + let input = + Input::new(vrf_input_data).ok_or(anyhow::anyhow!("Invalid input"))?; + let output_hash: [u8; 32] = signature.output.hash()[..32].try_into()?; + batch.push(input, signature.output, aux_data, &signature.proof); + outputs.push(output_hash); + } + + batch + .verify() + .map_err(|e| anyhow::anyhow!("Ring batch verification failure: {:?}", e))?; + Ok(outputs) + } + /// Non-Anonymous VRF signature verification. pub fn ietf_vrf_verify( &self, diff --git a/crates/crypto/tests/vrf.rs b/crates/crypto/tests/vrf.rs index 43e062bb..0cf9cabd 100644 --- a/crates/crypto/tests/vrf.rs +++ b/crates/crypto/tests/vrf.rs @@ -123,6 +123,43 @@ fn serde_vrf_signature() -> anyhow::Result<()> { Ok(()) } +#[test] +fn ring_vrf_verify_batch_matches_single() -> anyhow::Result<()> { + let ring: Vec<_> = (0..RING_SIZE) + .map(|i| KeyPair::from([i as u8; 32])) + .collect(); + let keys = ring + .iter() + .map(|k| k.public()) + .collect::>>()?; + let pkeys = ring.iter().map(|k| k.public).collect::>(); + let verifier = Verifier::new(pkeys); + + let cases: Vec<(usize, &[u8])> = vec![(0, b"alpha"), (1, b"beta"), (3, b"gamma")]; + let ad: &[u8] = b""; + + let sigs: Vec<[u8; 784]> = cases + .iter() + .map(|(signer, msg)| ring[*signer].ring_sign(keys.clone(), msg, ad)) + .collect::>()?; + + let expected: Vec<[u8; 32]> = cases + .iter() + .zip(sigs.iter()) + .map(|((_, msg), sig)| verifier.ring_vrf_verify(msg, ad, sig)) + .collect::>()?; + + let got = verifier.ring_vrf_verify_batch( + cases + .iter() + .zip(sigs.iter()) + .map(|((_, msg), sig)| (*msg, ad, sig.as_slice())), + )?; + + assert_eq!(got, expected); + Ok(()) +} + #[test] fn vrf_output_consistency() -> anyhow::Result<()> { let ring: Vec<_> = (0..RING_SIZE) diff --git a/crates/runtime/src/tx/assurance/mod.rs b/crates/runtime/src/tx/assurance/mod.rs index bed63412..459ba461 100644 --- a/crates/runtime/src/tx/assurance/mod.rs +++ b/crates/runtime/src/tx/assurance/mod.rs @@ -98,9 +98,7 @@ pub fn available( ) }) .collect(); - crypto::ed25519::batch_verify(&verify_items) - .inspect_err(|_| tracing::error!("bad signature in assurances")) - .map_err(|_| Error::BadSignature)?; + crypto::ed25519::batch_verify(&verify_items).map_err(|_| Error::BadSignature)?; // Check which cores reached 2/3 majority let mut available = Vec::new(); diff --git a/crates/runtime/src/tx/block/mod.rs b/crates/runtime/src/tx/block/mod.rs index 2e477df3..f50b7879 100644 --- a/crates/runtime/src/tx/block/mod.rs +++ b/crates/runtime/src/tx/block/mod.rs @@ -31,8 +31,7 @@ pub fn process(block: Block, storage: Arc) -> Result<()> ); match (vresult, sresult) { - (Err(e), _) => anyhow::bail!("failed to import block: {e:?}"), - (_, Err(e)) => anyhow::bail!("failed to import block: {e:?}"), + (Err(e), _) | (_, Err(e)) => Err(e), (Ok(()), Ok(diff)) => { storage.commit(Column::State, diff)?; Ok(()) diff --git a/crates/runtime/src/tx/mod.rs b/crates/runtime/src/tx/mod.rs index 0555615b..26c24c70 100644 --- a/crates/runtime/src/tx/mod.rs +++ b/crates/runtime/src/tx/mod.rs @@ -202,10 +202,11 @@ pub fn simulate_with_state( state.queue = accumulation.ready_queue; state.history = accumulation.accumulated_queue; state.validators.drawn = accumulation.validators; + let candidate = state + .safrole + .next(&state.validators.drawn, &state.disputes.offenders); + thread::spawn(move || ticket::lazy::drawn(&candidate)); - // lazy load vrf rings - let drawn = state.validators.drawn.clone(); - thread::spawn(move || ticket::lazy::drawn(&drawn)); state.statistics.merge_services(accumulation.records); state.logs = accumulation.logs; (accumulation.root, accumulation.accounts) diff --git a/crates/runtime/src/tx/ticket/lazy.rs b/crates/runtime/src/tx/ticket/lazy.rs index ab27b5bd..4a4aa9d0 100644 --- a/crates/runtime/src/tx/ticket/lazy.rs +++ b/crates/runtime/src/tx/ticket/lazy.rs @@ -10,19 +10,8 @@ static LAZY_RING: LazyLock, Arc LazyLock::new(|| Mutex::new(LruCache::new(NonZeroUsize::new(8).unwrap()))); /// Clear all cached data -pub async fn clear() { - if let Ok(mut map) = LAZY_RING.lock() { - map.clear(); - } -} - -/// Check if the lazy cache is empty -pub fn is_empty() -> bool { - if let Ok(cache) = LAZY_RING.lock() { - cache.len() == 0 - } else { - false - } +pub fn clear() { + lock().clear(); } /// Accept drawn validators after accumulation @@ -40,26 +29,18 @@ pub fn commitment(drawn: &Vec) -> BandersnatchRingCommitment /// Get the verifier of the next validators pub fn verifier(drawn: &Vec) -> Arc { - let Ok(mut map) = LAZY_RING.lock() else { - panic!("failed to lock ring, fix me later"); - }; - - if let Some(v) = map.get(drawn) { - return v.clone(); + if let Some(v) = lock().get(drawn).cloned() { + return v; } - - drop(map); - // build new verifier, expensive computation here. let verifier = Arc::new(crypto::ring::verifier(drawn)); - let Ok(mut map) = LAZY_RING.lock() else { - panic!("failed to lock ring, fix me later"); - }; - - // double-check after re-acquiring lock + let mut map = lock(); if let Some(v) = map.get(drawn) { return v.clone(); } - map.put(drawn.clone(), verifier.clone()); verifier } + +fn lock() -> std::sync::MutexGuard<'static, LruCache, Arc>> { + LAZY_RING.lock().unwrap_or_else(|p| p.into_inner()) +} diff --git a/crates/runtime/src/tx/ticket/verify.rs b/crates/runtime/src/tx/ticket/verify.rs index 7d66d536..af767435 100644 --- a/crates/runtime/src/tx/ticket/verify.rs +++ b/crates/runtime/src/tx/ticket/verify.rs @@ -1,12 +1,10 @@ //! Verification utilities for tickets use crate::tx::ticket::{Error, lazy}; -use rayon::iter::{IndexedParallelIterator, IntoParallelRefIterator, ParallelIterator}; use score::{ BandersnatchPublic, OpaqueHash, - extrinsic::{TicketBody, TicketEnvelope, TicketsAccumulator, TicketsExtrinsic}, + extrinsic::{TicketBody, TicketsAccumulator, TicketsExtrinsic}, }; -use std::{collections::BTreeMap, sync::Arc}; /// Verify tickets pub fn tickets( @@ -14,54 +12,46 @@ pub fn tickets( next: &Vec, tickets: &TicketsExtrinsic, ) -> Result { - let verifier = lazy::verifier(next); - let verified = tickets - .par_iter() - .enumerate() - .map(|(index, envelope)| self::ticket(index, envelope.clone(), entropy, verifier.clone())) - .collect::, Error>>()?; - - // Check for bad order: 6.32 & 6.33 - let new_tickets = verified.into_values().collect::>(); - let mut sorted = new_tickets.clone(); - sorted.sort_by_key(|a| a.id); - if sorted != new_tickets { - return Err(Error::BadTicketOrder); + // 1. Verify ticket attempts upfront (6.29) + for envelope in tickets.iter() { + if envelope.attempt >= score::TICKET_ENTRIES_PER_VALIDATOR as u8 { + return Err(Error::BadTicketAttempt); + } } - Ok(sorted) -} - -/// Verify a single ticket -fn ticket( - index: usize, - envelope: TicketEnvelope, - entropy: [OpaqueHash; 4], - verifier: Arc, -) -> Result<(usize, TicketBody), Error> { - // 1. Verify ticket attempt (6.29) - if envelope.attempt >= score::TICKET_ENTRIES_PER_VALIDATOR as u8 { - return Err(Error::BadTicketAttempt); - } - - // 2. Verify ring VRF signature and get ticket identifier - let id = verifier - .ring_vrf_verify( - &TicketBody::message(envelope.attempt, &entropy[2]), - &[], - &envelope.signature, + // 2. Batch-verify ring VRF signatures, harvesting per-ticket ids in order + let messages: Vec> = tickets + .iter() + .map(|e| TicketBody::message(e.attempt, &entropy[2])) + .collect(); + let verifier = lazy::verifier(next); + let ids = verifier + .ring_vrf_verify_batch( + messages + .iter() + .zip(tickets.iter()) + .map(|(msg, e)| (msg.as_slice(), [].as_slice(), e.signature.as_slice())), ) .map_err(|e| { - tracing::error!("failed to verify ring VRF signature: {:?}", e); + tracing::trace!("failed to batch-verify ring VRF signatures: {:?}", e); Error::BadTicketProof })?; - // 3. Store ticket for accumulation - Ok(( - index, - TicketBody { + let new_tickets: Vec = ids + .into_iter() + .zip(tickets.iter()) + .map(|(id, envelope)| TicketBody { id, attempt: envelope.attempt, - }, - )) + }) + .collect(); + + // 3. Check for bad order: 6.32 & 6.33 + let mut sorted = new_tickets.clone(); + sorted.sort_by_key(|a| a.id); + if sorted != new_tickets { + return Err(Error::BadTicketOrder); + } + + Ok(sorted) } diff --git a/crates/spacejam/src/fuzz/target.rs b/crates/spacejam/src/fuzz/target.rs index add89336..9f6e25b0 100644 --- a/crates/spacejam/src/fuzz/target.rs +++ b/crates/spacejam/src/fuzz/target.rs @@ -91,10 +91,12 @@ impl Target { match message { Message::Info(info) => self.info(info), Message::ImportBlock(block) => { + let slot = block.header.slot; if let Err(e) = self.import_block(block).await { - tracing::warn!("failed to import block: {e}"); + tracing::debug!("failed to import block#{slot}: {e}"); self.write_message(Message::Error(e.to_string()))?; } + tracing::info!("imported block#{slot}"); Ok(()) } Message::Initialize(state) => self.initialize(state).await, @@ -137,7 +139,7 @@ impl Target { /// Received set state request #[tracing::instrument(skip_all, name = "initialize")] pub async fn initialize(&mut self, state: Initialize) -> Result<()> { - lazy::clear().await; + lazy::clear(); self.chain = Default::default(); let root = self.chain.init(state.keyvals())?; if let Err(e) = self.init_state().await { From fb7e9c3f441a3f95d97545ee1c55438193d011d1 Mon Sep 17 00:00:00 2001 From: clearloop Date: Thu, 21 May 2026 15:16:35 +0800 Subject: [PATCH 07/18] fix(runtime): wrap authorization queue index --- Cargo.lock | 38 +++---- Cargo.toml | 2 +- crates/runtime/src/tx/guarantee/mod.rs | 8 +- crates/testing/build.rs | 145 ++++++++++++++----------- 4 files changed, 103 insertions(+), 90 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6400fb12..56e97df6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1016,7 +1016,7 @@ dependencies = [ [[package]] name = "cranelift-pvm" -version = "0.1.2-pre.4" +version = "0.1.2-pre.5" dependencies = [ "anyhow", "cranelift", @@ -2759,7 +2759,7 @@ dependencies = [ [[package]] name = "pvmc" -version = "0.1.2-pre.4" +version = "0.1.2-pre.5" dependencies = [ "anyhow", "bindgen", @@ -3294,7 +3294,7 @@ dependencies = [ [[package]] name = "serde-jam" -version = "0.1.2-pre.4" +version = "0.1.2-pre.5" dependencies = [ "anyhow", "serde", @@ -3462,7 +3462,7 @@ dependencies = [ [[package]] name = "spacejam" -version = "0.1.2-pre.4" +version = "0.1.2-pre.5" dependencies = [ "anyhow", "async-trait", @@ -3499,7 +3499,7 @@ dependencies = [ [[package]] name = "spacejam-account" -version = "0.1.2-pre.4" +version = "0.1.2-pre.5" dependencies = [ "anyhow", "serde-jam", @@ -3510,7 +3510,7 @@ dependencies = [ [[package]] name = "spacejam-core" -version = "0.1.2-pre.4" +version = "0.1.2-pre.5" dependencies = [ "anyhow", "erased-serde", @@ -3564,7 +3564,7 @@ dependencies = [ [[package]] name = "spacejam-network" -version = "0.1.2-pre.4" +version = "0.1.2-pre.5" dependencies = [ "anyhow", "clap", @@ -3587,7 +3587,7 @@ dependencies = [ [[package]] name = "spacejam-offchain" -version = "0.1.2-pre.4" +version = "0.1.2-pre.5" dependencies = [ "anyhow", "async-trait", @@ -3603,7 +3603,7 @@ dependencies = [ [[package]] name = "spacejam-rpc" -version = "0.1.2-pre.4" +version = "0.1.2-pre.5" dependencies = [ "anyhow", "jsonrpsee", @@ -3615,7 +3615,7 @@ dependencies = [ [[package]] name = "spacejam-runtime" -version = "0.1.2-pre.4" +version = "0.1.2-pre.5" dependencies = [ "anyhow", "hex", @@ -3636,7 +3636,7 @@ dependencies = [ [[package]] name = "spacejam-service" -version = "0.1.2-pre.4" +version = "0.1.2-pre.5" dependencies = [ "anyhow", "blake2b_simd", @@ -3650,14 +3650,14 @@ dependencies = [ [[package]] name = "spacejam-spec" -version = "0.1.2-pre.4" +version = "0.1.2-pre.5" dependencies = [ "serde", ] [[package]] name = "spacejam-testing" -version = "0.1.2-pre.4" +version = "0.1.2-pre.5" dependencies = [ "anyhow", "hex", @@ -3687,7 +3687,7 @@ dependencies = [ [[package]] name = "spacejam-testnet" -version = "0.1.2-pre.4" +version = "0.1.2-pre.5" dependencies = [ "anyhow", "clap", @@ -3698,7 +3698,7 @@ dependencies = [ [[package]] name = "spacejam-worker" -version = "0.1.2-pre.4" +version = "0.1.2-pre.5" dependencies = [ "anyhow", "pvm", @@ -3714,7 +3714,7 @@ dependencies = [ [[package]] name = "spacejson" -version = "0.1.2-pre.4" +version = "0.1.2-pre.5" dependencies = [ "anyhow", "hex", @@ -3725,7 +3725,7 @@ dependencies = [ [[package]] name = "spacejson-derive" -version = "0.1.2-pre.4" +version = "0.1.2-pre.5" dependencies = [ "proc-macro2", "quote", @@ -3734,7 +3734,7 @@ dependencies = [ [[package]] name = "spacevm" -version = "0.1.2-pre.4" +version = "0.1.2-pre.5" dependencies = [ "anyhow", "hex", @@ -3748,7 +3748,7 @@ dependencies = [ [[package]] name = "spacevm-export" -version = "0.1.2-pre.4" +version = "0.1.2-pre.5" dependencies = [ "pvm", "pvmc", diff --git a/Cargo.toml b/Cargo.toml index 5386c210..057a1b57 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -21,7 +21,7 @@ resolver = "1" [workspace.package] edition = "2024" -version = "0.1.2-pre.4" +version = "0.1.2-pre.5" homepage = "https://spacejam.app" repository = "https://github.com/spacejamapp/jade" license = "GPL-3.0" diff --git a/crates/runtime/src/tx/guarantee/mod.rs b/crates/runtime/src/tx/guarantee/mod.rs index 8be66412..2d27b45c 100644 --- a/crates/runtime/src/tx/guarantee/mod.rs +++ b/crates/runtime/src/tx/guarantee/mod.rs @@ -202,7 +202,7 @@ pub fn pools( >, guarantees: &GuaranteesExtrinsic, ) -> score::AuthorizationPools { - let slot = timeslot % score::EPOCH_LENGTH; + let slot = (timeslot % score::EPOCH_LENGTH) as usize % score::AUTH_QUEUE_SIZE; let mut new_pools = score::AuthorizationPools::default(); for (core_index, pool) in pools.iter().enumerate() { let mut new_pool = pool.clone(); @@ -218,10 +218,8 @@ pub fn pools( } } - // add new authorizer from queue at position H_t (current timeslot) - if let Some(auth) = authorizations[core_index].get(slot as usize) { - new_pool.push(*auth); - } + // add new authorizer from queue at position H_t mod Q + new_pool.push(authorizations[core_index][slot]); // truncate the pool to the max size if let Some(old) = new_pool.len().checked_sub(score::AUTH_POOL_MAX_SIZE) { diff --git a/crates/testing/build.rs b/crates/testing/build.rs index a00e701c..84a7659b 100644 --- a/crates/testing/build.rs +++ b/crates/testing/build.rs @@ -32,7 +32,7 @@ fn main() -> Result<()> { let scale = scale(); // set up the registry - try_download(&workspace)?; + util::try_download(&workspace)?; let registry = Registry::with_scale(workspace.join("res/jam-test-vectors"), scale); // build all tests @@ -54,7 +54,6 @@ fn main() -> Result<()> { build_tests(registry.shuffle()?, &out_dir.join("shuffle.rs"))?; build_tests(registry.statistics(scale)?, &out_dir.join("statistics.rs"))?; build_tests(registry.trie()?, &out_dir.join("trie.rs"))?; - // Trace and sequential tests are tiny-only if scale == Scale::Tiny { build_tests( registry.trace(Trace::Fallback)?, @@ -88,7 +87,6 @@ fn main() -> Result<()> { registry.trace(Trace::StorageLight)?, &out_dir.join("traces_storage_light.rs"), )?; - build_all_seq_test(&out_dir.join("traces_seq.rs"))?; } else { for name in [ "traces_fallback", @@ -99,31 +97,19 @@ fn main() -> Result<()> { "traces_preimages_light", "traces_storage", "traces_storage_light", - "traces_seq", ] { fs::write(out_dir.join(format!("{name}.rs")), "")?; } } + build_all_seq_test(&out_dir.join("traces_seq.rs"), scale)?; Ok(()) } -fn scale_constructor() -> proc_macro2::TokenStream { - if env::var("CARGO_FEATURE_FULL").is_ok() { - quote::quote!(specjam::Registry::with_scale( - "../../res/jam-test-vectors", - specjam::Scale::Full - )) - } else { - quote::quote!(specjam::Registry::new("../../res/jam-test-vectors")) - } -} - fn build_tests(entry: Entry, out: &Path) -> Result<()> { let mut tests: Vec = Vec::new(); let section = entry.section; let ss = section.as_ref(); - let registry = scale_constructor(); - + let registry = util::scale_constructor(); for (i, path) in entry.files.iter().enumerate() { let name = Entry::file_name(path)?; let test_name = Ident::new(&format!("test_{name}"), Span::call_site()); @@ -144,8 +130,7 @@ fn build_pvmc_tests(entry: Entry, out: &Path) -> Result<()> { let mut tests: Vec = Vec::new(); let section = entry.section; let ss = section.as_ref(); - let registry = scale_constructor(); - + let registry = util::scale_constructor(); for (i, path) in entry.files.iter().enumerate() { let name = Entry::file_name(path)?; let test_name = Ident::new(&format!("test_{name}"), Span::call_site()); @@ -163,25 +148,24 @@ fn build_pvmc_tests(entry: Entry, out: &Path) -> Result<()> { } /// Builds all sequential tests -fn build_all_seq_test(out: &Path) -> Result<()> { +fn build_all_seq_test(out: &Path, scale: Scale) -> Result<()> { let mut items = Vec::new(); - for entry in [TRACES, REPORTS] { - for entry in fs::read_dir(entry)? { - let entry = entry?; - let path = entry.path(); - // let dname = path.file_name().unwrap().to_string_lossy(); - /* if dname.contains("_4872") { - continue; - } */ - - if path.is_dir() { - let testset = path.to_str().expect("failed to get testset"); - items.push(build_seq_test(testset)?); + if scale == Scale::Tiny { + for entry in [TRACES, REPORTS] { + for entry in fs::read_dir(entry)? { + let entry = entry?; + let path = entry.path(); + if path.is_dir() { + let testset = path.to_str().expect("failed to get testset"); + items.push(build_seq_test(testset)?); + } } } } - if Path::new(REPORT).is_dir() { + // REPORT dir can hold either tiny- or full-spec traces; the spec is + // declared in its report.json metadata. Only include if it matches. + if Path::new(REPORT).is_dir() && util::report_scale()? == Some(scale) { items.push(build_seq_test(REPORT)?); } @@ -194,9 +178,6 @@ fn build_seq_test(entry: &str) -> Result { let fentry = Entry::seq(entry)?; let test_name = Path::new(entry).file_name().unwrap().to_str().unwrap(); let mut tests = BTreeSet::::new(); - - // Iterate over file paths directly instead of parsing JSON files to avoid - // reading thousands of files during build time. Extract test names from filenames. for path in &fentry.files { let name = Entry::file_name(path)?; let names = name.split('_').collect::>(); @@ -225,36 +206,70 @@ fn build_seq_test(entry: &str) -> Result { Ok(testfn) } -fn try_download(workspace: &Path) -> Result<()> { - fs::create_dir_all(workspace.join("res"))?; - clone_if_missing( - workspace, - "https://github.com/spacejamapp/jam-test-vectors", - "res/jam-test-vectors", - )?; - clone_if_missing( - workspace, - "https://github.com/spacejamapp/jam-conformance", - "res/jam-conformance", - )?; - Ok(()) -} +mod util { + use super::*; -fn clone_if_missing(workspace: &Path, url: &str, dest: &str) -> Result<()> { - if workspace.join(dest).exists() { - return Ok(()); + pub fn try_download(workspace: &Path) -> Result<()> { + fs::create_dir_all(workspace.join("res"))?; + clone_if_missing( + workspace, + "https://github.com/spacejamapp/jam-test-vectors", + "res/jam-test-vectors", + )?; + clone_if_missing( + workspace, + "https://github.com/spacejamapp/jam-conformance", + "res/jam-conformance", + )?; + Ok(()) } - let output = Command::new("git") - .args(["clone", url, dest, "--depth", "1"]) - .current_dir(workspace) - .output() - .map_err(|e| anyhow::anyhow!("failed to spawn `git clone {url}`: {e}"))?; - if !output.status.success() { - anyhow::bail!( - "git clone {url} into {dest} failed ({}): {}", - output.status, - String::from_utf8_lossy(&output.stderr).trim() - ); + + pub fn clone_if_missing(workspace: &Path, url: &str, dest: &str) -> Result<()> { + if workspace.join(dest).exists() { + return Ok(()); + } + let output = Command::new("git") + .args(["clone", url, dest, "--depth", "1"]) + .current_dir(workspace) + .output() + .map_err(|e| anyhow::anyhow!("failed to spawn `git clone {url}`: {e}"))?; + if !output.status.success() { + anyhow::bail!( + "git clone {url} into {dest} failed ({}): {}", + output.status, + String::from_utf8_lossy(&output.stderr).trim() + ); + } + Ok(()) + } + + /// Read `res/report/report.json` and return the spec scale it declares, + /// or None if the file is missing or unrecognized. + pub fn report_scale() -> Result> { + let path = Path::new(REPORT).join("report.json"); + if !path.exists() { + return Ok(None); + } + let content = fs::read_to_string(&path)?; + // Strip whitespace so we tolerate any pretty-printing of the JSON. + let stripped: String = content.chars().filter(|c| !c.is_whitespace()).collect(); + if stripped.contains(r#""jam_spec":{"full""#) { + Ok(Some(Scale::Full)) + } else if stripped.contains(r#""jam_spec":{"tiny""#) { + Ok(Some(Scale::Tiny)) + } else { + Ok(None) + } + } + + pub fn scale_constructor() -> proc_macro2::TokenStream { + if env::var("CARGO_FEATURE_FULL").is_ok() { + quote::quote!(specjam::Registry::with_scale( + "../../res/jam-test-vectors", + specjam::Scale::Full + )) + } else { + quote::quote!(specjam::Registry::new("../../res/jam-test-vectors")) + } } - Ok(()) } From 05e31e03d6b4a833dbc4fcc8b8ce7bf562b8b734 Mon Sep 17 00:00:00 2001 From: clearloop Date: Thu, 21 May 2026 22:12:38 +0800 Subject: [PATCH 08/18] fix(runtime): split preimage validate/integrate --- Cargo.lock | 38 ++++++++-------- Cargo.toml | 2 +- crates/core/account/src/registry.rs | 8 ++++ crates/runtime/src/tx/mod.rs | 9 ++-- crates/runtime/src/tx/preimage.rs | 68 ++++++++++------------------- crates/testing/src/preimage.rs | 24 +++++----- 6 files changed, 69 insertions(+), 80 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 56e97df6..f8ffd34f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1016,7 +1016,7 @@ dependencies = [ [[package]] name = "cranelift-pvm" -version = "0.1.2-pre.5" +version = "0.1.2-pre.7" dependencies = [ "anyhow", "cranelift", @@ -2759,7 +2759,7 @@ dependencies = [ [[package]] name = "pvmc" -version = "0.1.2-pre.5" +version = "0.1.2-pre.7" dependencies = [ "anyhow", "bindgen", @@ -3294,7 +3294,7 @@ dependencies = [ [[package]] name = "serde-jam" -version = "0.1.2-pre.5" +version = "0.1.2-pre.7" dependencies = [ "anyhow", "serde", @@ -3462,7 +3462,7 @@ dependencies = [ [[package]] name = "spacejam" -version = "0.1.2-pre.5" +version = "0.1.2-pre.7" dependencies = [ "anyhow", "async-trait", @@ -3499,7 +3499,7 @@ dependencies = [ [[package]] name = "spacejam-account" -version = "0.1.2-pre.5" +version = "0.1.2-pre.7" dependencies = [ "anyhow", "serde-jam", @@ -3510,7 +3510,7 @@ dependencies = [ [[package]] name = "spacejam-core" -version = "0.1.2-pre.5" +version = "0.1.2-pre.7" dependencies = [ "anyhow", "erased-serde", @@ -3564,7 +3564,7 @@ dependencies = [ [[package]] name = "spacejam-network" -version = "0.1.2-pre.5" +version = "0.1.2-pre.7" dependencies = [ "anyhow", "clap", @@ -3587,7 +3587,7 @@ dependencies = [ [[package]] name = "spacejam-offchain" -version = "0.1.2-pre.5" +version = "0.1.2-pre.7" dependencies = [ "anyhow", "async-trait", @@ -3603,7 +3603,7 @@ dependencies = [ [[package]] name = "spacejam-rpc" -version = "0.1.2-pre.5" +version = "0.1.2-pre.7" dependencies = [ "anyhow", "jsonrpsee", @@ -3615,7 +3615,7 @@ dependencies = [ [[package]] name = "spacejam-runtime" -version = "0.1.2-pre.5" +version = "0.1.2-pre.7" dependencies = [ "anyhow", "hex", @@ -3636,7 +3636,7 @@ dependencies = [ [[package]] name = "spacejam-service" -version = "0.1.2-pre.5" +version = "0.1.2-pre.7" dependencies = [ "anyhow", "blake2b_simd", @@ -3650,14 +3650,14 @@ dependencies = [ [[package]] name = "spacejam-spec" -version = "0.1.2-pre.5" +version = "0.1.2-pre.7" dependencies = [ "serde", ] [[package]] name = "spacejam-testing" -version = "0.1.2-pre.5" +version = "0.1.2-pre.7" dependencies = [ "anyhow", "hex", @@ -3687,7 +3687,7 @@ dependencies = [ [[package]] name = "spacejam-testnet" -version = "0.1.2-pre.5" +version = "0.1.2-pre.7" dependencies = [ "anyhow", "clap", @@ -3698,7 +3698,7 @@ dependencies = [ [[package]] name = "spacejam-worker" -version = "0.1.2-pre.5" +version = "0.1.2-pre.7" dependencies = [ "anyhow", "pvm", @@ -3714,7 +3714,7 @@ dependencies = [ [[package]] name = "spacejson" -version = "0.1.2-pre.5" +version = "0.1.2-pre.7" dependencies = [ "anyhow", "hex", @@ -3725,7 +3725,7 @@ dependencies = [ [[package]] name = "spacejson-derive" -version = "0.1.2-pre.5" +version = "0.1.2-pre.7" dependencies = [ "proc-macro2", "quote", @@ -3734,7 +3734,7 @@ dependencies = [ [[package]] name = "spacevm" -version = "0.1.2-pre.5" +version = "0.1.2-pre.7" dependencies = [ "anyhow", "hex", @@ -3748,7 +3748,7 @@ dependencies = [ [[package]] name = "spacevm-export" -version = "0.1.2-pre.5" +version = "0.1.2-pre.7" dependencies = [ "pvm", "pvmc", diff --git a/Cargo.toml b/Cargo.toml index 057a1b57..9a83077b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -21,7 +21,7 @@ resolver = "1" [workspace.package] edition = "2024" -version = "0.1.2-pre.5" +version = "0.1.2-pre.7" homepage = "https://spacejam.app" repository = "https://github.com/spacejamapp/jade" license = "GPL-3.0" diff --git a/crates/core/account/src/registry.rs b/crates/core/account/src/registry.rs index 47809fd7..db7f5817 100644 --- a/crates/core/account/src/registry.rs +++ b/crates/core/account/src/registry.rs @@ -23,6 +23,14 @@ pub trait Accounts: Clone + Send + Sync + 'static { self.get(index).is_some() } + /// Check if a lookup is providable + fn is_providable(&mut self, index: ServiceId, hash: OpaqueHash, len: u32) -> bool { + let Some(account) = self.get(index) else { + return false; + }; + matches!(account.lookup(hash, len), Some(Some(slots)) if slots.is_empty()) + } + /// Remove an account from the registry fn remove(&mut self, index: u32); diff --git a/crates/runtime/src/tx/mod.rs b/crates/runtime/src/tx/mod.rs index 26c24c70..77a3d1f7 100644 --- a/crates/runtime/src/tx/mod.rs +++ b/crates/runtime/src/tx/mod.rs @@ -68,7 +68,10 @@ pub fn simulate_with_state( } // The first round computation - let accounts = Accounts::new(storage); + let mut accounts = Accounts::new(storage); + + // (E_P) Validate preimages against the prior state (12.6) + preimage::validate(&mut accounts, &block.extrinsic.preimages)?; let (mut reports, reported, reporters) = { // (η') Update entropy (6.22) let entropy = crypto::vrf::ietf_output(block.header.entropy_source).unwrap_or_default(); @@ -230,8 +233,8 @@ pub fn simulate_with_state( .merge_reporters(&reporters, &state.validators.current.ed25519())?; } - // (δ') Update the accounts - let accounts = preimage::accounts(block.header.slot, &block.extrinsic.preimages, accounts)?; + // (δ') Integrate preimages into the post-transfer state + let accounts = preimage::accounts(block.header.slot, &block.extrinsic.preimages, accounts); let (updates, removals) = accounts.diff(); diff.extend_iter(updates, removals); diff --git a/crates/runtime/src/tx/preimage.rs b/crates/runtime/src/tx/preimage.rs index 3295bc9d..81c58f43 100644 --- a/crates/runtime/src/tx/preimage.rs +++ b/crates/runtime/src/tx/preimage.rs @@ -1,23 +1,11 @@ -//! Preimage handler +//! Preimage extrinsic handler use account::{Account, Accounts}; use anyhow::Result; use score::{TimeSlot, extrinsic::PreimagesExtrinsic}; -/// (δ') handle preimage extrinsic -/// -/// Validates preimages against post-transfer state and integrates valid ones. -/// Invalid preimages are disregarded without prejudice -/// -/// # Arguments -/// * `slot` - Current time slot (τ') -/// * `preimages` - Preimage extrinsic data -/// * `accounts` - Post-transfer account state -pub fn accounts( - slot: TimeSlot, - preimages: &PreimagesExtrinsic, - mut accounts: impl Accounts, -) -> Result { +/// Validate preimages against the prior state +pub fn validate(accounts: &mut A, preimages: &PreimagesExtrinsic) -> Result<()> { let mut prev: Option<&score::extrinsic::Preimage> = None; for preimage in preimages { if let Some(p) = prev @@ -26,41 +14,31 @@ pub fn accounts( anyhow::bail!("preimages not sorted or unique"); } prev = Some(preimage); - let Some(account) = accounts.get(preimage.requester) else { - anyhow::bail!("Preimage for non-existent account"); - }; - let blob_len = preimage.blob.len() as u32; let hash = crypto::blake2b(&preimage.blob); - tracing::debug!("lookup hash={} len={}", hex::encode(hash), blob_len); - let Some(slots) = account.lookup(hash, blob_len) else { - anyhow::bail!("Preimage lookup failed"); - }; - - // skip if the lookup is removed - let Some(slots) = slots else { - continue; - }; - - if !slots.is_empty() { - anyhow::bail!("Preimage already has non-empty lookup slots"); + let len = preimage.blob.len() as u32; + if !accounts.is_providable(preimage.requester, hash, len) { + anyhow::bail!("preimage not required"); } + } + Ok(()) +} - if account.preimage(hash).is_some() { - anyhow::bail!("Preimage already exists"); +/// (δ') Integrate providable preimages into the post-transfer state +pub fn accounts( + slot: TimeSlot, + preimages: &PreimagesExtrinsic, + mut accounts: A, +) -> A { + for preimage in preimages { + let hash = crypto::blake2b(&preimage.blob); + let len = preimage.blob.len() as u32; + if !accounts.is_providable(preimage.requester, hash, len) { + continue; } - - // Set lookup slots to [τ'] (current time slot) - let updated_slots = vec![slot]; - tracing::debug!( - "service={} inserting preimage hash={} len={}", - account.index(), - hex::encode(hash), - blob_len - ); + let account = accounts.get(preimage.requester).expect("just checked"); account.insert_preimage(hash, preimage.blob.clone()); - account.insert_lookup(hash, blob_len, updated_slots); + account.insert_lookup(hash, len, vec![slot]); } - - Ok(accounts) + accounts } diff --git a/crates/testing/src/preimage.rs b/crates/testing/src/preimage.rs index 68176a44..2d374658 100644 --- a/crates/testing/src/preimage.rs +++ b/crates/testing/src/preimage.rs @@ -17,20 +17,20 @@ pub fn run(test: &specjam::Test) -> anyhow::Result<()> { let output = TestOutput::from_json(&test.output)?; // Validate post state - let accounts = to_accounts(input.pre_state.accounts.clone()); - let result = tx::preimage::accounts(input.input.slot, &input.input.preimages, accounts); - if let Ok(accounts) = result { - assert_eq!( - accounts - .accounts() - .iter() - .map(|(id, account)| (*id, account.account())) - .collect::>(), - self::to_accounts(output.post_state.accounts) - ); - } else { + let mut accounts = to_accounts(input.pre_state.accounts.clone()); + if tx::preimage::validate(&mut accounts, &input.input.preimages).is_err() { assert_eq!(input.pre_state, output.post_state); + return Ok(()); } + let accounts = tx::preimage::accounts(input.input.slot, &input.input.preimages, accounts); + assert_eq!( + accounts + .accounts() + .iter() + .map(|(id, account)| (*id, account.account())) + .collect::>(), + self::to_accounts(output.post_state.accounts) + ); Ok(()) } From edf7ea7bb980df1d587e35d730327b5ef1d1379e Mon Sep 17 00:00:00 2001 From: clearloop Date: Thu, 21 May 2026 21:39:18 +0800 Subject: [PATCH 09/18] fix(runtime): propagate auth queue through accumulation --- crates/runtime/src/tx/guarantee/acc.rs | 5 ++++- crates/runtime/src/tx/guarantee/mod.rs | 24 ++++++++++++++++++++++-- crates/runtime/src/tx/mod.rs | 21 ++++++++++++--------- crates/testing/src/accumulate.rs | 2 ++ 4 files changed, 40 insertions(+), 12 deletions(-) diff --git a/crates/runtime/src/tx/guarantee/acc.rs b/crates/runtime/src/tx/guarantee/acc.rs index 0ae02ec4..bcabee1d 100644 --- a/crates/runtime/src/tx/guarantee/acc.rs +++ b/crates/runtime/src/tx/guarantee/acc.rs @@ -3,7 +3,7 @@ use account::Accounts; use pvm::{Account, AccumulateState}; use score::{ - Gas, OpaqueHash, ServiceId, + AUTH_QUEUE_SIZE, Array, CORES_COUNT, Gas, OpaqueHash, ServiceId, safrole::ValidatorsData, service::{AccumulatedQueue, Privileges, ReadyQueue, WorkReport}, statistic::{AccumulationRecord, ServiceActivityRecord}, @@ -149,6 +149,9 @@ pub struct Accumulation { /// (πS') The service records pub records: BTreeMap, + /// (φ') The authorization queue + pub authorization: Array, CORES_COUNT>, + /// (θ) The accumulation logs pub logs: CommitmentMap, } diff --git a/crates/runtime/src/tx/guarantee/mod.rs b/crates/runtime/src/tx/guarantee/mod.rs index 2d27b45c..92f8648f 100644 --- a/crates/runtime/src/tx/guarantee/mod.rs +++ b/crates/runtime/src/tx/guarantee/mod.rs @@ -5,7 +5,7 @@ use account::Accounts; use error::{Error, Result}; use pvm::{AccumulateState, Pvm}; use score::{ - CORES_COUNT, Ed25519Public, EntropyBuffer, OpaqueHash, TimeSlot, + AUTH_QUEUE_SIZE, Array, CORES_COUNT, Ed25519Public, EntropyBuffer, OpaqueHash, TimeSlot, extrinsic::GuaranteesExtrinsic, safrole::ValidatorsData, service::{ @@ -38,6 +38,8 @@ pub fn accumulate( privileges: &Privileges, // The validators to be drawn (ι) validators: &ValidatorsData, + // The authorization queue (φ) + authorization: &Array, CORES_COUNT>, // The account storage (δ) accounts: R, // The entropy (η) @@ -47,6 +49,14 @@ pub fn accumulate( let (accumulatable, queued) = queue::accumulatable(slot, reports, ready_queue, accumulated_queue); + // Seed φ from prior state so the assign host call mutates the real queue + let auth_init: score::AuthorizationPools = authorization + .iter() + .map(|q| q.to_vec()) + .collect::>() + .try_into() + .expect("authorization has CORES_COUNT entries"); + // (Δ+) run outer accumulation (12.18) let gas_limit = privileges.gas_limit(); let mut validators = validators.clone(); @@ -57,7 +67,7 @@ pub fn accumulate( AccumulateState { accounts, privileges: privileges.clone(), - authorization: Default::default(), + authorization: auth_init, entropy, timeslot: slot, }, @@ -76,6 +86,15 @@ pub fn accumulate( let next_ready_queue = self::ready_queue(ready_queue, &next_accumulated_queue, queued, tau, slot); + // (φ') Project the per-core Vec back to the fixed-size state shape + let next_authorization: Array, CORES_COUNT> = + std::mem::take(&mut accumulated.context.authorization) + .into_iter() + .map(|q| Array::try_from(q).expect("queue has AUTH_QUEUE_SIZE entries")) + .collect::>() + .try_into() + .expect("authorization has CORES_COUNT entries"); + Ok(Accumulation { root: accumulated.root(), ready_queue: next_ready_queue, @@ -84,6 +103,7 @@ pub fn accumulate( privileges: accumulated.context.privileges, validators, records, + authorization: next_authorization, logs: accumulated.pairings, }) } diff --git a/crates/runtime/src/tx/mod.rs b/crates/runtime/src/tx/mod.rs index 77a3d1f7..6542ca72 100644 --- a/crates/runtime/src/tx/mod.rs +++ b/crates/runtime/src/tx/mod.rs @@ -76,15 +76,8 @@ pub fn simulate_with_state( // (η') Update entropy (6.22) let entropy = crypto::vrf::ietf_output(block.header.entropy_source).unwrap_or_default(); state.entropy = ticket::eta(new_epoch, &state.entropy, entropy); - if new_epoch { - // (λ', κ') Update validator state (6.13) - state.validators.previous = std::mem::replace( - &mut state.validators.current, - state.safrole.validators.clone(), - ); - } - // (ψ') Update disputes and get marks + // (ψ') Update disputes against the prior validator sets (10.4) let marks = if block.extrinsic.disputes.is_empty() { if !block.header.offenders_mark.is_empty() { anyhow::bail!("offenders mark is not empty"); @@ -104,6 +97,14 @@ pub fn simulate_with_state( marks }; + if new_epoch { + // (λ', κ') Update validator state (6.13) + state.validators.previous = std::mem::replace( + &mut state.validators.current, + state.safrole.validators.clone(), + ); + } + // complete the state root of the last block in the history if let Some(last) = state.recent_blocks.history.last_mut() { last.state_root = block.header.parent_state_root; @@ -135,7 +136,7 @@ pub fn simulate_with_state( let (available, assurances) = { // (W) the sequence of new available work reports (11.16) let (available, assurances) = self::assurance::available( - &state.reports, + &reports, if new_epoch { &state.validators.previous } else { @@ -196,6 +197,7 @@ pub fn simulate_with_state( &state.history, &state.privileges, &state.validators.drawn, + &state.authorization, accounts, state.entropy, )?; @@ -205,6 +207,7 @@ pub fn simulate_with_state( state.queue = accumulation.ready_queue; state.history = accumulation.accumulated_queue; state.validators.drawn = accumulation.validators; + state.authorization = accumulation.authorization; let candidate = state .safrole .next(&state.validators.drawn, &state.disputes.offenders); diff --git a/crates/testing/src/accumulate.rs b/crates/testing/src/accumulate.rs index daf1585d..71742e9c 100644 --- a/crates/testing/src/accumulate.rs +++ b/crates/testing/src/accumulate.rs @@ -29,6 +29,7 @@ pub async fn run(test: &specjam::Test) -> Result<()> { &input.pre_state.accumulated, &input.pre_state.privileges.into(), &Default::default(), + &Default::default(), accounts.clone(), Default::default(), )? @@ -41,6 +42,7 @@ pub async fn run(test: &specjam::Test) -> Result<()> { &input.pre_state.accumulated, &input.pre_state.privileges.into(), &Default::default(), + &Default::default(), accounts.clone(), Default::default(), )? From 79c8b68aab36c8763969962fb4cdad0998603872 Mon Sep 17 00:00:00 2001 From: clearloop Date: Thu, 21 May 2026 22:24:08 +0800 Subject: [PATCH 10/18] fix(runtime): reject useless tickets in safrole extrinsic --- crates/runtime/src/tx/ticket/error.rs | 4 ++++ crates/runtime/src/tx/ticket/mod.rs | 21 +++++++++++++++++++-- crates/runtime/src/tx/ticket/verify.rs | 14 ++++++++------ 3 files changed, 31 insertions(+), 8 deletions(-) diff --git a/crates/runtime/src/tx/ticket/error.rs b/crates/runtime/src/tx/ticket/error.rs index 9c2c9ec5..c558b86f 100644 --- a/crates/runtime/src/tx/ticket/error.rs +++ b/crates/runtime/src/tx/ticket/error.rs @@ -19,6 +19,10 @@ pub enum Error { Reserved, /// Duplicate ticket DuplicateTicket, + /// Submitted ticket missing from posterior accumulator + UselessTicket, + /// Too many tickets in a single block + TooManyTickets, } impl std::fmt::Display for Error { diff --git a/crates/runtime/src/tx/ticket/mod.rs b/crates/runtime/src/tx/ticket/mod.rs index 443ebeba..09ac521e 100644 --- a/crates/runtime/src/tx/ticket/mod.rs +++ b/crates/runtime/src/tx/ticket/mod.rs @@ -61,8 +61,12 @@ pub fn safrole( } let slot_phase = slot % score::EPOCH_LENGTH; - if slot_phase >= score::TICKET_SUBMISSION_PERIOD && !tickets.is_empty() { - return Err(Error::UnexpectedTicket); + if slot_phase >= score::TICKET_SUBMISSION_PERIOD { + if !tickets.is_empty() { + return Err(Error::UnexpectedTicket); + } + } else if tickets.len() > score::MAX_TICKETS_PER_EXTRINSIC as usize { + return Err(Error::TooManyTickets); } let epoch = tau / score::EPOCH_LENGTH; @@ -105,6 +109,9 @@ pub fn accumulator( new_tickets = self::verify::tickets(entropy, next, tickets)?; } + // Snapshot submitted ids for the n ⊆ γ_a' check below + let submitted_ids: Vec = new_tickets.iter().map(|t| t.id).collect(); + // update the accumulator let mut accumulator = accumulator.clone(); if new_epoch { @@ -125,6 +132,16 @@ pub fn accumulator( // Take only the first E tickets (formula 6.35: truncate to E) accumulator.sort_by_key(|a| a.id); accumulator.truncate(score::EPOCH_LENGTH as usize); + + // (n ⊆ γ_a') Every submitted ticket must survive into the posterior + // accumulator; a submission that gets truncated out is "useless". + if submitted_ids + .iter() + .any(|id| accumulator.binary_search_by_key(id, |t| t.id).is_err()) + { + return Err(Error::UselessTicket); + } + Ok(accumulator) } diff --git a/crates/runtime/src/tx/ticket/verify.rs b/crates/runtime/src/tx/ticket/verify.rs index af767435..fe71ea31 100644 --- a/crates/runtime/src/tx/ticket/verify.rs +++ b/crates/runtime/src/tx/ticket/verify.rs @@ -46,12 +46,14 @@ pub fn tickets( }) .collect(); - // 3. Check for bad order: 6.32 & 6.33 - let mut sorted = new_tickets.clone(); - sorted.sort_by_key(|a| a.id); - if sorted != new_tickets { - return Err(Error::BadTicketOrder); + // 3. Strictly ascending by id, no duplicates (6.32 & 6.33) + for pair in new_tickets.windows(2) { + match pair[0].id.cmp(&pair[1].id) { + std::cmp::Ordering::Less => {} + std::cmp::Ordering::Equal => return Err(Error::DuplicateTicket), + std::cmp::Ordering::Greater => return Err(Error::BadTicketOrder), + } } - Ok(sorted) + Ok(new_tickets) } From 9b0625a2749a0cd0492541bc13ee38e1341055f0 Mon Sep 17 00:00:00 2001 From: clearloop Date: Thu, 21 May 2026 22:52:42 +0800 Subject: [PATCH 11/18] fix(runtime): reject duplicate work-packages across accumulation pipeline --- Cargo.toml | 2 +- crates/core/account/src/registry.rs | 9 +++++- crates/runtime/src/account/registry.rs | 11 +++++++ crates/runtime/src/tx/guarantee/mod.rs | 2 +- crates/runtime/src/tx/guarantee/validator.rs | 34 ++++++++++++++------ 5 files changed, 46 insertions(+), 12 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 9a83077b..0733ef18 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -21,7 +21,7 @@ resolver = "1" [workspace.package] edition = "2024" -version = "0.1.2-pre.7" +version = "0.1.2-pre.8" homepage = "https://spacejam.app" repository = "https://github.com/spacejamapp/jade" license = "GPL-3.0" diff --git a/crates/core/account/src/registry.rs b/crates/core/account/src/registry.rs index db7f5817..6da2500d 100644 --- a/crates/core/account/src/registry.rs +++ b/crates/core/account/src/registry.rs @@ -1,7 +1,7 @@ //! Account registry use crate::Account; -use score::{OpaqueHash, ServiceId, TrieKey, service::ServiceAccount}; +use score::{Gas, OpaqueHash, ServiceId, TrieKey, service::ServiceAccount}; use std::collections::{BTreeMap, BTreeSet}; /// Account registry @@ -12,6 +12,9 @@ pub trait Accounts: Clone + Send + Sync + 'static { /// Get the code hash of an account fn code_hash(&self, index: u32) -> Option; + /// Get the minimum gas required to invoke the accumulate entry-point + fn min_acc_gas(&self, index: u32) -> Option; + /// Create a new account fn upsert(&mut self, index: u32, account: impl Account); @@ -67,6 +70,10 @@ impl Accounts for BTreeMap { Some(self.get(&index)?.info.code) } + fn min_acc_gas(&self, index: u32) -> Option { + Some(self.get(&index)?.info.accumulate) + } + fn upsert(&mut self, index: u32, account: impl Account) { self.insert(index, account.account()); } diff --git a/crates/runtime/src/account/registry.rs b/crates/runtime/src/account/registry.rs index bcef633b..71bb5ee7 100644 --- a/crates/runtime/src/account/registry.rs +++ b/crates/runtime/src/account/registry.rs @@ -65,6 +65,17 @@ impl account::Accounts for Accounts { self.storage.account_info(index).ok().map(|info| info.code) } + fn min_acc_gas(&self, index: u32) -> Option { + if let Some(account) = self.accounts.get(&index) { + return Some(account.info.accumulate); + } + + self.storage + .account_info(index) + .ok() + .map(|info| info.accumulate) + } + fn upsert(&mut self, index: u32, account: impl account::Account) { let inherited = Account::inherit(self.storage.clone(), index, account); self.accounts.insert(index, inherited); diff --git a/crates/runtime/src/tx/guarantee/mod.rs b/crates/runtime/src/tx/guarantee/mod.rs index 92f8648f..f92b7a5e 100644 --- a/crates/runtime/src/tx/guarantee/mod.rs +++ b/crates/runtime/src/tx/guarantee/mod.rs @@ -222,7 +222,7 @@ pub fn pools( >, guarantees: &GuaranteesExtrinsic, ) -> score::AuthorizationPools { - let slot = (timeslot % score::EPOCH_LENGTH) as usize % score::AUTH_QUEUE_SIZE; + let slot = timeslot as usize % score::AUTH_QUEUE_SIZE; let mut new_pools = score::AuthorizationPools::default(); for (core_index, pool) in pools.iter().enumerate() { let mut new_pool = pool.clone(); diff --git a/crates/runtime/src/tx/guarantee/validator.rs b/crates/runtime/src/tx/guarantee/validator.rs index b440950c..851f6297 100644 --- a/crates/runtime/src/tx/guarantee/validator.rs +++ b/crates/runtime/src/tx/guarantee/validator.rs @@ -4,8 +4,8 @@ use crate::tx::guarantee::error::{Error, Result}; use account::{Account, Accounts}; use score::{ CORES_COUNT, CoreIndex, EPOCH_LENGTH, Ed25519Public, Entropy, MAX_DEPENDENCY_COUNT, - MAX_WORK_REPORT_OUTPUT_SIZE, OpaqueHash, ROTATION_PERIOD, SERVICE_ITEM_MIN_GAS, State, - TimeSlot, VALIDATORS_COUNT, WORK_REPORT_GAS_LIMIT, + MAX_WORK_REPORT_OUTPUT_SIZE, OpaqueHash, ROTATION_PERIOD, State, TimeSlot, VALIDATORS_COUNT, + WORK_REPORT_GAS_LIMIT, extrinsic::{GuaranteesExtrinsic, ReportGuarantee}, service::{ReportedWorkPackage, WorkExecResult}, }; @@ -251,14 +251,18 @@ impl<'s, R: Accounts> GuaranteeValidator<'s, R> { return Err(Error::WorkReportGasTooHigh); } - if result.accumulate_gas < SERVICE_ITEM_MIN_GAS { - return Err(Error::ServiceItemGasTooLow); - } - let Some(code_hash) = self.accounts.code_hash(result.service_id) else { return Err(Error::BadServiceId); }; + let min_gas = self + .accounts + .min_acc_gas(result.service_id) + .ok_or(Error::BadServiceId)?; + if result.accumulate_gas < min_gas { + return Err(Error::ServiceItemGasTooLow); + } + if code_hash != result.code_hash { tracing::debug!( "bad code hash for service {}: 0x{} != 0x{}", @@ -329,10 +333,22 @@ impl<'s, R: Accounts> GuaranteeValidator<'s, R> { Ok(()) } - // TODO: check if duplicated in service deps? pub fn duplicated(&self, hash: &OpaqueHash) -> bool { - self.recent.iter().any(|r| r.hash == *hash) - || self.reported.iter().filter(|h| *h == hash).count() > 1 + self.reported.iter().filter(|h| *h == hash).count() > 1 + || self.recent.iter().any(|r| r.hash == *hash) + || self.state.history.iter().flatten().any(|h| h == hash) + || self + .state + .queue + .iter() + .flatten() + .any(|r| r.report.spec.hash == *hash) + || self + .state + .reports + .iter() + .flatten() + .any(|a| a.report.spec.hash == *hash) } fn contains_dep(&self, dep: &OpaqueHash) -> bool { From eee8dd0201233dafe3e4fb2ff813a1e5ee7d5adb Mon Sep 17 00:00:00 2001 From: clearloop Date: Thu, 21 May 2026 23:17:48 +0800 Subject: [PATCH 12/18] fix(runtime): tighten verdict ordering and vote-split checks --- Cargo.lock | 38 ++++++++++++------------ crates/runtime/src/tx/assurance/mod.rs | 41 -------------------------- crates/runtime/src/tx/dispute/mod.rs | 25 ++++++---------- crates/runtime/src/tx/mod.rs | 1 - crates/testing/src/assurances.rs | 1 - 5 files changed, 28 insertions(+), 78 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f8ffd34f..a9224aa4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1016,7 +1016,7 @@ dependencies = [ [[package]] name = "cranelift-pvm" -version = "0.1.2-pre.7" +version = "0.1.2-pre.8" dependencies = [ "anyhow", "cranelift", @@ -2759,7 +2759,7 @@ dependencies = [ [[package]] name = "pvmc" -version = "0.1.2-pre.7" +version = "0.1.2-pre.8" dependencies = [ "anyhow", "bindgen", @@ -3294,7 +3294,7 @@ dependencies = [ [[package]] name = "serde-jam" -version = "0.1.2-pre.7" +version = "0.1.2-pre.8" dependencies = [ "anyhow", "serde", @@ -3462,7 +3462,7 @@ dependencies = [ [[package]] name = "spacejam" -version = "0.1.2-pre.7" +version = "0.1.2-pre.8" dependencies = [ "anyhow", "async-trait", @@ -3499,7 +3499,7 @@ dependencies = [ [[package]] name = "spacejam-account" -version = "0.1.2-pre.7" +version = "0.1.2-pre.8" dependencies = [ "anyhow", "serde-jam", @@ -3510,7 +3510,7 @@ dependencies = [ [[package]] name = "spacejam-core" -version = "0.1.2-pre.7" +version = "0.1.2-pre.8" dependencies = [ "anyhow", "erased-serde", @@ -3564,7 +3564,7 @@ dependencies = [ [[package]] name = "spacejam-network" -version = "0.1.2-pre.7" +version = "0.1.2-pre.8" dependencies = [ "anyhow", "clap", @@ -3587,7 +3587,7 @@ dependencies = [ [[package]] name = "spacejam-offchain" -version = "0.1.2-pre.7" +version = "0.1.2-pre.8" dependencies = [ "anyhow", "async-trait", @@ -3603,7 +3603,7 @@ dependencies = [ [[package]] name = "spacejam-rpc" -version = "0.1.2-pre.7" +version = "0.1.2-pre.8" dependencies = [ "anyhow", "jsonrpsee", @@ -3615,7 +3615,7 @@ dependencies = [ [[package]] name = "spacejam-runtime" -version = "0.1.2-pre.7" +version = "0.1.2-pre.8" dependencies = [ "anyhow", "hex", @@ -3636,7 +3636,7 @@ dependencies = [ [[package]] name = "spacejam-service" -version = "0.1.2-pre.7" +version = "0.1.2-pre.8" dependencies = [ "anyhow", "blake2b_simd", @@ -3650,14 +3650,14 @@ dependencies = [ [[package]] name = "spacejam-spec" -version = "0.1.2-pre.7" +version = "0.1.2-pre.8" dependencies = [ "serde", ] [[package]] name = "spacejam-testing" -version = "0.1.2-pre.7" +version = "0.1.2-pre.8" dependencies = [ "anyhow", "hex", @@ -3687,7 +3687,7 @@ dependencies = [ [[package]] name = "spacejam-testnet" -version = "0.1.2-pre.7" +version = "0.1.2-pre.8" dependencies = [ "anyhow", "clap", @@ -3698,7 +3698,7 @@ dependencies = [ [[package]] name = "spacejam-worker" -version = "0.1.2-pre.7" +version = "0.1.2-pre.8" dependencies = [ "anyhow", "pvm", @@ -3714,7 +3714,7 @@ dependencies = [ [[package]] name = "spacejson" -version = "0.1.2-pre.7" +version = "0.1.2-pre.8" dependencies = [ "anyhow", "hex", @@ -3725,7 +3725,7 @@ dependencies = [ [[package]] name = "spacejson-derive" -version = "0.1.2-pre.7" +version = "0.1.2-pre.8" dependencies = [ "proc-macro2", "quote", @@ -3734,7 +3734,7 @@ dependencies = [ [[package]] name = "spacevm" -version = "0.1.2-pre.7" +version = "0.1.2-pre.8" dependencies = [ "anyhow", "hex", @@ -3748,7 +3748,7 @@ dependencies = [ [[package]] name = "spacevm-export" -version = "0.1.2-pre.7" +version = "0.1.2-pre.8" dependencies = [ "pvm", "pvmc", diff --git a/crates/runtime/src/tx/assurance/mod.rs b/crates/runtime/src/tx/assurance/mod.rs index 459ba461..470e2260 100644 --- a/crates/runtime/src/tx/assurance/mod.rs +++ b/crates/runtime/src/tx/assurance/mod.rs @@ -8,7 +8,6 @@ use score::{ safrole::ValidatorData, service::{AvailabilityAssignments, WorkReport}, }; -use std::collections::HashSet; mod error; @@ -34,23 +33,11 @@ pub fn reports( pub fn available( reports: &AvailabilityAssignments, validators: &[ValidatorData], - slot: TimeSlot, parent: OpaqueHash, assurances: &[AvailAssurance], ) -> Result<(Vec, [u32; CORES_COUNT])> { // Track assurance count per core let mut core_assurance_counts = [0u32; CORES_COUNT]; - let mut stale_reports = HashSet::new(); - - // Check for stale reports - for (core_idx, assignment) in reports.iter().enumerate() { - if let Some(assignment) = assignment - && slot >= assignment.timeout + WORK_REPORT_TIMEOUT_PERIOD - { - stale_reports.insert(core_idx); - continue; - } - } // Check for engaged reports: cheap checks first, then batch verify sigs. let mut assuror = None; @@ -112,31 +99,3 @@ pub fn available( Ok((available, core_assurance_counts)) } - -/// Verifies the assurance -pub fn verify_assurance( - validators: &[ValidatorData], - assurance: &AvailAssurance, - parent: OpaqueHash, -) -> Result<()> { - if assurance.validator_index >= VALIDATORS_COUNT { - return Err(Error::BadValidatorIndex); - } - - if assurance.anchor != parent { - return Err(Error::BadAttestationParent); - } - - if crypto::ed25519::verify( - &assurance.singing_message(), - assurance.signature, - validators[assurance.validator_index as usize].ed25519, - ) - .is_err() - { - tracing::error!("bad signature for assurance: {:?}", assurance); - return Err(Error::BadSignature); - } - - Ok(()) -} diff --git a/crates/runtime/src/tx/dispute/mod.rs b/crates/runtime/src/tx/dispute/mod.rs index 5b31543c..890c611d 100644 --- a/crates/runtime/src/tx/dispute/mod.rs +++ b/crates/runtime/src/tx/dispute/mod.rs @@ -82,19 +82,18 @@ fn verdicts( verdicts: &[Verdict], ) -> Result { let mut records = DisputesRecords::default(); - let mut last_verdict = None; + let mut last_target: Option = None; for verdict in verdicts { if verdict.votes.len() != VALIDATORS_SUPER_MAJORITY as usize { return Err(Error::NotEnoughValidators); } - if let Some(last_verdict) = last_verdict.take() { - if verdict < last_verdict { - return Err(Error::VerdictsNotSortedUnique); - } - } else { - last_verdict = Some(verdict); + if let Some(last) = last_target + && verdict.target <= last + { + return Err(Error::VerdictsNotSortedUnique); } + last_target = Some(verdict.target); let mut aye = 0; let aye_message = verdict.signature_message(true); @@ -136,15 +135,9 @@ fn verdicts( .map_err(|_| Error::BadSignature)?; match aye { - aye if aye >= VALIDATORS_SUPER_MAJORITY => { - records.good.push(verdict.target); - } - aye if aye >= VALIDATORS_COUNT / 3 => { - records.wonky.push(verdict.target); - } - 0 => { - records.bad.push(verdict.target); - } + aye if aye == VALIDATORS_SUPER_MAJORITY => records.good.push(verdict.target), + aye if aye == VALIDATORS_COUNT / 3 => records.wonky.push(verdict.target), + 0 => records.bad.push(verdict.target), _ => { tracing::error!("Bad vote split in verdict: {aye}/{VALIDATORS_SUPER_MAJORITY}"); return Err(Error::BadVoteSplit); diff --git a/crates/runtime/src/tx/mod.rs b/crates/runtime/src/tx/mod.rs index 6542ca72..f9c7b97c 100644 --- a/crates/runtime/src/tx/mod.rs +++ b/crates/runtime/src/tx/mod.rs @@ -142,7 +142,6 @@ pub fn simulate_with_state( } else { &state.validators.current }, - block.header.slot, block.header.parent, &block.extrinsic.assurances, )?; diff --git a/crates/testing/src/assurances.rs b/crates/testing/src/assurances.rs index f425b8bc..187b3ad0 100644 --- a/crates/testing/src/assurances.rs +++ b/crates/testing/src/assurances.rs @@ -20,7 +20,6 @@ pub fn run(test: &specjam::Test) -> anyhow::Result<()> { let result = tx::assurance::available( &input.pre_state.avail_assignments, &input.pre_state.curr_validators, - input.input.slot, input.input.parent, &input.input.assurances, ); From 4a0cfcd127ecb288d830c0cad3385edb1b782644 Mon Sep 17 00:00:00 2001 From: clearloop Date: Fri, 22 May 2026 00:38:16 +0800 Subject: [PATCH 13/18] fix(runtime): reject epoch_mark on non-epoch-transition blocks --- crates/core/src/safrole/mod.rs | 10 +++++----- crates/core/src/statistic/mod.rs | 2 -- crates/runtime/src/tx/block/header.rs | 16 ++++++++++++++-- crates/runtime/src/tx/block/history.rs | 5 ----- crates/runtime/src/tx/mod.rs | 1 - crates/testing/src/history.rs | 4 +++- 6 files changed, 22 insertions(+), 16 deletions(-) diff --git a/crates/core/src/safrole/mod.rs b/crates/core/src/safrole/mod.rs index 6c6f571f..99906c9c 100644 --- a/crates/core/src/safrole/mod.rs +++ b/crates/core/src/safrole/mod.rs @@ -77,11 +77,11 @@ impl Safrole { let curr_slot_phase = slot % crate::EPOCH_LENGTH; let prev_slot_phase = tau % crate::EPOCH_LENGTH; - // Return true if: - // 1. Different epochs (e' ≠ e) - // 2. Previous slot not before submission period (m ≥ Y) - // 3. Current slot not after submission period (m' < Y) - // 4. Accumulator not full (|gamma_a| ≠ E) + // Return true if all of: + // 1. Same epoch (e' = e) + // 2. Previous slot before submission period (m < Y) + // 3. Current slot at or after submission period (m' ≥ Y) + // 4. Accumulator full (|gamma_a| = E) curr_epoch == prev_epoch && prev_slot_phase < crate::TICKET_SUBMISSION_PERIOD && curr_slot_phase >= crate::TICKET_SUBMISSION_PERIOD diff --git a/crates/core/src/statistic/mod.rs b/crates/core/src/statistic/mod.rs index 9321efba..fdd4d92f 100644 --- a/crates/core/src/statistic/mod.rs +++ b/crates/core/src/statistic/mod.rs @@ -108,9 +108,7 @@ impl Statistics { for reporter in reporters.iter() { let Some(index) = validators.iter().position(|v| v == reporter) else { continue; - // anyhow::bail!("reporter is invalid"); }; - self.vals_current[index].guarantees += 1; } Ok(()) diff --git a/crates/runtime/src/tx/block/header.rs b/crates/runtime/src/tx/block/header.rs index 6764e91f..4cd47f2d 100644 --- a/crates/runtime/src/tx/block/header.rs +++ b/crates/runtime/src/tx/block/header.rs @@ -137,9 +137,21 @@ pub fn check(state: &State, header: &Header, new_epoch: bool) -> anyhow::Result< // validate the epoch mark if let Some(epoch_mark) = &header.epoch_mark { + if !new_epoch { + anyhow::bail!("epoch mark present but not expected"); + } + + // Φ uses the posterior punish-set: prior offenders ∪ this block's + let offenders: Vec<_> = state + .disputes + .offenders + .iter() + .chain(header.offenders_mark.iter()) + .copied() + .collect(); let expected = state .safrole - .next(&state.validators.drawn, &header.offenders_mark) + .next(&state.validators.drawn, &offenders) .evals(); if epoch_mark.validators[..] != expected[..] { anyhow::bail!("epoch mark validators mismatch"); @@ -153,7 +165,7 @@ pub fn check(state: &State, header: &Header, new_epoch: bool) -> anyhow::Result< anyhow::bail!("epoch mark tickets entropy mismatch"); } } else if new_epoch { - anyhow::bail!("epoch mark is required"); + anyhow::bail!("epoch mark required but not present"); } let should_have_tickets_mark = state.safrole.has_tickets_mark(state.timeslot, header.slot); diff --git a/crates/runtime/src/tx/block/history.rs b/crates/runtime/src/tx/block/history.rs index 696ece4b..80c75099 100644 --- a/crates/runtime/src/tx/block/history.rs +++ b/crates/runtime/src/tx/block/history.rs @@ -11,14 +11,9 @@ use score::{ pub fn import( history: &mut History, header_hash: OpaqueHash, - parent_state_root: OpaqueHash, accumulate_root: OpaqueHash, reported: Vec, ) { - if let Some(last) = history.history.last_mut() { - last.state_root = parent_state_root; - } - history.mmr.peaks = mmr::append(history.mmr.peaks.clone(), accumulate_root); if history.history.is_empty() { let new_block = BlockInfo { diff --git a/crates/runtime/src/tx/mod.rs b/crates/runtime/src/tx/mod.rs index f9c7b97c..3a7e24cd 100644 --- a/crates/runtime/src/tx/mod.rs +++ b/crates/runtime/src/tx/mod.rs @@ -224,7 +224,6 @@ pub fn simulate_with_state( block::history::import( &mut state.recent_blocks, block.header.hash(), - block.header.parent_state_root, root, reported, ); diff --git a/crates/testing/src/history.rs b/crates/testing/src/history.rs index 2f58941b..eb2f2fd0 100644 --- a/crates/testing/src/history.rs +++ b/crates/testing/src/history.rs @@ -17,10 +17,12 @@ pub fn run(test: &specjam::Test) -> anyhow::Result<()> { let input = TestInput::from_json(&test.input)?; let output = TestOutput::from_json(&test.output)?; let mut history = input.pre_state.beta.clone(); + if let Some(last) = history.history.last_mut() { + last.state_root = input.input.parent_state_root; + } history::import( &mut history, input.input.header_hash, - input.input.parent_state_root, input.input.accumulate_root, input.input.work_packages.clone(), ); From d5e6d2e23e68ce8758f3e1d6a6f0ee066ccb6767 Mon Sep 17 00:00:00 2001 From: clearloop Date: Fri, 22 May 2026 14:05:01 +0800 Subject: [PATCH 14/18] fix(runtime): merge per-core authqueue mutations after accumulation --- crates/runtime/src/tx/guarantee/acc.rs | 15 +++------ crates/runtime/src/tx/guarantee/exec.rs | 44 +++++++++++++++++-------- crates/runtime/src/tx/guarantee/mod.rs | 13 +++----- 3 files changed, 40 insertions(+), 32 deletions(-) diff --git a/crates/runtime/src/tx/guarantee/acc.rs b/crates/runtime/src/tx/guarantee/acc.rs index bcabee1d..f3df5e81 100644 --- a/crates/runtime/src/tx/guarantee/acc.rs +++ b/crates/runtime/src/tx/guarantee/acc.rs @@ -48,7 +48,7 @@ impl Accumulated { } } - /// Get the service records + /// Compose the service activity records; caller passes the accumulated prefix. pub fn records( &mut self, accumulatable: &[WorkReport], @@ -64,17 +64,12 @@ impl Accumulated { } } - for transfer in self.transfers.iter() { - if records.contains_key(&transfer.recipient) - || !self.gas.contains_key(&transfer.recipient) - { + // Include services that consumed gas without contributing digests. + for (service, gas) in self.gas.iter() { + if records.contains_key(service) { continue; } - - let record = records.entry(transfer.recipient).or_default(); - if record.accumulate_gas_used == 0 { - record.accumulate_gas_used = *self.gas.get(&transfer.recipient).unwrap_or(&0); - } + records.entry(*service).or_default().accumulate_gas_used = *gas; } // update the last update time of the accounts diff --git a/crates/runtime/src/tx/guarantee/exec.rs b/crates/runtime/src/tx/guarantee/exec.rs index 3b730c7d..8cd6e8fc 100644 --- a/crates/runtime/src/tx/guarantee/exec.rs +++ b/crates/runtime/src/tx/guarantee/exec.rs @@ -34,7 +34,7 @@ pub fn outer( context: AccumulateState, validators: &mut score::safrole::ValidatorsData, gas_table: &BTreeMap, -) -> Accumulated { +) -> anyhow::Result> { let mut accumulated = Accumulated::new(context); let empty = BTreeMap::new(); let mut first = true; @@ -54,15 +54,19 @@ pub fn outer( break; } + // Deferred-transfer gas supplements the block budget for this round. + let xfer_gas: Gas = transfers.iter().map(|t| t.gas_limit).sum(); + let step = self::parallel::( accumulated.context.clone(), validators, &transfers, if index == 0 { &[] } else { &reports[..index] }, if first { gas_table } else { &empty }, - ); + )?; - gas_limit -= step.gas.values().sum::(); + let step_gas: Gas = step.gas.values().sum(); + gas_limit = gas_limit.saturating_add(xfer_gas).saturating_sub(step_gas); reports = &reports[index..]; transfers = step.transfers.clone(); accumulated.transfers.extend(step.transfers); @@ -79,7 +83,7 @@ pub fn outer( } } - accumulated + Ok(accumulated) } /// (Δ*) parallel accumulation @@ -89,7 +93,7 @@ pub fn parallel( transfers: &[DeferredTransfer], reports: &[WorkReport], table: &BTreeMap, -) -> Accumulated { +) -> anyhow::Result> { let mut services: BTreeSet = Default::default(); for report in reports { for result in &report.results { @@ -140,12 +144,16 @@ pub fn parallel( context.privileges.always_acc = mgr.always_acc.clone(); } - // Update assign services + // Update assign service and authorization queue from the prior c-th assigner. for c in 0..context.privileges.assign.len() { let old = context.privileges.assign[c]; let mgr_val = mgr.map(|m| m.assign[c]).unwrap_or(old); let svc_val = results.get(&old).map(|r| r.context.privileges.assign[c]); context.privileges.assign[c] = svc_val.map(|s| r(old, mgr_val, s)).unwrap_or(mgr_val); + + if let Some(result) = results.get(&old) { + context.authorization[c] = result.context.authorization[c].clone(); + } } // Update designate @@ -198,7 +206,8 @@ pub fn parallel( gas.insert(*service_id, result.gas); } - // Apply additions (n): the service's own account from every result, + // Apply additions (n); reject the same new id created by two services. + let mut created: BTreeSet = BTreeSet::new(); for (service_id, result) in results.iter() { if !removed.contains(service_id) && let Some(account) = result.context.accounts.accounts().get(service_id) @@ -214,12 +223,19 @@ pub fn parallel( if id == service_id { continue; // already handled above } - if !removed.contains(id) - && account.creation() == context.timeslot - && context.accounts.get(*id).is_none() - { - context.accounts.upsert(*id, account.clone()); + if removed.contains(id) || account.creation() != context.timeslot { + continue; + } + if context.accounts.get(*id).is_some() { + if created.contains(id) { + anyhow::bail!( + "duplicate account creation: service {id} created by multiple services" + ); + } + continue; } + context.accounts.upsert(*id, account.clone()); + created.insert(*id); } } @@ -228,13 +244,13 @@ pub fn parallel( context.accounts.remove(*service); } - Accumulated { + Ok(Accumulated { accumulated: reports.len(), context, transfers, pairings, gas, - } + }) } /// (Δ1) single accumulation (12.24) diff --git a/crates/runtime/src/tx/guarantee/mod.rs b/crates/runtime/src/tx/guarantee/mod.rs index f92b7a5e..9f6192de 100644 --- a/crates/runtime/src/tx/guarantee/mod.rs +++ b/crates/runtime/src/tx/guarantee/mod.rs @@ -73,14 +73,14 @@ pub fn accumulate( }, &mut validators, &privileges.always_acc, - ); + )?; // (πS') compose the service activity records - let records = accumulated.records(&accumulatable); + let n = accumulated.accumulated; + let records = accumulated.records(&accumulatable[..n]); // update the accumulated queue (ξ') - let next_accumulated_queue = - self::accumulated_history(accumulated_queue, accumulatable, accumulated.accumulated); + let next_accumulated_queue = self::accumulated_history(accumulated_queue, accumulatable, n); // update the ready queue (θ') let next_ready_queue = @@ -120,15 +120,12 @@ pub fn accumulated_history( next.remove(0); } - // Add new accumulated work report hashes + // Set of newly-accumulated work-package hashes, canonically sorted. let mut new_accumulated: Vec = accumulatable .iter() .take(accumulated) .map(|w| w.spec.hash) .collect(); - - // NOTE: Sort the new accumulated work report hashes again to align the test - // vectors, not sure if we missed anything that we have to do it here. new_accumulated.sort(); next.push(new_accumulated); From e30ac9a7cc932f276104cdf737d4d93dc30a8acb Mon Sep 17 00:00:00 2001 From: clearloop Date: Fri, 22 May 2026 17:11:21 +0800 Subject: [PATCH 15/18] fix(vm): honor gratis and registrar desired-id in new host call --- crates/vm/src/host/accumulate.rs | 65 ++++++++++++++++++++------------ crates/vm/src/host/general.rs | 6 +++ 2 files changed, 46 insertions(+), 25 deletions(-) diff --git a/crates/vm/src/host/accumulate.rs b/crates/vm/src/host/accumulate.rs index 927f76ba..c028d2eb 100644 --- a/crates/vm/src/host/accumulate.rs +++ b/crates/vm/src/host/accumulate.rs @@ -55,10 +55,10 @@ pub fn bless(ctx: &mut impl Argument) -> Result { } } - // (m, v, r) Check if bless and designate are valid service IDs + // (m, v, r) must fit in u32; GP allows any serviceid including the protected range. if [bless, designate, register] .iter() - .any(|&id| (id != 0 && id < score::MINIMUM_SERVICE_ID as u64) || id > u32::MAX as u64) + .any(|&id| id > u32::MAX as u64) { return Ok(Exit::Who as u64); } @@ -81,19 +81,16 @@ pub fn assign(ctx: &mut impl Argument) -> Result { let mut source = vec![0u8; 32 * score::QUEUE_ITEMS as usize]; ctx.read_into(o as u32, &mut source)?; - // Check core index first (before reading memory) if core >= score::CORES_COUNT as u64 { return Ok(Exit::Core as u64); } - // Check if the calling service is the current assign service for this core let privileges = ctx.privileges(); if ctx.service() != privileges.assign[core as usize] { return Ok(Exit::Huh as u64); } - // Validate assign parameter is a valid service ID - if (assign != 0 && assign < score::MINIMUM_SERVICE_ID as u64) || assign > u32::MAX as u64 { + if assign > u32::MAX as u64 { return Ok(Exit::Who as u64); } @@ -157,28 +154,38 @@ pub fn checkpoint(ctx: &mut impl Argument) -> Result { /// (ΩN) new pub fn new_(ctx: &mut impl Argument) -> Result { - let [o, length, accumulate_gas, transfer_gas, f] = [ + let [o, length, accumulate_gas, transfer_gas, gratis, desiredid] = [ ctx.rget(7), ctx.rget(8), ctx.rget(9), ctx.rget(10), ctx.rget(11), + ctx.rget(12), ]; - // check if the service is blessed + // Non-zero gratis requires manager privilege. let privileges = ctx.privileges(); - if f != 0 && privileges.bless != ctx.service() { + if gratis != 0 && privileges.bless != ctx.service() { return Ok(Exit::Huh as u64); } - // get the account code let code = ctx.read_hash(o as u32)?; if length > u32::MAX as u64 { crate::bail!("Invalid length"); } - // create a new account with proper storage accounting - let index = ctx.index(); + // Registrar can target a protected-range index directly via r_12. + let registrar_target = + ctx.service() == privileges.register && desiredid < score::MINIMUM_SERVICE_ID as u64; + let index = if registrar_target { + if ctx.account(desiredid).is_ok() { + return Ok(Exit::Full as u64); + } + desiredid as u32 + } else { + ctx.index() + }; + let mut created = ServiceAccount { index, info: ServiceInfo { @@ -186,6 +193,7 @@ pub fn new_(ctx: &mut impl Argument) -> Result { balance: score::BALANCE_PER_SERVICE, accumulate: accumulate_gas, transfer: transfer_gas, + offset: gratis, creation: ctx.timeslot(), update: 0, parent: ctx.service(), @@ -195,23 +203,24 @@ pub fn new_(ctx: &mut impl Argument) -> Result { }; created.insert_lookup(code, length as u32, vec![]); - // Calculate the full threshold cost let new_account_threshold = created.threshold(); let service = ctx.this()?; if service.balance() < service.threshold() + new_account_threshold { return Ok(Exit::Cash as u64); } - // Deduct full threshold from parent and give it to new account *service.balance_mut() -= new_account_threshold; created.info.balance = new_account_threshold; ctx.upsert(index, created); - // Check and find a free account index - let base = - score::MINIMUM_SERVICE_ID + (index - score::MINIMUM_SERVICE_ID + 42) % score::CHECK_SALT; - let new_index = ctx.check(base); - ctx.set_index(new_index); + // Auto-generated path bumps nextfreeid; registrar path doesn't. + if !registrar_target { + let base = score::MINIMUM_SERVICE_ID + + (index - score::MINIMUM_SERVICE_ID + 42) % score::CHECK_SALT; + let new_index = ctx.check(base); + ctx.set_index(new_index); + } + Ok(index as u64) } @@ -357,16 +366,24 @@ pub fn solicit(ctx: &mut impl Argument) -> Result { let timeslot = ctx.timeslot(); let account = ctx.this()?; let mut slots = vec![]; - if let Some(lookup) = account.lookup(hash, z as u32).flatten() { + let appending = if let Some(lookup) = account.lookup(hash, z as u32).flatten() { if lookup.len() == 2 { slots = vec![lookup[0], lookup[1], timeslot]; + true } else { return Ok(Exit::Huh as u64); } - } + } else { + false + }; - // pre-calculate the new threshold - let threshold = account.lookup_threshold(z).unwrap_or(u64::MAX); + // Appending to an existing 2-slot entry doesn't change items/octets; only + // a fresh insertion changes the threshold. + let threshold = if appending { + account.threshold() + } else { + account.lookup_threshold(z).unwrap_or(u64::MAX) + }; if account.balance() < threshold { return Ok(Exit::Full as u64); } @@ -440,8 +457,6 @@ pub fn provide(ctx: &mut impl Argument) -> Result { return Ok(Exit::Huh as u64); } - // FIXME: the lookup insert is not specified in graypper, could be a bug - // in the fuzzy tests or our implementation. account.insert_lookup(hash, size as u32, vec![timeslot]); account.insert_preimage(hash, preimage); Ok(Exit::Ok as u64) diff --git a/crates/vm/src/host/general.rs b/crates/vm/src/host/general.rs index 6b08cb33..f436081f 100644 --- a/crates/vm/src/host/general.rs +++ b/crates/vm/src/host/general.rs @@ -131,6 +131,12 @@ pub fn write(ctx: &mut impl Argument) -> Result { // check if the account has enough balance to cover the threshold let account = ctx.this()?; let prev = account.read(&key); + + // Removing a key that doesn't exist is a no-op; skip the threshold check. + if vz == 0 && prev.is_none() { + return Ok(Exit::None as u64); + } + let threshold = account .write_threshold(&key, &value, prev.as_deref()) .unwrap_or(u64::MAX); From f0628d30f6fbaa40e1b087fe9c8479e22ea36fcf Mon Sep 17 00:00:00 2001 From: clearloop Date: Fri, 22 May 2026 17:14:43 +0800 Subject: [PATCH 16/18] chore: make clippy happy --- crates/crypto/benches/blake2b.rs | 4 +-- crates/crypto/benches/ring_vrf.rs | 25 +++++++++++------ crates/crypto/src/merkle/trie31.rs | 4 ++- crates/crypto/src/vrf.rs | 3 +- crates/runtime/src/storage/multitree.rs | 37 +++++++++++++++---------- crates/runtime/src/tx/block/mod.rs | 3 +- crates/runtime/src/tx/preimage.rs | 6 +--- crates/runtime/tests/trie.rs | 5 ++-- crates/spacejam/src/storage/parity.rs | 4 +-- 9 files changed, 51 insertions(+), 40 deletions(-) diff --git a/crates/crypto/benches/blake2b.rs b/crates/crypto/benches/blake2b.rs index ab3d0f56..d9aecc42 100644 --- a/crates/crypto/benches/blake2b.rs +++ b/crates/crypto/benches/blake2b.rs @@ -3,14 +3,14 @@ //! at 8–64KiB). Compares the in-tree blake2b_simd impl against the RustCrypto //! `blake2` crate side by side. -use criterion::{BenchmarkId, Criterion, Throughput, black_box, criterion_group, criterion_main}; +use criterion::{black_box, criterion_group, criterion_main, BenchmarkId, Criterion, Throughput}; fn simd(input: &[u8]) -> [u8; 32] { spacejam_crypto::blake2b(input) } fn rustcrypto(input: &[u8]) -> [u8; 32] { - use blake2::{Blake2b, Digest, digest::consts::U32}; + use blake2::{digest::consts::U32, Blake2b, Digest}; let mut hasher = Blake2b::::new(); hasher.update(input); hasher.finalize().into() diff --git a/crates/crypto/benches/ring_vrf.rs b/crates/crypto/benches/ring_vrf.rs index 373e0e64..b3eda988 100644 --- a/crates/crypto/benches/ring_vrf.rs +++ b/crates/crypto/benches/ring_vrf.rs @@ -1,6 +1,6 @@ //! Microbench: single ring-vrf verify vs batched verify, swept across batch sizes. -use criterion::{BenchmarkId, Criterion, Throughput, black_box, criterion_group, criterion_main}; +use criterion::{black_box, criterion_group, criterion_main, BenchmarkId, Criterion, Throughput}; use spacejam_crypto::{ ring::RING_SIZE, vrf::{KeyPair, Verifier}, @@ -8,12 +8,14 @@ use spacejam_crypto::{ /// Generate `n` deterministic (message, signature) triples signed under distinct ring positions. fn gen_items(n: usize) -> (Verifier, Vec<(Vec, [u8; 784])>) { - assert!(n <= RING_SIZE, "batch size {n} exceeds RING_SIZE {RING_SIZE}"); - let ring: Vec = (0..RING_SIZE).map(|i| KeyPair::from([i as u8; 32])).collect(); - let pks = ring - .iter() - .map(|k| k.public().unwrap()) - .collect::>(); + assert!( + n <= RING_SIZE, + "batch size {n} exceeds RING_SIZE {RING_SIZE}" + ); + let ring: Vec = (0..RING_SIZE) + .map(|i| KeyPair::from([i as u8; 32])) + .collect(); + let pks = ring.iter().map(|k| k.public().unwrap()).collect::>(); let pkeys = ring.iter().map(|k| k.public).collect::>(); let verifier = Verifier::new(pkeys); @@ -32,7 +34,10 @@ fn bench(c: &mut Criterion) { let mut group = c.benchmark_group("ring_vrf"); group.sample_size(20); - for &n in [1usize, 3, 8, 16, 32, 64].iter().filter(|&&n| n <= RING_SIZE) { + for &n in [1usize, 3, 8, 16, 32, 64] + .iter() + .filter(|&&n| n <= RING_SIZE) + { let (verifier, items) = gen_items(n); group.throughput(Throughput::Elements(n as u64)); @@ -48,7 +53,9 @@ fn bench(c: &mut Criterion) { b.iter(|| { verifier .ring_vrf_verify_batch( - items.iter().map(|(m, s)| (m.as_slice(), [].as_slice(), s.as_slice())), + items + .iter() + .map(|(m, s)| (m.as_slice(), [].as_slice(), s.as_slice())), ) .unwrap() }) diff --git a/crates/crypto/src/merkle/trie31.rs b/crates/crypto/src/merkle/trie31.rs index ea778373..a8783b7c 100644 --- a/crates/crypto/src/merkle/trie31.rs +++ b/crates/crypto/src/merkle/trie31.rs @@ -38,7 +38,9 @@ fn merkle(kvs: &mut [([u8; 31], &[u8])], depth: usize) -> [u8; 32] { /// `bit=0` entries come before `bit=1` entries at `depth`. Linear scan — use /// this when the slice's relative order is known to be preserved. pub fn split_at_bit(keys: &[[u8; 31]], depth: usize) -> usize { - keys.iter().position(|k| bit(k, depth)).unwrap_or(keys.len()) + keys.iter() + .position(|k| bit(k, depth)) + .unwrap_or(keys.len()) } /// Recover `(left, right)` child slots for a branch given its encoded payload diff --git a/crates/crypto/src/vrf.rs b/crates/crypto/src/vrf.rs index 1b16116c..08c12d67 100644 --- a/crates/crypto/src/vrf.rs +++ b/crates/crypto/src/vrf.rs @@ -315,8 +315,7 @@ impl Verifier { let mut outputs = Vec::new(); for (vrf_input_data, aux_data, signature) in items { let signature = RingVrfSignature::deserialize_compressed_unchecked(signature)?; - let input = - Input::new(vrf_input_data).ok_or(anyhow::anyhow!("Invalid input"))?; + let input = Input::new(vrf_input_data).ok_or(anyhow::anyhow!("Invalid input"))?; let output_hash: [u8; 32] = signature.output.hash()[..32].try_into()?; batch.push(input, signature.output, aux_data, &signature.proof); outputs.push(output_hash); diff --git a/crates/runtime/src/storage/multitree.rs b/crates/runtime/src/storage/multitree.rs index 87a74b24..c94a1b7c 100644 --- a/crates/runtime/src/storage/multitree.rs +++ b/crates/runtime/src/storage/multitree.rs @@ -93,10 +93,11 @@ pub trait MultiTreeStore: Send + Sync + 'static { self.insert_tree(column, new_root, root_node)?; } - if let Some(prev) = prev_root { - if prev != ZERO_HASH && prev != new_root { - self.dereference_tree(column, prev)?; - } + if let Some(prev) = prev_root + && prev != ZERO_HASH + && prev != new_root + { + self.dereference_tree(column, prev)?; } Ok(new_root) @@ -120,7 +121,13 @@ pub trait MultiTreeStore: Send + Sync + 'static { let (k, v) = new_keys[0]; let data = trie31::leaf(k, v).to_vec(); let hash = crypto::blake2b(&data); - return Ok((Some(NodeRef::New(NewNode { data, children: vec![] })), hash)); + return Ok(( + Some(NodeRef::New(NewNode { + data, + children: vec![], + })), + hash, + )); } // Branch case: partition both keys and dirty by bit at `depth`. @@ -131,20 +138,20 @@ pub trait MultiTreeStore: Send + Sync + 'static { let (left_dirty, right_dirty) = new_dirty.split_at(dirty_mid); let (prev_left, prev_right) = match &prev_node { - Some((data, children)) if !trie31::is_leaf(data) => trie31::split_branch_children( - data, children, - ) - .ok_or_else(|| { - anyhow!( - "trie node shape mismatch at depth {depth}: children.len()={}", - children.len() - ) - })?, + Some((data, children)) if !trie31::is_leaf(data) => { + trie31::split_branch_children(data, children).ok_or_else(|| { + anyhow!( + "trie node shape mismatch at depth {depth}: children.len()={}", + children.len() + ) + })? + } _ => (None, None), }; let (l_ref, l_hash) = self.descend(column, prev_left, left_keys, left_dirty, depth + 1)?; - let (r_ref, r_hash) = self.descend(column, prev_right, right_keys, right_dirty, depth + 1)?; + let (r_ref, r_hash) = + self.descend(column, prev_right, right_keys, right_dirty, depth + 1)?; let data = trie31::branch(l_hash, r_hash).to_vec(); let hash = crypto::blake2b(&data); diff --git a/crates/runtime/src/tx/block/mod.rs b/crates/runtime/src/tx/block/mod.rs index f50b7879..bf43b1dd 100644 --- a/crates/runtime/src/tx/block/mod.rs +++ b/crates/runtime/src/tx/block/mod.rs @@ -158,8 +158,7 @@ impl TestChain { // Bootstrap the incremental trie. Every key is dirty on the first // build; the algorithm degenerates to a full retrie. let new_root = self.data.with_data(|data| { - let kvs: Vec<(TrieKey, &[u8])> = - data.iter().map(|(k, v)| (*k, v.as_slice())).collect(); + let kvs: Vec<(TrieKey, &[u8])> = data.iter().map(|(k, v)| (*k, v.as_slice())).collect(); let dirty: Vec = data.keys().copied().collect(); self.data.apply(Column::TrieNodes, None, &kvs, &dirty) })??; diff --git a/crates/runtime/src/tx/preimage.rs b/crates/runtime/src/tx/preimage.rs index 81c58f43..f5b6ac7d 100644 --- a/crates/runtime/src/tx/preimage.rs +++ b/crates/runtime/src/tx/preimage.rs @@ -25,11 +25,7 @@ pub fn validate(accounts: &mut A, preimages: &PreimagesExtrinsic) - } /// (δ') Integrate providable preimages into the post-transfer state -pub fn accounts( - slot: TimeSlot, - preimages: &PreimagesExtrinsic, - mut accounts: A, -) -> A { +pub fn accounts(slot: TimeSlot, preimages: &PreimagesExtrinsic, mut accounts: A) -> A { for preimage in preimages { let hash = crypto::blake2b(&preimage.blob); let len = preimage.blob.len() as u32; diff --git a/crates/runtime/tests/trie.rs b/crates/runtime/tests/trie.rs index 1dd1f812..5a4c4d60 100644 --- a/crates/runtime/tests/trie.rs +++ b/crates/runtime/tests/trie.rs @@ -2,8 +2,8 @@ //! reference [`crypto::merkle::trie31`] root. use crypto::{blake2b, merkle}; -use spacejam_runtime::storage::{Column, MemoryDb, MultiTreeStore}; use score::{OpaqueHash, TrieKey}; +use spacejam_runtime::storage::{Column, MemoryDb, MultiTreeStore}; use std::collections::BTreeMap; fn key(seed: u64) -> TrieKey { @@ -36,7 +36,8 @@ fn apply( dirty: &[TrieKey], ) -> OpaqueHash { let kvs: Vec<(TrieKey, &[u8])> = state.iter().map(|(k, v)| (*k, v.as_slice())).collect(); - db.apply(Column::TrieNodes, prev, &kvs, dirty).expect("apply") + db.apply(Column::TrieNodes, prev, &kvs, dirty) + .expect("apply") } #[test] diff --git a/crates/spacejam/src/storage/parity.rs b/crates/spacejam/src/storage/parity.rs index 7252f0af..2cd3eff7 100644 --- a/crates/spacejam/src/storage/parity.rs +++ b/crates/spacejam/src/storage/parity.rs @@ -2,8 +2,8 @@ use anyhow::Result; use parity_db::{ - BTreeIterator, ColumnOptions, Db, NewNode as PdNewNode, NodeRef as PdNodeRef, - Operation as Op, Options, + BTreeIterator, ColumnOptions, Db, NewNode as PdNewNode, NodeRef as PdNodeRef, Operation as Op, + Options, }; use runtime::storage::{ Column, Commit, KVStorage, MultiTreeStore, NewNode, NodeAddress, NodeRef, Operation, From ca3db996a57a739d6bff50705345591e5c2f43e1 Mon Sep 17 00:00:00 2001 From: clearloop Date: Fri, 22 May 2026 19:22:40 +0800 Subject: [PATCH 17/18] feat(crypto): introduce trait for MultiTree --- Cargo.lock | 1 + Cargo.toml | 2 + crates/crypto/Cargo.toml | 6 +- crates/crypto/src/merkle/mod.rs | 2 + crates/crypto/src/merkle/multitree.rs | 284 ++++++++++++++++++++++++ crates/crypto/src/merkle/trie31.rs | 42 +--- crates/crypto/tests/multitree.rs | 60 +++++ crates/runtime/src/storage/branch.rs | 28 +-- crates/runtime/src/storage/kv.rs | 159 ++----------- crates/runtime/src/storage/mod.rs | 3 +- crates/runtime/src/storage/multitree.rs | 199 ----------------- crates/runtime/src/tx/block/mod.rs | 19 +- crates/runtime/tests/trie.rs | 130 ----------- crates/spacejam/src/storage/parity.rs | 30 +-- 14 files changed, 409 insertions(+), 556 deletions(-) create mode 100644 crates/crypto/src/merkle/multitree.rs create mode 100644 crates/crypto/tests/multitree.rs delete mode 100644 crates/runtime/src/storage/multitree.rs delete mode 100644 crates/runtime/tests/trie.rs diff --git a/Cargo.lock b/Cargo.lock index a9224aa4..c3e1e9c0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3540,6 +3540,7 @@ dependencies = [ "ed25519-zebra", "hex", "once_cell", + "parking_lot", "rand 0.8.5", "rayon", "rcgen", diff --git a/Cargo.toml b/Cargo.toml index 0733ef18..d5aaccb6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -77,6 +77,7 @@ ark-serialize = { version = "0.5", default-features = false } ark-vrf = { version = "=0.2.1", features = ["bandersnatch", "ring", "parallel"] } async-trait = "0.1.89" bindgen = "0.72" +blake2 = "0.10.6" blake2b_simd = "1.0.4" blake3 = "1.8.2" cc = "1.2" @@ -103,6 +104,7 @@ log = "0.4.28" once_cell = "1.21.3" paste = "1.0.15" parity-db = "0.5.5" +parking_lot = "0.12" postcard = "1.1.3" proc-macro2 = "1.0.101" prometheus = { version = "0.23.1", package = "prometheus-client" } diff --git a/crates/crypto/Cargo.toml b/crates/crypto/Cargo.toml index 8571e9ab..cdf04894 100644 --- a/crates/crypto/Cargo.toml +++ b/crates/crypto/Cargo.toml @@ -13,6 +13,7 @@ blake2b_simd = { workspace = true, optional = true } blake3 = { workspace = true, optional = true } ed25519-zebra = { workspace = true, optional = true } once_cell = { workspace = true, optional = true } +parking_lot = { workspace = true, optional = true } rand = { workspace = true, optional = true } rayon = { workspace = true, optional = true } rcgen = { workspace = true, optional = true } @@ -28,12 +29,11 @@ blake3 = ["dep:blake3"] bls = ["w3f-bls"] ed25519 = ["anyhow", "ed25519-zebra", "rand", "rayon"] keccak = ["tiny-keccak"] -merkle = ["blake2", "keccak", "rayon"] +merkle = ["anyhow", "blake2", "keccak", "parking_lot", "rayon"] rand = ["dep:rand"] shuffle = ["blake2"] tls = ["rcgen", "ed25519-zebra/pkcs8"] vrf = [ - "anyhow", "ark-vrf", "ark-ec", "ark-ed-on-bls12-381-bandersnatch", @@ -42,7 +42,7 @@ vrf = [ ] [dev-dependencies] -blake2 = "0.10.6" +blake2.workspace = true criterion = { version = "0.5", default-features = false } hex = { workspace = true, features = ["std"] } serde.workspace = true diff --git a/crates/crypto/src/merkle/mod.rs b/crates/crypto/src/merkle/mod.rs index bbff16b8..2618ec57 100644 --- a/crates/crypto/src/merkle/mod.rs +++ b/crates/crypto/src/merkle/mod.rs @@ -4,9 +4,11 @@ mod binary; pub mod mmr; +pub mod multitree; mod trie; pub mod trie31; pub use binary::{broot, hroot, kroot, root, tree, MerkleTree}; +pub use multitree::{NewNode, NodeAddress, NodeRef}; pub use trie::merkle as trie; pub use trie31::trie as trie31; diff --git a/crates/crypto/src/merkle/multitree.rs b/crates/crypto/src/merkle/multitree.rs new file mode 100644 index 00000000..fa4a3dc0 --- /dev/null +++ b/crates/crypto/src/merkle/multitree.rs @@ -0,0 +1,284 @@ +//! Ref-counted Merkle tree primitives and the incremental builder. + +use crate::{blake2b, merkle::trie31}; +use anyhow::{anyhow, Result}; + +const ZERO_HASH: [u8; 32] = [0u8; 32]; + +/// Switch from sequential to parallel descent when a build step covers this +/// many keys or more. Mirrors `trie31::PARALLEL_THRESHOLD`. +const PARALLEL_THRESHOLD: usize = 64; + +/// Address of a node already persisted in the store. +pub type NodeAddress = u64; + +/// A persisted node: payload + child addresses. +pub type PersistedNode = (Vec, Vec); + +/// Reference to a node when building a tree: either a fresh subtree to write, +/// or a pointer to an existing node to reuse. +#[derive(Debug, Clone)] +pub enum NodeRef { + /// A fresh subtree. + New(NewNode), + /// A node already in the store, retained as-is. + Existing(NodeAddress), +} + +/// A newly-built node with its child references. +#[derive(Debug, Clone)] +pub struct NewNode { + /// Encoded node payload (leaf or branch). + pub data: Vec, + /// Child references, in canonical order. + pub children: Vec, +} + +/// Ref-counted Merkle tree storage. Implementors provide the IO surface; +/// the `apply` default impl owns the recursion, leaf/branch encoding, and +/// subtree-reuse logic. +pub trait MultiTree: Sync { + /// Look up a node by its persisted address. + fn get_node(&self, address: NodeAddress) -> Result>; + + /// Look up the root node of the tree identified by `hash`. + fn get_root(&self, hash: [u8; 32]) -> Result>; + + /// Persist a new tree under `hash`, bumping refcounts on existing children. + fn insert_tree(&self, hash: [u8; 32], root: NewNode) -> Result<()>; + + /// Release the tree at `hash`, GC'ing nodes whose refcount falls to zero. + fn dereference_tree(&self, hash: [u8; 32]) -> Result<()>; + + /// Build a new tree from `new_state`, reusing unchanged subtrees of `prev_root`. + fn apply( + &self, + prev_root: Option<[u8; 32]>, + new_state: &[([u8; 31], &[u8])], + dirty_keys: &[[u8; 31]], + ) -> Result<[u8; 32]> { + let prev_root_node = match prev_root { + Some(h) if h != ZERO_HASH => self.get_root(h)?, + _ => None, + }; + + let (root_ref, new_root) = self.build(prev_root_node, new_state, dirty_keys, 0)?; + + if matches!(prev_root, Some(p) if p == new_root) { + return Ok(new_root); + } + + if let Some(NodeRef::New(root_node)) = root_ref { + self.insert_tree(new_root, root_node)?; + } + + if let Some(prev) = prev_root.filter(|&p| p != ZERO_HASH && p != new_root) { + self.dereference_tree(prev)?; + } + + Ok(new_root) + } + + fn build( + &self, + prev_node: Option, + new_keys: &[([u8; 31], &[u8])], + new_dirty: &[[u8; 31]], + depth: usize, + ) -> Result<(Option, [u8; 32])> { + if new_keys.is_empty() { + return Ok((None, ZERO_HASH)); + } + + if new_keys.len() == 1 { + let (k, v) = new_keys[0]; + let data = trie31::leaf(k, v).to_vec(); + let hash = blake2b(&data); + return Ok(( + Some(NodeRef::New(NewNode { + data, + children: vec![], + })), + hash, + )); + } + + let mut buf = new_keys.to_vec(); + let key_mid = trie31::partition(&mut buf, depth); + let (left_keys, right_keys) = buf.split_at(key_mid); + let dirty_mid = trie31::split_at_bit(new_dirty, depth); + let (left_dirty, right_dirty) = new_dirty.split_at(dirty_mid); + let (prev_left, prev_right) = match &prev_node { + Some((data, children)) if !trie31::is_leaf(data) => { + trie31::split_branch_children(data, children).ok_or_else(|| { + anyhow!( + "trie node shape mismatch at depth {depth}: children.len()={}", + children.len() + ) + })? + } + _ => (None, None), + }; + + let left = || self.descend(prev_left, left_keys, left_dirty, depth + 1); + let right = || self.descend(prev_right, right_keys, right_dirty, depth + 1); + let (l, r) = if new_keys.len() >= PARALLEL_THRESHOLD { + rayon::join(left, right) + } else { + (left(), right()) + }; + + let (l_ref, l_hash) = l?; + let (r_ref, r_hash) = r?; + let data = trie31::branch(l_hash, r_hash).to_vec(); + let hash = blake2b(&data); + let children: Vec = [l_ref, r_ref].into_iter().flatten().collect(); + Ok((Some(NodeRef::New(NewNode { data, children })), hash)) + } + + fn descend( + &self, + prev_addr: Option, + new_keys: &[([u8; 31], &[u8])], + new_dirty: &[[u8; 31]], + depth: usize, + ) -> Result<(Option, [u8; 32])> { + if new_dirty.is_empty() { + if let Some(addr) = prev_addr { + let (data, _) = self + .get_node(addr)? + .ok_or_else(|| anyhow!("missing prev node at address {addr}"))?; + let hash = blake2b(&data); + return Ok((Some(NodeRef::Existing(addr)), hash)); + } + if new_keys.is_empty() { + return Ok((None, ZERO_HASH)); + } + } + + let prev_node = match prev_addr { + Some(addr) => self.get_node(addr)?, + None => None, + }; + self.build(prev_node, new_keys, new_dirty, depth) + } +} + +/// In-memory ref-counted node store. Used as the multitree backend for +/// `MemoryDb` and tests; mirrors the contract of parity-db's multitree column +/// without disk persistence. +#[derive(Default)] +pub struct MultiTreeMap { + inner: parking_lot::Mutex, +} + +#[derive(Default)] +struct Inner { + nodes: std::collections::HashMap, + roots: std::collections::HashMap<[u8; 32], NodeAddress>, + next_addr: NodeAddress, +} + +struct NodeEntry { + data: Vec, + children: Vec, + refcount: u32, +} + +impl Inner { + fn alloc(&mut self) -> NodeAddress { + let addr = self.next_addr; + self.next_addr += 1; + addr + } + + fn insert(&mut self, node_ref: NodeRef) -> anyhow::Result { + match node_ref { + NodeRef::Existing(addr) => { + let Some(entry) = self.nodes.get_mut(&addr) else { + anyhow::bail!("NodeRef::Existing({addr}) but node is gone"); + }; + + entry.refcount += 1; + Ok(addr) + } + NodeRef::New(NewNode { data, children }) => { + let child_addrs: Vec<_> = children + .into_iter() + .map(|c| self.insert(c)) + .collect::>>()?; + let addr = self.alloc(); + self.nodes.insert( + addr, + NodeEntry { + data, + children: child_addrs, + refcount: 1, + }, + ); + Ok(addr) + } + } + } + + fn dereference(&mut self, addr: NodeAddress) -> anyhow::Result<()> { + let drop_children = match self.nodes.get_mut(&addr) { + Some(entry) => { + entry.refcount = entry + .refcount + .checked_sub(1) + .ok_or_else(|| anyhow!("refcount underflow on addr {addr}"))?; + (entry.refcount == 0).then(|| std::mem::take(&mut entry.children)) + } + None => None, + }; + if let Some(children) = drop_children { + self.nodes.remove(&addr); + for child in children { + self.dereference(child)?; + } + } + + Ok(()) + } +} + +impl MultiTree for MultiTreeMap { + fn insert_tree(&self, key: [u8; 32], root: NewNode) -> Result<()> { + let mut inner = self.inner.lock(); + if inner.roots.contains_key(&key) { + anyhow::bail!("insert_tree: tree already present — caller must dereference_tree first"); + } + let addr = inner.insert(NodeRef::New(root))?; + inner.roots.insert(key, addr); + Ok(()) + } + + fn dereference_tree(&self, key: [u8; 32]) -> Result<()> { + let mut inner = self.inner.lock(); + if let Some(addr) = inner.roots.remove(&key) { + inner.dereference(addr)?; + } + Ok(()) + } + + fn get_root(&self, key: [u8; 32]) -> Result> { + let inner = self.inner.lock(); + let Some(&addr) = inner.roots.get(&key) else { + return Ok(None); + }; + Ok(inner + .nodes + .get(&addr) + .map(|n| (n.data.clone(), n.children.clone()))) + } + + fn get_node(&self, address: NodeAddress) -> Result> { + Ok(self + .inner + .lock() + .nodes + .get(&address) + .map(|n| (n.data.clone(), n.children.clone()))) + } +} diff --git a/crates/crypto/src/merkle/trie31.rs b/crates/crypto/src/merkle/trie31.rs index a8783b7c..636edded 100644 --- a/crates/crypto/src/merkle/trie31.rs +++ b/crates/crypto/src/merkle/trie31.rs @@ -1,28 +1,24 @@ -//! A space-efficient trie for storing key-value pairs. -//! Implements binary Patricia Merkle Trie as described in the graypaper appendix D. +//! Binary Patricia Merkle Trie (graypaper appendix D). use crate::blake2b; const PARALLEL_THRESHOLD: usize = 64; -/// Compute the Merkle root of a set of key-value pairs. (D.6) +/// Merkle root of a sorted key-value set (GP D.6). pub fn trie(kvs: &[([u8; 31], &[u8])]) -> [u8; 32] { let mut buf = kvs.to_vec(); merkle(&mut buf, 0) } -/// Compute the Merkle root of a set of key-value pairs with specified depth. (D.6) fn merkle(kvs: &mut [([u8; 31], &[u8])], depth: usize) -> [u8; 32] { if kvs.is_empty() { return [0; 32]; } - if kvs.len() == 1 { let (k, v) = kvs[0]; return blake2b(&leaf(k, v)); } - // In-place partition: entries with bit=0 (left) before entries with bit=1 (right) let len = kvs.len(); let mid = partition(kvs, depth); let (left, right) = kvs.split_at_mut(mid); @@ -34,20 +30,15 @@ fn merkle(kvs: &mut [([u8; 31], &[u8])], depth: usize) -> [u8; 32] { blake2b(&branch(l_hash, r_hash)) } -/// Find the partition index of a slice of keys already sorted such that -/// `bit=0` entries come before `bit=1` entries at `depth`. Linear scan — use -/// this when the slice's relative order is known to be preserved. +/// Partition index for an already-ordered key slice — linear scan, preserves order. pub fn split_at_bit(keys: &[[u8; 31]], depth: usize) -> usize { keys.iter() .position(|k| bit(k, depth)) .unwrap_or(keys.len()) } -/// Recover `(left, right)` child slots for a branch given its encoded payload -/// and stored child addresses (which omit empty subtrees). `None` if the -/// payload's empty-side encoding doesn't match the address count. Generic -/// over the address type so storage backends can keep their own -/// representation. +/// Recover `(left, right)` child slots from a branch payload and its stored +/// child addresses; `None` if the empty-side encoding doesn't match the count. pub fn split_branch_children( data: &[u8], children: &[T], @@ -63,8 +54,7 @@ pub fn split_branch_children( } } -/// Partition `kvs` in-place so that entries with bit 0 at `depth` come first. -/// Returns the index of the first "right" (bit=1) entry. +/// In-place partition of `kvs` so bit-0 entries at `depth` come first; returns the split index. pub fn partition(kvs: &mut [([u8; 31], &[u8])], depth: usize) -> usize { let mut left_end = 0; for i in 0..kvs.len() { @@ -76,44 +66,36 @@ pub fn partition(kvs: &mut [([u8; 31], &[u8])], depth: usize) -> usize { left_end } -/// Whether a node's encoded payload is a leaf (high bit of byte 0 set). +/// True when the payload's high bit of byte 0 is set (leaf marker). pub fn is_leaf(data: &[u8]) -> bool { data.first().is_some_and(|b| b & 0x80 != 0) } -/// Branch encoding (graypaper D.6). +/// Branch encoding (GP D.6). pub fn branch(l: [u8; 32], r: [u8; 32]) -> [u8; 64] { let mut encoded = [0u8; 64]; - encoded[0] = l[0] & 0x7F; // 0b01111111 + encoded[0] = l[0] & 0x7F; encoded[1..32].copy_from_slice(&l[1..]); encoded[32..64].copy_from_slice(&r); encoded } -/// Leaf encoding (graypaper D.6). +/// Leaf encoding (GP D.6). pub fn leaf(k: [u8; 31], v: &[u8]) -> [u8; 64] { let mut encoded = [0u8; 64]; - if v.len() <= 32 { - // 0x80 = 0b10000000, 0x3F = 0b00111111 encoded[0] = 0x80 | (v.len() as u8 & 0x3F); encoded[1..32].copy_from_slice(&k); encoded[32..(32 + v.len())].copy_from_slice(v); - - if v.len() < 32 { - encoded[(32 + v.len())..64].fill(0); - } } else { - encoded[0] = 0xC0; // 0b11000000 + encoded[0] = 0xC0; encoded[1..32].copy_from_slice(&k); encoded[32..64].copy_from_slice(&blake2b(v)); } - encoded } -/// Get the bit at the specified position in the key -/// For 31-byte keys, we have 248 bits (31*8) +/// Bit at `depth` (MSB-first) of a 248-bit key. pub fn bit(k: &[u8; 31], depth: usize) -> bool { let byte_idx = depth / 8; let bit_idx = 7 - (depth % 8); diff --git a/crates/crypto/tests/multitree.rs b/crates/crypto/tests/multitree.rs new file mode 100644 index 00000000..239cc2ab --- /dev/null +++ b/crates/crypto/tests/multitree.rs @@ -0,0 +1,60 @@ +//! `MultiTree::apply` produces the same root as `trie31::trie`, both from +//! scratch and after an incremental diff against a prior tree. +#![cfg(feature = "merkle")] + +use spacejam_crypto::{ + blake2b, + merkle::{ + multitree::{MultiTree, MultiTreeMap}, + trie31, + }, +}; + +fn key(seed: u64) -> [u8; 31] { + let mut k = [0u8; 31]; + k.copy_from_slice(&blake2b(&seed.to_le_bytes())[..31]); + k +} + +#[test] +fn from_scratch_matches_trie31() { + let store = MultiTreeMap::default(); + let pairs: Vec<([u8; 31], Vec)> = (0..500u64) + .map(|i| (key(i), i.to_le_bytes().repeat(5))) + .collect(); + let kvs: Vec<([u8; 31], &[u8])> = pairs.iter().map(|(k, v)| (*k, v.as_slice())).collect(); + let dirty: Vec<[u8; 31]> = kvs.iter().map(|(k, _)| *k).collect(); + + let root = store.apply(None, &kvs, &dirty).expect("apply"); + assert_eq!(root, trie31::trie(&kvs)); +} + +#[test] +fn incremental_matches_trie31() { + let store = MultiTreeMap::default(); + let mut pairs: Vec<([u8; 31], Vec)> = (0..500u64) + .map(|i| (key(i), i.to_le_bytes().repeat(5))) + .collect(); + pairs.sort_by_key(|p| p.0); + let kvs: Vec<([u8; 31], &[u8])> = pairs.iter().map(|(k, v)| (*k, v.as_slice())).collect(); + let dirty0: Vec<[u8; 31]> = kvs.iter().map(|(k, _)| *k).collect(); + + let root0 = store.apply(None, &kvs, &dirty0).expect("round 0"); + assert_eq!(root0, trie31::trie(&kvs)); + + let mut dirty1 = Vec::new(); + for i in 0..5u64 { + let k = key(i); + let v = (i + 1_000).to_le_bytes().repeat(7); + if let Some(p) = pairs.iter_mut().find(|p| p.0 == k) { + p.1 = v; + } + dirty1.push(k); + } + dirty1.sort(); + pairs.sort_by_key(|p| p.0); + let kvs1: Vec<([u8; 31], &[u8])> = pairs.iter().map(|(k, v)| (*k, v.as_slice())).collect(); + + let root1 = store.apply(Some(root0), &kvs1, &dirty1).expect("round 1"); + assert_eq!(root1, trie31::trie(&kvs1)); +} diff --git a/crates/runtime/src/storage/branch.rs b/crates/runtime/src/storage/branch.rs index 3a956e48..7c35bd72 100644 --- a/crates/runtime/src/storage/branch.rs +++ b/crates/runtime/src/storage/branch.rs @@ -2,7 +2,7 @@ use crate::{ Storage, - storage::{Column, Commit, KVStorage, MultiTreeStore, NewNode, NodeAddress, StateStorage}, + storage::{Column, Commit, KVStorage, MultiTree, NewNode, NodeAddress, StateStorage}, }; use anyhow::Result; use score::{OpaqueHash, TrieKey, state::StateKeyLike}; @@ -117,29 +117,21 @@ impl KVStorage for Branch { } } -impl MultiTreeStore for Branch { - fn insert_tree(&self, column: Column, key: OpaqueHash, root: NewNode) -> Result<()> { - self.state.insert_tree(column, key, root) +impl MultiTree for Branch { + fn insert_tree(&self, key: OpaqueHash, root: NewNode) -> Result<()> { + self.state.insert_tree(key, root) } - fn dereference_tree(&self, column: Column, key: OpaqueHash) -> Result<()> { - self.state.dereference_tree(column, key) + fn dereference_tree(&self, key: OpaqueHash) -> Result<()> { + self.state.dereference_tree(key) } - fn get_root( - &self, - column: Column, - key: OpaqueHash, - ) -> Result, Vec)>> { - self.state.get_root(column, key) + fn get_root(&self, key: OpaqueHash) -> Result, Vec)>> { + self.state.get_root(key) } - fn get_node( - &self, - column: Column, - address: NodeAddress, - ) -> Result, Vec)>> { - self.state.get_node(column, address) + fn get_node(&self, address: NodeAddress) -> Result, Vec)>> { + self.state.get_node(address) } } diff --git a/crates/runtime/src/storage/kv.rs b/crates/runtime/src/storage/kv.rs index db6846da..e7692540 100644 --- a/crates/runtime/src/storage/kv.rs +++ b/crates/runtime/src/storage/kv.rs @@ -1,10 +1,11 @@ //! Key-value storage abstraction -use crate::storage::{Column, Commit, MultiTreeStore, NewNode, NodeAddress, NodeRef}; +use crate::storage::{Column, Commit, MultiTree, NewNode, NodeAddress}; use anyhow::Result; +use crypto::merkle::multitree::MultiTreeMap; use score::{OpaqueHash, TrieKey, state::StateKeyLike}; use std::{ - collections::{BTreeMap, HashMap}, + collections::BTreeMap, sync::{Arc, RwLock}, }; @@ -40,108 +41,28 @@ pub trait KVStorage: Send + Sync + 'static { } } -/// In-memory key-value storage implementation -/// -/// This implementation stores all data in memory and is not persistent. -/// It's useful for testing and for situations where persistence isn't required. +/// In-memory key-value storage. #[derive(Default)] pub struct MemoryDb { data: Arc>>>, - tries: Arc>, -} - -/// Internal ref-counted node store backing [`MultiTreeStore`]. -#[derive(Default)] -struct TrieStore { - nodes: HashMap, - roots: HashMap, - next_addr: NodeAddress, -} - -struct NodeEntry { - data: Vec, - children: Vec, - refcount: u32, -} - -impl TrieStore { - fn alloc(&mut self) -> NodeAddress { - let addr = self.next_addr; - self.next_addr += 1; - addr - } - - /// Recursively materialize a `NodeRef` into the store, returning the - /// address of its root and bumping refcounts as we go. - fn insert(&mut self, node_ref: NodeRef) -> NodeAddress { - match node_ref { - NodeRef::Existing(addr) => { - let entry = self - .nodes - .get_mut(&addr) - .unwrap_or_else(|| panic!("NodeRef::Existing({addr}) but node is gone")); - entry.refcount += 1; - addr - } - NodeRef::New(NewNode { data, children }) => { - let child_addrs: Vec<_> = children.into_iter().map(|c| self.insert(c)).collect(); - let addr = self.alloc(); - self.nodes.insert( - addr, - NodeEntry { - data, - children: child_addrs, - refcount: 1, - }, - ); - addr - } - } - } - - /// Drop a reference to `addr`, recursively GC'ing descendants that hit 0. - fn dereference(&mut self, addr: NodeAddress) { - let drop_children = match self.nodes.get_mut(&addr) { - Some(entry) => { - entry.refcount = entry - .refcount - .checked_sub(1) - .unwrap_or_else(|| panic!("refcount underflow on addr {addr}")); - if entry.refcount == 0 { - Some(std::mem::take(&mut entry.children)) - } else { - None - } - } - None => None, - }; - if let Some(children) = drop_children { - self.nodes.remove(&addr); - for child in children { - self.dereference(child); - } - } - } + tries: MultiTreeMap, } impl MemoryDb { - /// Execute a closure with direct read access to the underlying data, - /// holding the read lock for the duration. + /// Run a closure under the data read lock. pub fn with_data(&self, f: F) -> Result where F: FnOnce(&BTreeMap>) -> R, { - let data = self + Ok(f(&*self .data .read() - .map_err(|_| anyhow::anyhow!("RwLock poisoned"))?; - Ok(f(&data)) + .map_err(|_| anyhow::anyhow!("RwLock poisoned"))?)) } /// Reset the memory database pub fn reset(&self, data: BTreeMap>) { - let mut curr = self.data.write().unwrap(); - *curr = data; + *self.data.write().unwrap() = data; } } @@ -216,64 +137,20 @@ impl KVStorage for MemoryDb { } } -impl MultiTreeStore for MemoryDb { - fn insert_tree(&self, _column: Column, key: OpaqueHash, root: NewNode) -> Result<()> { - let mut tries = self - .tries - .write() - .map_err(|_| anyhow::anyhow!("RwLock poisoned"))?; - if tries.roots.contains_key(&key) { - anyhow::bail!( - "insert_tree: tree already present at key 0x{} — caller must dereference_tree first", - hex::encode(key) - ); - } - let addr = tries.insert(NodeRef::New(root)); - tries.roots.insert(key, addr); - Ok(()) +impl MultiTree for MemoryDb { + fn insert_tree(&self, key: OpaqueHash, root: NewNode) -> Result<()> { + self.tries.insert_tree(key, root) } - fn dereference_tree(&self, _column: Column, key: OpaqueHash) -> Result<()> { - let mut tries = self - .tries - .write() - .map_err(|_| anyhow::anyhow!("RwLock poisoned"))?; - if let Some(addr) = tries.roots.remove(&key) { - tries.dereference(addr); - } - Ok(()) + fn dereference_tree(&self, key: OpaqueHash) -> Result<()> { + self.tries.dereference_tree(key) } - fn get_root( - &self, - _column: Column, - key: OpaqueHash, - ) -> Result, Vec)>> { - let tries = self - .tries - .read() - .map_err(|_| anyhow::anyhow!("RwLock poisoned"))?; - let Some(&addr) = tries.roots.get(&key) else { - return Ok(None); - }; - Ok(tries - .nodes - .get(&addr) - .map(|n| (n.data.clone(), n.children.clone()))) + fn get_root(&self, key: OpaqueHash) -> Result, Vec)>> { + self.tries.get_root(key) } - fn get_node( - &self, - _column: Column, - address: NodeAddress, - ) -> Result, Vec)>> { - let tries = self - .tries - .read() - .map_err(|_| anyhow::anyhow!("RwLock poisoned"))?; - Ok(tries - .nodes - .get(&address) - .map(|n| (n.data.clone(), n.children.clone()))) + fn get_node(&self, address: NodeAddress) -> Result, Vec)>> { + self.tries.get_node(address) } } diff --git a/crates/runtime/src/storage/mod.rs b/crates/runtime/src/storage/mod.rs index 830c038c..60e81f28 100644 --- a/crates/runtime/src/storage/mod.rs +++ b/crates/runtime/src/storage/mod.rs @@ -6,8 +6,8 @@ pub use { archive::{Archive, ArchiveStorage}, branch::Branch, commit::{Commit, Operation}, + crypto::merkle::multitree::{MultiTree, NewNode, NodeAddress, NodeRef}, kv::{KVStorage, MemoryDb}, - multitree::{MultiTreeStore, NewNode, NodeAddress, NodeRef}, state::StateStorage, sync::SyncStorage, }; @@ -16,7 +16,6 @@ mod archive; mod branch; mod commit; mod kv; -mod multitree; pub mod root; mod state; pub mod sync; diff --git a/crates/runtime/src/storage/multitree.rs b/crates/runtime/src/storage/multitree.rs deleted file mode 100644 index c94a1b7c..00000000 --- a/crates/runtime/src/storage/multitree.rs +++ /dev/null @@ -1,199 +0,0 @@ -//! Multi-tree (ref-counted Merkle node) storage abstraction. -//! -//! Mirrors parity-db's multitree API so the runtime can drive incremental trie -//! updates against either parity-db or the in-memory shim in [`MemoryDb`], -//! without depending on parity-db directly. - -use crate::storage::Column; -use anyhow::{Result, anyhow}; -use crypto::merkle::trie31; -use score::{OpaqueHash, TrieKey}; - -const ZERO_HASH: OpaqueHash = [0u8; 32]; - -/// Address of a node already persisted in the store. -pub type NodeAddress = u64; - -/// A reference used when building a tree to commit: either a brand-new subtree -/// that must be written, or a pointer to an existing node that the new tree -/// should reuse (and which the store will ref-count automatically). -#[derive(Debug, Clone)] -pub enum NodeRef { - /// A fresh subtree. - New(NewNode), - /// A node already in the store, retained as-is. - Existing(NodeAddress), -} - -/// A newly-built node with its child references. -#[derive(Debug, Clone)] -pub struct NewNode { - /// Encoded node payload (leaf or branch). - pub data: Vec, - /// Child references, in canonical order. - pub children: Vec, -} - -/// Ref-counted Merkle tree storage. -pub trait MultiTreeStore: Send + Sync + 'static { - /// Commit a new tree rooted at `key`. Nodes referenced via - /// [`NodeRef::Existing`] have their refcount bumped automatically. - fn insert_tree(&self, column: Column, key: OpaqueHash, root: NewNode) -> Result<()>; - - /// Drop the tree at `key`. Nodes whose refcount falls to zero are GC'd. - fn dereference_tree(&self, column: Column, key: OpaqueHash) -> Result<()>; - - /// Get the root node (encoded data + immediate children) for the tree at - /// `key`, if present. - fn get_root( - &self, - column: Column, - key: OpaqueHash, - ) -> Result, Vec)>>; - - /// Get a node by its address. - fn get_node( - &self, - column: Column, - address: NodeAddress, - ) -> Result, Vec)>>; - - /// Compute the new state root from `new_state`, reusing unchanged subtrees - /// of the tree at `prev_root`. Writes the new tree into the store and - /// dereferences the old one if it differs. - /// - /// - The tree shape matches `crypto::merkle::trie31` (graypaper D.6), so - /// the returned root is bit-identical to the non-incremental reference. - /// - `new_state` is the full sorted post-diff state. - /// - `dirty_keys` is the sorted set of keys touched by the diff (inserts, - /// updates, removals). Subtrees whose dirty slice is empty are reused - /// via [`NodeRef::Existing`]. - fn apply( - &self, - column: Column, - prev_root: Option, - new_state: &[(TrieKey, &[u8])], - dirty_keys: &[TrieKey], - ) -> Result { - let prev_root_node = match prev_root { - Some(h) if h != ZERO_HASH => self.get_root(column, h)?, - _ => None, - }; - - let (root_ref, new_root) = self.build(column, prev_root_node, new_state, dirty_keys, 0)?; - - // If the tree didn't change there's nothing to write or release; the - // existing tree at `new_root` already holds every node the rebuild - // would have referenced. - if matches!(prev_root, Some(p) if p == new_root) { - return Ok(new_root); - } - - if let Some(NodeRef::New(root_node)) = root_ref { - self.insert_tree(column, new_root, root_node)?; - } - - if let Some(prev) = prev_root - && prev != ZERO_HASH - && prev != new_root - { - self.dereference_tree(column, prev)?; - } - - Ok(new_root) - } - - /// Build the subtree at this position, with `prev_node` being the matching - /// node from the previous tree (if any). Internal helper for [`apply`]. - fn build( - &self, - column: Column, - prev_node: Option<(Vec, Vec)>, - new_keys: &[(TrieKey, &[u8])], - new_dirty: &[TrieKey], - depth: usize, - ) -> Result<(Option, OpaqueHash)> { - if new_keys.is_empty() { - return Ok((None, ZERO_HASH)); - } - - if new_keys.len() == 1 { - let (k, v) = new_keys[0]; - let data = trie31::leaf(k, v).to_vec(); - let hash = crypto::blake2b(&data); - return Ok(( - Some(NodeRef::New(NewNode { - data, - children: vec![], - })), - hash, - )); - } - - // Branch case: partition both keys and dirty by bit at `depth`. - let mut buf = new_keys.to_vec(); - let key_mid = trie31::partition(&mut buf, depth); - let (left_keys, right_keys) = buf.split_at(key_mid); - let dirty_mid = trie31::split_at_bit(new_dirty, depth); - let (left_dirty, right_dirty) = new_dirty.split_at(dirty_mid); - - let (prev_left, prev_right) = match &prev_node { - Some((data, children)) if !trie31::is_leaf(data) => { - trie31::split_branch_children(data, children).ok_or_else(|| { - anyhow!( - "trie node shape mismatch at depth {depth}: children.len()={}", - children.len() - ) - })? - } - _ => (None, None), - }; - - let (l_ref, l_hash) = self.descend(column, prev_left, left_keys, left_dirty, depth + 1)?; - let (r_ref, r_hash) = - self.descend(column, prev_right, right_keys, right_dirty, depth + 1)?; - - let data = trie31::branch(l_hash, r_hash).to_vec(); - let hash = crypto::blake2b(&data); - let mut children = Vec::with_capacity(2); - if let Some(l) = l_ref { - children.push(l); - } - if let Some(r) = r_ref { - children.push(r); - } - Ok((Some(NodeRef::New(NewNode { data, children })), hash)) - } - - /// Recurse into a child of the current branch. Reuses the subtree as-is - /// when no dirty key touches it. Internal helper for [`apply`]. - fn descend( - &self, - column: Column, - prev_addr: Option, - new_keys: &[(TrieKey, &[u8])], - new_dirty: &[TrieKey], - depth: usize, - ) -> Result<(Option, OpaqueHash)> { - if new_dirty.is_empty() { - if let Some(addr) = prev_addr { - let (data, _children) = self - .get_node(column, addr)? - .ok_or_else(|| anyhow!("missing prev node at address {addr}"))?; - let hash = crypto::blake2b(&data); - return Ok((Some(NodeRef::Existing(addr)), hash)); - } - if new_keys.is_empty() { - return Ok((None, ZERO_HASH)); - } - // Fall through: build fresh when caller's dirty tracking is - // over-conservative. - } - - let prev_node = match prev_addr { - Some(addr) => self.get_node(column, addr)?, - None => None, - }; - self.build(column, prev_node, new_keys, new_dirty, depth) - } -} diff --git a/crates/runtime/src/tx/block/mod.rs b/crates/runtime/src/tx/block/mod.rs index bf43b1dd..f0adff44 100644 --- a/crates/runtime/src/tx/block/mod.rs +++ b/crates/runtime/src/tx/block/mod.rs @@ -2,7 +2,7 @@ use crate::{ Storage, - storage::{Branch, Column, Commit, KVStorage, MemoryDb, MultiTreeStore, StateStorage, root}, + storage::{Branch, Column, Commit, KVStorage, MemoryDb, MultiTree, StateStorage, root}, tx, }; use anyhow::Result; @@ -74,13 +74,11 @@ impl TestChain { let dirty = collect_dirty_keys(&commit); self.data.commit(Column::State, commit)?; - // Incrementally rebuild the trie against the new `data` and prime - // the root cache so the next block's `state()` hits it. let prev = (self.current_root != EMPTY_ROOT).then_some(self.current_root); let new_root = self.data.with_data(|data| { let kvs: Vec<(TrieKey, &[u8])> = data.iter().map(|(k, v)| (*k, v.as_slice())).collect(); - self.data.apply(Column::TrieNodes, prev, &kvs, &dirty) + self.data.apply(prev, &kvs, &dirty) })??; root::set(parent, new_root); self.current_root = new_root; @@ -155,12 +153,10 @@ impl TestChain { .header_hash; self.finalized = head; - // Bootstrap the incremental trie. Every key is dirty on the first - // build; the algorithm degenerates to a full retrie. let new_root = self.data.with_data(|data| { let kvs: Vec<(TrieKey, &[u8])> = data.iter().map(|(k, v)| (*k, v.as_slice())).collect(); let dirty: Vec = data.keys().copied().collect(); - self.data.apply(Column::TrieNodes, None, &kvs, &dirty) + self.data.apply(None, &kvs, &dirty) })??; root::set(head, new_root); self.current_root = new_root; @@ -192,14 +188,7 @@ fn collect_dirty_keys(commit: &Commit>) -> Vec { keys } -/// Compute the state root from base data + overlay diff (no base clone). -/// -/// Both base and diff.update are sorted by TrieKey, so we merge-walk them in -/// O(N+M) and skip the final sort. On ties diff wins (it's the newer write); -/// keys present in diff.removal are dropped after selection so removals take -/// precedence over updates of the same key. -/// -/// NOTE: this method overrides the StateStorage::root for zero-copy. +/// Compute the state root from base data + overlay diff via merge-walk. fn handle_root_with_diff( head: OpaqueHash, base: &BTreeMap>, diff --git a/crates/runtime/tests/trie.rs b/crates/runtime/tests/trie.rs deleted file mode 100644 index 5a4c4d60..00000000 --- a/crates/runtime/tests/trie.rs +++ /dev/null @@ -1,130 +0,0 @@ -//! Cross-checks the incremental [`MultiTreeStore::apply`] root against the -//! reference [`crypto::merkle::trie31`] root. - -use crypto::{blake2b, merkle}; -use score::{OpaqueHash, TrieKey}; -use spacejam_runtime::storage::{Column, MemoryDb, MultiTreeStore}; -use std::collections::BTreeMap; - -fn key(seed: u64) -> TrieKey { - let mut k = [0u8; 31]; - let bytes = blake2b(&seed.to_le_bytes()); - k.copy_from_slice(&bytes[..31]); - k -} - -fn value(seed: u64, len: usize) -> Vec { - let mut v = Vec::with_capacity(len); - let mut s = seed; - while v.len() < len { - s = s.wrapping_mul(6364136223846793005).wrapping_add(1); - v.extend_from_slice(&s.to_le_bytes()); - } - v.truncate(len); - v -} - -fn reference_root(state: &BTreeMap>) -> OpaqueHash { - let kvs: Vec<(TrieKey, &[u8])> = state.iter().map(|(k, v)| (*k, v.as_slice())).collect(); - merkle::trie31(&kvs) -} - -fn apply( - db: &MemoryDb, - prev: Option, - state: &BTreeMap>, - dirty: &[TrieKey], -) -> OpaqueHash { - let kvs: Vec<(TrieKey, &[u8])> = state.iter().map(|(k, v)| (*k, v.as_slice())).collect(); - db.apply(Column::TrieNodes, prev, &kvs, dirty) - .expect("apply") -} - -#[test] -fn empty_state() { - let db = MemoryDb::default(); - let root = apply(&db, None, &BTreeMap::new(), &[]); - assert_eq!(root, [0u8; 32]); -} - -#[test] -fn single_leaf() { - let db = MemoryDb::default(); - let mut state = BTreeMap::new(); - state.insert(key(1), value(1, 10)); - let dirty: Vec<_> = state.keys().copied().collect(); - let root = apply(&db, None, &state, &dirty); - assert_eq!(root, reference_root(&state)); -} - -#[test] -fn bootstrap_matches_trie31() { - let db = MemoryDb::default(); - let mut state = BTreeMap::new(); - for i in 0..500u64 { - state.insert(key(i), value(i, 40)); - } - let dirty: Vec<_> = state.keys().copied().collect(); - let root = apply(&db, None, &state, &dirty); - assert_eq!(root, reference_root(&state)); -} - -#[test] -fn incremental_update_matches_trie31() { - let db = MemoryDb::default(); - let mut state = BTreeMap::new(); - for i in 0..500u64 { - state.insert(key(i), value(i, 40)); - } - let dirty0: Vec<_> = state.keys().copied().collect(); - let root0 = apply(&db, None, &state, &dirty0); - assert_eq!(root0, reference_root(&state)); - - let mut dirty = Vec::new(); - for i in 0..5u64 { - let k = key(i); - state.insert(k, value(i + 1000, 50)); - dirty.push(k); - } - for i in 500..503u64 { - let k = key(i); - state.insert(k, value(i, 60)); - dirty.push(k); - } - for i in 100..102u64 { - let k = key(i); - state.remove(&k); - dirty.push(k); - } - dirty.sort(); - - let root1 = apply(&db, Some(root0), &state, &dirty); - assert_eq!(root1, reference_root(&state)); -} - -#[test] -fn many_sequential_diffs_match_trie31() { - let db = MemoryDb::default(); - let mut state = BTreeMap::new(); - for i in 0..200u64 { - state.insert(key(i), value(i, 32)); - } - let dirty: Vec<_> = state.keys().copied().collect(); - let mut root = apply(&db, None, &state, &dirty); - assert_eq!(root, reference_root(&state)); - - for round in 1..=20u64 { - let mut dirty = Vec::new(); - for j in 0..10u64 { - let k = key((round * 7 + j) % 200); - // Offset by 1_000_000 so we never accidentally write back a - // bootstrap value and leave the root unchanged. - state.insert(k, value(round * 100 + j + 1_000_000, 32)); - dirty.push(k); - } - dirty.sort(); - dirty.dedup(); - root = apply(&db, Some(root), &state, &dirty); - assert_eq!(root, reference_root(&state), "round {round}"); - } -} diff --git a/crates/spacejam/src/storage/parity.rs b/crates/spacejam/src/storage/parity.rs index 2cd3eff7..f14570cf 100644 --- a/crates/spacejam/src/storage/parity.rs +++ b/crates/spacejam/src/storage/parity.rs @@ -6,11 +6,13 @@ use parity_db::{ Options, }; use runtime::storage::{ - Column, Commit, KVStorage, MultiTreeStore, NewNode, NodeAddress, NodeRef, Operation, + Column, Commit, KVStorage, MultiTree, NewNode, NodeAddress, NodeRef, Operation, }; use score::{OpaqueHash, TrieKey}; use std::path::PathBuf; +const TRIE_COL: u8 = Column::TrieNodes as u8; + /// The parity database storage pub struct Parity(Db); @@ -51,35 +53,27 @@ impl KVStorage for Parity { } } -impl MultiTreeStore for Parity { - fn insert_tree(&self, column: Column, key: OpaqueHash, root: NewNode) -> Result<()> { +impl MultiTree for Parity { + fn insert_tree(&self, key: OpaqueHash, root: NewNode) -> Result<()> { self.0.commit_changes([( - column as u8, + TRIE_COL, Op::InsertTree(key.to_vec(), to_pd_newnode(root)), )])?; Ok(()) } - fn dereference_tree(&self, column: Column, key: OpaqueHash) -> Result<()> { + fn dereference_tree(&self, key: OpaqueHash) -> Result<()> { self.0 - .commit_changes([(column as u8, Op::DereferenceTree(key.to_vec()))])?; + .commit_changes([(TRIE_COL, Op::DereferenceTree(key.to_vec()))])?; Ok(()) } - fn get_root( - &self, - column: Column, - key: OpaqueHash, - ) -> Result, Vec)>> { - Ok(self.0.get_root(column as u8, key.as_ref())?) + fn get_root(&self, key: OpaqueHash) -> Result, Vec)>> { + Ok(self.0.get_root(TRIE_COL, key.as_ref())?) } - fn get_node( - &self, - column: Column, - address: NodeAddress, - ) -> Result, Vec)>> { - Ok(self.0.get_node(column as u8, address)?) + fn get_node(&self, address: NodeAddress) -> Result, Vec)>> { + Ok(self.0.get_node(TRIE_COL, address)?) } } From ee2c69b0f52bc7baddd66eb8b457dc7201c50001 Mon Sep 17 00:00:00 2001 From: clearloop Date: Fri, 22 May 2026 19:29:29 +0800 Subject: [PATCH 18/18] chore: make clippy happy --- crates/spacejam/src/storage/parity.rs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/crates/spacejam/src/storage/parity.rs b/crates/spacejam/src/storage/parity.rs index f14570cf..2a21e329 100644 --- a/crates/spacejam/src/storage/parity.rs +++ b/crates/spacejam/src/storage/parity.rs @@ -55,10 +55,8 @@ impl KVStorage for Parity { impl MultiTree for Parity { fn insert_tree(&self, key: OpaqueHash, root: NewNode) -> Result<()> { - self.0.commit_changes([( - TRIE_COL, - Op::InsertTree(key.to_vec(), to_pd_newnode(root)), - )])?; + self.0 + .commit_changes([(TRIE_COL, Op::InsertTree(key.to_vec(), to_pd_newnode(root)))])?; Ok(()) }