diff --git a/Cargo.lock b/Cargo.lock index cf319ad4..c3e1e9c0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -24,6 +24,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" dependencies = [ "cfg-if", + "getrandom 0.3.3", "once_cell", "version_check", "zerocopy", @@ -399,9 +400,9 @@ dependencies = [ [[package]] name = "ark-vrf" -version = "0.1.0" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9501da18569b2afe0eb934fb7afd5a247d238b94116155af4dd068f319adfe6d" +checksum = "b722df2194fd06cd069bb3cb29cd4bfce8e023b6cace34d314ee2cf4bd4df9b5" dependencies = [ "ark-bls12-381 0.5.0", "ark-ec 0.5.0", @@ -568,6 +569,17 @@ dependencies = [ "digest", ] +[[package]] +name = "blake2b_simd" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b79834656f71332577234b50bfc009996f7449e0c056884e6a02492ded0ca2f3" +dependencies = [ + "arrayref", + "arrayvec", + "constant_time_eq 0.4.2", +] + [[package]] name = "blake3" version = "1.8.2" @@ -578,7 +590,7 @@ dependencies = [ "arrayvec", "cc", "cfg-if", - "constant_time_eq", + "constant_time_eq 0.3.1", ] [[package]] @@ -778,6 +790,12 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7c74b8349d32d297c9134b8c88677813a227df8f779daa29bfc29c183fe3dca6" +[[package]] +name = "constant_time_eq" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d52eff69cd5e647efe296129160853a42795992097e8af39800e1060caeea9b" + [[package]] name = "core-foundation" version = "0.10.1" @@ -998,7 +1016,7 @@ dependencies = [ [[package]] name = "cranelift-pvm" -version = "0.1.2-pre.1" +version = "0.1.2-pre.8" dependencies = [ "anyhow", "cranelift", @@ -2527,10 +2545,11 @@ checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" [[package]] name = "parity-db" -version = "0.5.4" +version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6985a45b0597d68448dac9db2907f9f72bbaf63fe3383d4ba15f99096c87212f" +checksum = "8b63063d738c6e39a9e29e0821fc7293f1d862e75bac6e39bcea131b44c03ade" dependencies = [ + "ahash", "blake2", "crc32fast", "fs2", @@ -2740,7 +2759,7 @@ dependencies = [ [[package]] name = "pvmc" -version = "0.1.2-pre.1" +version = "0.1.2-pre.8" dependencies = [ "anyhow", "bindgen", @@ -3275,7 +3294,7 @@ dependencies = [ [[package]] name = "serde-jam" -version = "0.1.2-pre.1" +version = "0.1.2-pre.8" dependencies = [ "anyhow", "serde", @@ -3443,7 +3462,7 @@ dependencies = [ [[package]] name = "spacejam" -version = "0.1.2-pre.1" +version = "0.1.2-pre.8" dependencies = [ "anyhow", "async-trait", @@ -3480,7 +3499,7 @@ dependencies = [ [[package]] name = "spacejam-account" -version = "0.1.2-pre.1" +version = "0.1.2-pre.8" dependencies = [ "anyhow", "serde-jam", @@ -3491,7 +3510,7 @@ dependencies = [ [[package]] name = "spacejam-core" -version = "0.1.2-pre.1" +version = "0.1.2-pre.8" dependencies = [ "anyhow", "erased-serde", @@ -3515,11 +3534,13 @@ dependencies = [ "ark-serialize 0.5.0", "ark-vrf", "blake2", + "blake2b_simd", "blake3", "criterion", "ed25519-zebra", "hex", "once_cell", + "parking_lot", "rand 0.8.5", "rayon", "rcgen", @@ -3528,7 +3549,6 @@ dependencies = [ "tiny-keccak", "tracing", "w3f-bls", - "w3f-ring-proof", ] [[package]] @@ -3545,7 +3565,7 @@ dependencies = [ [[package]] name = "spacejam-network" -version = "0.1.2-pre.1" +version = "0.1.2-pre.8" dependencies = [ "anyhow", "clap", @@ -3568,7 +3588,7 @@ dependencies = [ [[package]] name = "spacejam-offchain" -version = "0.1.2-pre.1" +version = "0.1.2-pre.8" dependencies = [ "anyhow", "async-trait", @@ -3584,7 +3604,7 @@ dependencies = [ [[package]] name = "spacejam-rpc" -version = "0.1.2-pre.1" +version = "0.1.2-pre.8" dependencies = [ "anyhow", "jsonrpsee", @@ -3596,7 +3616,7 @@ dependencies = [ [[package]] name = "spacejam-runtime" -version = "0.1.2-pre.1" +version = "0.1.2-pre.8" dependencies = [ "anyhow", "hex", @@ -3617,10 +3637,10 @@ dependencies = [ [[package]] name = "spacejam-service" -version = "0.1.2-pre.1" +version = "0.1.2-pre.8" dependencies = [ "anyhow", - "blake2", + "blake2b_simd", "hex", "serde", "serde-jam", @@ -3631,14 +3651,14 @@ dependencies = [ [[package]] name = "spacejam-spec" -version = "0.1.2-pre.1" +version = "0.1.2-pre.8" dependencies = [ "serde", ] [[package]] name = "spacejam-testing" -version = "0.1.2-pre.1" +version = "0.1.2-pre.8" dependencies = [ "anyhow", "hex", @@ -3668,7 +3688,7 @@ dependencies = [ [[package]] name = "spacejam-testnet" -version = "0.1.2-pre.1" +version = "0.1.2-pre.8" dependencies = [ "anyhow", "clap", @@ -3679,7 +3699,7 @@ dependencies = [ [[package]] name = "spacejam-worker" -version = "0.1.2-pre.1" +version = "0.1.2-pre.8" dependencies = [ "anyhow", "pvm", @@ -3695,7 +3715,7 @@ dependencies = [ [[package]] name = "spacejson" -version = "0.1.2-pre.1" +version = "0.1.2-pre.8" dependencies = [ "anyhow", "hex", @@ -3706,7 +3726,7 @@ dependencies = [ [[package]] name = "spacejson-derive" -version = "0.1.2-pre.1" +version = "0.1.2-pre.8" dependencies = [ "proc-macro2", "quote", @@ -3715,7 +3735,7 @@ dependencies = [ [[package]] name = "spacevm" -version = "0.1.2-pre.1" +version = "0.1.2-pre.8" dependencies = [ "anyhow", "hex", @@ -3729,7 +3749,7 @@ dependencies = [ [[package]] name = "spacevm-export" -version = "0.1.2-pre.1" +version = "0.1.2-pre.8" dependencies = [ "pvm", "pvmc", @@ -4308,9 +4328,9 @@ dependencies = [ [[package]] name = "w3f-pcs" -version = "0.0.2" +version = "0.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fbe7a8d5c914b69392ab3b267f679a2e546fe29afaddce47981772ac71bd02e1" +checksum = "3ea1046a1deb6d26c34ba2d1f1bab4222d695d126502ee765f80b021753cb674" dependencies = [ "ark-ec 0.5.0", "ark-ff 0.5.0", @@ -4323,9 +4343,9 @@ dependencies = [ [[package]] name = "w3f-plonk-common" -version = "0.0.2" +version = "0.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1aca389e494fe08c5c108b512e2328309036ee1c0bc7bdfdb743fef54d448c8c" +checksum = "077db25196f87773d7f0784c0ea5b11f18f38d336fa25c24bc67d7936af05d7a" dependencies = [ "ark-ec 0.5.0", "ark-ff 0.5.0", @@ -4340,9 +4360,9 @@ dependencies = [ [[package]] name = "w3f-ring-proof" -version = "0.0.2" +version = "0.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a639379402ad51504575dbd258740383291ac8147d3b15859bdf1ea48c677de" +checksum = "a3afac5d485a6eed7c1762be4b7b70a6f9b346bd6eebe485f48b7d909a6773f5" dependencies = [ "ark-ec 0.5.0", "ark-ff 0.5.0", diff --git a/Cargo.toml b/Cargo.toml index cbcb13d4..d5aaccb6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -21,7 +21,7 @@ resolver = "1" [workspace.package] edition = "2024" -version = "0.1.2-pre.1" +version = "0.1.2-pre.8" homepage = "https://spacejam.app" repository = "https://github.com/spacejamapp/jade" license = "GPL-3.0" @@ -74,10 +74,11 @@ ark-ed-on-bls12-381-bandersnatch = { version = "0.5", default-features = false } ark-ff = { version = "0.5", default-features = false } ark-std = { version = "0.5", default-features = false } ark-serialize = { version = "0.5", default-features = false } -ark-vrf = { version = "0.1.0", features = ["bandersnatch", "ring", "parallel"] } +ark-vrf = { version = "=0.2.1", features = ["bandersnatch", "ring", "parallel"] } async-trait = "0.1.89" bindgen = "0.72" -blake2 = { version = "0.10.6", default-features = false } +blake2 = "0.10.6" +blake2b_simd = "1.0.4" blake3 = "1.8.2" cc = "1.2" clap = "4.5.47" @@ -102,7 +103,8 @@ libc = "0.2" log = "0.4.28" once_cell = "1.21.3" paste = "1.0.15" -parity-db = "0.5.4" +parity-db = "0.5.5" +parking_lot = "0.12" postcard = "1.1.3" proc-macro2 = "1.0.101" prometheus = { version = "0.23.1", package = "prometheus-client" } @@ -112,7 +114,6 @@ rand = "0.8.5" rayon = "1.11.0" rcgen = { version = "0.13.2", default-features = false, features = ["crypto", "pem", "ring"] } reed-solomon = { package = "reed-solomon-simd", version = "3.0.1" } -ring = { package = "w3f-ring-proof", version = "0.0.2", default-features = false } rustls = { version = "0.23.31", default-features = false, features = ["std", "logging", "tls12", "ring"] } rustls-webpki = { version = "0.102.8", default-features = false, features = ["std", "ring"] } serde = { version = "1.0.219", features = ["derive"], default-features = false } diff --git a/crates/core/account/src/registry.rs b/crates/core/account/src/registry.rs index 47809fd7..6da2500d 100644 --- a/crates/core/account/src/registry.rs +++ b/crates/core/account/src/registry.rs @@ -1,7 +1,7 @@ //! Account registry use crate::Account; -use score::{OpaqueHash, ServiceId, TrieKey, service::ServiceAccount}; +use score::{Gas, OpaqueHash, ServiceId, TrieKey, service::ServiceAccount}; use std::collections::{BTreeMap, BTreeSet}; /// Account registry @@ -12,6 +12,9 @@ pub trait Accounts: Clone + Send + Sync + 'static { /// Get the code hash of an account fn code_hash(&self, index: u32) -> Option; + /// Get the minimum gas required to invoke the accumulate entry-point + fn min_acc_gas(&self, index: u32) -> Option; + /// Create a new account fn upsert(&mut self, index: u32, account: impl Account); @@ -23,6 +26,14 @@ pub trait Accounts: Clone + Send + Sync + 'static { self.get(index).is_some() } + /// Check if a lookup is providable + fn is_providable(&mut self, index: ServiceId, hash: OpaqueHash, len: u32) -> bool { + let Some(account) = self.get(index) else { + return false; + }; + matches!(account.lookup(hash, len), Some(Some(slots)) if slots.is_empty()) + } + /// Remove an account from the registry fn remove(&mut self, index: u32); @@ -59,6 +70,10 @@ impl Accounts for BTreeMap { Some(self.get(&index)?.info.code) } + fn min_acc_gas(&self, index: u32) -> Option { + Some(self.get(&index)?.info.accumulate) + } + fn upsert(&mut self, index: u32, account: impl Account) { self.insert(index, account.account()); } diff --git a/crates/core/service/Cargo.toml b/crates/core/service/Cargo.toml index 03cec4fa..f41d252e 100644 --- a/crates/core/service/Cargo.toml +++ b/crates/core/service/Cargo.toml @@ -17,7 +17,7 @@ serde.workspace = true spec.workspace = true # optional dependencies -blake2 = { workspace = true, optional = true } +blake2b_simd = { workspace = true, optional = true } hex = { workspace = true, optional = true } serde_json = { workspace = true, features = ["alloc"], optional = true } spacejson = { workspace = true, optional = true } @@ -26,14 +26,13 @@ spacejson = { workspace = true, optional = true } default = ["std", "tiny"] std = [ "anyhow/std", - "blake2?/std", "codec/std", "hex/std", "serde/std", "serde_json/std", "spacejson?/std", ] -blake2 = ["dep:blake2"] +blake2 = ["dep:blake2b_simd"] json = ["hex", "serde_json", "spacejson", "spacejson/codec"] tiny = ["spec/tiny"] full = ["spec/full"] diff --git a/crates/core/service/src/lib.rs b/crates/core/service/src/lib.rs index 4ddf1d73..7a218dad 100644 --- a/crates/core/service/src/lib.rs +++ b/crates/core/service/src/lib.rs @@ -75,9 +75,12 @@ pub type BlsPublic = [u8; 144]; #[cfg(feature = "blake2")] /// Compute the BLAKE2b 256-bit hash of a given input. pub fn blake2b(input: &[u8]) -> [u8; 32] { - use blake2::{Blake2b, Digest, digest::consts::U32}; - - let mut hasher = Blake2b::::new(); - hasher.update(input); - hasher.finalize().into() + let hash = blake2b_simd::Params::new() + .hash_length(32) + .to_state() + .update(input) + .finalize(); + let mut out = [0u8; 32]; + out.copy_from_slice(hash.as_bytes()); + out } diff --git a/crates/core/src/safrole/mod.rs b/crates/core/src/safrole/mod.rs index 6c6f571f..99906c9c 100644 --- a/crates/core/src/safrole/mod.rs +++ b/crates/core/src/safrole/mod.rs @@ -77,11 +77,11 @@ impl Safrole { let curr_slot_phase = slot % crate::EPOCH_LENGTH; let prev_slot_phase = tau % crate::EPOCH_LENGTH; - // Return true if: - // 1. Different epochs (e' ≠ e) - // 2. Previous slot not before submission period (m ≥ Y) - // 3. Current slot not after submission period (m' < Y) - // 4. Accumulator not full (|gamma_a| ≠ E) + // Return true if all of: + // 1. Same epoch (e' = e) + // 2. Previous slot before submission period (m < Y) + // 3. Current slot at or after submission period (m' ≥ Y) + // 4. Accumulator full (|gamma_a| = E) curr_epoch == prev_epoch && prev_slot_phase < crate::TICKET_SUBMISSION_PERIOD && curr_slot_phase >= crate::TICKET_SUBMISSION_PERIOD diff --git a/crates/core/src/statistic/mod.rs b/crates/core/src/statistic/mod.rs index 9321efba..fdd4d92f 100644 --- a/crates/core/src/statistic/mod.rs +++ b/crates/core/src/statistic/mod.rs @@ -108,9 +108,7 @@ impl Statistics { for reporter in reporters.iter() { let Some(index) = validators.iter().position(|v| v == reporter) else { continue; - // anyhow::bail!("reporter is invalid"); }; - self.vals_current[index].guarantees += 1; } Ok(()) diff --git a/crates/crypto/Cargo.toml b/crates/crypto/Cargo.toml index 65e92972..cdf04894 100644 --- a/crates/crypto/Cargo.toml +++ b/crates/crypto/Cargo.toml @@ -9,14 +9,14 @@ ark-ec = { workspace = true, optional = true } ark-ed-on-bls12-381-bandersnatch = { workspace = true, optional = true } ark-serialize = { workspace = true, optional = true } ark-vrf = { workspace = true, optional = true } -blake2 = { workspace = true, optional = true } +blake2b_simd = { workspace = true, optional = true } blake3 = { workspace = true, optional = true } ed25519-zebra = { workspace = true, optional = true } once_cell = { workspace = true, optional = true } +parking_lot = { workspace = true, optional = true } rand = { workspace = true, optional = true } rayon = { workspace = true, optional = true } rcgen = { workspace = true, optional = true } -ring = { workspace = true, optional = true } tiny-keccak = { workspace = true, optional = true } w3f-bls = { workspace = true, optional = true } tracing.workspace = true @@ -24,26 +24,25 @@ tracing.workspace = true [features] default = [] full = [] -blake2 = ["dep:blake2"] +blake2 = ["dep:blake2b_simd"] blake3 = ["dep:blake3"] bls = ["w3f-bls"] ed25519 = ["anyhow", "ed25519-zebra", "rand", "rayon"] keccak = ["tiny-keccak"] -merkle = ["blake2", "keccak", "rayon"] +merkle = ["anyhow", "blake2", "keccak", "parking_lot", "rayon"] rand = ["dep:rand"] shuffle = ["blake2"] tls = ["rcgen", "ed25519-zebra/pkcs8"] vrf = [ - "anyhow", "ark-vrf", "ark-ec", "ark-ed-on-bls12-381-bandersnatch", "ark-serialize", "once_cell", - "ring", ] [dev-dependencies] +blake2.workspace = true criterion = { version = "0.5", default-features = false } hex = { workspace = true, features = ["std"] } serde.workspace = true @@ -54,6 +53,16 @@ name = "ed25519" harness = false required-features = ["ed25519"] +[[bench]] +name = "blake2b" +harness = false +required-features = ["blake2"] + +[[bench]] +name = "ring_vrf" +harness = false +required-features = ["vrf"] + [build-dependencies] ark-serialize.workspace = true ark-vrf.workspace = true diff --git a/crates/crypto/benches/blake2b.rs b/crates/crypto/benches/blake2b.rs new file mode 100644 index 00000000..d9aecc42 --- /dev/null +++ b/crates/crypto/benches/blake2b.rs @@ -0,0 +1,35 @@ +//! Microbench: BLAKE2b-256 across input sizes representative of the hot paths +//! (trie31 leaves/branches at 32–64B, header/preimage at 100–1KiB, work bundles +//! at 8–64KiB). Compares the in-tree blake2b_simd impl against the RustCrypto +//! `blake2` crate side by side. + +use criterion::{black_box, criterion_group, criterion_main, BenchmarkId, Criterion, Throughput}; + +fn simd(input: &[u8]) -> [u8; 32] { + spacejam_crypto::blake2b(input) +} + +fn rustcrypto(input: &[u8]) -> [u8; 32] { + use blake2::{digest::consts::U32, Blake2b, Digest}; + let mut hasher = Blake2b::::new(); + hasher.update(input); + hasher.finalize().into() +} + +fn bench(c: &mut Criterion) { + let mut group = c.benchmark_group("blake2b"); + for &n in &[32usize, 64, 128, 512, 1024, 8192, 65536] { + let input = vec![0xa5u8; n]; + group.throughput(Throughput::Bytes(n as u64)); + group.bench_with_input(BenchmarkId::new("simd", n), &input, |b, input| { + b.iter(|| simd(black_box(input))); + }); + group.bench_with_input(BenchmarkId::new("rustcrypto", n), &input, |b, input| { + b.iter(|| rustcrypto(black_box(input))); + }); + } + group.finish(); +} + +criterion_group!(benches, bench); +criterion_main!(benches); diff --git a/crates/crypto/benches/ring_vrf.rs b/crates/crypto/benches/ring_vrf.rs new file mode 100644 index 00000000..b3eda988 --- /dev/null +++ b/crates/crypto/benches/ring_vrf.rs @@ -0,0 +1,69 @@ +//! Microbench: single ring-vrf verify vs batched verify, swept across batch sizes. + +use criterion::{black_box, criterion_group, criterion_main, BenchmarkId, Criterion, Throughput}; +use spacejam_crypto::{ + ring::RING_SIZE, + vrf::{KeyPair, Verifier}, +}; + +/// Generate `n` deterministic (message, signature) triples signed under distinct ring positions. +fn gen_items(n: usize) -> (Verifier, Vec<(Vec, [u8; 784])>) { + assert!( + n <= RING_SIZE, + "batch size {n} exceeds RING_SIZE {RING_SIZE}" + ); + let ring: Vec = (0..RING_SIZE) + .map(|i| KeyPair::from([i as u8; 32])) + .collect(); + let pks = ring.iter().map(|k| k.public().unwrap()).collect::>(); + let pkeys = ring.iter().map(|k| k.public).collect::>(); + let verifier = Verifier::new(pkeys); + + let items: Vec<(Vec, [u8; 784])> = (0..n) + .map(|i| { + let msg = format!("benchmark ring-vrf message {i:08}").into_bytes(); + let sig = ring[i].ring_sign(pks.clone(), &msg, &[]).unwrap(); + (msg, sig) + }) + .collect(); + + (verifier, items) +} + +fn bench(c: &mut Criterion) { + let mut group = c.benchmark_group("ring_vrf"); + group.sample_size(20); + + for &n in [1usize, 3, 8, 16, 32, 64] + .iter() + .filter(|&&n| n <= RING_SIZE) + { + let (verifier, items) = gen_items(n); + group.throughput(Throughput::Elements(n as u64)); + + group.bench_with_input(BenchmarkId::new("single", n), &items, |b, items| { + b.iter(|| { + for (m, s) in items { + verifier.ring_vrf_verify(black_box(m), &[], s).unwrap(); + } + }) + }); + + group.bench_with_input(BenchmarkId::new("batch", n), &items, |b, items| { + b.iter(|| { + verifier + .ring_vrf_verify_batch( + items + .iter() + .map(|(m, s)| (m.as_slice(), [].as_slice(), s.as_slice())), + ) + .unwrap() + }) + }); + } + + group.finish(); +} + +criterion_group!(benches, bench); +criterion_main!(benches); diff --git a/crates/crypto/src/lib.rs b/crates/crypto/src/lib.rs index cc670f28..b985a649 100644 --- a/crates/crypto/src/lib.rs +++ b/crates/crypto/src/lib.rs @@ -10,11 +10,14 @@ pub mod vrf; #[cfg(feature = "blake2")] /// Compute the BLAKE2b 256-bit hash of a given input. pub fn blake2b(input: &[u8]) -> [u8; 32] { - use blake2::{digest::consts::U32, Blake2b, Digest}; - - let mut hasher = Blake2b::::new(); - hasher.update(input); - hasher.finalize().into() + let hash = blake2b_simd::Params::new() + .hash_length(32) + .to_state() + .update(input) + .finalize(); + let mut out = [0u8; 32]; + out.copy_from_slice(hash.as_bytes()); + out } #[cfg(feature = "blake3")] diff --git a/crates/crypto/src/merkle/mod.rs b/crates/crypto/src/merkle/mod.rs index bbff16b8..2618ec57 100644 --- a/crates/crypto/src/merkle/mod.rs +++ b/crates/crypto/src/merkle/mod.rs @@ -4,9 +4,11 @@ mod binary; pub mod mmr; +pub mod multitree; mod trie; pub mod trie31; pub use binary::{broot, hroot, kroot, root, tree, MerkleTree}; +pub use multitree::{NewNode, NodeAddress, NodeRef}; pub use trie::merkle as trie; pub use trie31::trie as trie31; diff --git a/crates/crypto/src/merkle/multitree.rs b/crates/crypto/src/merkle/multitree.rs new file mode 100644 index 00000000..fa4a3dc0 --- /dev/null +++ b/crates/crypto/src/merkle/multitree.rs @@ -0,0 +1,284 @@ +//! Ref-counted Merkle tree primitives and the incremental builder. + +use crate::{blake2b, merkle::trie31}; +use anyhow::{anyhow, Result}; + +const ZERO_HASH: [u8; 32] = [0u8; 32]; + +/// Switch from sequential to parallel descent when a build step covers this +/// many keys or more. Mirrors `trie31::PARALLEL_THRESHOLD`. +const PARALLEL_THRESHOLD: usize = 64; + +/// Address of a node already persisted in the store. +pub type NodeAddress = u64; + +/// A persisted node: payload + child addresses. +pub type PersistedNode = (Vec, Vec); + +/// Reference to a node when building a tree: either a fresh subtree to write, +/// or a pointer to an existing node to reuse. +#[derive(Debug, Clone)] +pub enum NodeRef { + /// A fresh subtree. + New(NewNode), + /// A node already in the store, retained as-is. + Existing(NodeAddress), +} + +/// A newly-built node with its child references. +#[derive(Debug, Clone)] +pub struct NewNode { + /// Encoded node payload (leaf or branch). + pub data: Vec, + /// Child references, in canonical order. + pub children: Vec, +} + +/// Ref-counted Merkle tree storage. Implementors provide the IO surface; +/// the `apply` default impl owns the recursion, leaf/branch encoding, and +/// subtree-reuse logic. +pub trait MultiTree: Sync { + /// Look up a node by its persisted address. + fn get_node(&self, address: NodeAddress) -> Result>; + + /// Look up the root node of the tree identified by `hash`. + fn get_root(&self, hash: [u8; 32]) -> Result>; + + /// Persist a new tree under `hash`, bumping refcounts on existing children. + fn insert_tree(&self, hash: [u8; 32], root: NewNode) -> Result<()>; + + /// Release the tree at `hash`, GC'ing nodes whose refcount falls to zero. + fn dereference_tree(&self, hash: [u8; 32]) -> Result<()>; + + /// Build a new tree from `new_state`, reusing unchanged subtrees of `prev_root`. + fn apply( + &self, + prev_root: Option<[u8; 32]>, + new_state: &[([u8; 31], &[u8])], + dirty_keys: &[[u8; 31]], + ) -> Result<[u8; 32]> { + let prev_root_node = match prev_root { + Some(h) if h != ZERO_HASH => self.get_root(h)?, + _ => None, + }; + + let (root_ref, new_root) = self.build(prev_root_node, new_state, dirty_keys, 0)?; + + if matches!(prev_root, Some(p) if p == new_root) { + return Ok(new_root); + } + + if let Some(NodeRef::New(root_node)) = root_ref { + self.insert_tree(new_root, root_node)?; + } + + if let Some(prev) = prev_root.filter(|&p| p != ZERO_HASH && p != new_root) { + self.dereference_tree(prev)?; + } + + Ok(new_root) + } + + fn build( + &self, + prev_node: Option, + new_keys: &[([u8; 31], &[u8])], + new_dirty: &[[u8; 31]], + depth: usize, + ) -> Result<(Option, [u8; 32])> { + if new_keys.is_empty() { + return Ok((None, ZERO_HASH)); + } + + if new_keys.len() == 1 { + let (k, v) = new_keys[0]; + let data = trie31::leaf(k, v).to_vec(); + let hash = blake2b(&data); + return Ok(( + Some(NodeRef::New(NewNode { + data, + children: vec![], + })), + hash, + )); + } + + let mut buf = new_keys.to_vec(); + let key_mid = trie31::partition(&mut buf, depth); + let (left_keys, right_keys) = buf.split_at(key_mid); + let dirty_mid = trie31::split_at_bit(new_dirty, depth); + let (left_dirty, right_dirty) = new_dirty.split_at(dirty_mid); + let (prev_left, prev_right) = match &prev_node { + Some((data, children)) if !trie31::is_leaf(data) => { + trie31::split_branch_children(data, children).ok_or_else(|| { + anyhow!( + "trie node shape mismatch at depth {depth}: children.len()={}", + children.len() + ) + })? + } + _ => (None, None), + }; + + let left = || self.descend(prev_left, left_keys, left_dirty, depth + 1); + let right = || self.descend(prev_right, right_keys, right_dirty, depth + 1); + let (l, r) = if new_keys.len() >= PARALLEL_THRESHOLD { + rayon::join(left, right) + } else { + (left(), right()) + }; + + let (l_ref, l_hash) = l?; + let (r_ref, r_hash) = r?; + let data = trie31::branch(l_hash, r_hash).to_vec(); + let hash = blake2b(&data); + let children: Vec = [l_ref, r_ref].into_iter().flatten().collect(); + Ok((Some(NodeRef::New(NewNode { data, children })), hash)) + } + + fn descend( + &self, + prev_addr: Option, + new_keys: &[([u8; 31], &[u8])], + new_dirty: &[[u8; 31]], + depth: usize, + ) -> Result<(Option, [u8; 32])> { + if new_dirty.is_empty() { + if let Some(addr) = prev_addr { + let (data, _) = self + .get_node(addr)? + .ok_or_else(|| anyhow!("missing prev node at address {addr}"))?; + let hash = blake2b(&data); + return Ok((Some(NodeRef::Existing(addr)), hash)); + } + if new_keys.is_empty() { + return Ok((None, ZERO_HASH)); + } + } + + let prev_node = match prev_addr { + Some(addr) => self.get_node(addr)?, + None => None, + }; + self.build(prev_node, new_keys, new_dirty, depth) + } +} + +/// In-memory ref-counted node store. Used as the multitree backend for +/// `MemoryDb` and tests; mirrors the contract of parity-db's multitree column +/// without disk persistence. +#[derive(Default)] +pub struct MultiTreeMap { + inner: parking_lot::Mutex, +} + +#[derive(Default)] +struct Inner { + nodes: std::collections::HashMap, + roots: std::collections::HashMap<[u8; 32], NodeAddress>, + next_addr: NodeAddress, +} + +struct NodeEntry { + data: Vec, + children: Vec, + refcount: u32, +} + +impl Inner { + fn alloc(&mut self) -> NodeAddress { + let addr = self.next_addr; + self.next_addr += 1; + addr + } + + fn insert(&mut self, node_ref: NodeRef) -> anyhow::Result { + match node_ref { + NodeRef::Existing(addr) => { + let Some(entry) = self.nodes.get_mut(&addr) else { + anyhow::bail!("NodeRef::Existing({addr}) but node is gone"); + }; + + entry.refcount += 1; + Ok(addr) + } + NodeRef::New(NewNode { data, children }) => { + let child_addrs: Vec<_> = children + .into_iter() + .map(|c| self.insert(c)) + .collect::>>()?; + let addr = self.alloc(); + self.nodes.insert( + addr, + NodeEntry { + data, + children: child_addrs, + refcount: 1, + }, + ); + Ok(addr) + } + } + } + + fn dereference(&mut self, addr: NodeAddress) -> anyhow::Result<()> { + let drop_children = match self.nodes.get_mut(&addr) { + Some(entry) => { + entry.refcount = entry + .refcount + .checked_sub(1) + .ok_or_else(|| anyhow!("refcount underflow on addr {addr}"))?; + (entry.refcount == 0).then(|| std::mem::take(&mut entry.children)) + } + None => None, + }; + if let Some(children) = drop_children { + self.nodes.remove(&addr); + for child in children { + self.dereference(child)?; + } + } + + Ok(()) + } +} + +impl MultiTree for MultiTreeMap { + fn insert_tree(&self, key: [u8; 32], root: NewNode) -> Result<()> { + let mut inner = self.inner.lock(); + if inner.roots.contains_key(&key) { + anyhow::bail!("insert_tree: tree already present — caller must dereference_tree first"); + } + let addr = inner.insert(NodeRef::New(root))?; + inner.roots.insert(key, addr); + Ok(()) + } + + fn dereference_tree(&self, key: [u8; 32]) -> Result<()> { + let mut inner = self.inner.lock(); + if let Some(addr) = inner.roots.remove(&key) { + inner.dereference(addr)?; + } + Ok(()) + } + + fn get_root(&self, key: [u8; 32]) -> Result> { + let inner = self.inner.lock(); + let Some(&addr) = inner.roots.get(&key) else { + return Ok(None); + }; + Ok(inner + .nodes + .get(&addr) + .map(|n| (n.data.clone(), n.children.clone()))) + } + + fn get_node(&self, address: NodeAddress) -> Result> { + Ok(self + .inner + .lock() + .nodes + .get(&address) + .map(|n| (n.data.clone(), n.children.clone()))) + } +} diff --git a/crates/crypto/src/merkle/trie31.rs b/crates/crypto/src/merkle/trie31.rs index 0b0986c0..636edded 100644 --- a/crates/crypto/src/merkle/trie31.rs +++ b/crates/crypto/src/merkle/trie31.rs @@ -1,28 +1,24 @@ -//! A space-efficient trie for storing key-value pairs. -//! Implements binary Patricia Merkle Trie as described in the graypaper appendix D. +//! Binary Patricia Merkle Trie (graypaper appendix D). use crate::blake2b; const PARALLEL_THRESHOLD: usize = 64; -/// Compute the Merkle root of a set of key-value pairs. (D.6) +/// Merkle root of a sorted key-value set (GP D.6). pub fn trie(kvs: &[([u8; 31], &[u8])]) -> [u8; 32] { let mut buf = kvs.to_vec(); merkle(&mut buf, 0) } -/// Compute the Merkle root of a set of key-value pairs with specified depth. (D.6) fn merkle(kvs: &mut [([u8; 31], &[u8])], depth: usize) -> [u8; 32] { if kvs.is_empty() { return [0; 32]; } - if kvs.len() == 1 { let (k, v) = kvs[0]; return blake2b(&leaf(k, v)); } - // In-place partition: entries with bit=0 (left) before entries with bit=1 (right) let len = kvs.len(); let mid = partition(kvs, depth); let (left, right) = kvs.split_at_mut(mid); @@ -34,9 +30,32 @@ fn merkle(kvs: &mut [([u8; 31], &[u8])], depth: usize) -> [u8; 32] { blake2b(&branch(l_hash, r_hash)) } -/// Partition `kvs` in-place so that entries with bit 0 at `depth` come first. -/// Returns the index of the first "right" (bit=1) entry. -fn partition(kvs: &mut [([u8; 31], &[u8])], depth: usize) -> usize { +/// Partition index for an already-ordered key slice — linear scan, preserves order. +pub fn split_at_bit(keys: &[[u8; 31]], depth: usize) -> usize { + keys.iter() + .position(|k| bit(k, depth)) + .unwrap_or(keys.len()) +} + +/// Recover `(left, right)` child slots from a branch payload and its stored +/// child addresses; `None` if the empty-side encoding doesn't match the count. +pub fn split_branch_children( + data: &[u8], + children: &[T], +) -> Option<(Option, Option)> { + let l_empty = data[0] == 0 && data[1..32].iter().all(|&b| b == 0); + let r_empty = data[32..64].iter().all(|&b| b == 0); + match (l_empty, r_empty, children.len()) { + (false, false, 2) => Some((Some(children[0]), Some(children[1]))), + (true, false, 1) => Some((None, Some(children[0]))), + (false, true, 1) => Some((Some(children[0]), None)), + (true, true, 0) => Some((None, None)), + _ => None, + } +} + +/// In-place partition of `kvs` so bit-0 entries at `depth` come first; returns the split index. +pub fn partition(kvs: &mut [([u8; 31], &[u8])], depth: usize) -> usize { let mut left_end = 0; for i in 0..kvs.len() { if !bit(&kvs[i].0, depth) { @@ -47,40 +66,37 @@ fn partition(kvs: &mut [([u8; 31], &[u8])], depth: usize) -> usize { left_end } -/// Branch encoding -fn branch(l: [u8; 32], r: [u8; 32]) -> [u8; 64] { +/// True when the payload's high bit of byte 0 is set (leaf marker). +pub fn is_leaf(data: &[u8]) -> bool { + data.first().is_some_and(|b| b & 0x80 != 0) +} + +/// Branch encoding (GP D.6). +pub fn branch(l: [u8; 32], r: [u8; 32]) -> [u8; 64] { let mut encoded = [0u8; 64]; - encoded[0] = l[0] & 0x7F; // 0b01111111 + encoded[0] = l[0] & 0x7F; encoded[1..32].copy_from_slice(&l[1..]); encoded[32..64].copy_from_slice(&r); encoded } -/// Leaf encoding -fn leaf(k: [u8; 31], v: &[u8]) -> [u8; 64] { +/// Leaf encoding (GP D.6). +pub fn leaf(k: [u8; 31], v: &[u8]) -> [u8; 64] { let mut encoded = [0u8; 64]; - if v.len() <= 32 { - // 0x80 = 0b10000000, 0x3F = 0b00111111 encoded[0] = 0x80 | (v.len() as u8 & 0x3F); encoded[1..32].copy_from_slice(&k); encoded[32..(32 + v.len())].copy_from_slice(v); - - if v.len() < 32 { - encoded[(32 + v.len())..64].fill(0); - } } else { - encoded[0] = 0xC0; // 0b11000000 + encoded[0] = 0xC0; encoded[1..32].copy_from_slice(&k); encoded[32..64].copy_from_slice(&blake2b(v)); } - encoded } -/// Get the bit at the specified position in the key -/// For 31-byte keys, we have 248 bits (31*8) -fn bit(k: &[u8; 31], depth: usize) -> bool { +/// Bit at `depth` (MSB-first) of a 248-bit key. +pub fn bit(k: &[u8; 31], depth: usize) -> bool { let byte_idx = depth / 8; let bit_idx = 7 - (depth % 8); (k[byte_idx] & (1 << bit_idx)) != 0 diff --git a/crates/crypto/src/vrf.rs b/crates/crypto/src/vrf.rs index 96fefdf7..08c12d67 100644 --- a/crates/crypto/src/vrf.rs +++ b/crates/crypto/src/vrf.rs @@ -298,6 +298,35 @@ impl Verifier { Ok(vrf_output_hash) } + /// Batched anonymous VRF signature verification. + /// + /// Collapses N independent ring verifies into one randomized pairing check, + /// amortizing the dominant cost across the batch. On success, returns the + /// per-item VRF output hashes in iteration order. On failure, the entire + /// batch is rejected without per-item attribution. + pub fn ring_vrf_verify_batch<'a>( + &self, + items: impl IntoIterator, + ) -> anyhow::Result> { + let verifier_key = RING_CTX.verifier_key_from_commitment(self.commitment.clone()); + let ring_verifier = RING_CTX.verifier(verifier_key); + let mut batch = ark_vrf::ring::BatchVerifier::new(ring_verifier); + + let mut outputs = Vec::new(); + for (vrf_input_data, aux_data, signature) in items { + let signature = RingVrfSignature::deserialize_compressed_unchecked(signature)?; + let input = Input::new(vrf_input_data).ok_or(anyhow::anyhow!("Invalid input"))?; + let output_hash: [u8; 32] = signature.output.hash()[..32].try_into()?; + batch.push(input, signature.output, aux_data, &signature.proof); + outputs.push(output_hash); + } + + batch + .verify() + .map_err(|e| anyhow::anyhow!("Ring batch verification failure: {:?}", e))?; + Ok(outputs) + } + /// Non-Anonymous VRF signature verification. pub fn ietf_vrf_verify( &self, diff --git a/crates/crypto/tests/multitree.rs b/crates/crypto/tests/multitree.rs new file mode 100644 index 00000000..239cc2ab --- /dev/null +++ b/crates/crypto/tests/multitree.rs @@ -0,0 +1,60 @@ +//! `MultiTree::apply` produces the same root as `trie31::trie`, both from +//! scratch and after an incremental diff against a prior tree. +#![cfg(feature = "merkle")] + +use spacejam_crypto::{ + blake2b, + merkle::{ + multitree::{MultiTree, MultiTreeMap}, + trie31, + }, +}; + +fn key(seed: u64) -> [u8; 31] { + let mut k = [0u8; 31]; + k.copy_from_slice(&blake2b(&seed.to_le_bytes())[..31]); + k +} + +#[test] +fn from_scratch_matches_trie31() { + let store = MultiTreeMap::default(); + let pairs: Vec<([u8; 31], Vec)> = (0..500u64) + .map(|i| (key(i), i.to_le_bytes().repeat(5))) + .collect(); + let kvs: Vec<([u8; 31], &[u8])> = pairs.iter().map(|(k, v)| (*k, v.as_slice())).collect(); + let dirty: Vec<[u8; 31]> = kvs.iter().map(|(k, _)| *k).collect(); + + let root = store.apply(None, &kvs, &dirty).expect("apply"); + assert_eq!(root, trie31::trie(&kvs)); +} + +#[test] +fn incremental_matches_trie31() { + let store = MultiTreeMap::default(); + let mut pairs: Vec<([u8; 31], Vec)> = (0..500u64) + .map(|i| (key(i), i.to_le_bytes().repeat(5))) + .collect(); + pairs.sort_by_key(|p| p.0); + let kvs: Vec<([u8; 31], &[u8])> = pairs.iter().map(|(k, v)| (*k, v.as_slice())).collect(); + let dirty0: Vec<[u8; 31]> = kvs.iter().map(|(k, _)| *k).collect(); + + let root0 = store.apply(None, &kvs, &dirty0).expect("round 0"); + assert_eq!(root0, trie31::trie(&kvs)); + + let mut dirty1 = Vec::new(); + for i in 0..5u64 { + let k = key(i); + let v = (i + 1_000).to_le_bytes().repeat(7); + if let Some(p) = pairs.iter_mut().find(|p| p.0 == k) { + p.1 = v; + } + dirty1.push(k); + } + dirty1.sort(); + pairs.sort_by_key(|p| p.0); + let kvs1: Vec<([u8; 31], &[u8])> = pairs.iter().map(|(k, v)| (*k, v.as_slice())).collect(); + + let root1 = store.apply(Some(root0), &kvs1, &dirty1).expect("round 1"); + assert_eq!(root1, trie31::trie(&kvs1)); +} diff --git a/crates/crypto/tests/vrf.rs b/crates/crypto/tests/vrf.rs index 0a36ea8f..0cf9cabd 100644 --- a/crates/crypto/tests/vrf.rs +++ b/crates/crypto/tests/vrf.rs @@ -24,7 +24,7 @@ fn test_vrf() -> anyhow::Result<()> { let prover_key_index = 3; // NOTE: any key can be replaced with the padding point - let padding_point = Public::from(RingProofParams::padding_point()); + let padding_point = Public(RingProofParams::padding_point()); ring[2] = padding_point; ring[5] = padding_point; @@ -123,6 +123,43 @@ fn serde_vrf_signature() -> anyhow::Result<()> { Ok(()) } +#[test] +fn ring_vrf_verify_batch_matches_single() -> anyhow::Result<()> { + let ring: Vec<_> = (0..RING_SIZE) + .map(|i| KeyPair::from([i as u8; 32])) + .collect(); + let keys = ring + .iter() + .map(|k| k.public()) + .collect::>>()?; + let pkeys = ring.iter().map(|k| k.public).collect::>(); + let verifier = Verifier::new(pkeys); + + let cases: Vec<(usize, &[u8])> = vec![(0, b"alpha"), (1, b"beta"), (3, b"gamma")]; + let ad: &[u8] = b""; + + let sigs: Vec<[u8; 784]> = cases + .iter() + .map(|(signer, msg)| ring[*signer].ring_sign(keys.clone(), msg, ad)) + .collect::>()?; + + let expected: Vec<[u8; 32]> = cases + .iter() + .zip(sigs.iter()) + .map(|((_, msg), sig)| verifier.ring_vrf_verify(msg, ad, sig)) + .collect::>()?; + + let got = verifier.ring_vrf_verify_batch( + cases + .iter() + .zip(sigs.iter()) + .map(|((_, msg), sig)| (*msg, ad, sig.as_slice())), + )?; + + assert_eq!(got, expected); + Ok(()) +} + #[test] fn vrf_output_consistency() -> anyhow::Result<()> { let ring: Vec<_> = (0..RING_SIZE) diff --git a/crates/runtime/src/account/registry.rs b/crates/runtime/src/account/registry.rs index bcef633b..71bb5ee7 100644 --- a/crates/runtime/src/account/registry.rs +++ b/crates/runtime/src/account/registry.rs @@ -65,6 +65,17 @@ impl account::Accounts for Accounts { self.storage.account_info(index).ok().map(|info| info.code) } + fn min_acc_gas(&self, index: u32) -> Option { + if let Some(account) = self.accounts.get(&index) { + return Some(account.info.accumulate); + } + + self.storage + .account_info(index) + .ok() + .map(|info| info.accumulate) + } + fn upsert(&mut self, index: u32, account: impl account::Account) { let inherited = Account::inherit(self.storage.clone(), index, account); self.accounts.insert(index, inherited); diff --git a/crates/runtime/src/storage/branch.rs b/crates/runtime/src/storage/branch.rs index eefdd9ae..7c35bd72 100644 --- a/crates/runtime/src/storage/branch.rs +++ b/crates/runtime/src/storage/branch.rs @@ -2,10 +2,10 @@ use crate::{ Storage, - storage::{Column, Commit, KVStorage, StateStorage}, + storage::{Column, Commit, KVStorage, MultiTree, NewNode, NodeAddress, StateStorage}, }; use anyhow::Result; -use score::{TrieKey, state::StateKeyLike}; +use score::{OpaqueHash, TrieKey, state::StateKeyLike}; use std::{ collections::{BTreeMap, btree_map::IntoIter}, mem, @@ -117,6 +117,24 @@ impl KVStorage for Branch { } } +impl MultiTree for Branch { + fn insert_tree(&self, key: OpaqueHash, root: NewNode) -> Result<()> { + self.state.insert_tree(key, root) + } + + fn dereference_tree(&self, key: OpaqueHash) -> Result<()> { + self.state.dereference_tree(key) + } + + fn get_root(&self, key: OpaqueHash) -> Result, Vec)>> { + self.state.get_root(key) + } + + fn get_node(&self, address: NodeAddress) -> Result, Vec)>> { + self.state.get_node(address) + } +} + impl Clone for Branch { fn clone(&self) -> Self { Self { diff --git a/crates/runtime/src/storage/kv.rs b/crates/runtime/src/storage/kv.rs index b5bec88a..e7692540 100644 --- a/crates/runtime/src/storage/kv.rs +++ b/crates/runtime/src/storage/kv.rs @@ -1,10 +1,11 @@ //! Key-value storage abstraction -use crate::storage::{Column, Commit}; +use crate::storage::{Column, Commit, MultiTree, NewNode, NodeAddress}; use anyhow::Result; -use score::TrieKey; +use crypto::merkle::multitree::MultiTreeMap; +use score::{OpaqueHash, TrieKey, state::StateKeyLike}; use std::{ - collections::HashMap, + collections::BTreeMap, sync::{Arc, RwLock}, }; @@ -40,33 +41,28 @@ pub trait KVStorage: Send + Sync + 'static { } } -/// In-memory key-value storage implementation -/// -/// This implementation stores all data in memory and is not persistent. -/// It's useful for testing and for situations where persistence isn't required. +/// In-memory key-value storage. #[derive(Default)] pub struct MemoryDb { - data: Arc, Vec>>>, + data: Arc>>>, + tries: MultiTreeMap, } impl MemoryDb { - /// Execute a closure with direct read access to the underlying data, - /// holding the read lock for the duration. + /// Run a closure under the data read lock. pub fn with_data(&self, f: F) -> Result where - F: FnOnce(&HashMap, Vec>) -> R, + F: FnOnce(&BTreeMap>) -> R, { - let data = self + Ok(f(&*self .data .read() - .map_err(|_| anyhow::anyhow!("RwLock poisoned"))?; - Ok(f(&data)) + .map_err(|_| anyhow::anyhow!("RwLock poisoned"))?)) } /// Reset the memory database - pub fn reset(&self, data: HashMap, Vec>) { - let mut curr = self.data.write().unwrap(); - *curr = data; + pub fn reset(&self, data: BTreeMap>) { + *self.data.write().unwrap() = data; } } @@ -78,11 +74,11 @@ impl KVStorage for MemoryDb { .map_err(|_| anyhow::anyhow!("RwLock poisoned"))?; for (key, value) in commit.iset() { - data.insert(key.to_vec(), value.clone()); + data.insert(*key, value.clone()); } for key in commit.iremoval() { - data.remove(key.as_ref()); + data.remove(key); } Ok(()) @@ -94,7 +90,7 @@ impl KVStorage for MemoryDb { .write() .map_err(|_| anyhow::anyhow!("RwLock poisoned"))?; - data.insert(key.as_ref().to_vec(), value.as_ref().to_vec()); + data.insert(key.as_ref().as_state_key(), value.as_ref().to_vec()); Ok(()) } @@ -103,7 +99,7 @@ impl KVStorage for MemoryDb { .data .read() .map_err(|_| anyhow::anyhow!("RwLock poisoned"))?; - Ok(data.get(key.as_ref()).cloned()) + Ok(data.get(&key.as_ref().as_state_key()).cloned()) } fn iter(&self, _column: Column) -> Result, Vec)>>> { @@ -114,7 +110,7 @@ impl KVStorage for MemoryDb { // Clone all entries to avoid holding the lock during iteration let entries: Vec<(Vec, Vec)> = - data.iter().map(|(k, v)| (k.clone(), v.clone())).collect(); + data.iter().map(|(k, v)| (k.to_vec(), v.clone())).collect(); Ok(entries.into_iter().map(Ok)) } @@ -128,15 +124,33 @@ impl KVStorage for MemoryDb { .data .read() .map_err(|_| anyhow::anyhow!("RwLock poisoned"))?; - let prefix_bytes = prefix.as_ref().to_vec(); + let prefix_bytes = prefix.as_ref(); // Clone all matching entries to avoid holding the lock during iteration let matches: Vec<(Vec, Vec)> = data .iter() - .filter(|(k, _)| k.starts_with(&prefix_bytes)) - .map(|(k, v)| (k.clone(), v.clone())) + .filter(|(k, _)| k.starts_with(prefix_bytes)) + .map(|(k, v)| (k.to_vec(), v.clone())) .collect(); Ok(matches.into_iter().map(Ok)) } } + +impl MultiTree for MemoryDb { + fn insert_tree(&self, key: OpaqueHash, root: NewNode) -> Result<()> { + self.tries.insert_tree(key, root) + } + + fn dereference_tree(&self, key: OpaqueHash) -> Result<()> { + self.tries.dereference_tree(key) + } + + fn get_root(&self, key: OpaqueHash) -> Result, Vec)>> { + self.tries.get_root(key) + } + + fn get_node(&self, address: NodeAddress) -> Result, Vec)>> { + self.tries.get_node(address) + } +} diff --git a/crates/runtime/src/storage/mod.rs b/crates/runtime/src/storage/mod.rs index 9f01baaf..60e81f28 100644 --- a/crates/runtime/src/storage/mod.rs +++ b/crates/runtime/src/storage/mod.rs @@ -6,6 +6,7 @@ pub use { archive::{Archive, ArchiveStorage}, branch::Branch, commit::{Commit, Operation}, + crypto::merkle::multitree::{MultiTree, NewNode, NodeAddress, NodeRef}, kv::{KVStorage, MemoryDb}, state::StateStorage, sync::SyncStorage, @@ -23,14 +24,17 @@ pub mod sync; #[repr(u8)] #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] pub enum Column { - /// The column for the sync + /// The column for the state State = 0, - /// The column for the state + /// The column for the sync Sync = 1, /// The column for the archive Archive = 2, + + /// The column for incremental trie nodes (parity-db multitree). + TrieNodes = 3, } /// The storage of SpaceJam diff --git a/crates/runtime/src/tx/assurance/mod.rs b/crates/runtime/src/tx/assurance/mod.rs index bed63412..470e2260 100644 --- a/crates/runtime/src/tx/assurance/mod.rs +++ b/crates/runtime/src/tx/assurance/mod.rs @@ -8,7 +8,6 @@ use score::{ safrole::ValidatorData, service::{AvailabilityAssignments, WorkReport}, }; -use std::collections::HashSet; mod error; @@ -34,23 +33,11 @@ pub fn reports( pub fn available( reports: &AvailabilityAssignments, validators: &[ValidatorData], - slot: TimeSlot, parent: OpaqueHash, assurances: &[AvailAssurance], ) -> Result<(Vec, [u32; CORES_COUNT])> { // Track assurance count per core let mut core_assurance_counts = [0u32; CORES_COUNT]; - let mut stale_reports = HashSet::new(); - - // Check for stale reports - for (core_idx, assignment) in reports.iter().enumerate() { - if let Some(assignment) = assignment - && slot >= assignment.timeout + WORK_REPORT_TIMEOUT_PERIOD - { - stale_reports.insert(core_idx); - continue; - } - } // Check for engaged reports: cheap checks first, then batch verify sigs. let mut assuror = None; @@ -98,9 +85,7 @@ pub fn available( ) }) .collect(); - crypto::ed25519::batch_verify(&verify_items) - .inspect_err(|_| tracing::error!("bad signature in assurances")) - .map_err(|_| Error::BadSignature)?; + crypto::ed25519::batch_verify(&verify_items).map_err(|_| Error::BadSignature)?; // Check which cores reached 2/3 majority let mut available = Vec::new(); @@ -114,31 +99,3 @@ pub fn available( Ok((available, core_assurance_counts)) } - -/// Verifies the assurance -pub fn verify_assurance( - validators: &[ValidatorData], - assurance: &AvailAssurance, - parent: OpaqueHash, -) -> Result<()> { - if assurance.validator_index >= VALIDATORS_COUNT { - return Err(Error::BadValidatorIndex); - } - - if assurance.anchor != parent { - return Err(Error::BadAttestationParent); - } - - if crypto::ed25519::verify( - &assurance.singing_message(), - assurance.signature, - validators[assurance.validator_index as usize].ed25519, - ) - .is_err() - { - tracing::error!("bad signature for assurance: {:?}", assurance); - return Err(Error::BadSignature); - } - - Ok(()) -} diff --git a/crates/runtime/src/tx/block/header.rs b/crates/runtime/src/tx/block/header.rs index 6764e91f..4cd47f2d 100644 --- a/crates/runtime/src/tx/block/header.rs +++ b/crates/runtime/src/tx/block/header.rs @@ -137,9 +137,21 @@ pub fn check(state: &State, header: &Header, new_epoch: bool) -> anyhow::Result< // validate the epoch mark if let Some(epoch_mark) = &header.epoch_mark { + if !new_epoch { + anyhow::bail!("epoch mark present but not expected"); + } + + // Φ uses the posterior punish-set: prior offenders ∪ this block's + let offenders: Vec<_> = state + .disputes + .offenders + .iter() + .chain(header.offenders_mark.iter()) + .copied() + .collect(); let expected = state .safrole - .next(&state.validators.drawn, &header.offenders_mark) + .next(&state.validators.drawn, &offenders) .evals(); if epoch_mark.validators[..] != expected[..] { anyhow::bail!("epoch mark validators mismatch"); @@ -153,7 +165,7 @@ pub fn check(state: &State, header: &Header, new_epoch: bool) -> anyhow::Result< anyhow::bail!("epoch mark tickets entropy mismatch"); } } else if new_epoch { - anyhow::bail!("epoch mark is required"); + anyhow::bail!("epoch mark required but not present"); } let should_have_tickets_mark = state.safrole.has_tickets_mark(state.timeslot, header.slot); diff --git a/crates/runtime/src/tx/block/history.rs b/crates/runtime/src/tx/block/history.rs index 696ece4b..80c75099 100644 --- a/crates/runtime/src/tx/block/history.rs +++ b/crates/runtime/src/tx/block/history.rs @@ -11,14 +11,9 @@ use score::{ pub fn import( history: &mut History, header_hash: OpaqueHash, - parent_state_root: OpaqueHash, accumulate_root: OpaqueHash, reported: Vec, ) { - if let Some(last) = history.history.last_mut() { - last.state_root = parent_state_root; - } - history.mmr.peaks = mmr::append(history.mmr.peaks.clone(), accumulate_root); if history.history.is_empty() { let new_block = BlockInfo { diff --git a/crates/runtime/src/tx/block/mod.rs b/crates/runtime/src/tx/block/mod.rs index 29a28319..f0adff44 100644 --- a/crates/runtime/src/tx/block/mod.rs +++ b/crates/runtime/src/tx/block/mod.rs @@ -2,14 +2,20 @@ use crate::{ Storage, - storage::{Branch, Column, KVStorage, MemoryDb, StateStorage, root}, + storage::{Branch, Column, Commit, KVStorage, MemoryDb, MultiTree, StateStorage, root}, tx, }; use anyhow::Result; use crypto::merkle; use pvm::Pvm; -use score::{Block, OpaqueHash, state::StateKeyLike}; -use std::{collections::HashMap, sync::Arc}; +use score::{Block, OpaqueHash, TrieKey, state::StateKeyLike}; +use std::{ + collections::{BTreeMap, HashMap}, + sync::Arc, +}; + +/// Zero hash sentinel — `current_root` before [`TestChain::init`]. +const EMPTY_ROOT: OpaqueHash = [0; 32]; pub mod header; pub mod history; @@ -25,8 +31,7 @@ pub fn process(block: Block, storage: Arc) -> Result<()> ); match (vresult, sresult) { - (Err(e), _) => anyhow::bail!("failed to import block: {e:?}"), - (_, Err(e)) => anyhow::bail!("failed to import block: {e:?}"), + (Err(e), _) | (_, Err(e)) => Err(e), (Ok(()), Ok(diff)) => { storage.commit(Column::State, diff)?; Ok(()) @@ -46,6 +51,10 @@ pub struct TestChain { /// The forks and their states (diff only). pub forks: HashMap, + + /// The state root corresponding to `data`. Tracked incrementally via the + /// multitree column so we can skip the O(N log N) full retrie per block. + current_root: OpaqueHash, } impl TestChain { @@ -62,7 +71,18 @@ impl TestChain { .read() .map_err(|_| anyhow::anyhow!("lock poisoned"))? .clone(); + let dirty = collect_dirty_keys(&commit); self.data.commit(Column::State, commit)?; + + let prev = (self.current_root != EMPTY_ROOT).then_some(self.current_root); + let new_root = self.data.with_data(|data| { + let kvs: Vec<(TrieKey, &[u8])> = + data.iter().map(|(k, v)| (*k, v.as_slice())).collect(); + self.data.apply(prev, &kvs, &dirty) + })??; + root::set(parent, new_root); + self.current_root = new_root; + self.finalized = parent; self.forks.clear(); } @@ -120,6 +140,10 @@ impl TestChain { /// Initialize the chain with the given block. pub fn init(&mut self, state: HashMap, Vec>) -> anyhow::Result { + let state: BTreeMap> = state + .into_iter() + .map(|(k, v)| (k.as_slice().as_state_key(), v)) + .collect(); self.data.reset(state); let head = self .data @@ -128,7 +152,15 @@ impl TestChain { .ok_or(anyhow::anyhow!("no recent blocks"))? .header_hash; self.finalized = head; - self.data.with_data(|data| handle_root(head, data)) + + let new_root = self.data.with_data(|data| { + let kvs: Vec<(TrieKey, &[u8])> = data.iter().map(|(k, v)| (*k, v.as_slice())).collect(); + let dirty: Vec = data.keys().copied().collect(); + self.data.apply(None, &kvs, &dirty) + })??; + root::set(head, new_root); + self.current_root = new_root; + Ok(new_root) } } @@ -138,58 +170,59 @@ impl Default for TestChain { finalized: Default::default(), data: Arc::new(MemoryDb::default()), forks: HashMap::new(), + current_root: EMPTY_ROOT, } } } -/// Compute the state root and cache it for the given header hash. -/// -/// NOTE: this method overrides the StateStorage::root for zero-copy. -fn handle_root(head: OpaqueHash, data: &HashMap, Vec>) -> OpaqueHash { - let mut kvs: Vec<([u8; 31], &[u8])> = data - .iter() - .map(|(k, v)| (k.as_slice().as_state_key(), v.as_slice())) +/// Sorted, deduplicated union of the keys an overlay commit touches. +fn collect_dirty_keys(commit: &Commit>) -> Vec { + let mut keys: Vec = commit + .update + .keys() + .copied() + .chain(commit.removal.iter().copied()) .collect(); - kvs.sort_by_key(|a| a.0); - let state_root = merkle::trie31(&kvs); - root::set(head, state_root); - state_root + keys.sort_unstable(); + keys.dedup(); + keys } -/// Compute the state root from base data + overlay diff (no base clone). -/// -/// NOTE: this method overrides the StateStorage::root for zero-copy. +/// Compute the state root from base data + overlay diff via merge-walk. fn handle_root_with_diff( head: OpaqueHash, - base: &HashMap, Vec>, - diff: &crate::storage::Commit>, + base: &BTreeMap>, + diff: &crate::storage::Commit>, ) -> OpaqueHash { - let mut kvs: Vec<([u8; 31], &[u8])> = Vec::with_capacity(base.len() + diff.update.len()); - - // Add base entries, applying overlay - for (k, v) in base.iter() { - let trie_key = k.as_slice().as_state_key(); - if diff.removal.contains(&trie_key) { - continue; + let mut kvs: Vec<(TrieKey, &[u8])> = Vec::with_capacity(base.len() + diff.update.len()); + let mut base_iter = base.iter(); + let mut diff_iter = diff.update.iter(); + let mut b = base_iter.next(); + let mut d = diff_iter.next(); + + while let Some((key, value)) = match (b, d) { + (Some((bk, _)), Some((dk, dv))) if dk <= bk => { + if dk == bk { + b = base_iter.next(); + } + d = diff_iter.next(); + Some((dk, dv.as_slice())) } - if let Some(updated) = diff.update.get(&trie_key) { - kvs.push((trie_key, updated.as_slice())); - } else { - kvs.push((trie_key, v.as_slice())); + (Some((bk, bv)), _) => { + b = base_iter.next(); + Some((bk, bv.as_slice())) } - } - - // Add new keys from overlay (not in base) - for (trie_key, v) in diff.update.iter() { - if diff.removal.contains(trie_key) { - continue; + (None, Some((dk, dv))) => { + d = diff_iter.next(); + Some((dk, dv.as_slice())) } - if !base.contains_key(trie_key.as_ref()) { - kvs.push((*trie_key, v.as_slice())); + (None, None) => None, + } { + if !diff.removal.contains(key) { + kvs.push((*key, value)); } } - kvs.sort_by_key(|a| a.0); let state_root = merkle::trie31(&kvs); root::set(head, state_root); state_root diff --git a/crates/runtime/src/tx/dispute/mod.rs b/crates/runtime/src/tx/dispute/mod.rs index 5b31543c..890c611d 100644 --- a/crates/runtime/src/tx/dispute/mod.rs +++ b/crates/runtime/src/tx/dispute/mod.rs @@ -82,19 +82,18 @@ fn verdicts( verdicts: &[Verdict], ) -> Result { let mut records = DisputesRecords::default(); - let mut last_verdict = None; + let mut last_target: Option = None; for verdict in verdicts { if verdict.votes.len() != VALIDATORS_SUPER_MAJORITY as usize { return Err(Error::NotEnoughValidators); } - if let Some(last_verdict) = last_verdict.take() { - if verdict < last_verdict { - return Err(Error::VerdictsNotSortedUnique); - } - } else { - last_verdict = Some(verdict); + if let Some(last) = last_target + && verdict.target <= last + { + return Err(Error::VerdictsNotSortedUnique); } + last_target = Some(verdict.target); let mut aye = 0; let aye_message = verdict.signature_message(true); @@ -136,15 +135,9 @@ fn verdicts( .map_err(|_| Error::BadSignature)?; match aye { - aye if aye >= VALIDATORS_SUPER_MAJORITY => { - records.good.push(verdict.target); - } - aye if aye >= VALIDATORS_COUNT / 3 => { - records.wonky.push(verdict.target); - } - 0 => { - records.bad.push(verdict.target); - } + aye if aye == VALIDATORS_SUPER_MAJORITY => records.good.push(verdict.target), + aye if aye == VALIDATORS_COUNT / 3 => records.wonky.push(verdict.target), + 0 => records.bad.push(verdict.target), _ => { tracing::error!("Bad vote split in verdict: {aye}/{VALIDATORS_SUPER_MAJORITY}"); return Err(Error::BadVoteSplit); diff --git a/crates/runtime/src/tx/guarantee/acc.rs b/crates/runtime/src/tx/guarantee/acc.rs index 0ae02ec4..f3df5e81 100644 --- a/crates/runtime/src/tx/guarantee/acc.rs +++ b/crates/runtime/src/tx/guarantee/acc.rs @@ -3,7 +3,7 @@ use account::Accounts; use pvm::{Account, AccumulateState}; use score::{ - Gas, OpaqueHash, ServiceId, + AUTH_QUEUE_SIZE, Array, CORES_COUNT, Gas, OpaqueHash, ServiceId, safrole::ValidatorsData, service::{AccumulatedQueue, Privileges, ReadyQueue, WorkReport}, statistic::{AccumulationRecord, ServiceActivityRecord}, @@ -48,7 +48,7 @@ impl Accumulated { } } - /// Get the service records + /// Compose the service activity records; caller passes the accumulated prefix. pub fn records( &mut self, accumulatable: &[WorkReport], @@ -64,17 +64,12 @@ impl Accumulated { } } - for transfer in self.transfers.iter() { - if records.contains_key(&transfer.recipient) - || !self.gas.contains_key(&transfer.recipient) - { + // Include services that consumed gas without contributing digests. + for (service, gas) in self.gas.iter() { + if records.contains_key(service) { continue; } - - let record = records.entry(transfer.recipient).or_default(); - if record.accumulate_gas_used == 0 { - record.accumulate_gas_used = *self.gas.get(&transfer.recipient).unwrap_or(&0); - } + records.entry(*service).or_default().accumulate_gas_used = *gas; } // update the last update time of the accounts @@ -149,6 +144,9 @@ pub struct Accumulation { /// (πS') The service records pub records: BTreeMap, + /// (φ') The authorization queue + pub authorization: Array, CORES_COUNT>, + /// (θ) The accumulation logs pub logs: CommitmentMap, } diff --git a/crates/runtime/src/tx/guarantee/exec.rs b/crates/runtime/src/tx/guarantee/exec.rs index 3b730c7d..8cd6e8fc 100644 --- a/crates/runtime/src/tx/guarantee/exec.rs +++ b/crates/runtime/src/tx/guarantee/exec.rs @@ -34,7 +34,7 @@ pub fn outer( context: AccumulateState, validators: &mut score::safrole::ValidatorsData, gas_table: &BTreeMap, -) -> Accumulated { +) -> anyhow::Result> { let mut accumulated = Accumulated::new(context); let empty = BTreeMap::new(); let mut first = true; @@ -54,15 +54,19 @@ pub fn outer( break; } + // Deferred-transfer gas supplements the block budget for this round. + let xfer_gas: Gas = transfers.iter().map(|t| t.gas_limit).sum(); + let step = self::parallel::( accumulated.context.clone(), validators, &transfers, if index == 0 { &[] } else { &reports[..index] }, if first { gas_table } else { &empty }, - ); + )?; - gas_limit -= step.gas.values().sum::(); + let step_gas: Gas = step.gas.values().sum(); + gas_limit = gas_limit.saturating_add(xfer_gas).saturating_sub(step_gas); reports = &reports[index..]; transfers = step.transfers.clone(); accumulated.transfers.extend(step.transfers); @@ -79,7 +83,7 @@ pub fn outer( } } - accumulated + Ok(accumulated) } /// (Δ*) parallel accumulation @@ -89,7 +93,7 @@ pub fn parallel( transfers: &[DeferredTransfer], reports: &[WorkReport], table: &BTreeMap, -) -> Accumulated { +) -> anyhow::Result> { let mut services: BTreeSet = Default::default(); for report in reports { for result in &report.results { @@ -140,12 +144,16 @@ pub fn parallel( context.privileges.always_acc = mgr.always_acc.clone(); } - // Update assign services + // Update assign service and authorization queue from the prior c-th assigner. for c in 0..context.privileges.assign.len() { let old = context.privileges.assign[c]; let mgr_val = mgr.map(|m| m.assign[c]).unwrap_or(old); let svc_val = results.get(&old).map(|r| r.context.privileges.assign[c]); context.privileges.assign[c] = svc_val.map(|s| r(old, mgr_val, s)).unwrap_or(mgr_val); + + if let Some(result) = results.get(&old) { + context.authorization[c] = result.context.authorization[c].clone(); + } } // Update designate @@ -198,7 +206,8 @@ pub fn parallel( gas.insert(*service_id, result.gas); } - // Apply additions (n): the service's own account from every result, + // Apply additions (n); reject the same new id created by two services. + let mut created: BTreeSet = BTreeSet::new(); for (service_id, result) in results.iter() { if !removed.contains(service_id) && let Some(account) = result.context.accounts.accounts().get(service_id) @@ -214,12 +223,19 @@ pub fn parallel( if id == service_id { continue; // already handled above } - if !removed.contains(id) - && account.creation() == context.timeslot - && context.accounts.get(*id).is_none() - { - context.accounts.upsert(*id, account.clone()); + if removed.contains(id) || account.creation() != context.timeslot { + continue; + } + if context.accounts.get(*id).is_some() { + if created.contains(id) { + anyhow::bail!( + "duplicate account creation: service {id} created by multiple services" + ); + } + continue; } + context.accounts.upsert(*id, account.clone()); + created.insert(*id); } } @@ -228,13 +244,13 @@ pub fn parallel( context.accounts.remove(*service); } - Accumulated { + Ok(Accumulated { accumulated: reports.len(), context, transfers, pairings, gas, - } + }) } /// (Δ1) single accumulation (12.24) diff --git a/crates/runtime/src/tx/guarantee/mod.rs b/crates/runtime/src/tx/guarantee/mod.rs index 8be66412..9f6192de 100644 --- a/crates/runtime/src/tx/guarantee/mod.rs +++ b/crates/runtime/src/tx/guarantee/mod.rs @@ -5,7 +5,7 @@ use account::Accounts; use error::{Error, Result}; use pvm::{AccumulateState, Pvm}; use score::{ - CORES_COUNT, Ed25519Public, EntropyBuffer, OpaqueHash, TimeSlot, + AUTH_QUEUE_SIZE, Array, CORES_COUNT, Ed25519Public, EntropyBuffer, OpaqueHash, TimeSlot, extrinsic::GuaranteesExtrinsic, safrole::ValidatorsData, service::{ @@ -38,6 +38,8 @@ pub fn accumulate( privileges: &Privileges, // The validators to be drawn (ι) validators: &ValidatorsData, + // The authorization queue (φ) + authorization: &Array, CORES_COUNT>, // The account storage (δ) accounts: R, // The entropy (η) @@ -47,6 +49,14 @@ pub fn accumulate( let (accumulatable, queued) = queue::accumulatable(slot, reports, ready_queue, accumulated_queue); + // Seed φ from prior state so the assign host call mutates the real queue + let auth_init: score::AuthorizationPools = authorization + .iter() + .map(|q| q.to_vec()) + .collect::>() + .try_into() + .expect("authorization has CORES_COUNT entries"); + // (Δ+) run outer accumulation (12.18) let gas_limit = privileges.gas_limit(); let mut validators = validators.clone(); @@ -57,25 +67,34 @@ pub fn accumulate( AccumulateState { accounts, privileges: privileges.clone(), - authorization: Default::default(), + authorization: auth_init, entropy, timeslot: slot, }, &mut validators, &privileges.always_acc, - ); + )?; // (πS') compose the service activity records - let records = accumulated.records(&accumulatable); + let n = accumulated.accumulated; + let records = accumulated.records(&accumulatable[..n]); // update the accumulated queue (ξ') - let next_accumulated_queue = - self::accumulated_history(accumulated_queue, accumulatable, accumulated.accumulated); + let next_accumulated_queue = self::accumulated_history(accumulated_queue, accumulatable, n); // update the ready queue (θ') let next_ready_queue = self::ready_queue(ready_queue, &next_accumulated_queue, queued, tau, slot); + // (φ') Project the per-core Vec back to the fixed-size state shape + let next_authorization: Array, CORES_COUNT> = + std::mem::take(&mut accumulated.context.authorization) + .into_iter() + .map(|q| Array::try_from(q).expect("queue has AUTH_QUEUE_SIZE entries")) + .collect::>() + .try_into() + .expect("authorization has CORES_COUNT entries"); + Ok(Accumulation { root: accumulated.root(), ready_queue: next_ready_queue, @@ -84,6 +103,7 @@ pub fn accumulate( privileges: accumulated.context.privileges, validators, records, + authorization: next_authorization, logs: accumulated.pairings, }) } @@ -100,15 +120,12 @@ pub fn accumulated_history( next.remove(0); } - // Add new accumulated work report hashes + // Set of newly-accumulated work-package hashes, canonically sorted. let mut new_accumulated: Vec = accumulatable .iter() .take(accumulated) .map(|w| w.spec.hash) .collect(); - - // NOTE: Sort the new accumulated work report hashes again to align the test - // vectors, not sure if we missed anything that we have to do it here. new_accumulated.sort(); next.push(new_accumulated); @@ -202,7 +219,7 @@ pub fn pools( >, guarantees: &GuaranteesExtrinsic, ) -> score::AuthorizationPools { - let slot = timeslot % score::EPOCH_LENGTH; + let slot = timeslot as usize % score::AUTH_QUEUE_SIZE; let mut new_pools = score::AuthorizationPools::default(); for (core_index, pool) in pools.iter().enumerate() { let mut new_pool = pool.clone(); @@ -218,10 +235,8 @@ pub fn pools( } } - // add new authorizer from queue at position H_t (current timeslot) - if let Some(auth) = authorizations[core_index].get(slot as usize) { - new_pool.push(*auth); - } + // add new authorizer from queue at position H_t mod Q + new_pool.push(authorizations[core_index][slot]); // truncate the pool to the max size if let Some(old) = new_pool.len().checked_sub(score::AUTH_POOL_MAX_SIZE) { diff --git a/crates/runtime/src/tx/guarantee/validator.rs b/crates/runtime/src/tx/guarantee/validator.rs index b440950c..851f6297 100644 --- a/crates/runtime/src/tx/guarantee/validator.rs +++ b/crates/runtime/src/tx/guarantee/validator.rs @@ -4,8 +4,8 @@ use crate::tx::guarantee::error::{Error, Result}; use account::{Account, Accounts}; use score::{ CORES_COUNT, CoreIndex, EPOCH_LENGTH, Ed25519Public, Entropy, MAX_DEPENDENCY_COUNT, - MAX_WORK_REPORT_OUTPUT_SIZE, OpaqueHash, ROTATION_PERIOD, SERVICE_ITEM_MIN_GAS, State, - TimeSlot, VALIDATORS_COUNT, WORK_REPORT_GAS_LIMIT, + MAX_WORK_REPORT_OUTPUT_SIZE, OpaqueHash, ROTATION_PERIOD, State, TimeSlot, VALIDATORS_COUNT, + WORK_REPORT_GAS_LIMIT, extrinsic::{GuaranteesExtrinsic, ReportGuarantee}, service::{ReportedWorkPackage, WorkExecResult}, }; @@ -251,14 +251,18 @@ impl<'s, R: Accounts> GuaranteeValidator<'s, R> { return Err(Error::WorkReportGasTooHigh); } - if result.accumulate_gas < SERVICE_ITEM_MIN_GAS { - return Err(Error::ServiceItemGasTooLow); - } - let Some(code_hash) = self.accounts.code_hash(result.service_id) else { return Err(Error::BadServiceId); }; + let min_gas = self + .accounts + .min_acc_gas(result.service_id) + .ok_or(Error::BadServiceId)?; + if result.accumulate_gas < min_gas { + return Err(Error::ServiceItemGasTooLow); + } + if code_hash != result.code_hash { tracing::debug!( "bad code hash for service {}: 0x{} != 0x{}", @@ -329,10 +333,22 @@ impl<'s, R: Accounts> GuaranteeValidator<'s, R> { Ok(()) } - // TODO: check if duplicated in service deps? pub fn duplicated(&self, hash: &OpaqueHash) -> bool { - self.recent.iter().any(|r| r.hash == *hash) - || self.reported.iter().filter(|h| *h == hash).count() > 1 + self.reported.iter().filter(|h| *h == hash).count() > 1 + || self.recent.iter().any(|r| r.hash == *hash) + || self.state.history.iter().flatten().any(|h| h == hash) + || self + .state + .queue + .iter() + .flatten() + .any(|r| r.report.spec.hash == *hash) + || self + .state + .reports + .iter() + .flatten() + .any(|a| a.report.spec.hash == *hash) } fn contains_dep(&self, dep: &OpaqueHash) -> bool { diff --git a/crates/runtime/src/tx/mod.rs b/crates/runtime/src/tx/mod.rs index 0555615b..3a7e24cd 100644 --- a/crates/runtime/src/tx/mod.rs +++ b/crates/runtime/src/tx/mod.rs @@ -68,20 +68,16 @@ pub fn simulate_with_state( } // The first round computation - let accounts = Accounts::new(storage); + let mut accounts = Accounts::new(storage); + + // (E_P) Validate preimages against the prior state (12.6) + preimage::validate(&mut accounts, &block.extrinsic.preimages)?; let (mut reports, reported, reporters) = { // (η') Update entropy (6.22) let entropy = crypto::vrf::ietf_output(block.header.entropy_source).unwrap_or_default(); state.entropy = ticket::eta(new_epoch, &state.entropy, entropy); - if new_epoch { - // (λ', κ') Update validator state (6.13) - state.validators.previous = std::mem::replace( - &mut state.validators.current, - state.safrole.validators.clone(), - ); - } - // (ψ') Update disputes and get marks + // (ψ') Update disputes against the prior validator sets (10.4) let marks = if block.extrinsic.disputes.is_empty() { if !block.header.offenders_mark.is_empty() { anyhow::bail!("offenders mark is not empty"); @@ -101,6 +97,14 @@ pub fn simulate_with_state( marks }; + if new_epoch { + // (λ', κ') Update validator state (6.13) + state.validators.previous = std::mem::replace( + &mut state.validators.current, + state.safrole.validators.clone(), + ); + } + // complete the state root of the last block in the history if let Some(last) = state.recent_blocks.history.last_mut() { last.state_root = block.header.parent_state_root; @@ -132,13 +136,12 @@ pub fn simulate_with_state( let (available, assurances) = { // (W) the sequence of new available work reports (11.16) let (available, assurances) = self::assurance::available( - &state.reports, + &reports, if new_epoch { &state.validators.previous } else { &state.validators.current }, - block.header.slot, block.header.parent, &block.extrinsic.assurances, )?; @@ -193,6 +196,7 @@ pub fn simulate_with_state( &state.history, &state.privileges, &state.validators.drawn, + &state.authorization, accounts, state.entropy, )?; @@ -202,10 +206,12 @@ pub fn simulate_with_state( state.queue = accumulation.ready_queue; state.history = accumulation.accumulated_queue; state.validators.drawn = accumulation.validators; + state.authorization = accumulation.authorization; + let candidate = state + .safrole + .next(&state.validators.drawn, &state.disputes.offenders); + thread::spawn(move || ticket::lazy::drawn(&candidate)); - // lazy load vrf rings - let drawn = state.validators.drawn.clone(); - thread::spawn(move || ticket::lazy::drawn(&drawn)); state.statistics.merge_services(accumulation.records); state.logs = accumulation.logs; (accumulation.root, accumulation.accounts) @@ -218,7 +224,6 @@ pub fn simulate_with_state( block::history::import( &mut state.recent_blocks, block.header.hash(), - block.header.parent_state_root, root, reported, ); @@ -229,8 +234,8 @@ pub fn simulate_with_state( .merge_reporters(&reporters, &state.validators.current.ed25519())?; } - // (δ') Update the accounts - let accounts = preimage::accounts(block.header.slot, &block.extrinsic.preimages, accounts)?; + // (δ') Integrate preimages into the post-transfer state + let accounts = preimage::accounts(block.header.slot, &block.extrinsic.preimages, accounts); let (updates, removals) = accounts.diff(); diff.extend_iter(updates, removals); diff --git a/crates/runtime/src/tx/preimage.rs b/crates/runtime/src/tx/preimage.rs index 3295bc9d..f5b6ac7d 100644 --- a/crates/runtime/src/tx/preimage.rs +++ b/crates/runtime/src/tx/preimage.rs @@ -1,23 +1,11 @@ -//! Preimage handler +//! Preimage extrinsic handler use account::{Account, Accounts}; use anyhow::Result; use score::{TimeSlot, extrinsic::PreimagesExtrinsic}; -/// (δ') handle preimage extrinsic -/// -/// Validates preimages against post-transfer state and integrates valid ones. -/// Invalid preimages are disregarded without prejudice -/// -/// # Arguments -/// * `slot` - Current time slot (τ') -/// * `preimages` - Preimage extrinsic data -/// * `accounts` - Post-transfer account state -pub fn accounts( - slot: TimeSlot, - preimages: &PreimagesExtrinsic, - mut accounts: impl Accounts, -) -> Result { +/// Validate preimages against the prior state +pub fn validate(accounts: &mut A, preimages: &PreimagesExtrinsic) -> Result<()> { let mut prev: Option<&score::extrinsic::Preimage> = None; for preimage in preimages { if let Some(p) = prev @@ -26,41 +14,27 @@ pub fn accounts( anyhow::bail!("preimages not sorted or unique"); } prev = Some(preimage); - let Some(account) = accounts.get(preimage.requester) else { - anyhow::bail!("Preimage for non-existent account"); - }; - let blob_len = preimage.blob.len() as u32; let hash = crypto::blake2b(&preimage.blob); - tracing::debug!("lookup hash={} len={}", hex::encode(hash), blob_len); - let Some(slots) = account.lookup(hash, blob_len) else { - anyhow::bail!("Preimage lookup failed"); - }; - - // skip if the lookup is removed - let Some(slots) = slots else { - continue; - }; - - if !slots.is_empty() { - anyhow::bail!("Preimage already has non-empty lookup slots"); + let len = preimage.blob.len() as u32; + if !accounts.is_providable(preimage.requester, hash, len) { + anyhow::bail!("preimage not required"); } + } + Ok(()) +} - if account.preimage(hash).is_some() { - anyhow::bail!("Preimage already exists"); +/// (δ') Integrate providable preimages into the post-transfer state +pub fn accounts(slot: TimeSlot, preimages: &PreimagesExtrinsic, mut accounts: A) -> A { + for preimage in preimages { + let hash = crypto::blake2b(&preimage.blob); + let len = preimage.blob.len() as u32; + if !accounts.is_providable(preimage.requester, hash, len) { + continue; } - - // Set lookup slots to [τ'] (current time slot) - let updated_slots = vec![slot]; - tracing::debug!( - "service={} inserting preimage hash={} len={}", - account.index(), - hex::encode(hash), - blob_len - ); + let account = accounts.get(preimage.requester).expect("just checked"); account.insert_preimage(hash, preimage.blob.clone()); - account.insert_lookup(hash, blob_len, updated_slots); + account.insert_lookup(hash, len, vec![slot]); } - - Ok(accounts) + accounts } diff --git a/crates/runtime/src/tx/ticket/error.rs b/crates/runtime/src/tx/ticket/error.rs index 9c2c9ec5..c558b86f 100644 --- a/crates/runtime/src/tx/ticket/error.rs +++ b/crates/runtime/src/tx/ticket/error.rs @@ -19,6 +19,10 @@ pub enum Error { Reserved, /// Duplicate ticket DuplicateTicket, + /// Submitted ticket missing from posterior accumulator + UselessTicket, + /// Too many tickets in a single block + TooManyTickets, } impl std::fmt::Display for Error { diff --git a/crates/runtime/src/tx/ticket/lazy.rs b/crates/runtime/src/tx/ticket/lazy.rs index ab27b5bd..4a4aa9d0 100644 --- a/crates/runtime/src/tx/ticket/lazy.rs +++ b/crates/runtime/src/tx/ticket/lazy.rs @@ -10,19 +10,8 @@ static LAZY_RING: LazyLock, Arc LazyLock::new(|| Mutex::new(LruCache::new(NonZeroUsize::new(8).unwrap()))); /// Clear all cached data -pub async fn clear() { - if let Ok(mut map) = LAZY_RING.lock() { - map.clear(); - } -} - -/// Check if the lazy cache is empty -pub fn is_empty() -> bool { - if let Ok(cache) = LAZY_RING.lock() { - cache.len() == 0 - } else { - false - } +pub fn clear() { + lock().clear(); } /// Accept drawn validators after accumulation @@ -40,26 +29,18 @@ pub fn commitment(drawn: &Vec) -> BandersnatchRingCommitment /// Get the verifier of the next validators pub fn verifier(drawn: &Vec) -> Arc { - let Ok(mut map) = LAZY_RING.lock() else { - panic!("failed to lock ring, fix me later"); - }; - - if let Some(v) = map.get(drawn) { - return v.clone(); + if let Some(v) = lock().get(drawn).cloned() { + return v; } - - drop(map); - // build new verifier, expensive computation here. let verifier = Arc::new(crypto::ring::verifier(drawn)); - let Ok(mut map) = LAZY_RING.lock() else { - panic!("failed to lock ring, fix me later"); - }; - - // double-check after re-acquiring lock + let mut map = lock(); if let Some(v) = map.get(drawn) { return v.clone(); } - map.put(drawn.clone(), verifier.clone()); verifier } + +fn lock() -> std::sync::MutexGuard<'static, LruCache, Arc>> { + LAZY_RING.lock().unwrap_or_else(|p| p.into_inner()) +} diff --git a/crates/runtime/src/tx/ticket/mod.rs b/crates/runtime/src/tx/ticket/mod.rs index 443ebeba..09ac521e 100644 --- a/crates/runtime/src/tx/ticket/mod.rs +++ b/crates/runtime/src/tx/ticket/mod.rs @@ -61,8 +61,12 @@ pub fn safrole( } let slot_phase = slot % score::EPOCH_LENGTH; - if slot_phase >= score::TICKET_SUBMISSION_PERIOD && !tickets.is_empty() { - return Err(Error::UnexpectedTicket); + if slot_phase >= score::TICKET_SUBMISSION_PERIOD { + if !tickets.is_empty() { + return Err(Error::UnexpectedTicket); + } + } else if tickets.len() > score::MAX_TICKETS_PER_EXTRINSIC as usize { + return Err(Error::TooManyTickets); } let epoch = tau / score::EPOCH_LENGTH; @@ -105,6 +109,9 @@ pub fn accumulator( new_tickets = self::verify::tickets(entropy, next, tickets)?; } + // Snapshot submitted ids for the n ⊆ γ_a' check below + let submitted_ids: Vec = new_tickets.iter().map(|t| t.id).collect(); + // update the accumulator let mut accumulator = accumulator.clone(); if new_epoch { @@ -125,6 +132,16 @@ pub fn accumulator( // Take only the first E tickets (formula 6.35: truncate to E) accumulator.sort_by_key(|a| a.id); accumulator.truncate(score::EPOCH_LENGTH as usize); + + // (n ⊆ γ_a') Every submitted ticket must survive into the posterior + // accumulator; a submission that gets truncated out is "useless". + if submitted_ids + .iter() + .any(|id| accumulator.binary_search_by_key(id, |t| t.id).is_err()) + { + return Err(Error::UselessTicket); + } + Ok(accumulator) } diff --git a/crates/runtime/src/tx/ticket/verify.rs b/crates/runtime/src/tx/ticket/verify.rs index 7d66d536..fe71ea31 100644 --- a/crates/runtime/src/tx/ticket/verify.rs +++ b/crates/runtime/src/tx/ticket/verify.rs @@ -1,12 +1,10 @@ //! Verification utilities for tickets use crate::tx::ticket::{Error, lazy}; -use rayon::iter::{IndexedParallelIterator, IntoParallelRefIterator, ParallelIterator}; use score::{ BandersnatchPublic, OpaqueHash, - extrinsic::{TicketBody, TicketEnvelope, TicketsAccumulator, TicketsExtrinsic}, + extrinsic::{TicketBody, TicketsAccumulator, TicketsExtrinsic}, }; -use std::{collections::BTreeMap, sync::Arc}; /// Verify tickets pub fn tickets( @@ -14,54 +12,48 @@ pub fn tickets( next: &Vec, tickets: &TicketsExtrinsic, ) -> Result { - let verifier = lazy::verifier(next); - let verified = tickets - .par_iter() - .enumerate() - .map(|(index, envelope)| self::ticket(index, envelope.clone(), entropy, verifier.clone())) - .collect::, Error>>()?; - - // Check for bad order: 6.32 & 6.33 - let new_tickets = verified.into_values().collect::>(); - let mut sorted = new_tickets.clone(); - sorted.sort_by_key(|a| a.id); - if sorted != new_tickets { - return Err(Error::BadTicketOrder); + // 1. Verify ticket attempts upfront (6.29) + for envelope in tickets.iter() { + if envelope.attempt >= score::TICKET_ENTRIES_PER_VALIDATOR as u8 { + return Err(Error::BadTicketAttempt); + } } - Ok(sorted) -} - -/// Verify a single ticket -fn ticket( - index: usize, - envelope: TicketEnvelope, - entropy: [OpaqueHash; 4], - verifier: Arc, -) -> Result<(usize, TicketBody), Error> { - // 1. Verify ticket attempt (6.29) - if envelope.attempt >= score::TICKET_ENTRIES_PER_VALIDATOR as u8 { - return Err(Error::BadTicketAttempt); - } - - // 2. Verify ring VRF signature and get ticket identifier - let id = verifier - .ring_vrf_verify( - &TicketBody::message(envelope.attempt, &entropy[2]), - &[], - &envelope.signature, + // 2. Batch-verify ring VRF signatures, harvesting per-ticket ids in order + let messages: Vec> = tickets + .iter() + .map(|e| TicketBody::message(e.attempt, &entropy[2])) + .collect(); + let verifier = lazy::verifier(next); + let ids = verifier + .ring_vrf_verify_batch( + messages + .iter() + .zip(tickets.iter()) + .map(|(msg, e)| (msg.as_slice(), [].as_slice(), e.signature.as_slice())), ) .map_err(|e| { - tracing::error!("failed to verify ring VRF signature: {:?}", e); + tracing::trace!("failed to batch-verify ring VRF signatures: {:?}", e); Error::BadTicketProof })?; - // 3. Store ticket for accumulation - Ok(( - index, - TicketBody { + let new_tickets: Vec = ids + .into_iter() + .zip(tickets.iter()) + .map(|(id, envelope)| TicketBody { id, attempt: envelope.attempt, - }, - )) + }) + .collect(); + + // 3. Strictly ascending by id, no duplicates (6.32 & 6.33) + for pair in new_tickets.windows(2) { + match pair[0].id.cmp(&pair[1].id) { + std::cmp::Ordering::Less => {} + std::cmp::Ordering::Equal => return Err(Error::DuplicateTicket), + std::cmp::Ordering::Greater => return Err(Error::BadTicketOrder), + } + } + + Ok(new_tickets) } diff --git a/crates/spacejam/src/fuzz/env.rs b/crates/spacejam/src/fuzz/env.rs index f8ac3e90..c4dde299 100644 --- a/crates/spacejam/src/fuzz/env.rs +++ b/crates/spacejam/src/fuzz/env.rs @@ -117,7 +117,7 @@ fn require_env(name: &str) -> Result { fn init_logger(level: Option<&str>) { let Some(level) = level else { return }; let _ = tracing_subscriber::fmt() - .with_env_filter(EnvFilter::new(level)) + .with_env_filter(EnvFilter::new(format!("spacejam={level}"))) .with_target(false) .with_ansi(false) .try_init(); diff --git a/crates/spacejam/src/fuzz/target.rs b/crates/spacejam/src/fuzz/target.rs index add89336..9f6e25b0 100644 --- a/crates/spacejam/src/fuzz/target.rs +++ b/crates/spacejam/src/fuzz/target.rs @@ -91,10 +91,12 @@ impl Target { match message { Message::Info(info) => self.info(info), Message::ImportBlock(block) => { + let slot = block.header.slot; if let Err(e) = self.import_block(block).await { - tracing::warn!("failed to import block: {e}"); + tracing::debug!("failed to import block#{slot}: {e}"); self.write_message(Message::Error(e.to_string()))?; } + tracing::info!("imported block#{slot}"); Ok(()) } Message::Initialize(state) => self.initialize(state).await, @@ -137,7 +139,7 @@ impl Target { /// Received set state request #[tracing::instrument(skip_all, name = "initialize")] pub async fn initialize(&mut self, state: Initialize) -> Result<()> { - lazy::clear().await; + lazy::clear(); self.chain = Default::default(); let root = self.chain.init(state.keyvals())?; if let Err(e) = self.init_state().await { diff --git a/crates/spacejam/src/storage/parity.rs b/crates/spacejam/src/storage/parity.rs index ae88fe56..2a21e329 100644 --- a/crates/spacejam/src/storage/parity.rs +++ b/crates/spacejam/src/storage/parity.rs @@ -1,11 +1,18 @@ //! The parity database storage use anyhow::Result; -use parity_db::{BTreeIterator, ColumnOptions, Db, Operation as Op, Options}; -use runtime::storage::{Column, Commit, KVStorage, Operation}; -use score::TrieKey; +use parity_db::{ + BTreeIterator, ColumnOptions, Db, NewNode as PdNewNode, NodeRef as PdNodeRef, Operation as Op, + Options, +}; +use runtime::storage::{ + Column, Commit, KVStorage, MultiTree, NewNode, NodeAddress, NodeRef, Operation, +}; +use score::{OpaqueHash, TrieKey}; use std::path::PathBuf; +const TRIE_COL: u8 = Column::TrieNodes as u8; + /// The parity database storage pub struct Parity(Db); @@ -46,6 +53,42 @@ impl KVStorage for Parity { } } +impl MultiTree for Parity { + fn insert_tree(&self, key: OpaqueHash, root: NewNode) -> Result<()> { + self.0 + .commit_changes([(TRIE_COL, Op::InsertTree(key.to_vec(), to_pd_newnode(root)))])?; + Ok(()) + } + + fn dereference_tree(&self, key: OpaqueHash) -> Result<()> { + self.0 + .commit_changes([(TRIE_COL, Op::DereferenceTree(key.to_vec()))])?; + Ok(()) + } + + fn get_root(&self, key: OpaqueHash) -> Result, Vec)>> { + Ok(self.0.get_root(TRIE_COL, key.as_ref())?) + } + + fn get_node(&self, address: NodeAddress) -> Result, Vec)>> { + Ok(self.0.get_node(TRIE_COL, address)?) + } +} + +fn to_pd_newnode(n: NewNode) -> PdNewNode { + PdNewNode { + data: n.data, + children: n.children.into_iter().map(to_pd_noderef).collect(), + } +} + +fn to_pd_noderef(r: NodeRef) -> PdNodeRef { + match r { + NodeRef::New(n) => PdNodeRef::New(to_pd_newnode(n)), + NodeRef::Existing(addr) => PdNodeRef::Existing(addr), + } +} + /// The iterator wrapper pub struct ParityIter<'a>(BTreeIterator<'a>); @@ -64,18 +107,28 @@ impl TryFrom for Parity { let options = Options { path, columns: vec![ + // Column::State ColumnOptions { btree_index: true, ..Default::default() }, + // Column::Sync ColumnOptions { btree_index: true, ..Default::default() }, + // Column::Archive ColumnOptions { btree_index: true, ..Default::default() }, + // Column::TrieNodes (multitree, content-addressed trie nodes) + ColumnOptions { + multitree: true, + preimage: true, + allow_direct_node_access: true, + ..Default::default() + }, ], sync_wal: true, sync_data: true, diff --git a/crates/testing/build.rs b/crates/testing/build.rs index a00e701c..84a7659b 100644 --- a/crates/testing/build.rs +++ b/crates/testing/build.rs @@ -32,7 +32,7 @@ fn main() -> Result<()> { let scale = scale(); // set up the registry - try_download(&workspace)?; + util::try_download(&workspace)?; let registry = Registry::with_scale(workspace.join("res/jam-test-vectors"), scale); // build all tests @@ -54,7 +54,6 @@ fn main() -> Result<()> { build_tests(registry.shuffle()?, &out_dir.join("shuffle.rs"))?; build_tests(registry.statistics(scale)?, &out_dir.join("statistics.rs"))?; build_tests(registry.trie()?, &out_dir.join("trie.rs"))?; - // Trace and sequential tests are tiny-only if scale == Scale::Tiny { build_tests( registry.trace(Trace::Fallback)?, @@ -88,7 +87,6 @@ fn main() -> Result<()> { registry.trace(Trace::StorageLight)?, &out_dir.join("traces_storage_light.rs"), )?; - build_all_seq_test(&out_dir.join("traces_seq.rs"))?; } else { for name in [ "traces_fallback", @@ -99,31 +97,19 @@ fn main() -> Result<()> { "traces_preimages_light", "traces_storage", "traces_storage_light", - "traces_seq", ] { fs::write(out_dir.join(format!("{name}.rs")), "")?; } } + build_all_seq_test(&out_dir.join("traces_seq.rs"), scale)?; Ok(()) } -fn scale_constructor() -> proc_macro2::TokenStream { - if env::var("CARGO_FEATURE_FULL").is_ok() { - quote::quote!(specjam::Registry::with_scale( - "../../res/jam-test-vectors", - specjam::Scale::Full - )) - } else { - quote::quote!(specjam::Registry::new("../../res/jam-test-vectors")) - } -} - fn build_tests(entry: Entry, out: &Path) -> Result<()> { let mut tests: Vec = Vec::new(); let section = entry.section; let ss = section.as_ref(); - let registry = scale_constructor(); - + let registry = util::scale_constructor(); for (i, path) in entry.files.iter().enumerate() { let name = Entry::file_name(path)?; let test_name = Ident::new(&format!("test_{name}"), Span::call_site()); @@ -144,8 +130,7 @@ fn build_pvmc_tests(entry: Entry, out: &Path) -> Result<()> { let mut tests: Vec = Vec::new(); let section = entry.section; let ss = section.as_ref(); - let registry = scale_constructor(); - + let registry = util::scale_constructor(); for (i, path) in entry.files.iter().enumerate() { let name = Entry::file_name(path)?; let test_name = Ident::new(&format!("test_{name}"), Span::call_site()); @@ -163,25 +148,24 @@ fn build_pvmc_tests(entry: Entry, out: &Path) -> Result<()> { } /// Builds all sequential tests -fn build_all_seq_test(out: &Path) -> Result<()> { +fn build_all_seq_test(out: &Path, scale: Scale) -> Result<()> { let mut items = Vec::new(); - for entry in [TRACES, REPORTS] { - for entry in fs::read_dir(entry)? { - let entry = entry?; - let path = entry.path(); - // let dname = path.file_name().unwrap().to_string_lossy(); - /* if dname.contains("_4872") { - continue; - } */ - - if path.is_dir() { - let testset = path.to_str().expect("failed to get testset"); - items.push(build_seq_test(testset)?); + if scale == Scale::Tiny { + for entry in [TRACES, REPORTS] { + for entry in fs::read_dir(entry)? { + let entry = entry?; + let path = entry.path(); + if path.is_dir() { + let testset = path.to_str().expect("failed to get testset"); + items.push(build_seq_test(testset)?); + } } } } - if Path::new(REPORT).is_dir() { + // REPORT dir can hold either tiny- or full-spec traces; the spec is + // declared in its report.json metadata. Only include if it matches. + if Path::new(REPORT).is_dir() && util::report_scale()? == Some(scale) { items.push(build_seq_test(REPORT)?); } @@ -194,9 +178,6 @@ fn build_seq_test(entry: &str) -> Result { let fentry = Entry::seq(entry)?; let test_name = Path::new(entry).file_name().unwrap().to_str().unwrap(); let mut tests = BTreeSet::::new(); - - // Iterate over file paths directly instead of parsing JSON files to avoid - // reading thousands of files during build time. Extract test names from filenames. for path in &fentry.files { let name = Entry::file_name(path)?; let names = name.split('_').collect::>(); @@ -225,36 +206,70 @@ fn build_seq_test(entry: &str) -> Result { Ok(testfn) } -fn try_download(workspace: &Path) -> Result<()> { - fs::create_dir_all(workspace.join("res"))?; - clone_if_missing( - workspace, - "https://github.com/spacejamapp/jam-test-vectors", - "res/jam-test-vectors", - )?; - clone_if_missing( - workspace, - "https://github.com/spacejamapp/jam-conformance", - "res/jam-conformance", - )?; - Ok(()) -} +mod util { + use super::*; -fn clone_if_missing(workspace: &Path, url: &str, dest: &str) -> Result<()> { - if workspace.join(dest).exists() { - return Ok(()); + pub fn try_download(workspace: &Path) -> Result<()> { + fs::create_dir_all(workspace.join("res"))?; + clone_if_missing( + workspace, + "https://github.com/spacejamapp/jam-test-vectors", + "res/jam-test-vectors", + )?; + clone_if_missing( + workspace, + "https://github.com/spacejamapp/jam-conformance", + "res/jam-conformance", + )?; + Ok(()) } - let output = Command::new("git") - .args(["clone", url, dest, "--depth", "1"]) - .current_dir(workspace) - .output() - .map_err(|e| anyhow::anyhow!("failed to spawn `git clone {url}`: {e}"))?; - if !output.status.success() { - anyhow::bail!( - "git clone {url} into {dest} failed ({}): {}", - output.status, - String::from_utf8_lossy(&output.stderr).trim() - ); + + pub fn clone_if_missing(workspace: &Path, url: &str, dest: &str) -> Result<()> { + if workspace.join(dest).exists() { + return Ok(()); + } + let output = Command::new("git") + .args(["clone", url, dest, "--depth", "1"]) + .current_dir(workspace) + .output() + .map_err(|e| anyhow::anyhow!("failed to spawn `git clone {url}`: {e}"))?; + if !output.status.success() { + anyhow::bail!( + "git clone {url} into {dest} failed ({}): {}", + output.status, + String::from_utf8_lossy(&output.stderr).trim() + ); + } + Ok(()) + } + + /// Read `res/report/report.json` and return the spec scale it declares, + /// or None if the file is missing or unrecognized. + pub fn report_scale() -> Result> { + let path = Path::new(REPORT).join("report.json"); + if !path.exists() { + return Ok(None); + } + let content = fs::read_to_string(&path)?; + // Strip whitespace so we tolerate any pretty-printing of the JSON. + let stripped: String = content.chars().filter(|c| !c.is_whitespace()).collect(); + if stripped.contains(r#""jam_spec":{"full""#) { + Ok(Some(Scale::Full)) + } else if stripped.contains(r#""jam_spec":{"tiny""#) { + Ok(Some(Scale::Tiny)) + } else { + Ok(None) + } + } + + pub fn scale_constructor() -> proc_macro2::TokenStream { + if env::var("CARGO_FEATURE_FULL").is_ok() { + quote::quote!(specjam::Registry::with_scale( + "../../res/jam-test-vectors", + specjam::Scale::Full + )) + } else { + quote::quote!(specjam::Registry::new("../../res/jam-test-vectors")) + } } - Ok(()) } diff --git a/crates/testing/src/accumulate.rs b/crates/testing/src/accumulate.rs index daf1585d..71742e9c 100644 --- a/crates/testing/src/accumulate.rs +++ b/crates/testing/src/accumulate.rs @@ -29,6 +29,7 @@ pub async fn run(test: &specjam::Test) -> Result<()> { &input.pre_state.accumulated, &input.pre_state.privileges.into(), &Default::default(), + &Default::default(), accounts.clone(), Default::default(), )? @@ -41,6 +42,7 @@ pub async fn run(test: &specjam::Test) -> Result<()> { &input.pre_state.accumulated, &input.pre_state.privileges.into(), &Default::default(), + &Default::default(), accounts.clone(), Default::default(), )? diff --git a/crates/testing/src/assurances.rs b/crates/testing/src/assurances.rs index f425b8bc..187b3ad0 100644 --- a/crates/testing/src/assurances.rs +++ b/crates/testing/src/assurances.rs @@ -20,7 +20,6 @@ pub fn run(test: &specjam::Test) -> anyhow::Result<()> { let result = tx::assurance::available( &input.pre_state.avail_assignments, &input.pre_state.curr_validators, - input.input.slot, input.input.parent, &input.input.assurances, ); diff --git a/crates/testing/src/history.rs b/crates/testing/src/history.rs index 2f58941b..eb2f2fd0 100644 --- a/crates/testing/src/history.rs +++ b/crates/testing/src/history.rs @@ -17,10 +17,12 @@ pub fn run(test: &specjam::Test) -> anyhow::Result<()> { let input = TestInput::from_json(&test.input)?; let output = TestOutput::from_json(&test.output)?; let mut history = input.pre_state.beta.clone(); + if let Some(last) = history.history.last_mut() { + last.state_root = input.input.parent_state_root; + } history::import( &mut history, input.input.header_hash, - input.input.parent_state_root, input.input.accumulate_root, input.input.work_packages.clone(), ); diff --git a/crates/testing/src/preimage.rs b/crates/testing/src/preimage.rs index 68176a44..2d374658 100644 --- a/crates/testing/src/preimage.rs +++ b/crates/testing/src/preimage.rs @@ -17,20 +17,20 @@ pub fn run(test: &specjam::Test) -> anyhow::Result<()> { let output = TestOutput::from_json(&test.output)?; // Validate post state - let accounts = to_accounts(input.pre_state.accounts.clone()); - let result = tx::preimage::accounts(input.input.slot, &input.input.preimages, accounts); - if let Ok(accounts) = result { - assert_eq!( - accounts - .accounts() - .iter() - .map(|(id, account)| (*id, account.account())) - .collect::>(), - self::to_accounts(output.post_state.accounts) - ); - } else { + let mut accounts = to_accounts(input.pre_state.accounts.clone()); + if tx::preimage::validate(&mut accounts, &input.input.preimages).is_err() { assert_eq!(input.pre_state, output.post_state); + return Ok(()); } + let accounts = tx::preimage::accounts(input.input.slot, &input.input.preimages, accounts); + assert_eq!( + accounts + .accounts() + .iter() + .map(|(id, account)| (*id, account.account())) + .collect::>(), + self::to_accounts(output.post_state.accounts) + ); Ok(()) } diff --git a/crates/vm/src/host/accumulate.rs b/crates/vm/src/host/accumulate.rs index 927f76ba..c028d2eb 100644 --- a/crates/vm/src/host/accumulate.rs +++ b/crates/vm/src/host/accumulate.rs @@ -55,10 +55,10 @@ pub fn bless(ctx: &mut impl Argument) -> Result { } } - // (m, v, r) Check if bless and designate are valid service IDs + // (m, v, r) must fit in u32; GP allows any serviceid including the protected range. if [bless, designate, register] .iter() - .any(|&id| (id != 0 && id < score::MINIMUM_SERVICE_ID as u64) || id > u32::MAX as u64) + .any(|&id| id > u32::MAX as u64) { return Ok(Exit::Who as u64); } @@ -81,19 +81,16 @@ pub fn assign(ctx: &mut impl Argument) -> Result { let mut source = vec![0u8; 32 * score::QUEUE_ITEMS as usize]; ctx.read_into(o as u32, &mut source)?; - // Check core index first (before reading memory) if core >= score::CORES_COUNT as u64 { return Ok(Exit::Core as u64); } - // Check if the calling service is the current assign service for this core let privileges = ctx.privileges(); if ctx.service() != privileges.assign[core as usize] { return Ok(Exit::Huh as u64); } - // Validate assign parameter is a valid service ID - if (assign != 0 && assign < score::MINIMUM_SERVICE_ID as u64) || assign > u32::MAX as u64 { + if assign > u32::MAX as u64 { return Ok(Exit::Who as u64); } @@ -157,28 +154,38 @@ pub fn checkpoint(ctx: &mut impl Argument) -> Result { /// (ΩN) new pub fn new_(ctx: &mut impl Argument) -> Result { - let [o, length, accumulate_gas, transfer_gas, f] = [ + let [o, length, accumulate_gas, transfer_gas, gratis, desiredid] = [ ctx.rget(7), ctx.rget(8), ctx.rget(9), ctx.rget(10), ctx.rget(11), + ctx.rget(12), ]; - // check if the service is blessed + // Non-zero gratis requires manager privilege. let privileges = ctx.privileges(); - if f != 0 && privileges.bless != ctx.service() { + if gratis != 0 && privileges.bless != ctx.service() { return Ok(Exit::Huh as u64); } - // get the account code let code = ctx.read_hash(o as u32)?; if length > u32::MAX as u64 { crate::bail!("Invalid length"); } - // create a new account with proper storage accounting - let index = ctx.index(); + // Registrar can target a protected-range index directly via r_12. + let registrar_target = + ctx.service() == privileges.register && desiredid < score::MINIMUM_SERVICE_ID as u64; + let index = if registrar_target { + if ctx.account(desiredid).is_ok() { + return Ok(Exit::Full as u64); + } + desiredid as u32 + } else { + ctx.index() + }; + let mut created = ServiceAccount { index, info: ServiceInfo { @@ -186,6 +193,7 @@ pub fn new_(ctx: &mut impl Argument) -> Result { balance: score::BALANCE_PER_SERVICE, accumulate: accumulate_gas, transfer: transfer_gas, + offset: gratis, creation: ctx.timeslot(), update: 0, parent: ctx.service(), @@ -195,23 +203,24 @@ pub fn new_(ctx: &mut impl Argument) -> Result { }; created.insert_lookup(code, length as u32, vec![]); - // Calculate the full threshold cost let new_account_threshold = created.threshold(); let service = ctx.this()?; if service.balance() < service.threshold() + new_account_threshold { return Ok(Exit::Cash as u64); } - // Deduct full threshold from parent and give it to new account *service.balance_mut() -= new_account_threshold; created.info.balance = new_account_threshold; ctx.upsert(index, created); - // Check and find a free account index - let base = - score::MINIMUM_SERVICE_ID + (index - score::MINIMUM_SERVICE_ID + 42) % score::CHECK_SALT; - let new_index = ctx.check(base); - ctx.set_index(new_index); + // Auto-generated path bumps nextfreeid; registrar path doesn't. + if !registrar_target { + let base = score::MINIMUM_SERVICE_ID + + (index - score::MINIMUM_SERVICE_ID + 42) % score::CHECK_SALT; + let new_index = ctx.check(base); + ctx.set_index(new_index); + } + Ok(index as u64) } @@ -357,16 +366,24 @@ pub fn solicit(ctx: &mut impl Argument) -> Result { let timeslot = ctx.timeslot(); let account = ctx.this()?; let mut slots = vec![]; - if let Some(lookup) = account.lookup(hash, z as u32).flatten() { + let appending = if let Some(lookup) = account.lookup(hash, z as u32).flatten() { if lookup.len() == 2 { slots = vec![lookup[0], lookup[1], timeslot]; + true } else { return Ok(Exit::Huh as u64); } - } + } else { + false + }; - // pre-calculate the new threshold - let threshold = account.lookup_threshold(z).unwrap_or(u64::MAX); + // Appending to an existing 2-slot entry doesn't change items/octets; only + // a fresh insertion changes the threshold. + let threshold = if appending { + account.threshold() + } else { + account.lookup_threshold(z).unwrap_or(u64::MAX) + }; if account.balance() < threshold { return Ok(Exit::Full as u64); } @@ -440,8 +457,6 @@ pub fn provide(ctx: &mut impl Argument) -> Result { return Ok(Exit::Huh as u64); } - // FIXME: the lookup insert is not specified in graypper, could be a bug - // in the fuzzy tests or our implementation. account.insert_lookup(hash, size as u32, vec![timeslot]); account.insert_preimage(hash, preimage); Ok(Exit::Ok as u64) diff --git a/crates/vm/src/host/general.rs b/crates/vm/src/host/general.rs index 6b08cb33..f436081f 100644 --- a/crates/vm/src/host/general.rs +++ b/crates/vm/src/host/general.rs @@ -131,6 +131,12 @@ pub fn write(ctx: &mut impl Argument) -> Result { // check if the account has enough balance to cover the threshold let account = ctx.this()?; let prev = account.read(&key); + + // Removing a key that doesn't exist is a no-op; skip the threshold check. + if vz == 0 && prev.is_none() { + return Ok(Exit::None as u64); + } + let threshold = account .write_threshold(&key, &value, prev.as_deref()) .unwrap_or(u64::MAX);