From 4f8cc58c186557b8270462028e5289d84dccb740 Mon Sep 17 00:00:00 2001 From: Gabriel Coutinho de Paula Date: Thu, 26 Feb 2026 16:22:40 -0300 Subject: [PATCH 1/6] feat: implement l2_tx_broadcaster --- AGENTS.md | 6 +- Cargo.lock | 90 +++++++ README.md | 28 ++- TODO.md | 6 +- justfile | 36 +++ sequencer/Cargo.toml | 6 +- sequencer/src/api/mod.rs | 164 +++++++++++- sequencer/src/l2_tx_broadcaster/mod.rs | 235 ++++++++++++++++++ sequencer/src/lib.rs | 1 + sequencer/src/main.rs | 34 +++ sequencer/src/storage/db.rs | 105 ++++++-- ...select_ordered_l2_txs_page_from_offset.sql | 5 + sequencer/src/storage/sql.rs | 28 ++- sequencer/tests/ws_broadcaster.rs | 233 +++++++++++++++++ 14 files changed, 941 insertions(+), 36 deletions(-) create mode 100644 justfile create mode 100644 sequencer/src/l2_tx_broadcaster/mod.rs create mode 100644 sequencer/src/storage/queries/select_ordered_l2_txs_page_from_offset.sql create mode 100644 sequencer/tests/ws_broadcaster.rs diff --git a/AGENTS.md b/AGENTS.md index 9e11288..3ecaf6f 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -33,12 +33,13 @@ Primary objective in this phase: make sequencer behavior, safety checks, and per ## Architecture Map - `sequencer/src/main.rs`: process bootstrap, env config, queue wiring, HTTP server. -- `sequencer/src/api/mod.rs`: `POST /tx` endpoint, JSON decode, signature recovery, enqueue + wait for commit result. +- `sequencer/src/api/mod.rs`: `POST /tx` and `GET /ws/subscribe` endpoints (tx ingress + replay broadcaster). - `sequencer/src/api/error.rs`: API error model + HTTP mapping. - `sequencer/src/inclusion_lane/mod.rs`: inclusion-lane exports and public surface. - `sequencer/src/inclusion_lane/lane.rs`: batched execution/commit loop (single lane). - `sequencer/src/inclusion_lane/types.rs`: inclusion-lane queue item and pipeline error types. - `sequencer/src/inclusion_lane/error.rs`: inclusion-lane runtime and catch-up error types. +- `sequencer/src/l2_tx_broadcaster.rs`: centralized ordered-L2Tx poller + live fanout to WS subscribers. - `sequencer/src/storage/mod.rs`: DB open, migrations, frame persistence, and direct-input broker APIs. - `sequencer/src/storage/migrations/`: DB schema/bootstrapping (`0001`) and views (`0002`). - `app-core/src/application/mod.rs`: app execution interface (`Application`) and wallet prototype. @@ -120,6 +121,9 @@ Key env vars: - `SEQ_INCLUSION_LANE_IDLE_POLL_INTERVAL_MS` (preferred) - `SEQ_INCLUSION_LANE_TICK_INTERVAL_MS` (legacy alias) - `SEQ_COMMIT_LANE_TICK_INTERVAL_MS` (legacy alias) +- `SEQ_BROADCASTER_IDLE_POLL_INTERVAL_MS` +- `SEQ_BROADCASTER_PAGE_SIZE` +- `SEQ_BROADCASTER_SUBSCRIBER_BUFFER_CAPACITY` - `SEQ_MAX_BODY_BYTES` - `SEQ_SQLITE_SYNCHRONOUS` - `SEQ_DOMAIN_NAME` diff --git a/Cargo.lock b/Cargo.lock index ee907cf..91749df 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -367,6 +367,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b52af3cb4058c895d37317bb27508dccc8e5f2d39454016b297bf4a400597b8" dependencies = [ "axum-core", + "base64", "bytes", "form_urlencoded", "futures-util", @@ -385,8 +386,10 @@ dependencies = [ "serde_json", "serde_path_to_error", "serde_urlencoded", + "sha1", "sync_wrapper", "tokio", + "tokio-tungstenite", "tower", "tower-layer", "tower-service", @@ -418,6 +421,12 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4c7f02d4ea65f2c1853089ffd8d2787bdbc63de2f0d29dedbcf8ccdfa0ccd4cf" +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + [[package]] name = "base64ct" version = "1.8.3" @@ -634,6 +643,12 @@ dependencies = [ "syn 2.0.114", ] +[[package]] +name = "data-encoding" +version = "2.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7a1e2f27636f116493b8b860f5546edb47c8d8f8ea73e1d2a20be88e28d1fea" + [[package]] name = "der" version = "0.7.10" @@ -942,6 +957,23 @@ version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e" +[[package]] +name = "futures-macro" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.114", +] + +[[package]] +name = "futures-sink" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e575fab7d1e0dcb8d0c7bcf9a63ee213816ab51902e6d244a95819acacf1d4f7" + [[package]] name = "futures-task" version = "0.3.31" @@ -955,9 +987,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81" dependencies = [ "futures-core", + "futures-macro", + "futures-sink", "futures-task", "pin-project-lite", "pin-utils", + "slab", ] [[package]] @@ -1858,12 +1893,14 @@ dependencies = [ "axum", "ethereum_ssz", "ethereum_ssz_derive", + "futures-util", "rusqlite", "rusqlite_migration", "serde", "serde_json", "thiserror 1.0.69", "tokio", + "tokio-tungstenite", "tower-http", "tracing", "tracing-subscriber", @@ -1935,6 +1972,17 @@ dependencies = [ "serde", ] +[[package]] +name = "sha1" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest 0.10.7", +] + [[package]] name = "sha2" version = "0.10.9" @@ -1991,6 +2039,12 @@ dependencies = [ "rand_core 0.6.4", ] +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + [[package]] name = "smallvec" version = "1.15.1" @@ -2161,6 +2215,7 @@ version = "1.49.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72a2903cd7736441aac9df9d7688bd0ce48edccaadf181c3b90be801e81d3d86" dependencies = [ + "bytes", "libc", "mio", "pin-project-lite", @@ -2180,6 +2235,18 @@ dependencies = [ "syn 2.0.114", ] +[[package]] +name = "tokio-tungstenite" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d25a406cddcc431a75d3d9afc6a7c0f7428d4891dd973e4d54c56b46127bf857" +dependencies = [ + "futures-util", + "log", + "tokio", + "tungstenite", +] + [[package]] name = "toml_datetime" version = "0.7.5+spec-1.1.0" @@ -2316,6 +2383,23 @@ dependencies = [ "tracing-log", ] +[[package]] +name = "tungstenite" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8628dcc84e5a09eb3d8423d6cb682965dea9133204e8fb3efee74c2a0c259442" +dependencies = [ + "bytes", + "data-encoding", + "http", + "httparse", + "log", + "rand 0.9.2", + "sha1", + "thiserror 2.0.18", + "utf-8", +] + [[package]] name = "typenum" version = "1.19.0" @@ -2364,6 +2448,12 @@ version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" +[[package]] +name = "utf-8" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" + [[package]] name = "valuable" version = "0.1.1" diff --git a/README.md b/README.md index 640acdb..0c34734 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ Current focus is reliability of sequencing, persistence, and replay semantics. ## Status - Language: Rust (edition 2024) -- API: Axum (`POST /tx`) +- API: Axum (`POST /tx`, `GET /ws/subscribe`) - Hot path: single blocking inclusion lane - Storage: SQLite (`rusqlite`, WAL) - Signing: EIP-712 (`alloy`) @@ -61,13 +61,33 @@ Request shape: } ``` -Notes: +POST notes: - `signature` must be 65 bytes. - `sender` is optional; if provided, it must match recovered signer. - `message.data` is SSZ-encoded method payload bytes. - payload size is bounded at ingress; oversized requests are rejected before they enter hot path. +### `GET /ws/subscribe?from_offset=` + +WebSocket stream of sequenced L2 transactions from persisted order. + +Notes: + +- `from_offset` is optional (defaults to `0`). +- messages are JSON text frames. +- binary fields are hex-encoded (`0x`-prefixed). + +Message shapes: + +```json +{ "kind": "user_op", "offset": 10, "sender": "0x...", "fee": 1, "data": "0x..." } +``` + +```json +{ "kind": "direct_input", "offset": 11, "payload": "0x..." } +``` + Success response: ```json @@ -94,6 +114,9 @@ Main environment variables: - `SEQ_INCLUSION_LANE_IDLE_POLL_INTERVAL_MS` - `SEQ_INCLUSION_LANE_TICK_INTERVAL_MS` (legacy alias) - `SEQ_COMMIT_LANE_TICK_INTERVAL_MS` (legacy alias) +- `SEQ_BROADCASTER_IDLE_POLL_INTERVAL_MS` +- `SEQ_BROADCASTER_PAGE_SIZE` +- `SEQ_BROADCASTER_SUBSCRIBER_BUFFER_CAPACITY` - `SEQ_MAX_BODY_BYTES` - `SEQ_SQLITE_SYNCHRONOUS` - `SEQ_DOMAIN_NAME` @@ -120,6 +143,7 @@ Views: - `sequencer/src/main.rs`: bootstrap, env config, HTTP server + lane lifecycle - `sequencer/src/api/`: HTTP API and error mapping - `sequencer/src/inclusion_lane/`: hot-path inclusion loop, chunk/frame/batch rotation, catch-up +- `sequencer/src/l2_tx_broadcaster.rs`: centralized ordered-L2Tx poller + subscriber fanout - `sequencer/src/storage/`: schema, migrations, SQLite persistence and replay reads - `app-core/src/application/`: app execution/validation interfaces + wallet prototype - `app-core/src/user_op.rs`: signed user-op and EIP-712 payload model diff --git a/TODO.md b/TODO.md index 3f7c28a..cb67121 100644 --- a/TODO.md +++ b/TODO.md @@ -10,13 +10,15 @@ Build a robust sequencer prototype for a future DeFi stack, with deterministic o ### 1) Sequencer -- Keep and harden write path: API -> inclusion lane -> app execution -> persistence. +- Keep and harden write path: API -> inclusion lane -> app execution -> persistence -> ack. - Implement direct-input reader from blockchain (ingests into `direct_inputs`). - Implement batch submitter (reads closed batches and submits on-chain). - Implement `L2Tx` broadcaster (WebSocket fanout of ordered `L2Tx`s). - Implement inclusion fee estimator module that updates the suggested fee in DB (`recommended_fees`). - Add API endpoint to query current suggested inclusion fee. +- Add API endpoint to query user current tx count. - Keep storage/replay semantics deterministic and catch-up-safe. +- Change `drain_n` design to a "safe block" design. --- @@ -35,7 +37,7 @@ Build a robust sequencer prototype for a future DeFi stack, with deterministic o - Measure ack latency and end-to-end latency. - Report p50 / p95 / p99. - Measure idle and under-load behavior. -- Include network-aware runs (client/server on different hosts). +- Include network-aware runs (client/server on different hosts) like network latency. - Note: end-to-end depends on `L2Tx` broadcaster being available. --- diff --git a/justfile b/justfile new file mode 100644 index 0000000..1a6c158 --- /dev/null +++ b/justfile @@ -0,0 +1,36 @@ +set shell := ["bash", "-euo", "pipefail", "-c"] + +default: + @just --list + +check: + cargo check --workspace + +check-all-targets: + cargo check --workspace --all-targets + +test: + cargo test --workspace + +test-sequencer: + cargo test -p sequencer --lib --tests + +fmt: + cargo fmt --all + +fmt-check: + cargo fmt --all --check + +clippy: + cargo clippy --workspace --all-targets --all-features -- -D warnings + +verify: fmt-check check test clippy + +ci: + cargo check --workspace --all-targets --locked + cargo build --workspace --all-targets --locked + cargo fmt --all -- --check + cargo test --workspace --all-targets --all-features --locked + +run addr="127.0.0.1:3000" db="sequencer.db": + SEQ_HTTP_ADDR={{addr}} SEQ_DB_PATH={{db}} cargo run -p sequencer diff --git a/sequencer/Cargo.toml b/sequencer/Cargo.toml index 4adbf62..d38fbfd 100644 --- a/sequencer/Cargo.toml +++ b/sequencer/Cargo.toml @@ -11,7 +11,7 @@ authors.workspace = true [dependencies] app-core = { path = "../app-core" } -axum = "0.8.8" +axum = { version = "0.8.8", features = ["ws"] } tokio = { version = "1.35", features = ["macros", "rt-multi-thread", "sync", "time", "net"] } serde = { version = "1", features = ["derive"] } serde_json = "1" @@ -25,3 +25,7 @@ alloy-sol-types = "1.4.1" thiserror = "1" ssz = { package = "ethereum_ssz", version = "0.10" } ssz_derive = { package = "ethereum_ssz_derive", version = "0.10" } + +[dev-dependencies] +futures-util = "0.3" +tokio-tungstenite = "0.28" diff --git a/sequencer/src/api/mod.rs b/sequencer/src/api/mod.rs index 802fccf..c4a2a8a 100644 --- a/sequencer/src/api/mod.rs +++ b/sequencer/src/api/mod.rs @@ -7,19 +7,23 @@ use std::sync::Arc; use std::time::{Duration, SystemTime}; use axum::Router; -use axum::extract::{DefaultBodyLimit, Json, State}; -use axum::routing::post; +use axum::extract::ws::{Message, WebSocket, WebSocketUpgrade}; +use axum::extract::{DefaultBodyLimit, Json, Query, State}; +use axum::response::IntoResponse; +use axum::routing::{get, post}; use serde::{Deserialize, Serialize}; use tokio::sync::mpsc; use tokio::sync::mpsc::error::SendTimeoutError; use tokio::sync::oneshot; use tower_http::trace::TraceLayer; -use tracing::info; +use tracing::{info, warn}; use alloy_primitives::{Address, Signature}; use alloy_sol_types::{Eip712Domain, SolStruct}; use crate::inclusion_lane::{InclusionLaneInput, PendingUserOp}; +use crate::l2_tx_broadcaster::{BroadcastTxMessage, L2TxBroadcaster}; +use crate::storage::Storage; use crate::user_op::{SignedUserOp, UserOp}; pub use error::ApiError; @@ -29,6 +33,7 @@ pub struct AppState { pub tx_sender: mpsc::Sender, pub domain: Eip712Domain, pub queue_timeout: Duration, + pub broadcaster: L2TxBroadcaster, } #[derive(Debug, Deserialize)] @@ -46,9 +51,15 @@ struct TxResponse { nonce: u32, } +#[derive(Debug, Deserialize)] +struct SubscribeQuery { + from_offset: Option, +} + pub fn router(state: Arc, max_body_bytes: usize) -> Router { Router::new() .route("/tx", post(submit_tx)) + .route("/ws/subscribe", get(subscribe_l2_txs)) .with_state(state) .layer(DefaultBodyLimit::max(max_body_bytes)) .layer(TraceLayer::new_for_http()) @@ -172,3 +183,150 @@ async fn enqueue_tx(state: &AppState, tx: PendingUserOp) -> Result<(), ApiError> } } } + +async fn subscribe_l2_txs( + State(state): State>, + Query(query): Query, + ws: WebSocketUpgrade, +) -> impl IntoResponse { + let from_offset = query.from_offset.unwrap_or(0); + let broadcaster = state.broadcaster.clone(); + ws.on_upgrade(move |socket| run_broadcaster_session(broadcaster, socket, from_offset)) +} + +async fn run_broadcaster_session( + broadcaster: L2TxBroadcaster, + mut socket: WebSocket, + from_offset: u64, +) { + let mut subscription = broadcaster.subscribe(); + let mut next_offset = from_offset; + + if next_offset < subscription.live_start_offset { + if send_catch_up( + &broadcaster, + &mut socket, + next_offset, + subscription.live_start_offset, + ) + .await + .is_err() + { + return; + } + next_offset = subscription.live_start_offset; + } + + loop { + tokio::select! { + maybe_event = subscription.receiver.recv() => { + let Some(event) = maybe_event else { + break; + }; + let offset = event.offset(); + if offset < next_offset { + continue; + } + if offset != next_offset { + warn!( + expected_offset = next_offset, + received_offset = offset, + "broadcaster detected gap in live stream" + ); + break; + } + if send_ws_event(&mut socket, &event).await.is_err() { + break; + } + next_offset = next_offset.saturating_add(1); + } + inbound = socket.recv() => { + match inbound { + Some(Ok(Message::Close(_))) | None => break, + Some(Ok(Message::Ping(payload))) => { + if socket.send(Message::Pong(payload)).await.is_err() { + break; + } + } + Some(Ok(_)) => {} + Some(Err(_)) => break, + } + } + } + } +} + +async fn send_catch_up( + broadcaster: &L2TxBroadcaster, + socket: &mut WebSocket, + from_offset: u64, + to_offset: u64, +) -> Result<(), ()> { + if from_offset >= to_offset { + return Ok(()); + } + + let (events_tx, mut events_rx) = mpsc::channel::(1024); + let db_path = broadcaster.db_path(); + let page_size = broadcaster.page_size(); + let worker = tokio::task::spawn_blocking(move || -> Result<(), String> { + let mut storage = Storage::open_read_only(&db_path) + .map_err(|err| format!("open catch-up storage failed: {err}"))?; + let mut next_offset = from_offset; + + while next_offset < to_offset { + let remaining = (to_offset - next_offset) as usize; + let page_limit = remaining.min(page_size.max(1)); + let txs = storage + .load_ordered_l2_txs_page_from(next_offset, page_limit) + .map_err(|err| format!("read catch-up page from {next_offset} failed: {err}"))?; + if txs.is_empty() { + return Err(format!( + "catch-up reached sparse range [{next_offset}, {to_offset})" + )); + } + + for tx in txs { + let event = BroadcastTxMessage::from_offset_and_tx(next_offset, tx); + next_offset = next_offset.saturating_add(1); + if events_tx.blocking_send(event).is_err() { + return Ok(()); + } + } + } + Ok(()) + }); + + while let Some(event) = events_rx.recv().await { + if send_ws_event(socket, &event).await.is_err() { + return Err(()); + } + } + + match worker.await { + Ok(Ok(())) => Ok(()), + Ok(Err(reason)) => { + warn!(reason, "broadcaster catch-up worker exited with error"); + Err(()) + } + Err(err) => { + warn!(error = %err, "broadcaster catch-up worker join failed"); + Err(()) + } + } +} + +async fn send_ws_event(socket: &mut WebSocket, event: &BroadcastTxMessage) -> Result<(), ()> { + let payload = match serde_json::to_string(event) { + Ok(value) => value, + Err(err) => { + warn!(error = %err, "broadcaster failed to serialize tx event"); + return Err(()); + } + }; + + if socket.send(Message::Text(payload.into())).await.is_err() { + return Err(()); + } + Ok(()) +} diff --git a/sequencer/src/l2_tx_broadcaster/mod.rs b/sequencer/src/l2_tx_broadcaster/mod.rs new file mode 100644 index 0000000..3e241ce --- /dev/null +++ b/sequencer/src/l2_tx_broadcaster/mod.rs @@ -0,0 +1,235 @@ +// (c) Cartesi and individual authors (see AUTHORS) +// SPDX-License-Identifier: Apache-2.0 (see LICENSE) + +use std::collections::HashMap; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::{Arc, Mutex}; +use std::time::Duration; + +use serde::Serialize; +use tokio::sync::mpsc; +use tracing::warn; + +use crate::l2_tx::SequencedL2Tx; +use crate::storage::Storage; + +#[derive(Debug, Clone, Copy)] +pub struct L2TxBroadcasterConfig { + pub idle_poll_interval: Duration, + pub page_size: usize, + pub subscriber_buffer_capacity: usize, +} + +#[derive(Clone)] +pub struct L2TxBroadcaster { + inner: Arc, +} + +pub struct LiveSubscription { + pub receiver: mpsc::Receiver, + pub live_start_offset: u64, +} + +#[derive(Debug, Clone, Serialize)] +#[serde(tag = "kind", rename_all = "snake_case")] +pub enum BroadcastTxMessage { + UserOp { + offset: u64, + sender: String, + fee: u64, + data: String, + }, + DirectInput { + offset: u64, + payload: String, + }, +} + +struct L2TxBroadcasterInner { + db_path: String, + page_size: usize, + subscriber_buffer_capacity: usize, + head_offset: AtomicU64, + next_subscriber_id: AtomicU64, + subscribers: Mutex>>, +} + +impl L2TxBroadcaster { + pub fn start( + db_path: String, + config: L2TxBroadcasterConfig, + ) -> std::result::Result { + let mut storage = Storage::open_read_only(&db_path) + .map_err(|err| format!("open broadcaster storage failed: {err}"))?; + let head_offset = storage + .ordered_l2_tx_count() + .map_err(|err| format!("load broadcaster head offset failed: {err}"))?; + + let inner = Arc::new(L2TxBroadcasterInner { + db_path, + page_size: config.page_size.max(1), + subscriber_buffer_capacity: config.subscriber_buffer_capacity.max(1), + head_offset: AtomicU64::new(head_offset), + next_subscriber_id: AtomicU64::new(0), + subscribers: Mutex::new(HashMap::new()), + }); + + let worker_inner = Arc::clone(&inner); + tokio::task::spawn_blocking(move || { + run_poller(worker_inner, config.idle_poll_interval); + }); + + Ok(Self { inner }) + } + + pub fn subscribe(&self) -> LiveSubscription { + let (tx, rx) = mpsc::channel(self.inner.subscriber_buffer_capacity); + let subscriber_id = self + .inner + .next_subscriber_id + .fetch_add(1, Ordering::Relaxed); + let live_start_offset = self.inner.head_offset.load(Ordering::Acquire); + + let mut subscribers = self + .inner + .subscribers + .lock() + .expect("l2 tx broadcaster subscribers mutex poisoned"); + subscribers.insert(subscriber_id, tx); + + LiveSubscription { + receiver: rx, + live_start_offset, + } + } + + pub fn db_path(&self) -> String { + self.inner.db_path.clone() + } + + pub fn page_size(&self) -> usize { + self.inner.page_size + } +} + +impl BroadcastTxMessage { + pub fn offset(&self) -> u64 { + match self { + Self::UserOp { offset, .. } => *offset, + Self::DirectInput { offset, .. } => *offset, + } + } + + pub fn from_offset_and_tx(offset: u64, tx: SequencedL2Tx) -> Self { + match tx { + SequencedL2Tx::UserOp(user_op) => Self::UserOp { + offset, + sender: user_op.sender.to_string(), + fee: user_op.fee, + data: alloy_primitives::hex::encode_prefixed(user_op.data.as_slice()), + }, + SequencedL2Tx::Direct(direct) => Self::DirectInput { + offset, + payload: alloy_primitives::hex::encode_prefixed(direct.payload.as_slice()), + }, + } + } +} + +fn run_poller(inner: Arc, idle_poll_interval: Duration) { + let mut storage = match Storage::open_read_only(inner.db_path.as_str()) { + Ok(storage) => storage, + Err(err) => { + warn!(error = %err, "l2 tx broadcaster failed to open read-only storage"); + return; + } + }; + let mut next_offset = inner.head_offset.load(Ordering::Acquire); + + loop { + let txs = match storage.load_ordered_l2_txs_page_from(next_offset, inner.page_size) { + Ok(value) => value, + Err(err) => { + warn!( + error = %err, + offset = next_offset, + "l2 tx broadcaster failed to read ordered tx page" + ); + std::thread::sleep(idle_poll_interval); + continue; + } + }; + + if txs.is_empty() { + std::thread::sleep(idle_poll_interval); + continue; + } + + for tx in txs { + let event = BroadcastTxMessage::from_offset_and_tx(next_offset, tx); + next_offset = next_offset.saturating_add(1); + inner.head_offset.store(next_offset, Ordering::Release); + fanout_event(Arc::as_ref(&inner), event); + } + } +} + +fn fanout_event(inner: &L2TxBroadcasterInner, event: BroadcastTxMessage) { + let mut to_remove = Vec::new(); + let mut subscribers = inner + .subscribers + .lock() + .expect("l2 tx broadcaster subscribers mutex poisoned"); + + for (subscriber_id, sender) in subscribers.iter() { + if sender.try_send(event.clone()).is_err() { + to_remove.push(*subscriber_id); + } + } + + for subscriber_id in to_remove { + subscribers.remove(&subscriber_id); + warn!( + subscriber_id, + "l2 tx broadcaster dropped subscriber due to closed/full channel" + ); + } +} + +#[cfg(test)] +mod tests { + use super::BroadcastTxMessage; + use crate::l2_tx::{DirectInput, SequencedL2Tx, ValidUserOp}; + use alloy_primitives::Address; + + #[test] + fn broadcast_user_op_serializes_with_hex_data() { + let msg = BroadcastTxMessage::from_offset_and_tx( + 7, + SequencedL2Tx::UserOp(ValidUserOp { + sender: Address::from_slice(&[0x11; 20]), + fee: 3, + data: vec![0xaa, 0xbb], + }), + ); + let json = serde_json::to_string(&msg).expect("serialize"); + assert!(json.contains("\"kind\":\"user_op\"")); + assert!(json.contains("\"offset\":7")); + assert!(json.contains("\"fee\":3")); + assert!(json.contains("\"data\":\"0xaabb\"")); + } + + #[test] + fn broadcast_direct_input_serializes_with_hex_payload() { + let msg = BroadcastTxMessage::from_offset_and_tx( + 9, + SequencedL2Tx::Direct(DirectInput { + payload: vec![0xcc, 0xdd], + }), + ); + let json = serde_json::to_string(&msg).expect("serialize"); + assert!(json.contains("\"kind\":\"direct_input\"")); + assert!(json.contains("\"offset\":9")); + assert!(json.contains("\"payload\":\"0xccdd\"")); + } +} diff --git a/sequencer/src/lib.rs b/sequencer/src/lib.rs index acf8f8c..1959ece 100644 --- a/sequencer/src/lib.rs +++ b/sequencer/src/lib.rs @@ -9,5 +9,6 @@ pub mod api; pub mod application; pub mod inclusion_lane; pub mod l2_tx; +pub mod l2_tx_broadcaster; pub mod storage; pub mod user_op; diff --git a/sequencer/src/main.rs b/sequencer/src/main.rs index 609045e..b35c116 100644 --- a/sequencer/src/main.rs +++ b/sequencer/src/main.rs @@ -13,6 +13,7 @@ use sequencer::application::{WalletApp, WalletConfig}; use sequencer::inclusion_lane::{ InclusionLane, InclusionLaneConfig, InclusionLaneError, InclusionLaneInput, }; +use sequencer::l2_tx_broadcaster::{L2TxBroadcaster, L2TxBroadcasterConfig}; use sequencer::storage; const DEFAULT_HTTP_ADDR: &str = "127.0.0.1:3000"; @@ -24,6 +25,9 @@ const DEFAULT_SAFE_DIRECT_BUFFER_CAPACITY: usize = 256; const DEFAULT_MAX_BATCH_OPEN_DURATION: Duration = Duration::from_secs(2 * 60 * 60); const DEFAULT_MAX_BATCH_USER_OP_BYTES: usize = 1_048_576; // 1 MiB const DEFAULT_INCLUSION_LANE_IDLE_POLL_INTERVAL: Duration = Duration::from_millis(2); +const DEFAULT_BROADCASTER_IDLE_POLL_INTERVAL: Duration = Duration::from_millis(20); +const DEFAULT_BROADCASTER_PAGE_SIZE: usize = 256; +const DEFAULT_BROADCASTER_SUBSCRIBER_BUFFER_CAPACITY: usize = 1024; const DEFAULT_MAX_BODY_BYTES: usize = 128 * 1024; const DEFAULT_SQLITE_SYNCHRONOUS: &str = "NORMAL"; const DEFAULT_DOMAIN_NAME: &str = "CartesiAppSequencer"; @@ -58,11 +62,21 @@ async fn main() -> Result<(), Box> { }, ); let (mut inclusion_lane_handle, inclusion_lane_stop) = inclusion_lane.spawn(); + let broadcaster = L2TxBroadcaster::start( + config.db_path.clone(), + L2TxBroadcasterConfig { + idle_poll_interval: config.broadcaster_idle_poll_interval, + page_size: config.broadcaster_page_size, + subscriber_buffer_capacity: config.broadcaster_subscriber_buffer_capacity, + }, + ) + .map_err(|reason| format!("failed to start l2 tx broadcaster: {reason}"))?; let state = Arc::new(AppState { tx_sender: tx, domain, queue_timeout: std::time::Duration::from_millis(config.queue_timeout_ms), + broadcaster, }); let app = sequencer::api::router(state, config.max_body_bytes); @@ -105,6 +119,9 @@ struct Config { max_batch_open: Duration, max_batch_user_op_bytes: usize, inclusion_lane_idle_poll_interval: Duration, + broadcaster_idle_poll_interval: Duration, + broadcaster_page_size: usize, + broadcaster_subscriber_buffer_capacity: usize, max_body_bytes: usize, sqlite_synchronous: String, domain_name: String, @@ -155,6 +172,23 @@ impl Config { ) .max(1), ), + broadcaster_idle_poll_interval: Duration::from_millis( + env_u64( + "SEQ_BROADCASTER_IDLE_POLL_INTERVAL_MS", + DEFAULT_BROADCASTER_IDLE_POLL_INTERVAL.as_millis() as u64, + ) + .max(1), + ), + broadcaster_page_size: env_usize( + "SEQ_BROADCASTER_PAGE_SIZE", + DEFAULT_BROADCASTER_PAGE_SIZE, + ) + .max(1), + broadcaster_subscriber_buffer_capacity: env_usize( + "SEQ_BROADCASTER_SUBSCRIBER_BUFFER_CAPACITY", + DEFAULT_BROADCASTER_SUBSCRIBER_BUFFER_CAPACITY, + ) + .max(1), max_body_bytes: env_usize("SEQ_MAX_BODY_BYTES", DEFAULT_MAX_BODY_BYTES), sqlite_synchronous: env_string("SEQ_SQLITE_SYNCHRONOUS", DEFAULT_SQLITE_SYNCHRONOUS), domain_name: env_string("SEQ_DOMAIN_NAME", DEFAULT_DOMAIN_NAME), diff --git a/sequencer/src/storage/db.rs b/sequencer/src/storage/db.rs index e7b9cc0..9132c13 100644 --- a/sequencer/src/storage/db.rs +++ b/sequencer/src/storage/db.rs @@ -1,7 +1,7 @@ // (c) Cartesi and individual authors (see AUTHORS) // SPDX-License-Identifier: Apache-2.0 (see LICENSE) -use rusqlite::{Connection, Result, Transaction, TransactionBehavior}; +use rusqlite::{Connection, OpenFlags, Result, Transaction, TransactionBehavior}; use rusqlite_migration::{M, Migrations}; use std::time::{Duration, SystemTime, UNIX_EPOCH}; @@ -9,7 +9,8 @@ use super::sql::{ sql_count_user_ops_for_frame, sql_insert_direct_inputs_batch, sql_insert_frame_drain, sql_insert_open_batch, sql_insert_open_frame, sql_insert_user_ops_batch, sql_select_latest_batch_with_user_op_count, sql_select_latest_frame_in_batch_for_batch, - sql_select_max_direct_input_index, sql_select_ordered_l2_txs_from_offset, + sql_select_max_direct_input_index, sql_select_ordered_l2_tx_count, + sql_select_ordered_l2_txs_from_offset, sql_select_ordered_l2_txs_page_from_offset, sql_select_recommended_fee, sql_select_safe_inputs_range, sql_select_total_drained_direct_inputs, sql_update_recommended_fee, }; @@ -31,6 +32,19 @@ impl Storage { Ok(Self { conn }) } + pub fn open_without_migrations( + path: &str, + synchronous: &str, + ) -> std::result::Result { + let conn = Self::open_connection(path, synchronous)?; + Ok(Self { conn }) + } + + pub fn open_read_only(path: &str) -> std::result::Result { + let conn = Self::open_connection_read_only(path)?; + Ok(Self { conn }) + } + pub fn open_connection( path: &str, synchronous: &str, @@ -43,6 +57,15 @@ impl Storage { Ok(conn) } + pub fn open_connection_read_only( + path: &str, + ) -> std::result::Result { + let conn = Connection::open_with_flags(path, OpenFlags::SQLITE_OPEN_READ_ONLY)?; + conn.pragma_update(None, "query_only", "ON")?; + conn.pragma_update(None, "busy_timeout", 5000)?; + Ok(conn) + } + pub fn open_connection_with_migrations( path: &str, synchronous: &str, @@ -238,36 +261,62 @@ impl Storage { pub fn load_ordered_l2_txs_from(&mut self, offset: u64) -> Result> { // Read the persisted total order used by catch-up and downstream broadcasters. let rows = sql_select_ordered_l2_txs_from_offset(&self.conn, u64_to_i64(offset))?; - let mut out = Vec::new(); + Ok(decode_ordered_l2_txs(rows)) + } - for row in rows { - if row.kind == 0 { - let sender_bytes = row.sender.expect("ordered replay row: missing sender"); - assert_eq!( - sender_bytes.len(), - 20, - "ordered replay row: sender must be 20 bytes" - ); - - let entry = ValidUserOp { - sender: Address::from_slice(sender_bytes.as_slice()), - // Replay uses the persisted batch fee to mirror canonical execution. - fee: i64_to_u64(row.fee.expect("ordered replay row: missing fee")), - data: row.data.expect("ordered replay row: missing data"), - }; - out.push(SequencedL2Tx::UserOp(entry)); - } else { - let direct = DirectInput { - payload: row.payload.expect("ordered replay row: missing payload"), - }; - out.push(SequencedL2Tx::Direct(direct)); - } + pub fn load_ordered_l2_txs_page_from( + &mut self, + offset: u64, + limit: usize, + ) -> Result> { + if limit == 0 { + return Ok(Vec::new()); } - Ok(out) + let rows = sql_select_ordered_l2_txs_page_from_offset( + &self.conn, + u64_to_i64(offset), + usize_to_i64(limit), + )?; + Ok(decode_ordered_l2_txs(rows)) + } + + pub fn ordered_l2_tx_count(&mut self) -> Result { + let value = sql_select_ordered_l2_tx_count(&self.conn)?; + Ok(i64_to_u64(value)) } } +fn decode_ordered_l2_txs(rows: Vec) -> Vec { + let mut out = Vec::new(); + + for row in rows { + if row.kind == 0 { + let sender_bytes = row.sender.expect("ordered replay row: missing sender"); + assert_eq!( + sender_bytes.len(), + 20, + "ordered replay row: sender must be 20 bytes" + ); + + let entry = ValidUserOp { + sender: Address::from_slice(sender_bytes.as_slice()), + // Replay uses the persisted batch fee to mirror canonical execution. + fee: i64_to_u64(row.fee.expect("ordered replay row: missing fee")), + data: row.data.expect("ordered replay row: missing data"), + }; + out.push(SequencedL2Tx::UserOp(entry)); + } else { + let direct = DirectInput { + payload: row.payload.expect("ordered replay row: missing payload"), + }; + out.push(SequencedL2Tx::Direct(direct)); + } + } + + out +} + fn load_current_write_head(tx: &Transaction<'_>) -> Result { let (batch_index, batch_created_at, batch_fee, batch_user_op_count) = query_latest_batch(tx)?; let frame_in_batch = query_latest_frame_in_batch(tx, batch_index)?; @@ -399,6 +448,10 @@ fn u64_to_i64(value: u64) -> i64 { i64::try_from(value).unwrap_or(i64::MAX) } +fn usize_to_i64(value: usize) -> i64 { + i64::try_from(value).unwrap_or(i64::MAX) +} + fn i64_to_u64(value: i64) -> u64 { value.max(0) as u64 } diff --git a/sequencer/src/storage/queries/select_ordered_l2_txs_page_from_offset.sql b/sequencer/src/storage/queries/select_ordered_l2_txs_page_from_offset.sql new file mode 100644 index 0000000..eda939b --- /dev/null +++ b/sequencer/src/storage/queries/select_ordered_l2_txs_page_from_offset.sql @@ -0,0 +1,5 @@ +SELECT kind, sender, data, fee, payload +FROM ordered_sequenced_l2_txs +-- `kind ASC` guarantees user_ops (0) are replayed before drained directs (1) in each frame. +ORDER BY batch_index ASC, frame_in_batch ASC, kind ASC, pos ASC +LIMIT ?2 OFFSET ?1 diff --git a/sequencer/src/storage/sql.rs b/sequencer/src/storage/sql.rs index a91d437..de4dfc7 100644 --- a/sequencer/src/storage/sql.rs +++ b/sequencer/src/storage/sql.rs @@ -10,6 +10,8 @@ use crate::inclusion_lane::PendingUserOp; const SQL_SELECT_SAFE_INPUTS_RANGE: &str = include_str!("queries/select_safe_inputs_range.sql"); const SQL_SELECT_ORDERED_L2_TXS_FROM_OFFSET: &str = include_str!("queries/select_ordered_l2_txs_from_offset.sql"); +const SQL_SELECT_ORDERED_L2_TXS_PAGE_FROM_OFFSET: &str = + include_str!("queries/select_ordered_l2_txs_page_from_offset.sql"); const SQL_SELECT_LATEST_BATCH_WITH_USER_OP_COUNT: &str = include_str!("queries/select_latest_batch_with_user_op_count.sql"); const SQL_SELECT_LATEST_FRAME_IN_BATCH_FOR_BATCH: &str = @@ -17,6 +19,7 @@ const SQL_SELECT_LATEST_FRAME_IN_BATCH_FOR_BATCH: &str = const SQL_SELECT_USER_OP_COUNT_FOR_FRAME: &str = include_str!("queries/select_user_op_count_for_frame.sql"); const SQL_SELECT_MAX_DIRECT_INPUT_INDEX: &str = "SELECT MAX(direct_input_index) FROM direct_inputs"; +const SQL_SELECT_ORDERED_L2_TX_COUNT: &str = "SELECT COUNT(*) FROM ordered_sequenced_l2_txs"; const SQL_SELECT_RECOMMENDED_FEE: &str = "SELECT fee FROM recommended_fees WHERE singleton_id = 0 LIMIT 1"; const SQL_INSERT_DIRECT_INPUT: &str = @@ -131,6 +134,20 @@ pub(super) fn sql_select_ordered_l2_txs_from_offset( mapped.collect() } +pub(super) fn sql_select_ordered_l2_txs_page_from_offset( + conn: &Connection, + offset: i64, + limit: i64, +) -> Result> { + let mut stmt = conn.prepare_cached(SQL_SELECT_ORDERED_L2_TXS_PAGE_FROM_OFFSET)?; + let mapped = stmt.query_map(params![offset, limit], convert_row_to_ordered_l2_tx_row)?; + mapped.collect() +} + +pub(super) fn sql_select_ordered_l2_tx_count(conn: &Connection) -> Result { + conn.query_row(SQL_SELECT_ORDERED_L2_TX_COUNT, [], |row| row.get(0)) +} + pub(super) fn sql_select_latest_batch_with_user_op_count( tx: &Transaction<'_>, ) -> Result<(i64, i64, i64, i64)> { @@ -239,7 +256,8 @@ mod tests { sql_count_user_ops_for_frame, sql_insert_direct_inputs_batch, sql_insert_frame_drain, sql_insert_open_batch, sql_insert_open_frame, sql_insert_user_ops_batch, sql_select_latest_batch_with_user_op_count, sql_select_latest_frame_in_batch_for_batch, - sql_select_max_direct_input_index, sql_select_ordered_l2_txs_from_offset, + sql_select_max_direct_input_index, sql_select_ordered_l2_tx_count, + sql_select_ordered_l2_txs_from_offset, sql_select_ordered_l2_txs_page_from_offset, sql_select_recommended_fee, sql_select_safe_inputs_range, sql_select_total_drained_direct_inputs, sql_update_recommended_fee, }; @@ -368,6 +386,14 @@ mod tests { assert_eq!(rows[0].fee, Some(1)); assert_eq!(rows[1].kind, 1); assert_eq!(rows[1].fee, None); + + let paged = sql_select_ordered_l2_txs_page_from_offset(&conn, 1, 1).expect("query page"); + assert_eq!(paged.len(), 1); + assert_eq!(paged[0].kind, 1); + assert_eq!( + sql_select_ordered_l2_tx_count(&conn).expect("query ordered count"), + 2 + ); } #[test] diff --git a/sequencer/tests/ws_broadcaster.rs b/sequencer/tests/ws_broadcaster.rs new file mode 100644 index 0000000..a02abda --- /dev/null +++ b/sequencer/tests/ws_broadcaster.rs @@ -0,0 +1,233 @@ +// (c) Cartesi and individual authors (see AUTHORS) +// SPDX-License-Identifier: Apache-2.0 (see LICENSE) + +use std::io::ErrorKind; +use std::path::PathBuf; +use std::sync::Arc; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; + +use alloy_primitives::{Address, B256, Signature}; +use alloy_sol_types::Eip712Domain; +use futures_util::StreamExt; +use sequencer::api::{AppState, router}; +use sequencer::inclusion_lane::{InclusionLaneInput, PendingUserOp, SequencerError}; +use sequencer::l2_tx_broadcaster::{L2TxBroadcaster, L2TxBroadcasterConfig}; +use sequencer::storage::{IndexedDirectInput, Storage}; +use sequencer::user_op::{SignedUserOp, UserOp}; +use serde::Deserialize; +use tokio::sync::{mpsc, oneshot}; +use tokio_tungstenite::connect_async; +use tokio_tungstenite::tungstenite::Message; + +#[derive(Debug, Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case")] +enum WsTxMessage { + UserOp { + offset: u64, + sender: String, + fee: u64, + data: String, + }, + DirectInput { + offset: u64, + payload: String, + }, +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn ws_subscribe_streams_ordered_txs_from_offset_zero() { + let db_path = temp_db_path("ws-subscribe-zero"); + seed_ordered_txs(&db_path); + + let Some((addr, shutdown_tx, server_task)) = start_test_server(&db_path).await else { + return; + }; + let url = format!("ws://{addr}/ws/subscribe?from_offset=0"); + let (mut ws, _) = connect_async(url).await.expect("connect websocket"); + + let first = recv_tx_message(&mut ws).await; + let second = recv_tx_message(&mut ws).await; + drop(ws); + + shutdown_tx.send(()).expect("request shutdown"); + server_task.await.expect("join server task"); + + match first { + WsTxMessage::UserOp { + offset, + sender, + fee, + data, + } => { + assert_eq!(offset, 0); + assert_eq!(fee, 1); + assert_eq!(decode_hex_prefixed(data.as_str()), vec![0x42]); + assert_eq!( + decode_hex_prefixed(sender.as_str()), + vec![0x11; 20], + "sender should match persisted user-op sender" + ); + } + value => panic!("expected user_op at offset 0, got {value:?}"), + } + + match second { + WsTxMessage::DirectInput { offset, payload } => { + assert_eq!(offset, 1); + assert_eq!(decode_hex_prefixed(payload.as_str()), vec![0xaa]); + } + value => panic!("expected direct_input at offset 1, got {value:?}"), + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn ws_subscribe_resumes_from_given_offset() { + let db_path = temp_db_path("ws-subscribe-resume"); + seed_ordered_txs(&db_path); + + let Some((addr, shutdown_tx, server_task)) = start_test_server(&db_path).await else { + return; + }; + let url = format!("ws://{addr}/ws/subscribe?from_offset=1"); + let (mut ws, _) = connect_async(url).await.expect("connect websocket"); + + let first = recv_tx_message(&mut ws).await; + drop(ws); + + shutdown_tx.send(()).expect("request shutdown"); + server_task.await.expect("join server task"); + + match first { + WsTxMessage::DirectInput { offset, payload } => { + assert_eq!(offset, 1); + assert_eq!(decode_hex_prefixed(payload.as_str()), vec![0xaa]); + } + value => panic!("expected direct_input at offset 1, got {value:?}"), + } +} + +fn seed_ordered_txs(db_path: &str) { + let mut storage = Storage::open(db_path, "NORMAL").expect("open storage"); + let mut head = storage.load_open_state().expect("load open state"); + + let (respond_to, _recv) = oneshot::channel::>(); + let pending = PendingUserOp { + signed: SignedUserOp { + sender: Address::from_slice(&[0x11; 20]), + signature: Signature::test_signature(), + user_op: UserOp { + nonce: 0, + max_fee: 3, + data: vec![0x42].into(), + }, + }, + tx_hash: B256::from([0x77; 32]), + respond_to, + received_at: SystemTime::now(), + }; + + storage + .append_user_ops_chunk(&mut head, &[pending]) + .expect("append user-op chunk"); + storage + .append_safe_direct_inputs(&[IndexedDirectInput { + index: 0, + payload: vec![0xaa], + }]) + .expect("append direct input"); + storage + .close_frame_only(&mut head, 1) + .expect("close frame with one drained direct input"); +} + +async fn start_test_server( + db_path: &str, +) -> Option<( + std::net::SocketAddr, + oneshot::Sender<()>, + tokio::task::JoinHandle<()>, +)> { + let listener = match tokio::net::TcpListener::bind("127.0.0.1:0").await { + Ok(value) => value, + Err(err) if err.kind() == ErrorKind::PermissionDenied => { + eprintln!( + "skipping ws integration test: cannot bind test listener in this environment" + ); + return None; + } + Err(err) => panic!("bind test listener: {err}"), + }; + let addr = listener.local_addr().expect("read listener addr"); + + let (tx_sender, _rx) = mpsc::channel::(1); + let broadcaster = L2TxBroadcaster::start( + db_path.to_string(), + L2TxBroadcasterConfig { + idle_poll_interval: Duration::from_millis(2), + page_size: 64, + subscriber_buffer_capacity: 256, + }, + ) + .expect("start broadcaster"); + let state = Arc::new(AppState { + tx_sender, + domain: Eip712Domain { + name: None, + version: None, + chain_id: None, + verifying_contract: None, + salt: None, + }, + queue_timeout: Duration::from_millis(50), + broadcaster, + }); + let app = router(state, 128 * 1024); + let (shutdown_tx, shutdown_rx) = oneshot::channel::<()>(); + + let server = axum::serve(listener, app).with_graceful_shutdown(async move { + let _ = shutdown_rx.await; + }); + let task = tokio::spawn(async move { + server.await.expect("run test server"); + }); + + Some((addr, shutdown_tx, task)) +} + +async fn recv_tx_message( + ws: &mut tokio_tungstenite::WebSocketStream< + tokio_tungstenite::MaybeTlsStream, + >, +) -> WsTxMessage { + let received = tokio::time::timeout(Duration::from_secs(2), ws.next()) + .await + .expect("wait for websocket message") + .expect("websocket stream ended") + .expect("receive websocket frame"); + + let text = match received { + Message::Text(value) => value, + other => panic!("expected text frame, got {other:?}"), + }; + + serde_json::from_str(text.as_str()).expect("parse websocket tx message") +} + +fn decode_hex_prefixed(value: &str) -> Vec { + assert!(value.starts_with("0x"), "hex field must be 0x-prefixed"); + alloy_primitives::hex::decode(value).expect("decode hex") +} + +fn temp_db_path(name: &str) -> String { + let mut path = std::env::temp_dir(); + let unique = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_nanos(); + path.push(format!("sequencer-ws-broadcaster-{name}-{unique}.sqlite")); + path_to_string(path) +} + +fn path_to_string(path: PathBuf) -> String { + path.to_string_lossy().into_owned() +} From b7a461c40291cc1710085c995548593d68be463a Mon Sep 17 00:00:00 2001 From: Gabriel Coutinho de Paula Date: Thu, 26 Feb 2026 16:43:14 -0300 Subject: [PATCH 2/6] fix: remove l2_tx_broadcaster race condition --- sequencer/src/l2_tx_broadcaster/mod.rs | 64 +++++++++++++- sequencer/tests/ws_broadcaster.rs | 118 ++++++++++++++++++++++++- 2 files changed, 180 insertions(+), 2 deletions(-) diff --git a/sequencer/src/l2_tx_broadcaster/mod.rs b/sequencer/src/l2_tx_broadcaster/mod.rs index 3e241ce..6ff6c6c 100644 --- a/sequencer/src/l2_tx_broadcaster/mod.rs +++ b/sequencer/src/l2_tx_broadcaster/mod.rs @@ -88,7 +88,6 @@ impl L2TxBroadcaster { .inner .next_subscriber_id .fetch_add(1, Ordering::Relaxed); - let live_start_offset = self.inner.head_offset.load(Ordering::Acquire); let mut subscribers = self .inner @@ -96,6 +95,7 @@ impl L2TxBroadcaster { .lock() .expect("l2 tx broadcaster subscribers mutex poisoned"); subscribers.insert(subscriber_id, tx); + let live_start_offset = self.inner.head_offset.load(Ordering::Acquire); LiveSubscription { receiver: rx, @@ -199,8 +199,13 @@ fn fanout_event(inner: &L2TxBroadcasterInner, event: BroadcastTxMessage) { #[cfg(test)] mod tests { use super::BroadcastTxMessage; + use super::{L2TxBroadcaster, L2TxBroadcasterInner}; use crate::l2_tx::{DirectInput, SequencedL2Tx, ValidUserOp}; use alloy_primitives::Address; + use std::collections::HashMap; + use std::sync::atomic::AtomicU64; + use std::sync::{Arc, Mutex}; + use std::time::Duration; #[test] fn broadcast_user_op_serializes_with_hex_data() { @@ -232,4 +237,61 @@ mod tests { assert!(json.contains("\"offset\":9")); assert!(json.contains("\"payload\":\"0xccdd\"")); } + + #[test] + fn subscribe_observes_live_start_after_registering_subscriber() { + let broadcaster = L2TxBroadcaster { + inner: Arc::new(L2TxBroadcasterInner { + db_path: ":memory:".to_string(), + page_size: 1, + subscriber_buffer_capacity: 1, + head_offset: AtomicU64::new(0), + next_subscriber_id: AtomicU64::new(0), + subscribers: Mutex::new(HashMap::new()), + }), + }; + + for _ in 0..16 { + broadcaster + .inner + .head_offset + .store(0, std::sync::atomic::Ordering::Release); + broadcaster + .inner + .subscribers + .lock() + .expect("subscribers mutex") + .clear(); + + let guard = broadcaster + .inner + .subscribers + .lock() + .expect("subscribers mutex"); + let (tx, rx) = std::sync::mpsc::channel(); + let cloned = broadcaster.clone(); + let join = std::thread::spawn(move || { + let subscription = cloned.subscribe(); + tx.send(subscription.live_start_offset) + .expect("send live start offset"); + }); + + std::thread::sleep(Duration::from_millis(2)); + broadcaster + .inner + .head_offset + .store(1, std::sync::atomic::Ordering::Release); + drop(guard); + + let observed = rx + .recv_timeout(Duration::from_secs(1)) + .expect("recv live start offset"); + join.join().expect("join subscribe thread"); + + assert_eq!( + observed, 1, + "subscriber must observe current head after it is visible in subscriber set" + ); + } + } } diff --git a/sequencer/tests/ws_broadcaster.rs b/sequencer/tests/ws_broadcaster.rs index a02abda..6c9a9f4 100644 --- a/sequencer/tests/ws_broadcaster.rs +++ b/sequencer/tests/ws_broadcaster.rs @@ -8,7 +8,7 @@ use std::time::{Duration, SystemTime, UNIX_EPOCH}; use alloy_primitives::{Address, B256, Signature}; use alloy_sol_types::Eip712Domain; -use futures_util::StreamExt; +use futures_util::{SinkExt, StreamExt}; use sequencer::api::{AppState, router}; use sequencer::inclusion_lane::{InclusionLaneInput, PendingUserOp, SequencerError}; use sequencer::l2_tx_broadcaster::{L2TxBroadcaster, L2TxBroadcasterConfig}; @@ -106,6 +106,99 @@ async fn ws_subscribe_resumes_from_given_offset() { } } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn ws_subscribe_receives_live_events_after_subscribing() { + let db_path = temp_db_path("ws-subscribe-live"); + seed_ordered_txs(&db_path); + + let Some((addr, shutdown_tx, server_task)) = start_test_server(&db_path).await else { + return; + }; + + // Existing persisted offsets are [0, 2). Subscribe at 2 to exercise live-only delivery. + let url = format!("ws://{addr}/ws/subscribe?from_offset=2"); + let (mut ws, _) = connect_async(url).await.expect("connect websocket"); + + append_drained_direct_input(&db_path, 1, vec![0xbb]); + let live = recv_tx_message(&mut ws).await; + drop(ws); + + shutdown_tx.send(()).expect("request shutdown"); + server_task.await.expect("join server task"); + + match live { + WsTxMessage::DirectInput { offset, payload } => { + assert_eq!(offset, 2); + assert_eq!(decode_hex_prefixed(payload.as_str()), vec![0xbb]); + } + value => panic!("expected live direct_input at offset 2, got {value:?}"), + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn ws_subscribe_fanout_delivers_live_event_to_multiple_subscribers() { + let db_path = temp_db_path("ws-subscribe-fanout"); + seed_ordered_txs(&db_path); + + let Some((addr, shutdown_tx, server_task)) = start_test_server(&db_path).await else { + return; + }; + + let url = format!("ws://{addr}/ws/subscribe?from_offset=2"); + let (mut ws_a, _) = connect_async(url.as_str()) + .await + .expect("connect websocket A"); + let (mut ws_b, _) = connect_async(url).await.expect("connect websocket B"); + + append_drained_direct_input(&db_path, 1, vec![0xcd]); + + let event_a = recv_tx_message(&mut ws_a).await; + let event_b = recv_tx_message(&mut ws_b).await; + drop(ws_a); + drop(ws_b); + + shutdown_tx.send(()).expect("request shutdown"); + server_task.await.expect("join server task"); + + let assert_event = |event: WsTxMessage| match event { + WsTxMessage::DirectInput { offset, payload } => { + assert_eq!(offset, 2); + assert_eq!(decode_hex_prefixed(payload.as_str()), vec![0xcd]); + } + value => panic!("expected live direct_input at offset 2, got {value:?}"), + }; + assert_event(event_a); + assert_event(event_b); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn ws_subscribe_replies_with_pong_on_ping() { + let db_path = temp_db_path("ws-subscribe-ping-pong"); + seed_ordered_txs(&db_path); + + let Some((addr, shutdown_tx, server_task)) = start_test_server(&db_path).await else { + return; + }; + + let url = format!("ws://{addr}/ws/subscribe?from_offset=2"); + let (mut ws, _) = connect_async(url).await.expect("connect websocket"); + + ws.send(Message::Ping(vec![0x01, 0x02].into())) + .await + .expect("send ping frame"); + + let frame = recv_raw_message(&mut ws).await; + drop(ws); + + shutdown_tx.send(()).expect("request shutdown"); + server_task.await.expect("join server task"); + + match frame { + Message::Pong(payload) => assert_eq!(payload.as_ref(), [0x01, 0x02]), + value => panic!("expected pong frame, got {value:?}"), + } +} + fn seed_ordered_txs(db_path: &str) { let mut storage = Storage::open(db_path, "NORMAL").expect("open storage"); let mut head = storage.load_open_state().expect("load open state"); @@ -140,6 +233,17 @@ fn seed_ordered_txs(db_path: &str) { .expect("close frame with one drained direct input"); } +fn append_drained_direct_input(db_path: &str, index: u64, payload: Vec) { + let mut storage = Storage::open(db_path, "NORMAL").expect("open storage"); + let mut head = storage.load_open_state().expect("load open state"); + storage + .append_safe_direct_inputs(&[IndexedDirectInput { index, payload }]) + .expect("append direct input"); + storage + .close_frame_only(&mut head, 1) + .expect("close frame with one drained direct input"); +} + async fn start_test_server( db_path: &str, ) -> Option<( @@ -213,6 +317,18 @@ async fn recv_tx_message( serde_json::from_str(text.as_str()).expect("parse websocket tx message") } +async fn recv_raw_message( + ws: &mut tokio_tungstenite::WebSocketStream< + tokio_tungstenite::MaybeTlsStream, + >, +) -> Message { + tokio::time::timeout(Duration::from_secs(2), ws.next()) + .await + .expect("wait for websocket message") + .expect("websocket stream ended") + .expect("receive websocket frame") +} + fn decode_hex_prefixed(value: &str) -> Vec { assert!(value.starts_with("0x"), "hex field must be 0x-prefixed"); alloy_primitives::hex::decode(value).expect("decode hex") From 29987a4e4293eda7569b6efed526121619d36ec1 Mon Sep 17 00:00:00 2001 From: Gabriel Coutinho de Paula Date: Thu, 26 Feb 2026 21:33:00 -0300 Subject: [PATCH 3/6] test: add e2e tests --- Cargo.lock | 2 + sequencer/Cargo.toml | 1 + sequencer/tests/e2e_sequencer.rs | 360 +++++++++++++++++++++++++++++++ 3 files changed, 363 insertions(+) create mode 100644 sequencer/tests/e2e_sequencer.rs diff --git a/Cargo.lock b/Cargo.lock index 91749df..688d0c7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1254,6 +1254,7 @@ dependencies = [ "elliptic-curve", "once_cell", "sha2", + "signature", ] [[package]] @@ -1894,6 +1895,7 @@ dependencies = [ "ethereum_ssz", "ethereum_ssz_derive", "futures-util", + "k256", "rusqlite", "rusqlite_migration", "serde", diff --git a/sequencer/Cargo.toml b/sequencer/Cargo.toml index d38fbfd..1e1c962 100644 --- a/sequencer/Cargo.toml +++ b/sequencer/Cargo.toml @@ -29,3 +29,4 @@ ssz_derive = { package = "ethereum_ssz_derive", version = "0.10" } [dev-dependencies] futures-util = "0.3" tokio-tungstenite = "0.28" +k256 = "0.13.4" diff --git a/sequencer/tests/e2e_sequencer.rs b/sequencer/tests/e2e_sequencer.rs new file mode 100644 index 0000000..232f2cf --- /dev/null +++ b/sequencer/tests/e2e_sequencer.rs @@ -0,0 +1,360 @@ +// (c) Cartesi and individual authors (see AUTHORS) +// SPDX-License-Identifier: Apache-2.0 (see LICENSE) + +use std::io::ErrorKind; +use std::path::PathBuf; +use std::sync::Arc; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; + +use alloy_primitives::{Address, Signature, U256}; +use alloy_sol_types::{Eip712Domain, SolStruct}; +use futures_util::StreamExt; +use k256::ecdsa::SigningKey; +use k256::ecdsa::signature::hazmat::PrehashSigner; +use sequencer::api::{AppState, router}; +use sequencer::application::{Deposit, Method, WalletApp, WalletConfig}; +use sequencer::inclusion_lane::{ + InclusionLane, InclusionLaneConfig, InclusionLaneError, InclusionLaneInput, +}; +use sequencer::l2_tx_broadcaster::{L2TxBroadcaster, L2TxBroadcasterConfig}; +use sequencer::storage::Storage; +use sequencer::user_op::UserOp; +use serde::Deserialize; +use tokio::io::{AsyncReadExt, AsyncWriteExt}; +use tokio::sync::{mpsc, oneshot}; +use tokio_tungstenite::connect_async; +use tokio_tungstenite::tungstenite::Message; + +#[derive(Debug, Deserialize)] +struct TxResponse { + ok: bool, + tx_hash: String, + sender: String, + nonce: u32, +} + +#[derive(Debug, Deserialize)] +#[allow(dead_code)] +#[serde(tag = "kind", rename_all = "snake_case")] +enum WsTxMessage { + UserOp { + offset: u64, + sender: String, + fee: u64, + data: String, + }, + DirectInput { + offset: u64, + payload: String, + }, +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn e2e_submit_tx_ack_and_broadcast() { + let db_path = temp_db_path("full-e2e"); + let domain = test_domain(); + bootstrap_open_batch_fee_zero(&db_path); + + let Some(runtime) = start_full_server(&db_path, domain.clone()).await else { + return; + }; + + let ws_url = format!("ws://{}/ws/subscribe?from_offset=0", runtime.addr); + let (mut ws, _) = connect_async(ws_url).await.expect("connect websocket"); + + let signing_key = SigningKey::from_bytes((&[7_u8; 32]).into()).expect("create signing key"); + let sender = address_from_signing_key(&signing_key); + let method = Method::Deposit(Deposit { + amount: U256::from(5_u64), + to: sender, + }); + let user_op = UserOp { + nonce: 0, + max_fee: 0, + data: ssz::Encode::as_ssz_bytes(&method).into(), + }; + let signature_hex = sign_user_op_hex(&domain, &user_op, &signing_key); + + let request_body = serde_json::json!({ + "message": user_op, + "signature": signature_hex, + "sender": sender.to_string(), + }); + + let (status, response_body) = post_json(runtime.addr, "/tx", request_body.to_string()).await; + assert_eq!( + status, 200, + "submit tx should succeed: body={response_body}" + ); + + let response: TxResponse = + serde_json::from_str(response_body.as_str()).expect("parse response"); + assert!(response.ok); + assert_eq!(response.nonce, 0); + assert_eq!(response.sender, sender.to_string()); + assert!( + response.tx_hash.starts_with("0x"), + "response tx hash should be 0x-prefixed" + ); + + let first_message = recv_ws_message(&mut ws).await; + match first_message { + WsTxMessage::UserOp { + offset, + sender: ws_sender, + fee, + data, + } => { + assert_eq!(offset, 0); + assert_eq!(ws_sender, sender.to_string()); + assert_eq!(fee, 0); + assert_eq!( + decode_hex_prefixed(data.as_str()), + ssz::Encode::as_ssz_bytes(&method) + ); + } + value => panic!("expected user_op at offset 0, got {value:?}"), + } + + drop(ws); + shutdown_runtime(runtime).await; +} + +struct FullServerRuntime { + addr: std::net::SocketAddr, + shutdown_tx: oneshot::Sender<()>, + server_task: tokio::task::JoinHandle<()>, + lane_stop: sequencer::inclusion_lane::InclusionLaneStop, + lane_handle: tokio::task::JoinHandle, +} + +async fn start_full_server(db_path: &str, domain: Eip712Domain) -> Option { + let listener = match tokio::net::TcpListener::bind("127.0.0.1:0").await { + Ok(value) => value, + Err(err) if err.kind() == ErrorKind::PermissionDenied => { + eprintln!( + "skipping e2e integration test: cannot bind test listener in this environment" + ); + return None; + } + Err(err) => panic!("bind test listener: {err}"), + }; + let addr = listener.local_addr().expect("read listener addr"); + + let storage = Storage::open(db_path, "NORMAL").expect("open storage"); + let (tx, rx) = mpsc::channel::(128); + + let inclusion_lane = InclusionLane::new( + rx, + WalletApp::new(WalletConfig), + storage, + InclusionLaneConfig { + max_user_ops_per_chunk: 32, + safe_direct_buffer_capacity: 32, + max_batch_open: Duration::from_secs(60 * 60), + max_batch_user_op_bytes: 1_048_576, + idle_poll_interval: Duration::from_millis(2), + }, + ); + let (lane_handle, lane_stop) = inclusion_lane.spawn(); + + let broadcaster = L2TxBroadcaster::start( + db_path.to_string(), + L2TxBroadcasterConfig { + idle_poll_interval: Duration::from_millis(2), + page_size: 64, + subscriber_buffer_capacity: 256, + }, + ) + .expect("start broadcaster"); + + let state = Arc::new(AppState { + tx_sender: tx, + domain, + queue_timeout: Duration::from_millis(100), + broadcaster, + }); + let app = router(state, 128 * 1024); + + let (shutdown_tx, shutdown_rx) = oneshot::channel::<()>(); + let server = axum::serve(listener, app).with_graceful_shutdown(async move { + let _ = shutdown_rx.await; + }); + let server_task = tokio::spawn(async move { + server.await.expect("run test server"); + }); + + Some(FullServerRuntime { + addr, + shutdown_tx, + server_task, + lane_stop, + lane_handle, + }) +} + +async fn shutdown_runtime(runtime: FullServerRuntime) { + runtime + .shutdown_tx + .send(()) + .expect("request server shutdown"); + runtime.server_task.await.expect("join server task"); + runtime.lane_stop.request_shutdown(); + let lane_result = tokio::time::timeout(Duration::from_secs(2), runtime.lane_handle) + .await + .expect("wait for inclusion lane") + .expect("join inclusion lane task"); + assert!( + matches!(lane_result, InclusionLaneError::ShutdownRequested), + "expected shutdown result, got {lane_result}" + ); +} + +fn bootstrap_open_batch_fee_zero(db_path: &str) { + let mut storage = Storage::open(db_path, "NORMAL").expect("open storage"); + storage.set_recommended_fee(0).expect("set recommended fee"); + let mut head = storage.load_open_state().expect("load open state"); + storage + .close_frame_and_batch(&mut head, 0) + .expect("rotate batch to fee=0"); + assert_eq!(head.batch_fee, 0); +} + +fn sign_user_op_hex(domain: &Eip712Domain, user_op: &UserOp, signing_key: &SigningKey) -> String { + let hash = user_op.eip712_signing_hash(domain); + let k256_sig = signing_key + .sign_prehash(hash.as_slice()) + .expect("sign user op hash"); + + let sender = address_from_signing_key(signing_key); + let signature = [false, true] + .into_iter() + .map(|parity| Signature::from_signature_and_parity(k256_sig, parity)) + .find(|candidate| { + candidate + .recover_address_from_prehash(&hash) + .ok() + .map(|value| value == sender) + .unwrap_or(false) + }) + .expect("recoverable parity for signature"); + + alloy_primitives::hex::encode_prefixed(signature.as_bytes()) +} + +fn address_from_signing_key(signing_key: &SigningKey) -> Address { + let verifying = signing_key.verifying_key().to_encoded_point(false); + Address::from_raw_public_key(&verifying.as_bytes()[1..]) +} + +async fn post_json(addr: std::net::SocketAddr, path: &str, body: String) -> (u16, String) { + let mut stream = tokio::net::TcpStream::connect(addr) + .await + .expect("connect http socket"); + let request = format!( + "POST {path} HTTP/1.1\r\nHost: {addr}\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", + body.len() + ); + stream + .write_all(request.as_bytes()) + .await + .expect("write http request"); + stream.flush().await.expect("flush http request"); + + let mut response = Vec::new(); + let mut chunk = [0_u8; 1024]; + loop { + let read_result = tokio::time::timeout(Duration::from_secs(2), stream.read(&mut chunk)) + .await + .expect("timed out while reading http response") + .expect("read http response"); + if read_result == 0 { + break; + } + response.extend_from_slice(&chunk[..read_result]); + + if let Some((header_end, content_length)) = response_content_len(response.as_slice()) + && response.len() >= header_end.saturating_add(content_length) + { + break; + } + } + parse_http_response(response.as_slice()) +} + +fn parse_http_response(raw: &[u8]) -> (u16, String) { + let text = String::from_utf8(raw.to_vec()).expect("http response utf8"); + let mut sections = text.splitn(2, "\r\n\r\n"); + let headers = sections.next().unwrap_or_default(); + let body = sections.next().unwrap_or_default().to_string(); + + let mut header_lines = headers.lines(); + let status_line = header_lines.next().expect("http status line"); + let status = status_line + .split_whitespace() + .nth(1) + .expect("status code") + .parse::() + .expect("parse status code"); + (status, body) +} + +fn response_content_len(raw: &[u8]) -> Option<(usize, usize)> { + let header_end = raw.windows(4).position(|window| window == b"\r\n\r\n")? + 4; + let headers = std::str::from_utf8(&raw[..header_end]).ok()?; + let mut content_length = None; + for line in headers.lines() { + if let Some((name, value)) = line.split_once(':') + && name.eq_ignore_ascii_case("content-length") + { + content_length = value.trim().parse::().ok(); + break; + } + } + content_length.map(|len| (header_end, len)) +} + +async fn recv_ws_message( + ws: &mut tokio_tungstenite::WebSocketStream< + tokio_tungstenite::MaybeTlsStream, + >, +) -> WsTxMessage { + let frame = tokio::time::timeout(Duration::from_secs(2), ws.next()) + .await + .expect("wait for websocket frame") + .expect("websocket stream ended") + .expect("receive websocket frame"); + match frame { + Message::Text(value) => serde_json::from_str(value.as_str()).expect("parse ws payload"), + other => panic!("expected text ws frame, got {other:?}"), + } +} + +fn decode_hex_prefixed(value: &str) -> Vec { + assert!(value.starts_with("0x"), "hex field must be 0x-prefixed"); + alloy_primitives::hex::decode(value).expect("decode hex") +} + +fn test_domain() -> Eip712Domain { + Eip712Domain { + name: Some("CartesiAppSequencer".to_string().into()), + version: Some("1".to_string().into()), + chain_id: Some(U256::from(1_u64)), + verifying_contract: Some(Address::from_slice(&[0_u8; 20])), + salt: None, + } +} + +fn temp_db_path(name: &str) -> String { + let mut path = std::env::temp_dir(); + let unique = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_nanos(); + path.push(format!("sequencer-full-e2e-{name}-{unique}.sqlite")); + path_to_string(path) +} + +fn path_to_string(path: PathBuf) -> String { + path.to_string_lossy().into_owned() +} From 1ae02dca1d720d2d3cc75c5b42cd8e5fa5f7059c Mon Sep 17 00:00:00 2001 From: Gabriel Coutinho de Paula Date: Thu, 26 Feb 2026 22:43:56 -0300 Subject: [PATCH 4/6] perf: add benchmarks and optimisations --- .github/workflows/ci.yml | 2 + .gitignore | 4 + AGENTS.md | 29 +- Cargo.lock | 190 ++- Cargo.toml | 7 +- README.md | 40 +- TODO.md | 14 +- benchmarks/.gitignore | 3 +- benchmarks/BENCHMARK_SPEC.md | 142 ++ benchmarks/Cargo.toml | 25 + benchmarks/README.md | 65 + benchmarks/justfile | 55 + benchmarks/src/bin/ack_latency.rs | 263 ++++ benchmarks/src/bin/compare_latest.rs | 474 +++++++ benchmarks/src/bin/e2e_latency.rs | 299 ++++ benchmarks/src/bin/sweep.rs | 675 +++++++++ benchmarks/src/bin/unit_hot_path.rs | 58 + benchmarks/src/lib.rs | 1223 +++++++++++++++++ benchmarks/src/runtime.rs | 535 +++++++ examples/app-core/Cargo.toml | 15 + .../app-core/src/application/mod.rs | 4 +- .../app-core}/src/application/wallet.rs | 14 +- .../l2_tx.rs => examples/app-core/src/lib.rs | 2 +- .../canonical-app}/Cargo.toml | 2 +- .../canonical-app}/src/main.rs | 0 justfile | 10 +- {tools => sdk}/cli/.gitignore | 0 sdk/rust-client/.gitignore | 3 +- sdk/rust-client/Cargo.toml | 17 + sdk/rust-client/src/errors.rs | 66 + sdk/rust-client/src/lib.rs | 217 +++ {app-core => sequencer-core}/Cargo.toml | 2 +- sequencer-core/src/api.rs | 32 + .../src/application/method.rs | 0 .../src/application/mod.rs | 2 - sequencer-core/src/broadcast.rs | 44 + {app-core => sequencer-core}/src/l2_tx.rs | 2 +- {app-core => sequencer-core}/src/lib.rs | 2 + {app-core => sequencer-core}/src/user_op.rs | 3 +- sequencer/Cargo.toml | 7 +- sequencer/src/api/error.rs | 8 + sequencer/src/api/mod.rs | 195 ++- sequencer/src/application.rs | 4 - sequencer/src/inclusion_lane/error.rs | 2 +- sequencer/src/inclusion_lane/lane.rs | 205 ++- sequencer/src/inclusion_lane/mod.rs | 1 + sequencer/src/inclusion_lane/profiling.rs | 263 ++++ sequencer/src/inclusion_lane/types.rs | 5 +- sequencer/src/l2_tx_broadcaster/mod.rs | 138 +- sequencer/src/l2_tx_broadcaster/profiling.rs | 199 +++ sequencer/src/lib.rs | 3 - sequencer/src/main.rs | 676 +++++++-- sequencer/src/storage/db.rs | 165 ++- .../src/storage/migrations/0001_schema.sql | 65 +- .../src/storage/migrations/0002_views.sql | 66 - sequencer/src/storage/mod.rs | 11 +- .../queries/insert_sequenced_direct_input.sql | 6 + .../queries/insert_sequenced_user_op.sql | 6 + .../src/storage/queries/insert_user_op.sql | 3 +- ...select_latest_batch_with_user_op_count.sql | 15 +- ...select_latest_frame_in_batch_for_batch.sql | 4 +- .../select_ordered_l2_txs_from_offset.sql | 23 +- ...select_ordered_l2_txs_page_from_offset.sql | 24 +- .../select_user_op_count_for_frame.sql | 12 +- sequencer/src/storage/sql.rs | 218 ++- sequencer/tests/e2e_sequencer.rs | 574 +++++--- sequencer/tests/ws_broadcaster.rs | 409 ++++-- 67 files changed, 6958 insertions(+), 889 deletions(-) create mode 100644 benchmarks/BENCHMARK_SPEC.md create mode 100644 benchmarks/Cargo.toml create mode 100644 benchmarks/README.md create mode 100644 benchmarks/justfile create mode 100644 benchmarks/src/bin/ack_latency.rs create mode 100644 benchmarks/src/bin/compare_latest.rs create mode 100644 benchmarks/src/bin/e2e_latency.rs create mode 100644 benchmarks/src/bin/sweep.rs create mode 100644 benchmarks/src/bin/unit_hot_path.rs create mode 100644 benchmarks/src/lib.rs create mode 100644 benchmarks/src/runtime.rs create mode 100644 examples/app-core/Cargo.toml rename sequencer/src/user_op.rs => examples/app-core/src/application/mod.rs (65%) rename {app-core => examples/app-core}/src/application/wallet.rs (93%) rename sequencer/src/l2_tx.rs => examples/app-core/src/lib.rs (79%) rename {canonical-app => examples/canonical-app}/Cargo.toml (90%) rename {canonical-app => examples/canonical-app}/src/main.rs (100%) rename {tools => sdk}/cli/.gitignore (100%) create mode 100644 sdk/rust-client/Cargo.toml create mode 100644 sdk/rust-client/src/errors.rs create mode 100644 sdk/rust-client/src/lib.rs rename {app-core => sequencer-core}/Cargo.toml (95%) create mode 100644 sequencer-core/src/api.rs rename {app-core => sequencer-core}/src/application/method.rs (100%) rename {app-core => sequencer-core}/src/application/mod.rs (98%) create mode 100644 sequencer-core/src/broadcast.rs rename {app-core => sequencer-core}/src/l2_tx.rs (83%) rename {app-core => sequencer-core}/src/lib.rs (83%) rename {app-core => sequencer-core}/src/user_op.rs (86%) delete mode 100644 sequencer/src/application.rs create mode 100644 sequencer/src/inclusion_lane/profiling.rs create mode 100644 sequencer/src/l2_tx_broadcaster/profiling.rs delete mode 100644 sequencer/src/storage/migrations/0002_views.sql create mode 100644 sequencer/src/storage/queries/insert_sequenced_direct_input.sql create mode 100644 sequencer/src/storage/queries/insert_sequenced_user_op.sql diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9b1ff85..719ace2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -7,6 +7,7 @@ on: jobs: rust: runs-on: ubuntu-latest + timeout-minutes: 30 steps: - name: Checkout @@ -30,4 +31,5 @@ jobs: run: cargo fmt --all -- --check - name: Test + timeout-minutes: 15 run: cargo test --workspace --all-targets --all-features --locked diff --git a/.gitignore b/.gitignore index ea8c4bf..7d4ba02 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,5 @@ /target +sequencer.db +sequencer.db-shm +sequencer.db-wal +benchmarks/results/ diff --git a/AGENTS.md b/AGENTS.md index 3ecaf6f..3a05c17 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -39,22 +39,22 @@ Primary objective in this phase: make sequencer behavior, safety checks, and per - `sequencer/src/inclusion_lane/lane.rs`: batched execution/commit loop (single lane). - `sequencer/src/inclusion_lane/types.rs`: inclusion-lane queue item and pipeline error types. - `sequencer/src/inclusion_lane/error.rs`: inclusion-lane runtime and catch-up error types. -- `sequencer/src/l2_tx_broadcaster.rs`: centralized ordered-L2Tx poller + live fanout to WS subscribers. +- `sequencer/src/l2_tx_broadcaster/mod.rs`: centralized ordered-L2Tx poller + live fanout to WS subscribers. - `sequencer/src/storage/mod.rs`: DB open, migrations, frame persistence, and direct-input broker APIs. -- `sequencer/src/storage/migrations/`: DB schema/bootstrapping (`0001`) and views (`0002`). -- `app-core/src/application/mod.rs`: app execution interface (`Application`) and wallet prototype. -- `app-core/src/user_op.rs`: signed user-op domain types and EIP-712 payload. -- `app-core/src/l2_tx.rs`: sequenced L2 transaction types used for replay/broadcast boundaries. -- `canonical-app/src/main.rs`: placeholder canonical scheduler binary entrypoint. +- `sequencer/src/storage/migrations/`: DB schema/bootstrapping (`0001`). +- `sequencer-core/src/`: shared domain types/interfaces (`Application`, `SignedUserOp`, `SequencedL2Tx`, broadcast message model). +- `examples/app-core/src/application/mod.rs`: wallet prototype implementing `Application`. +- `examples/canonical-app/src/main.rs`: placeholder canonical scheduler binary entrypoint. ## Domain Truths (Important) - This is a **sequencer prototype**, not a full DeFi stack yet. - API validates signature and enqueues signed `UserOp`; method decoding happens during application execution. - Rejections (`InvalidNonce`, fee cap too low, insufficient gas balance) produce no state mutation and are not persisted. -- Included txs are persisted as frame/batch data in `batches`, `frames`, `user_ops`, `direct_inputs`, and `frame_drains`. -- Batch fee is persisted in `batches.fee` and is fixed for the lifetime of that batch. -- The next batch fee is sampled from `recommended_fees` when rotating to a new batch (default bootstrap value is `1`). +- Included txs are persisted as frame/batch data in `batches`, `frames`, `user_ops`, `direct_inputs`, and `sequenced_l2_txs`. +- Frame fee is persisted in `frames.fee` and is fixed for the lifetime of that frame. +- The next frame fee is sampled from `recommended_fees` when rotating to a new frame (default bootstrap value is `0`). +- `/ws/subscribe` has soft operational guardrails: subscriber cap (`SEQ_WS_MAX_SUBSCRIBERS`, default `64`) and catch-up cap (`SEQ_WS_MAX_CATCHUP_EVENTS`, default `50000`). - Wallet state (balances/nonces) is in-memory right now (not persisted). ## Hot-Path Invariants @@ -69,9 +69,10 @@ Primary objective in this phase: make sequencer behavior, safety checks, and per - Storage model is append-oriented; avoid mutable status flags for open/closed entities. - Open batch/frame are derived by “latest row” convention. -- `drain_n` is represented by `frame_drains` rows and is derivable from stored data. +- `drain_n` is derivable from `sequenced_l2_txs` by counting direct-input rows per frame. - Safe cursor/head values should be derived from persisted facts when possible, not duplicated as mutable fields. -- Replay/catch-up must use persisted ordering plus persisted batch fee (`batches.fee`) to mirror inclusion semantics. +- Replay/catch-up must use persisted ordering plus persisted frame fee (`frames.fee`) to mirror inclusion semantics. +- Included user-op identity is constrained by `UNIQUE(sender, nonce)`. ## Type Boundaries @@ -112,7 +113,7 @@ Key env vars: - `SEQ_HTTP_ADDR` - `SEQ_DB_PATH` - `SEQ_QUEUE_CAP` -- `SEQ_QUEUE_TIMEOUT_MS` +- `SEQ_OVERLOAD_MAX_INFLIGHT_SUBMISSIONS` - `SEQ_MAX_USER_OPS_PER_CHUNK` (preferred) - `SEQ_MAX_BATCH` (legacy alias) - `SEQ_SAFE_DIRECT_BUFFER_CAPACITY` @@ -124,6 +125,10 @@ Key env vars: - `SEQ_BROADCASTER_IDLE_POLL_INTERVAL_MS` - `SEQ_BROADCASTER_PAGE_SIZE` - `SEQ_BROADCASTER_SUBSCRIBER_BUFFER_CAPACITY` +- `SEQ_WS_MAX_SUBSCRIBERS` +- `SEQ_WS_MAX_CATCHUP_EVENTS` +- `SEQ_RUNTIME_METRICS_ENABLED` +- `SEQ_RUNTIME_METRICS_LOG_INTERVAL_MS` - `SEQ_MAX_BODY_BYTES` - `SEQ_SQLITE_SYNCHRONOUS` - `SEQ_DOMAIN_NAME` diff --git a/Cargo.lock b/Cargo.lock index 688d0c7..c0bddad 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -130,16 +130,63 @@ dependencies = [ "serde", ] +[[package]] +name = "anstream" +version = "0.6.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43d5b281e737544384e969a5ccad3f1cdd24b48086a0fc1b2a5262a26b8f4f4a" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5192cca8006f1fd4f7237516f40fa183bb07f8fbdfedaa0036de5ea9b0b45e78" + +[[package]] +name = "anstyle-parse" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e7644824f0aa2c7b9384579234ef10eb7efb6a0deb83f9630a49594dd9c15c2" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" +dependencies = [ + "anstyle", + "once_cell_polyfill", + "windows-sys 0.61.2", +] + [[package]] name = "app-core" version = "0.1.0" dependencies = [ "alloy-primitives", - "alloy-sol-types", "ethereum_ssz", - "ethereum_ssz_derive", - "serde", - "thiserror 1.0.69", + "sequencer-core", ] [[package]] @@ -433,6 +480,25 @@ version = "1.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06" +[[package]] +name = "benchmarks" +version = "0.1.0" +dependencies = [ + "alloy-primitives", + "alloy-sol-types", + "clap", + "ethereum_ssz", + "futures-util", + "k256", + "sequencer-core", + "sequencer-rust-client", + "serde", + "serde_json", + "tempfile", + "tokio", + "tokio-tungstenite", +] + [[package]] name = "bit-set" version = "0.8.0" @@ -525,6 +591,52 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" +[[package]] +name = "clap" +version = "4.5.60" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2797f34da339ce31042b27d23607e051786132987f595b02ba4f6a6dffb7030a" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.5.60" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24a241312cea5059b13574bb9b3861cabf758b879c15190b37b6d6fd63ab6876" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.5.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a92793da1a46a5f2a02a6f4c46c6496b28c43638adea8306fcb0caa1634f24e5" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 2.0.114", +] + +[[package]] +name = "clap_lex" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a822ea5bc7590f9d40f1ba12c0dc3c2760f3482c6984db1573ad11031420831" + +[[package]] +name = "colorchoice" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b05b61dc5112cbb17e4b6cd61790d9845d13888356391624cbe7e41efeac1e75" + [[package]] name = "const-hex" version = "1.17.0" @@ -1200,6 +1312,12 @@ dependencies = [ "serde_core", ] +[[package]] +name = "is_terminal_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" + [[package]] name = "itertools" version = "0.10.5" @@ -1410,6 +1528,12 @@ version = "1.21.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" +[[package]] +name = "once_cell_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" + [[package]] name = "parity-scale-codec" version = "3.7.5" @@ -1892,22 +2016,50 @@ dependencies = [ "alloy-sol-types", "app-core", "axum", + "clap", "ethereum_ssz", "ethereum_ssz_derive", "futures-util", "k256", "rusqlite", "rusqlite_migration", + "sequencer-core", + "sequencer-rust-client", "serde", "serde_json", + "tempfile", "thiserror 1.0.69", "tokio", "tokio-tungstenite", + "tower", "tower-http", "tracing", "tracing-subscriber", ] +[[package]] +name = "sequencer-core" +version = "0.1.0" +dependencies = [ + "alloy-primitives", + "alloy-sol-types", + "ethereum_ssz", + "ethereum_ssz_derive", + "serde", + "thiserror 1.0.69", +] + +[[package]] +name = "sequencer-rust-client" +version = "0.1.0" +dependencies = [ + "sequencer-core", + "serde_json", + "thiserror 1.0.69", + "tokio", + "tokio-tungstenite", +] + [[package]] name = "serde" version = "1.0.228" @@ -2031,6 +2183,15 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" +[[package]] +name = "signal-hook-registry" +version = "1.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2a4719bff48cee6b39d12c020eeb490953ad2443b7055bd0b21fca26bd8c28b" +dependencies = [ + "libc", +] + [[package]] name = "signature" version = "2.2.0" @@ -2221,6 +2382,7 @@ dependencies = [ "libc", "mio", "pin-project-lite", + "signal-hook-registry", "socket2", "tokio-macros", "windows-sys 0.61.2", @@ -2249,6 +2411,19 @@ dependencies = [ "tungstenite", ] +[[package]] +name = "tokio-util" +version = "0.7.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ae9cec805b01e8fc3fd2fe289f89149a9b66dd16786abd8b19cfa7b48cb0098" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "pin-project-lite", + "tokio", +] + [[package]] name = "toml_datetime" version = "0.7.5+spec-1.1.0" @@ -2290,6 +2465,7 @@ dependencies = [ "pin-project-lite", "sync_wrapper", "tokio", + "tokio-util", "tower-layer", "tower-service", "tracing", @@ -2456,6 +2632,12 @@ version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + [[package]] name = "valuable" version = "0.1.1" diff --git a/Cargo.toml b/Cargo.toml index 6d0867f..35b93f8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,8 +2,11 @@ resolver = "2" members = [ "sequencer", - "app-core", - "canonical-app", + "sequencer-core", + "sdk/rust-client", + "examples/app-core", + "examples/canonical-app", + "benchmarks", ] default-members = ["sequencer"] diff --git a/README.md b/README.md index 0c34734..5b15766 100644 --- a/README.md +++ b/README.md @@ -16,13 +16,13 @@ Current focus is reliability of sequencing, persistence, and replay semantics. ## Core Design - **User ops** arrive through the API, are validated, executed, and persisted by the inclusion lane. -- **Direct inputs** are stored in SQLite (`direct_inputs`) and drained by the inclusion lane into frame boundaries (`frame_drains`). -- **Ordering** is deterministic and persisted. Replay/catch-up reads `ordered_sequenced_l2_txs`. -- **Batch fee** is fixed per batch (`batches.fee`): +- **Direct inputs** are stored in SQLite (`direct_inputs`) and sequenced in append-only replay order (`sequenced_l2_txs`). +- **Ordering** is deterministic and persisted. Replay/catch-up reads `sequenced_l2_txs` (joined with `user_ops` / `direct_inputs`). +- **Frame fee** is fixed per frame (`frames.fee`): - users sign `max_fee` - - inclusion validates `max_fee >= current_batch_fee` - - execution charges `current_batch_fee` (not signed max) - - next batch fee is sampled from `recommended_fees` when rotating to a new batch + - inclusion validates `max_fee >= current_frame_fee` + - execution charges `current_frame_fee` (not signed max) + - next frame fee is sampled from `recommended_fees` when rotating to a new frame ## Quick Start @@ -77,6 +77,8 @@ Notes: - `from_offset` is optional (defaults to `0`). - messages are JSON text frames. - binary fields are hex-encoded (`0x`-prefixed). +- handshake is rejected with `429` when `SEQ_WS_MAX_SUBSCRIBERS` is exceeded (default `64`). +- connections with `live_start_offset - from_offset > SEQ_WS_MAX_CATCHUP_EVENTS` are closed immediately (default `50000`). Message shapes: @@ -93,7 +95,6 @@ Success response: ```json { "ok": true, - "tx_hash": "0x...", "sender": "0x...", "nonce": 0 } @@ -106,7 +107,7 @@ Main environment variables: - `SEQ_HTTP_ADDR` - `SEQ_DB_PATH` - `SEQ_QUEUE_CAP` -- `SEQ_QUEUE_TIMEOUT_MS` +- `SEQ_OVERLOAD_MAX_INFLIGHT_SUBMISSIONS` - `SEQ_MAX_USER_OPS_PER_CHUNK` (`SEQ_MAX_BATCH` is legacy alias) - `SEQ_SAFE_DIRECT_BUFFER_CAPACITY` - `SEQ_MAX_BATCH_OPEN_MS` @@ -117,6 +118,8 @@ Main environment variables: - `SEQ_BROADCASTER_IDLE_POLL_INTERVAL_MS` - `SEQ_BROADCASTER_PAGE_SIZE` - `SEQ_BROADCASTER_SUBSCRIBER_BUFFER_CAPACITY` +- `SEQ_WS_MAX_SUBSCRIBERS` +- `SEQ_WS_MAX_CATCHUP_EVENTS` - `SEQ_MAX_BODY_BYTES` - `SEQ_SQLITE_SYNCHRONOUS` - `SEQ_DOMAIN_NAME` @@ -126,29 +129,26 @@ Main environment variables: ## Storage Model (high level) -- `batches`: batch metadata + committed batch fee +- `batches`: batch metadata - `frames`: frame boundaries within each batch +- `frames.fee`: committed fee for each frame - `user_ops`: included user operations - `direct_inputs`: direct-input payload stream -- `frame_drains`: per-frame `drain_n` -- `recommended_fees`: singleton mutable recommendation for next batch fee +- `sequenced_l2_txs`: append-only ordered replay rows (`UserOp` xor `DirectInput`) +- `recommended_fees`: singleton mutable recommendation for next frame fee -Views: - -- `ordered_sequenced_l2_txs`: canonical ordered replay stream (`UserOp | DirectInput`) -- `frame_drain_ranges`, `batch_user_op_counts`, `frame_user_op_counts` +No SQL views are required in the current prototype schema. ## Project Layout - `sequencer/src/main.rs`: bootstrap, env config, HTTP server + lane lifecycle - `sequencer/src/api/`: HTTP API and error mapping - `sequencer/src/inclusion_lane/`: hot-path inclusion loop, chunk/frame/batch rotation, catch-up -- `sequencer/src/l2_tx_broadcaster.rs`: centralized ordered-L2Tx poller + subscriber fanout +- `sequencer/src/l2_tx_broadcaster/`: centralized ordered-L2Tx poller + subscriber fanout - `sequencer/src/storage/`: schema, migrations, SQLite persistence and replay reads -- `app-core/src/application/`: app execution/validation interfaces + wallet prototype -- `app-core/src/user_op.rs`: signed user-op and EIP-712 payload model -- `app-core/src/l2_tx.rs`: replay/fanout transaction domain types -- `canonical-app/src/main.rs`: placeholder canonical runtime entrypoint +- `sequencer-core/src/`: shared sequencer domain types and interfaces (`Application`, `SignedUserOp`, `SequencedL2Tx`, broadcaster message types) +- `examples/app-core/src/`: wallet prototype implementing the shared `Application` trait +- `examples/canonical-app/src/main.rs`: placeholder canonical runtime entrypoint ## Prototype Limits diff --git a/TODO.md b/TODO.md index cb67121..85c45dd 100644 --- a/TODO.md +++ b/TODO.md @@ -24,7 +24,7 @@ Build a robust sequencer prototype for a future DeFi stack, with deterministic o ### 2) Canonical App / Scheduler -- Implement scheduler behavior in `canonical-app` using shared `app-core`. +- Implement scheduler behavior in `examples/canonical-app` using shared `sequencer-core` + `examples/app-core`. - Ensure deterministic ordering model compatible with persisted sequencer order. - Canonical app is the state-transition artifact used by verification flow (Cartesi Machine / RISC-V path), not by sequencer runtime itself. - Add focused tests for queue/drain/backstop behavior and ordering invariants. @@ -39,16 +39,26 @@ Build a robust sequencer prototype for a future DeFi stack, with deterministic o - Measure idle and under-load behavior. - Include network-aware runs (client/server on different hosts) like network latency. - Note: end-to-end depends on `L2Tx` broadcaster being available. +- Possible optimization idea (later): adaptive chunk sizing in inclusion lane based on queue pressure and latency budget. --- ## Post-MVP (Nice to Have / Dogfooding Artifacts) - `sdk/ts-client/`: TypeScript client library for browser/server JS callers. -- `tools/cli/`: Rust CLI for manual tx submission and debugging flows. +- `sdk/cli/`: Rust CLI for manual tx submission and debugging flows. - `examples/web-demo/`: browser demo app consuming `sdk/ts-client`. Notes: - These are intentionally outside MVP scope. - Still valuable for dogfooding and contributor onboarding. + + + + +Dev endpoint for direct inputs? + +Endpoints for domain, nonce, fee?? + +Implement health check?? Ready check?? diff --git a/benchmarks/.gitignore b/benchmarks/.gitignore index d6b7ef3..2f7896d 100644 --- a/benchmarks/.gitignore +++ b/benchmarks/.gitignore @@ -1,2 +1 @@ -* -!.gitignore +target/ diff --git a/benchmarks/BENCHMARK_SPEC.md b/benchmarks/BENCHMARK_SPEC.md new file mode 100644 index 0000000..a3b15eb --- /dev/null +++ b/benchmarks/BENCHMARK_SPEC.md @@ -0,0 +1,142 @@ +# Benchmark Spec + +This document defines benchmark intent, measurement definitions, methodology, target scenario, target goals, and reporting requirements for the sequencer. + +## 1. Purpose + +The external requirement is abstract: `latency <= 500ms`. +This spec makes that requirement measurable and repeatable. + +## 2. Measurement Catalog + +### 2.1 Latency Metrics + +1. `ack_latency_ms`: elapsed time from client send of `POST /tx` to successful HTTP ack (`200`). +2. `soft_confirm_latency_ms`: elapsed time from client send of `POST /tx` to receipt of matching `user_op` event on `GET /ws/subscribe`. + +### 2.2 Throughput and Volume Metrics + +1. `accepted_tps`: accepted tx count divided by total run duration. +2. `accepted_count`: number of accepted requests. +3. `rejected_count`: number of rejected requests. +4. `rejection_rate`: `rejected_count / (accepted_count + rejected_count) * 100`. + +### 2.3 Capacity and Overload Metrics + +1. `max_sustainable_tps_at_0_rejections`: highest accepted TPS observed while `rejection_rate == 0%`. +2. `tps_at_first_non_200`: throughput point where first non-`200` response appears. +3. `tps_at_first_429`: throughput point where first `429 OVERLOADED` response appears. + +### 2.4 Memory Metrics + +1. `rss_start_mb`: sequencer process RSS at run start. +2. `rss_peak_mb`: peak sequencer process RSS during run. +3. `rss_end_mb`: sequencer process RSS at run end. +4. `rss_growth_mb`: `rss_end_mb - rss_start_mb`. +5. `rss_growth_per_1k_accepted_tx_mb`: RSS growth normalized by accepted tx volume. + +## 3. Measurement Method + +### 3.1 Percentiles and Sample Size + +1. Required latency percentiles: p50, p95, p99, p99.9. +2. Target percentile is `p99` for both latency goals. +3. `p99.9` is diagnostic in this phase. +4. Runs are valid for target evaluation only when `accepted_count >= 5,000`. +5. `p99.9` is considered reliable when `accepted_count >= 10,000`; otherwise it must be marked diagnostic-low-confidence. + +### 3.2 Request Outcome Classification + +1. `accepted`: HTTP status `200` from `POST /tx`. +2. `rejected`: non-`200`, timeout, or network/client failure. + +### 3.3 Workload Validity Rules + +1. Target evaluation must use valid-only benchmark traffic (no intentionally invalid transactions). +2. Non-`200` responses must be broken down by status code so overload (`429`) is distinguishable from invalid-input failures (`400`/`422`). +3. If no non-`200` appears, `tps_at_first_non_200` and `tps_at_first_429` must be reported as `not reached`. + +### 3.4 Memory Collection Rules + +1. Memory metrics must be collected for the sequencer process. +2. Reports must include collection method/tool (`ps`, `/proc`, container stats, etc.). +3. Reports must include sampling interval (recommended `250ms` to `1000ms`). +4. Memory has no hard pass/fail threshold in this phase; it is profiling-critical. + +### 3.5 Sanity Assertions (Correctness, Non-Metric) + +1. For end-to-end runs, every HTTP-accepted request must have a matching WS `user_op` within the configured wait timeout. +2. Any sanity assertion failure indicates a correctness bug and invalidates the benchmark run. + +## 4. Benchmark Scenarios + +### 4.1 Profiling Scenarios + +1. Same-host baseline (`no injected latency`) for low-noise branch comparisons. +2. Canonical network-aware scenario. +3. Throughput/concurrency sweeps to locate knee and overload behavior. + +### 4.2 Canonical Network-Aware Scenario (Target Evaluation) + +1. The injected profile must be applied to both HTTP `POST /tx` and WS `GET /ws/subscribe` paths. +2. Base delay: `+50ms one-way` in each direction (`~100ms RTT`). +3. Jitter: `+/-10ms` around base delay. +4. Packet loss: `0%`. +5. No artificial bandwidth cap. +6. Packet reordering, duplication, and corruption injection disabled. +7. Reports must include the network shaping tool and exact shaping config. + +## 5. Targets and Verdicts + +### 5.1 Target Goals + +1. `ack_latency_ms p99 <= 500ms` +2. `soft_confirm_latency_ms p99 <= 1000ms` + +### 5.2 Target Evaluation Conditions + +The latency goals above are evaluated under these conditions: + +1. `rejection_rate == 0%` (equivalently `rejected_count == 0`). +2. Canonical network-aware scenario is satisfied. +3. Run is valid per sample-size policy. + +### 5.3 Separate Verdicts + +1. `ACK_TARGET` is `PASS` when all hold: + - target evaluation conditions are satisfied + - `ack_latency_ms p99 <= 500ms` +2. `SOFT_CONFIRM_TARGET` is `PASS` when all hold: + - target evaluation conditions are satisfied + - `soft_confirm_latency_ms p99 <= 1000ms` + - sanity assertions are satisfied + +## 6. Required Report Content + +Each benchmark report must include: + +1. Latency percentiles for measured metrics. +2. Accepted/rejected counts and rejection rate. +3. Accepted TPS and total run duration. +4. Rejection reason breakdown (when available). +5. Network profile and shaping method/config. +6. Memory metrics and memory sampling method/interval. +7. Full run configuration (command, concurrency/arrival settings, timeout settings). +8. Target verdict lines: `ACK_TARGET` and `SOFT_CONFIRM_TARGET` (when target evaluation is performed). +9. Sanity assertion status and failure summary (if any). + +## 7. Mapping to Current Harnesses + +1. `ack_latency`: primary path for `ack_latency_ms`. +2. `e2e_latency`: primary path for `soft_confirm_latency_ms`. +3. `sweep`: capacity exploration and overload discovery. +4. `unit_hot_path`: micro profiling. + +## 8. Current Gaps + +1. Add first-class support for canonical injected-latency runs. +2. Add explicit `p99.9` export in sweep CSV/JSON outputs. +3. Add standard `ACK_TARGET` and `SOFT_CONFIRM_TARGET` verdict lines in benchmark outputs. +4. Add rejection-reason breakdown consistently across relevant reports. +5. Ensure reports always include network shaping tool and exact shaping config. +6. Add first-class memory collection/reporting in benchmark tooling. diff --git a/benchmarks/Cargo.toml b/benchmarks/Cargo.toml new file mode 100644 index 0000000..970c94a --- /dev/null +++ b/benchmarks/Cargo.toml @@ -0,0 +1,25 @@ +[package] +name = "benchmarks" +version.workspace = true +edition.workspace = true +license.workspace = true +description.workspace = true +homepage.workspace = true +repository.workspace = true +readme.workspace = true +authors.workspace = true + +[dependencies] +sequencer-rust-client = { path = "../sdk/rust-client" } +sequencer-core = { path = "../sequencer-core" } +alloy-primitives = { version = "1.4.1", features = ["k256"] } +alloy-sol-types = "1.4.1" +futures-util = "0.3" +k256 = "0.13.4" +clap = { version = "4", features = ["derive"] } +serde = { version = "1", features = ["derive"] } +serde_json = "1" +ssz = { package = "ethereum_ssz", version = "0.10" } +tokio = { version = "1.35", features = ["macros", "rt-multi-thread", "time", "net", "io-util", "process", "sync"] } +tokio-tungstenite = "0.28" +tempfile = "3" diff --git a/benchmarks/README.md b/benchmarks/README.md new file mode 100644 index 0000000..9b2c3d1 --- /dev/null +++ b/benchmarks/README.md @@ -0,0 +1,65 @@ +# Benchmarks + +This crate contains executable benchmark harnesses for the sequencer API. + +Benchmark goals, UX-facing metrics, and initial SLO targets are defined in +[`BENCHMARK_SPEC.md`](./BENCHMARK_SPEC.md). + +## Commands + +From repository root: + +```bash +just --justfile benchmarks/justfile bench-unit +just --justfile benchmarks/justfile bench-ack +just --justfile benchmarks/justfile bench-e2e +just --justfile benchmarks/justfile bench-hammer +just --justfile benchmarks/justfile bench-sweep +just --justfile benchmarks/justfile bench-compare-latest +just --justfile benchmarks/justfile all +just --justfile benchmarks/justfile all-and-compare +``` + +Or from inside `benchmarks/`: + +```bash +just bench-unit +just bench-ack +just bench-e2e +just bench-hammer +just bench-sweep +just bench-compare-latest +just all +just all-and-compare +``` + +Or directly with `cargo`: + +```bash +cargo run -p benchmarks --bin unit_hot_path -- --count 10000 --max-fee 0 +cargo run -p benchmarks --bin ack_latency -- --http-url http://127.0.0.1:3000 --count 200 --max-fee 0 --concurrency 1 +cargo run -p benchmarks --bin e2e_latency -- --http-url http://127.0.0.1:3000 --count 100 --max-fee 0 --from-offset 0 --concurrency 1 +cargo run -p benchmarks --bin sweep --release -- --mode e2e --count 1000 --url http://127.0.0.1:3000 --max-fee 0 --from-offset 0 --concurrency-list "1 2 4 8 16 32 64 96 128" +cargo run -p benchmarks --bin compare_latest --release -- --results-dir benchmarks/results --kind all +``` + +## Notes + +- `unit_hot_path`: measures local signing + request-encoding costs (no network). +- `ack_latency`: measures `POST /tx` acknowledgment latency for accepted txs. +- `e2e_latency`: measures submit-to-broadcast latency (`POST /tx` to `GET /ws/subscribe` message) for accepted txs. +- `bench-hammer`: high-concurrency e2e run that hammers the sequencer and verifies each accepted tx is observed on WS. +- `bench-sweep`: runs a concurrency sweep (default `1..128`, `count=1000`) and emits a CSV plus an estimated knee. +- `bench-compare-latest`: compares the latest two `ack`, `e2e`, and `sweep` artifacts and prints deltas. +- For newly generated benchmark accounts, included transactions usually require sequencer frame fee `0`. +- Networked benches fail by default if any tx is rejected. Pass `--allow-rejections` to inspect mixed traffic. +- `e2e_latency` drains existing WS backlog before timing to reduce stale-history noise. +- Sweep CSV columns: `concurrency,completed_per_s,p95_ms,rejected`. +- `bench-sweep mode=e2e` carries `from_offset` forward across rounds to avoid re-reading old WS history. +- If sweep hits `Too many open files`, increase shell limit (`ulimit -n 4096`) or lower `conc_list`. +- Self-contained variants automatically spawn a sequencer and persist logs/results. +- For non-self-contained networked benches, run a sequencer instance beforehand, for example: + +```bash +just run +``` diff --git a/benchmarks/justfile b/benchmarks/justfile new file mode 100644 index 0000000..65251c5 --- /dev/null +++ b/benchmarks/justfile @@ -0,0 +1,55 @@ +set shell := ["bash", "-euo", "pipefail", "-c"] +set working-directory := ".." + +default: + @just --justfile benchmarks/justfile --list + +bench-unit count="10000" max_fee="0": + cargo run -p benchmarks --bin unit_hot_path --release -- --count {{count}} --max-fee {{max_fee}} + +bench-ack count="200" url="http://127.0.0.1:3000" max_fee="0" concurrency="1" extra="": + cargo run -p benchmarks --bin ack_latency --release -- --endpoint {{url}} --count {{count}} --max-fee {{max_fee}} --concurrency {{concurrency}} {{extra}} + +bench-ack-self count="200" max_fee="0" concurrency="1" extra="": + cargo build -p sequencer --release + cargo run -p benchmarks --bin ack_latency --release -- --self-contained --count {{count}} --max-fee {{max_fee}} --concurrency {{concurrency}} {{extra}} + +bench-e2e count="100" url="http://127.0.0.1:3000" max_fee="0" from_offset="0" concurrency="1" extra="": + cargo run -p benchmarks --bin e2e_latency --release -- --endpoint {{url}} --count {{count}} --max-fee {{max_fee}} --from-offset {{from_offset}} --concurrency {{concurrency}} {{extra}} + +bench-e2e-self count="100" max_fee="0" from_offset="0" concurrency="1" extra="": + cargo build -p sequencer --release + cargo run -p benchmarks --bin e2e_latency --release -- --self-contained --count {{count}} --max-fee {{max_fee}} --from-offset {{from_offset}} --concurrency {{concurrency}} {{extra}} + +bench-hammer count="20000" url="http://127.0.0.1:3000" max_fee="0" concurrency="10" from_offset="0" workload="funded-transfer" extra="": + cargo run -p benchmarks --bin e2e_latency --release -- --endpoint {{url}} --count {{count}} --max-fee {{max_fee}} --from-offset {{from_offset}} --concurrency {{concurrency}} --request-timeout-ms 10000 --max-ws-wait-ms 20000 --workload {{workload}} {{extra}} + +bench-hammer-self count="20000" max_fee="0" concurrency="10" from_offset="0" workload="funded-transfer" out="benchmarks/results/hammer-self-latest.json" extra="": + cargo build -p sequencer --release + cargo run -p benchmarks --bin e2e_latency --release -- --self-contained --count {{count}} --max-fee {{max_fee}} --from-offset {{from_offset}} --concurrency {{concurrency}} --request-timeout-ms 10000 --max-ws-wait-ms 20000 --workload {{workload}} --json-out {{out}} {{extra}} + +bench-sweep mode="e2e" count="1000" url="http://127.0.0.1:3000" max_fee="0" from_offset="0" conc_list="1 2 4 8 16 32 64 96 128" extra="": + cargo run -p benchmarks --bin sweep --release -- --mode {{mode}} --count {{count}} --endpoint {{url}} --max-fee {{max_fee}} --from-offset {{from_offset}} --concurrency-list "{{conc_list}}" {{extra}} + +bench-sweep-self mode="e2e" count="1000" max_fee="0" from_offset="0" conc_list="1 2 4 8 16 32 64 96 128" extra="": + cargo build -p sequencer --release + cargo run -p benchmarks --bin sweep --release -- --self-contained --mode {{mode}} --count {{count}} --max-fee {{max_fee}} --from-offset {{from_offset}} --concurrency-list "{{conc_list}}" {{extra}} + +bench-soak-low-lat-self count="20000" max_fee="0" out="benchmarks/results/soak-low-lat-self.json" extra="": + just --justfile benchmarks/justfile bench-hammer-self {{count}} {{max_fee}} 10 0 funded-transfer {{out}} '{{extra}}' + +bench-soak-high-throughput-self count="20000" max_fee="0" out="benchmarks/results/soak-high-throughput-self.json" extra="": + cargo build -p sequencer --release + cargo run -p benchmarks --bin e2e_latency --release -- --self-contained --count {{count}} --max-fee {{max_fee}} --from-offset 0 --concurrency 96 --request-timeout-ms 10000 --max-ws-wait-ms 20000 --workload synthetic --json-out {{out}} {{extra}} + +bench-capacity-sweep-self count="1000" max_fee="0" from_offset="0" conc_range="32:512:32" out="benchmarks/results/capacity-sweep-self.json" extra="": + cargo build -p sequencer --release + cargo run -p benchmarks --bin sweep --release -- --self-contained --mode e2e --count {{count}} --max-fee {{max_fee}} --from-offset {{from_offset}} --workload synthetic --concurrency-range "{{conc_range}}" --stop-on-first-non-200 --json-out {{out}} {{extra}} + +bench-compare-latest kind="all" results_dir="benchmarks/results": + cargo run -p benchmarks --bin compare_latest --release -- --results-dir {{results_dir}} --kind {{kind}} + +all: bench-unit bench-ack-self bench-e2e-self bench-soak-low-lat-self bench-soak-high-throughput-self bench-capacity-sweep-self + +all-and-compare: all + just --justfile benchmarks/justfile bench-compare-latest all diff --git a/benchmarks/src/bin/ack_latency.rs b/benchmarks/src/bin/ack_latency.rs new file mode 100644 index 0000000..6de274a --- /dev/null +++ b/benchmarks/src/bin/ack_latency.rs @@ -0,0 +1,263 @@ +// (c) Cartesi and individual authors (see AUTHORS) +// SPDX-License-Identifier: Apache-2.0 (see LICENSE) + +use benchmarks::{ + AckRunConfig, BenchResult, DEFAULT_ENDPOINT, DEFAULT_WORKLOAD_INITIAL_BALANCE, + DEFAULT_WORKLOAD_TRANSFER_AMOUNT, WorkloadConfig, WorkloadKind, default_seed_offset, + print_ack_report, run_ack_benchmark, + runtime::{ + DEFAULT_MEMORY_SAMPLE_INTERVAL_MS, DEFAULT_RUNTIME_METRICS_LOG_INTERVAL_MS, + DEFAULT_SEQUENCER_BIN, DEFAULT_SEQUENCER_SHUTDOWN_TIMEOUT_MS, + DEFAULT_SEQUENCER_START_TIMEOUT_MS, ManagedSequencer, ManagedSequencerConfig, + MemorySampler, default_sequencer_log_path, parse_inclusion_lane_profile_from_log, + }, +}; +use clap::{Parser, ValueEnum}; +use serde::Serialize; +use std::fs; +use std::path::Path; +use std::path::PathBuf; +use std::time::Duration; + +#[derive(Debug, Parser)] +#[command( + name = "ack_latency", + about = "ack latency benchmark", + version, + after_help = "Examples:\n cargo run -p benchmarks --bin ack_latency -- --endpoint http://127.0.0.1:3000 --count 1000 --concurrency 32 --max-fee 0\n cargo run -p benchmarks --bin ack_latency --release -- --count 5000 --allow-rejections" +)] +struct Args { + #[arg(long, visible_alias = "http-url", default_value = DEFAULT_ENDPOINT)] + endpoint: String, + #[arg(long, default_value_t = false)] + self_contained: bool, + #[arg(long, default_value = DEFAULT_SEQUENCER_BIN)] + sequencer_bin: String, + #[arg(long, default_value_t = DEFAULT_SEQUENCER_START_TIMEOUT_MS)] + sequencer_start_timeout_ms: u64, + #[arg(long, default_value_t = DEFAULT_SEQUENCER_SHUTDOWN_TIMEOUT_MS)] + sequencer_shutdown_timeout_ms: u64, + #[arg(long, default_value_t = true)] + temp_db: bool, + #[arg(long)] + sequencer_log_path: Option, + #[arg(long, default_value_t = true)] + sequencer_runtime_metrics_enabled: bool, + #[arg(long, default_value_t = DEFAULT_RUNTIME_METRICS_LOG_INTERVAL_MS)] + sequencer_runtime_metrics_log_interval_ms: u64, + #[arg(long, default_value = "info")] + sequencer_rust_log: String, + #[arg(long, default_value_t = DEFAULT_MEMORY_SAMPLE_INTERVAL_MS)] + memory_sample_interval_ms: u64, + #[arg(long, value_enum, default_value_t = CliWorkload::Synthetic)] + workload: CliWorkload, + #[arg(long)] + accounts_file: Option, + #[arg(long, default_value_t = DEFAULT_WORKLOAD_INITIAL_BALANCE)] + initial_balance: u64, + #[arg(long, default_value_t = DEFAULT_WORKLOAD_TRANSFER_AMOUNT)] + transfer_amount: u64, + #[arg(long, default_value_t = 200_u64)] + count: u64, + #[arg(long, default_value_t = 1_usize)] + concurrency: usize, + #[arg(long)] + seed_offset: Option, + #[arg(long, default_value_t = 0_u32)] + max_fee: u32, + #[arg(long, default_value_t = 3_000_u64)] + request_timeout_ms: u64, + #[arg(long, default_value_t = 0_u64)] + progress_every: u64, + #[arg(long, default_value_t = false)] + allow_rejections: bool, + #[arg(long)] + json_out: Option, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, ValueEnum)] +enum CliWorkload { + #[value(name = "synthetic")] + Synthetic, + #[value(name = "funded-transfer")] + FundedTransfer, +} + +impl From for WorkloadKind { + fn from(value: CliWorkload) -> Self { + match value { + CliWorkload::Synthetic => Self::Synthetic, + CliWorkload::FundedTransfer => Self::FundedTransfer, + } + } +} + +impl CliWorkload { + fn as_str(self) -> &'static str { + match self { + Self::Synthetic => "synthetic", + Self::FundedTransfer => "funded-transfer", + } + } +} + +#[derive(Debug, Serialize)] +struct AckJsonConfig { + endpoint: String, + self_contained: bool, + count: u64, + concurrency: usize, + max_fee: u32, + request_timeout_ms: u64, + allow_rejections: bool, + workload: String, + accounts_file: Option, + initial_balance: u64, + transfer_amount: u64, +} + +#[derive(Debug, Serialize)] +struct AckJsonOutput { + benchmark: &'static str, + config: AckJsonConfig, + report: benchmarks::AckRunReport, +} + +#[tokio::main] +async fn main() -> BenchResult<()> { + let args = Args::parse(); + let json_out = args.json_out.clone().or_else(|| { + args.self_contained + .then(|| default_json_output_path("ack-latency")) + }); + let mut managed = if args.self_contained { + Some( + ManagedSequencer::spawn(ManagedSequencerConfig { + sequencer_bin: args.sequencer_bin.clone(), + start_timeout: Duration::from_millis(args.sequencer_start_timeout_ms), + shutdown_timeout: Duration::from_millis(args.sequencer_shutdown_timeout_ms), + temp_db: args.temp_db, + log_path: args + .sequencer_log_path + .clone() + .or_else(|| Some(default_sequencer_log_path("ack-latency-self-contained"))), + runtime_metrics_enabled: args.sequencer_runtime_metrics_enabled, + runtime_metrics_log_interval: Duration::from_millis( + args.sequencer_runtime_metrics_log_interval_ms, + ), + rust_log: args.sequencer_rust_log.clone(), + }) + .await?, + ) + } else { + None + }; + let endpoint = managed + .as_ref() + .map(|value| value.endpoint.clone()) + .unwrap_or_else(|| args.endpoint.clone()); + + println!( + "ack config: endpoint={}, self_contained={}, count={}, concurrency={}, max_fee={}, request_timeout_ms={}, allow_rejections={}, workload={:?}", + endpoint, + args.self_contained, + args.count, + args.concurrency.max(1), + args.max_fee, + args.request_timeout_ms, + args.allow_rejections, + args.workload + ); + + let memory_sampler = managed.as_ref().and_then(|value| value.pid()).map(|pid| { + MemorySampler::start(pid, Duration::from_millis(args.memory_sample_interval_ms)) + }); + + let config = AckRunConfig { + endpoint, + count: args.count, + concurrency: args.concurrency.max(1), + seed_offset: args.seed_offset.unwrap_or_else(default_seed_offset), + max_fee: args.max_fee, + request_timeout_ms: args.request_timeout_ms, + progress_every: args.progress_every, + fail_on_rejection: !args.allow_rejections, + workload: WorkloadConfig { + kind: args.workload.into(), + accounts_file: args.accounts_file.clone(), + initial_balance: args.initial_balance, + transfer_amount: args.transfer_amount, + }, + }; + + let mut report_result = run_ack_benchmark(config).await; + if let Some(path) = managed + .as_ref() + .map(|value| value.log_path().to_string_lossy().to_string()) + && let Ok(report) = report_result.as_mut() + { + report.sequencer_log_path = Some(path); + } + if let Some(sampler) = memory_sampler { + match report_result.as_mut() { + Ok(report) => report.memory = Some(sampler.stop(report.accepted).await?), + Err(_) => { + let _ = sampler.stop(0).await; + } + } + } + if let Some(value) = managed.take() { + let shutdown_result = value.shutdown().await; + if let Err(err) = shutdown_result + && report_result.is_ok() + { + return Err(err); + } + } + + if let Some(path) = report_result + .as_ref() + .ok() + .and_then(|report| report.sequencer_log_path.clone()) + && let Some(profile) = parse_inclusion_lane_profile_from_log(PathBuf::from(path).as_path())? + && let Ok(report) = report_result.as_mut() + { + report.inclusion_lane_profile = Some(profile); + } + + let report = report_result?; + print_ack_report(&report); + if let Some(path) = json_out.as_ref() { + if let Some(parent) = Path::new(path).parent() { + fs::create_dir_all(parent)?; + } + let payload = AckJsonOutput { + benchmark: "ack_latency", + config: AckJsonConfig { + endpoint: report.endpoint.clone(), + self_contained: args.self_contained, + count: args.count, + concurrency: args.concurrency.max(1), + max_fee: args.max_fee, + request_timeout_ms: args.request_timeout_ms, + allow_rejections: args.allow_rejections, + workload: args.workload.as_str().to_string(), + accounts_file: args.accounts_file.clone(), + initial_balance: args.initial_balance, + transfer_amount: args.transfer_amount, + }, + report, + }; + fs::write(path, serde_json::to_vec_pretty(&payload)?)?; + println!("ack json: {path}"); + } + Ok(()) +} + +fn default_json_output_path(prefix: &str) -> String { + let ts = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|value| value.as_secs()) + .unwrap_or(0); + format!("benchmarks/results/{prefix}-{ts}.json") +} diff --git a/benchmarks/src/bin/compare_latest.rs b/benchmarks/src/bin/compare_latest.rs new file mode 100644 index 0000000..a58b53e --- /dev/null +++ b/benchmarks/src/bin/compare_latest.rs @@ -0,0 +1,474 @@ +// (c) Cartesi and individual authors (see AUTHORS) +// SPDX-License-Identifier: Apache-2.0 (see LICENSE) + +use benchmarks::BenchResult; +use clap::{Parser, ValueEnum}; +use serde::Deserialize; +use std::collections::BTreeMap; +use std::fs; +use std::path::{Path, PathBuf}; + +#[derive(Debug, Clone, Copy, ValueEnum)] +enum CompareKind { + Ack, + E2e, + Sweep, + All, +} + +#[derive(Debug, Parser)] +#[command(name = "compare_latest")] +#[command(about = "Compare the latest two benchmark result files")] +struct Cli { + #[arg(long, default_value = "benchmarks/results")] + results_dir: PathBuf, + #[arg(long, value_enum, default_value_t = CompareKind::All)] + kind: CompareKind, +} + +#[derive(Debug, Deserialize)] +struct DurationJson { + secs: u64, + nanos: u32, +} + +impl DurationJson { + fn as_secs_f64(&self) -> f64 { + self.secs as f64 + f64::from(self.nanos) / 1_000_000_000.0 + } + + fn as_ms_f64(&self) -> f64 { + self.as_secs_f64() * 1_000.0 + } +} + +#[derive(Debug, Deserialize)] +struct StatsJson { + p50: DurationJson, + p95: DurationJson, + p99: DurationJson, + p999: DurationJson, + max: DurationJson, +} + +#[derive(Debug, Deserialize)] +struct MemoryJson { + rss_start_mb: f64, + rss_peak_mb: f64, + rss_growth_mb: f64, +} + +#[derive(Debug, Deserialize)] +struct AckFileJson { + report: AckReportJson, +} + +#[derive(Debug, Deserialize)] +struct AckReportJson { + accepted: u64, + rejected: u64, + total_wall: DurationJson, + ack_latency_accepted: StatsJson, + memory: Option, +} + +#[derive(Debug, Deserialize)] +struct E2eFileJson { + report: E2eReportJson, +} + +#[derive(Debug, Deserialize)] +struct E2eReportJson { + accepted: u64, + rejected: u64, + total_wall: DurationJson, + ack_latency_accepted: StatsJson, + e2e_latency_accepted: StatsJson, + memory: Option, +} + +#[derive(Debug, Clone)] +struct SweepCsvRow { + concurrency: u64, + accepted_tps: f64, + rejected_count: u64, + p95_ms: f64, + p99_ms: f64, + p999_ms: f64, +} + +fn main() -> BenchResult<()> { + let cli = Cli::parse(); + match cli.kind { + CompareKind::Ack => compare_ack(&cli.results_dir)?, + CompareKind::E2e => compare_e2e(&cli.results_dir)?, + CompareKind::Sweep => compare_sweep(&cli.results_dir)?, + CompareKind::All => { + compare_ack(&cli.results_dir)?; + println!(); + compare_e2e(&cli.results_dir)?; + println!(); + compare_sweep(&cli.results_dir)?; + } + } + Ok(()) +} + +fn compare_ack(results_dir: &Path) -> BenchResult<()> { + let (old_path, new_path) = latest_two_files(results_dir, "ack-latency-", ".json")?; + let old = read_json::(&old_path)?; + let new = read_json::(&new_path)?; + + println!( + "ACK latest two:\n old: {}\n new: {}", + old_path.display(), + new_path.display() + ); + print_common( + old.report.accepted, + old.report.rejected, + &old.report.total_wall, + ); + print_common_delta( + old.report.accepted, + old.report.rejected, + &old.report.total_wall, + new.report.accepted, + new.report.rejected, + &new.report.total_wall, + ); + print_stats_delta( + "ack latency", + &old.report.ack_latency_accepted, + &new.report.ack_latency_accepted, + ); + print_memory_delta(old.report.memory.as_ref(), new.report.memory.as_ref()); + Ok(()) +} + +fn compare_e2e(results_dir: &Path) -> BenchResult<()> { + let (old_path, new_path) = latest_two_files(results_dir, "e2e-latency-", ".json")?; + let old = read_json::(&old_path)?; + let new = read_json::(&new_path)?; + + println!( + "E2E latest two:\n old: {}\n new: {}", + old_path.display(), + new_path.display() + ); + print_common( + old.report.accepted, + old.report.rejected, + &old.report.total_wall, + ); + print_common_delta( + old.report.accepted, + old.report.rejected, + &old.report.total_wall, + new.report.accepted, + new.report.rejected, + &new.report.total_wall, + ); + print_stats_delta( + "ack latency", + &old.report.ack_latency_accepted, + &new.report.ack_latency_accepted, + ); + print_stats_delta( + "e2e latency", + &old.report.e2e_latency_accepted, + &new.report.e2e_latency_accepted, + ); + print_memory_delta(old.report.memory.as_ref(), new.report.memory.as_ref()); + Ok(()) +} + +fn compare_sweep(results_dir: &Path) -> BenchResult<()> { + let (old_path, new_path) = latest_two_files(results_dir, "e2e-sweep-", ".csv")?; + let old_rows = read_sweep_rows(&old_path)?; + let new_rows = read_sweep_rows(&new_path)?; + + println!( + "SWEEP latest two:\n old: {}\n new: {}", + old_path.display(), + new_path.display() + ); + + let old_by_concurrency = map_sweep_rows(&old_rows); + let new_by_concurrency = map_sweep_rows(&new_rows); + + println!(" deltas by concurrency:"); + println!(" c,accepted_tps_delta,p95_ms_delta,p99_ms_delta,p999_ms_delta,rejected_delta"); + + let mut all_concurrency: Vec = old_by_concurrency + .keys() + .chain(new_by_concurrency.keys()) + .copied() + .collect(); + all_concurrency.sort_unstable(); + all_concurrency.dedup(); + + for concurrency in all_concurrency { + let old = old_by_concurrency.get(&concurrency); + let new = new_by_concurrency.get(&concurrency); + if let (Some(old), Some(new)) = (old, new) { + println!( + " {},{:+.3},{:+.3},{:+.3},{:+.3},{:+}", + concurrency, + new.accepted_tps - old.accepted_tps, + new.p95_ms - old.p95_ms, + new.p99_ms - old.p99_ms, + new.p999_ms - old.p999_ms, + i128::from(new.rejected_count) - i128::from(old.rejected_count), + ); + } else { + println!(" {concurrency},n/a,n/a,n/a,n/a,n/a"); + } + } + + let old_max_zero_rejection_tps = max_zero_rejection_tps(&old_rows); + let new_max_zero_rejection_tps = max_zero_rejection_tps(&new_rows); + let old_first_rejection_tps = first_rejection_tps(&old_rows); + let new_first_rejection_tps = first_rejection_tps(&new_rows); + + println!(" summary:"); + println!( + " max_sustainable_tps_at_0_rejections: old={:.3}, new={:.3}, delta={:+.3}", + old_max_zero_rejection_tps, + new_max_zero_rejection_tps, + new_max_zero_rejection_tps - old_max_zero_rejection_tps + ); + println!( + " tps_at_first_non_200: old={}, new={}", + fmt_opt(old_first_rejection_tps), + fmt_opt(new_first_rejection_tps), + ); + Ok(()) +} + +fn latest_two_files( + results_dir: &Path, + prefix: &str, + suffix: &str, +) -> BenchResult<(PathBuf, PathBuf)> { + let mut candidates = Vec::new(); + for entry in fs::read_dir(results_dir)? { + let entry = entry?; + let path = entry.path(); + if !path.is_file() { + continue; + } + let Some(file_name) = path.file_name().and_then(|s| s.to_str()) else { + continue; + }; + if file_name.starts_with(prefix) && file_name.ends_with(suffix) { + let timestamp = trailing_number(file_name).unwrap_or(0); + candidates.push((timestamp, file_name.to_string(), path)); + } + } + + candidates.sort_by(|a, b| a.cmp(b)); + if candidates.len() < 2 { + return Err(std::io::Error::other(format!( + "need at least 2 files for pattern {prefix}*{suffix} in {}", + results_dir.display() + )) + .into()); + } + let old = candidates[candidates.len() - 2].2.clone(); + let new = candidates[candidates.len() - 1].2.clone(); + Ok((old, new)) +} + +fn trailing_number(file_name: &str) -> Option { + let stem = file_name + .rsplit_once('.') + .map(|(left, _)| left) + .unwrap_or(file_name); + let reversed_digits: String = stem + .chars() + .rev() + .take_while(|c| c.is_ascii_digit()) + .collect(); + if reversed_digits.is_empty() { + return None; + } + let digits: String = reversed_digits.chars().rev().collect(); + digits.parse::().ok() +} + +fn read_json Deserialize<'de>>(path: &Path) -> BenchResult { + let raw = fs::read_to_string(path)?; + let parsed = serde_json::from_str::(&raw)?; + Ok(parsed) +} + +fn read_sweep_rows(path: &Path) -> BenchResult> { + let content = fs::read_to_string(path)?; + let mut lines = content.lines(); + let Some(header_line) = lines.next() else { + return Err(std::io::Error::other(format!("empty CSV file: {}", path.display())).into()); + }; + let headers: Vec<&str> = header_line.split(',').collect(); + let mut header_idx = BTreeMap::new(); + for (idx, header) in headers.iter().enumerate() { + header_idx.insert(*header, idx); + } + + let mut rows = Vec::new(); + for line in lines { + if line.trim().is_empty() { + continue; + } + let cols: Vec<&str> = line.split(',').collect(); + let concurrency = parse_csv_u64(&cols, &header_idx, "concurrency")?; + let accepted_tps = parse_csv_f64(&cols, &header_idx, "accepted_tps")?; + let rejected_count = parse_csv_u64(&cols, &header_idx, "rejected_count")?; + let p95_ms = parse_csv_f64(&cols, &header_idx, "p95_ms")?; + let p99_ms = parse_csv_f64(&cols, &header_idx, "p99_ms")?; + let p999_ms = parse_csv_f64(&cols, &header_idx, "p999_ms")?; + rows.push(SweepCsvRow { + concurrency, + accepted_tps, + rejected_count, + p95_ms, + p99_ms, + p999_ms, + }); + } + Ok(rows) +} + +fn parse_csv_u64(cols: &[&str], headers: &BTreeMap<&str, usize>, key: &str) -> BenchResult { + let idx = *headers + .get(key) + .ok_or_else(|| std::io::Error::other(format!("missing CSV column: {key}")))?; + let value = cols + .get(idx) + .ok_or_else(|| std::io::Error::other(format!("missing CSV value at column: {key}")))?; + let parsed = value + .parse::() + .map_err(|e| std::io::Error::other(format!("invalid u64 for {key}: {value} ({e})")))?; + Ok(parsed) +} + +fn parse_csv_f64(cols: &[&str], headers: &BTreeMap<&str, usize>, key: &str) -> BenchResult { + let idx = *headers + .get(key) + .ok_or_else(|| std::io::Error::other(format!("missing CSV column: {key}")))?; + let value = cols + .get(idx) + .ok_or_else(|| std::io::Error::other(format!("missing CSV value at column: {key}")))?; + let parsed = value + .parse::() + .map_err(|e| std::io::Error::other(format!("invalid f64 for {key}: {value} ({e})")))?; + Ok(parsed) +} + +fn map_sweep_rows(rows: &[SweepCsvRow]) -> BTreeMap { + let mut out = BTreeMap::new(); + for row in rows { + out.insert(row.concurrency, row.clone()); + } + out +} + +fn max_zero_rejection_tps(rows: &[SweepCsvRow]) -> f64 { + rows.iter() + .filter(|row| row.rejected_count == 0) + .map(|row| row.accepted_tps) + .fold(0.0, f64::max) +} + +fn first_rejection_tps(rows: &[SweepCsvRow]) -> Option { + rows.iter() + .find(|row| row.rejected_count > 0) + .map(|row| row.accepted_tps) +} + +fn print_common(accepted: u64, rejected: u64, total_wall: &DurationJson) { + let throughput = throughput(accepted, total_wall); + println!(" old summary:"); + println!(" accepted: {accepted}"); + println!(" rejected: {rejected}"); + println!(" completed_per_s: {:.2} tx/s", throughput); +} + +fn print_common_delta( + old_accepted: u64, + old_rejected: u64, + old_total_wall: &DurationJson, + new_accepted: u64, + new_rejected: u64, + new_total_wall: &DurationJson, +) { + let old_tps = throughput(old_accepted, old_total_wall); + let new_tps = throughput(new_accepted, new_total_wall); + println!(" new summary:"); + println!( + " accepted: {new_accepted} (delta {:+})", + i128::from(new_accepted) - i128::from(old_accepted) + ); + println!( + " rejected: {new_rejected} (delta {:+})", + i128::from(new_rejected) - i128::from(old_rejected) + ); + println!( + " completed_per_s: {:.2} tx/s (delta {:+.2}, {:+.2}%)", + new_tps, + new_tps - old_tps, + pct(new_tps, old_tps) + ); +} + +fn print_stats_delta(name: &str, old: &StatsJson, new: &StatsJson) { + println!(" {name} delta (new - old):"); + print_metric_delta("p50", old.p50.as_ms_f64(), new.p50.as_ms_f64()); + print_metric_delta("p95", old.p95.as_ms_f64(), new.p95.as_ms_f64()); + print_metric_delta("p99", old.p99.as_ms_f64(), new.p99.as_ms_f64()); + print_metric_delta("p99.9", old.p999.as_ms_f64(), new.p999.as_ms_f64()); + print_metric_delta("max", old.max.as_ms_f64(), new.max.as_ms_f64()); +} + +fn print_memory_delta(old: Option<&MemoryJson>, new: Option<&MemoryJson>) { + let (Some(old), Some(new)) = (old, new) else { + return; + }; + println!(" memory delta (new - old):"); + print_metric_delta("rss_start_mb", old.rss_start_mb, new.rss_start_mb); + print_metric_delta("rss_peak_mb", old.rss_peak_mb, new.rss_peak_mb); + print_metric_delta("rss_growth_mb", old.rss_growth_mb, new.rss_growth_mb); +} + +fn print_metric_delta(label: &str, old_value: f64, new_value: f64) { + println!( + " {label}: {:.3} -> {:.3} (delta {:+.3}, {:+.2}%)", + old_value, + new_value, + new_value - old_value, + pct(new_value, old_value) + ); +} + +fn throughput(accepted: u64, total_wall: &DurationJson) -> f64 { + let secs = total_wall.as_secs_f64(); + if secs == 0.0 { + return 0.0; + } + accepted as f64 / secs +} + +fn pct(new_value: f64, old_value: f64) -> f64 { + if old_value == 0.0 { + 0.0 + } else { + ((new_value / old_value) - 1.0) * 100.0 + } +} + +fn fmt_opt(value: Option) -> String { + match value { + Some(v) => format!("{v:.3}"), + None => "n/a".to_string(), + } +} diff --git a/benchmarks/src/bin/e2e_latency.rs b/benchmarks/src/bin/e2e_latency.rs new file mode 100644 index 0000000..23a47cc --- /dev/null +++ b/benchmarks/src/bin/e2e_latency.rs @@ -0,0 +1,299 @@ +// (c) Cartesi and individual authors (see AUTHORS) +// SPDX-License-Identifier: Apache-2.0 (see LICENSE) + +use benchmarks::{ + BenchResult, DEFAULT_ENDPOINT, DEFAULT_WORKLOAD_INITIAL_BALANCE, + DEFAULT_WORKLOAD_TRANSFER_AMOUNT, E2eRunConfig, WorkloadConfig, WorkloadKind, + default_seed_offset, print_e2e_report, run_e2e_benchmark, + runtime::{ + DEFAULT_MEMORY_SAMPLE_INTERVAL_MS, DEFAULT_RUNTIME_METRICS_LOG_INTERVAL_MS, + DEFAULT_SEQUENCER_BIN, DEFAULT_SEQUENCER_SHUTDOWN_TIMEOUT_MS, + DEFAULT_SEQUENCER_START_TIMEOUT_MS, ManagedSequencer, ManagedSequencerConfig, + MemorySampler, default_sequencer_log_path, parse_inclusion_lane_profile_from_log, + }, +}; +use clap::{Parser, ValueEnum}; +use serde::Serialize; +use std::fs; +use std::path::Path; +use std::path::PathBuf; +use std::time::Duration; + +#[derive(Debug, Parser)] +#[command( + name = "e2e_latency", + about = "end-to-end latency benchmark", + version, + after_help = "Examples:\n cargo run -p benchmarks --bin e2e_latency -- --endpoint http://127.0.0.1:3000 --count 1000 --concurrency 16 --max-fee 0 --from-offset 0\n cargo run -p benchmarks --bin e2e_latency --release -- --count 2000 --concurrency 64 --allow-rejections" +)] +struct Args { + #[arg(long, visible_alias = "http-url", default_value = DEFAULT_ENDPOINT)] + endpoint: String, + #[arg(long, default_value_t = false)] + self_contained: bool, + #[arg(long, default_value = DEFAULT_SEQUENCER_BIN)] + sequencer_bin: String, + #[arg(long, default_value_t = DEFAULT_SEQUENCER_START_TIMEOUT_MS)] + sequencer_start_timeout_ms: u64, + #[arg(long, default_value_t = DEFAULT_SEQUENCER_SHUTDOWN_TIMEOUT_MS)] + sequencer_shutdown_timeout_ms: u64, + #[arg(long, default_value_t = true)] + temp_db: bool, + #[arg(long)] + sequencer_log_path: Option, + #[arg(long, default_value_t = true)] + sequencer_runtime_metrics_enabled: bool, + #[arg(long, default_value_t = DEFAULT_RUNTIME_METRICS_LOG_INTERVAL_MS)] + sequencer_runtime_metrics_log_interval_ms: u64, + #[arg(long, default_value = "info")] + sequencer_rust_log: String, + #[arg(long, default_value_t = DEFAULT_MEMORY_SAMPLE_INTERVAL_MS)] + memory_sample_interval_ms: u64, + #[arg(long, value_enum, default_value_t = CliWorkload::Synthetic)] + workload: CliWorkload, + #[arg(long)] + accounts_file: Option, + #[arg(long, default_value_t = DEFAULT_WORKLOAD_INITIAL_BALANCE)] + initial_balance: u64, + #[arg(long, default_value_t = DEFAULT_WORKLOAD_TRANSFER_AMOUNT)] + transfer_amount: u64, + #[arg(long, visible_alias = "ws-url")] + ws_subscribe_url: Option, + #[arg(long, default_value_t = 0_u64)] + from_offset: u64, + #[arg(long, default_value_t = 100_u64)] + count: u64, + #[arg(long, default_value_t = 1_usize)] + concurrency: usize, + #[arg(long)] + seed_offset: Option, + #[arg(long, default_value_t = 0_u32)] + max_fee: u32, + #[arg(long, default_value_t = 3_000_u64)] + request_timeout_ms: u64, + #[arg(long, default_value_t = 5_000_u64)] + max_ws_wait_ms: u64, + #[arg(long, default_value_t = 0_u64)] + progress_every: u64, + #[arg(long, default_value_t = false)] + skip_backlog_drain: bool, + #[arg(long, default_value_t = 25_u64)] + backlog_drain_idle_ms: u64, + #[arg(long, default_value_t = 2_000_u64)] + backlog_drain_max_ms: u64, + #[arg(long, default_value_t = false)] + allow_rejections: bool, + #[arg(long)] + json_out: Option, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, ValueEnum)] +enum CliWorkload { + #[value(name = "synthetic")] + Synthetic, + #[value(name = "funded-transfer")] + FundedTransfer, +} + +impl From for WorkloadKind { + fn from(value: CliWorkload) -> Self { + match value { + CliWorkload::Synthetic => Self::Synthetic, + CliWorkload::FundedTransfer => Self::FundedTransfer, + } + } +} + +impl CliWorkload { + fn as_str(self) -> &'static str { + match self { + Self::Synthetic => "synthetic", + Self::FundedTransfer => "funded-transfer", + } + } +} + +#[derive(Debug, Serialize)] +struct E2eJsonConfig { + endpoint: String, + ws_subscribe_url: Option, + self_contained: bool, + from_offset: u64, + count: u64, + concurrency: usize, + max_fee: u32, + request_timeout_ms: u64, + max_ws_wait_ms: u64, + allow_rejections: bool, + workload: String, + accounts_file: Option, + initial_balance: u64, + transfer_amount: u64, +} + +#[derive(Debug, Serialize)] +struct E2eJsonOutput { + benchmark: &'static str, + config: E2eJsonConfig, + report: benchmarks::E2eRunReport, +} + +#[tokio::main] +async fn main() -> BenchResult<()> { + let args = Args::parse(); + let json_out = args.json_out.clone().or_else(|| { + args.self_contained + .then(|| default_json_output_path("e2e-latency")) + }); + let mut managed = if args.self_contained { + Some( + ManagedSequencer::spawn(ManagedSequencerConfig { + sequencer_bin: args.sequencer_bin.clone(), + start_timeout: Duration::from_millis(args.sequencer_start_timeout_ms), + shutdown_timeout: Duration::from_millis(args.sequencer_shutdown_timeout_ms), + temp_db: args.temp_db, + log_path: args + .sequencer_log_path + .clone() + .or_else(|| Some(default_sequencer_log_path("e2e-latency-self-contained"))), + runtime_metrics_enabled: args.sequencer_runtime_metrics_enabled, + runtime_metrics_log_interval: Duration::from_millis( + args.sequencer_runtime_metrics_log_interval_ms, + ), + rust_log: args.sequencer_rust_log.clone(), + }) + .await?, + ) + } else { + None + }; + let endpoint = managed + .as_ref() + .map(|value| value.endpoint.clone()) + .unwrap_or_else(|| args.endpoint.clone()); + let ws_subscribe_url = if args.self_contained { + managed.as_ref().map(|value| value.ws_subscribe_url.clone()) + } else { + args.ws_subscribe_url.clone() + }; + + println!( + "e2e config: endpoint={}, self_contained={}, ws_subscribe_url={:?}, from_offset={}, count={}, concurrency={}, max_fee={}, request_timeout_ms={}, max_ws_wait_ms={}, allow_rejections={}, workload={:?}", + endpoint, + args.self_contained, + ws_subscribe_url, + args.from_offset, + args.count, + args.concurrency.max(1), + args.max_fee, + args.request_timeout_ms, + args.max_ws_wait_ms, + args.allow_rejections, + args.workload + ); + + let memory_sampler = managed.as_ref().and_then(|value| value.pid()).map(|pid| { + MemorySampler::start(pid, Duration::from_millis(args.memory_sample_interval_ms)) + }); + + let config = E2eRunConfig { + endpoint, + ws_subscribe_url, + from_offset: args.from_offset, + count: args.count, + concurrency: args.concurrency.max(1), + seed_offset: args.seed_offset.unwrap_or_else(default_seed_offset), + max_fee: args.max_fee, + request_timeout_ms: args.request_timeout_ms, + max_ws_wait_ms: args.max_ws_wait_ms, + progress_every: args.progress_every, + drain_backlog_before_bench: !args.skip_backlog_drain, + backlog_drain_idle_ms: args.backlog_drain_idle_ms, + backlog_drain_max_ms: args.backlog_drain_max_ms, + fail_on_rejection: !args.allow_rejections, + workload: WorkloadConfig { + kind: args.workload.into(), + accounts_file: args.accounts_file.clone(), + initial_balance: args.initial_balance, + transfer_amount: args.transfer_amount, + }, + }; + + let mut report_result = run_e2e_benchmark(config).await; + if let Some(path) = managed + .as_ref() + .map(|value| value.log_path().to_string_lossy().to_string()) + && let Ok(report) = report_result.as_mut() + { + report.sequencer_log_path = Some(path); + } + if let Some(sampler) = memory_sampler { + match report_result.as_mut() { + Ok(report) => report.memory = Some(sampler.stop(report.accepted).await?), + Err(_) => { + let _ = sampler.stop(0).await; + } + } + } + if let Some(value) = managed.take() { + let shutdown_result = value.shutdown().await; + if let Err(err) = shutdown_result + && report_result.is_ok() + { + return Err(err); + } + } + + if let Some(path) = report_result + .as_ref() + .ok() + .and_then(|report| report.sequencer_log_path.clone()) + && let Some(profile) = parse_inclusion_lane_profile_from_log(PathBuf::from(path).as_path())? + && let Ok(report) = report_result.as_mut() + { + report.inclusion_lane_profile = Some(profile); + } + + let report = report_result?; + print_e2e_report(&report); + if let Some(path) = json_out.as_ref() { + if let Some(parent) = Path::new(path).parent() { + fs::create_dir_all(parent)?; + } + let payload = E2eJsonOutput { + benchmark: "e2e_latency", + config: E2eJsonConfig { + endpoint: report.endpoint.clone(), + ws_subscribe_url: if args.self_contained { + Some(report.ws_subscribe_url.clone()) + } else { + args.ws_subscribe_url.clone() + }, + self_contained: args.self_contained, + from_offset: args.from_offset, + count: args.count, + concurrency: args.concurrency.max(1), + max_fee: args.max_fee, + request_timeout_ms: args.request_timeout_ms, + max_ws_wait_ms: args.max_ws_wait_ms, + allow_rejections: args.allow_rejections, + workload: args.workload.as_str().to_string(), + accounts_file: args.accounts_file.clone(), + initial_balance: args.initial_balance, + transfer_amount: args.transfer_amount, + }, + report, + }; + fs::write(path, serde_json::to_vec_pretty(&payload)?)?; + println!("e2e json: {path}"); + } + Ok(()) +} + +fn default_json_output_path(prefix: &str) -> String { + let ts = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|value| value.as_secs()) + .unwrap_or(0); + format!("benchmarks/results/{prefix}-{ts}.json") +} diff --git a/benchmarks/src/bin/sweep.rs b/benchmarks/src/bin/sweep.rs new file mode 100644 index 0000000..510e886 --- /dev/null +++ b/benchmarks/src/bin/sweep.rs @@ -0,0 +1,675 @@ +// (c) Cartesi and individual authors (see AUTHORS) +// SPDX-License-Identifier: Apache-2.0 (see LICENSE) + +use benchmarks::{ + AckRunConfig, BenchResult, DEFAULT_ENDPOINT, DEFAULT_WORKLOAD_INITIAL_BALANCE, + DEFAULT_WORKLOAD_TRANSFER_AMOUNT, E2eRunConfig, WorkloadConfig, WorkloadKind, + default_seed_offset, print_ack_report, print_e2e_report, run_ack_benchmark, run_e2e_benchmark, + runtime::{ + DEFAULT_MEMORY_SAMPLE_INTERVAL_MS, DEFAULT_RUNTIME_METRICS_LOG_INTERVAL_MS, + DEFAULT_SEQUENCER_BIN, DEFAULT_SEQUENCER_SHUTDOWN_TIMEOUT_MS, + DEFAULT_SEQUENCER_START_TIMEOUT_MS, ManagedSequencer, ManagedSequencerConfig, + MemorySampler, default_sequencer_log_path, parse_inclusion_lane_profile_from_log, + }, +}; +use clap::{Parser, ValueEnum}; +use serde::Serialize; +use std::collections::BTreeMap; +use std::fs; +use std::path::PathBuf; +use std::process::Command; +use std::time::Duration; +use std::time::{SystemTime, UNIX_EPOCH}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, ValueEnum)] +enum SweepMode { + #[value(name = "ack")] + Ack, + #[value(name = "e2e")] + E2e, +} + +impl SweepMode { + fn as_str(self) -> &'static str { + match self { + Self::Ack => "ack", + Self::E2e => "e2e", + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, ValueEnum)] +enum CliWorkload { + #[value(name = "synthetic")] + Synthetic, + #[value(name = "funded-transfer")] + FundedTransfer, +} + +impl From for WorkloadKind { + fn from(value: CliWorkload) -> Self { + match value { + CliWorkload::Synthetic => Self::Synthetic, + CliWorkload::FundedTransfer => Self::FundedTransfer, + } + } +} + +#[derive(Debug, Clone, Parser)] +#[command( + name = "sweep", + about = "benchmark sweep runner", + version, + after_help = "Examples:\n cargo run -p benchmarks --bin sweep -- --mode e2e --endpoint http://127.0.0.1:3000 --count 1000 --concurrency-list \"1 2 4 8 16 32 64\"\n cargo run -p benchmarks --bin sweep -- --mode e2e --concurrency-range 1:128:8 --json-out benchmarks/results/e2e-latest.json\n cargo run -p benchmarks --bin sweep --release -- --mode ack --count 5000 --concurrency-list \"1 2 4 8 16 32 64 96 128\"" +)] +struct Args { + #[arg(long, value_enum, default_value_t = SweepMode::E2e)] + mode: SweepMode, + #[arg(long, default_value_t = 1_000_u64)] + count: u64, + #[arg(long, visible_alias = "url", default_value = DEFAULT_ENDPOINT)] + endpoint: String, + #[arg(long, default_value_t = false)] + self_contained: bool, + #[arg(long, default_value = DEFAULT_SEQUENCER_BIN)] + sequencer_bin: String, + #[arg(long, default_value_t = DEFAULT_SEQUENCER_START_TIMEOUT_MS)] + sequencer_start_timeout_ms: u64, + #[arg(long, default_value_t = DEFAULT_SEQUENCER_SHUTDOWN_TIMEOUT_MS)] + sequencer_shutdown_timeout_ms: u64, + #[arg(long, default_value_t = true)] + temp_db: bool, + #[arg(long)] + sequencer_log_path: Option, + #[arg(long, default_value_t = true)] + sequencer_runtime_metrics_enabled: bool, + #[arg(long, default_value_t = DEFAULT_RUNTIME_METRICS_LOG_INTERVAL_MS)] + sequencer_runtime_metrics_log_interval_ms: u64, + #[arg(long, default_value = "info")] + sequencer_rust_log: String, + #[arg(long, default_value_t = DEFAULT_MEMORY_SAMPLE_INTERVAL_MS)] + memory_sample_interval_ms: u64, + #[arg(long, value_enum, default_value_t = CliWorkload::Synthetic)] + workload: CliWorkload, + #[arg(long)] + accounts_file: Option, + #[arg(long, default_value_t = DEFAULT_WORKLOAD_INITIAL_BALANCE)] + initial_balance: u64, + #[arg(long, default_value_t = DEFAULT_WORKLOAD_TRANSFER_AMOUNT)] + transfer_amount: u64, + #[arg(long, default_value_t = 0_u32)] + max_fee: u32, + #[arg(long, default_value_t = 0_u64)] + from_offset: u64, + #[arg( + long, + conflicts_with = "concurrency_range", + value_delimiter = ' ', + num_args = 1.. + )] + concurrency_list: Option>, + #[arg( + long, + conflicts_with = "concurrency_list", + value_name = "START:END:STEP" + )] + concurrency_range: Option, + #[arg(long, default_value = "benchmarks/results")] + results_dir: String, + #[arg(long)] + json_out: Option, + #[arg(long, default_value_t = 10_000_u64)] + e2e_request_timeout_ms: u64, + #[arg(long, default_value_t = 20_000_u64)] + e2e_max_ws_wait_ms: u64, + #[arg(long, default_value_t = false)] + stop_on_first_non_200: bool, +} + +#[derive(Debug, Clone, Serialize)] +struct SweepRow { + concurrency: usize, + accepted_tps: f64, + accepted_count: u64, + rejected_count: u64, + rejection_rate: f64, + p95_ms: f64, + p99_ms: f64, + p999_ms: f64, + rejection_breakdown: BTreeMap, +} + +#[derive(Debug, Clone, Serialize)] +struct SweepSummary { + tps_at_first_non_200: Option, + tps_at_first_429: Option, + max_sustainable_tps_at_0_rejections: Option, +} + +#[derive(Debug, Clone, Serialize)] +struct SweepJson { + mode: String, + endpoint: String, + count: u64, + max_fee: u32, + from_offset: u64, + rows: Vec, + summary: SweepSummary, + memory: Option, + sequencer_log_path: Option, + inclusion_lane_profile: Option, +} + +#[tokio::main] +async fn main() -> BenchResult<()> { + let args = Args::parse(); + let json_out = args.json_out.clone().or_else(|| { + args.self_contained + .then(|| default_json_output_path("sweep")) + }); + let concurrencies = resolve_concurrency_list(&args)?; + if concurrencies.is_empty() { + return Err(std::io::Error::other("concurrency list cannot be empty").into()); + } + + fs::create_dir_all(args.results_dir.as_str())?; + let timestamp = timestamp_string(); + let csv_path = format!( + "{}/{}-sweep-{}.csv", + args.results_dir, + args.mode.as_str(), + timestamp + ); + + let mut managed = if args.self_contained { + Some( + ManagedSequencer::spawn(ManagedSequencerConfig { + sequencer_bin: args.sequencer_bin.clone(), + start_timeout: Duration::from_millis(args.sequencer_start_timeout_ms), + shutdown_timeout: Duration::from_millis(args.sequencer_shutdown_timeout_ms), + temp_db: args.temp_db, + log_path: args + .sequencer_log_path + .clone() + .or_else(|| Some(default_sequencer_log_path("sweep-self-contained"))), + runtime_metrics_enabled: args.sequencer_runtime_metrics_enabled, + runtime_metrics_log_interval: Duration::from_millis( + args.sequencer_runtime_metrics_log_interval_ms, + ), + rust_log: args.sequencer_rust_log.clone(), + }) + .await?, + ) + } else { + None + }; + + let endpoint = managed + .as_ref() + .map(|value| value.endpoint.clone()) + .unwrap_or_else(|| args.endpoint.clone()); + let ws_subscribe_url = managed.as_ref().map(|value| value.ws_subscribe_url.clone()); + let sequencer_log_path = managed + .as_ref() + .map(|value| value.log_path().to_string_lossy().to_string()); + let memory_sampler = managed.as_ref().and_then(|value| value.pid()).map(|pid| { + MemorySampler::start(pid, Duration::from_millis(args.memory_sample_interval_ms)) + }); + + println!( + "starting {} sweep: endpoint={} self_contained={} count={} max_fee={} workload={:?} stop_on_first_non_200={} concs={:?}", + args.mode.as_str(), + endpoint, + args.self_contained, + args.count, + args.max_fee, + args.workload, + args.stop_on_first_non_200, + concurrencies + ); + println!("host fd soft limit (ulimit -n): {}", fd_soft_limit_string()); + + let mut rows = Vec::new(); + let mut total_accepted = 0_u64; + let mut current_from_offset = args.from_offset; + let mut seed_offset = default_seed_offset(); + + let workload = WorkloadConfig { + kind: args.workload.into(), + accounts_file: args.accounts_file.clone(), + initial_balance: args.initial_balance, + transfer_amount: args.transfer_amount, + }; + + let mut run_error: Option> = None; + + for concurrency in concurrencies.iter().copied() { + println!(); + println!( + "=== sweep mode={} concurrency={} ===", + args.mode.as_str(), + concurrency + ); + + let result = match args.mode { + SweepMode::Ack => { + let config = AckRunConfig { + endpoint: endpoint.clone(), + count: args.count, + concurrency, + seed_offset, + max_fee: args.max_fee, + request_timeout_ms: 3_000, + progress_every: 500, + fail_on_rejection: false, + workload: workload.clone(), + }; + run_ack_benchmark(config).await.map(|report| { + seed_offset = seed_offset.saturating_add(args.count); + print_ack_report(&report); + SweepRow { + concurrency, + accepted_tps: tx_per_second(report.accepted as usize, report.total_wall), + accepted_count: report.accepted, + rejected_count: report.rejected, + rejection_rate: report.rejection_rate, + p95_ms: report.ack_latency_accepted.p95.as_secs_f64() * 1000.0, + p99_ms: report.ack_latency_accepted.p99.as_secs_f64() * 1000.0, + p999_ms: report.ack_latency_accepted.p999.as_secs_f64() * 1000.0, + rejection_breakdown: report.rejection_breakdown, + } + }) + } + SweepMode::E2e => { + let config = E2eRunConfig { + endpoint: endpoint.clone(), + ws_subscribe_url: ws_subscribe_url.clone(), + from_offset: current_from_offset, + count: args.count, + concurrency, + seed_offset, + max_fee: args.max_fee, + request_timeout_ms: args.e2e_request_timeout_ms, + max_ws_wait_ms: args.e2e_max_ws_wait_ms, + progress_every: 500, + drain_backlog_before_bench: true, + backlog_drain_idle_ms: 25, + backlog_drain_max_ms: 2_000, + fail_on_rejection: false, + workload: workload.clone(), + }; + run_e2e_benchmark(config).await.map(|report| { + seed_offset = seed_offset.saturating_add(args.count); + current_from_offset = + current_from_offset.saturating_add(report.consumed_ws_events_total); + print_e2e_report(&report); + SweepRow { + concurrency, + accepted_tps: tx_per_second(report.accepted as usize, report.total_wall), + accepted_count: report.accepted, + rejected_count: report.rejected, + rejection_rate: report.rejection_rate, + p95_ms: report.e2e_latency_accepted.p95.as_secs_f64() * 1000.0, + p99_ms: report.e2e_latency_accepted.p99.as_secs_f64() * 1000.0, + p999_ms: report.e2e_latency_accepted.p999.as_secs_f64() * 1000.0, + rejection_breakdown: report.rejection_breakdown, + } + }) + } + }; + + match result { + Ok(row) => { + total_accepted = total_accepted.saturating_add(row.accepted_count); + let should_stop = args.stop_on_first_non_200 && row.rejected_count > 0; + rows.push(row); + if should_stop { + println!("stopping sweep at first non-200 response"); + break; + } + } + Err(err) => { + let message = err.to_string(); + if message.contains("Too many open files") { + println!(); + println!( + "sweep stopped: hit file-descriptor limit at concurrency={concurrency}" + ); + println!( + "hint: raise soft limit before running sweep (example: ulimit -n 4096)" + ); + println!("or use a smaller conc_list."); + } + run_error = Some(err); + break; + } + } + } + + let mut memory_report = None; + if let Some(sampler) = memory_sampler { + memory_report = Some(sampler.stop(total_accepted).await?); + } + + if let Some(value) = managed.take() { + let shutdown_result = value.shutdown().await; + if run_error.is_none() && shutdown_result.is_err() { + return shutdown_result; + } + } + let inclusion_lane_profile = if let Some(path) = sequencer_log_path.as_ref() { + parse_inclusion_lane_profile_from_log(PathBuf::from(path).as_path())? + } else { + None + }; + + if let Some(err) = run_error { + return Err(err); + } + + write_csv(csv_path.as_str(), rows.as_slice())?; + let summary = compute_capacity_summary(rows.as_slice()); + + println!(); + println!("sweep csv: {csv_path}"); + println!( + "tps_at_first_non_200: {}", + format_optional(summary.tps_at_first_non_200) + ); + println!( + "tps_at_first_429: {}", + format_optional(summary.tps_at_first_429) + ); + println!( + "max_sustainable_tps_at_0_rejections: {}", + format_optional(summary.max_sustainable_tps_at_0_rejections) + ); + if let Some(memory) = memory_report.as_ref() { + benchmarks::print_memory_report(memory); + } + if let Some(path) = sequencer_log_path.as_ref() { + println!("sequencer_log_path: {path}"); + } + if let Some(profile) = inclusion_lane_profile.as_ref() { + println!("inclusion_lane_profile:"); + println!(" samples: {}", profile.samples); + println!( + " latest_user_op_app_share_pct_of_app_plus_persist: {}", + format_optional(profile.latest_user_op_app_share_pct_of_app_plus_persist) + ); + println!( + " latest_user_op_persist_share_pct_of_app_plus_persist: {}", + format_optional(profile.latest_user_op_persist_share_pct_of_app_plus_persist) + ); + println!( + " avg_user_op_app_share_pct_of_app_plus_persist: {}", + format_optional(profile.avg_user_op_app_share_pct_of_app_plus_persist) + ); + println!( + " avg_user_op_persist_share_pct_of_app_plus_persist: {}", + format_optional(profile.avg_user_op_persist_share_pct_of_app_plus_persist) + ); + } + + if let Some(path) = json_out.as_ref() { + let json_path = PathBuf::from(path); + if let Some(parent) = json_path.parent() { + fs::create_dir_all(parent)?; + } + let payload = SweepJson { + mode: args.mode.as_str().to_string(), + endpoint, + count: args.count, + max_fee: args.max_fee, + from_offset: args.from_offset, + rows, + summary, + memory: memory_report, + sequencer_log_path, + inclusion_lane_profile, + }; + fs::write(path, serde_json::to_vec_pretty(&payload)?)?; + println!("sweep json: {path}"); + } + Ok(()) +} + +fn resolve_concurrency_list(args: &Args) -> BenchResult> { + if let Some(values) = args.concurrency_list.as_ref() { + return Ok(values.iter().copied().filter(|value| *value > 0).collect()); + } + if let Some(range) = args.concurrency_range.as_ref() { + return parse_concurrency_range(range); + } + Ok(vec![1, 2, 4, 8, 16, 32, 64, 96, 128]) +} + +fn parse_concurrency_range(value: &str) -> BenchResult> { + let parts: Vec<&str> = value.split(':').collect(); + if parts.len() != 3 { + return Err(std::io::Error::other(format!( + "invalid --concurrency-range '{value}', expected START:END:STEP" + )) + .into()); + } + + let start = parts[0].parse::().map_err(|_| { + std::io::Error::other(format!( + "invalid range start in --concurrency-range: '{value}'" + )) + })?; + let end = parts[1].parse::().map_err(|_| { + std::io::Error::other(format!( + "invalid range end in --concurrency-range: '{value}'" + )) + })?; + let step = parts[2].parse::().map_err(|_| { + std::io::Error::other(format!( + "invalid range step in --concurrency-range: '{value}'" + )) + })?; + + if start == 0 || end == 0 || step == 0 { + return Err(std::io::Error::other("concurrency range values must all be > 0").into()); + } + if start > end { + return Err(std::io::Error::other("concurrency range start must be <= end").into()); + } + + let mut out = Vec::new(); + let mut current = start; + while current <= end { + out.push(current); + current = current.saturating_add(step); + if current == usize::MAX { + break; + } + } + Ok(out) +} + +fn write_csv(path: &str, rows: &[SweepRow]) -> BenchResult<()> { + let mut out = String::from( + "concurrency,accepted_tps,accepted_count,rejected_count,rejection_rate,p95_ms,p99_ms,p999_ms\n", + ); + for row in rows { + out.push_str( + format!( + "{},{:.6},{},{},{:.6},{:.6},{:.6},{:.6}\n", + row.concurrency, + row.accepted_tps, + row.accepted_count, + row.rejected_count, + row.rejection_rate, + row.p95_ms, + row.p99_ms, + row.p999_ms, + ) + .as_str(), + ); + } + fs::write(path, out)?; + Ok(()) +} + +fn compute_capacity_summary(rows: &[SweepRow]) -> SweepSummary { + let tps_at_first_non_200 = rows + .iter() + .find(|row| row.rejected_count > 0) + .map(|row| row.accepted_tps); + + let tps_at_first_429 = rows + .iter() + .find(|row| { + row.rejection_breakdown + .get("http_429") + .copied() + .unwrap_or(0) + > 0 + }) + .map(|row| row.accepted_tps); + + let max_sustainable_tps_at_0_rejections = rows + .iter() + .filter(|row| row.rejected_count == 0) + .map(|row| row.accepted_tps) + .max_by(|a, b| a.total_cmp(b)); + + SweepSummary { + tps_at_first_non_200, + tps_at_first_429, + max_sustainable_tps_at_0_rejections, + } +} + +fn tx_per_second(count: usize, total_wall: std::time::Duration) -> f64 { + if total_wall.is_zero() { + 0.0 + } else { + count as f64 / total_wall.as_secs_f64() + } +} + +fn timestamp_string() -> String { + let secs = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_secs(); + secs.to_string() +} + +fn default_json_output_path(prefix: &str) -> String { + let ts = SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|value| value.as_secs()) + .unwrap_or(0); + format!("benchmarks/results/{prefix}-{ts}.json") +} + +fn format_optional(value: Option) -> String { + match value { + Some(v) => format!("{v:.2}"), + None => "not reached".to_string(), + } +} + +fn fd_soft_limit_string() -> String { + #[cfg(unix)] + { + let out = Command::new("sh") + .arg("-c") + .arg("ulimit -n") + .output() + .ok() + .and_then(|value| String::from_utf8(value.stdout).ok()) + .map(|value| value.trim().to_string()); + out.unwrap_or_else(|| "unknown".to_string()) + } + #[cfg(not(unix))] + { + "n/a".to_string() + } +} + +#[cfg(test)] +mod tests { + use super::{SweepRow, compute_capacity_summary}; + use std::collections::BTreeMap; + + #[test] + fn capacity_summary_equal_case() { + let rows = vec![ + SweepRow { + concurrency: 1, + accepted_tps: 10.0, + accepted_count: 100, + rejected_count: 0, + rejection_rate: 0.0, + p95_ms: 1.0, + p99_ms: 1.0, + p999_ms: 1.0, + rejection_breakdown: BTreeMap::new(), + }, + SweepRow { + concurrency: 2, + accepted_tps: 20.0, + accepted_count: 100, + rejected_count: 1, + rejection_rate: 1.0, + p95_ms: 2.0, + p99_ms: 2.0, + p999_ms: 2.0, + rejection_breakdown: BTreeMap::from([("http_429".to_string(), 1_u64)]), + }, + ]; + + let summary = compute_capacity_summary(rows.as_slice()); + assert_eq!(summary.tps_at_first_non_200, Some(20.0)); + assert_eq!(summary.tps_at_first_429, Some(20.0)); + assert_eq!(summary.max_sustainable_tps_at_0_rejections, Some(10.0)); + } + + #[test] + fn capacity_summary_diverging_case() { + let rows = vec![ + SweepRow { + concurrency: 1, + accepted_tps: 10.0, + accepted_count: 100, + rejected_count: 0, + rejection_rate: 0.0, + p95_ms: 1.0, + p99_ms: 1.0, + p999_ms: 1.0, + rejection_breakdown: BTreeMap::new(), + }, + SweepRow { + concurrency: 2, + accepted_tps: 18.0, + accepted_count: 100, + rejected_count: 1, + rejection_rate: 1.0, + p95_ms: 1.5, + p99_ms: 1.5, + p999_ms: 1.5, + rejection_breakdown: BTreeMap::from([("http_422".to_string(), 1_u64)]), + }, + SweepRow { + concurrency: 4, + accepted_tps: 25.0, + accepted_count: 100, + rejected_count: 1, + rejection_rate: 1.0, + p95_ms: 2.0, + p99_ms: 2.0, + p999_ms: 2.0, + rejection_breakdown: BTreeMap::from([("http_429".to_string(), 1_u64)]), + }, + ]; + + let summary = compute_capacity_summary(rows.as_slice()); + assert_eq!(summary.tps_at_first_non_200, Some(18.0)); + assert_eq!(summary.tps_at_first_429, Some(25.0)); + assert_eq!(summary.max_sustainable_tps_at_0_rejections, Some(10.0)); + } +} diff --git a/benchmarks/src/bin/unit_hot_path.rs b/benchmarks/src/bin/unit_hot_path.rs new file mode 100644 index 0000000..ebecc68 --- /dev/null +++ b/benchmarks/src/bin/unit_hot_path.rs @@ -0,0 +1,58 @@ +// (c) Cartesi and individual authors (see AUTHORS) +// SPDX-License-Identifier: Apache-2.0 (see LICENSE) + +use benchmarks::{ + BenchResult, default_domain, make_signed_fixture, now, print_stats, summarize, + throughput_tx_per_s, +}; +use clap::Parser; + +#[derive(Debug, Parser)] +#[command( + name = "unit_hot_path", + about = "unit benchmark for signing + request JSON encoding", + version, + after_help = "Examples:\n cargo run -p benchmarks --bin unit_hot_path -- --count 10000 --max-fee 0\n cargo run -p benchmarks --bin unit_hot_path --release -- --count 50000" +)] +struct Args { + #[arg(long, default_value_t = 10_000_u64)] + count: u64, + #[arg(long, default_value_t = 0_u32)] + max_fee: u32, +} + +fn main() -> BenchResult<()> { + let args = Args::parse(); + println!( + "unit config: count={}, max_fee={}", + args.count, args.max_fee + ); + let domain = default_domain(); + + let mut fixture_build_samples = Vec::with_capacity(args.count as usize); + let mut json_encode_samples = Vec::with_capacity(args.count as usize); + let started = now(); + + for i in 0..args.count { + let build_started = now(); + let fixture = make_signed_fixture(i, args.max_fee, &domain)?; + fixture_build_samples.push(build_started.elapsed()); + + let json_started = now(); + let _json = serde_json::to_string(&fixture.request)?; + json_encode_samples.push(json_started.elapsed()); + } + + let total_wall = started.elapsed(); + let build_stats = summarize(fixture_build_samples.as_slice())?; + let json_stats = summarize(json_encode_samples.as_slice())?; + + println!("unit hot-path benchmark completed: count={}", args.count); + println!( + "unit_ops_per_s: {:.2}", + throughput_tx_per_s(args.count as usize, total_wall) + ); + print_stats("fixture_build (sign+encode payload)", &build_stats); + print_stats("request_json_encode", &json_stats); + Ok(()) +} diff --git a/benchmarks/src/lib.rs b/benchmarks/src/lib.rs new file mode 100644 index 0000000..fd5f3d8 --- /dev/null +++ b/benchmarks/src/lib.rs @@ -0,0 +1,1223 @@ +// (c) Cartesi and individual authors (see AUTHORS) +// SPDX-License-Identifier: Apache-2.0 (see LICENSE) + +pub mod runtime; + +use alloy_primitives::{Address, Signature, U256}; +use alloy_sol_types::{Eip712Domain, SolStruct}; +use futures_util::StreamExt; +use futures_util::future::join_all; +use k256::ecdsa::SigningKey; +use k256::ecdsa::signature::hazmat::PrehashSigner; +use sequencer_core::api::{TxRequest, TxResponse, WsTxMessage}; +use sequencer_core::application::{Deposit, Method, Transfer, Withdrawal}; +use sequencer_core::user_op::{SignedUserOp, UserOp}; +use sequencer_rust_client::{SequencerClient, SubmitRejected, SubmitTxError}; +use serde::{Deserialize, Serialize}; +use std::collections::{BTreeMap, HashMap}; +use std::error::Error; +use std::fs; +use std::time::{Duration, Instant}; +use std::time::{SystemTime, UNIX_EPOCH}; +use tokio_tungstenite::tungstenite::Message; +use tokio_tungstenite::{MaybeTlsStream, WebSocketStream, connect_async}; + +pub type BenchResult = Result>; +pub const DEFAULT_ENDPOINT: &str = "http://127.0.0.1:3000"; +pub const DEFAULT_WORKLOAD_INITIAL_BALANCE: u64 = 1_000_000; +pub const DEFAULT_WORKLOAD_TRANSFER_AMOUNT: u64 = 1; + +const ANVIL_DEFAULT_PRIVATE_KEYS: [&str; 10] = [ + "0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80", + "0x59c6995e998f97a5a0044966f0945389dc9e86dae88c7a8412f4603b6b78690d", + "0x5de4111afa1a4b94908f83103eb1f1706367c2e68ca870fc3fb9a804cdab365a", + "0x7c852118294e51e653712a81e05800f419141751be58f605c371e15141b007a6", + "0x47e179ec197488593b187f80a00eb0da91f1b9d0b13f8733639f19c30a34926a", + "0x8b3a350cf5c34c9194ca85829a2df0ec3153be0318b5e2d3348e872092edffba", + "0x92db14e403b83dfe3df233f83dfa3a0d7096f21ca9b0d6d6b8d88b2b4ec1564e", + "0x4bbbf85ce3377467afe5d46f804f221813b2bb87f24d81f60f1fcdbf7cbf4356", + "0xdbda1821b80551c9d65939329250298aa3472ba22feea921c0cf5d620ea67b97", + "0x2a871d0798f97d79848a013d4936a73bf4cc922c825d33c1cf7073dff6d409c6", +]; + +#[derive(Debug, Clone)] +pub struct SignedTxFixture { + pub request: TxRequest, + pub expected_sender: String, + pub expected_data_hex: String, +} + +#[derive(Debug, Clone, Serialize)] +pub struct Stats { + pub count: usize, + pub min: Duration, + pub max: Duration, + pub mean: Duration, + pub p50: Duration, + pub p95: Duration, + pub p99: Duration, + pub p999: Duration, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum WorkloadKind { + Synthetic, + FundedTransfer, +} + +#[derive(Debug, Clone, Serialize)] +pub struct WorkloadConfig { + pub kind: WorkloadKind, + pub accounts_file: Option, + pub initial_balance: u64, + pub transfer_amount: u64, +} + +impl Default for WorkloadConfig { + fn default() -> Self { + Self { + kind: WorkloadKind::Synthetic, + accounts_file: None, + initial_balance: DEFAULT_WORKLOAD_INITIAL_BALANCE, + transfer_amount: DEFAULT_WORKLOAD_TRANSFER_AMOUNT, + } + } +} + +#[derive(Debug, Clone)] +pub struct AckRunConfig { + pub endpoint: String, + pub count: u64, + pub concurrency: usize, + pub seed_offset: u64, + pub max_fee: u32, + pub request_timeout_ms: u64, + pub progress_every: u64, + pub fail_on_rejection: bool, + pub workload: WorkloadConfig, +} + +#[derive(Debug, Clone, Serialize)] +pub struct AckRunReport { + pub count: u64, + pub endpoint: String, + pub concurrency: usize, + pub accepted: u64, + pub rejected: u64, + pub rejection_rate: f64, + pub rejection_breakdown: BTreeMap, + pub first_rejection: Option, + pub total_wall: Duration, + pub ack_latency_accepted: Stats, + pub ack_latency_rejected: Option, + pub memory: Option, + pub sequencer_log_path: Option, + pub inclusion_lane_profile: Option, +} + +#[derive(Debug, Clone)] +pub struct E2eRunConfig { + pub endpoint: String, + pub ws_subscribe_url: Option, + pub from_offset: u64, + pub count: u64, + pub concurrency: usize, + pub seed_offset: u64, + pub max_fee: u32, + pub request_timeout_ms: u64, + pub max_ws_wait_ms: u64, + pub progress_every: u64, + pub drain_backlog_before_bench: bool, + pub backlog_drain_idle_ms: u64, + pub backlog_drain_max_ms: u64, + pub fail_on_rejection: bool, + pub workload: WorkloadConfig, +} + +#[derive(Debug, Clone, Serialize)] +pub struct E2eRunReport { + pub count: u64, + pub endpoint: String, + pub ws_subscribe_url: String, + pub concurrency: usize, + pub accepted: u64, + pub rejected: u64, + pub rejection_rate: f64, + pub rejection_breakdown: BTreeMap, + pub first_rejection: Option, + pub drained_ws_backlog_events: u64, + pub consumed_ws_events_total: u64, + pub total_wall: Duration, + pub ack_latency_accepted: Stats, + pub ack_latency_rejected: Option, + pub e2e_latency_accepted: Stats, + pub memory: Option, + pub sequencer_log_path: Option, + pub inclusion_lane_profile: Option, +} + +struct RejectionOutcome { + key: String, + detail: String, +} + +struct WorkloadState { + inner: WorkloadStateInner, +} + +enum WorkloadStateInner { + Synthetic { + next_seed: u64, + }, + FundedTransfer { + accounts: Vec, + round_robin_index: usize, + transfer_amount: u64, + }, +} + +#[derive(Clone)] +struct FundedAccount { + signing_key: SigningKey, + sender: Address, + next_nonce: u32, +} + +impl WorkloadState { + async fn initialize( + config: &WorkloadConfig, + seed_offset: u64, + client: &SequencerClient, + max_fee: u32, + domain: &Eip712Domain, + ) -> BenchResult { + match config.kind { + WorkloadKind::Synthetic => Ok(Self { + inner: WorkloadStateInner::Synthetic { + next_seed: seed_offset, + }, + }), + WorkloadKind::FundedTransfer => { + let mut accounts = load_funded_accounts(config.accounts_file.as_deref())?; + setup_funded_accounts( + client, + max_fee, + U256::from(config.initial_balance), + domain, + accounts.as_mut_slice(), + ) + .await?; + Ok(Self { + inner: WorkloadStateInner::FundedTransfer { + accounts, + round_robin_index: 0, + transfer_amount: config.transfer_amount, + }, + }) + } + } + } + + fn next_fixture( + &mut self, + max_fee: u32, + domain: &Eip712Domain, + ) -> BenchResult { + match &mut self.inner { + WorkloadStateInner::Synthetic { next_seed } => { + let fixture = make_signed_fixture(*next_seed, max_fee, domain)?; + *next_seed = next_seed.wrapping_add(1); + Ok(fixture) + } + WorkloadStateInner::FundedTransfer { + accounts, + round_robin_index, + transfer_amount, + } => { + if accounts.is_empty() { + return Err(err("funded workload has zero accounts")); + } + let sender_index = *round_robin_index % accounts.len(); + let recipient_index = (sender_index + 1) % accounts.len(); + let recipient = accounts[recipient_index].sender; + let sender = &mut accounts[sender_index]; + + let amount = + U256::from((*transfer_amount).saturating_add(u64::from(sender.next_nonce))); + let method = Method::Transfer(Transfer { + amount, + to: recipient, + }); + let data = ssz::Encode::as_ssz_bytes(&method); + if data.len() > SignedUserOp::MAX_METHOD_PAYLOAD_BYTES { + return Err(err(format!( + "funded transfer payload too large: {} > {}", + data.len(), + SignedUserOp::MAX_METHOD_PAYLOAD_BYTES + ))); + } + + let user_op = UserOp { + nonce: sender.next_nonce, + max_fee, + data: data.into(), + }; + let fixture = + make_signed_fixture_from_signing_key(&sender.signing_key, user_op, domain)?; + sender.next_nonce = sender.next_nonce.wrapping_add(1); + *round_robin_index = (*round_robin_index + 1) % accounts.len(); + Ok(fixture) + } + } + } + + fn concurrency_cap(&self) -> Option { + match &self.inner { + WorkloadStateInner::Synthetic { .. } => None, + WorkloadStateInner::FundedTransfer { accounts, .. } => Some(accounts.len().max(1)), + } + } +} + +pub async fn run_ack_benchmark(config: AckRunConfig) -> BenchResult { + let domain = default_domain(); + let timeout = Duration::from_millis(config.request_timeout_ms); + let client = SequencerClient::new_with_timeout(config.endpoint.clone(), timeout) + .map_err(|e| err(format!("invalid endpoint '{}': {e}", config.endpoint)))?; + let mut workload = WorkloadState::initialize( + &config.workload, + config.seed_offset, + &client, + config.max_fee, + &domain, + ) + .await?; + let effective_concurrency = if let Some(cap) = workload.concurrency_cap() { + let capped = config.concurrency.min(cap); + if capped < config.concurrency { + println!( + "workload concurrency capped: requested={}, effective={}, funded_accounts={}", + config.concurrency, capped, cap + ); + } + capped + } else { + config.concurrency + }; + let mut accepted_ack_samples = Vec::with_capacity(config.count as usize); + let mut rejected_ack_samples = Vec::new(); + let mut accepted = 0_u64; + let mut rejected = 0_u64; + let mut first_rejection: Option = None; + let mut rejection_breakdown = BTreeMap::::new(); + let started = now(); + + while accepted.saturating_add(rejected) < config.count { + let remaining = config + .count + .saturating_sub(accepted.saturating_add(rejected)); + let batch_size = remaining.min(effective_concurrency as u64) as usize; + + let mut inflight = Vec::with_capacity(batch_size); + for _ in 0..batch_size { + let fixture = workload.next_fixture(config.max_fee, &domain)?; + let client = client.clone(); + let sent_at = now(); + inflight.push(async move { + let outcome = client.submit_tx_with_status(&fixture.request).await; + (sent_at.elapsed(), outcome) + }); + } + + for (ack_latency, outcome) in join_all(inflight).await { + match classify_rejection(outcome) { + None => { + accepted = accepted.saturating_add(1); + accepted_ack_samples.push(ack_latency); + } + Some(rejection) => { + rejected = rejected.saturating_add(1); + rejected_ack_samples.push(ack_latency); + *rejection_breakdown + .entry(rejection.key.clone()) + .or_insert(0) += 1; + if first_rejection.is_none() { + first_rejection = Some(rejection.detail); + } + } + } + } + + let processed = accepted.saturating_add(rejected); + if config.progress_every > 0 + && processed > 0 + && processed.is_multiple_of(config.progress_every) + { + println!( + "progress: processed={processed}/{}, accepted={accepted}, rejected={rejected}", + config.count + ); + } + } + + if config.fail_on_rejection && rejected > 0 { + let reason = first_rejection + .clone() + .unwrap_or_else(|| "unknown rejection".to_string()); + return Err(std::io::Error::other(format!( + "ack benchmark saw {rejected} rejection(s): {reason}" + )) + .into()); + } + + if accepted_ack_samples.is_empty() { + return Err(std::io::Error::other("ack benchmark had no accepted txs").into()); + } + + let total_wall = started.elapsed(); + let ack_stats = summarize(accepted_ack_samples.as_slice())?; + let rejected_stats = if rejected_ack_samples.is_empty() { + None + } else { + Some(summarize(rejected_ack_samples.as_slice())?) + }; + + Ok(AckRunReport { + count: config.count, + endpoint: config.endpoint, + concurrency: config.concurrency, + accepted, + rejected, + rejection_rate: rejection_rate(accepted, rejected), + rejection_breakdown, + first_rejection, + total_wall, + ack_latency_accepted: ack_stats, + ack_latency_rejected: rejected_stats, + memory: None, + sequencer_log_path: None, + inclusion_lane_profile: None, + }) +} + +pub fn print_ack_report(report: &AckRunReport) { + println!( + "ack benchmark completed: count={}, endpoint={}, concurrency={}", + report.count, report.endpoint, report.concurrency + ); + println!(" accepted: {}", report.accepted); + println!(" rejected: {}", report.rejected); + println!(" rejection_rate: {:.4}%", report.rejection_rate); + println!( + "accepted_completed_per_s: {:.2} tx/s", + throughput_tx_per_s(report.ack_latency_accepted.count, report.total_wall) + ); + if let Some(reason) = report.first_rejection.as_ref() { + println!(" first_rejection: {reason}"); + } + if !report.rejection_breakdown.is_empty() { + println!(" rejection_breakdown:"); + for (key, count) in &report.rejection_breakdown { + println!(" {key}: {count}"); + } + } + print_stats("ack_latency_accepted", &report.ack_latency_accepted); + if let Some(stats) = report.ack_latency_rejected.as_ref() { + print_stats("ack_latency_rejected", stats); + } + if let Some(memory) = report.memory.as_ref() { + print_memory_report(memory); + } + if let Some(path) = report.sequencer_log_path.as_ref() { + println!("sequencer_log_path: {path}"); + } + if let Some(profile) = report.inclusion_lane_profile.as_ref() { + println!("inclusion_lane_profile:"); + println!(" samples: {}", profile.samples); + println!( + " latest_user_op_app_share_pct_of_app_plus_persist: {}", + format_optional_f64(profile.latest_user_op_app_share_pct_of_app_plus_persist) + ); + println!( + " latest_user_op_persist_share_pct_of_app_plus_persist: {}", + format_optional_f64(profile.latest_user_op_persist_share_pct_of_app_plus_persist) + ); + println!( + " avg_user_op_app_share_pct_of_app_plus_persist: {}", + format_optional_f64(profile.avg_user_op_app_share_pct_of_app_plus_persist) + ); + println!( + " avg_user_op_persist_share_pct_of_app_plus_persist: {}", + format_optional_f64(profile.avg_user_op_persist_share_pct_of_app_plus_persist) + ); + } +} + +pub async fn run_e2e_benchmark(config: E2eRunConfig) -> BenchResult { + let timeout = Duration::from_millis(config.request_timeout_ms); + let client = SequencerClient::new_with_timeout(config.endpoint.clone(), timeout) + .map_err(|e| err(format!("invalid endpoint '{}': {e}", config.endpoint)))?; + let ws_subscribe_url = config + .ws_subscribe_url + .clone() + .map(|base| append_from_offset(base.as_str(), config.from_offset)) + .unwrap_or_else(|| client.ws_subscribe_url(config.from_offset)); + let domain = default_domain(); + let mut workload = WorkloadState::initialize( + &config.workload, + config.seed_offset, + &client, + config.max_fee, + &domain, + ) + .await?; + let effective_concurrency = if let Some(cap) = workload.concurrency_cap() { + let capped = config.concurrency.min(cap); + if capped < config.concurrency { + println!( + "workload concurrency capped: requested={}, effective={}, funded_accounts={}", + config.concurrency, capped, cap + ); + } + capped + } else { + config.concurrency + }; + + let mut ws = if config.ws_subscribe_url.is_some() { + connect_async(ws_subscribe_url.as_str()) + .await + .map(|(stream, _)| stream) + .map_err(|e| { + io_err(format!( + "ws connect failed: url={ws_subscribe_url}, error={e}" + )) + })? + } else { + client.subscribe(config.from_offset).await.map_err(|e| { + io_err(format!( + "ws connect failed: url={ws_subscribe_url}, error={e}" + )) + })? + }; + let mut consumed_ws_events_total = 0_u64; + let mut drained_ws_backlog_events = 0_u64; + + if config.drain_backlog_before_bench { + let drained = drain_existing_ws_backlog( + &mut ws, + Duration::from_millis(config.backlog_drain_idle_ms), + Duration::from_millis(config.backlog_drain_max_ms), + ) + .await?; + consumed_ws_events_total = consumed_ws_events_total.saturating_add(drained); + drained_ws_backlog_events = drained; + println!("drained_ws_backlog_events: {drained}"); + } + + let mut accepted_ack_samples = Vec::with_capacity(config.count as usize); + let mut rejected_ack_samples = Vec::new(); + let mut e2e_samples = Vec::with_capacity(config.count as usize); + let mut accepted = 0_u64; + let mut rejected = 0_u64; + let mut first_rejection: Option = None; + let mut rejection_breakdown = BTreeMap::::new(); + let started = now(); + + let mut processed = 0_u64; + while processed < config.count { + let remaining = config.count.saturating_sub(processed); + let batch_size = remaining.min(effective_concurrency as u64) as usize; + + let mut inflight = Vec::with_capacity(batch_size); + for _ in 0..batch_size { + let fixture = workload.next_fixture(config.max_fee, &domain)?; + let match_key = fixture_match_key( + fixture.expected_sender.as_str(), + fixture.expected_data_hex.as_str(), + ); + let client = client.clone(); + let submit_started = now(); + inflight.push(async move { + let outcome = client.submit_tx_with_status(&fixture.request).await; + (match_key, submit_started, submit_started.elapsed(), outcome) + }); + } + + let mut expected_submit_starts = + HashMap::>::with_capacity(batch_size); + for (match_key, submit_started, ack_latency, outcome) in join_all(inflight).await { + match classify_rejection(outcome) { + None => { + accepted = accepted.saturating_add(1); + accepted_ack_samples.push(ack_latency); + expected_submit_starts + .entry(match_key) + .or_default() + .push(submit_started); + } + Some(rejection) => { + rejected = rejected.saturating_add(1); + rejected_ack_samples.push(ack_latency); + *rejection_breakdown + .entry(rejection.key.clone()) + .or_insert(0) += 1; + if first_rejection.is_none() { + first_rejection = Some(rejection.detail); + } + } + } + } + + if !expected_submit_starts.is_empty() { + let mut matched = wait_for_matching_user_ops( + &mut ws, + &mut expected_submit_starts, + Duration::from_millis(config.max_ws_wait_ms), + ) + .await?; + consumed_ws_events_total = + consumed_ws_events_total.saturating_add(matched.consumed_events); + e2e_samples.append(&mut matched.e2e_samples); + } + + processed = processed.saturating_add(batch_size as u64); + if config.progress_every > 0 + && processed > 0 + && processed.is_multiple_of(config.progress_every) + { + println!( + "progress: processed={processed}/{}, accepted={accepted}, rejected={rejected}", + config.count + ); + } + } + + if config.fail_on_rejection && rejected > 0 { + let reason = first_rejection + .clone() + .unwrap_or_else(|| "unknown rejection".to_string()); + return Err(std::io::Error::other(format!( + "e2e benchmark saw {rejected} rejection(s): {reason}" + )) + .into()); + } + + if accepted_ack_samples.is_empty() { + return Err(std::io::Error::other("e2e benchmark had no accepted txs").into()); + } + if e2e_samples.len() != accepted as usize { + return Err(std::io::Error::other(format!( + "e2e sample mismatch: accepted={accepted}, matched_ws_events={}", + e2e_samples.len() + )) + .into()); + } + + let total_wall = started.elapsed(); + let ack_stats = summarize(accepted_ack_samples.as_slice())?; + let e2e_stats = summarize(e2e_samples.as_slice())?; + let rejected_stats = if rejected_ack_samples.is_empty() { + None + } else { + Some(summarize(rejected_ack_samples.as_slice())?) + }; + + Ok(E2eRunReport { + count: config.count, + endpoint: config.endpoint, + ws_subscribe_url, + concurrency: config.concurrency, + accepted, + rejected, + rejection_rate: rejection_rate(accepted, rejected), + rejection_breakdown, + first_rejection, + drained_ws_backlog_events, + consumed_ws_events_total, + total_wall, + ack_latency_accepted: ack_stats, + ack_latency_rejected: rejected_stats, + e2e_latency_accepted: e2e_stats, + memory: None, + sequencer_log_path: None, + inclusion_lane_profile: None, + }) +} + +pub fn print_e2e_report(report: &E2eRunReport) { + println!( + "e2e benchmark completed: count={}, endpoint={}, ws={}, concurrency={}", + report.count, report.endpoint, report.ws_subscribe_url, report.concurrency + ); + println!(" accepted: {}", report.accepted); + println!(" rejected: {}", report.rejected); + println!(" rejection_rate: {:.4}%", report.rejection_rate); + println!( + "accepted_completed_per_s: {:.2} tx/s", + throughput_tx_per_s(report.e2e_latency_accepted.count, report.total_wall) + ); + println!( + "consumed_ws_events_total: {}", + report.consumed_ws_events_total + ); + if let Some(reason) = report.first_rejection.as_ref() { + println!(" first_rejection: {reason}"); + } + if !report.rejection_breakdown.is_empty() { + println!(" rejection_breakdown:"); + for (key, count) in &report.rejection_breakdown { + println!(" {key}: {count}"); + } + } + print_stats("ack_latency_accepted", &report.ack_latency_accepted); + if let Some(stats) = report.ack_latency_rejected.as_ref() { + print_stats("ack_latency_rejected", stats); + } + print_stats("e2e_latency_accepted", &report.e2e_latency_accepted); + if let Some(memory) = report.memory.as_ref() { + print_memory_report(memory); + } + if let Some(path) = report.sequencer_log_path.as_ref() { + println!("sequencer_log_path: {path}"); + } + if let Some(profile) = report.inclusion_lane_profile.as_ref() { + println!("inclusion_lane_profile:"); + println!(" samples: {}", profile.samples); + println!( + " latest_user_op_app_share_pct_of_app_plus_persist: {}", + format_optional_f64(profile.latest_user_op_app_share_pct_of_app_plus_persist) + ); + println!( + " latest_user_op_persist_share_pct_of_app_plus_persist: {}", + format_optional_f64(profile.latest_user_op_persist_share_pct_of_app_plus_persist) + ); + println!( + " avg_user_op_app_share_pct_of_app_plus_persist: {}", + format_optional_f64(profile.avg_user_op_app_share_pct_of_app_plus_persist) + ); + println!( + " avg_user_op_persist_share_pct_of_app_plus_persist: {}", + format_optional_f64(profile.avg_user_op_persist_share_pct_of_app_plus_persist) + ); + } +} + +pub fn print_memory_report(memory: &runtime::MemoryReport) { + println!("memory:"); + println!(" method: {}", memory.method); + println!(" sample_interval_ms: {}", memory.sample_interval_ms); + println!( + " rss_start_mb: {}", + format_optional_f64(memory.rss_start_mb) + ); + println!(" rss_peak_mb: {}", format_optional_f64(memory.rss_peak_mb)); + println!(" rss_end_mb: {}", format_optional_f64(memory.rss_end_mb)); + println!( + " rss_growth_mb: {}", + format_optional_f64(memory.rss_growth_mb) + ); + println!( + " rss_growth_per_1k_accepted_tx_mb: {}", + format_optional_f64(memory.rss_growth_per_1k_accepted_tx_mb) + ); +} + +pub fn default_domain() -> Eip712Domain { + Eip712Domain { + name: Some("CartesiAppSequencer".to_string().into()), + version: Some("1".to_string().into()), + chain_id: Some(U256::from(1_u64)), + verifying_contract: Some(Address::from_slice(&[0_u8; 20])), + salt: None, + } +} + +pub fn make_signed_fixture( + seed: u64, + max_fee: u32, + domain: &Eip712Domain, +) -> BenchResult { + let signing_key = signing_key_for_seed(seed)?; + let sender = address_from_signing_key(&signing_key); + let method = Method::Withdrawal(Withdrawal { + amount: U256::from(seed.saturating_add(1)), + }); + let data = ssz::Encode::as_ssz_bytes(&method); + if data.len() > SignedUserOp::MAX_METHOD_PAYLOAD_BYTES { + return Err(err(format!( + "benchmark payload too large: {} > {}", + data.len(), + SignedUserOp::MAX_METHOD_PAYLOAD_BYTES + ))); + } + let message = UserOp { + nonce: 0, + max_fee, + data: data.clone().into(), + }; + + let fixture = make_signed_fixture_from_signing_key(&signing_key, message, domain)?; + if fixture.expected_sender != sender.to_string() { + return Err(err("unexpected synthetic sender mismatch")); + } + Ok(fixture) +} + +pub async fn submit_tx( + endpoint: &str, + req: &TxRequest, + timeout: Duration, +) -> BenchResult { + let client = SequencerClient::new_with_timeout(endpoint.to_string(), timeout) + .map_err(|e| err(format!("invalid endpoint '{endpoint}': {e}")))?; + match client.submit_tx(req).await { + Ok(response) => Ok(response), + Err(SubmitRejected::Transport(transport_err)) => { + Err(err(format!("tx submit failed: {transport_err}"))) + } + Err(SubmitRejected::Http { status, body }) => Err(err(format!( + "/tx rejected with status {status}: {body}. Hint: sequencer frame fee and payload-size bounds must allow these txs." + ))), + Err(SubmitRejected::Decode(reason)) => { + Err(err(format!("tx submit decode failed: {reason}"))) + } + } +} + +async fn setup_funded_accounts( + client: &SequencerClient, + max_fee: u32, + initial_balance: U256, + domain: &Eip712Domain, + accounts: &mut [FundedAccount], +) -> BenchResult<()> { + for account in accounts { + let method = Method::Deposit(Deposit { + amount: initial_balance, + to: account.sender, + }); + let data = ssz::Encode::as_ssz_bytes(&method); + let user_op = UserOp { + nonce: 0, + max_fee, + data: data.into(), + }; + let fixture = make_signed_fixture_from_signing_key(&account.signing_key, user_op, domain)?; + let (status, body) = client + .submit_tx_with_status(&fixture.request) + .await + .map_err(|e| { + err(format!( + "funded setup tx failed for {}: {e}", + account.sender + )) + })?; + if status != 200 { + return Err(err(format!( + "funded setup tx rejected for {}: status={status}, body={body}. Hint: self-contained mode (fresh db) avoids nonce conflicts for funded workload.", + account.sender + ))); + } + account.next_nonce = 1; + } + Ok(()) +} + +fn load_funded_accounts(accounts_file: Option<&str>) -> BenchResult> { + let keys = match accounts_file { + Some(path) => load_private_keys_from_file(path)?, + None => ANVIL_DEFAULT_PRIVATE_KEYS + .iter() + .map(|s| s.to_string()) + .collect(), + }; + if keys.is_empty() { + return Err(err("no private keys available for funded workload")); + } + + let mut accounts = Vec::with_capacity(keys.len()); + for key_hex in keys { + let signing_key = signing_key_from_hex(key_hex.as_str())?; + let sender = address_from_signing_key(&signing_key); + accounts.push(FundedAccount { + signing_key, + sender, + next_nonce: 0, + }); + } + Ok(accounts) +} + +fn load_private_keys_from_file(path: &str) -> BenchResult> { + let contents = fs::read_to_string(path) + .map_err(|e| err(format!("failed reading accounts file '{path}': {e}")))?; + let mut keys = Vec::new(); + for line in contents.lines() { + let mut candidate = line.trim(); + if let Some((_, rhs)) = candidate.split_once(')') { + candidate = rhs.trim(); + } + if candidate.starts_with("0x") && candidate.len() == 66 { + let is_hex = candidate + .as_bytes() + .iter() + .skip(2) + .all(|b| b.is_ascii_hexdigit()); + if is_hex { + keys.push(candidate.to_string()); + } + } + } + if keys.is_empty() { + return Err(err(format!( + "accounts file '{path}' did not contain any 32-byte hex private keys" + ))); + } + Ok(keys) +} + +fn signing_key_from_hex(hex: &str) -> BenchResult { + let bytes = alloy_primitives::hex::decode(hex) + .map_err(|e| err(format!("invalid private key hex '{hex}': {e}")))?; + if bytes.len() != 32 { + return Err(err(format!( + "invalid private key length: expected 32 bytes, got {}", + bytes.len() + ))); + } + let mut key_bytes = [0_u8; 32]; + key_bytes.copy_from_slice(&bytes); + SigningKey::from_bytes((&key_bytes).into()) + .map_err(|e| err(format!("invalid private key material: {e}"))) +} + +fn make_signed_fixture_from_signing_key( + signing_key: &SigningKey, + user_op: UserOp, + domain: &Eip712Domain, +) -> BenchResult { + let sender = address_from_signing_key(signing_key); + let signature = sign_user_op(domain, &user_op, signing_key)?; + let data = user_op.data.to_vec(); + Ok(SignedTxFixture { + request: TxRequest { + message: user_op, + signature, + sender: sender.to_string(), + }, + expected_sender: sender.to_string(), + expected_data_hex: alloy_primitives::hex::encode_prefixed(data), + }) +} + +fn fixture_match_key(sender: &str, data_hex: &str) -> String { + format!( + "{}|{}", + sender.to_ascii_lowercase(), + data_hex.to_ascii_lowercase() + ) +} + +fn event_match_key(sender: &str, data_hex: &str) -> String { + fixture_match_key(sender, data_hex) +} + +fn classify_rejection(outcome: Result<(u16, String), SubmitTxError>) -> Option { + match outcome { + Ok((200, _body)) => None, + Ok((status, body)) => Some(RejectionOutcome { + key: format!("http_{status}"), + detail: format!("status={status}, body={body}"), + }), + Err(err) => Some(RejectionOutcome { + key: err.breakdown_key().to_string(), + detail: err.to_string(), + }), + } +} + +pub fn summarize(samples: &[Duration]) -> BenchResult { + if samples.is_empty() { + return Err(err("cannot summarize empty sample set")); + } + + let mut nanos: Vec = samples.iter().map(Duration::as_nanos).collect(); + nanos.sort_unstable(); + let sum: u128 = nanos.iter().copied().sum(); + let count = nanos.len(); + + Ok(Stats { + count, + min: duration_from_nanos(nanos[0]), + max: duration_from_nanos(nanos[count - 1]), + mean: duration_from_nanos(sum / count as u128), + p50: duration_from_nanos(percentile(&nanos, 0.50)), + p95: duration_from_nanos(percentile(&nanos, 0.95)), + p99: duration_from_nanos(percentile(&nanos, 0.99)), + p999: duration_from_nanos(percentile(&nanos, 0.999)), + }) +} + +pub fn print_stats(name: &str, stats: &Stats) { + println!("{name}:"); + println!(" count: {}", stats.count); + println!(" min: {}", format_ms(stats.min)); + println!(" p50: {}", format_ms(stats.p50)); + println!(" p95: {}", format_ms(stats.p95)); + println!(" p99: {}", format_ms(stats.p99)); + println!(" p99.9: {}", format_ms(stats.p999)); + println!(" max: {}", format_ms(stats.max)); + println!(" mean: {}", format_ms(stats.mean)); +} + +pub fn throughput_tx_per_s(accepted_count: usize, total_wall: Duration) -> f64 { + if total_wall.is_zero() { + 0.0 + } else { + accepted_count as f64 / total_wall.as_secs_f64() + } +} + +pub fn rejection_rate(accepted: u64, rejected: u64) -> f64 { + let total = accepted.saturating_add(rejected); + if total == 0 { + 0.0 + } else { + (rejected as f64 / total as f64) * 100.0 + } +} + +pub fn now() -> Instant { + Instant::now() +} + +pub fn default_seed_offset() -> u64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_nanos() as u64 +} + +fn signing_key_for_seed(seed: u64) -> BenchResult { + let mut bytes = [0_u8; 32]; + bytes[24..32].copy_from_slice(&seed.saturating_add(1).to_be_bytes()); + SigningKey::from_bytes((&bytes).into()) + .map_err(|e| err(format!("build signing key failed: {e}"))) +} + +fn sign_user_op( + domain: &Eip712Domain, + user_op: &UserOp, + signing_key: &SigningKey, +) -> BenchResult { + let hash = user_op.eip712_signing_hash(domain); + let k256_sig = signing_key + .sign_prehash(hash.as_slice()) + .map_err(|e| err(format!("sign user op prehash failed: {e}")))?; + + let expected_sender = address_from_signing_key(signing_key); + let signature = [false, true] + .into_iter() + .map(|parity| Signature::from_signature_and_parity(k256_sig, parity)) + .find(|candidate| { + candidate + .recover_address_from_prehash(&hash) + .ok() + .map(|sender| sender == expected_sender) + .unwrap_or(false) + }) + .ok_or_else(|| err("could not recover parity for signature"))?; + + Ok(alloy_primitives::hex::encode_prefixed(signature.as_bytes())) +} + +fn address_from_signing_key(signing_key: &SigningKey) -> Address { + let verifying = signing_key.verifying_key().to_encoded_point(false); + Address::from_raw_public_key(&verifying.as_bytes()[1..]) +} + +fn percentile(sorted_nanos: &[u128], p: f64) -> u128 { + let last = sorted_nanos.len() - 1; + let rank = (p * last as f64).ceil() as usize; + sorted_nanos[rank.min(last)] +} + +fn duration_from_nanos(value: u128) -> Duration { + let nanos = u64::try_from(value).unwrap_or(u64::MAX); + Duration::from_nanos(nanos) +} + +fn format_ms(value: Duration) -> String { + format!("{:.3} ms", value.as_secs_f64() * 1000.0) +} + +fn format_optional_f64(value: Option) -> String { + match value { + Some(v) => format!("{v:.3}"), + None => "n/a".to_string(), + } +} + +fn append_from_offset(base_ws_subscribe_url: &str, from_offset: u64) -> String { + let separator = if base_ws_subscribe_url.contains('?') { + '&' + } else { + '?' + }; + format!("{base_ws_subscribe_url}{separator}from_offset={from_offset}") +} + +struct MatchResult { + e2e_samples: Vec, + consumed_events: u64, +} + +async fn wait_for_matching_user_ops( + ws: &mut WebSocketStream>, + expected_submit_starts: &mut HashMap>, + max_wait: Duration, +) -> BenchResult { + let deadline = tokio::time::Instant::now() + max_wait; + let expected_total: usize = expected_submit_starts.values().map(Vec::len).sum(); + let mut e2e_samples = Vec::with_capacity(expected_total); + let mut consumed_events = 0_u64; + + while expected_submit_starts + .values() + .any(|entries| !entries.is_empty()) + { + let now = tokio::time::Instant::now(); + if now >= deadline { + let pending: usize = expected_submit_starts.values().map(Vec::len).sum(); + return Err(io_err(format!( + "timed out waiting for {pending} ws event(s)" + ))); + } + let remaining = deadline - now; + let maybe_frame = tokio::time::timeout(remaining, ws.next()) + .await + .map_err(|_| io_err("ws timeout"))?; + let frame = maybe_frame + .ok_or_else(|| io_err("ws stream closed"))? + .map_err(|err| io_err(format!("ws frame read failed: {err}")))?; + + let Message::Text(text) = frame else { + continue; + }; + let event: WsTxMessage = serde_json::from_str(text.as_str())?; + consumed_events = consumed_events.saturating_add(1); + + if let WsTxMessage::UserOp { sender, data, .. } = event { + let key = event_match_key(sender.as_str(), data.as_str()); + if let Some(entries) = expected_submit_starts.get_mut(key.as_str()) + && let Some(submit_started) = entries.pop() + { + e2e_samples.push(submit_started.elapsed()); + } + } + } + + Ok(MatchResult { + e2e_samples, + consumed_events, + }) +} + +async fn drain_existing_ws_backlog( + ws: &mut WebSocketStream>, + idle_quiet_window: Duration, + max_total: Duration, +) -> BenchResult { + let mut drained = 0_u64; + let hard_deadline = tokio::time::Instant::now() + max_total; + + loop { + let now = tokio::time::Instant::now(); + if now >= hard_deadline { + break; + } + let remaining_until_deadline = hard_deadline - now; + let poll_timeout = remaining_until_deadline.min(idle_quiet_window); + + match tokio::time::timeout(poll_timeout, ws.next()).await { + Err(_) => break, + Ok(None) => return Err(io_err("ws stream closed while draining backlog")), + Ok(Some(Err(err))) => { + return Err(io_err(format!( + "ws frame read failed while draining backlog: {err}" + ))); + } + Ok(Some(Ok(_))) => { + drained = drained.saturating_add(1); + } + } + } + + Ok(drained) +} + +fn io_err(message: impl Into) -> Box { + Box::new(std::io::Error::other(message.into())) +} + +fn err(message: impl Into) -> Box { + Box::new(std::io::Error::other(message.into())) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn summarize_includes_p999() { + let samples: Vec = (1_u64..=10_000).map(Duration::from_micros).collect(); + let stats = summarize(samples.as_slice()).expect("summarize"); + assert_eq!(stats.count, 10_000); + assert!(stats.p999 >= stats.p99); + assert!(stats.p999 <= stats.max); + } + + #[test] + fn classify_rejection_maps_http_and_transport() { + let http = classify_rejection(Ok((429, "overloaded".to_string()))).expect("http rejection"); + assert_eq!(http.key, "http_429"); + + let transport = + classify_rejection(Err(SubmitTxError::TimeoutRead)).expect("transport rejection"); + assert_eq!(transport.key, "timeout_read"); + } + + #[test] + fn funded_transfer_round_robin_nonce_progression() { + let mut accounts = Vec::new(); + for key in ANVIL_DEFAULT_PRIVATE_KEYS.iter().take(2) { + let signing_key = signing_key_from_hex(key).expect("signing key"); + accounts.push(FundedAccount { + sender: address_from_signing_key(&signing_key), + signing_key, + next_nonce: 1, + }); + } + + let domain = default_domain(); + let mut state = WorkloadState { + inner: WorkloadStateInner::FundedTransfer { + accounts, + round_robin_index: 0, + transfer_amount: 1, + }, + }; + + let one = state.next_fixture(0, &domain).expect("fixture 1"); + let two = state.next_fixture(0, &domain).expect("fixture 2"); + let three = state.next_fixture(0, &domain).expect("fixture 3"); + + assert_ne!(one.expected_sender, two.expected_sender); + assert_eq!(one.expected_sender, three.expected_sender); + assert_eq!(one.request.message.nonce, 1); + assert_eq!(two.request.message.nonce, 1); + assert_eq!(three.request.message.nonce, 2); + } +} diff --git a/benchmarks/src/runtime.rs b/benchmarks/src/runtime.rs new file mode 100644 index 0000000..fe7a7df --- /dev/null +++ b/benchmarks/src/runtime.rs @@ -0,0 +1,535 @@ +// (c) Cartesi and individual authors (see AUTHORS) +// SPDX-License-Identifier: Apache-2.0 (see LICENSE) + +use serde::Serialize; +use std::fs::{self, OpenOptions}; +use std::io::{BufRead, BufReader}; +use std::path::{Path, PathBuf}; +use std::process::Stdio; +use std::time::Duration; +use tempfile::TempDir; +use tokio::io::{AsyncReadExt, AsyncWriteExt}; +use tokio::net::TcpStream; +use tokio::process::{Child, Command}; +use tokio::sync::oneshot; +use tokio::task::JoinHandle; + +use crate::BenchResult; + +pub const DEFAULT_SEQUENCER_BIN: &str = "target/release/sequencer"; +pub const DEFAULT_SEQUENCER_START_TIMEOUT_MS: u64 = 10_000; +pub const DEFAULT_SEQUENCER_SHUTDOWN_TIMEOUT_MS: u64 = 3_000; +pub const DEFAULT_MEMORY_SAMPLE_INTERVAL_MS: u64 = 500; +pub const DEFAULT_RUNTIME_METRICS_LOG_INTERVAL_MS: u64 = 5_000; +pub const DEFAULT_SEQUENCER_LOGS_DIR: &str = "benchmarks/results"; + +#[derive(Debug, Clone)] +pub struct ManagedSequencerConfig { + pub sequencer_bin: String, + pub start_timeout: Duration, + pub shutdown_timeout: Duration, + pub temp_db: bool, + pub log_path: Option, + pub runtime_metrics_enabled: bool, + pub runtime_metrics_log_interval: Duration, + pub rust_log: String, +} + +pub struct ManagedSequencer { + child: Child, + shutdown_timeout: Duration, + temp_dir: Option, + pub endpoint: String, + pub ws_subscribe_url: String, + pub db_path: PathBuf, + log_path: PathBuf, +} + +impl ManagedSequencer { + pub async fn spawn(config: ManagedSequencerConfig) -> BenchResult { + let (endpoint, http_addr) = build_local_endpoint()?; + let ws_subscribe_url = format!("{}/ws/subscribe", endpoint.replacen("http://", "ws://", 1)); + + let (temp_dir, db_path) = if config.temp_db { + let dir = tempfile::tempdir()?; + let path = dir_path_join(dir.path(), "sequencer.db"); + (Some(dir), path) + } else { + (None, PathBuf::from("sequencer.bench.db")) + }; + + let log_path = config + .log_path + .unwrap_or_else(|| default_sequencer_log_path("sequencer-self-contained")); + if let Some(parent) = log_path.parent() { + fs::create_dir_all(parent)?; + } + let stdout_log = OpenOptions::new() + .create(true) + .truncate(true) + .write(true) + .open(log_path.as_path())?; + let stderr_log = stdout_log.try_clone()?; + + let mut child = Command::new(config.sequencer_bin.as_str()) + .arg("--http-addr") + .arg(http_addr) + .arg("--db-path") + .arg(path_as_str(db_path.as_path())?) + .env( + "SEQ_RUNTIME_METRICS_ENABLED", + if config.runtime_metrics_enabled { + "true" + } else { + "false" + }, + ) + .env( + "SEQ_RUNTIME_METRICS_LOG_INTERVAL_MS", + config.runtime_metrics_log_interval.as_millis().to_string(), + ) + .env("RUST_LOG", config.rust_log.as_str()) + .stdout(Stdio::from(stdout_log)) + .stderr(Stdio::from(stderr_log)) + .spawn() + .map_err(|err| { + io_other(format!( + "failed to spawn sequencer binary '{}': {err}", + config.sequencer_bin + )) + })?; + + wait_for_readiness(endpoint.as_str(), &mut child, config.start_timeout).await?; + + Ok(Self { + child, + shutdown_timeout: config.shutdown_timeout, + temp_dir, + endpoint, + ws_subscribe_url, + db_path, + log_path, + }) + } + + pub fn pid(&self) -> Option { + self.child.id() + } + + pub fn log_path(&self) -> &Path { + self.log_path.as_path() + } + + pub async fn shutdown(mut self) -> BenchResult<()> { + let _ = self.temp_dir.take(); + send_graceful_terminate(&mut self.child).await; + match tokio::time::timeout(self.shutdown_timeout, self.child.wait()).await { + Ok(wait_result) => { + let _ = wait_result?; + Ok(()) + } + Err(_) => { + self.child.start_kill()?; + let _ = self.child.wait().await; + Ok(()) + } + } + } +} + +#[derive(Debug, Clone, Serialize, Default)] +pub struct MemoryReport { + pub method: String, + pub sample_interval_ms: u64, + pub rss_start_mb: Option, + pub rss_peak_mb: Option, + pub rss_end_mb: Option, + pub rss_growth_mb: Option, + pub rss_growth_per_1k_accepted_tx_mb: Option, +} + +pub struct MemorySampler { + stop_tx: Option>, + join: JoinHandle, +} + +#[derive(Debug, Clone, Serialize, Default)] +pub struct InclusionLaneProfileReport { + pub samples: u64, + pub latest_window_ms: Option, + pub latest_user_op_app_execute_phase_ms: Option, + pub latest_user_op_persist_phase_ms: Option, + pub latest_user_op_app_share_pct_of_app_plus_persist: Option, + pub latest_user_op_persist_share_pct_of_app_plus_persist: Option, + pub avg_user_op_app_share_pct_of_app_plus_persist: Option, + pub avg_user_op_persist_share_pct_of_app_plus_persist: Option, +} + +impl MemorySampler { + pub fn start(pid: u32, sample_interval: Duration) -> Self { + let (stop_tx, stop_rx) = oneshot::channel::(); + let join = + tokio::spawn( + async move { sample_memory_until_stop(pid, sample_interval, stop_rx).await }, + ); + Self { + stop_tx: Some(stop_tx), + join, + } + } + + pub async fn stop(mut self, accepted_count: u64) -> BenchResult { + if let Some(stop_tx) = self.stop_tx.take() { + let _ = stop_tx.send(accepted_count); + } + let report = self + .join + .await + .map_err(|err| io_other(format!("memory sampler task join failed: {err}")))?; + Ok(report) + } +} + +async fn sample_memory_until_stop( + pid: u32, + sample_interval: Duration, + mut stop_rx: oneshot::Receiver, +) -> MemoryReport { + let mut start_mb = sample_rss_mb(pid); + let mut end_mb = start_mb; + let mut peak_mb = start_mb; + let mut accepted_count = 0_u64; + + loop { + tokio::select! { + maybe_accepted = &mut stop_rx => { + if let Ok(value) = maybe_accepted { + accepted_count = value; + } + end_mb = sample_rss_mb(pid).or(end_mb); + if let Some(end) = end_mb { + peak_mb = Some(peak_mb.unwrap_or(end).max(end)); + } + break; + } + _ = tokio::time::sleep(sample_interval) => { + if let Some(current) = sample_rss_mb(pid) { + if start_mb.is_none() { + start_mb = Some(current); + } + end_mb = Some(current); + peak_mb = Some(peak_mb.unwrap_or(current).max(current)); + } + } + } + } + + let rss_growth_mb = match (start_mb, end_mb) { + (Some(start), Some(end)) => Some(end - start), + _ => None, + }; + let rss_growth_per_1k_accepted_tx_mb = match (rss_growth_mb, accepted_count) { + (Some(growth), value) if value > 0 => Some(growth / (value as f64 / 1000.0)), + _ => None, + }; + + MemoryReport { + method: "ps-rss-sampling".to_string(), + sample_interval_ms: u64::try_from(sample_interval.as_millis()).unwrap_or(u64::MAX), + rss_start_mb: start_mb, + rss_peak_mb: peak_mb, + rss_end_mb: end_mb, + rss_growth_mb, + rss_growth_per_1k_accepted_tx_mb, + } +} + +fn sample_rss_mb(pid: u32) -> Option { + let output = std::process::Command::new("ps") + .arg("-o") + .arg("rss=") + .arg("-p") + .arg(pid.to_string()) + .output() + .ok()?; + if !output.status.success() { + return None; + } + let text = String::from_utf8(output.stdout).ok()?; + let rss_kib = text.trim().parse::().ok()?; + Some(rss_kib / 1024.0) +} + +pub fn parse_inclusion_lane_profile_from_log( + log_path: &Path, +) -> BenchResult> { + if !log_path.exists() { + return Ok(None); + } + let file = std::fs::File::open(log_path)?; + let reader = BufReader::new(file); + + let mut report = InclusionLaneProfileReport::default(); + let mut app_share_sum = 0.0_f64; + let mut persist_share_sum = 0.0_f64; + let mut app_share_samples = 0_u64; + let mut persist_share_samples = 0_u64; + + for line_result in reader.lines() { + let line = line_result?; + let line = strip_ansi_escapes(line.as_str()); + if !line.contains("inclusion lane metrics") { + continue; + } + + report.samples = report.samples.saturating_add(1); + report.latest_window_ms = parse_u64_field(line.as_str(), "window_ms"); + report.latest_user_op_app_execute_phase_ms = + parse_u64_field(line.as_str(), "user_op_app_execute_phase_ms"); + report.latest_user_op_persist_phase_ms = + parse_u64_field(line.as_str(), "user_op_persist_phase_ms"); + report.latest_user_op_app_share_pct_of_app_plus_persist = + parse_f64_field(line.as_str(), "user_op_app_share_pct_of_app_plus_persist"); + report.latest_user_op_persist_share_pct_of_app_plus_persist = parse_f64_field( + line.as_str(), + "user_op_persist_share_pct_of_app_plus_persist", + ); + + if let Some(value) = report.latest_user_op_app_share_pct_of_app_plus_persist { + app_share_sum += value; + app_share_samples = app_share_samples.saturating_add(1); + } + if let Some(value) = report.latest_user_op_persist_share_pct_of_app_plus_persist { + persist_share_sum += value; + persist_share_samples = persist_share_samples.saturating_add(1); + } + } + + if report.samples == 0 { + return Ok(None); + } + if app_share_samples > 0 { + report.avg_user_op_app_share_pct_of_app_plus_persist = + Some(app_share_sum / app_share_samples as f64); + } + if persist_share_samples > 0 { + report.avg_user_op_persist_share_pct_of_app_plus_persist = + Some(persist_share_sum / persist_share_samples as f64); + } + + Ok(Some(report)) +} + +pub fn default_sequencer_log_path(prefix: &str) -> PathBuf { + let ts = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|value| value.as_millis()) + .unwrap_or(0); + PathBuf::from(format!("{DEFAULT_SEQUENCER_LOGS_DIR}/{prefix}-{ts}.log")) +} + +fn parse_u64_field(line: &str, key: &str) -> Option { + let needle = format!("{key}="); + line.split_whitespace() + .find_map(|token| token.strip_prefix(needle.as_str())) + .and_then(clean_token_value) + .and_then(|value| value.parse::().ok()) +} + +fn parse_f64_field(line: &str, key: &str) -> Option { + let needle = format!("{key}="); + line.split_whitespace() + .find_map(|token| token.strip_prefix(needle.as_str())) + .and_then(clean_token_value) + .and_then(|value| value.parse::().ok()) +} + +fn clean_token_value(raw: &str) -> Option { + let value = raw + .trim() + .trim_matches(',') + .trim_matches('"') + .trim_matches('\''); + (!value.is_empty()).then(|| value.to_string()) +} + +fn strip_ansi_escapes(input: &str) -> String { + let mut out = String::with_capacity(input.len()); + let bytes = input.as_bytes(); + let mut i = 0_usize; + + while i < bytes.len() { + if bytes[i] == 0x1b { + i += 1; + if i < bytes.len() && bytes[i] == b'[' { + i += 1; + while i < bytes.len() { + let b = bytes[i]; + i += 1; + if b.is_ascii_alphabetic() { + break; + } + } + continue; + } + continue; + } + + out.push(bytes[i] as char); + i += 1; + } + + out +} + +async fn wait_for_readiness( + endpoint: &str, + child: &mut Child, + timeout: Duration, +) -> BenchResult<()> { + let deadline = tokio::time::Instant::now() + timeout; + loop { + if let Some(status) = child.try_wait()? { + return Err(io_other(format!( + "sequencer exited before readiness: status={status}" + )) + .into()); + } + if http_endpoint_is_ready(endpoint).await { + return Ok(()); + } + if tokio::time::Instant::now() >= deadline { + return Err(io_other(format!( + "timed out waiting for sequencer readiness at {endpoint}" + )) + .into()); + } + tokio::time::sleep(Duration::from_millis(100)).await; + } +} + +async fn http_endpoint_is_ready(endpoint: &str) -> bool { + let Some(host_port) = endpoint.strip_prefix("http://") else { + return false; + }; + let mut stream = + match tokio::time::timeout(Duration::from_millis(300), TcpStream::connect(host_port)).await + { + Ok(Ok(value)) => value, + _ => return false, + }; + + let request = format!("GET /tx HTTP/1.1\r\nHost: {host_port}\r\nConnection: close\r\n\r\n"); + if stream.write_all(request.as_bytes()).await.is_err() { + return false; + } + let mut head = [0_u8; 64]; + match tokio::time::timeout(Duration::from_millis(300), stream.read(&mut head)).await { + Ok(Ok(read)) if read > 0 => std::str::from_utf8(&head[..read]) + .ok() + .map(|text| text.starts_with("HTTP/1.1") || text.starts_with("HTTP/1.0")) + .unwrap_or(false), + _ => false, + } +} + +async fn send_graceful_terminate(child: &mut Child) { + let Some(pid) = child.id() else { + return; + }; + + #[cfg(unix)] + { + let _ = std::process::Command::new("kill") + .arg("-TERM") + .arg(pid.to_string()) + .status(); + } + + #[cfg(not(unix))] + { + let _ = child.start_kill(); + } +} + +fn build_local_endpoint() -> BenchResult<(String, String)> { + let listener = std::net::TcpListener::bind("127.0.0.1:0")?; + let addr = listener.local_addr()?; + drop(listener); + let http_addr = format!("127.0.0.1:{}", addr.port()); + let endpoint = format!("http://{http_addr}"); + Ok((endpoint, http_addr)) +} + +fn dir_path_join(base: &Path, file: &str) -> PathBuf { + let mut path = base.to_path_buf(); + path.push(file); + path +} + +fn path_as_str(path: &Path) -> BenchResult<&str> { + path.to_str() + .ok_or_else(|| io_other(format!("path is not valid UTF-8: {}", path.display())).into()) +} + +fn io_other(message: impl Into) -> std::io::Error { + std::io::Error::other(message.into()) +} + +#[cfg(test)] +mod tests { + use super::{default_sequencer_log_path, parse_inclusion_lane_profile_from_log}; + use std::fs; + + #[test] + fn parses_inclusion_lane_profile_summary_from_logs() { + let temp = tempfile::tempdir().expect("tempdir"); + let log_path = temp.path().join("sequencer.log"); + let content = r#" +2026-03-01T00:00:00Z INFO x: inclusion lane metrics window_ms=5000 user_op_app_execute_phase_ms=120 user_op_persist_phase_ms=80 user_op_app_share_pct_of_app_plus_persist=60.0 user_op_persist_share_pct_of_app_plus_persist=40.0 +2026-03-01T00:00:05Z INFO x: inclusion lane metrics window_ms=5000 user_op_app_execute_phase_ms=140 user_op_persist_phase_ms=60 user_op_app_share_pct_of_app_plus_persist=70.0 user_op_persist_share_pct_of_app_plus_persist=30.0 +"#; + fs::write(log_path.as_path(), content).expect("write log"); + + let report = parse_inclusion_lane_profile_from_log(log_path.as_path()) + .expect("parse result") + .expect("profile present"); + + assert_eq!(report.samples, 2); + assert_eq!(report.latest_user_op_app_execute_phase_ms, Some(140)); + assert_eq!(report.latest_user_op_persist_phase_ms, Some(60)); + assert_eq!( + report.avg_user_op_app_share_pct_of_app_plus_persist, + Some(65.0) + ); + assert_eq!( + report.avg_user_op_persist_share_pct_of_app_plus_persist, + Some(35.0) + ); + } + + #[test] + fn default_log_path_uses_results_dir() { + let value = default_sequencer_log_path("ack-latency"); + assert!(value.to_string_lossy().contains("benchmarks/results/")); + assert!(value.to_string_lossy().contains("ack-latency")); + } + + #[test] + fn parses_ansi_colored_profile_lines() { + let temp = tempfile::tempdir().expect("tempdir"); + let log_path = temp.path().join("sequencer.log"); + let content = "\u{1b}[2m2026-03-01T00:00:00Z\u{1b}[0m \u{1b}[32mINFO\u{1b}[0m x: inclusion lane metrics \u{1b}[3mwindow_ms\u{1b}[0m\u{1b}[2m=\u{1b}[0m5000 \u{1b}[3muser_op_app_share_pct_of_app_plus_persist\u{1b}[0m\u{1b}[2m=\u{1b}[0m60.5 \u{1b}[3muser_op_persist_share_pct_of_app_plus_persist\u{1b}[0m\u{1b}[2m=\u{1b}[0m39.5\n"; + fs::write(log_path.as_path(), content).expect("write log"); + + let report = parse_inclusion_lane_profile_from_log(log_path.as_path()) + .expect("parse result") + .expect("profile present"); + assert_eq!(report.samples, 1); + assert_eq!(report.latest_window_ms, Some(5000)); + assert_eq!( + report.latest_user_op_app_share_pct_of_app_plus_persist, + Some(60.5) + ); + } +} diff --git a/examples/app-core/Cargo.toml b/examples/app-core/Cargo.toml new file mode 100644 index 0000000..e59e74f --- /dev/null +++ b/examples/app-core/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "app-core" +version.workspace = true +edition.workspace = true +license.workspace = true +description.workspace = true +homepage.workspace = true +repository.workspace = true +readme = "../../README.md" +authors.workspace = true + +[dependencies] +sequencer-core = { path = "../../sequencer-core" } +alloy-primitives = { version = "1.4.1", features = ["serde", "k256"] } +ssz = { package = "ethereum_ssz", version = "0.10" } diff --git a/sequencer/src/user_op.rs b/examples/app-core/src/application/mod.rs similarity index 65% rename from sequencer/src/user_op.rs rename to examples/app-core/src/application/mod.rs index 7ec00c7..2780534 100644 --- a/sequencer/src/user_op.rs +++ b/examples/app-core/src/application/mod.rs @@ -1,4 +1,6 @@ // (c) Cartesi and individual authors (see AUTHORS) // SPDX-License-Identifier: Apache-2.0 (see LICENSE) -pub use app_core::user_op::*; +mod wallet; + +pub use wallet::{WalletApp, WalletConfig}; diff --git a/app-core/src/application/wallet.rs b/examples/app-core/src/application/wallet.rs similarity index 93% rename from app-core/src/application/wallet.rs rename to examples/app-core/src/application/wallet.rs index 47f2b66..bf30dcb 100644 --- a/app-core/src/application/wallet.rs +++ b/examples/app-core/src/application/wallet.rs @@ -6,9 +6,9 @@ use std::collections::HashMap; use alloy_primitives::{Address, U256}; use ssz::Decode; -use crate::application::{AppError, Application, InvalidReason, Method}; -use crate::l2_tx::ValidUserOp; -use crate::user_op::UserOp; +use sequencer_core::application::{AppError, Application, InvalidReason, Method}; +use sequencer_core::l2_tx::ValidUserOp; +use sequencer_core::user_op::UserOp; #[derive(Debug, Clone, Copy)] pub struct WalletConfig; @@ -93,7 +93,7 @@ impl Application for WalletApp { } let max_fee = user_op.max_fee; - // Users sign a cap; sequencer executes against the batch fee. + // Users sign a cap; sequencer executes against the committed frame fee. if u64::from(max_fee) < current_fee { return Err(InvalidReason::InvalidMaxFee { max_fee, @@ -159,10 +159,10 @@ impl Application for WalletApp { #[cfg(test)] mod tests { use super::{WalletApp, WalletConfig}; - use crate::application::{Application, InvalidReason}; - use crate::l2_tx::ValidUserOp; - use crate::user_op::UserOp; use alloy_primitives::{Address, U256}; + use sequencer_core::application::{Application, InvalidReason}; + use sequencer_core::l2_tx::ValidUserOp; + use sequencer_core::user_op::UserOp; #[test] fn validate_rejects_when_max_fee_below_current_fee() { diff --git a/sequencer/src/l2_tx.rs b/examples/app-core/src/lib.rs similarity index 79% rename from sequencer/src/l2_tx.rs rename to examples/app-core/src/lib.rs index 6497917..90eb57f 100644 --- a/sequencer/src/l2_tx.rs +++ b/examples/app-core/src/lib.rs @@ -1,4 +1,4 @@ // (c) Cartesi and individual authors (see AUTHORS) // SPDX-License-Identifier: Apache-2.0 (see LICENSE) -pub use app_core::l2_tx::*; +pub mod application; diff --git a/canonical-app/Cargo.toml b/examples/canonical-app/Cargo.toml similarity index 90% rename from canonical-app/Cargo.toml rename to examples/canonical-app/Cargo.toml index 11f0890..1d694f0 100644 --- a/canonical-app/Cargo.toml +++ b/examples/canonical-app/Cargo.toml @@ -6,7 +6,7 @@ license.workspace = true description.workspace = true homepage.workspace = true repository.workspace = true -readme.workspace = true +readme = "../../README.md" authors.workspace = true [dependencies] diff --git a/canonical-app/src/main.rs b/examples/canonical-app/src/main.rs similarity index 100% rename from canonical-app/src/main.rs rename to examples/canonical-app/src/main.rs diff --git a/justfile b/justfile index 1a6c158..4351b10 100644 --- a/justfile +++ b/justfile @@ -13,7 +13,12 @@ test: cargo test --workspace test-sequencer: - cargo test -p sequencer --lib --tests + cargo test -p sequencer --lib + cargo test -p sequencer --test e2e_sequencer -- --test-threads=1 + cargo test -p sequencer --test ws_broadcaster -- --test-threads=1 + +bench target="all": + just --justfile benchmarks/justfile {{target}} fmt: cargo fmt --all @@ -33,4 +38,5 @@ ci: cargo test --workspace --all-targets --all-features --locked run addr="127.0.0.1:3000" db="sequencer.db": - SEQ_HTTP_ADDR={{addr}} SEQ_DB_PATH={{db}} cargo run -p sequencer + rm -f {{db}} {{db}}-shm {{db}}-wal + SEQ_HTTP_ADDR={{addr}} SEQ_DB_PATH={{db}} cargo run -p sequencer --release diff --git a/tools/cli/.gitignore b/sdk/cli/.gitignore similarity index 100% rename from tools/cli/.gitignore rename to sdk/cli/.gitignore diff --git a/sdk/rust-client/.gitignore b/sdk/rust-client/.gitignore index d6b7ef3..ea8c4bf 100644 --- a/sdk/rust-client/.gitignore +++ b/sdk/rust-client/.gitignore @@ -1,2 +1 @@ -* -!.gitignore +/target diff --git a/sdk/rust-client/Cargo.toml b/sdk/rust-client/Cargo.toml new file mode 100644 index 0000000..e11cb6e --- /dev/null +++ b/sdk/rust-client/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "sequencer-rust-client" +version.workspace = true +edition.workspace = true +license.workspace = true +description.workspace = true +homepage.workspace = true +repository.workspace = true +readme = "../../README.md" +authors.workspace = true + +[dependencies] +sequencer-core = { path = "../../sequencer-core" } +serde_json = "1" +thiserror = "1" +tokio = { version = "1.35", features = ["time", "net", "io-util"] } +tokio-tungstenite = "0.28" diff --git a/sdk/rust-client/src/errors.rs b/sdk/rust-client/src/errors.rs new file mode 100644 index 0000000..6ad6812 --- /dev/null +++ b/sdk/rust-client/src/errors.rs @@ -0,0 +1,66 @@ +// (c) Cartesi and individual authors (see AUTHORS) +// SPDX-License-Identifier: Apache-2.0 (see LICENSE) + +use thiserror::Error; + +#[derive(Debug, Error)] +pub enum ClientBuildError { + #[error("invalid endpoint: {0}")] + InvalidEndpoint(String), +} + +#[derive(Debug, Error)] +pub enum SubmitTxError { + #[error("tcp connect timeout")] + TimeoutConnect, + #[error("tcp write timeout")] + TimeoutWrite, + #[error("tcp flush timeout")] + TimeoutFlush, + #[error("tcp read timeout")] + TimeoutRead, + #[error("tcp connect failed: {0}")] + IoConnect(String), + #[error("tcp write failed: {0}")] + IoWrite(String), + #[error("tcp flush failed: {0}")] + IoFlush(String), + #[error("tcp read failed: {0}")] + IoRead(String), + #[error("parse failed: {0}")] + Parse(String), +} + +impl SubmitTxError { + pub fn breakdown_key(&self) -> &'static str { + match self { + Self::TimeoutConnect => "timeout_connect", + Self::TimeoutWrite => "timeout_write", + Self::TimeoutFlush => "timeout_flush", + Self::TimeoutRead => "timeout_read", + Self::IoConnect(_) => "io_connect", + Self::IoWrite(_) => "io_write", + Self::IoFlush(_) => "io_flush", + Self::IoRead(_) => "io_read", + Self::Parse(_) => "parse_error", + } + } +} + +#[derive(Debug, Error)] +pub enum SubmitRejected { + #[error("tx submit failed: {0}")] + Transport(#[from] SubmitTxError), + #[error("/tx rejected with status {status}: {body}")] + Http { status: u16, body: String }, + #[error("invalid /tx success body: {0}")] + Decode(String), +} + +#[derive(Debug, Error)] +pub enum SubscribeError { + #[error("invalid endpoint: {0}")] + InvalidEndpoint(String), + #[error("ws connect failed: {0}")] + Connect(String), +} diff --git a/sdk/rust-client/src/lib.rs b/sdk/rust-client/src/lib.rs new file mode 100644 index 0000000..9eb8855 --- /dev/null +++ b/sdk/rust-client/src/lib.rs @@ -0,0 +1,217 @@ +// (c) Cartesi and individual authors (see AUTHORS) +// SPDX-License-Identifier: Apache-2.0 (see LICENSE) + +mod errors; + +pub use errors::{ClientBuildError, SubmitRejected, SubmitTxError, SubscribeError}; + +use sequencer_core::api::{TxRequest, TxResponse}; +use std::time::Duration; +use tokio::io::{AsyncReadExt, AsyncWriteExt}; +use tokio::net::TcpStream; +use tokio_tungstenite::{MaybeTlsStream, WebSocketStream, connect_async}; + +pub type SubscribeStream = WebSocketStream>; + +#[derive(Debug, Clone)] +pub struct SequencerClient { + endpoint: String, + host_port: String, + path_prefix: String, + request_timeout: Duration, +} + +impl SequencerClient { + pub fn new(endpoint: impl Into) -> Result { + Self::new_with_timeout(endpoint, Duration::from_secs(3)) + } + + pub fn new_with_timeout( + endpoint: impl Into, + request_timeout: Duration, + ) -> Result { + let endpoint = endpoint.into(); + let (host_port, path_prefix) = + parse_http_url(endpoint.as_str()).map_err(ClientBuildError::InvalidEndpoint)?; + Ok(Self { + endpoint, + host_port, + path_prefix, + request_timeout, + }) + } + + pub fn endpoint(&self) -> &str { + self.endpoint.as_str() + } + + pub fn host_port(&self) -> &str { + self.host_port.as_str() + } + + pub fn request_timeout(&self) -> Duration { + self.request_timeout + } + + pub fn with_request_timeout(mut self, request_timeout: Duration) -> Self { + self.request_timeout = request_timeout; + self + } + + pub fn ws_subscribe_url(&self, from_offset: u64) -> String { + with_from_offset( + default_ws_subscribe_url_for_http(self.endpoint.as_str()).as_str(), + from_offset, + ) + } + + pub async fn submit_tx_with_status( + &self, + req: &TxRequest, + ) -> Result<(u16, String), SubmitTxError> { + submit_tx_with_status_parsed( + self.host_port.as_str(), + self.path_prefix.as_str(), + req, + self.request_timeout, + ) + .await + } + + pub async fn submit_tx(&self, req: &TxRequest) -> Result { + let (status, body) = self.submit_tx_with_status(req).await?; + if status != 200 { + return Err(SubmitRejected::Http { status, body }); + } + serde_json::from_str::(&body).map_err(|e| SubmitRejected::Decode(e.to_string())) + } + + pub async fn subscribe(&self, from_offset: u64) -> Result { + let url = self.ws_subscribe_url(from_offset); + let (stream, _response) = connect_async(url.as_str()) + .await + .map_err(|e| SubscribeError::Connect(e.to_string()))?; + Ok(stream) + } +} + +fn default_ws_subscribe_url_for_http(http_url: &str) -> String { + let scheme_replaced = if let Some(rest) = http_url.strip_prefix("https://") { + format!("wss://{rest}") + } else if let Some(rest) = http_url.strip_prefix("http://") { + format!("ws://{rest}") + } else { + format!("ws://{}", http_url.trim_end_matches('/')) + }; + format!("{}/ws/subscribe", scheme_replaced.trim_end_matches('/')) +} + +fn with_from_offset(ws_subscribe_url: &str, from_offset: u64) -> String { + let separator = if ws_subscribe_url.contains('?') { + '&' + } else { + '?' + }; + format!("{ws_subscribe_url}{separator}from_offset={from_offset}") +} + +async fn submit_tx_with_status_parsed( + host_port: &str, + path_prefix: &str, + req: &TxRequest, + timeout: Duration, +) -> Result<(u16, String), SubmitTxError> { + let path = format!("{}/tx", path_prefix.trim_end_matches('/')); + let body = serde_json::to_string(req).map_err(|e| SubmitTxError::Parse(e.to_string()))?; + let request = format!( + "POST {path} HTTP/1.1\r\nHost: {host_port}\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", + body.len() + ); + + let mut stream = tokio::time::timeout(timeout, TcpStream::connect(host_port)) + .await + .map_err(|_| SubmitTxError::TimeoutConnect)? + .map_err(|e| SubmitTxError::IoConnect(e.to_string()))?; + + tokio::time::timeout(timeout, stream.write_all(request.as_bytes())) + .await + .map_err(|_| SubmitTxError::TimeoutWrite)? + .map_err(|e| SubmitTxError::IoWrite(e.to_string()))?; + + tokio::time::timeout(timeout, stream.flush()) + .await + .map_err(|_| SubmitTxError::TimeoutFlush)? + .map_err(|e| SubmitTxError::IoFlush(e.to_string()))?; + + let mut response = Vec::new(); + let mut chunk = [0_u8; 1024]; + loop { + let read = tokio::time::timeout(timeout, stream.read(&mut chunk)) + .await + .map_err(|_| SubmitTxError::TimeoutRead)? + .map_err(|e| SubmitTxError::IoRead(e.to_string()))?; + if read == 0 { + break; + } + response.extend_from_slice(&chunk[..read]); + + if let Some((header_end, content_length)) = response_content_len(response.as_slice()) + && response.len() >= header_end.saturating_add(content_length) + { + break; + } + } + + parse_http_response(response.as_slice()).map_err(SubmitTxError::Parse) +} + +fn parse_http_url(http_url: &str) -> Result<(String, String), String> { + let stripped = http_url + .trim_end_matches('/') + .strip_prefix("http://") + .ok_or_else(|| "only http:// URLs are supported".to_string())?; + + let (host_port, path_prefix) = if let Some((host, path)) = stripped.split_once('/') { + (host.to_string(), format!("/{}", path)) + } else { + (stripped.to_string(), String::new()) + }; + if host_port.is_empty() { + return Err("missing host in http URL".to_string()); + } + Ok((host_port, path_prefix)) +} + +fn parse_http_response(raw: &[u8]) -> Result<(u16, String), String> { + let text = String::from_utf8(raw.to_vec()).map_err(|e| e.to_string())?; + let mut sections = text.splitn(2, "\r\n\r\n"); + let headers = sections.next().unwrap_or_default(); + let body = sections.next().unwrap_or_default().to_string(); + + let status_line = headers + .lines() + .next() + .ok_or_else(|| "missing HTTP status line".to_string())?; + let status = status_line + .split_whitespace() + .nth(1) + .ok_or_else(|| "missing HTTP status code".to_string())? + .parse::() + .map_err(|e| e.to_string())?; + Ok((status, body)) +} + +fn response_content_len(raw: &[u8]) -> Option<(usize, usize)> { + let header_end = raw.windows(4).position(|window| window == b"\r\n\r\n")? + 4; + let headers = std::str::from_utf8(&raw[..header_end]).ok()?; + let mut content_length = None; + for line in headers.lines() { + if let Some((name, value)) = line.split_once(':') + && name.eq_ignore_ascii_case("content-length") + { + content_length = value.trim().parse::().ok(); + break; + } + } + content_length.map(|len| (header_end, len)) +} diff --git a/app-core/Cargo.toml b/sequencer-core/Cargo.toml similarity index 95% rename from app-core/Cargo.toml rename to sequencer-core/Cargo.toml index f0f3b35..a1ae326 100644 --- a/app-core/Cargo.toml +++ b/sequencer-core/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "app-core" +name = "sequencer-core" version.workspace = true edition.workspace = true license.workspace = true diff --git a/sequencer-core/src/api.rs b/sequencer-core/src/api.rs new file mode 100644 index 0000000..955f382 --- /dev/null +++ b/sequencer-core/src/api.rs @@ -0,0 +1,32 @@ +// (c) Cartesi and individual authors (see AUTHORS) +// SPDX-License-Identifier: Apache-2.0 (see LICENSE) + +use crate::broadcast::BroadcastTxMessage; +use crate::user_op::{SignedUserOp, UserOp}; +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct TxRequest { + pub message: UserOp, + pub signature: String, + pub sender: String, +} + +impl TxRequest { + pub const HEX_PREFIX_LEN: usize = 2; + pub const ADDRESS_BYTES: usize = 20; + pub const SIGNATURE_HEX_LEN: usize = Self::HEX_PREFIX_LEN + (SignedUserOp::SIGNATURE_BYTES * 2); + pub const ADDRESS_HEX_LEN: usize = Self::HEX_PREFIX_LEN + (Self::ADDRESS_BYTES * 2); + // Conservative wire-level cap for TxRequest JSON. It intentionally leaves headroom for field + // names, quotes, separators, and decimal nonce/max_fee rendering. + pub const MAX_JSON_BYTES_RECOMMENDED: usize = 4 * 1024; +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct TxResponse { + pub ok: bool, + pub sender: String, + pub nonce: u32, +} + +pub type WsTxMessage = BroadcastTxMessage; diff --git a/app-core/src/application/method.rs b/sequencer-core/src/application/method.rs similarity index 100% rename from app-core/src/application/method.rs rename to sequencer-core/src/application/method.rs diff --git a/app-core/src/application/mod.rs b/sequencer-core/src/application/mod.rs similarity index 98% rename from app-core/src/application/mod.rs rename to sequencer-core/src/application/mod.rs index 5e2947c..77a7514 100644 --- a/app-core/src/application/mod.rs +++ b/sequencer-core/src/application/mod.rs @@ -2,7 +2,6 @@ // SPDX-License-Identifier: Apache-2.0 (see LICENSE) mod method; -mod wallet; use crate::l2_tx::ValidUserOp; use crate::user_op::UserOp; @@ -11,7 +10,6 @@ use std::fmt; use thiserror::Error; pub use method::{Deposit, Method, Transfer, Withdrawal}; -pub use wallet::{WalletApp, WalletConfig}; #[derive(Debug, Error)] pub enum AppError { diff --git a/sequencer-core/src/broadcast.rs b/sequencer-core/src/broadcast.rs new file mode 100644 index 0000000..84f92f4 --- /dev/null +++ b/sequencer-core/src/broadcast.rs @@ -0,0 +1,44 @@ +// (c) Cartesi and individual authors (see AUTHORS) +// SPDX-License-Identifier: Apache-2.0 (see LICENSE) + +use crate::l2_tx::SequencedL2Tx; +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case")] +pub enum BroadcastTxMessage { + UserOp { + offset: u64, + sender: String, + fee: u64, + data: String, + }, + DirectInput { + offset: u64, + payload: String, + }, +} + +impl BroadcastTxMessage { + pub fn offset(&self) -> u64 { + match self { + Self::UserOp { offset, .. } => *offset, + Self::DirectInput { offset, .. } => *offset, + } + } + + pub fn from_offset_and_tx(offset: u64, tx: SequencedL2Tx) -> Self { + match tx { + SequencedL2Tx::UserOp(user_op) => Self::UserOp { + offset, + sender: user_op.sender.to_string(), + fee: user_op.fee, + data: alloy_primitives::hex::encode_prefixed(user_op.data.as_slice()), + }, + SequencedL2Tx::Direct(direct) => Self::DirectInput { + offset, + payload: alloy_primitives::hex::encode_prefixed(direct.payload.as_slice()), + }, + } + } +} diff --git a/app-core/src/l2_tx.rs b/sequencer-core/src/l2_tx.rs similarity index 83% rename from app-core/src/l2_tx.rs rename to sequencer-core/src/l2_tx.rs index af1e49d..7a2a63c 100644 --- a/app-core/src/l2_tx.rs +++ b/sequencer-core/src/l2_tx.rs @@ -11,7 +11,7 @@ pub struct DirectInput { #[derive(Debug, Clone)] pub struct ValidUserOp { pub sender: Address, - // Fee committed by the sequencer for the batch that contains this user-op. + // Fee committed by the sequencer for the batch/frame that contains this user-op. pub fee: u64, pub data: Vec, } diff --git a/app-core/src/lib.rs b/sequencer-core/src/lib.rs similarity index 83% rename from app-core/src/lib.rs rename to sequencer-core/src/lib.rs index c300caa..1f2fa25 100644 --- a/app-core/src/lib.rs +++ b/sequencer-core/src/lib.rs @@ -1,6 +1,8 @@ // (c) Cartesi and individual authors (see AUTHORS) // SPDX-License-Identifier: Apache-2.0 (see LICENSE) +pub mod api; pub mod application; +pub mod broadcast; pub mod l2_tx; pub mod user_op; diff --git a/app-core/src/user_op.rs b/sequencer-core/src/user_op.rs similarity index 86% rename from app-core/src/user_op.rs rename to sequencer-core/src/user_op.rs index f3af5ed..a22ebd9 100644 --- a/app-core/src/user_op.rs +++ b/sequencer-core/src/user_op.rs @@ -25,7 +25,8 @@ impl SignedUserOp { pub const SIGNATURE_BYTES: usize = 65; pub const NONCE_BYTES: usize = 4; pub const MAX_FEE_BYTES: usize = 4; - pub const MAX_METHOD_PAYLOAD_BYTES: usize = 32 + 20; + // Method is SSZ enum-union encoded; Transfer includes a 1-byte union tag + 32-byte amount + 20-byte recipient. + pub const MAX_METHOD_PAYLOAD_BYTES: usize = 1 + 32 + 20; pub const MAX_BATCH_BYTES_UPPER_BOUND: usize = Self::SIGNATURE_BYTES + Self::NONCE_BYTES + Self::MAX_FEE_BYTES diff --git a/sequencer/Cargo.toml b/sequencer/Cargo.toml index 1e1c962..e345f96 100644 --- a/sequencer/Cargo.toml +++ b/sequencer/Cargo.toml @@ -10,7 +10,8 @@ readme.workspace = true authors.workspace = true [dependencies] -app-core = { path = "../app-core" } +app-core = { path = "../examples/app-core" } +sequencer-core = { path = "../sequencer-core" } axum = { version = "0.8.8", features = ["ws"] } tokio = { version = "1.35", features = ["macros", "rt-multi-thread", "sync", "time", "net"] } serde = { version = "1", features = ["derive"] } @@ -18,6 +19,7 @@ serde_json = "1" tracing = "0.1" tracing-subscriber = { version = "0.3", features = ["env-filter"] } tower-http = { version = "0.6.8", features = ["trace"] } +tower = { version = "0.5", features = ["limit", "load-shed", "util"] } rusqlite = { version = "0.38.0", features = ["bundled"] } rusqlite_migration = "2.3.0" alloy-primitives = { version = "1.4.1", features = ["serde", "k256"] } @@ -25,8 +27,11 @@ alloy-sol-types = "1.4.1" thiserror = "1" ssz = { package = "ethereum_ssz", version = "0.10" } ssz_derive = { package = "ethereum_ssz_derive", version = "0.10" } +clap = { version = "4", features = ["derive", "env"] } [dev-dependencies] futures-util = "0.3" tokio-tungstenite = "0.28" k256 = "0.13.4" +tempfile = "3" +sequencer-rust-client = { path = "../sdk/rust-client" } diff --git a/sequencer/src/api/error.rs b/sequencer/src/api/error.rs index b4f036a..60696a7 100644 --- a/sequencer/src/api/error.rs +++ b/sequencer/src/api/error.rs @@ -14,6 +14,8 @@ pub enum ApiError { #[error("{0}")] BadRequest(String), #[error("{0}")] + PayloadTooLarge(String), + #[error("{0}")] InvalidSignature(String), #[error("{0}")] ExecutionRejected(String), @@ -35,6 +37,10 @@ impl ApiError { Self::BadRequest(message.into()) } + pub fn payload_too_large(message: impl Into) -> Self { + Self::PayloadTooLarge(message.into()) + } + pub fn invalid_signature(message: impl Into) -> Self { Self::InvalidSignature(message.into()) } @@ -50,6 +56,7 @@ impl ApiError { pub fn status(&self) -> StatusCode { match self { Self::BadRequest(_) | Self::InvalidSignature(_) => StatusCode::BAD_REQUEST, + Self::PayloadTooLarge(_) => StatusCode::PAYLOAD_TOO_LARGE, Self::ExecutionRejected(_) => StatusCode::UNPROCESSABLE_ENTITY, Self::InternalError(_) => StatusCode::INTERNAL_SERVER_ERROR, Self::Overloaded(_) => StatusCode::TOO_MANY_REQUESTS, @@ -59,6 +66,7 @@ impl ApiError { pub fn code(&self) -> &'static str { match self { Self::BadRequest(_) => "BAD_REQUEST", + Self::PayloadTooLarge(_) => "PAYLOAD_TOO_LARGE", Self::InvalidSignature(_) => "INVALID_SIGNATURE", Self::ExecutionRejected(_) => "EXECUTION_REJECTED", Self::InternalError(_) => "INTERNAL_ERROR", diff --git a/sequencer/src/api/mod.rs b/sequencer/src/api/mod.rs index c4a2a8a..d7351ff 100644 --- a/sequencer/src/api/mod.rs +++ b/sequencer/src/api/mod.rs @@ -4,27 +4,34 @@ mod error; use std::sync::Arc; -use std::time::{Duration, SystemTime}; +use std::time::SystemTime; use axum::Router; -use axum::extract::ws::{Message, WebSocket, WebSocketUpgrade}; +use axum::error_handling::HandleErrorLayer; +use axum::extract::ws::{CloseFrame, Message, WebSocket, WebSocketUpgrade, close_code}; use axum::extract::{DefaultBodyLimit, Json, Query, State}; -use axum::response::IntoResponse; +use axum::http::StatusCode; +use axum::response::{IntoResponse, Response}; use axum::routing::{get, post}; -use serde::{Deserialize, Serialize}; +use serde::Deserialize; use tokio::sync::mpsc; -use tokio::sync::mpsc::error::SendTimeoutError; +use tokio::sync::mpsc::error::TrySendError; use tokio::sync::oneshot; +use tokio::sync::{OwnedSemaphorePermit, Semaphore}; +use tower::limit::ConcurrencyLimitLayer; +use tower::load_shed::LoadShedLayer; +use tower::{BoxError, ServiceBuilder}; use tower_http::trace::TraceLayer; -use tracing::{info, warn}; +use tracing::{debug, warn}; use alloy_primitives::{Address, Signature}; use alloy_sol_types::{Eip712Domain, SolStruct}; +use sequencer_core::api::{TxRequest, TxResponse}; +use sequencer_core::user_op::SignedUserOp; use crate::inclusion_lane::{InclusionLaneInput, PendingUserOp}; use crate::l2_tx_broadcaster::{BroadcastTxMessage, L2TxBroadcaster}; use crate::storage::Storage; -use crate::user_op::{SignedUserOp, UserOp}; pub use error::ApiError; @@ -32,35 +39,31 @@ pub use error::ApiError; pub struct AppState { pub tx_sender: mpsc::Sender, pub domain: Eip712Domain, - pub queue_timeout: Duration, + pub overload_max_inflight_submissions: usize, + pub ws_subscriber_limit: Arc, + pub ws_max_catchup_events: u64, pub broadcaster: L2TxBroadcaster, } -#[derive(Debug, Deserialize)] -struct TxRequest { - message: UserOp, - signature: String, - sender: Option, -} - -#[derive(Debug, Serialize)] -struct TxResponse { - ok: bool, - tx_hash: String, - sender: String, - nonce: u32, -} - #[derive(Debug, Deserialize)] struct SubscribeQuery { from_offset: Option, } pub fn router(state: Arc, max_body_bytes: usize) -> Router { + let tx_concurrency_limit = state.overload_max_inflight_submissions; + let tx_route = post(submit_tx).layer( + ServiceBuilder::new() + .layer(HandleErrorLayer::new(handle_tx_route_error)) + .layer(LoadShedLayer::new()) + .layer(ConcurrencyLimitLayer::new(tx_concurrency_limit)), + ); + Router::new() - .route("/tx", post(submit_tx)) + .route("/tx", tx_route) .route("/ws/subscribe", get(subscribe_l2_txs)) .with_state(state) + // Enforces a raw request-body cap before JSON deserialization, including whitespace. .layer(DefaultBodyLimit::max(max_body_bytes)) .layer(TraceLayer::new_for_http()) } @@ -69,18 +72,26 @@ async fn submit_tx( State(state): State>, req: Result, axum::extract::rejection::JsonRejection>, ) -> Result, ApiError> { - let Json(req) = req.map_err(|err| ApiError::bad_request(format!("invalid JSON: {err}")))?; + let Json(req) = req.map_err(map_json_rejection)?; - let signature_bytes = decode_hex_0x(&req.signature).map_err(ApiError::bad_request)?; - if signature_bytes.len() != 65 { - return Err(ApiError::bad_request("signature must be 65 bytes")); + if req.signature.len() != TxRequest::SIGNATURE_HEX_LEN { + return Err(ApiError::bad_request(format!( + "signature must be {} hex chars (0x + 65 bytes)", + TxRequest::SIGNATURE_HEX_LEN + ))); + } + if req.sender.len() != TxRequest::ADDRESS_HEX_LEN { + return Err(ApiError::bad_request(format!( + "sender must be {} hex chars (0x + 20 bytes)", + TxRequest::ADDRESS_HEX_LEN + ))); } - let signature = parse_signature(&signature_bytes)?; let user_op = req.message; let user_op_data_len = user_op.data.len(); let user_op_size_upper_bound = SignedUserOp::batch_bytes_upper_bound_for_data_len(user_op_data_len); + // Keep over-sized payloads out of the hot path so chunk-level batch checks can stay simple. if user_op_size_upper_bound > SignedUserOp::max_batch_bytes_upper_bound() { return Err(ApiError::bad_request(format!( @@ -89,18 +100,22 @@ async fn submit_tx( user_op_data_len ))); } + + let signature_bytes = decode_hex_0x(&req.signature).map_err(ApiError::bad_request)?; + if signature_bytes.len() != 65 { + return Err(ApiError::bad_request("signature must be 65 bytes")); + } + let signature = parse_signature(&signature_bytes)?; let nonce = user_op.nonce; - let tx_hash = user_op.eip712_signing_hash(&state.domain); + let signing_hash = user_op.eip712_signing_hash(&state.domain); let sender = signature - .recover_address_from_prehash(&tx_hash) + .recover_address_from_prehash(&signing_hash) .map_err(|_| ApiError::invalid_signature("cannot recover sender"))?; - if let Some(sender_hex) = req.sender.as_deref() { - let expected = parse_address(sender_hex).map_err(ApiError::bad_request)?; - if expected != sender { - return Err(ApiError::invalid_signature("sender mismatch")); - } + let expected = parse_address(req.sender.as_str()).map_err(ApiError::bad_request)?; + if expected != sender { + return Err(ApiError::invalid_signature("sender mismatch")); } let signed = SignedUserOp { @@ -112,23 +127,20 @@ async fn submit_tx( let (respond_to, recv) = oneshot::channel(); let enqueued = PendingUserOp { signed, - tx_hash, respond_to, received_at: SystemTime::now(), }; - enqueue_tx(&state, enqueued).await?; + enqueue_tx(&state, enqueued)?; let commit_result = recv .await .map_err(|_| ApiError::internal_error("inclusion lane dropped response"))?; commit_result.map_err(ApiError::from)?; - - info!(tx_hash = %encode_hex(&tx_hash), sender = %sender, "tx committed"); + debug!(sender = %sender, nonce, "tx committed"); Ok(Json(TxResponse { ok: true, - tx_hash: encode_hex(&tx_hash), sender: sender.to_string(), nonce, })) @@ -166,21 +178,29 @@ fn parse_signature(bytes: &[u8]) -> Result { }) } -fn encode_hex(value: &alloy_primitives::B256) -> String { - alloy_primitives::hex::encode_prefixed(value.as_slice()) +fn enqueue_tx(state: &AppState, tx: PendingUserOp) -> Result<(), ApiError> { + match state.tx_sender.try_send(InclusionLaneInput::UserOp(tx)) { + Ok(()) => Ok(()), + Err(TrySendError::Full(_)) => Err(ApiError::overloaded("queue full")), + Err(TrySendError::Closed(_)) => Err(ApiError::internal_error("inclusion lane unavailable")), + } } -async fn enqueue_tx(state: &AppState, tx: PendingUserOp) -> Result<(), ApiError> { - match state - .tx_sender - .send_timeout(InclusionLaneInput::UserOp(tx), state.queue_timeout) - .await - { - Ok(()) => Ok(()), - Err(SendTimeoutError::Timeout(_)) => Err(ApiError::overloaded("queue full")), - Err(SendTimeoutError::Closed(_)) => { - Err(ApiError::internal_error("inclusion lane unavailable")) - } +async fn handle_tx_route_error(err: BoxError) -> impl IntoResponse { + if err.is::() { + ApiError::overloaded("tx endpoint overloaded").into_response() + } else { + warn!(error = %err, "tx endpoint middleware error"); + ApiError::internal_error("tx endpoint unavailable").into_response() + } +} + +// Keep non-413 JSON extractor failures normalized to 400 for a stable API contract. +fn map_json_rejection(err: axum::extract::rejection::JsonRejection) -> ApiError { + if err.status() == StatusCode::PAYLOAD_TOO_LARGE { + ApiError::payload_too_large(format!("request body too large: {err}")) + } else { + ApiError::bad_request(format!("invalid JSON: {err}")) } } @@ -188,21 +208,53 @@ async fn subscribe_l2_txs( State(state): State>, Query(query): Query, ws: WebSocketUpgrade, -) -> impl IntoResponse { +) -> Response { let from_offset = query.from_offset.unwrap_or(0); + let permit = match Arc::clone(&state.ws_subscriber_limit).try_acquire_owned() { + Ok(permit) => permit, + Err(_) => return ApiError::overloaded("ws subscriber limit reached").into_response(), + }; let broadcaster = state.broadcaster.clone(); - ws.on_upgrade(move |socket| run_broadcaster_session(broadcaster, socket, from_offset)) + let ws_max_catchup_events = state.ws_max_catchup_events; + ws.on_upgrade(move |socket| { + run_broadcaster_session( + broadcaster, + socket, + from_offset, + permit, + ws_max_catchup_events, + ) + }) + .into_response() } async fn run_broadcaster_session( broadcaster: L2TxBroadcaster, mut socket: WebSocket, from_offset: u64, + _subscriber_permit: OwnedSemaphorePermit, + ws_max_catchup_events: u64, ) { let mut subscription = broadcaster.subscribe(); let mut next_offset = from_offset; if next_offset < subscription.live_start_offset { + let catchup_events = subscription.live_start_offset - next_offset; + if catchup_events > ws_max_catchup_events { + warn!( + requested_offset = next_offset, + live_start_offset = subscription.live_start_offset, + max_catchup_events = ws_max_catchup_events, + "ws catch-up window exceeded; closing subscriber" + ); + let _ = socket + .send(Message::Close(Some(CloseFrame { + code: close_code::POLICY, + reason: "catch-up window exceeded".into(), + }))) + .await; + return; + } if send_catch_up( &broadcaster, &mut socket, @@ -256,6 +308,37 @@ async fn run_broadcaster_session( } } +#[cfg(test)] +mod tests { + use super::*; + use axum::body::to_bytes; + + #[tokio::test] + async fn tx_route_internal_errors_are_sanitized() { + let err: BoxError = std::io::Error::other("sensitive middleware detail").into(); + let response = handle_tx_route_error(err).await.into_response(); + assert_eq!(response.status(), StatusCode::INTERNAL_SERVER_ERROR); + + let body = to_bytes(response.into_body(), usize::MAX) + .await + .expect("read response body"); + let body = String::from_utf8(body.to_vec()).expect("utf8 response body"); + + assert!( + body.contains("INTERNAL_ERROR"), + "expected internal error code in body: {body}" + ); + assert!( + body.contains("tx endpoint unavailable"), + "expected sanitized internal message in body: {body}" + ); + assert!( + !body.contains("sensitive middleware detail"), + "middleware internals leaked in body: {body}" + ); + } +} + async fn send_catch_up( broadcaster: &L2TxBroadcaster, socket: &mut WebSocket, diff --git a/sequencer/src/application.rs b/sequencer/src/application.rs deleted file mode 100644 index a14fdfb..0000000 --- a/sequencer/src/application.rs +++ /dev/null @@ -1,4 +0,0 @@ -// (c) Cartesi and individual authors (see AUTHORS) -// SPDX-License-Identifier: Apache-2.0 (see LICENSE) - -pub use app_core::application::*; diff --git a/sequencer/src/inclusion_lane/error.rs b/sequencer/src/inclusion_lane/error.rs index e1cd6a1..42cf9ec 100644 --- a/sequencer/src/inclusion_lane/error.rs +++ b/sequencer/src/inclusion_lane/error.rs @@ -1,7 +1,7 @@ // (c) Cartesi and individual authors (see AUTHORS) // SPDX-License-Identifier: Apache-2.0 (see LICENSE) -use crate::application::AppError; +use sequencer_core::application::AppError; use thiserror::Error; #[derive(Debug, Error)] diff --git a/sequencer/src/inclusion_lane/lane.rs b/sequencer/src/inclusion_lane/lane.rs index 58b3936..3ecd1fa 100644 --- a/sequencer/src/inclusion_lane/lane.rs +++ b/sequencer/src/inclusion_lane/lane.rs @@ -1,18 +1,19 @@ // (c) Cartesi and individual authors (see AUTHORS) // SPDX-License-Identifier: Apache-2.0 (see LICENSE) -use crate::application::{AppError, Application, ExecutionOutcome}; -use crate::l2_tx::SequencedL2Tx; use crate::storage::{IndexedDirectInput, Storage, WriteHead}; -use crate::user_op::SignedUserOp; +use sequencer_core::application::{AppError, Application, ExecutionOutcome}; +use sequencer_core::l2_tx::SequencedL2Tx; +use sequencer_core::user_op::SignedUserOp; use std::sync::Arc; use std::sync::atomic::{AtomicBool, Ordering}; use std::thread; -use std::time::{Duration, SystemTime}; +use std::time::{Duration, Instant, SystemTime}; use tokio::sync::mpsc; use tokio::task::JoinHandle; use super::error::CatchUpError; +use super::profiling::InclusionLaneMetrics; use super::{InclusionLaneError, InclusionLaneInput, PendingUserOp, SequencerError}; #[derive(Debug, Clone, Copy)] @@ -31,6 +32,8 @@ pub struct InclusionLaneConfig { pub max_batch_user_op_bytes: usize, pub idle_poll_interval: Duration, + pub metrics_enabled: bool, + pub metrics_log_interval: Duration, } #[derive(Debug, Clone, Default)] @@ -87,31 +90,62 @@ impl InclusionLane { fn run_forever(&mut self) -> Result<(), InclusionLaneError> { self.run_catch_up()?; let (mut next_safe_input_index, mut head) = self.load_lane_state()?; + let mut metrics = InclusionLaneMetrics::new( + self.config.metrics_enabled, + self.config.metrics_log_interval, + ); let mut included = Vec::with_capacity(self.config.max_user_ops_per_chunk.max(1)); let mut safe_directs = Vec::with_capacity(self.config.safe_direct_buffer_capacity.max(1)); while !self.stop.is_shutdown_requested() { + metrics.on_loop_start(self.rx.len()); + // Canonical per-iteration order: include user-ops first, then drain direct inputs. - let included_user_op_count = self.process_user_op_chunk(&mut head, &mut included)?; + let user_op_started = metrics.phase_started_at(); + let included_user_op_count = + self.process_user_op_chunk(&mut head, &mut included, &mut metrics)?; + metrics.on_user_ops_phase_end(user_op_started, included_user_op_count as u64); + + let direct_started = metrics.phase_started_at(); let drained_safe_direct_count = self.drain_and_execute_safe_direct_inputs( &mut next_safe_input_index, &mut safe_directs, )?; + metrics.on_directs_phase_end(direct_started, drained_safe_direct_count as u64); + let drained_safe_direct_start_index = next_safe_input_index + .checked_sub(drained_safe_direct_count as u64) + .expect("drained direct-input count cannot exceed next safe-input index"); if head.should_close_batch(&self.config) { - self.close_frame_and_batch(&mut head, drained_safe_direct_count)?; + let close_started = metrics.phase_started_at(); + self.close_frame_and_batch( + &mut head, + drained_safe_direct_start_index, + drained_safe_direct_count, + )?; + metrics.on_close_phase_end(close_started, true); } else if drained_safe_direct_count > 0 { - self.close_frame_only(&mut head, drained_safe_direct_count)?; + let close_started = metrics.phase_started_at(); + self.close_frame_only( + &mut head, + drained_safe_direct_start_index, + drained_safe_direct_count, + )?; + metrics.on_close_phase_end(close_started, false); } if included_user_op_count == 0 && drained_safe_direct_count == 0 { + let sleep_started = metrics.phase_started_at(); thread::sleep(self.config.idle_poll_interval); + metrics.on_idle_sleep_end(sleep_started); } safe_directs.clear(); + metrics.maybe_log_window(); } + metrics.log_final(); Err(InclusionLaneError::ShutdownRequested) } @@ -138,20 +172,28 @@ impl InclusionLane { &mut self, head: &mut WriteHead, included: &mut Vec, + metrics: &mut InclusionLaneMetrics, ) -> Result { - dequeue_and_execute_user_op_chunk( + let timing = dequeue_and_execute_user_op_chunk( &mut self.rx, &mut self.app, - head.batch_fee, + head.frame_fee, self.config.max_user_ops_per_chunk.max(1), included, )?; + metrics.on_user_op_dequeue_end(timing.dequeue); + metrics.on_user_op_app_execute_end(timing.app_execute); let included_count = included.len(); + let persist_started = metrics.phase_started_at(); self.persist_included_user_ops(head, included)?; + metrics.on_user_op_persist_end(persist_started); + + let ack_started = metrics.phase_started_at(); for item in included.drain(..) { let _ = item.respond_to.send(Ok(())); } + metrics.on_user_op_ack_end(ack_started); Ok(included_count) } @@ -209,20 +251,22 @@ impl InclusionLane { fn close_frame_and_batch( &mut self, head: &mut WriteHead, + drained_direct_start_index: u64, drained_direct_count: usize, ) -> Result<(), InclusionLaneError> { self.storage - .close_frame_and_batch(head, drained_direct_count) + .close_frame_and_batch(head, drained_direct_start_index, drained_direct_count) .map_err(|source| InclusionLaneError::CloseFrameRotate { source }) } fn close_frame_only( &mut self, head: &mut WriteHead, + drained_direct_start_index: u64, drained_direct_count: usize, ) -> Result<(), InclusionLaneError> { self.storage - .close_frame_only(head, drained_direct_count) + .close_frame_only(head, drained_direct_start_index, drained_direct_count) .map_err(|source| InclusionLaneError::CloseFrameRotate { source }) } @@ -259,13 +303,13 @@ impl WriteHead { fn execute_user_op( app: &mut impl Application, item: PendingUserOp, - current_batch_fee: u64, + current_frame_fee: u64, included: &mut Vec, ) { match app.validate_and_execute_user_op( item.signed.sender, &item.signed.user_op, - current_batch_fee, + current_frame_fee, ) { Ok(ExecutionOutcome::Included) => included.push(item), Ok(ExecutionOutcome::Invalid(reason)) => { @@ -282,28 +326,45 @@ fn execute_user_op( fn dequeue_and_execute_user_op_chunk( rx: &mut mpsc::Receiver, app: &mut impl Application, - current_batch_fee: u64, + current_frame_fee: u64, max_chunk: usize, included: &mut Vec, -) -> Result<(), InclusionLaneError> { +) -> Result { let mut executed_user_ops = 0_usize; + let mut timing = UserOpChunkTiming::default(); while executed_user_ops < max_chunk { + let dequeue_started = Instant::now(); match rx.try_recv() { Ok(InclusionLaneInput::UserOp(item)) => { - execute_user_op(app, item, current_batch_fee, included); + timing.dequeue = timing.dequeue.saturating_add(dequeue_started.elapsed()); + let app_exec_started = Instant::now(); + execute_user_op(app, item, current_frame_fee, included); + timing.app_execute = timing + .app_execute + .saturating_add(app_exec_started.elapsed()); executed_user_ops = executed_user_ops.saturating_add(1); } - Err(mpsc::error::TryRecvError::Empty) => return Ok(()), + Err(mpsc::error::TryRecvError::Empty) => { + timing.dequeue = timing.dequeue.saturating_add(dequeue_started.elapsed()); + return Ok(timing); + } Err(mpsc::error::TryRecvError::Disconnected) => { + timing.dequeue = timing.dequeue.saturating_add(dequeue_started.elapsed()); if executed_user_ops == 0 { return Err(InclusionLaneError::ChannelClosed); } - return Ok(()); + return Ok(timing); } } } - Ok(()) + Ok(timing) +} + +#[derive(Debug, Default, Clone, Copy)] +struct UserOpChunkTiming { + dequeue: Duration, + app_execute: Duration, } fn catch_up_application( @@ -346,15 +407,15 @@ mod tests { InclusionLane, InclusionLaneConfig, InclusionLaneError, InclusionLaneInput, InclusionLaneStop, PendingUserOp, dequeue_and_execute_user_op_chunk, }; - use crate::application::{AppError, Application, InvalidReason}; - use crate::l2_tx::ValidUserOp; use crate::storage::Storage; - use crate::user_op::{SignedUserOp, UserOp}; - use alloy_primitives::{Address, B256, Signature, U256}; - use rusqlite::{OptionalExtension, params}; + use alloy_primitives::{Address, Signature, U256}; + use rusqlite::params; + use sequencer_core::application::{AppError, Application, InvalidReason}; + use sequencer_core::l2_tx::ValidUserOp; + use sequencer_core::user_op::{SignedUserOp, UserOp}; use std::collections::HashMap; - use std::path::PathBuf; - use std::time::{Duration, SystemTime, UNIX_EPOCH}; + use std::time::{Duration, SystemTime}; + use tempfile::TempDir; use tokio::sync::{mpsc, oneshot}; #[derive(Default)] @@ -398,18 +459,21 @@ mod tests { } } - fn temp_db_path(name: &str) -> String { - let mut path = std::env::temp_dir(); - let unique = SystemTime::now() - .duration_since(UNIX_EPOCH) - .unwrap_or_default() - .as_nanos(); - path.push(format!("sequencer-inclusion-lane-{name}-{unique}.sqlite")); - path_to_string(path) + struct TestDb { + _dir: TempDir, + path: String, } - fn path_to_string(path: PathBuf) -> String { - path.to_string_lossy().into_owned() + fn temp_db(name: &str) -> TestDb { + let dir = tempfile::Builder::new() + .prefix(format!("sequencer-inclusion-lane-{name}-").as_str()) + .tempdir() + .expect("create temporary test directory"); + let path = dir.path().join("sequencer.sqlite"); + TestDb { + _dir: dir, + path: path.to_string_lossy().into_owned(), + } } fn default_test_config() -> InclusionLaneConfig { @@ -419,6 +483,8 @@ mod tests { max_batch_open: Duration::MAX, max_batch_user_op_bytes: 1_000_000_000, idle_poll_interval: Duration::from_millis(2), + metrics_enabled: false, + metrics_log_interval: Duration::from_secs(5), } } @@ -457,7 +523,6 @@ mod tests { signature: Signature::test_signature(), user_op, }, - tx_hash: B256::from([seed; 32]), respond_to, received_at: SystemTime::now(), }, @@ -472,15 +537,17 @@ mod tests { .expect("count rows") } - fn read_frame_drain(db_path: &str, batch_index: i64, frame_in_batch: i64) -> Option { + fn read_frame_direct_count(db_path: &str, batch_index: i64, frame_in_batch: i64) -> i64 { let conn = Storage::open_connection(db_path, "NORMAL").expect("open sqlite reader"); conn.query_row( - "SELECT drain_n FROM frame_drains WHERE batch_index = ?1 AND frame_in_batch = ?2", + "SELECT COUNT(*) FROM sequenced_l2_txs + WHERE batch_index = ?1 + AND frame_in_batch = ?2 + AND direct_input_index IS NOT NULL", params![batch_index, frame_in_batch], |row| row.get(0), ) - .optional() - .expect("query frame drain") + .expect("query frame direct count") } async fn wait_until(timeout: Duration, mut predicate: impl FnMut() -> bool) -> bool { @@ -508,8 +575,8 @@ mod tests { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn ack_happens_after_chunk_commit_without_closing_frame() { - let db_path = temp_db_path("ack-chunk-commit"); - let (tx, lane_stop, lane_handle) = start_lane(&db_path, default_test_config()); + let db = temp_db("ack-chunk-commit"); + let (tx, lane_stop, lane_handle) = start_lane(db.path.as_str(), default_test_config()); let (pending, recv) = make_pending_user_op(0x11); tx.send(InclusionLaneInput::UserOp(pending)) @@ -519,23 +586,24 @@ mod tests { .await .expect("wait for ack") .expect("ack channel open"); - let user_ops_count = read_count(&db_path, "user_ops"); - let frame_drains_count = read_count(&db_path, "frame_drains"); + let user_ops_count = read_count(db.path.as_str(), "user_ops"); + let frame0_direct_count = read_frame_direct_count(db.path.as_str(), 0, 0); shutdown_lane(&lane_stop, lane_handle).await; assert!(ack.is_ok(), "user op should be included"); assert_eq!(user_ops_count, 1); assert_eq!( - frame_drains_count, 0, + frame0_direct_count, 0, "frame should stay open when no directs and no batch close" ); } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn direct_inputs_close_frame_and_persist_drain() { - let db_path = temp_db_path("directs-close-frame"); - let (_tx, lane_stop, lane_handle) = start_lane(&db_path, default_test_config()); - let mut feeder_storage = Storage::open(&db_path, "NORMAL").expect("open feeder storage"); + let db = temp_db("directs-close-frame"); + let (_tx, lane_stop, lane_handle) = start_lane(db.path.as_str(), default_test_config()); + let mut feeder_storage = + Storage::open(db.path.as_str(), "NORMAL").expect("open feeder storage"); feeder_storage .append_safe_direct_inputs(&[crate::storage::IndexedDirectInput { @@ -545,23 +613,24 @@ mod tests { .expect("append safe direct input"); let drained = wait_until(Duration::from_secs(2), || { - read_frame_drain(&db_path, 0, 0) == Some(1) + read_frame_direct_count(db.path.as_str(), 0, 0) == 1 }) .await; - let frames_count = read_count(&db_path, "frames"); + let frames_count = read_count(db.path.as_str(), "frames"); shutdown_lane(&lane_stop, lane_handle).await; - assert!(drained, "expected frame drain row with drain_n=1"); + assert!(drained, "expected one drained direct input in frame 0"); assert_eq!(frames_count, 2); } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn direct_inputs_are_paginated_by_buffer_capacity() { - let db_path = temp_db_path("directs-pagination"); + let db = temp_db("directs-pagination"); let mut config = default_test_config(); config.safe_direct_buffer_capacity = 2; - let (_tx, lane_stop, lane_handle) = start_lane(&db_path, config); - let mut feeder_storage = Storage::open(&db_path, "NORMAL").expect("open feeder storage"); + let (_tx, lane_stop, lane_handle) = start_lane(db.path.as_str(), config); + let mut feeder_storage = + Storage::open(db.path.as_str(), "NORMAL").expect("open feeder storage"); let mut directs = Vec::new(); for index in 0..5_u64 { @@ -575,22 +644,22 @@ mod tests { .expect("append safe direct inputs"); let drained = wait_until(Duration::from_secs(2), || { - read_frame_drain(&db_path, 0, 0) == Some(5) + read_frame_direct_count(db.path.as_str(), 0, 0) == 5 }) .await; - let frames_count = read_count(&db_path, "frames"); + let frames_count = read_count(db.path.as_str(), "frames"); shutdown_lane(&lane_stop, lane_handle).await; - assert!(drained, "expected frame drain row with drain_n=5"); + assert!(drained, "expected five drained direct inputs in frame 0"); assert_eq!(frames_count, 2); } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn batch_closes_when_max_open_time_is_reached() { - let db_path = temp_db_path("batch-close-time"); + let db = temp_db("batch-close-time"); let mut config = default_test_config(); config.max_batch_open = Duration::from_millis(20); - let (tx, lane_stop, lane_handle) = start_lane(&db_path, config); + let (tx, lane_stop, lane_handle) = start_lane(db.path.as_str(), config); let (pending, recv) = make_pending_user_op(0x22); tx.send(InclusionLaneInput::UserOp(pending)) @@ -601,23 +670,23 @@ mod tests { .expect("wait for ack") .expect("ack channel open"); let rotated = wait_until(Duration::from_secs(2), || { - read_count(&db_path, "batches") >= 2 + read_count(db.path.as_str(), "batches") >= 2 }) .await; - let drain = read_frame_drain(&db_path, 0, 0); + let drain = read_frame_direct_count(db.path.as_str(), 0, 0); shutdown_lane(&lane_stop, lane_handle).await; assert!(ack.is_ok(), "user op should be included"); assert!(rotated, "expected batch rotation by time"); - assert_eq!(drain, Some(0)); + assert_eq!(drain, 0); } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn batch_closes_when_max_user_op_bytes_is_reached() { - let db_path = temp_db_path("batch-close-size"); + let db = temp_db("batch-close-size"); let mut config = default_test_config(); config.max_batch_user_op_bytes = SignedUserOp::max_batch_bytes_upper_bound(); - let (tx, lane_stop, lane_handle) = start_lane(&db_path, config); + let (tx, lane_stop, lane_handle) = start_lane(db.path.as_str(), config); let (pending, recv) = make_pending_user_op(0x33); tx.send(InclusionLaneInput::UserOp(pending)) @@ -628,15 +697,15 @@ mod tests { .expect("wait for ack") .expect("ack channel open"); let rotated = wait_until(Duration::from_secs(2), || { - read_count(&db_path, "batches") >= 2 + read_count(db.path.as_str(), "batches") >= 2 }) .await; - let drain = read_frame_drain(&db_path, 0, 0); + let drain = read_frame_direct_count(db.path.as_str(), 0, 0); shutdown_lane(&lane_stop, lane_handle).await; assert!(ack.is_ok(), "user op should be included"); assert!(rotated, "expected batch rotation by size"); - assert_eq!(drain, Some(0)); + assert_eq!(drain, 0); } #[test] diff --git a/sequencer/src/inclusion_lane/mod.rs b/sequencer/src/inclusion_lane/mod.rs index 73ce6e5..10cb443 100644 --- a/sequencer/src/inclusion_lane/mod.rs +++ b/sequencer/src/inclusion_lane/mod.rs @@ -3,6 +3,7 @@ mod error; mod lane; +mod profiling; mod types; pub use error::InclusionLaneError; diff --git a/sequencer/src/inclusion_lane/profiling.rs b/sequencer/src/inclusion_lane/profiling.rs new file mode 100644 index 0000000..0aace24 --- /dev/null +++ b/sequencer/src/inclusion_lane/profiling.rs @@ -0,0 +1,263 @@ +// (c) Cartesi and individual authors (see AUTHORS) +// SPDX-License-Identifier: Apache-2.0 (see LICENSE) + +use std::time::{Duration, Instant}; +use tracing::info; + +#[derive(Debug)] +pub(super) struct InclusionLaneMetrics { + enabled: bool, + log_interval: Duration, + window_started_at: Instant, + loops: u64, + included_user_ops: u64, + drained_direct_inputs: u64, + frame_only_closes: u64, + frame_and_batch_closes: u64, + idle_sleeps: u64, + max_queue_depth: usize, + user_op_phase: Duration, + user_op_dequeue_phase: Duration, + user_op_app_execute_phase: Duration, + user_op_persist_phase: Duration, + user_op_ack_phase: Duration, + direct_phase: Duration, + close_phase: Duration, + idle_sleep: Duration, +} + +impl InclusionLaneMetrics { + pub(super) fn new(enabled: bool, log_interval: Duration) -> Self { + Self { + enabled, + log_interval, + window_started_at: Instant::now(), + loops: 0, + included_user_ops: 0, + drained_direct_inputs: 0, + frame_only_closes: 0, + frame_and_batch_closes: 0, + idle_sleeps: 0, + max_queue_depth: 0, + user_op_phase: Duration::ZERO, + user_op_dequeue_phase: Duration::ZERO, + user_op_app_execute_phase: Duration::ZERO, + user_op_persist_phase: Duration::ZERO, + user_op_ack_phase: Duration::ZERO, + direct_phase: Duration::ZERO, + close_phase: Duration::ZERO, + idle_sleep: Duration::ZERO, + } + } + + pub(super) fn phase_started_at(&self) -> Option { + self.enabled.then(Instant::now) + } + + pub(super) fn on_loop_start(&mut self, queue_depth: usize) { + if !self.enabled { + return; + } + self.loops = self.loops.saturating_add(1); + self.max_queue_depth = self.max_queue_depth.max(queue_depth); + } + + pub(super) fn on_user_ops_phase_end( + &mut self, + started_at: Option, + included_user_ops: u64, + ) { + if !self.enabled { + return; + } + self.included_user_ops = self.included_user_ops.saturating_add(included_user_ops); + self.user_op_phase = self + .user_op_phase + .saturating_add(elapsed_or_zero(started_at)); + } + + pub(super) fn on_user_op_dequeue_end(&mut self, elapsed: Duration) { + if !self.enabled { + return; + } + self.user_op_dequeue_phase = self.user_op_dequeue_phase.saturating_add(elapsed); + } + + pub(super) fn on_user_op_app_execute_end(&mut self, elapsed: Duration) { + if !self.enabled { + return; + } + self.user_op_app_execute_phase = self.user_op_app_execute_phase.saturating_add(elapsed); + } + + pub(super) fn on_user_op_persist_end(&mut self, started_at: Option) { + if !self.enabled { + return; + } + self.user_op_persist_phase = self + .user_op_persist_phase + .saturating_add(elapsed_or_zero(started_at)); + } + + pub(super) fn on_user_op_ack_end(&mut self, started_at: Option) { + if !self.enabled { + return; + } + self.user_op_ack_phase = self + .user_op_ack_phase + .saturating_add(elapsed_or_zero(started_at)); + } + + pub(super) fn on_directs_phase_end( + &mut self, + started_at: Option, + drained_direct_inputs: u64, + ) { + if !self.enabled { + return; + } + self.drained_direct_inputs = self + .drained_direct_inputs + .saturating_add(drained_direct_inputs); + self.direct_phase = self + .direct_phase + .saturating_add(elapsed_or_zero(started_at)); + } + + pub(super) fn on_close_phase_end(&mut self, started_at: Option, closed_batch: bool) { + if !self.enabled { + return; + } + if closed_batch { + self.frame_and_batch_closes = self.frame_and_batch_closes.saturating_add(1); + } else { + self.frame_only_closes = self.frame_only_closes.saturating_add(1); + } + self.close_phase = self.close_phase.saturating_add(elapsed_or_zero(started_at)); + } + + pub(super) fn on_idle_sleep_end(&mut self, started_at: Option) { + if !self.enabled { + return; + } + self.idle_sleeps = self.idle_sleeps.saturating_add(1); + self.idle_sleep = self.idle_sleep.saturating_add(elapsed_or_zero(started_at)); + } + + pub(super) fn maybe_log_window(&mut self) { + if !self.enabled { + return; + } + let elapsed = self.window_started_at.elapsed(); + if elapsed < self.log_interval { + return; + } + self.log_window(elapsed, false); + self.reset_window(); + } + + pub(super) fn log_final(&mut self) { + if !self.enabled { + return; + } + let elapsed = self.window_started_at.elapsed(); + if elapsed.is_zero() && self.loops == 0 { + return; + } + self.log_window(elapsed, true); + } + + fn log_window(&self, elapsed: Duration, final_window: bool) { + let elapsed_secs = elapsed.as_secs_f64(); + let included_tps = if elapsed_secs > 0.0 { + self.included_user_ops as f64 / elapsed_secs + } else { + 0.0 + }; + let user_op_dequeue_share_pct = percentage( + self.user_op_dequeue_phase.as_nanos(), + self.user_op_phase.as_nanos(), + ); + let user_op_app_execute_share_pct = percentage( + self.user_op_app_execute_phase.as_nanos(), + self.user_op_phase.as_nanos(), + ); + let user_op_persist_share_pct = percentage( + self.user_op_persist_phase.as_nanos(), + self.user_op_phase.as_nanos(), + ); + let user_op_ack_share_pct = percentage( + self.user_op_ack_phase.as_nanos(), + self.user_op_phase.as_nanos(), + ); + let app_plus_persist = self + .user_op_app_execute_phase + .saturating_add(self.user_op_persist_phase); + let user_op_app_share_pct_of_app_plus_persist = percentage( + self.user_op_app_execute_phase.as_nanos(), + app_plus_persist.as_nanos(), + ); + let user_op_persist_share_pct_of_app_plus_persist = percentage( + self.user_op_persist_phase.as_nanos(), + app_plus_persist.as_nanos(), + ); + info!( + final_window, + window_ms = elapsed.as_millis() as u64, + loops = self.loops, + included_user_ops = self.included_user_ops, + drained_direct_inputs = self.drained_direct_inputs, + included_tps = included_tps, + frame_only_closes = self.frame_only_closes, + frame_and_batch_closes = self.frame_and_batch_closes, + idle_sleeps = self.idle_sleeps, + max_queue_depth = self.max_queue_depth, + user_op_phase_ms = self.user_op_phase.as_millis() as u64, + user_op_dequeue_phase_ms = self.user_op_dequeue_phase.as_millis() as u64, + user_op_app_execute_phase_ms = self.user_op_app_execute_phase.as_millis() as u64, + user_op_persist_phase_ms = self.user_op_persist_phase.as_millis() as u64, + user_op_ack_phase_ms = self.user_op_ack_phase.as_millis() as u64, + user_op_dequeue_share_pct = user_op_dequeue_share_pct, + user_op_app_execute_share_pct = user_op_app_execute_share_pct, + user_op_persist_share_pct = user_op_persist_share_pct, + user_op_ack_share_pct = user_op_ack_share_pct, + user_op_app_share_pct_of_app_plus_persist = user_op_app_share_pct_of_app_plus_persist, + user_op_persist_share_pct_of_app_plus_persist = + user_op_persist_share_pct_of_app_plus_persist, + direct_phase_ms = self.direct_phase.as_millis() as u64, + close_phase_ms = self.close_phase.as_millis() as u64, + idle_sleep_ms = self.idle_sleep.as_millis() as u64, + "inclusion lane metrics" + ); + } + + fn reset_window(&mut self) { + self.window_started_at = Instant::now(); + self.loops = 0; + self.included_user_ops = 0; + self.drained_direct_inputs = 0; + self.frame_only_closes = 0; + self.frame_and_batch_closes = 0; + self.idle_sleeps = 0; + self.max_queue_depth = 0; + self.user_op_phase = Duration::ZERO; + self.user_op_dequeue_phase = Duration::ZERO; + self.user_op_app_execute_phase = Duration::ZERO; + self.user_op_persist_phase = Duration::ZERO; + self.user_op_ack_phase = Duration::ZERO; + self.direct_phase = Duration::ZERO; + self.close_phase = Duration::ZERO; + self.idle_sleep = Duration::ZERO; + } +} + +fn elapsed_or_zero(started_at: Option) -> Duration { + started_at.map_or(Duration::ZERO, |value| value.elapsed()) +} + +fn percentage(part: u128, total: u128) -> f64 { + if total == 0 { + return 0.0; + } + (part as f64) * 100.0 / (total as f64) +} diff --git a/sequencer/src/inclusion_lane/types.rs b/sequencer/src/inclusion_lane/types.rs index 9756fc0..204f52b 100644 --- a/sequencer/src/inclusion_lane/types.rs +++ b/sequencer/src/inclusion_lane/types.rs @@ -3,16 +3,13 @@ use std::time::SystemTime; -use alloy_primitives::B256; +use sequencer_core::user_op::SignedUserOp; use thiserror::Error; use tokio::sync::oneshot; -use crate::user_op::SignedUserOp; - #[derive(Debug)] pub struct PendingUserOp { pub signed: SignedUserOp, - pub tx_hash: B256, pub respond_to: oneshot::Sender>, pub received_at: SystemTime, } diff --git a/sequencer/src/l2_tx_broadcaster/mod.rs b/sequencer/src/l2_tx_broadcaster/mod.rs index 6ff6c6c..5e441e0 100644 --- a/sequencer/src/l2_tx_broadcaster/mod.rs +++ b/sequencer/src/l2_tx_broadcaster/mod.rs @@ -1,16 +1,19 @@ // (c) Cartesi and individual authors (see AUTHORS) // SPDX-License-Identifier: Apache-2.0 (see LICENSE) +mod profiling; + use std::collections::HashMap; -use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; use std::sync::{Arc, Mutex}; use std::time::Duration; -use serde::Serialize; +pub use sequencer_core::broadcast::BroadcastTxMessage; use tokio::sync::mpsc; +use tokio::sync::mpsc::error::TrySendError; use tracing::warn; -use crate::l2_tx::SequencedL2Tx; +use self::profiling::{BroadcasterMetrics, FanoutOutcome}; use crate::storage::Storage; #[derive(Debug, Clone, Copy)] @@ -18,6 +21,8 @@ pub struct L2TxBroadcasterConfig { pub idle_poll_interval: Duration, pub page_size: usize, pub subscriber_buffer_capacity: usize, + pub metrics_enabled: bool, + pub metrics_log_interval: Duration, } #[derive(Clone)] @@ -30,27 +35,13 @@ pub struct LiveSubscription { pub live_start_offset: u64, } -#[derive(Debug, Clone, Serialize)] -#[serde(tag = "kind", rename_all = "snake_case")] -pub enum BroadcastTxMessage { - UserOp { - offset: u64, - sender: String, - fee: u64, - data: String, - }, - DirectInput { - offset: u64, - payload: String, - }, -} - struct L2TxBroadcasterInner { db_path: String, page_size: usize, subscriber_buffer_capacity: usize, head_offset: AtomicU64, next_subscriber_id: AtomicU64, + stop_requested: AtomicBool, subscribers: Mutex>>, } @@ -71,17 +62,27 @@ impl L2TxBroadcaster { subscriber_buffer_capacity: config.subscriber_buffer_capacity.max(1), head_offset: AtomicU64::new(head_offset), next_subscriber_id: AtomicU64::new(0), + stop_requested: AtomicBool::new(false), subscribers: Mutex::new(HashMap::new()), }); let worker_inner = Arc::clone(&inner); tokio::task::spawn_blocking(move || { - run_poller(worker_inner, config.idle_poll_interval); + run_poller( + worker_inner, + config.idle_poll_interval, + config.metrics_enabled, + config.metrics_log_interval, + ); }); Ok(Self { inner }) } + pub fn request_shutdown(&self) { + self.inner.stop_requested.store(true, Ordering::Relaxed); + } + pub fn subscribe(&self) -> LiveSubscription { let (tx, rx) = mpsc::channel(self.inner.subscriber_buffer_capacity); let subscriber_id = self @@ -112,31 +113,12 @@ impl L2TxBroadcaster { } } -impl BroadcastTxMessage { - pub fn offset(&self) -> u64 { - match self { - Self::UserOp { offset, .. } => *offset, - Self::DirectInput { offset, .. } => *offset, - } - } - - pub fn from_offset_and_tx(offset: u64, tx: SequencedL2Tx) -> Self { - match tx { - SequencedL2Tx::UserOp(user_op) => Self::UserOp { - offset, - sender: user_op.sender.to_string(), - fee: user_op.fee, - data: alloy_primitives::hex::encode_prefixed(user_op.data.as_slice()), - }, - SequencedL2Tx::Direct(direct) => Self::DirectInput { - offset, - payload: alloy_primitives::hex::encode_prefixed(direct.payload.as_slice()), - }, - } - } -} - -fn run_poller(inner: Arc, idle_poll_interval: Duration) { +fn run_poller( + inner: Arc, + idle_poll_interval: Duration, + metrics_enabled: bool, + metrics_log_interval: Duration, +) { let mut storage = match Storage::open_read_only(inner.db_path.as_str()) { Ok(storage) => storage, Err(err) => { @@ -145,23 +127,41 @@ fn run_poller(inner: Arc, idle_poll_interval: Duration) { } }; let mut next_offset = inner.head_offset.load(Ordering::Acquire); - - loop { + let mut metrics = BroadcasterMetrics::new( + metrics_enabled, + metrics_log_interval, + inner.page_size, + inner.subscriber_buffer_capacity, + idle_poll_interval, + ); + + while !inner.stop_requested.load(Ordering::Relaxed) { + metrics.on_loop_start(); + let read_started = metrics.phase_started_at(); let txs = match storage.load_ordered_l2_txs_page_from(next_offset, inner.page_size) { Ok(value) => value, Err(err) => { + metrics.on_read_error(read_started); warn!( error = %err, offset = next_offset, "l2 tx broadcaster failed to read ordered tx page" ); + let sleep_started = metrics.phase_started_at(); std::thread::sleep(idle_poll_interval); + metrics.on_idle_sleep_end(sleep_started); + metrics.maybe_log_window(); continue; } }; + metrics.on_read_end(read_started, txs.len() as u64); if txs.is_empty() { + metrics.on_empty_poll(); + let sleep_started = metrics.phase_started_at(); std::thread::sleep(idle_poll_interval); + metrics.on_idle_sleep_end(sleep_started); + metrics.maybe_log_window(); continue; } @@ -169,30 +169,55 @@ fn run_poller(inner: Arc, idle_poll_interval: Duration) { let event = BroadcastTxMessage::from_offset_and_tx(next_offset, tx); next_offset = next_offset.saturating_add(1); inner.head_offset.store(next_offset, Ordering::Release); - fanout_event(Arc::as_ref(&inner), event); + let fanout_started = metrics.phase_started_at(); + let outcome = fanout_event(Arc::as_ref(&inner), event); + metrics.on_fanout_end(fanout_started, outcome); } + metrics.maybe_log_window(); } + metrics.log_final(); } -fn fanout_event(inner: &L2TxBroadcasterInner, event: BroadcastTxMessage) { +fn fanout_event(inner: &L2TxBroadcasterInner, event: BroadcastTxMessage) -> FanoutOutcome { let mut to_remove = Vec::new(); let mut subscribers = inner .subscribers .lock() .expect("l2 tx broadcaster subscribers mutex poisoned"); + let subscriber_count_before = subscribers.len(); + let mut dropped_closed = 0_u64; + let mut dropped_full = 0_u64; + let mut delivered = 0_u64; for (subscriber_id, sender) in subscribers.iter() { - if sender.try_send(event.clone()).is_err() { - to_remove.push(*subscriber_id); + match sender.try_send(event.clone()) { + Ok(()) => delivered = delivered.saturating_add(1), + Err(TrySendError::Closed(_)) => { + to_remove.push(*subscriber_id); + dropped_closed = dropped_closed.saturating_add(1); + warn!(subscriber_id, "l2 tx broadcaster removed closed subscriber"); + } + Err(TrySendError::Full(_)) => { + to_remove.push(*subscriber_id); + dropped_full = dropped_full.saturating_add(1); + warn!( + subscriber_id, + "l2 tx broadcaster dropped slow subscriber due to full channel" + ); + } } } for subscriber_id in to_remove { subscribers.remove(&subscriber_id); - warn!( - subscriber_id, - "l2 tx broadcaster dropped subscriber due to closed/full channel" - ); + } + + FanoutOutcome { + delivered, + dropped_closed, + dropped_full, + subscriber_count_before: subscriber_count_before as u64, + subscriber_count_after: subscribers.len() as u64, } } @@ -200,10 +225,10 @@ fn fanout_event(inner: &L2TxBroadcasterInner, event: BroadcastTxMessage) { mod tests { use super::BroadcastTxMessage; use super::{L2TxBroadcaster, L2TxBroadcasterInner}; - use crate::l2_tx::{DirectInput, SequencedL2Tx, ValidUserOp}; use alloy_primitives::Address; + use sequencer_core::l2_tx::{DirectInput, SequencedL2Tx, ValidUserOp}; use std::collections::HashMap; - use std::sync::atomic::AtomicU64; + use std::sync::atomic::{AtomicBool, AtomicU64}; use std::sync::{Arc, Mutex}; use std::time::Duration; @@ -247,6 +272,7 @@ mod tests { subscriber_buffer_capacity: 1, head_offset: AtomicU64::new(0), next_subscriber_id: AtomicU64::new(0), + stop_requested: AtomicBool::new(false), subscribers: Mutex::new(HashMap::new()), }), }; diff --git a/sequencer/src/l2_tx_broadcaster/profiling.rs b/sequencer/src/l2_tx_broadcaster/profiling.rs new file mode 100644 index 0000000..0110e92 --- /dev/null +++ b/sequencer/src/l2_tx_broadcaster/profiling.rs @@ -0,0 +1,199 @@ +// (c) Cartesi and individual authors (see AUTHORS) +// SPDX-License-Identifier: Apache-2.0 (see LICENSE) + +use std::time::{Duration, Instant}; +use tracing::info; + +#[derive(Debug, Clone, Copy)] +pub(super) struct FanoutOutcome { + pub(super) delivered: u64, + pub(super) dropped_closed: u64, + pub(super) dropped_full: u64, + pub(super) subscriber_count_before: u64, + pub(super) subscriber_count_after: u64, +} + +#[derive(Debug)] +pub(super) struct BroadcasterMetrics { + enabled: bool, + log_interval: Duration, + page_size: usize, + subscriber_buffer_capacity: usize, + idle_poll_interval: Duration, + window_started_at: Instant, + loops: u64, + empty_polls: u64, + read_errors: u64, + loaded_txs: u64, + fanout_delivered: u64, + dropped_closed: u64, + dropped_full: u64, + max_subscribers_before: u64, + max_subscribers_after: u64, + read_phase: Duration, + fanout_phase: Duration, + idle_sleep: Duration, +} + +impl BroadcasterMetrics { + pub(super) fn new( + enabled: bool, + log_interval: Duration, + page_size: usize, + subscriber_buffer_capacity: usize, + idle_poll_interval: Duration, + ) -> Self { + Self { + enabled, + log_interval, + page_size, + subscriber_buffer_capacity, + idle_poll_interval, + window_started_at: Instant::now(), + loops: 0, + empty_polls: 0, + read_errors: 0, + loaded_txs: 0, + fanout_delivered: 0, + dropped_closed: 0, + dropped_full: 0, + max_subscribers_before: 0, + max_subscribers_after: 0, + read_phase: Duration::ZERO, + fanout_phase: Duration::ZERO, + idle_sleep: Duration::ZERO, + } + } + + pub(super) fn phase_started_at(&self) -> Option { + self.enabled.then(Instant::now) + } + + pub(super) fn on_loop_start(&mut self) { + if !self.enabled { + return; + } + self.loops = self.loops.saturating_add(1); + } + + pub(super) fn on_empty_poll(&mut self) { + if !self.enabled { + return; + } + self.empty_polls = self.empty_polls.saturating_add(1); + } + + pub(super) fn on_read_error(&mut self, started_at: Option) { + if !self.enabled { + return; + } + self.read_errors = self.read_errors.saturating_add(1); + self.read_phase = self.read_phase.saturating_add(elapsed_or_zero(started_at)); + } + + pub(super) fn on_read_end(&mut self, started_at: Option, loaded_txs: u64) { + if !self.enabled { + return; + } + self.loaded_txs = self.loaded_txs.saturating_add(loaded_txs); + self.read_phase = self.read_phase.saturating_add(elapsed_or_zero(started_at)); + } + + pub(super) fn on_fanout_end(&mut self, started_at: Option, outcome: FanoutOutcome) { + if !self.enabled { + return; + } + self.fanout_delivered = self.fanout_delivered.saturating_add(outcome.delivered); + self.dropped_closed = self.dropped_closed.saturating_add(outcome.dropped_closed); + self.dropped_full = self.dropped_full.saturating_add(outcome.dropped_full); + self.max_subscribers_before = self + .max_subscribers_before + .max(outcome.subscriber_count_before); + self.max_subscribers_after = self + .max_subscribers_after + .max(outcome.subscriber_count_after); + self.fanout_phase = self + .fanout_phase + .saturating_add(elapsed_or_zero(started_at)); + } + + pub(super) fn on_idle_sleep_end(&mut self, started_at: Option) { + if !self.enabled { + return; + } + self.idle_sleep = self.idle_sleep.saturating_add(elapsed_or_zero(started_at)); + } + + pub(super) fn maybe_log_window(&mut self) { + if !self.enabled { + return; + } + let elapsed = self.window_started_at.elapsed(); + if elapsed < self.log_interval { + return; + } + self.log_window(elapsed, false); + self.reset_window(); + } + + pub(super) fn log_final(&mut self) { + if !self.enabled { + return; + } + let elapsed = self.window_started_at.elapsed(); + if elapsed.is_zero() && self.loops == 0 { + return; + } + self.log_window(elapsed, true); + } + + fn log_window(&self, elapsed: Duration, final_window: bool) { + let elapsed_secs = elapsed.as_secs_f64(); + let loaded_tps = if elapsed_secs > 0.0 { + self.loaded_txs as f64 / elapsed_secs + } else { + 0.0 + }; + info!( + final_window, + window_ms = elapsed.as_millis() as u64, + page_size = self.page_size, + subscriber_buffer_capacity = self.subscriber_buffer_capacity, + idle_poll_interval_ms = self.idle_poll_interval.as_millis() as u64, + loops = self.loops, + empty_polls = self.empty_polls, + read_errors = self.read_errors, + loaded_txs = self.loaded_txs, + loaded_tps = loaded_tps, + fanout_delivered = self.fanout_delivered, + dropped_closed = self.dropped_closed, + dropped_full = self.dropped_full, + max_subscribers_before = self.max_subscribers_before, + max_subscribers_after = self.max_subscribers_after, + read_phase_ms = self.read_phase.as_millis() as u64, + fanout_phase_ms = self.fanout_phase.as_millis() as u64, + idle_sleep_ms = self.idle_sleep.as_millis() as u64, + "l2 tx broadcaster metrics" + ); + } + + fn reset_window(&mut self) { + self.window_started_at = Instant::now(); + self.loops = 0; + self.empty_polls = 0; + self.read_errors = 0; + self.loaded_txs = 0; + self.fanout_delivered = 0; + self.dropped_closed = 0; + self.dropped_full = 0; + self.max_subscribers_before = 0; + self.max_subscribers_after = 0; + self.read_phase = Duration::ZERO; + self.fanout_phase = Duration::ZERO; + self.idle_sleep = Duration::ZERO; + } +} + +fn elapsed_or_zero(started_at: Option) -> Duration { + started_at.map_or(Duration::ZERO, |value| value.elapsed()) +} diff --git a/sequencer/src/lib.rs b/sequencer/src/lib.rs index 1959ece..9d711eb 100644 --- a/sequencer/src/lib.rs +++ b/sequencer/src/lib.rs @@ -6,9 +6,6 @@ //! Flow: API -> inclusion lane -> SQLite -> catch-up replay. //! The inclusion lane is the single writer that defines execution order. pub mod api; -pub mod application; pub mod inclusion_lane; -pub mod l2_tx; pub mod l2_tx_broadcaster; pub mod storage; -pub mod user_op; diff --git a/sequencer/src/main.rs b/sequencer/src/main.rs index b35c116..fe87046 100644 --- a/sequencer/src/main.rs +++ b/sequencer/src/main.rs @@ -6,10 +6,15 @@ use std::time::Duration; use alloy_primitives::{Address, U256}; use alloy_sol_types::Eip712Domain; +use app_core::application::{WalletApp, WalletConfig}; +use clap::{Args, Parser, Subcommand, ValueEnum}; +use sequencer_core::api::TxRequest; +use sequencer_core::user_op::SignedUserOp; +use serde::Serialize; +use tokio::sync::Semaphore; use tracing_subscriber::EnvFilter; use sequencer::api::AppState; -use sequencer::application::{WalletApp, WalletConfig}; use sequencer::inclusion_lane::{ InclusionLane, InclusionLaneConfig, InclusionLaneError, InclusionLaneInput, }; @@ -18,35 +23,56 @@ use sequencer::storage; const DEFAULT_HTTP_ADDR: &str = "127.0.0.1:3000"; const DEFAULT_DB_PATH: &str = "sequencer.db"; -const DEFAULT_QUEUE_CAP: usize = 1024; -const DEFAULT_QUEUE_TIMEOUT_MS: u64 = 100; -const DEFAULT_MAX_USER_OPS_PER_CHUNK: usize = 64; -const DEFAULT_SAFE_DIRECT_BUFFER_CAPACITY: usize = 256; +const DEFAULT_QUEUE_CAP: usize = 8192; +const DEFAULT_MAX_USER_OPS_PER_CHUNK: usize = 1024; +const DEFAULT_SAFE_DIRECT_BUFFER_CAPACITY: usize = 2048; const DEFAULT_MAX_BATCH_OPEN_DURATION: Duration = Duration::from_secs(2 * 60 * 60); const DEFAULT_MAX_BATCH_USER_OP_BYTES: usize = 1_048_576; // 1 MiB const DEFAULT_INCLUSION_LANE_IDLE_POLL_INTERVAL: Duration = Duration::from_millis(2); const DEFAULT_BROADCASTER_IDLE_POLL_INTERVAL: Duration = Duration::from_millis(20); const DEFAULT_BROADCASTER_PAGE_SIZE: usize = 256; -const DEFAULT_BROADCASTER_SUBSCRIBER_BUFFER_CAPACITY: usize = 1024; -const DEFAULT_MAX_BODY_BYTES: usize = 128 * 1024; -const DEFAULT_SQLITE_SYNCHRONOUS: &str = "NORMAL"; +const DEFAULT_BROADCASTER_SUBSCRIBER_BUFFER_CAPACITY: usize = 32_768; +const DEFAULT_WS_MAX_SUBSCRIBERS: usize = 64; +const DEFAULT_WS_MAX_CATCHUP_EVENTS: u64 = 50_000; +const DEFAULT_OVERLOAD_MAX_INFLIGHT_MULTIPLIER: usize = 2; +const DEFAULT_RUNTIME_METRICS_ENABLED: bool = false; +const DEFAULT_RUNTIME_METRICS_LOG_INTERVAL: Duration = Duration::from_secs(5); +const DEFAULT_MAX_BODY_BYTES: usize = TxRequest::MAX_JSON_BYTES_RECOMMENDED; const DEFAULT_DOMAIN_NAME: &str = "CartesiAppSequencer"; const DEFAULT_DOMAIN_VERSION: &str = "1"; const DEFAULT_DOMAIN_CHAIN_ID: u64 = 1; const DEFAULT_DOMAIN_VERIFYING_CONTRACT: &str = "0x0000000000000000000000000000000000000000"; +fn default_overload_max_inflight_submissions(queue_capacity: usize) -> usize { + queue_capacity + .saturating_mul(DEFAULT_OVERLOAD_MAX_INFLIGHT_MULTIPLIER) + .max(1) +} + #[tokio::main] async fn main() -> Result<(), Box> { + let cli = Cli::parse(); + if let Some(command) = cli.command { + return handle_config_command(command); + } + + run(cli.run).await +} + +async fn run(args: RunArgs) -> Result<(), Box> { tracing_subscriber::fmt() .with_env_filter( EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("info")), ) .init(); - let config = Config::from_env(); + let config = Config::from_run_args(args) + .map_err(|reason| std::io::Error::other(format!("invalid configuration: {reason}")))?; + log_effective_config(&config); let domain = config.build_domain()?; - let storage = storage::Storage::open(&config.db_path, &config.sqlite_synchronous)?; + let mut storage = storage::Storage::open(&config.db_path, config.sqlite_synchronous.pragma())?; + force_zero_frame_fee_for_now(&mut storage)?; let (tx, rx) = tokio::sync::mpsc::channel::(config.queue_capacity); let inclusion_lane = InclusionLane::new( @@ -59,6 +85,8 @@ async fn main() -> Result<(), Box> { max_batch_open: config.max_batch_open, max_batch_user_op_bytes: config.max_batch_user_op_bytes, idle_poll_interval: config.inclusion_lane_idle_poll_interval, + metrics_enabled: config.runtime_metrics_enabled, + metrics_log_interval: config.runtime_metrics_log_interval, }, ); let (mut inclusion_lane_handle, inclusion_lane_stop) = inclusion_lane.spawn(); @@ -68,14 +96,19 @@ async fn main() -> Result<(), Box> { idle_poll_interval: config.broadcaster_idle_poll_interval, page_size: config.broadcaster_page_size, subscriber_buffer_capacity: config.broadcaster_subscriber_buffer_capacity, + metrics_enabled: config.runtime_metrics_enabled, + metrics_log_interval: config.runtime_metrics_log_interval, }, ) .map_err(|reason| format!("failed to start l2 tx broadcaster: {reason}"))?; + let broadcaster_shutdown = broadcaster.clone(); let state = Arc::new(AppState { tx_sender: tx, domain, - queue_timeout: std::time::Duration::from_millis(config.queue_timeout_ms), + overload_max_inflight_submissions: config.overload_max_inflight_submissions, + ws_subscriber_limit: Arc::new(Semaphore::new(config.ws_max_subscribers)), + ws_max_catchup_events: config.ws_max_catchup_events, broadcaster, }); @@ -85,6 +118,7 @@ async fn main() -> Result<(), Box> { tracing::info!(address = %config.http_addr, "listening"); tokio::select! { server_result = axum::serve(listener, app) => { + broadcaster_shutdown.request_shutdown(); inclusion_lane_stop.request_shutdown(); let lane_result = inclusion_lane_handle.await; match lane_result { @@ -97,6 +131,7 @@ async fn main() -> Result<(), Box> { server_result?; } lane_result = &mut inclusion_lane_handle => { + broadcaster_shutdown.request_shutdown(); match lane_result { Ok(err) => return Err(format!("inclusion lane exited: {err}").into()), Err(join_err) => { @@ -109,11 +144,39 @@ async fn main() -> Result<(), Box> { Ok(()) } +fn handle_config_command(command: Command) -> Result<(), Box> { + match command { + Command::Config { + command: ConfigCommand::Print(args), + } => { + let config = Config::from_run_args(args).map_err(std::io::Error::other)?; + println!("{}", serde_json::to_string_pretty(&config.effective())?); + Ok(()) + } + Command::Config { + command: ConfigCommand::Validate(args), + } => { + let config = Config::from_run_args(args).map_err(std::io::Error::other)?; + println!("configuration is valid"); + println!("{}", serde_json::to_string_pretty(&config.effective())?); + Ok(()) + } + } +} + +fn log_effective_config(config: &Config) { + match serde_json::to_string(&config.effective()) { + Ok(json) => tracing::info!(effective_config = %json, "resolved sequencer config"), + Err(err) => tracing::warn!(%err, "failed to serialize effective sequencer config"), + } +} + struct Config { + profile: Profile, http_addr: String, db_path: String, queue_capacity: usize, - queue_timeout_ms: u64, + overload_max_inflight_submissions: usize, max_user_ops_per_chunk: usize, safe_direct_buffer_capacity: usize, max_batch_open: Duration, @@ -122,83 +185,376 @@ struct Config { broadcaster_idle_poll_interval: Duration, broadcaster_page_size: usize, broadcaster_subscriber_buffer_capacity: usize, + ws_max_subscribers: usize, + ws_max_catchup_events: u64, + runtime_metrics_enabled: bool, + runtime_metrics_log_interval: Duration, max_body_bytes: usize, - sqlite_synchronous: String, + sqlite_synchronous: SqliteSynchronous, domain_name: String, domain_version: String, domain_chain_id: u64, domain_verifying_contract: String, } +#[derive(Debug, Parser)] +#[command( + name = "sequencer", + about = "Deterministic sequencer prototype with low-latency soft confirmations", + version, + after_help = "Examples:\n sequencer --profile dev\n sequencer --profile bench --max-user-ops-per-chunk 4096 --max-batch-open 30m\n sequencer config print --profile safe\n sequencer config validate --sqlite-synchronous FULL" +)] +struct Cli { + #[command(subcommand)] + command: Option, + #[command(flatten)] + run: RunArgs, +} + +#[derive(Debug, Subcommand)] +enum Command { + Config { + #[command(subcommand)] + command: ConfigCommand, + }, +} + +#[derive(Debug, Subcommand)] +enum ConfigCommand { + Print(RunArgs), + Validate(RunArgs), +} + +#[derive(Debug, Clone, Args)] +struct RunArgs { + #[arg(long, env = "SEQ_PROFILE", value_enum, default_value_t = Profile::Dev)] + profile: Profile, + #[arg(long, env = "SEQ_HTTP_ADDR")] + http_addr: Option, + #[arg(long, env = "SEQ_DB_PATH")] + db_path: Option, + #[arg(long, env = "SEQ_QUEUE_CAP")] + queue_capacity: Option, + #[arg(long, env = "SEQ_OVERLOAD_MAX_INFLIGHT_SUBMISSIONS")] + overload_max_inflight_submissions: Option, + #[arg(long, env = "SEQ_MAX_USER_OPS_PER_CHUNK")] + max_user_ops_per_chunk: Option, + #[arg(long, env = "SEQ_MAX_BATCH", hide = true)] + legacy_max_batch: Option, + #[arg(long, env = "SEQ_SAFE_DIRECT_BUFFER_CAPACITY")] + safe_direct_buffer_capacity: Option, + #[arg( + long, + env = "SEQ_MAX_BATCH_OPEN_MS", + value_name = "DURATION", + value_parser = parse_duration_ms_or_unit + )] + max_batch_open: Option, + #[arg(long, env = "SEQ_MAX_BATCH_USER_OP_BYTES")] + max_batch_user_op_bytes: Option, + #[arg( + long, + env = "SEQ_INCLUSION_LANE_IDLE_POLL_INTERVAL_MS", + value_name = "DURATION", + value_parser = parse_duration_ms_or_unit + )] + inclusion_lane_idle_poll_interval: Option, + #[arg( + long, + env = "SEQ_INCLUSION_LANE_TICK_INTERVAL_MS", + hide = true, + value_parser = parse_duration_ms_or_unit + )] + legacy_inclusion_lane_tick_interval: Option, + #[arg( + long, + env = "SEQ_COMMIT_LANE_TICK_INTERVAL_MS", + hide = true, + value_parser = parse_duration_ms_or_unit + )] + legacy_commit_lane_tick_interval: Option, + #[arg( + long, + env = "SEQ_BROADCASTER_IDLE_POLL_INTERVAL_MS", + value_name = "DURATION", + value_parser = parse_duration_ms_or_unit + )] + broadcaster_idle_poll_interval: Option, + #[arg(long, env = "SEQ_BROADCASTER_PAGE_SIZE")] + broadcaster_page_size: Option, + #[arg(long, env = "SEQ_BROADCASTER_SUBSCRIBER_BUFFER_CAPACITY")] + broadcaster_subscriber_buffer_capacity: Option, + #[arg(long, env = "SEQ_WS_MAX_SUBSCRIBERS")] + ws_max_subscribers: Option, + #[arg(long, env = "SEQ_WS_MAX_CATCHUP_EVENTS")] + ws_max_catchup_events: Option, + #[arg(long, env = "SEQ_RUNTIME_METRICS_ENABLED")] + runtime_metrics_enabled: Option, + #[arg( + long, + env = "SEQ_RUNTIME_METRICS_LOG_INTERVAL_MS", + value_name = "DURATION", + value_parser = parse_duration_ms_or_unit + )] + runtime_metrics_log_interval: Option, + #[arg(long, env = "SEQ_MAX_BODY_BYTES")] + max_body_bytes: Option, + #[arg(long, env = "SEQ_SQLITE_SYNCHRONOUS", value_enum)] + sqlite_synchronous: Option, + #[arg(long, env = "SEQ_DOMAIN_NAME")] + domain_name: Option, + #[arg(long, env = "SEQ_DOMAIN_VERSION")] + domain_version: Option, + #[arg(long, env = "SEQ_DOMAIN_CHAIN_ID")] + domain_chain_id: Option, + #[arg(long, env = "SEQ_DOMAIN_VERIFYING_CONTRACT")] + domain_verifying_contract: Option, +} + +#[derive(Debug, Clone, Copy, Serialize, ValueEnum)] +#[serde(rename_all = "snake_case")] +enum Profile { + Dev, + Bench, + Safe, +} + +#[derive(Debug, Clone, Copy, Serialize, ValueEnum)] +#[serde(rename_all = "UPPERCASE")] +enum SqliteSynchronous { + #[value(name = "OFF", alias = "off")] + Off, + #[value(name = "NORMAL", alias = "normal")] + Normal, + #[value(name = "FULL", alias = "full")] + Full, + #[value(name = "EXTRA", alias = "extra")] + Extra, +} + +impl SqliteSynchronous { + fn pragma(self) -> &'static str { + match self { + Self::Off => "OFF", + Self::Normal => "NORMAL", + Self::Full => "FULL", + Self::Extra => "EXTRA", + } + } +} + +struct ProfileDefaults { + queue_capacity: usize, + max_user_ops_per_chunk: usize, + safe_direct_buffer_capacity: usize, + max_batch_open: Duration, + max_batch_user_op_bytes: usize, + inclusion_lane_idle_poll_interval: Duration, + broadcaster_idle_poll_interval: Duration, + broadcaster_page_size: usize, + broadcaster_subscriber_buffer_capacity: usize, + ws_max_subscribers: usize, + ws_max_catchup_events: u64, + runtime_metrics_enabled: bool, + runtime_metrics_log_interval: Duration, + max_body_bytes: usize, + sqlite_synchronous: SqliteSynchronous, +} + +impl Profile { + fn defaults(self) -> ProfileDefaults { + match self { + Self::Dev => ProfileDefaults { + queue_capacity: DEFAULT_QUEUE_CAP, + max_user_ops_per_chunk: DEFAULT_MAX_USER_OPS_PER_CHUNK, + safe_direct_buffer_capacity: DEFAULT_SAFE_DIRECT_BUFFER_CAPACITY, + max_batch_open: DEFAULT_MAX_BATCH_OPEN_DURATION, + max_batch_user_op_bytes: DEFAULT_MAX_BATCH_USER_OP_BYTES, + inclusion_lane_idle_poll_interval: DEFAULT_INCLUSION_LANE_IDLE_POLL_INTERVAL, + broadcaster_idle_poll_interval: DEFAULT_BROADCASTER_IDLE_POLL_INTERVAL, + broadcaster_page_size: DEFAULT_BROADCASTER_PAGE_SIZE, + broadcaster_subscriber_buffer_capacity: + DEFAULT_BROADCASTER_SUBSCRIBER_BUFFER_CAPACITY, + ws_max_subscribers: DEFAULT_WS_MAX_SUBSCRIBERS, + ws_max_catchup_events: DEFAULT_WS_MAX_CATCHUP_EVENTS, + runtime_metrics_enabled: DEFAULT_RUNTIME_METRICS_ENABLED, + runtime_metrics_log_interval: DEFAULT_RUNTIME_METRICS_LOG_INTERVAL, + max_body_bytes: DEFAULT_MAX_BODY_BYTES, + sqlite_synchronous: SqliteSynchronous::Normal, + }, + Self::Bench => ProfileDefaults { + queue_capacity: 32_768, + max_user_ops_per_chunk: 4_096, + safe_direct_buffer_capacity: 8_192, + max_batch_open: DEFAULT_MAX_BATCH_OPEN_DURATION, + max_batch_user_op_bytes: 1_572_864, // 1.5 MiB + inclusion_lane_idle_poll_interval: Duration::from_millis(1), + broadcaster_idle_poll_interval: Duration::from_millis(5), + broadcaster_page_size: 1_024, + broadcaster_subscriber_buffer_capacity: 131_072, + ws_max_subscribers: DEFAULT_WS_MAX_SUBSCRIBERS, + ws_max_catchup_events: DEFAULT_WS_MAX_CATCHUP_EVENTS, + runtime_metrics_enabled: DEFAULT_RUNTIME_METRICS_ENABLED, + runtime_metrics_log_interval: DEFAULT_RUNTIME_METRICS_LOG_INTERVAL, + max_body_bytes: DEFAULT_MAX_BODY_BYTES, + sqlite_synchronous: SqliteSynchronous::Normal, + }, + Self::Safe => ProfileDefaults { + queue_capacity: DEFAULT_QUEUE_CAP, + max_user_ops_per_chunk: DEFAULT_MAX_USER_OPS_PER_CHUNK, + safe_direct_buffer_capacity: DEFAULT_SAFE_DIRECT_BUFFER_CAPACITY, + max_batch_open: Duration::from_secs(60 * 60), + max_batch_user_op_bytes: DEFAULT_MAX_BATCH_USER_OP_BYTES, + inclusion_lane_idle_poll_interval: Duration::from_millis(5), + broadcaster_idle_poll_interval: Duration::from_millis(25), + broadcaster_page_size: DEFAULT_BROADCASTER_PAGE_SIZE, + broadcaster_subscriber_buffer_capacity: + DEFAULT_BROADCASTER_SUBSCRIBER_BUFFER_CAPACITY, + ws_max_subscribers: DEFAULT_WS_MAX_SUBSCRIBERS, + ws_max_catchup_events: DEFAULT_WS_MAX_CATCHUP_EVENTS, + runtime_metrics_enabled: DEFAULT_RUNTIME_METRICS_ENABLED, + runtime_metrics_log_interval: DEFAULT_RUNTIME_METRICS_LOG_INTERVAL, + max_body_bytes: DEFAULT_MAX_BODY_BYTES, + sqlite_synchronous: SqliteSynchronous::Full, + }, + } + } +} + impl Config { - fn from_env() -> Self { - Self { - http_addr: env_string("SEQ_HTTP_ADDR", DEFAULT_HTTP_ADDR), - db_path: env_string("SEQ_DB_PATH", DEFAULT_DB_PATH), - queue_capacity: env_usize("SEQ_QUEUE_CAP", DEFAULT_QUEUE_CAP).max(1), - queue_timeout_ms: env_u64("SEQ_QUEUE_TIMEOUT_MS", DEFAULT_QUEUE_TIMEOUT_MS), - max_user_ops_per_chunk: env_usize( - "SEQ_MAX_USER_OPS_PER_CHUNK", - env_usize("SEQ_MAX_BATCH", DEFAULT_MAX_USER_OPS_PER_CHUNK), - ) - .max(1), - safe_direct_buffer_capacity: env_usize( - "SEQ_SAFE_DIRECT_BUFFER_CAPACITY", - DEFAULT_SAFE_DIRECT_BUFFER_CAPACITY, - ) - .max(1), - max_batch_open: Duration::from_millis( - env_u64( - "SEQ_MAX_BATCH_OPEN_MS", - DEFAULT_MAX_BATCH_OPEN_DURATION.as_millis() as u64, - ) - .max(1), - ), - max_batch_user_op_bytes: env_usize( - "SEQ_MAX_BATCH_USER_OP_BYTES", - DEFAULT_MAX_BATCH_USER_OP_BYTES, - ) - .max(1), - inclusion_lane_idle_poll_interval: Duration::from_millis( - env_u64( - "SEQ_INCLUSION_LANE_IDLE_POLL_INTERVAL_MS", - env_u64( - "SEQ_INCLUSION_LANE_TICK_INTERVAL_MS", - env_u64( - "SEQ_COMMIT_LANE_TICK_INTERVAL_MS", - DEFAULT_INCLUSION_LANE_IDLE_POLL_INTERVAL.as_millis() as u64, - ), - ), - ) - .max(1), - ), - broadcaster_idle_poll_interval: Duration::from_millis( - env_u64( - "SEQ_BROADCASTER_IDLE_POLL_INTERVAL_MS", - DEFAULT_BROADCASTER_IDLE_POLL_INTERVAL.as_millis() as u64, - ) - .max(1), - ), - broadcaster_page_size: env_usize( - "SEQ_BROADCASTER_PAGE_SIZE", - DEFAULT_BROADCASTER_PAGE_SIZE, - ) - .max(1), - broadcaster_subscriber_buffer_capacity: env_usize( - "SEQ_BROADCASTER_SUBSCRIBER_BUFFER_CAPACITY", - DEFAULT_BROADCASTER_SUBSCRIBER_BUFFER_CAPACITY, - ) - .max(1), - max_body_bytes: env_usize("SEQ_MAX_BODY_BYTES", DEFAULT_MAX_BODY_BYTES), - sqlite_synchronous: env_string("SEQ_SQLITE_SYNCHRONOUS", DEFAULT_SQLITE_SYNCHRONOUS), - domain_name: env_string("SEQ_DOMAIN_NAME", DEFAULT_DOMAIN_NAME), - domain_version: env_string("SEQ_DOMAIN_VERSION", DEFAULT_DOMAIN_VERSION), - domain_chain_id: env_u64("SEQ_DOMAIN_CHAIN_ID", DEFAULT_DOMAIN_CHAIN_ID), - domain_verifying_contract: env_string( - "SEQ_DOMAIN_VERIFYING_CONTRACT", - DEFAULT_DOMAIN_VERIFYING_CONTRACT, - ), + fn from_run_args(args: RunArgs) -> Result { + let defaults = args.profile.defaults(); + let queue_capacity = args.queue_capacity.unwrap_or(defaults.queue_capacity); + let overload_max_inflight_submissions = args + .overload_max_inflight_submissions + .unwrap_or_else(|| default_overload_max_inflight_submissions(queue_capacity)); + + let config = Self { + profile: args.profile, + http_addr: args + .http_addr + .unwrap_or_else(|| DEFAULT_HTTP_ADDR.to_string()), + db_path: args.db_path.unwrap_or_else(|| DEFAULT_DB_PATH.to_string()), + queue_capacity, + overload_max_inflight_submissions, + max_user_ops_per_chunk: args + .max_user_ops_per_chunk + .or(args.legacy_max_batch) + .unwrap_or(defaults.max_user_ops_per_chunk), + safe_direct_buffer_capacity: args + .safe_direct_buffer_capacity + .unwrap_or(defaults.safe_direct_buffer_capacity), + max_batch_open: args.max_batch_open.unwrap_or(defaults.max_batch_open), + max_batch_user_op_bytes: args + .max_batch_user_op_bytes + .unwrap_or(defaults.max_batch_user_op_bytes), + inclusion_lane_idle_poll_interval: args + .inclusion_lane_idle_poll_interval + .or(args.legacy_inclusion_lane_tick_interval) + .or(args.legacy_commit_lane_tick_interval) + .unwrap_or(defaults.inclusion_lane_idle_poll_interval), + broadcaster_idle_poll_interval: args + .broadcaster_idle_poll_interval + .unwrap_or(defaults.broadcaster_idle_poll_interval), + broadcaster_page_size: args + .broadcaster_page_size + .unwrap_or(defaults.broadcaster_page_size), + broadcaster_subscriber_buffer_capacity: args + .broadcaster_subscriber_buffer_capacity + .unwrap_or(defaults.broadcaster_subscriber_buffer_capacity), + ws_max_subscribers: args + .ws_max_subscribers + .unwrap_or(defaults.ws_max_subscribers), + ws_max_catchup_events: args + .ws_max_catchup_events + .unwrap_or(defaults.ws_max_catchup_events), + runtime_metrics_enabled: args + .runtime_metrics_enabled + .unwrap_or(defaults.runtime_metrics_enabled), + runtime_metrics_log_interval: args + .runtime_metrics_log_interval + .unwrap_or(defaults.runtime_metrics_log_interval), + max_body_bytes: args.max_body_bytes.unwrap_or(defaults.max_body_bytes), + sqlite_synchronous: args + .sqlite_synchronous + .unwrap_or(defaults.sqlite_synchronous), + domain_name: args + .domain_name + .unwrap_or_else(|| DEFAULT_DOMAIN_NAME.to_string()), + domain_version: args + .domain_version + .unwrap_or_else(|| DEFAULT_DOMAIN_VERSION.to_string()), + domain_chain_id: args.domain_chain_id.unwrap_or(DEFAULT_DOMAIN_CHAIN_ID), + domain_verifying_contract: args + .domain_verifying_contract + .unwrap_or_else(|| DEFAULT_DOMAIN_VERIFYING_CONTRACT.to_string()), + }; + config.validate()?; + Ok(config) + } + + fn validate(&self) -> Result<(), String> { + if self.http_addr.trim().is_empty() { + return Err("http_addr cannot be empty".to_string()); + } + if self.db_path.trim().is_empty() { + return Err("db_path cannot be empty".to_string()); + } + if self.queue_capacity == 0 { + return Err("queue_capacity must be > 0".to_string()); + } + if self.overload_max_inflight_submissions == 0 { + return Err("overload_max_inflight_submissions must be > 0".to_string()); + } + if self.max_user_ops_per_chunk == 0 { + return Err("max_user_ops_per_chunk must be > 0".to_string()); } + if self.safe_direct_buffer_capacity == 0 { + return Err("safe_direct_buffer_capacity must be > 0".to_string()); + } + if self.max_batch_open.is_zero() { + return Err("max_batch_open must be > 0".to_string()); + } + if self.max_batch_user_op_bytes == 0 { + return Err("max_batch_user_op_bytes must be > 0".to_string()); + } + if self.max_batch_user_op_bytes < SignedUserOp::max_batch_bytes_upper_bound() { + return Err(format!( + "max_batch_user_op_bytes must be >= {} (one max-sized user op)", + SignedUserOp::max_batch_bytes_upper_bound() + )); + } + if self.inclusion_lane_idle_poll_interval.is_zero() { + return Err("inclusion_lane_idle_poll_interval must be > 0".to_string()); + } + if self.broadcaster_idle_poll_interval.is_zero() { + return Err("broadcaster_idle_poll_interval must be > 0".to_string()); + } + if self.broadcaster_page_size == 0 { + return Err("broadcaster_page_size must be > 0".to_string()); + } + if self.broadcaster_subscriber_buffer_capacity == 0 { + return Err("broadcaster_subscriber_buffer_capacity must be > 0".to_string()); + } + if self.ws_max_subscribers == 0 { + return Err("ws_max_subscribers must be > 0".to_string()); + } + if self.ws_max_catchup_events == 0 { + return Err("ws_max_catchup_events must be > 0".to_string()); + } + if self.runtime_metrics_log_interval.is_zero() { + return Err("runtime_metrics_log_interval must be > 0".to_string()); + } + if self.max_body_bytes == 0 { + return Err("max_body_bytes must be > 0".to_string()); + } + if self.domain_name.trim().is_empty() { + return Err("domain_name cannot be empty".to_string()); + } + if self.domain_version.trim().is_empty() { + return Err("domain_version cannot be empty".to_string()); + } + Ok(()) } fn build_domain(&self) -> Result { @@ -211,24 +567,80 @@ impl Config { salt: None, }) } + + fn effective(&self) -> EffectiveConfig { + EffectiveConfig { + profile: self.profile, + http_addr: self.http_addr.clone(), + db_path: self.db_path.clone(), + queue_capacity: self.queue_capacity, + overload_max_inflight_submissions: self.overload_max_inflight_submissions, + max_user_ops_per_chunk: self.max_user_ops_per_chunk, + safe_direct_buffer_capacity: self.safe_direct_buffer_capacity, + max_batch_open: DurationValue::from(self.max_batch_open), + max_batch_user_op_bytes: self.max_batch_user_op_bytes, + inclusion_lane_idle_poll_interval: DurationValue::from( + self.inclusion_lane_idle_poll_interval, + ), + broadcaster_idle_poll_interval: DurationValue::from( + self.broadcaster_idle_poll_interval, + ), + broadcaster_page_size: self.broadcaster_page_size, + broadcaster_subscriber_buffer_capacity: self.broadcaster_subscriber_buffer_capacity, + ws_max_subscribers: self.ws_max_subscribers, + ws_max_catchup_events: self.ws_max_catchup_events, + runtime_metrics_enabled: self.runtime_metrics_enabled, + runtime_metrics_log_interval: DurationValue::from(self.runtime_metrics_log_interval), + max_body_bytes: self.max_body_bytes, + sqlite_synchronous: self.sqlite_synchronous, + domain_name: self.domain_name.clone(), + domain_version: self.domain_version.clone(), + domain_chain_id: self.domain_chain_id, + domain_verifying_contract: self.domain_verifying_contract.clone(), + } + } } -fn env_string(key: &str, default: &str) -> String { - std::env::var(key).unwrap_or_else(|_| default.to_string()) +#[derive(Debug, Clone, Serialize)] +struct EffectiveConfig { + profile: Profile, + http_addr: String, + db_path: String, + queue_capacity: usize, + overload_max_inflight_submissions: usize, + max_user_ops_per_chunk: usize, + safe_direct_buffer_capacity: usize, + max_batch_open: DurationValue, + max_batch_user_op_bytes: usize, + inclusion_lane_idle_poll_interval: DurationValue, + broadcaster_idle_poll_interval: DurationValue, + broadcaster_page_size: usize, + broadcaster_subscriber_buffer_capacity: usize, + ws_max_subscribers: usize, + ws_max_catchup_events: u64, + runtime_metrics_enabled: bool, + runtime_metrics_log_interval: DurationValue, + max_body_bytes: usize, + sqlite_synchronous: SqliteSynchronous, + domain_name: String, + domain_version: String, + domain_chain_id: u64, + domain_verifying_contract: String, } -fn env_usize(key: &str, default: usize) -> usize { - std::env::var(key) - .ok() - .and_then(|value| value.parse().ok()) - .unwrap_or(default) +#[derive(Debug, Clone, Serialize)] +struct DurationValue { + ms: u64, + human: String, } -fn env_u64(key: &str, default: u64) -> u64 { - std::env::var(key) - .ok() - .and_then(|value| value.parse().ok()) - .unwrap_or(default) +impl From for DurationValue { + fn from(value: Duration) -> Self { + Self { + ms: duration_millis_u64(value), + human: format_duration(value), + } + } } fn parse_address(value: &str) -> Result { @@ -242,3 +654,83 @@ fn parse_address(value: &str) -> Result { } Ok(Address::from_slice(&bytes)) } + +fn parse_duration_ms_or_unit(raw: &str) -> Result { + let value = raw.trim(); + if value.is_empty() { + return Err("duration cannot be empty".to_string()); + } + + if let Ok(ms) = value.parse::() { + return Ok(Duration::from_millis(ms)); + } + + if let Some(ms) = value.strip_suffix("ms") { + let value = ms + .trim() + .parse::() + .map_err(|_| format!("invalid milliseconds duration: {raw}"))?; + return Ok(Duration::from_millis(value)); + } + if let Some(seconds) = value.strip_suffix('s') { + let value = seconds + .trim() + .parse::() + .map_err(|_| format!("invalid seconds duration: {raw}"))?; + return Ok(Duration::from_secs(value)); + } + if let Some(minutes) = value.strip_suffix('m') { + let value = minutes + .trim() + .parse::() + .map_err(|_| format!("invalid minutes duration: {raw}"))?; + let seconds = value + .checked_mul(60) + .ok_or_else(|| format!("duration overflow: {raw}"))?; + return Ok(Duration::from_secs(seconds)); + } + if let Some(hours) = value.strip_suffix('h') { + let value = hours + .trim() + .parse::() + .map_err(|_| format!("invalid hours duration: {raw}"))?; + let seconds = value + .checked_mul(60 * 60) + .ok_or_else(|| format!("duration overflow: {raw}"))?; + return Ok(Duration::from_secs(seconds)); + } + + Err(format!( + "invalid duration '{raw}'. Use plain milliseconds (e.g. 250) or suffix: ms, s, m, h" + )) +} + +fn format_duration(value: Duration) -> String { + let ms = duration_millis_u64(value); + if ms.is_multiple_of(60 * 60 * 1000) { + return format!("{}h", ms / (60 * 60 * 1000)); + } + if ms.is_multiple_of(60 * 1000) { + return format!("{}m", ms / (60 * 1000)); + } + if ms.is_multiple_of(1000) { + return format!("{}s", ms / 1000); + } + format!("{ms}ms") +} + +fn duration_millis_u64(value: Duration) -> u64 { + u64::try_from(value.as_millis()).unwrap_or(u64::MAX) +} + +fn force_zero_frame_fee_for_now( + storage: &mut storage::Storage, +) -> Result<(), Box> { + // Temporary prototype policy: keep sequencer frame fee at zero until fee estimation lands. + storage.set_recommended_fee(0)?; + let mut head = storage.load_open_state()?; + if head.frame_fee != 0 { + storage.close_frame_only(&mut head, 0, 0)?; + } + Ok(()) +} diff --git a/sequencer/src/storage/db.rs b/sequencer/src/storage/db.rs index 9132c13..5d1b490 100644 --- a/sequencer/src/storage/db.rs +++ b/sequencer/src/storage/db.rs @@ -6,21 +6,21 @@ use rusqlite_migration::{M, Migrations}; use std::time::{Duration, SystemTime, UNIX_EPOCH}; use super::sql::{ - sql_count_user_ops_for_frame, sql_insert_direct_inputs_batch, sql_insert_frame_drain, - sql_insert_open_batch, sql_insert_open_frame, sql_insert_user_ops_batch, - sql_select_latest_batch_with_user_op_count, sql_select_latest_frame_in_batch_for_batch, - sql_select_max_direct_input_index, sql_select_ordered_l2_tx_count, - sql_select_ordered_l2_txs_from_offset, sql_select_ordered_l2_txs_page_from_offset, - sql_select_recommended_fee, sql_select_safe_inputs_range, - sql_select_total_drained_direct_inputs, sql_update_recommended_fee, + sql_count_user_ops_for_frame, sql_insert_direct_inputs_batch, sql_insert_open_batch, + sql_insert_open_frame, sql_insert_sequenced_direct_inputs_for_frame, + sql_insert_user_ops_and_sequenced_batch, sql_select_latest_batch_with_user_op_count, + sql_select_latest_frame_in_batch_for_batch, sql_select_max_direct_input_index, + sql_select_ordered_l2_tx_count, sql_select_ordered_l2_txs_from_offset, + sql_select_ordered_l2_txs_page_from_offset, sql_select_recommended_fee, + sql_select_safe_inputs_range, sql_select_total_drained_direct_inputs, + sql_update_recommended_fee, }; use super::{IndexedDirectInput, StorageOpenError, WriteHead}; use crate::inclusion_lane::PendingUserOp; -use crate::l2_tx::{DirectInput, SequencedL2Tx, ValidUserOp}; use alloy_primitives::Address; +use sequencer_core::l2_tx::{DirectInput, SequencedL2Tx, ValidUserOp}; const MIGRATION_0001_SCHEMA: &str = include_str!("migrations/0001_schema.sql"); -const MIGRATION_0002_VIEWS: &str = include_str!("migrations/0002_views.sql"); pub struct Storage { conn: Connection, @@ -62,7 +62,8 @@ impl Storage { ) -> std::result::Result { let conn = Connection::open_with_flags(path, OpenFlags::SQLITE_OPEN_READ_ONLY)?; conn.pragma_update(None, "query_only", "ON")?; - conn.pragma_update(None, "busy_timeout", 5000)?; + // Readers should fail fast under write pressure to keep tail latency bounded. + conn.pragma_update(None, "busy_timeout", 50)?; Ok(conn) } @@ -76,8 +77,7 @@ impl Storage { } pub fn run_migrations(conn: &mut Connection) -> std::result::Result<(), StorageOpenError> { - Migrations::from_slice(&[M::up(MIGRATION_0001_SCHEMA), M::up(MIGRATION_0002_VIEWS)]) - .to_latest(conn)?; + Migrations::from_slice(&[M::up(MIGRATION_0001_SCHEMA)]).to_latest(conn)?; Ok(()) } @@ -206,7 +206,7 @@ impl Storage { let frame_user_op_count = query_frame_user_op_count(&tx, head.batch_index, head.frame_in_batch)?; - sql_insert_user_ops_batch( + sql_insert_user_ops_and_sequenced_batch( &tx, u64_to_i64(head.batch_index), i64::from(head.frame_in_batch), @@ -222,6 +222,7 @@ impl Storage { pub fn close_frame_only( &mut self, head: &mut WriteHead, + drained_direct_start_index: u64, drained_direct_count: usize, ) -> Result<()> { let tx = self @@ -229,17 +230,25 @@ impl Storage { .transaction_with_behavior(TransactionBehavior::Immediate)?; assert_write_head_matches_open_state(&tx, head)?; let now_ms = now_unix_ms(); - persist_frame_drain(&tx, head, drained_direct_count)?; + let next_frame_fee = query_recommended_fee(&tx)?; + persist_frame_direct_sequence(&tx, head, drained_direct_start_index, drained_direct_count)?; let next_frame_in_batch = head.frame_in_batch.saturating_add(1); - insert_open_frame(&tx, head.batch_index, next_frame_in_batch, now_ms)?; + insert_open_frame( + &tx, + head.batch_index, + next_frame_in_batch, + now_ms, + next_frame_fee, + )?; tx.commit()?; - head.advance_frame(); + head.advance_frame(next_frame_fee); Ok(()) } pub fn close_frame_and_batch( &mut self, head: &mut WriteHead, + drained_direct_start_index: u64, drained_direct_count: usize, ) -> Result<()> { let tx = self @@ -247,14 +256,14 @@ impl Storage { .transaction_with_behavior(TransactionBehavior::Immediate)?; assert_write_head_matches_open_state(&tx, head)?; let now_ms = now_unix_ms(); - // Batch fee is committed here: we sample the current recommendation once and - // assign it to the newly opened batch. - let next_batch_fee = query_recommended_fee(&tx)?; - persist_frame_drain(&tx, head, drained_direct_count)?; - let next_batch_index = insert_open_batch(&tx, now_ms, next_batch_fee)?; - insert_open_frame(&tx, next_batch_index, 0, now_ms)?; + // Frame fee is committed here: we sample the current recommendation once and + // assign it to the newly opened frame. + let next_frame_fee = query_recommended_fee(&tx)?; + persist_frame_direct_sequence(&tx, head, drained_direct_start_index, drained_direct_count)?; + let next_batch_index = insert_open_batch(&tx, now_ms)?; + insert_open_frame(&tx, next_batch_index, 0, now_ms, next_frame_fee)?; tx.commit()?; - head.move_to_next_batch(next_batch_index, from_unix_ms(now_ms), next_batch_fee); + head.move_to_next_batch(next_batch_index, from_unix_ms(now_ms), next_frame_fee); Ok(()) } @@ -301,7 +310,7 @@ fn decode_ordered_l2_txs(rows: Vec) -> Vec) -> Vec) -> Result { - let (batch_index, batch_created_at, batch_fee, batch_user_op_count) = query_latest_batch(tx)?; - let frame_in_batch = query_latest_frame_in_batch(tx, batch_index)?; + let (batch_index, batch_created_at, batch_user_op_count) = query_latest_batch(tx)?; + let (frame_in_batch, frame_fee) = query_latest_frame_in_batch(tx, batch_index)?; Ok(WriteHead { batch_index, batch_created_at, - batch_fee, + frame_fee, batch_user_op_count, frame_in_batch, }) @@ -344,8 +353,8 @@ fn assert_write_head_matches_open_state(tx: &Transaction<'_>, expected: &WriteHe "stale WriteHead: batch_user_op_count mismatch" ); assert_eq!( - expected.batch_fee, actual.batch_fee, - "stale WriteHead: batch_fee mismatch" + expected.frame_fee, actual.frame_fee, + "stale WriteHead: frame_fee mismatch" ); assert_eq!( to_unix_ms(expected.batch_created_at), @@ -355,20 +364,20 @@ fn assert_write_head_matches_open_state(tx: &Transaction<'_>, expected: &WriteHe Ok(()) } -fn query_latest_batch(tx: &Transaction<'_>) -> Result<(u64, SystemTime, u64, u64)> { - let (batch_index, batch_created_at_ms, batch_fee, batch_user_op_count) = +fn query_latest_batch(tx: &Transaction<'_>) -> Result<(u64, SystemTime, u64)> { + let (batch_index, batch_created_at_ms, batch_user_op_count) = sql_select_latest_batch_with_user_op_count(tx)?; Ok(( i64_to_u64(batch_index), from_unix_ms(batch_created_at_ms), - i64_to_u64(batch_fee), i64_to_u64(batch_user_op_count), )) } -fn query_latest_frame_in_batch(tx: &Transaction<'_>, batch_index: u64) -> Result { - let value = sql_select_latest_frame_in_batch_for_batch(tx, u64_to_i64(batch_index))?; - Ok(i64_to_u32(value)) +fn query_latest_frame_in_batch(tx: &Transaction<'_>, batch_index: u64) -> Result<(u32, u64)> { + let (frame_in_batch, frame_fee) = + sql_select_latest_frame_in_batch_for_batch(tx, u64_to_i64(batch_index))?; + Ok((i64_to_u32(frame_in_batch), i64_to_u64(frame_fee))) } fn query_frame_user_op_count( @@ -394,21 +403,23 @@ fn query_recommended_fee(tx: &Transaction<'_>) -> Result { Ok(i64_to_u64(value)) } -fn persist_frame_drain( +fn persist_frame_direct_sequence( tx: &Transaction<'_>, head: &WriteHead, + drained_direct_start_index: u64, drained_direct_count: usize, ) -> Result<()> { - sql_insert_frame_drain( + sql_insert_sequenced_direct_inputs_for_frame( tx, u64_to_i64(head.batch_index), i64::from(head.frame_in_batch), - u64_to_i64(drained_direct_count as u64), + drained_direct_start_index, + drained_direct_count, ) } -fn insert_open_batch(tx: &Transaction<'_>, created_at_ms: i64, fee: u64) -> Result { - sql_insert_open_batch(tx, created_at_ms, u64_to_i64(fee))?; +fn insert_open_batch(tx: &Transaction<'_>, created_at_ms: i64) -> Result { + sql_insert_open_batch(tx, created_at_ms)?; Ok(i64_to_u64(tx.last_insert_rowid())) } @@ -417,12 +428,14 @@ fn insert_open_frame( batch_index: u64, frame_in_batch: u32, created_at_ms: i64, + frame_fee: u64, ) -> Result<()> { sql_insert_open_frame( tx, u64_to_i64(batch_index), i64::from(frame_in_batch), created_at_ms, + u64_to_i64(frame_fee), )?; Ok(()) } @@ -463,74 +476,76 @@ fn i64_to_u32(value: i64) -> u32 { #[cfg(test)] mod tests { use super::Storage; - use crate::l2_tx::SequencedL2Tx; use crate::storage::IndexedDirectInput; - use std::path::PathBuf; - use std::time::{SystemTime, UNIX_EPOCH}; - - fn temp_db_path(name: &str) -> String { - let mut path = std::env::temp_dir(); - let unique = SystemTime::now() - .duration_since(UNIX_EPOCH) - .unwrap_or_default() - .as_nanos(); - path.push(format!("sequencer-{name}-{unique}.sqlite")); - path_to_string(path) + use sequencer_core::l2_tx::SequencedL2Tx; + use tempfile::TempDir; + + struct TestDb { + _dir: TempDir, + path: String, } - fn path_to_string(path: PathBuf) -> String { - path.to_string_lossy().into_owned() + fn temp_db(name: &str) -> TestDb { + let dir = tempfile::Builder::new() + .prefix(format!("sequencer-{name}-").as_str()) + .tempdir() + .expect("create temporary test directory"); + let path = dir.path().join("sequencer.sqlite"); + TestDb { + _dir: dir, + path: path.to_string_lossy().into_owned(), + } } #[test] fn open_state_is_idempotent_and_rotation_is_atomic() { - let db_path = temp_db_path("open-state"); - let mut storage = Storage::open(&db_path, "NORMAL").expect("open storage"); + let db = temp_db("open-state"); + let mut storage = Storage::open(db.path.as_str(), "NORMAL").expect("open storage"); let head_a = storage.load_open_state().expect("load open state"); let head_b = storage.load_open_state().expect("load existing open state"); assert_eq!(head_a.batch_index, head_b.batch_index); assert_eq!(head_a.frame_in_batch, head_b.frame_in_batch); - assert_eq!(head_a.batch_fee, head_b.batch_fee); - assert_eq!(head_a.batch_fee, 1); + assert_eq!(head_a.frame_fee, head_b.frame_fee); + assert_eq!(head_a.frame_fee, 0); let mut head_c = head_b; storage - .close_frame_only(&mut head_c, 0) + .close_frame_only(&mut head_c, 0, 0) .expect("rotate within same batch"); assert_eq!(head_c.batch_index, head_b.batch_index); assert_eq!(head_c.frame_in_batch, 1); let mut head_d = head_c; storage - .close_frame_and_batch(&mut head_d, 0) + .close_frame_and_batch(&mut head_d, 0, 0) .expect("close batch and rotate"); assert!(head_d.batch_index > head_c.batch_index); assert_eq!(head_d.frame_in_batch, 0); } #[test] - fn next_batch_fee_comes_from_recommended_fee_singleton() { - let db_path = temp_db_path("recommended-fee"); - let mut storage = Storage::open(&db_path, "NORMAL").expect("open storage"); - assert_eq!(storage.recommended_fee().expect("default recommended"), 1); + fn next_frame_fee_comes_from_recommended_fee_singleton() { + let db = temp_db("recommended-fee"); + let mut storage = Storage::open(db.path.as_str(), "NORMAL").expect("open storage"); + assert_eq!(storage.recommended_fee().expect("default recommended"), 0); storage.set_recommended_fee(7).expect("set recommended fee"); let mut head = storage.load_open_state().expect("load open state"); storage - .close_frame_and_batch(&mut head, 0) + .close_frame_and_batch(&mut head, 0, 0) .expect("rotate batch"); - assert_eq!(head.batch_fee, 7); + assert_eq!(head.frame_fee, 7); assert_eq!(storage.recommended_fee().expect("read recommended"), 7); } #[test] fn replay_returns_direct_inputs_in_drain_order() { - let db_path = temp_db_path("replay-order"); - let mut storage = Storage::open(&db_path, "NORMAL").expect("open storage"); + let db = temp_db("replay-order"); + let mut storage = Storage::open(db.path.as_str(), "NORMAL").expect("open storage"); let head = storage.load_open_state().expect("load open state"); let drained = vec![ @@ -548,7 +563,7 @@ mod tests { .expect("insert direct inputs"); let mut head = head; storage - .close_frame_only(&mut head, drained.len()) + .close_frame_only(&mut head, 0, drained.len()) .expect("close frame with directs"); let replay = storage.load_ordered_l2_txs_from(0).expect("load replay"); @@ -564,9 +579,9 @@ mod tests { } #[test] - fn next_undrained_direct_input_index_is_derived_from_frame_drains() { - let db_path = temp_db_path("safe-cursor"); - let mut storage = Storage::open(&db_path, "NORMAL").expect("open storage"); + fn next_undrained_direct_input_index_is_derived_from_sequenced_directs() { + let db = temp_db("safe-cursor"); + let mut storage = Storage::open(db.path.as_str(), "NORMAL").expect("open storage"); assert_eq!( storage .load_next_undrained_direct_input_index() @@ -590,7 +605,7 @@ mod tests { .expect("insert direct inputs"); let mut head = head; storage - .close_frame_only(&mut head, drained.len()) + .close_frame_only(&mut head, 0, drained.len()) .expect("close frame with directs"); assert_eq!( @@ -603,8 +618,8 @@ mod tests { #[test] fn safe_input_api_uses_half_open_intervals() { - let db_path = temp_db_path("safe-input-api"); - let mut storage = Storage::open(&db_path, "NORMAL").expect("open storage"); + let db = temp_db("safe-input-api"); + let mut storage = Storage::open(db.path.as_str(), "NORMAL").expect("open storage"); assert_eq!(storage.safe_input_end_exclusive().expect("safe head"), 0); let mut out = Vec::new(); diff --git a/sequencer/src/storage/migrations/0001_schema.sql b/sequencer/src/storage/migrations/0001_schema.sql index ee237f2..1a9818f 100644 --- a/sequencer/src/storage/migrations/0001_schema.sql +++ b/sequencer/src/storage/migrations/0001_schema.sql @@ -1,14 +1,14 @@ CREATE TABLE IF NOT EXISTS batches ( batch_index INTEGER PRIMARY KEY, - created_at_ms INTEGER NOT NULL, - -- Fee committed by the sequencer for this whole batch. - fee INTEGER NOT NULL CHECK (fee >= 0) + created_at_ms INTEGER NOT NULL ); CREATE TABLE IF NOT EXISTS frames ( batch_index INTEGER NOT NULL REFERENCES batches(batch_index), frame_in_batch INTEGER NOT NULL, created_at_ms INTEGER NOT NULL, + -- Fee committed by the sequencer for this whole frame. + fee INTEGER NOT NULL CHECK (fee >= 0), PRIMARY KEY(batch_index, frame_in_batch) ); @@ -16,7 +16,6 @@ CREATE TABLE IF NOT EXISTS user_ops ( batch_index INTEGER NOT NULL, frame_in_batch INTEGER NOT NULL, pos_in_frame INTEGER NOT NULL, - tx_hash BLOB NOT NULL, sender BLOB NOT NULL, nonce INTEGER NOT NULL, max_fee INTEGER NOT NULL, @@ -25,7 +24,7 @@ CREATE TABLE IF NOT EXISTS user_ops ( received_at_ms INTEGER NOT NULL, PRIMARY KEY(batch_index, frame_in_batch, pos_in_frame), FOREIGN KEY(batch_index, frame_in_batch) REFERENCES frames(batch_index, frame_in_batch), - UNIQUE(tx_hash) + UNIQUE(sender, nonce) ); CREATE TABLE IF NOT EXISTS direct_inputs ( @@ -33,33 +32,51 @@ CREATE TABLE IF NOT EXISTS direct_inputs ( payload BLOB NOT NULL ); -CREATE TABLE IF NOT EXISTS frame_drains ( - batch_index INTEGER NOT NULL, - frame_in_batch INTEGER NOT NULL, - drain_n INTEGER NOT NULL, - PRIMARY KEY(batch_index, frame_in_batch), - FOREIGN KEY(batch_index, frame_in_batch) REFERENCES frames(batch_index, frame_in_batch), - CHECK(drain_n >= 0) +CREATE TABLE IF NOT EXISTS sequenced_l2_txs ( + -- Global append-only replay order consumed by catch-up and broadcaster. + offset INTEGER PRIMARY KEY, + batch_index INTEGER NOT NULL, + frame_in_batch INTEGER NOT NULL, + + -- User-op branch: references user_ops(..., pos_in_frame). + user_op_pos_in_frame INTEGER, + + -- Direct-input branch: references direct_inputs(direct_input_index). + direct_input_index INTEGER, + + FOREIGN KEY(batch_index, frame_in_batch) + REFERENCES frames(batch_index, frame_in_batch), + FOREIGN KEY(batch_index, frame_in_batch, user_op_pos_in_frame) + REFERENCES user_ops(batch_index, frame_in_batch, pos_in_frame), + FOREIGN KEY(direct_input_index) + REFERENCES direct_inputs(direct_input_index), + + -- XOR invariant: row is either a sequenced user-op OR a drained direct input. + CHECK ( + (user_op_pos_in_frame IS NOT NULL AND direct_input_index IS NULL) OR + (user_op_pos_in_frame IS NULL AND direct_input_index IS NOT NULL) + ), + + -- At most one sequenced user-op row for each user-op key. + UNIQUE(batch_index, frame_in_batch, user_op_pos_in_frame), + -- A direct input can only be sequenced once. + UNIQUE(direct_input_index) ); -CREATE INDEX IF NOT EXISTS idx_frames_batch_order - ON frames(batch_index, frame_in_batch); -CREATE INDEX IF NOT EXISTS idx_user_ops_frame_pos - ON user_ops(batch_index, frame_in_batch, pos_in_frame); -CREATE INDEX IF NOT EXISTS idx_frame_drains_frame - ON frame_drains(batch_index, frame_in_batch); +CREATE INDEX IF NOT EXISTS idx_sequenced_l2_txs_frame + ON sequenced_l2_txs(batch_index, frame_in_batch); CREATE TABLE IF NOT EXISTS recommended_fees ( singleton_id INTEGER PRIMARY KEY CHECK (singleton_id = 0), - -- Mutable recommendation consumed when opening the next batch. + -- Mutable recommendation consumed when opening the next frame. fee INTEGER NOT NULL CHECK (fee >= 0) ); INSERT OR IGNORE INTO recommended_fees (singleton_id, fee) -VALUES (0, 1); +VALUES (0, 0); -INSERT OR IGNORE INTO batches (batch_index, created_at_ms, fee) -VALUES (0, 0, 1); +INSERT OR IGNORE INTO batches (batch_index, created_at_ms) +VALUES (0, 0); -INSERT OR IGNORE INTO frames (batch_index, frame_in_batch, created_at_ms) -VALUES (0, 0, 0); +INSERT OR IGNORE INTO frames (batch_index, frame_in_batch, created_at_ms, fee) +VALUES (0, 0, 0, 0); diff --git a/sequencer/src/storage/migrations/0002_views.sql b/sequencer/src/storage/migrations/0002_views.sql deleted file mode 100644 index 1fe26a0..0000000 --- a/sequencer/src/storage/migrations/0002_views.sql +++ /dev/null @@ -1,66 +0,0 @@ -CREATE VIEW IF NOT EXISTS frame_drain_ranges AS -SELECT - batch_index, - frame_in_batch, - COALESCE( - SUM(drain_n) OVER ( - ORDER BY batch_index, frame_in_batch - ROWS BETWEEN UNBOUNDED PRECEDING AND 1 PRECEDING - ), - 0 - ) AS direct_start_index, - drain_n -FROM frame_drains; - -CREATE VIEW IF NOT EXISTS ordered_sequenced_l2_txs AS -SELECT - u.batch_index AS batch_index, - u.frame_in_batch AS frame_in_batch, - 0 AS kind, - u.pos_in_frame AS pos, - u.sender AS sender, - u.data AS data, - b.fee AS fee, - NULL AS payload -FROM user_ops u -JOIN batches b - ON b.batch_index = u.batch_index - -UNION ALL - -SELECT - d.batch_index AS batch_index, - d.frame_in_batch AS frame_in_batch, - 1 AS kind, - (i.direct_input_index - d.direct_start_index) AS pos, - NULL AS sender, - NULL AS data, - NULL AS fee, - i.payload AS payload -FROM frame_drain_ranges d -JOIN direct_inputs i - ON i.direct_input_index >= d.direct_start_index - AND i.direct_input_index < (d.direct_start_index + d.drain_n); - -CREATE VIEW IF NOT EXISTS batch_user_op_counts AS -SELECT - b.batch_index AS batch_index, - b.created_at_ms AS created_at_ms, - b.fee AS fee, - COALESCE(c.user_op_count, 0) AS user_op_count -FROM batches b -LEFT JOIN ( - SELECT - u.batch_index AS batch_index, - COUNT(*) AS user_op_count - FROM user_ops u - GROUP BY u.batch_index -) c ON c.batch_index = b.batch_index; - -CREATE VIEW IF NOT EXISTS frame_user_op_counts AS -SELECT - u.batch_index AS batch_index, - u.frame_in_batch AS frame_in_batch, - COUNT(*) AS user_op_count -FROM user_ops u -GROUP BY u.batch_index, u.frame_in_batch; diff --git a/sequencer/src/storage/mod.rs b/sequencer/src/storage/mod.rs index 1af4fb7..9994c1e 100644 --- a/sequencer/src/storage/mod.rs +++ b/sequencer/src/storage/mod.rs @@ -27,8 +27,8 @@ pub enum StorageOpenError { pub struct WriteHead { pub batch_index: u64, pub batch_created_at: SystemTime, - // Sequencer-chosen fee committed for this open batch. - pub batch_fee: u64, + // Sequencer-chosen fee committed for this open frame. + pub frame_fee: u64, pub batch_user_op_count: u64, pub frame_in_batch: u32, } @@ -38,19 +38,20 @@ impl WriteHead { self.batch_user_op_count = self.batch_user_op_count.saturating_add(count as u64); } - pub fn advance_frame(&mut self) { + pub fn advance_frame(&mut self, frame_fee: u64) { self.frame_in_batch = self.frame_in_batch.saturating_add(1); + self.frame_fee = frame_fee; } pub fn move_to_next_batch( &mut self, batch_index: u64, batch_created_at: SystemTime, - batch_fee: u64, + frame_fee: u64, ) { self.batch_index = batch_index; self.batch_created_at = batch_created_at; - self.batch_fee = batch_fee; + self.frame_fee = frame_fee; self.batch_user_op_count = 0; self.frame_in_batch = 0; } diff --git a/sequencer/src/storage/queries/insert_sequenced_direct_input.sql b/sequencer/src/storage/queries/insert_sequenced_direct_input.sql new file mode 100644 index 0000000..0c34ee2 --- /dev/null +++ b/sequencer/src/storage/queries/insert_sequenced_direct_input.sql @@ -0,0 +1,6 @@ +INSERT INTO sequenced_l2_txs ( + batch_index, + frame_in_batch, + user_op_pos_in_frame, + direct_input_index +) VALUES (?1, ?2, NULL, ?3) diff --git a/sequencer/src/storage/queries/insert_sequenced_user_op.sql b/sequencer/src/storage/queries/insert_sequenced_user_op.sql new file mode 100644 index 0000000..53eb402 --- /dev/null +++ b/sequencer/src/storage/queries/insert_sequenced_user_op.sql @@ -0,0 +1,6 @@ +INSERT INTO sequenced_l2_txs ( + batch_index, + frame_in_batch, + user_op_pos_in_frame, + direct_input_index +) VALUES (?1, ?2, ?3, NULL) diff --git a/sequencer/src/storage/queries/insert_user_op.sql b/sequencer/src/storage/queries/insert_user_op.sql index 9defa87..d86a72a 100644 --- a/sequencer/src/storage/queries/insert_user_op.sql +++ b/sequencer/src/storage/queries/insert_user_op.sql @@ -2,11 +2,10 @@ INSERT INTO user_ops ( batch_index, frame_in_batch, pos_in_frame, - tx_hash, sender, nonce, max_fee, data, sig, received_at_ms -) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10) +) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9) diff --git a/sequencer/src/storage/queries/select_latest_batch_with_user_op_count.sql b/sequencer/src/storage/queries/select_latest_batch_with_user_op_count.sql index 3782f48..ca7f9d0 100644 --- a/sequencer/src/storage/queries/select_latest_batch_with_user_op_count.sql +++ b/sequencer/src/storage/queries/select_latest_batch_with_user_op_count.sql @@ -1,8 +1,11 @@ SELECT - batch_index, - created_at_ms, - fee, - user_op_count -FROM batch_user_op_counts -ORDER BY batch_index DESC + b.batch_index, + b.created_at_ms, + ( + SELECT COUNT(*) + FROM user_ops u + WHERE u.batch_index = b.batch_index + ) AS user_op_count +FROM batches b +ORDER BY b.batch_index DESC LIMIT 1 diff --git a/sequencer/src/storage/queries/select_latest_frame_in_batch_for_batch.sql b/sequencer/src/storage/queries/select_latest_frame_in_batch_for_batch.sql index 4199113..759aff6 100644 --- a/sequencer/src/storage/queries/select_latest_frame_in_batch_for_batch.sql +++ b/sequencer/src/storage/queries/select_latest_frame_in_batch_for_batch.sql @@ -1,4 +1,6 @@ -SELECT f.frame_in_batch +SELECT + f.frame_in_batch, + f.fee FROM frames f WHERE f.batch_index = ?1 ORDER BY f.frame_in_batch DESC diff --git a/sequencer/src/storage/queries/select_ordered_l2_txs_from_offset.sql b/sequencer/src/storage/queries/select_ordered_l2_txs_from_offset.sql index 11190f6..941fcd6 100644 --- a/sequencer/src/storage/queries/select_ordered_l2_txs_from_offset.sql +++ b/sequencer/src/storage/queries/select_ordered_l2_txs_from_offset.sql @@ -1,5 +1,18 @@ -SELECT kind, sender, data, fee, payload -FROM ordered_sequenced_l2_txs --- `kind ASC` guarantees user_ops (0) are replayed before drained directs (1) in each frame. -ORDER BY batch_index ASC, frame_in_batch ASC, kind ASC, pos ASC -LIMIT -1 OFFSET ?1 +SELECT + CASE WHEN s.user_op_pos_in_frame IS NOT NULL THEN 0 ELSE 1 END AS kind, + CASE WHEN s.user_op_pos_in_frame IS NOT NULL THEN u.sender ELSE NULL END AS sender, + CASE WHEN s.user_op_pos_in_frame IS NOT NULL THEN u.data ELSE NULL END AS data, + CASE WHEN s.user_op_pos_in_frame IS NOT NULL THEN f.fee ELSE NULL END AS fee, + CASE WHEN s.direct_input_index IS NOT NULL THEN d.payload ELSE NULL END AS payload +FROM sequenced_l2_txs s +LEFT JOIN user_ops u + ON u.batch_index = s.batch_index + AND u.frame_in_batch = s.frame_in_batch + AND u.pos_in_frame = s.user_op_pos_in_frame +LEFT JOIN frames f + ON f.batch_index = s.batch_index + AND f.frame_in_batch = s.frame_in_batch +LEFT JOIN direct_inputs d + ON d.direct_input_index = s.direct_input_index +WHERE s.offset >= ?1 +ORDER BY s.offset ASC diff --git a/sequencer/src/storage/queries/select_ordered_l2_txs_page_from_offset.sql b/sequencer/src/storage/queries/select_ordered_l2_txs_page_from_offset.sql index eda939b..ded3cce 100644 --- a/sequencer/src/storage/queries/select_ordered_l2_txs_page_from_offset.sql +++ b/sequencer/src/storage/queries/select_ordered_l2_txs_page_from_offset.sql @@ -1,5 +1,19 @@ -SELECT kind, sender, data, fee, payload -FROM ordered_sequenced_l2_txs --- `kind ASC` guarantees user_ops (0) are replayed before drained directs (1) in each frame. -ORDER BY batch_index ASC, frame_in_batch ASC, kind ASC, pos ASC -LIMIT ?2 OFFSET ?1 +SELECT + CASE WHEN s.user_op_pos_in_frame IS NOT NULL THEN 0 ELSE 1 END AS kind, + CASE WHEN s.user_op_pos_in_frame IS NOT NULL THEN u.sender ELSE NULL END AS sender, + CASE WHEN s.user_op_pos_in_frame IS NOT NULL THEN u.data ELSE NULL END AS data, + CASE WHEN s.user_op_pos_in_frame IS NOT NULL THEN f.fee ELSE NULL END AS fee, + CASE WHEN s.direct_input_index IS NOT NULL THEN d.payload ELSE NULL END AS payload +FROM sequenced_l2_txs s +LEFT JOIN user_ops u + ON u.batch_index = s.batch_index + AND u.frame_in_batch = s.frame_in_batch + AND u.pos_in_frame = s.user_op_pos_in_frame +LEFT JOIN frames f + ON f.batch_index = s.batch_index + AND f.frame_in_batch = s.frame_in_batch +LEFT JOIN direct_inputs d + ON d.direct_input_index = s.direct_input_index +WHERE s.offset >= ?1 +ORDER BY s.offset ASC +LIMIT ?2 diff --git a/sequencer/src/storage/queries/select_user_op_count_for_frame.sql b/sequencer/src/storage/queries/select_user_op_count_for_frame.sql index 8774330..e28ada7 100644 --- a/sequencer/src/storage/queries/select_user_op_count_for_frame.sql +++ b/sequencer/src/storage/queries/select_user_op_count_for_frame.sql @@ -1,9 +1,3 @@ -SELECT - COALESCE( - ( - SELECT user_op_count - FROM frame_user_op_counts - WHERE batch_index = ?1 AND frame_in_batch = ?2 - ), - 0 - ) +SELECT COUNT(*) +FROM user_ops +WHERE batch_index = ?1 AND frame_in_batch = ?2 diff --git a/sequencer/src/storage/sql.rs b/sequencer/src/storage/sql.rs index de4dfc7..a9a122f 100644 --- a/sequencer/src/storage/sql.rs +++ b/sequencer/src/storage/sql.rs @@ -19,14 +19,15 @@ const SQL_SELECT_LATEST_FRAME_IN_BATCH_FOR_BATCH: &str = const SQL_SELECT_USER_OP_COUNT_FOR_FRAME: &str = include_str!("queries/select_user_op_count_for_frame.sql"); const SQL_SELECT_MAX_DIRECT_INPUT_INDEX: &str = "SELECT MAX(direct_input_index) FROM direct_inputs"; -const SQL_SELECT_ORDERED_L2_TX_COUNT: &str = "SELECT COUNT(*) FROM ordered_sequenced_l2_txs"; +const SQL_SELECT_ORDERED_L2_TX_COUNT: &str = "SELECT COUNT(*) FROM sequenced_l2_txs"; const SQL_SELECT_RECOMMENDED_FEE: &str = "SELECT fee FROM recommended_fees WHERE singleton_id = 0 LIMIT 1"; const SQL_INSERT_DIRECT_INPUT: &str = "INSERT INTO direct_inputs (direct_input_index, payload) VALUES (?1, ?2)"; const SQL_INSERT_USER_OP: &str = include_str!("queries/insert_user_op.sql"); -const SQL_INSERT_FRAME_DRAIN: &str = - "INSERT INTO frame_drains (batch_index, frame_in_batch, drain_n) VALUES (?1, ?2, ?3)"; +const SQL_INSERT_SEQUENCED_USER_OP: &str = include_str!("queries/insert_sequenced_user_op.sql"); +const SQL_INSERT_SEQUENCED_DIRECT_INPUT: &str = + include_str!("queries/insert_sequenced_direct_input.sql"); const SQL_UPDATE_RECOMMENDED_FEE: &str = "UPDATE recommended_fees SET fee = ?1 WHERE singleton_id = 0"; @@ -46,7 +47,7 @@ pub(super) struct SafeInputRow { } pub(super) fn sql_select_total_drained_direct_inputs(conn: &Connection) -> Result { - const SQL: &str = "SELECT COALESCE(SUM(drain_n), 0) FROM frame_drains"; + const SQL: &str = "SELECT COUNT(*) FROM sequenced_l2_txs WHERE direct_input_index IS NOT NULL"; conn.query_row(SQL, [], |row| row.get(0)) } @@ -94,7 +95,7 @@ pub(super) fn sql_insert_direct_inputs_batch( Ok(()) } -pub(super) fn sql_insert_user_ops_batch( +pub(super) fn sql_insert_user_ops_and_sequenced_batch( tx: &Transaction<'_>, batch_index: i64, frame_in_batch: i64, @@ -105,15 +106,15 @@ pub(super) fn sql_insert_user_ops_batch( return Ok(()); } - let mut stmt = tx.prepare_cached(SQL_INSERT_USER_OP)?; + let mut user_ops_stmt = tx.prepare_cached(SQL_INSERT_USER_OP)?; + let mut sequenced_stmt = tx.prepare_cached(SQL_INSERT_SEQUENCED_USER_OP)?; for (offset, item) in user_ops.iter().enumerate() { let pos_in_frame = frame_pos_start.saturating_add(offset as u32); let sig = item.signed.signature.as_bytes(); - stmt.execute(params![ + user_ops_stmt.execute(params![ batch_index, frame_in_batch, i64::from(pos_in_frame), - item.tx_hash.as_slice(), item.signed.sender.as_slice(), i64::from(item.signed.user_op.nonce), i64::from(item.signed.user_op.max_fee), @@ -121,6 +122,34 @@ pub(super) fn sql_insert_user_ops_batch( &sig[..], to_unix_ms(item.received_at), ])?; + sequenced_stmt.execute(params![ + batch_index, + frame_in_batch, + i64::from(pos_in_frame), + ])?; + } + Ok(()) +} + +pub(super) fn sql_insert_sequenced_direct_inputs( + tx: &Transaction<'_>, + batch_index: i64, + frame_in_batch: i64, + direct_start_index: u64, + direct_input_count: usize, +) -> Result<()> { + if direct_input_count == 0 { + return Ok(()); + } + + let mut stmt = tx.prepare_cached(SQL_INSERT_SEQUENCED_DIRECT_INPUT)?; + for offset in 0..direct_input_count { + let direct_input_index = direct_start_index.saturating_add(offset as u64); + stmt.execute(params![ + batch_index, + frame_in_batch, + u64_to_i64(direct_input_index), + ])?; } Ok(()) } @@ -150,7 +179,7 @@ pub(super) fn sql_select_ordered_l2_tx_count(conn: &Connection) -> Result { pub(super) fn sql_select_latest_batch_with_user_op_count( tx: &Transaction<'_>, -) -> Result<(i64, i64, i64, i64)> { +) -> Result<(i64, i64, i64)> { tx.query_row( SQL_SELECT_LATEST_BATCH_WITH_USER_OP_COUNT, [], @@ -161,11 +190,11 @@ pub(super) fn sql_select_latest_batch_with_user_op_count( pub(super) fn sql_select_latest_frame_in_batch_for_batch( tx: &Transaction<'_>, batch_index: i64, -) -> Result { +) -> Result<(i64, i64)> { tx.query_row( SQL_SELECT_LATEST_FRAME_IN_BATCH_FOR_BATCH, params![batch_index], - |row| row.get(0), + |row| Ok((row.get(0)?, row.get(1)?)), ) } @@ -181,24 +210,25 @@ pub(super) fn sql_count_user_ops_for_frame( ) } -pub(super) fn sql_insert_frame_drain( +pub(super) fn sql_insert_sequenced_direct_inputs_for_frame( tx: &Transaction<'_>, batch_index: i64, frame_in_batch: i64, - drain_n: i64, + direct_start_index: u64, + direct_input_count: usize, ) -> Result<()> { - let mut stmt = tx.prepare_cached(SQL_INSERT_FRAME_DRAIN)?; - stmt.execute(params![batch_index, frame_in_batch, drain_n])?; - Ok(()) + sql_insert_sequenced_direct_inputs( + tx, + batch_index, + frame_in_batch, + direct_start_index, + direct_input_count, + ) } -pub(super) fn sql_insert_open_batch( - tx: &Transaction<'_>, - created_at_ms: i64, - fee: i64, -) -> Result { - const SQL: &str = "INSERT INTO batches (created_at_ms, fee) VALUES (?1, ?2)"; - tx.execute(SQL, params![created_at_ms, fee]) +pub(super) fn sql_insert_open_batch(tx: &Transaction<'_>, created_at_ms: i64) -> Result { + const SQL: &str = "INSERT INTO batches (created_at_ms) VALUES (?1)"; + tx.execute(SQL, params![created_at_ms]) } pub(super) fn sql_insert_open_frame( @@ -206,10 +236,13 @@ pub(super) fn sql_insert_open_frame( batch_index: i64, frame_in_batch: i64, created_at_ms: i64, + fee: i64, ) -> Result { - const SQL: &str = - "INSERT INTO frames (batch_index, frame_in_batch, created_at_ms) VALUES (?1, ?2, ?3)"; - tx.execute(SQL, params![batch_index, frame_in_batch, created_at_ms]) + const SQL: &str = "INSERT INTO frames (batch_index, frame_in_batch, created_at_ms, fee) VALUES (?1, ?2, ?3, ?4)"; + tx.execute( + SQL, + params![batch_index, frame_in_batch, created_at_ms, fee], + ) } fn convert_row_to_optional_i64(row: &Row<'_>) -> Result> { @@ -233,8 +266,8 @@ fn convert_row_to_ordered_l2_tx_row(row: &Row<'_>) -> Result { }) } -fn convert_row_to_latest_batch_with_user_op_count(row: &Row<'_>) -> Result<(i64, i64, i64, i64)> { - Ok((row.get(0)?, row.get(1)?, row.get(2)?, row.get(3)?)) +fn convert_row_to_latest_batch_with_user_op_count(row: &Row<'_>) -> Result<(i64, i64, i64)> { + Ok((row.get(0)?, row.get(1)?, row.get(2)?)) } fn to_unix_ms(time: SystemTime) -> i64 { @@ -252,21 +285,22 @@ fn u64_to_i64(value: u64) -> i64 { #[cfg(test)] mod tests { use super::{ - SQL_INSERT_DIRECT_INPUT, SQL_INSERT_FRAME_DRAIN, SQL_INSERT_USER_OP, - sql_count_user_ops_for_frame, sql_insert_direct_inputs_batch, sql_insert_frame_drain, - sql_insert_open_batch, sql_insert_open_frame, sql_insert_user_ops_batch, - sql_select_latest_batch_with_user_op_count, sql_select_latest_frame_in_batch_for_batch, - sql_select_max_direct_input_index, sql_select_ordered_l2_tx_count, - sql_select_ordered_l2_txs_from_offset, sql_select_ordered_l2_txs_page_from_offset, - sql_select_recommended_fee, sql_select_safe_inputs_range, - sql_select_total_drained_direct_inputs, sql_update_recommended_fee, + SQL_INSERT_DIRECT_INPUT, SQL_INSERT_SEQUENCED_DIRECT_INPUT, SQL_INSERT_SEQUENCED_USER_OP, + SQL_INSERT_USER_OP, sql_count_user_ops_for_frame, sql_insert_direct_inputs_batch, + sql_insert_open_batch, sql_insert_open_frame, sql_insert_sequenced_direct_inputs_for_frame, + sql_insert_user_ops_and_sequenced_batch, sql_select_latest_batch_with_user_op_count, + sql_select_latest_frame_in_batch_for_batch, sql_select_max_direct_input_index, + sql_select_ordered_l2_tx_count, sql_select_ordered_l2_txs_from_offset, + sql_select_ordered_l2_txs_page_from_offset, sql_select_recommended_fee, + sql_select_safe_inputs_range, sql_select_total_drained_direct_inputs, + sql_update_recommended_fee, }; use crate::inclusion_lane::PendingUserOp; use crate::storage::IndexedDirectInput; use crate::storage::db::Storage; - use crate::user_op::{SignedUserOp, UserOp}; - use alloy_primitives::{Address, B256, Signature}; + use alloy_primitives::{Address, Signature}; use rusqlite::{Connection, params}; + use sequencer_core::user_op::{SignedUserOp, UserOp}; use std::time::SystemTime; use tokio::sync::oneshot; @@ -290,7 +324,6 @@ mod tests { data: vec![seed].into(), }, }, - tx_hash: B256::from([seed; 32]), respond_to, received_at: SystemTime::now(), } @@ -319,8 +352,11 @@ mod tests { ); let tx = conn.transaction().expect("start tx"); - tx.execute(SQL_INSERT_FRAME_DRAIN, params![0_i64, 0_i64, 1_i64]) - .expect("insert frame drain"); + tx.execute( + SQL_INSERT_SEQUENCED_DIRECT_INPUT, + params![0_i64, 0_i64, 0_i64], + ) + .expect("insert sequenced direct input"); tx.commit().expect("commit tx"); assert_eq!( @@ -365,7 +401,6 @@ mod tests { 0_i64, 0_i64, 0_i64, - vec![0x10_u8; 32], vec![0x20_u8; 20], 0_i64, 1_i64, @@ -377,17 +412,22 @@ mod tests { .expect("insert user op"); conn.execute(SQL_INSERT_DIRECT_INPUT, params![0_i64, vec![0xaa_u8]]) .expect("insert direct input"); - conn.execute(SQL_INSERT_FRAME_DRAIN, params![0_i64, 0_i64, 1_i64]) - .expect("insert frame drain"); + conn.execute(SQL_INSERT_SEQUENCED_USER_OP, params![0_i64, 0_i64, 0_i64]) + .expect("insert sequenced user op"); + conn.execute( + SQL_INSERT_SEQUENCED_DIRECT_INPUT, + params![0_i64, 0_i64, 0_i64], + ) + .expect("insert sequenced direct input"); let rows = sql_select_ordered_l2_txs_from_offset(&conn, 0).expect("query ordered l2"); assert_eq!(rows.len(), 2); assert_eq!(rows[0].kind, 0); - assert_eq!(rows[0].fee, Some(1)); + assert_eq!(rows[0].fee, Some(0)); assert_eq!(rows[1].kind, 1); assert_eq!(rows[1].fee, None); - let paged = sql_select_ordered_l2_txs_page_from_offset(&conn, 1, 1).expect("query page"); + let paged = sql_select_ordered_l2_txs_page_from_offset(&conn, 2, 1).expect("query page"); assert_eq!(paged.len(), 1); assert_eq!(paged[0].kind, 1); assert_eq!( @@ -401,15 +441,15 @@ mod tests { let mut conn = setup_conn(); let tx = conn.transaction().expect("start tx"); - let (batch_index, _created_at_ms, fee, user_op_count) = + let (batch_index, _created_at_ms, user_op_count) = sql_select_latest_batch_with_user_op_count(&tx).expect("query latest batch"); assert_eq!(batch_index, 0); - assert_eq!(fee, 1); assert_eq!(user_op_count, 0); - let frame_in_batch = + let (frame_in_batch, frame_fee) = sql_select_latest_frame_in_batch_for_batch(&tx, batch_index).expect("latest frame"); assert_eq!(frame_in_batch, 0); + assert_eq!(frame_fee, 0); let frame_user_op_count = sql_count_user_ops_for_frame(&tx, batch_index, frame_in_batch).expect("count user ops"); @@ -421,9 +461,9 @@ mod tests { let mut conn = setup_conn(); let tx = conn.transaction().expect("start tx"); - sql_insert_open_batch(&tx, 123, 7).expect("insert open batch"); + sql_insert_open_batch(&tx, 123).expect("insert open batch"); let new_batch = tx.last_insert_rowid(); - sql_insert_open_frame(&tx, new_batch, 0, 123).expect("insert open frame"); + sql_insert_open_frame(&tx, new_batch, 0, 123, 7).expect("insert open frame"); tx.commit().expect("commit tx"); let batch_count: i64 = conn @@ -441,7 +481,7 @@ mod tests { let conn = setup_conn(); assert_eq!( sql_select_recommended_fee(&conn).expect("read recommended"), - 1 + 0 ); sql_update_recommended_fee(&conn, 9).expect("update recommended"); assert_eq!(sql_select_recommended_fee(&conn).expect("read updated"), 9); @@ -469,10 +509,11 @@ mod tests { sample_pending_user_op(0x20, 0, 1), sample_pending_user_op(0x21, 1, 1), ]; - sql_insert_user_ops_batch(&tx, 0, 0, 0, user_ops.as_slice()) - .expect("insert user ops batch"); + sql_insert_user_ops_and_sequenced_batch(&tx, 0, 0, 0, user_ops.as_slice()) + .expect("insert user ops + sequenced batch"); - sql_insert_frame_drain(&tx, 0, 0, direct_inputs.len() as i64).expect("insert frame drain"); + sql_insert_sequenced_direct_inputs_for_frame(&tx, 0, 0, 0, direct_inputs.len()) + .expect("insert sequenced direct inputs batch"); tx.commit().expect("commit tx"); @@ -482,12 +523,71 @@ mod tests { let user_ops_count: i64 = conn .query_row("SELECT COUNT(*) FROM user_ops", [], |row| row.get(0)) .expect("count user ops"); - let frame_drains_count: i64 = conn - .query_row("SELECT COUNT(*) FROM frame_drains", [], |row| row.get(0)) - .expect("count frame drains"); + let sequenced_count: i64 = conn + .query_row("SELECT COUNT(*) FROM sequenced_l2_txs", [], |row| { + row.get(0) + }) + .expect("count sequenced l2 txs"); assert_eq!(direct_inputs_count, 2); assert_eq!(user_ops_count, 2); - assert_eq!(frame_drains_count, 1); + assert_eq!(sequenced_count, 4); + } + + #[test] + fn user_op_uniqueness_is_sender_nonce() { + let conn = setup_conn(); + + // Same nonce with different senders should be accepted. + conn.execute( + SQL_INSERT_USER_OP, + params![ + 0_i64, + 0_i64, + 0_i64, + vec![0x11_u8; 20], + 0_i64, + 0_i64, + vec![0x01_u8], + vec![0x55_u8; 65], + 0_i64 + ], + ) + .expect("insert first user op"); + conn.execute( + SQL_INSERT_USER_OP, + params![ + 0_i64, + 0_i64, + 1_i64, + vec![0x22_u8; 20], + 0_i64, + 0_i64, + vec![0x02_u8], + vec![0x66_u8; 65], + 0_i64 + ], + ) + .expect("insert second user op with same nonce and different sender"); + + // Same sender + nonce should violate uniqueness. + let duplicate_sender_nonce = conn.execute( + SQL_INSERT_USER_OP, + params![ + 0_i64, + 0_i64, + 2_i64, + vec![0x11_u8; 20], + 0_i64, + 0_i64, + vec![0x03_u8], + vec![0x77_u8; 65], + 0_i64 + ], + ); + assert!( + duplicate_sender_nonce.is_err(), + "duplicate (sender, nonce) should fail" + ); } } diff --git a/sequencer/tests/e2e_sequencer.rs b/sequencer/tests/e2e_sequencer.rs index 232f2cf..48c37bd 100644 --- a/sequencer/tests/e2e_sequencer.rs +++ b/sequencer/tests/e2e_sequencer.rs @@ -2,71 +2,54 @@ // SPDX-License-Identifier: Apache-2.0 (see LICENSE) use std::io::ErrorKind; -use std::path::PathBuf; use std::sync::Arc; -use std::time::{Duration, SystemTime, UNIX_EPOCH}; +use std::time::Duration; use alloy_primitives::{Address, Signature, U256}; use alloy_sol_types::{Eip712Domain, SolStruct}; +use app_core::application::{WalletApp, WalletConfig}; use futures_util::StreamExt; use k256::ecdsa::SigningKey; use k256::ecdsa::signature::hazmat::PrehashSigner; use sequencer::api::{AppState, router}; -use sequencer::application::{Deposit, Method, WalletApp, WalletConfig}; use sequencer::inclusion_lane::{ InclusionLane, InclusionLaneConfig, InclusionLaneError, InclusionLaneInput, }; use sequencer::l2_tx_broadcaster::{L2TxBroadcaster, L2TxBroadcasterConfig}; use sequencer::storage::Storage; -use sequencer::user_op::UserOp; -use serde::Deserialize; +use sequencer_core::api::{TxRequest, TxResponse, WsTxMessage}; +use sequencer_core::application::{Method, Withdrawal}; +use sequencer_core::user_op::UserOp; +use sequencer_rust_client::SequencerClient; +use tempfile::TempDir; use tokio::io::{AsyncReadExt, AsyncWriteExt}; -use tokio::sync::{mpsc, oneshot}; +use tokio::sync::{Semaphore, mpsc, oneshot}; use tokio_tungstenite::connect_async; use tokio_tungstenite::tungstenite::Message; -#[derive(Debug, Deserialize)] -struct TxResponse { - ok: bool, - tx_hash: String, - sender: String, - nonce: u32, -} - -#[derive(Debug, Deserialize)] -#[allow(dead_code)] -#[serde(tag = "kind", rename_all = "snake_case")] -enum WsTxMessage { - UserOp { - offset: u64, - sender: String, - fee: u64, - data: String, - }, - DirectInput { - offset: u64, - payload: String, - }, -} - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn e2e_submit_tx_ack_and_broadcast() { - let db_path = temp_db_path("full-e2e"); + let db = temp_db("full-e2e"); let domain = test_domain(); - bootstrap_open_batch_fee_zero(&db_path); + bootstrap_open_frame_fee_zero(db.path.as_str()); - let Some(runtime) = start_full_server(&db_path, domain.clone()).await else { + let Some(runtime) = start_full_server(db.path.as_str(), domain.clone()).await else { return; }; - let ws_url = format!("ws://{}/ws/subscribe?from_offset=0", runtime.addr); - let (mut ws, _) = connect_async(ws_url).await.expect("connect websocket"); + let endpoint = format!("http://{}", runtime.addr); + let client = SequencerClient::new_with_timeout(endpoint.clone(), Duration::from_secs(2)) + .expect("build sequencer client"); + let ws_url = client.ws_subscribe_url(0); + let (mut ws, _) = tokio::time::timeout(Duration::from_secs(5), connect_async(ws_url)) + .await + .expect("timeout connecting websocket") + .expect("connect websocket"); let signing_key = SigningKey::from_bytes((&[7_u8; 32]).into()).expect("create signing key"); let sender = address_from_signing_key(&signing_key); - let method = Method::Deposit(Deposit { - amount: U256::from(5_u64), - to: sender, + let method = Method::Withdrawal(Withdrawal { + amount: U256::from(0_u64), }); let user_op = UserOp { nonce: 0, @@ -75,13 +58,16 @@ async fn e2e_submit_tx_ack_and_broadcast() { }; let signature_hex = sign_user_op_hex(&domain, &user_op, &signing_key); - let request_body = serde_json::json!({ - "message": user_op, - "signature": signature_hex, - "sender": sender.to_string(), - }); + let request_body = TxRequest { + message: user_op, + signature: signature_hex, + sender: sender.to_string(), + }; - let (status, response_body) = post_json(runtime.addr, "/tx", request_body.to_string()).await; + let (status, response_body) = client + .submit_tx_with_status(&request_body) + .await + .expect("submit tx"); assert_eq!( status, 200, "submit tx should succeed: body={response_body}" @@ -92,10 +78,6 @@ async fn e2e_submit_tx_ack_and_broadcast() { assert!(response.ok); assert_eq!(response.nonce, 0); assert_eq!(response.sender, sender.to_string()); - assert!( - response.tx_hash.starts_with("0x"), - "response tx hash should be 0x-prefixed" - ); let first_message = recv_ws_message(&mut ws).await; match first_message { @@ -120,15 +102,230 @@ async fn e2e_submit_tx_ack_and_broadcast() { shutdown_runtime(runtime).await; } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn api_rejects_signature_with_wrong_hex_length() { + let db = temp_db("bad-signature-hex-len"); + let domain = test_domain(); + bootstrap_open_frame_fee_zero(db.path.as_str()); + + let Some(runtime) = start_full_server(db.path.as_str(), domain.clone()).await else { + return; + }; + + let endpoint = format!("http://{}", runtime.addr); + let client = SequencerClient::new_with_timeout(endpoint, Duration::from_secs(2)) + .expect("build sequencer client"); + + let mut request = make_valid_request(&domain); + request.signature = "0xdeadbeef".to_string(); + let (status, body) = client + .submit_tx_with_status(&request) + .await + .expect("submit tx"); + + assert_eq!( + status, 400, + "unexpected status for bad signature len: {body}" + ); + assert!( + body.contains("signature must be"), + "expected signature length message, got: {body}" + ); + + shutdown_runtime(runtime).await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn api_rejects_sender_with_wrong_hex_length() { + let db = temp_db("bad-sender-hex-len"); + let domain = test_domain(); + bootstrap_open_frame_fee_zero(db.path.as_str()); + + let Some(runtime) = start_full_server(db.path.as_str(), domain.clone()).await else { + return; + }; + + let endpoint = format!("http://{}", runtime.addr); + let client = SequencerClient::new_with_timeout(endpoint, Duration::from_secs(2)) + .expect("build sequencer client"); + + let mut request = make_valid_request(&domain); + request.sender = "0x1234".to_string(); + let (status, body) = client + .submit_tx_with_status(&request) + .await + .expect("submit tx"); + + assert_eq!(status, 400, "unexpected status for bad sender len: {body}"); + assert!( + body.contains("sender must be"), + "expected sender length message, got: {body}" + ); + + shutdown_runtime(runtime).await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn api_rejects_oversized_json_body_before_parsing() { + let db = temp_db("oversized-json"); + let domain = test_domain(); + bootstrap_open_frame_fee_zero(db.path.as_str()); + + let Some(runtime) = start_full_server_with_max_body(db.path.as_str(), domain, 256).await else { + return; + }; + + let oversized_json = format!( + r#"{{"message":{{"nonce":0,"max_fee":0,"data":"0x"}},"signature":"0x{}","sender":null,"pad":"{}"}}"#, + "00".repeat(65), + " ".repeat(4096) + ); + let (status, body) = post_raw_json(runtime.addr, oversized_json.as_str()).await; + + assert_eq!( + status, 413, + "unexpected status for oversized JSON body: {body}" + ); + assert!( + body.contains("PAYLOAD_TOO_LARGE"), + "expected payload-too-large error code, got: {body}" + ); + + shutdown_runtime(runtime).await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn api_rejects_malformed_json_as_bad_request() { + let db = temp_db("malformed-json"); + let domain = test_domain(); + bootstrap_open_frame_fee_zero(db.path.as_str()); + + let Some(runtime) = start_full_server_with_max_body(db.path.as_str(), domain, 128 * 1024).await + else { + return; + }; + + let malformed_json = + r#"{"message":{"nonce":0,"max_fee":0,"data":"0x"},"signature":"0x00","sender":"0x1234""#; + let (status, body) = post_raw_json(runtime.addr, malformed_json).await; + + assert_eq!( + status, 400, + "unexpected status for malformed JSON body: {body}" + ); + assert!( + body.contains("BAD_REQUEST"), + "expected bad-request error code, got: {body}" + ); + + shutdown_runtime(runtime).await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn api_returns_429_when_tx_middleware_concurrency_is_exceeded() { + let db = temp_db("tx-middleware-overload"); + let domain = test_domain(); + bootstrap_open_frame_fee_zero(db.path.as_str()); + + let Some(runtime) = + start_api_only_server(db.path.as_str(), domain.clone(), 128 * 1024, 8, 1).await + else { + return; + }; + + let request = make_valid_request(&domain); + let request_json = serde_json::to_string(&request).expect("serialize valid request"); + let first = tokio::spawn({ + let body = request_json.clone(); + let addr = runtime.addr; + async move { post_raw_json(addr, body.as_str()).await } + }); + tokio::time::sleep(Duration::from_millis(50)).await; + + let (status, body) = post_raw_json(runtime.addr, request_json.as_str()).await; + assert_eq!(status, 429, "expected 429 for middleware overload: {body}"); + assert!( + body.contains("OVERLOADED"), + "expected OVERLOADED code for middleware overload: {body}" + ); + + first.abort(); + shutdown_runtime(runtime).await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn api_returns_429_when_queue_is_full() { + let db = temp_db("queue-full-overload"); + let domain = test_domain(); + bootstrap_open_frame_fee_zero(db.path.as_str()); + + let Some(runtime) = + start_api_only_server(db.path.as_str(), domain.clone(), 128 * 1024, 1, 8).await + else { + return; + }; + + let request = make_valid_request(&domain); + let request_json = serde_json::to_string(&request).expect("serialize valid request"); + let first = tokio::spawn({ + let body = request_json.clone(); + let addr = runtime.addr; + async move { post_raw_json(addr, body.as_str()).await } + }); + tokio::time::sleep(Duration::from_millis(50)).await; + + let (status, body) = post_raw_json(runtime.addr, request_json.as_str()).await; + assert_eq!(status, 429, "expected 429 for queue-full overload: {body}"); + assert!( + body.contains("queue full"), + "expected queue full message in overload response: {body}" + ); + assert!( + body.contains("OVERLOADED"), + "expected OVERLOADED code for queue-full overload: {body}" + ); + + first.abort(); + shutdown_runtime(runtime).await; +} + struct FullServerRuntime { addr: std::net::SocketAddr, - shutdown_tx: oneshot::Sender<()>, - server_task: tokio::task::JoinHandle<()>, - lane_stop: sequencer::inclusion_lane::InclusionLaneStop, - lane_handle: tokio::task::JoinHandle, + broadcaster: L2TxBroadcaster, + shutdown_tx: Option>, + server_task: Option>, + lane_stop: Option, + lane_handle: Option>, + _parked_rx: Option>, +} + +impl Drop for FullServerRuntime { + fn drop(&mut self) { + self.broadcaster.request_shutdown(); + if let Some(tx) = self.shutdown_tx.take() { + let _ = tx.send(()); + } + if let Some(stop) = self.lane_stop.take() { + stop.request_shutdown(); + } + if let Some(task) = self.server_task.take() { + task.abort(); + } + if let Some(task) = self.lane_handle.take() { + task.abort(); + } + } } async fn start_full_server(db_path: &str, domain: Eip712Domain) -> Option { + start_full_server_with_max_body(db_path, domain, 128 * 1024).await +} + +async fn start_full_server_with_max_body( + db_path: &str, + domain: Eip712Domain, + max_body_bytes: usize, +) -> Option { let listener = match tokio::net::TcpListener::bind("127.0.0.1:0").await { Ok(value) => value, Err(err) if err.kind() == ErrorKind::PermissionDenied => { @@ -154,6 +351,8 @@ async fn start_full_server(db_path: &str, domain: Eip712Domain) -> Option Option Option(); + let server = axum::serve(listener, app).with_graceful_shutdown(async move { + let _ = shutdown_rx.await; + }); + let server_task = tokio::spawn(async move { + server.await.expect("run test server"); + }); + + Some(FullServerRuntime { + addr, broadcaster, + shutdown_tx: Some(shutdown_tx), + server_task: Some(server_task), + lane_stop: Some(lane_stop), + lane_handle: Some(lane_handle), + _parked_rx: None, + }) +} + +async fn start_api_only_server( + db_path: &str, + domain: Eip712Domain, + max_body_bytes: usize, + queue_capacity: usize, + overload_max_inflight_submissions: usize, +) -> Option { + let listener = match tokio::net::TcpListener::bind("127.0.0.1:0").await { + Ok(value) => value, + Err(err) if err.kind() == ErrorKind::PermissionDenied => { + eprintln!("skipping api overload test: cannot bind test listener in this environment"); + return None; + } + Err(err) => panic!("bind test listener: {err}"), + }; + let addr = listener.local_addr().expect("read listener addr"); + + let _storage = Storage::open(db_path, "NORMAL").expect("open storage"); + let (tx, rx) = mpsc::channel::(queue_capacity); + let broadcaster = L2TxBroadcaster::start( + db_path.to_string(), + L2TxBroadcasterConfig { + idle_poll_interval: Duration::from_millis(2), + page_size: 64, + subscriber_buffer_capacity: 256, + metrics_enabled: false, + metrics_log_interval: Duration::from_secs(5), + }, + ) + .expect("start broadcaster"); + let state = Arc::new(AppState { + tx_sender: tx, + domain, + overload_max_inflight_submissions, + ws_subscriber_limit: Arc::new(Semaphore::new(64)), + ws_max_catchup_events: 50_000, + broadcaster: broadcaster.clone(), }); - let app = router(state, 128 * 1024); + let app = router(state, max_body_bytes); let (shutdown_tx, shutdown_rx) = oneshot::channel::<()>(); let server = axum::serve(listener, app).with_graceful_shutdown(async move { @@ -186,38 +448,108 @@ async fn start_full_server(db_path: &str, domain: Eip712Domain) -> Option TxRequest { + let signing_key = SigningKey::from_bytes((&[7_u8; 32]).into()).expect("create signing key"); + let sender = address_from_signing_key(&signing_key); + let method = Method::Withdrawal(Withdrawal { + amount: U256::from(0_u64), + }); + let user_op = UserOp { + nonce: 0, + max_fee: 0, + data: ssz::Encode::as_ssz_bytes(&method).into(), + }; + let signature_hex = sign_user_op_hex(domain, &user_op, &signing_key); + TxRequest { + message: user_op, + signature: signature_hex, + sender: sender.to_string(), + } +} + +async fn post_raw_json(addr: std::net::SocketAddr, body: &str) -> (u16, String) { + let host_port = addr.to_string(); + let mut stream = tokio::net::TcpStream::connect(host_port.as_str()) + .await + .expect("connect test http socket"); + let request = format!( + "POST /tx HTTP/1.1\r\nHost: {host_port}\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", + body.len() + ); + stream + .write_all(request.as_bytes()) + .await + .expect("write raw request"); + stream.flush().await.expect("flush raw request"); + + let mut response = Vec::new(); + stream + .read_to_end(&mut response) + .await + .expect("read raw response"); + parse_http_response(response.as_slice()) +} + +fn parse_http_response(raw: &[u8]) -> (u16, String) { + let text = String::from_utf8(raw.to_vec()).expect("response is valid utf8"); + let mut sections = text.splitn(2, "\r\n\r\n"); + let headers = sections.next().unwrap_or_default(); + let body = sections.next().unwrap_or_default().to_string(); + let status = headers + .lines() + .next() + .expect("status line exists") + .split_whitespace() + .nth(1) + .expect("status code exists") + .parse::() + .expect("status code parses"); + (status, body) } fn sign_user_op_hex(domain: &Eip712Domain, user_op: &UserOp, signing_key: &SigningKey) -> String { @@ -247,73 +579,6 @@ fn address_from_signing_key(signing_key: &SigningKey) -> Address { Address::from_raw_public_key(&verifying.as_bytes()[1..]) } -async fn post_json(addr: std::net::SocketAddr, path: &str, body: String) -> (u16, String) { - let mut stream = tokio::net::TcpStream::connect(addr) - .await - .expect("connect http socket"); - let request = format!( - "POST {path} HTTP/1.1\r\nHost: {addr}\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", - body.len() - ); - stream - .write_all(request.as_bytes()) - .await - .expect("write http request"); - stream.flush().await.expect("flush http request"); - - let mut response = Vec::new(); - let mut chunk = [0_u8; 1024]; - loop { - let read_result = tokio::time::timeout(Duration::from_secs(2), stream.read(&mut chunk)) - .await - .expect("timed out while reading http response") - .expect("read http response"); - if read_result == 0 { - break; - } - response.extend_from_slice(&chunk[..read_result]); - - if let Some((header_end, content_length)) = response_content_len(response.as_slice()) - && response.len() >= header_end.saturating_add(content_length) - { - break; - } - } - parse_http_response(response.as_slice()) -} - -fn parse_http_response(raw: &[u8]) -> (u16, String) { - let text = String::from_utf8(raw.to_vec()).expect("http response utf8"); - let mut sections = text.splitn(2, "\r\n\r\n"); - let headers = sections.next().unwrap_or_default(); - let body = sections.next().unwrap_or_default().to_string(); - - let mut header_lines = headers.lines(); - let status_line = header_lines.next().expect("http status line"); - let status = status_line - .split_whitespace() - .nth(1) - .expect("status code") - .parse::() - .expect("parse status code"); - (status, body) -} - -fn response_content_len(raw: &[u8]) -> Option<(usize, usize)> { - let header_end = raw.windows(4).position(|window| window == b"\r\n\r\n")? + 4; - let headers = std::str::from_utf8(&raw[..header_end]).ok()?; - let mut content_length = None; - for line in headers.lines() { - if let Some((name, value)) = line.split_once(':') - && name.eq_ignore_ascii_case("content-length") - { - content_length = value.trim().parse::().ok(); - break; - } - } - content_length.map(|len| (header_end, len)) -} - async fn recv_ws_message( ws: &mut tokio_tungstenite::WebSocketStream< tokio_tungstenite::MaybeTlsStream, @@ -345,16 +610,19 @@ fn test_domain() -> Eip712Domain { } } -fn temp_db_path(name: &str) -> String { - let mut path = std::env::temp_dir(); - let unique = SystemTime::now() - .duration_since(UNIX_EPOCH) - .unwrap_or_default() - .as_nanos(); - path.push(format!("sequencer-full-e2e-{name}-{unique}.sqlite")); - path_to_string(path) +struct TestDb { + _dir: TempDir, + path: String, } -fn path_to_string(path: PathBuf) -> String { - path.to_string_lossy().into_owned() +fn temp_db(name: &str) -> TestDb { + let dir = tempfile::Builder::new() + .prefix(format!("sequencer-full-e2e-{name}-").as_str()) + .tempdir() + .expect("create temporary test directory"); + let path = dir.path().join("sequencer.sqlite"); + TestDb { + _dir: dir, + path: path.to_string_lossy().into_owned(), + } } diff --git a/sequencer/tests/ws_broadcaster.rs b/sequencer/tests/ws_broadcaster.rs index 6c9a9f4..b48167c 100644 --- a/sequencer/tests/ws_broadcaster.rs +++ b/sequencer/tests/ws_broadcaster.rs @@ -2,186 +2,167 @@ // SPDX-License-Identifier: Apache-2.0 (see LICENSE) use std::io::ErrorKind; -use std::path::PathBuf; use std::sync::Arc; -use std::time::{Duration, SystemTime, UNIX_EPOCH}; +use std::time::{Duration, SystemTime}; -use alloy_primitives::{Address, B256, Signature}; +use alloy_primitives::{Address, Signature}; use alloy_sol_types::Eip712Domain; use futures_util::{SinkExt, StreamExt}; use sequencer::api::{AppState, router}; use sequencer::inclusion_lane::{InclusionLaneInput, PendingUserOp, SequencerError}; use sequencer::l2_tx_broadcaster::{L2TxBroadcaster, L2TxBroadcasterConfig}; use sequencer::storage::{IndexedDirectInput, Storage}; -use sequencer::user_op::{SignedUserOp, UserOp}; -use serde::Deserialize; -use tokio::sync::{mpsc, oneshot}; +use sequencer_core::api::WsTxMessage; +use sequencer_core::l2_tx::SequencedL2Tx; +use sequencer_core::user_op::{SignedUserOp, UserOp}; +use sequencer_rust_client::SequencerClient; +use tempfile::TempDir; +use tokio::sync::{Semaphore, mpsc, oneshot}; use tokio_tungstenite::connect_async; use tokio_tungstenite::tungstenite::Message; -#[derive(Debug, Deserialize)] -#[serde(tag = "kind", rename_all = "snake_case")] -enum WsTxMessage { - UserOp { - offset: u64, - sender: String, - fee: u64, - data: String, - }, - DirectInput { - offset: u64, - payload: String, - }, -} - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn ws_subscribe_streams_ordered_txs_from_offset_zero() { - let db_path = temp_db_path("ws-subscribe-zero"); - seed_ordered_txs(&db_path); + let db = temp_db("ws-subscribe-zero"); + seed_ordered_txs(db.path.as_str()); + let expected = load_ordered_l2_txs_page(db.path.as_str(), 0, 2); + assert_eq!(expected.len(), 2, "seeded replay must contain two txs"); - let Some((addr, shutdown_tx, server_task)) = start_test_server(&db_path).await else { + let Some(runtime) = start_test_server(db.path.as_str()).await else { return; }; - let url = format!("ws://{addr}/ws/subscribe?from_offset=0"); - let (mut ws, _) = connect_async(url).await.expect("connect websocket"); + let url = ws_subscribe_url(runtime.addr, 0); + let (mut ws, _) = tokio::time::timeout(Duration::from_secs(5), connect_async(url)) + .await + .expect("timeout connecting websocket") + .expect("connect websocket"); let first = recv_tx_message(&mut ws).await; let second = recv_tx_message(&mut ws).await; drop(ws); - shutdown_tx.send(()).expect("request shutdown"); - server_task.await.expect("join server task"); - - match first { - WsTxMessage::UserOp { - offset, - sender, - fee, - data, - } => { - assert_eq!(offset, 0); - assert_eq!(fee, 1); - assert_eq!(decode_hex_prefixed(data.as_str()), vec![0x42]); - assert_eq!( - decode_hex_prefixed(sender.as_str()), - vec![0x11; 20], - "sender should match persisted user-op sender" - ); - } - value => panic!("expected user_op at offset 0, got {value:?}"), - } + shutdown_runtime(runtime).await; - match second { - WsTxMessage::DirectInput { offset, payload } => { - assert_eq!(offset, 1); - assert_eq!(decode_hex_prefixed(payload.as_str()), vec![0xaa]); - } - value => panic!("expected direct_input at offset 1, got {value:?}"), - } + assert_ws_message_matches_tx(first, &expected[0], 0); + assert_ws_message_matches_tx(second, &expected[1], 1); } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn ws_subscribe_resumes_from_given_offset() { - let db_path = temp_db_path("ws-subscribe-resume"); - seed_ordered_txs(&db_path); - - let Some((addr, shutdown_tx, server_task)) = start_test_server(&db_path).await else { + let db = temp_db("ws-subscribe-resume"); + seed_ordered_txs(db.path.as_str()); + let expected = load_ordered_l2_txs_page(db.path.as_str(), 1, 1); + assert_eq!( + expected.len(), + 1, + "resume snapshot must contain one event at offset 1" + ); + + let Some(runtime) = start_test_server(db.path.as_str()).await else { return; }; - let url = format!("ws://{addr}/ws/subscribe?from_offset=1"); - let (mut ws, _) = connect_async(url).await.expect("connect websocket"); + let url = ws_subscribe_url(runtime.addr, 1); + let (mut ws, _) = tokio::time::timeout(Duration::from_secs(5), connect_async(url)) + .await + .expect("timeout connecting websocket") + .expect("connect websocket"); let first = recv_tx_message(&mut ws).await; drop(ws); - shutdown_tx.send(()).expect("request shutdown"); - server_task.await.expect("join server task"); + shutdown_runtime(runtime).await; - match first { - WsTxMessage::DirectInput { offset, payload } => { - assert_eq!(offset, 1); - assert_eq!(decode_hex_prefixed(payload.as_str()), vec![0xaa]); - } - value => panic!("expected direct_input at offset 1, got {value:?}"), - } + assert_ws_message_matches_tx(first, &expected[0], 1); } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn ws_subscribe_receives_live_events_after_subscribing() { - let db_path = temp_db_path("ws-subscribe-live"); - seed_ordered_txs(&db_path); + let db = temp_db("ws-subscribe-live"); + seed_ordered_txs(db.path.as_str()); + let base_offset = ordered_l2_tx_count(db.path.as_str()); - let Some((addr, shutdown_tx, server_task)) = start_test_server(&db_path).await else { + let Some(runtime) = start_test_server(db.path.as_str()).await else { return; }; - // Existing persisted offsets are [0, 2). Subscribe at 2 to exercise live-only delivery. - let url = format!("ws://{addr}/ws/subscribe?from_offset=2"); - let (mut ws, _) = connect_async(url).await.expect("connect websocket"); - - append_drained_direct_input(&db_path, 1, vec![0xbb]); + // Subscribe at the current DB head to exercise live-only delivery. + let url = ws_subscribe_url(runtime.addr, base_offset); + let (mut ws, _) = tokio::time::timeout(Duration::from_secs(5), connect_async(url)) + .await + .expect("timeout connecting websocket") + .expect("connect websocket"); + + append_drained_direct_input(db.path.as_str(), 1, vec![0xbb]); + let expected = load_ordered_l2_txs_page(db.path.as_str(), base_offset, 1); + assert_eq!( + expected.len(), + 1, + "appending one drained direct input must add one sequenced tx" + ); let live = recv_tx_message(&mut ws).await; drop(ws); - shutdown_tx.send(()).expect("request shutdown"); - server_task.await.expect("join server task"); + shutdown_runtime(runtime).await; - match live { - WsTxMessage::DirectInput { offset, payload } => { - assert_eq!(offset, 2); - assert_eq!(decode_hex_prefixed(payload.as_str()), vec![0xbb]); - } - value => panic!("expected live direct_input at offset 2, got {value:?}"), - } + assert_ws_message_matches_tx(live, &expected[0], base_offset); } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn ws_subscribe_fanout_delivers_live_event_to_multiple_subscribers() { - let db_path = temp_db_path("ws-subscribe-fanout"); - seed_ordered_txs(&db_path); + let db = temp_db("ws-subscribe-fanout"); + seed_ordered_txs(db.path.as_str()); + let base_offset = ordered_l2_tx_count(db.path.as_str()); - let Some((addr, shutdown_tx, server_task)) = start_test_server(&db_path).await else { + let Some(runtime) = start_test_server(db.path.as_str()).await else { return; }; - let url = format!("ws://{addr}/ws/subscribe?from_offset=2"); - let (mut ws_a, _) = connect_async(url.as_str()) + let url = ws_subscribe_url(runtime.addr, base_offset); + let (mut ws_a, _) = tokio::time::timeout(Duration::from_secs(5), connect_async(url.as_str())) .await + .expect("timeout connecting websocket A") .expect("connect websocket A"); - let (mut ws_b, _) = connect_async(url).await.expect("connect websocket B"); + let (mut ws_b, _) = tokio::time::timeout(Duration::from_secs(5), connect_async(url)) + .await + .expect("timeout connecting websocket B") + .expect("connect websocket B"); - append_drained_direct_input(&db_path, 1, vec![0xcd]); + append_drained_direct_input(db.path.as_str(), 1, vec![0xcd]); + let expected = load_ordered_l2_txs_page(db.path.as_str(), base_offset, 1); + assert_eq!( + expected.len(), + 1, + "appending one drained direct input must add one sequenced tx" + ); let event_a = recv_tx_message(&mut ws_a).await; let event_b = recv_tx_message(&mut ws_b).await; drop(ws_a); drop(ws_b); - shutdown_tx.send(()).expect("request shutdown"); - server_task.await.expect("join server task"); + shutdown_runtime(runtime).await; - let assert_event = |event: WsTxMessage| match event { - WsTxMessage::DirectInput { offset, payload } => { - assert_eq!(offset, 2); - assert_eq!(decode_hex_prefixed(payload.as_str()), vec![0xcd]); - } - value => panic!("expected live direct_input at offset 2, got {value:?}"), - }; - assert_event(event_a); - assert_event(event_b); + assert_ws_message_matches_tx(event_a, &expected[0], base_offset); + assert_ws_message_matches_tx(event_b, &expected[0], base_offset); } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn ws_subscribe_replies_with_pong_on_ping() { - let db_path = temp_db_path("ws-subscribe-ping-pong"); - seed_ordered_txs(&db_path); + let db = temp_db("ws-subscribe-ping-pong"); + seed_ordered_txs(db.path.as_str()); + // Use a far-future offset so this test validates ping/pong without + // interleaving replay/live tx frames. + let from_offset = u64::MAX; - let Some((addr, shutdown_tx, server_task)) = start_test_server(&db_path).await else { + let Some(runtime) = start_test_server(db.path.as_str()).await else { return; }; - let url = format!("ws://{addr}/ws/subscribe?from_offset=2"); - let (mut ws, _) = connect_async(url).await.expect("connect websocket"); + let url = ws_subscribe_url(runtime.addr, from_offset); + let (mut ws, _) = tokio::time::timeout(Duration::from_secs(5), connect_async(url)) + .await + .expect("timeout connecting websocket") + .expect("connect websocket"); ws.send(Message::Ping(vec![0x01, 0x02].into())) .await @@ -190,8 +171,7 @@ async fn ws_subscribe_replies_with_pong_on_ping() { let frame = recv_raw_message(&mut ws).await; drop(ws); - shutdown_tx.send(()).expect("request shutdown"); - server_task.await.expect("join server task"); + shutdown_runtime(runtime).await; match frame { Message::Pong(payload) => assert_eq!(payload.as_ref(), [0x01, 0x02]), @@ -199,6 +179,68 @@ async fn ws_subscribe_replies_with_pong_on_ping() { } } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn ws_subscribe_rejects_when_subscriber_limit_is_reached() { + let db = temp_db("ws-subscriber-limit"); + seed_ordered_txs(db.path.as_str()); + let base_offset = ordered_l2_tx_count(db.path.as_str()); + + let Some(runtime) = start_test_server_with_limits(db.path.as_str(), 1, 50_000).await else { + return; + }; + + let url = ws_subscribe_url(runtime.addr, base_offset); + let (ws_a, _) = tokio::time::timeout(Duration::from_secs(5), connect_async(url.as_str())) + .await + .expect("timeout connecting websocket A") + .expect("connect websocket A"); + + let second = tokio::time::timeout(Duration::from_secs(5), connect_async(url)) + .await + .expect("timeout connecting websocket B"); + match second { + Ok((_ws, _resp)) => panic!("expected second subscriber to be rejected with 429"), + Err(tokio_tungstenite::tungstenite::Error::Http(response)) => { + assert_eq!(response.status().as_u16(), 429); + } + Err(other) => panic!("expected HTTP 429 handshake rejection, got {other:?}"), + } + + drop(ws_a); + shutdown_runtime(runtime).await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn ws_subscribe_closes_when_catchup_window_exceeds_limit() { + let db = temp_db("ws-catchup-limit"); + seed_ordered_txs(db.path.as_str()); + + let Some(runtime) = start_test_server_with_limits(db.path.as_str(), 64, 1).await else { + return; + }; + + let url = ws_subscribe_url(runtime.addr, 0); + let (mut ws, _) = tokio::time::timeout(Duration::from_secs(5), connect_async(url)) + .await + .expect("timeout connecting websocket") + .expect("connect websocket"); + + let frame = recv_raw_message(&mut ws).await; + match frame { + Message::Close(Some(close_frame)) => { + assert!( + close_frame.reason.contains("catch-up window exceeded"), + "expected close reason to mention catch-up window: {:?}", + close_frame.reason + ); + } + other => panic!("expected close frame for catch-up limit, got {other:?}"), + } + + drop(ws); + shutdown_runtime(runtime).await; +} + fn seed_ordered_txs(db_path: &str) { let mut storage = Storage::open(db_path, "NORMAL").expect("open storage"); let mut head = storage.load_open_state().expect("load open state"); @@ -214,7 +256,6 @@ fn seed_ordered_txs(db_path: &str) { data: vec![0x42].into(), }, }, - tx_hash: B256::from([0x77; 32]), respond_to, received_at: SystemTime::now(), }; @@ -229,7 +270,7 @@ fn seed_ordered_txs(db_path: &str) { }]) .expect("append direct input"); storage - .close_frame_only(&mut head, 1) + .close_frame_only(&mut head, 0, 1) .expect("close frame with one drained direct input"); } @@ -240,17 +281,38 @@ fn append_drained_direct_input(db_path: &str, index: u64, payload: Vec) { .append_safe_direct_inputs(&[IndexedDirectInput { index, payload }]) .expect("append direct input"); storage - .close_frame_only(&mut head, 1) + .close_frame_only(&mut head, index, 1) .expect("close frame with one drained direct input"); } -async fn start_test_server( +struct WsServerRuntime { + addr: std::net::SocketAddr, + broadcaster: L2TxBroadcaster, + shutdown_tx: Option>, + server_task: Option>, +} + +impl Drop for WsServerRuntime { + fn drop(&mut self) { + self.broadcaster.request_shutdown(); + if let Some(tx) = self.shutdown_tx.take() { + let _ = tx.send(()); + } + if let Some(task) = self.server_task.take() { + task.abort(); + } + } +} + +async fn start_test_server(db_path: &str) -> Option { + start_test_server_with_limits(db_path, 64, 50_000).await +} + +async fn start_test_server_with_limits( db_path: &str, -) -> Option<( - std::net::SocketAddr, - oneshot::Sender<()>, - tokio::task::JoinHandle<()>, -)> { + ws_max_subscribers: usize, + ws_max_catchup_events: u64, +) -> Option { let listener = match tokio::net::TcpListener::bind("127.0.0.1:0").await { Ok(value) => value, Err(err) if err.kind() == ErrorKind::PermissionDenied => { @@ -270,6 +332,8 @@ async fn start_test_server( idle_poll_interval: Duration::from_millis(2), page_size: 64, subscriber_buffer_capacity: 256, + metrics_enabled: false, + metrics_log_interval: Duration::from_secs(5), }, ) .expect("start broadcaster"); @@ -282,8 +346,10 @@ async fn start_test_server( verifying_contract: None, salt: None, }, - queue_timeout: Duration::from_millis(50), - broadcaster, + overload_max_inflight_submissions: 16, + ws_subscriber_limit: Arc::new(Semaphore::new(ws_max_subscribers)), + ws_max_catchup_events, + broadcaster: broadcaster.clone(), }); let app = router(state, 128 * 1024); let (shutdown_tx, shutdown_rx) = oneshot::channel::<()>(); @@ -295,7 +361,25 @@ async fn start_test_server( server.await.expect("run test server"); }); - Some((addr, shutdown_tx, task)) + Some(WsServerRuntime { + addr, + broadcaster, + shutdown_tx: Some(shutdown_tx), + server_task: Some(task), + }) +} + +async fn shutdown_runtime(mut runtime: WsServerRuntime) { + runtime.broadcaster.request_shutdown(); + if let Some(tx) = runtime.shutdown_tx.take() { + let _ = tx.send(()); + } + if let Some(task) = runtime.server_task.take() { + tokio::time::timeout(Duration::from_secs(3), task) + .await + .expect("wait for server task") + .expect("join server task"); + } } async fn recv_tx_message( @@ -334,16 +418,75 @@ fn decode_hex_prefixed(value: &str) -> Vec { alloy_primitives::hex::decode(value).expect("decode hex") } -fn temp_db_path(name: &str) -> String { - let mut path = std::env::temp_dir(); - let unique = SystemTime::now() - .duration_since(UNIX_EPOCH) - .unwrap_or_default() - .as_nanos(); - path.push(format!("sequencer-ws-broadcaster-{name}-{unique}.sqlite")); - path_to_string(path) +fn ws_subscribe_url(addr: std::net::SocketAddr, from_offset: u64) -> String { + let endpoint = format!("http://{addr}"); + let client = SequencerClient::new(endpoint).expect("build sequencer client"); + client.ws_subscribe_url(from_offset) +} + +fn ordered_l2_tx_count(db_path: &str) -> u64 { + let mut storage = Storage::open_read_only(db_path).expect("open read-only storage"); + storage + .ordered_l2_tx_count() + .expect("query ordered l2 count") +} + +fn load_ordered_l2_txs_page(db_path: &str, from_offset: u64, limit: usize) -> Vec { + let mut storage = Storage::open_read_only(db_path).expect("open read-only storage"); + storage + .load_ordered_l2_txs_page_from(from_offset, limit) + .expect("load ordered l2 tx page") +} + +fn assert_ws_message_matches_tx( + actual: WsTxMessage, + expected: &SequencedL2Tx, + expected_offset: u64, +) { + match (actual, expected) { + ( + WsTxMessage::UserOp { + offset, + sender, + fee, + data, + }, + SequencedL2Tx::UserOp(expected), + ) => { + assert_eq!(offset, expected_offset); + assert_eq!( + decode_hex_prefixed(sender.as_str()), + expected.sender.as_slice() + ); + assert_eq!(fee, expected.fee); + assert_eq!(decode_hex_prefixed(data.as_str()), expected.data.as_slice()); + } + (WsTxMessage::DirectInput { offset, payload }, SequencedL2Tx::Direct(expected)) => { + assert_eq!(offset, expected_offset); + assert_eq!( + decode_hex_prefixed(payload.as_str()), + expected.payload.as_slice() + ); + } + (actual, expected) => { + panic!("ws message type mismatch; actual={actual:?}, expected={expected:?}") + } + } +} + +struct TestDb { + _dir: TempDir, + path: String, } -fn path_to_string(path: PathBuf) -> String { - path.to_string_lossy().into_owned() +fn temp_db(name: &str) -> TestDb { + let dir = tempfile::Builder::new() + .prefix(format!("sequencer-ws-broadcaster-{name}-").as_str()) + .tempdir() + .expect("create temporary test directory"); + let path = dir.path().join("sequencer.sqlite"); + TestDb { + _dir: dir, + path: path.to_string_lossy().into_owned(), + } } From cdd182f07907032463d486cf7fb95bb48f31fe7c Mon Sep 17 00:00:00 2001 From: Gabriel Coutinho de Paula Date: Mon, 2 Mar 2026 11:24:05 -0300 Subject: [PATCH 5/6] chore: fix linter and readme --- README.md | 2 +- benchmarks/src/bin/compare_latest.rs | 2 +- sequencer/src/api/mod.rs | 62 ++++++++++++++-------------- 3 files changed, 33 insertions(+), 33 deletions(-) diff --git a/README.md b/README.md index 5b15766..2cb8f9a 100644 --- a/README.md +++ b/README.md @@ -64,7 +64,7 @@ Request shape: POST notes: - `signature` must be 65 bytes. -- `sender` is optional; if provided, it must match recovered signer. +- `sender` is required and must match recovered signer. - `message.data` is SSZ-encoded method payload bytes. - payload size is bounded at ingress; oversized requests are rejected before they enter hot path. diff --git a/benchmarks/src/bin/compare_latest.rs b/benchmarks/src/bin/compare_latest.rs index a58b53e..e5ce067 100644 --- a/benchmarks/src/bin/compare_latest.rs +++ b/benchmarks/src/bin/compare_latest.rs @@ -267,7 +267,7 @@ fn latest_two_files( } } - candidates.sort_by(|a, b| a.cmp(b)); + candidates.sort(); if candidates.len() < 2 { return Err(std::io::Error::other(format!( "need at least 2 files for pattern {prefix}*{suffix} in {}", diff --git a/sequencer/src/api/mod.rs b/sequencer/src/api/mod.rs index d7351ff..5fef7e0 100644 --- a/sequencer/src/api/mod.rs +++ b/sequencer/src/api/mod.rs @@ -308,37 +308,6 @@ async fn run_broadcaster_session( } } -#[cfg(test)] -mod tests { - use super::*; - use axum::body::to_bytes; - - #[tokio::test] - async fn tx_route_internal_errors_are_sanitized() { - let err: BoxError = std::io::Error::other("sensitive middleware detail").into(); - let response = handle_tx_route_error(err).await.into_response(); - assert_eq!(response.status(), StatusCode::INTERNAL_SERVER_ERROR); - - let body = to_bytes(response.into_body(), usize::MAX) - .await - .expect("read response body"); - let body = String::from_utf8(body.to_vec()).expect("utf8 response body"); - - assert!( - body.contains("INTERNAL_ERROR"), - "expected internal error code in body: {body}" - ); - assert!( - body.contains("tx endpoint unavailable"), - "expected sanitized internal message in body: {body}" - ); - assert!( - !body.contains("sensitive middleware detail"), - "middleware internals leaked in body: {body}" - ); - } -} - async fn send_catch_up( broadcaster: &L2TxBroadcaster, socket: &mut WebSocket, @@ -413,3 +382,34 @@ async fn send_ws_event(socket: &mut WebSocket, event: &BroadcastTxMessage) -> Re } Ok(()) } + +#[cfg(test)] +mod tests { + use super::*; + use axum::body::to_bytes; + + #[tokio::test] + async fn tx_route_internal_errors_are_sanitized() { + let err: BoxError = std::io::Error::other("sensitive middleware detail").into(); + let response = handle_tx_route_error(err).await.into_response(); + assert_eq!(response.status(), StatusCode::INTERNAL_SERVER_ERROR); + + let body = to_bytes(response.into_body(), usize::MAX) + .await + .expect("read response body"); + let body = String::from_utf8(body.to_vec()).expect("utf8 response body"); + + assert!( + body.contains("INTERNAL_ERROR"), + "expected internal error code in body: {body}" + ); + assert!( + body.contains("tx endpoint unavailable"), + "expected sanitized internal message in body: {body}" + ); + assert!( + !body.contains("sensitive middleware detail"), + "middleware internals leaked in body: {body}" + ); + } +} From d0362a0a50d261be1e0ecfefc31eb6cdfd1e47b1 Mon Sep 17 00:00:00 2001 From: Gabriel Coutinho de Paula Date: Wed, 4 Mar 2026 06:37:07 -0300 Subject: [PATCH 6/6] chore: join ws catch-up worker; log closed ws channel --- sequencer/src/api/mod.rs | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/sequencer/src/api/mod.rs b/sequencer/src/api/mod.rs index 5fef7e0..dd78058 100644 --- a/sequencer/src/api/mod.rs +++ b/sequencer/src/api/mod.rs @@ -342,6 +342,7 @@ async fn send_catch_up( let event = BroadcastTxMessage::from_offset_and_tx(next_offset, tx); next_offset = next_offset.saturating_add(1); if events_tx.blocking_send(event).is_err() { + debug!("broadcaster catch-up worker stopping early: receiver channel closed"); return Ok(()); } } @@ -349,13 +350,16 @@ async fn send_catch_up( Ok(()) }); + let mut ws_send_failed = false; while let Some(event) = events_rx.recv().await { if send_ws_event(socket, &event).await.is_err() { - return Err(()); + ws_send_failed = true; + break; } } + drop(events_rx); - match worker.await { + let worker_result = match worker.await { Ok(Ok(())) => Ok(()), Ok(Err(reason)) => { warn!(reason, "broadcaster catch-up worker exited with error"); @@ -365,7 +369,12 @@ async fn send_catch_up( warn!(error = %err, "broadcaster catch-up worker join failed"); Err(()) } + }; + + if ws_send_failed { + return Err(()); } + worker_result } async fn send_ws_event(socket: &mut WebSocket, event: &BroadcastTxMessage) -> Result<(), ()> {