From 8a14542e3c116ffc3f8dd6cbea2637fe1564fa24 Mon Sep 17 00:00:00 2001 From: Valter Balegas Date: Mon, 6 Jul 2026 17:06:53 +0100 Subject: [PATCH 1/3] fix: complete DS_UNSAFE_FAST_FSYNC rename in wal/segment.rs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The rename in #4675 missed segment.rs's copy of fast_fsync_enabled() — the fsync that actually gates WAL append acks — so the documented DS_UNSAFE_FAST_FSYNC name only affected store/checkpoint fsyncs. Verified with the bench-latency harness: WAL acks now drop ~3.2ms -> ~0.3ms under the new name on macOS. Co-Authored-By: Claude Fable 5 --- packages/durable-streams-rust/src/wal/segment.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/durable-streams-rust/src/wal/segment.rs b/packages/durable-streams-rust/src/wal/segment.rs index bd74249960..af6ae46b92 100644 --- a/packages/durable-streams-rust/src/wal/segment.rs +++ b/packages/durable-streams-rust/src/wal/segment.rs @@ -133,19 +133,19 @@ impl FileSegment { /// preserves the ORIGINAL F_FULLFSYNC errno in context — the fallback's errno /// alone would mislead durability diagnostics. (Shared by `seal_to` and /// `FileSegment::fdatasync`; mirrors `store::barrier_fsync`.) -/// BENCH-ONLY: whether `DS_BENCH_FAST_FSYNC` requests plain `fsync` over +/// BENCH-ONLY: whether `DS_UNSAFE_FAST_FSYNC` requests plain `fsync` over /// `F_FULLFSYNC` on macOS. Read once and cached. Mirrors the gate in /// `store::barrier_fsync`. #[cfg(target_os = "macos")] fn fast_fsync_enabled() -> bool { use std::sync::OnceLock; static ON: OnceLock = OnceLock::new(); - *ON.get_or_init(|| std::env::var_os("DS_BENCH_FAST_FSYNC").is_some()) + *ON.get_or_init(|| std::env::var_os("DS_UNSAFE_FAST_FSYNC").is_some()) } #[cfg(target_os = "macos")] fn macos_full_fsync(fd: libc::c_int) -> io::Result<()> { - // BENCH-ONLY (`DS_BENCH_FAST_FSYNC`): plain `fsync` instead of the + // BENCH-ONLY (`DS_UNSAFE_FAST_FSYNC`): plain `fsync` instead of the // `F_FULLFSYNC` drive barrier so the committer's hot fsync is cheap on a RAM // disk and the per-shard LOCK becomes the bottleneck (the Linux+NVMe regime // this build studies). NOT power-loss durable; never set in production. See From 9e9fdf041438dff94f4609f84a4093031b134a34 Mon Sep 17 00:00:00 2001 From: Valter Balegas Date: Mon, 6 Jul 2026 17:08:39 +0100 Subject: [PATCH 2/3] chore: add changeset for the fast-fsync rename fix Co-Authored-By: Claude Fable 5 --- .changeset/unsafe-fast-fsync-rename.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/unsafe-fast-fsync-rename.md diff --git a/.changeset/unsafe-fast-fsync-rename.md b/.changeset/unsafe-fast-fsync-rename.md new file mode 100644 index 0000000000..a895a472f7 --- /dev/null +++ b/.changeset/unsafe-fast-fsync-rename.md @@ -0,0 +1,5 @@ +--- +"@electric-ax/durable-streams-server-rust": patch +--- + +Complete the `DS_BENCH_FAST_FSYNC` → `DS_UNSAFE_FAST_FSYNC` rename in `wal/segment.rs`. The previous rename missed the WAL segment's copy of the check — the fsync that gates append acks — so the documented `DS_UNSAFE_FAST_FSYNC` name had no effect on WAL-mode write latency (macOS bench/experimentation only; never set in production). From 0d33d956408a8556fab7be316c32e2e8df807f60 Mon Sep 17 00:00:00 2001 From: Valter Balegas Date: Mon, 6 Jul 2026 17:09:50 +0100 Subject: [PATCH 3/3] docs: remove the WAL contention investigation tracking doc Investigation is concluded (fixes landed in #4675); drop the tracking doc and its dangling cross-reference in CARDINALITY_1M.md. Co-Authored-By: Claude Fable 5 --- .../durable-streams-rust/CARDINALITY_1M.md | 4 +- .../CONTENTION_INVESTIGATION.md | 105 ------------------ 2 files changed, 2 insertions(+), 107 deletions(-) delete mode 100644 packages/durable-streams-rust/CONTENTION_INVESTIGATION.md diff --git a/packages/durable-streams-rust/CARDINALITY_1M.md b/packages/durable-streams-rust/CARDINALITY_1M.md index 3fd0a3a91a..f3b2b94b54 100644 --- a/packages/durable-streams-rust/CARDINALITY_1M.md +++ b/packages/durable-streams-rust/CARDINALITY_1M.md @@ -1,7 +1,7 @@ # 1M-stream cardinality fixes — findings + results (2026-07-02) -Follow-up to `WRITE_BOTTLENECKS_1M.md` (bottleneck #2: stream cardinality) and -`CONTENTION_INVESTIGATION.md`. Server commit: `662b0c845` on +Follow-up to the write-bottleneck and WAL-contention investigations +(bottleneck #2: stream cardinality). Server commit: `662b0c845` on `perf/combined-t1a-t1c-t2a`. **Outcome: 1M streams reaches 1,114,644 ops/s on a 16 vCPU `c4d-standard-16-lssd` (ladder unsaturated), and the 500k→1M degradation at equal load is −17% (was a cliff).** diff --git a/packages/durable-streams-rust/CONTENTION_INVESTIGATION.md b/packages/durable-streams-rust/CONTENTION_INVESTIGATION.md deleted file mode 100644 index 217df32354..0000000000 --- a/packages/durable-streams-rust/CONTENTION_INVESTIGATION.md +++ /dev/null @@ -1,105 +0,0 @@ -# WAL write-saturation contention investigation - -Tracking doc for the investigation into the write-throughput ceiling reported in -`ds-bench/results/run-durable-pool2/FINDINGS.md` (server plateaus at ~80% CPU and -then _declines_ under more load — a ceiling set by the commit path, not compute). - -## Hypothesis - -Sharding, group-commit, and the network reactor are **not isolated**: they are -multiplexed over one shared work-stealing Tokio runtime, and each WAL shard is -guarded by cross-thread blocking `std::sync::Mutex`es. So a "shard" is a _lock_, -not a _core_. At saturation the cost is lock contention + committer scheduling + -a durability-wakeup thundering herd, not CPU. - -Per-append contended state (all on the shard a stream hashes to): - -- `shard.dirty` Mutex + HashMap insert — **every append** (`register_dirty`). -- `shard.inner` Mutex — **twice** per append (reserve + mark_written). -- `durable_tx` watch — `publish_durable` wakes **every** parked waiter on the shard. - -## Phase 0 — telemetry (DONE) - -Added always-on, dependency-free contention telemetry (independent of the heavy -`telemetry`/OTLP feature): - -- `ShardStats` gained per-shard counters: `inner`/`dirty` lock-wait nanos + - acquire counts, records `staged`, and durability `waiters_woken` - (`src/wal/telemetry.rs`). -- Instrumented the hot path (`src/wal/shard.rs`): `register_dirty`, - `reserve_and_stage` (both `inner` acquisitions), `publish_durable`. -- Runtime gate `--wal-stats `: arms the hot-path timing (one relaxed - atomic load when off — no clock reads in a default run) and spawns a stderr - emitter printing per-interval `WAL_CONT` lines: - - ``` - WAL_CONT staged/s=… fsync/s=… batch_avg=… inner_wait_us=… inner_wait_load=… \ - dirty_wait_us=… dirty_wait_load=… waiters_woken_avg=… - ``` - - `*_wait_load` = fraction of a core-second spent purely _waiting_ on that lock - (>1.0 ⇒ more than a whole core lost to parking on it). - -## Phase 0 — local reproduction (DONE / caveated) - -Local reproduction drives the server with the `ds-bench multi-stream` pool -client and reads throughput + CPU + steady-state `WAL_CONT`. - -**macOS caveats (why a Linux harness is also needed):** - -- `F_FULLFSYNC` is a true drive barrier (~tens of ms) and dominates the commit - path, masking the lock. Added a **bench-only** `DS_UNSAFE_FAST_FSYNC` env - (`src/store.rs`, `src/wal/segment.rs`) that uses plain `fsync` on macOS so a - RAM-disk data dir gives cheap fsync (the Linux+NVMe regime). NOT durable; never - set in production. -- The 10-core dev box co-locates client + server, so the _absolute_ throughput - ceiling is confounded (a flat ~1600 ops/s independent of shards/connections). - The **contention telemetry signals are valid** on macOS (use them for relative - before/after of a change); the **throughput-ceiling** comparison must run on - Linux with a tmpfs data dir and CPU isolation. - -Use a RAM disk for cheap fsync on macOS (point the server's data dir at it): - -``` -DEV=$(hdiutil attach -nomount ram://6291456 | awk '{print $1}') -diskutil erasevolume HFS+ dsram "$DEV" # → /Volumes/dsram -``` - -## Baseline (Linux harness, 6 server cores / 4 client cores, tmpfs, conn=256) - -Reproduces the findings' signature — a hard throughput ceiling at **~80% CPU** -(480–500 of 600), barely helped by more shards, with CPU left on the table: - -| shards | ops/s | cpu% | fsync/s | batch | inner_wait_load | waiters_woken | -| ------ | ------ | ---- | ------- | ----- | --------------- | ------------- | -| 1 | 45,543 | 482 | 6,113 | 7.5 | 0.02 | 25.8 | -| 2 | 46,905 | 504 | 12,384 | 3.8 | 0.01 | 12.4 | -| 6 | 48,825 | 495 | 24,177 | 2.0 | 0.01 | 5.2 | - -Read: fsync is cheap (tmpfs), the inner lock is not yet the gate at 6 cores -(`inner_wait_load`≈0.02), so the ceiling here is the **commit + durability-wakeup -coordination machinery** burning CPU/scheduling (25.8 waiters woken per commit at -1 shard). The lock itself becomes the gate at the findings' 32-core scale; both -are targeted below. (Numbers are this dev box; use deltas, not absolutes.) - -## How to judge a candidate change - -A change is good if it **lifts the Linux throughput ceiling** AND drives the -contention metric it targets toward zero: - -- lock-free `register_dirty` → `dirty_wait_load` → ~0 -- atomic reserve → `inner_wait_load` drops -- coalesced wakeups → `waiters_woken_avg` → ~1 -- dedicated committer / io_uring → higher `fsync/s` without CPU saturation - -## Candidate architectures (Phase 1, parallel worktrees) - -- **T1a** lock-free `register_dirty` (atomic dirty bit + lock-free push on 0→1). -- **T1b** atomic reserve (packed `fetch_add` for lsn+write_pos; lock only on roll). -- **T1c** coalesced durability wakeups (wake only satisfied waiters, not broadcast). -- **T2a** dedicated committer thread(s) off the shared runtime / drop per-commit - `spawn_blocking`. -- **T2b** io_uring WAL writes + fsync (Linux). -- **T3** shared-nothing thread-per-core spike (shard→core, per-core epoll via - `SO_REUSEPORT`, no cross-core lock; SPSC handoff for the pool client's - all-shards-per-connection access pattern).