From 6cfd6cb795a82cbbbfbf29f103862bb11e7a0f43 Mon Sep 17 00:00:00 2001 From: Valter Balegas Date: Mon, 13 Jul 2026 18:34:30 +0100 Subject: [PATCH 1/8] bench: split-lane + guaranteed-QoS manifests, STATIC_CPU pool, checkpoint-fix suites + findings - gke/durable-streams-splitlane{,-guaranteed}.yaml: stream data on its own NVMe lane, WAL shards on dedicated lanes; guaranteed variant = requests==limits everywhere + integer server CPU for exclusive cores under static CPU manager. Recreate strategy + hostPath base volume (emptyDir teardown recursed through stale lane mounts and wiped the next server's dirs). - cluster-up.sh: STATIC_CPU=1 -> kubelet cpuManagerPolicy=static on server pool. - lib-bench.sh: SPLITLANE=1 / GUARANTEED=1 manifest selection. - suites + results: wal-decomp-lane0, wal-splitlane, wal-sizetrigger, wal-cpubind (cliff eliminated: 10.4k -> 328k @100k streams; see reports + AGENTS.md). Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01Y3x7bcT9vLGiT4tXZeQpnk --- AGENTS.md | 129 +++++++- MULTILANE_SETUP.md | 179 +++++++++++ gke/durable-streams-multilane.yaml | 251 +++++++++++++++ gke/durable-streams-splitlane-guaranteed.yaml | 271 ++++++++++++++++ gke/durable-streams-splitlane.yaml | 256 +++++++++++++++ results/wal-batched-mw-local/aggregate.csv | 3 + results/wal-batched-mw-local/aggregate.json | 76 +++++ .../wal-batched-mw-local/baseline/cells.json | 37 +++ .../batched-mw/cells.json | 37 +++ results/wal-batched-mw-local/report.md | 38 +++ results/wal-checkpoint-fix-100k/aggregate.csv | 3 + .../wal-checkpoint-fix-100k/aggregate.json | 64 ++++ .../wal-checkpoint-fix-100k/both/cells.json | 30 ++ results/wal-checkpoint-fix-100k/report.md | 44 +++ .../wal-checkpoint-fix-100k/syncfs/cells.json | 30 ++ .../wal-checkpoint-fix-ab.run1/aggregate.csv | 9 + .../wal-checkpoint-fix-ab.run1/aggregate.json | 304 ++++++++++++++++++ .../baseline/cells.json | 74 +++++ .../both/cells.json | 68 ++++ results/wal-checkpoint-fix-ab.run1/report.md | 47 +++ .../stagger/cells.json | 68 ++++ .../syncfs/cells.json | 69 ++++ .../wal-checkpoint-fix-ab/baseline/cells.json | 30 ++ results/wal-cpubind/aggregate.csv | 3 + results/wal-cpubind/aggregate.json | 70 ++++ results/wal-cpubind/bound-3s/cells.json | 64 ++++ results/wal-cpubind/report.md | 41 +++ results/wal-decomp-lane0/aggregate.csv | 9 + results/wal-decomp-lane0/aggregate.json | 250 ++++++++++++++ results/wal-decomp-lane0/ckpt-off/cells.json | 58 ++++ results/wal-decomp-lane0/memory/cells.json | 58 ++++ results/wal-decomp-lane0/nofsync/cells.json | 58 ++++ results/wal-decomp-lane0/ref-3s/cells.json | 58 ++++ results/wal-decomp-lane0/report.md | 47 +++ results/wal-fanout-sweep/aggregate.csv | 5 + results/wal-fanout-sweep/aggregate.json | 192 +++++++++++ results/wal-fanout-sweep/report.md | 40 +++ results/wal-fanout-sweep/s4-f1/cells.json | 49 +++ results/wal-fanout-sweep/s4-f2/cells.json | 43 +++ results/wal-fanout-sweep/s4-f4/cells.json | 49 +++ results/wal-fanout-sweep/s4-f8/cells.json | 49 +++ results/wal-fsync-diag-local/aggregate.csv | 7 + results/wal-fsync-diag-local/aggregate.json | 200 ++++++++++++ .../wal-fsync-diag-local/memory/cells.json | 58 ++++ results/wal-fsync-diag-local/report.md | 45 +++ .../wal-nofsync/cells.json | 64 ++++ .../wal-normal/cells.json | 64 ++++ .../wal-lazy-checkpoint-local/aggregate.csv | 9 + .../wal-lazy-checkpoint-local/aggregate.json | 274 ++++++++++++++++ .../lazy-syncfs-30s/cells.json | 64 ++++ .../lazy-syncfs-60s/cells.json | 64 ++++ .../ref-baseline-3s/cells.json | 64 ++++ .../ref-syncfs-3s/cells.json | 64 ++++ results/wal-lazy-checkpoint-local/report.md | 47 +++ .../lazy-120s/cells.json | 36 +++ .../lazy-30s/cells.json | 68 ++++ .../lazy-60s/cells.json | 68 ++++ .../ref-syncfs-3s/cells.json | 68 ++++ .../aggregate.csv | 2 + .../aggregate.json | 39 +++ .../wal-machinery-baseline-local/report.md | 37 +++ .../wal-baseline/cells.json | 37 +++ results/wal-mem-diag-local/aggregate.csv | 5 + results/wal-mem-diag-local/aggregate.json | 126 ++++++++ results/wal-mem-diag-local/memory/cells.json | 58 ++++ results/wal-mem-diag-local/report.md | 43 +++ .../wal-mem-diag-local/wal-syncfs/cells.json | 58 ++++ results/wal-multilane-shards/s1/cells.json | 170 ++++++++++ results/wal-multilane-shards/s2/cells.json | 138 ++++++++ results/wal-multilane-shards/s4/cells.json | 144 +++++++++ results/wal-multilane-shards/s6/cells.json | 36 +++ results/wal-shard-sweep/aggregate.csv | 6 + results/wal-shard-sweep/aggregate.json | 271 ++++++++++++++++ results/wal-shard-sweep/report.md | 41 +++ results/wal-shard-sweep/wal-s1/cells.json | 55 ++++ results/wal-shard-sweep/wal-s16/cells.json | 55 ++++ results/wal-shard-sweep/wal-s24/cells.json | 55 ++++ results/wal-shard-sweep/wal-s4/cells.json | 55 ++++ results/wal-shard-sweep/wal-s8/cells.json | 49 +++ results/wal-sizetrigger/aggregate.csv | 5 + results/wal-sizetrigger/aggregate.json | 138 ++++++++ results/wal-sizetrigger/ref-3s/cells.json | 64 ++++ results/wal-sizetrigger/report.md | 43 +++ results/wal-sizetrigger/size-1g/cells.json | 64 ++++ results/wal-splitlane/aggregate.csv | 7 + results/wal-splitlane/aggregate.json | 206 ++++++++++++ results/wal-splitlane/ckpt-off/cells.json | 64 ++++ results/wal-splitlane/nofsync/cells.json | 64 ++++ results/wal-splitlane/ref-3s/cells.json | 64 ++++ results/wal-splitlane/report.md | 53 +++ results/wal-syncfs-local/aggregate.csv | 5 + results/wal-syncfs-local/aggregate.json | 138 ++++++++ results/wal-syncfs-local/baseline/cells.json | 64 ++++ results/wal-syncfs-local/report.md | 43 +++ results/wal-syncfs-local/syncfs/cells.json | 64 ++++ .../write-cliff-nvme/memory-gated/cells.json | 80 +++++ .../write-cliff-nvme/memory-nogate/cells.json | 86 +++++ .../write-cliff-nvme/wal-fsync1/cells.json | 104 ++++++ .../write-cliff-nvme/wal-fsync16/cells.json | 54 ++++ .../write-nvme-bottleneck/memory/cells.json | 48 +++ .../write-nvme-bottleneck/wal-s4/cells.json | 36 +++ scripts/cluster-up.sh | 38 ++- scripts/lib-bench.sh | 22 +- suites/wal-batched-mw-local.json | 42 +++ suites/wal-checkpoint-fix-100k.json | 48 +++ suites/wal-checkpoint-fix-ab.json | 63 ++++ suites/wal-cpubind.json | 50 +++ suites/wal-decomp-lane0.json | 61 ++++ suites/wal-fanout-sweep.json | 59 ++++ suites/wal-fsync-diag-local.json | 51 +++ suites/wal-lazy-checkpoint-local.json | 55 ++++ suites/wal-lazy-checkpoint-multilane.json | 63 ++++ suites/wal-machinery-baseline-local.json | 38 +++ suites/wal-mem-diag-local.json | 47 +++ suites/wal-multilane-shards.json | 79 +++++ suites/wal-shard-sweep.json | 64 ++++ suites/wal-sizetrigger.json | 54 ++++ suites/wal-splitlane.json | 58 ++++ suites/wal-syncfs-local.json | 47 +++ suites/wal-syncfs-nvme.json | 57 ++++ suites/write-cliff-nvme.json | 55 ++++ suites/write-nvme-bottleneck.json | 60 ++++ 122 files changed, 8598 insertions(+), 11 deletions(-) create mode 100644 MULTILANE_SETUP.md create mode 100644 gke/durable-streams-multilane.yaml create mode 100644 gke/durable-streams-splitlane-guaranteed.yaml create mode 100644 gke/durable-streams-splitlane.yaml create mode 100644 results/wal-batched-mw-local/aggregate.csv create mode 100644 results/wal-batched-mw-local/aggregate.json create mode 100644 results/wal-batched-mw-local/baseline/cells.json create mode 100644 results/wal-batched-mw-local/batched-mw/cells.json create mode 100644 results/wal-batched-mw-local/report.md create mode 100644 results/wal-checkpoint-fix-100k/aggregate.csv create mode 100644 results/wal-checkpoint-fix-100k/aggregate.json create mode 100644 results/wal-checkpoint-fix-100k/both/cells.json create mode 100644 results/wal-checkpoint-fix-100k/report.md create mode 100644 results/wal-checkpoint-fix-100k/syncfs/cells.json create mode 100644 results/wal-checkpoint-fix-ab.run1/aggregate.csv create mode 100644 results/wal-checkpoint-fix-ab.run1/aggregate.json create mode 100644 results/wal-checkpoint-fix-ab.run1/baseline/cells.json create mode 100644 results/wal-checkpoint-fix-ab.run1/both/cells.json create mode 100644 results/wal-checkpoint-fix-ab.run1/report.md create mode 100644 results/wal-checkpoint-fix-ab.run1/stagger/cells.json create mode 100644 results/wal-checkpoint-fix-ab.run1/syncfs/cells.json create mode 100644 results/wal-checkpoint-fix-ab/baseline/cells.json create mode 100644 results/wal-cpubind/aggregate.csv create mode 100644 results/wal-cpubind/aggregate.json create mode 100644 results/wal-cpubind/bound-3s/cells.json create mode 100644 results/wal-cpubind/report.md create mode 100644 results/wal-decomp-lane0/aggregate.csv create mode 100644 results/wal-decomp-lane0/aggregate.json create mode 100644 results/wal-decomp-lane0/ckpt-off/cells.json create mode 100644 results/wal-decomp-lane0/memory/cells.json create mode 100644 results/wal-decomp-lane0/nofsync/cells.json create mode 100644 results/wal-decomp-lane0/ref-3s/cells.json create mode 100644 results/wal-decomp-lane0/report.md create mode 100644 results/wal-fanout-sweep/aggregate.csv create mode 100644 results/wal-fanout-sweep/aggregate.json create mode 100644 results/wal-fanout-sweep/report.md create mode 100644 results/wal-fanout-sweep/s4-f1/cells.json create mode 100644 results/wal-fanout-sweep/s4-f2/cells.json create mode 100644 results/wal-fanout-sweep/s4-f4/cells.json create mode 100644 results/wal-fanout-sweep/s4-f8/cells.json create mode 100644 results/wal-fsync-diag-local/aggregate.csv create mode 100644 results/wal-fsync-diag-local/aggregate.json create mode 100644 results/wal-fsync-diag-local/memory/cells.json create mode 100644 results/wal-fsync-diag-local/report.md create mode 100644 results/wal-fsync-diag-local/wal-nofsync/cells.json create mode 100644 results/wal-fsync-diag-local/wal-normal/cells.json create mode 100644 results/wal-lazy-checkpoint-local/aggregate.csv create mode 100644 results/wal-lazy-checkpoint-local/aggregate.json create mode 100644 results/wal-lazy-checkpoint-local/lazy-syncfs-30s/cells.json create mode 100644 results/wal-lazy-checkpoint-local/lazy-syncfs-60s/cells.json create mode 100644 results/wal-lazy-checkpoint-local/ref-baseline-3s/cells.json create mode 100644 results/wal-lazy-checkpoint-local/ref-syncfs-3s/cells.json create mode 100644 results/wal-lazy-checkpoint-local/report.md create mode 100644 results/wal-lazy-checkpoint-multilane/lazy-120s/cells.json create mode 100644 results/wal-lazy-checkpoint-multilane/lazy-30s/cells.json create mode 100644 results/wal-lazy-checkpoint-multilane/lazy-60s/cells.json create mode 100644 results/wal-lazy-checkpoint-multilane/ref-syncfs-3s/cells.json create mode 100644 results/wal-machinery-baseline-local/aggregate.csv create mode 100644 results/wal-machinery-baseline-local/aggregate.json create mode 100644 results/wal-machinery-baseline-local/report.md create mode 100644 results/wal-machinery-baseline-local/wal-baseline/cells.json create mode 100644 results/wal-mem-diag-local/aggregate.csv create mode 100644 results/wal-mem-diag-local/aggregate.json create mode 100644 results/wal-mem-diag-local/memory/cells.json create mode 100644 results/wal-mem-diag-local/report.md create mode 100644 results/wal-mem-diag-local/wal-syncfs/cells.json create mode 100644 results/wal-multilane-shards/s1/cells.json create mode 100644 results/wal-multilane-shards/s2/cells.json create mode 100644 results/wal-multilane-shards/s4/cells.json create mode 100644 results/wal-multilane-shards/s6/cells.json create mode 100644 results/wal-shard-sweep/aggregate.csv create mode 100644 results/wal-shard-sweep/aggregate.json create mode 100644 results/wal-shard-sweep/report.md create mode 100644 results/wal-shard-sweep/wal-s1/cells.json create mode 100644 results/wal-shard-sweep/wal-s16/cells.json create mode 100644 results/wal-shard-sweep/wal-s24/cells.json create mode 100644 results/wal-shard-sweep/wal-s4/cells.json create mode 100644 results/wal-shard-sweep/wal-s8/cells.json create mode 100644 results/wal-sizetrigger/aggregate.csv create mode 100644 results/wal-sizetrigger/aggregate.json create mode 100644 results/wal-sizetrigger/ref-3s/cells.json create mode 100644 results/wal-sizetrigger/report.md create mode 100644 results/wal-sizetrigger/size-1g/cells.json create mode 100644 results/wal-splitlane/aggregate.csv create mode 100644 results/wal-splitlane/aggregate.json create mode 100644 results/wal-splitlane/ckpt-off/cells.json create mode 100644 results/wal-splitlane/nofsync/cells.json create mode 100644 results/wal-splitlane/ref-3s/cells.json create mode 100644 results/wal-splitlane/report.md create mode 100644 results/wal-syncfs-local/aggregate.csv create mode 100644 results/wal-syncfs-local/aggregate.json create mode 100644 results/wal-syncfs-local/baseline/cells.json create mode 100644 results/wal-syncfs-local/report.md create mode 100644 results/wal-syncfs-local/syncfs/cells.json create mode 100644 results/write-cliff-nvme/memory-gated/cells.json create mode 100644 results/write-cliff-nvme/memory-nogate/cells.json create mode 100644 results/write-cliff-nvme/wal-fsync1/cells.json create mode 100644 results/write-cliff-nvme/wal-fsync16/cells.json create mode 100644 results/write-nvme-bottleneck/memory/cells.json create mode 100644 results/write-nvme-bottleneck/wal-s4/cells.json create mode 100644 suites/wal-batched-mw-local.json create mode 100644 suites/wal-checkpoint-fix-100k.json create mode 100644 suites/wal-checkpoint-fix-ab.json create mode 100644 suites/wal-cpubind.json create mode 100644 suites/wal-decomp-lane0.json create mode 100644 suites/wal-fanout-sweep.json create mode 100644 suites/wal-fsync-diag-local.json create mode 100644 suites/wal-lazy-checkpoint-local.json create mode 100644 suites/wal-lazy-checkpoint-multilane.json create mode 100644 suites/wal-machinery-baseline-local.json create mode 100644 suites/wal-mem-diag-local.json create mode 100644 suites/wal-multilane-shards.json create mode 100644 suites/wal-shard-sweep.json create mode 100644 suites/wal-sizetrigger.json create mode 100644 suites/wal-splitlane.json create mode 100644 suites/wal-syncfs-local.json create mode 100644 suites/wal-syncfs-nvme.json create mode 100644 suites/write-cliff-nvme.json create mode 100644 suites/write-nvme-bottleneck.json diff --git a/AGENTS.md b/AGENTS.md index 527b90e..e6f3227 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -350,9 +350,12 @@ The client fleet is ~4× the server's cost, so optimize there. In descending imp | 8 vCPU | 100k | 43k | 5.0 ms | 526k | 1.2 ms | | 8 vCPU | 500k | 29k | 6.9 ms | 323k | 1.3 ms | -- **wal does not scale with the CPU pin** at shards = cores (fsync-lane bound at - ~25–30 % CPU); the shard count is the parallel-fsync knob (`run-durable-tune`: - s16t4 ≈ 380k @ 200k streams). **memory scales with cores** at ~1–2 ms p50. +- **wal does not scale with the CPU pin** at shards = cores (fsync-bound at + ~15–30 % CPU). Note: shard count is **not** the throughput knob either — a + controlled sweep (below, `results/wal-shard-sweep/`) shows s1→s24 flat within 5 % + (~72–75k @ 200k, 8 vCPU); the old `run-durable-tune` "s16t4 ≈ 380k" does not + reproduce. wal is bounded by the disk's `fdatasync`/s rate; the lever is + group-commit `batch_avg` (offered load). **memory scales with cores** at ~1–2 ms p50. - **Cardinality cliff:** 100k → 500k streams costs wal ~33 % and memory ~28–39 % in both pins — present in every build (registry/page-cache/fd physics, see `WRITE_BOTTLENECKS_1M.md` in the server crate), NOT coordination. @@ -362,6 +365,126 @@ The client fleet is ~4× the server's cost, so optimize there. In descending imp - **Cost** (list, europe-west4, Spot): fleet 5–7×`n2d-standard-32` ≈ $2–3/hr · server ≈ $0.5/hr · GKE ≈ $0.1/hr; the corrected 2-suite campaign ≈ $6–8. +### Fleet config by bottleneck (telemetry-backed, use `--server-stats`) + +Add `--server-stats 3` to the server args to emit a `SRV_STATS` line every 3 s: +`cpu_cores` (busy cores from `/proc/self/stat` utime+stime), `inflight` (appends +in-flight), `svc_us` (mean service time), `durwait_us` (mean time blocked in +`wait_durable_lsn`). This tells you which resource is the ceiling **on the actual +NVMe box**, so you size the fleet to the bottleneck instead of guessing. + +- **memory is CPU-bound.** On NVMe at saturation: `cpu_cores≈3.4/4`, `inflight≈0`, + `durwait≈0`, `svc_us≈8–15`. It scales with cores → **`server_cpus` is the + throughput knob.** Give memory pins real CPU (8–16 vCPU) and drive with enough + fleet to keep the cores busy; don't waste money on high shard counts. +- **wal (2026-07-13, SUPERSEDES the single-device analysis below): STORAGE + LAYOUT is the #1 lever — split stream data and WAL onto separate NVMe devices.** + The wal-decomp-lane0 + wal-splitlane suites (c4d-standard-64-lssd raw-block, + `SPLITLANE=1` = `gke/durable-streams-splitlane.yaml`: device 0 → stream files, + devices 1–5 → one WAL shard each, `--data-dir /data/wal/0 --wal-shards 5 + --wal-checkpoint-syncfs on`) measured: + + | config @100k streams | peak ops/s | + |---|---| + | original (streams on PD boot disk!) | 10.4k | + | everything on ONE shared NVMe lane | 46k | + | **split-lane + syncfs @3s** | **271.6k (flat vs 286k @10k)** | + | split-lane, checkpoint off | 306k | + | memory mode (ceiling) | 512k | + + The old "wal is fsync-bound at ~1000 fdatasync/s" ceiling was DEVICE CONTENTION: + commit fdatasync and checkpoint writeback fighting one queue — on dedicated WAL + lanes the commit-fsync tax is ~zero (ckpt-off ≥ nofsync) and the cardinality + cliff disappears (−5% from 10k→100k streams vs −90% before). Mandates: + - **Benchmark wal ONLY on multi-device instances** (`c4d-standard-64-lssd`, + `SERVER_LOCAL_SSD_BLOCK=1`); a single-lane or PD-backed box mismeasures wal + by 5–26×. CRITICAL: the base `/data` (emptyDir) sits on the PD boot disk with + raw-block pools — stream files MUST be routed onto an NVMe lane + (`--data-dir /data/wal/0`), or the checkpoint hammers the PD. + - **`--wal-shards` = number of dedicated WAL lanes** (5 on a 6-device box, one + lane reserved for data). On a single shared device, shards remain a non-lever + (the sweep below stands for that topology). + - **`--wal-checkpoint-syncfs on`** always (one barrier per checkpoint instead of + O(N-touched) fdatasync; PR #4697). + - **CPU binding: +21–24% (wal-cpubind, 2026-07-13).** Exclusive pinned cores + (STATIC_CPU=1 node pool = kubelet cpuManagerPolicy=static, deploy with + GUARANTEED=1 = requests==limits everywhere + integer server CPU) measured + 356k @10k / 328k @100k vs 286k/272k on shared cores, same layout/image/args. + Now that wal isn't fsync-bound, bind the server's cores for wal benches. + - **Checkpoint size trigger ≈ free checkpointing (wal-sizetrigger, PR #4704).** + `--wal-checkpoint-wal-bytes 1073741824` (+60s fallback interval) hits the + checkpoint-off ceiling (303k vs 306k @100k) while bounding replay to ≤1 GiB + retained WAL per shard. Prefer it over the 3s timer for wal benches. +- **wal on a SINGLE shared device is fsync-bound, CPU sits idle — and shard + count is NOT the lever there.** + (This corrects an earlier draft of this section that called `--wal-shards` the + knob and cited "s16 ≈ 380k"; a controlled sweep does not reproduce that.) + Controlled `--wal-shards` sweep on Titanium NVMe (`c4d-standard-16-lssd`, 8 vCPU + pin, 200k streams, 256 conns/pod; `results/wal-shard-sweep/`): + + | shards | 1 | 4 | 8 | 16 | 24 | + |---|---|---|---|---|---| + | peak ops/s | 72k | **75k** | 73k | 73k | 71k | + + All within 5 %. Live telemetry at every shard count: SRV_STATS `cpu_cores≈1.1–1.8/8` + (idle), `durwait_us ≈ 97–99 % of svc_us`; WAL_CONT `fsync/s≈900–1000`, + `batch_avg` 53→88. The ceiling is the disk's **durability-barrier rate** + (~1000 `fdatasync`/s) — one shared resource, not a per-shard lane. More shards + don't add it; at low load they *hurt* (s24 @1 pod = 34k vs s4 @1 pod = 49k: + offered load split across more committers → thinner group-commit batches). So: + - **Throughput = `fsync/s × batch_avg`.** `fsync/s` is a hardware constant of the + box's disk; the only software lever is `batch_avg`, which rises with offered + load (`connections`×pods) at the cost of latency (36 ms p50 at 3072 in-flight). + Tune `connections` up to your latency SLO — that is the wal throughput knob. + - **Keep `--wal-shards` small (2–4); do NOT tie it to cores.** The server default + is `= core count` (one committer OS thread per shard), which over-fragments + batches on high-core boxes for zero ceiling gain. `min(cores, 4)` is a better + default; only raise it if a shard sweep shows aggregate `fsync/s` still climbing. + - **`--wal-fsync-parallel` does NOT help — it regresses.** Controlled sweep at s4 + (200k, 8 vCPU, NVMe; `results/wal-fanout-sweep/`): f1=75k, f2=73k, f4=75k, + **f8=67k** — small fanout is noise, ≥8 regresses (earlier: f16 = 66k→59k @100k, + −19 % on 2-vCPU virtiofs). Default serial (fanout=1). It parallelizes the + *checkpoint* per-stream fsync storm, which only steals more device budget from + the commit fsyncs — the opposite of what you want. + - **Don't pay for high `server_cpus`** — CPU is idle; 4–8 vCPU is plenty for a + fsync-bound wal server. Spend the budget on the fleet. + +Rule of thumb: **memory → raise `server_cpus`; wal → split-lane layout first +(streams + WAL on separate NVMe devices, shards = WAL lanes, syncfs on); only on +a single shared device fall back to: `server_cpus` low + shards small (2–4) + +raise `connections` to your latency budget.** + +### Calibrating wal on a NEW cluster (don't port numbers — port this loop) + +The numbers above are properties of *this* cluster's disk, not universal. A +different disk (network PD, a faster/slower NVMe) shifts `fsync/s` and therefore +every derived number. Never copy shard/connection values across clusters — run this +~20-min calibration and read the counters. `suites/wal-shard-sweep.json` IS the +harness; point it at the new cluster and watch SRV_STATS/WAL_CONT (capture with a +`kubectl -n ds-bench logs --since=6s deploy/durable-streams | grep -E 'SRV_STATS|WAL_'` +poll loop). + +1. **Classify the bottleneck** from `--server-stats` (SRV_STATS) under load: + - `durwait_us ≈ svc_us` **and** `cpu_cores ≪ pin` → **fsync-bound** (usual wal) → step 2. + - `cpu_cores ≈ pin` → **CPU-bound** → raise vCPU pin / `--worker-threads`; shards may now help. + - `applock_us` large → **commit-path lock** (a code issue, not a knob). + - `inflight` low while the client offers more → **client/network** → add pods/connections. +2. **Measure the disk's flush rate** from `--wal-stats` (WAL_CONT `fsync/s`). This is + your ceiling divisor — measure it per cluster (~1000 here; could be hundreds on a + PD). If `inner_wait_us`/`dirty_wait_us` are non-trivial the committer is lock-blocked, + capping you *below* the disk rate — shards won't fix that. +3. **Tune shards to the measured `fsync/s`, not to cores.** Sweep {1,2,4,8}, watch + aggregate `fsync/s`: if one committer already saturates the disk → keep shards low; + if `fsync/s` keeps rising with shards → the commit path was serialized, add shards + until it plateaus, then stop. The plateau is the optimum. +4. **Raise throughput via `batch_avg`** — increase `connections`/pods until `durwait_us` + (latency) hits your SLO. `fsync/s` is fixed by hardware, so this is the only lever. +5. **Watch checkpoint contention** (WAL_CKPT `touched`/`fsync_us`) at high cardinality: + a large `fsync_us` fraction means the per-stream fdatasync storm is stealing device + budget from commits (the cliff). `--wal-fsync-parallel` does NOT help (f8/f16 + regress — it just adds concurrent checkpoint fsyncs); the real fix is coalescing + the per-stream durability barrier to O(1) syscalls (`syncfs`/`sync_file_range`, open). + **Older (pre-barrier) reference points are inflated** — treat the 2026-06-30 "1.48M @ 200k / 1.15M @ 500k on 32 vCPU" numbers (`run-durable-pool2/FINDINGS.md`) as upper bounds only: they predate the start barrier, and misaligned fleet windows diff --git a/MULTILANE_SETUP.md b/MULTILANE_SETUP.md new file mode 100644 index 0000000..e906131 --- /dev/null +++ b/MULTILANE_SETUP.md @@ -0,0 +1,179 @@ +# Multi-lane local-NVMe WAL benchmark setup + +Purpose: run the durable-streams server on a GKE node that has **multiple, +physically-separate** local NVMe devices, with **one device per WAL shard +directory**, so each shard's `fdatasync` hits an independent device queue +(an independent *fsync lane*). This re-tests whether WAL throughput scales +with `--wal-shards` when the disk is genuinely multi-lane. + +The single-device `c4d-standard-16-lssd` runs we did before could NOT show +shard scaling: `--ephemeral-storage-local-ssd` makes GKE **RAID0-stripe all +local SSDs into ONE filesystem**, so every shard's fsync serialises behind the +same single md/dm write barrier. That is a single-lane artifact, not a real +ceiling. To break it we must attach the NVMe as **raw block** (one `/dev` node +per physical device) and lay one filesystem per device under each shard dir. + +--- + +## 1. Machine + provisioning command + +Machine: **`c4d-standard-64-lssd`** — 64 vCPU, 6 physically-attached Titanium +local NVMe devices (each device is its own NVMe controller = independent queue). + +### The raw-block flag (researched) + +For **3rd/4th-generation** machine series (C3, C3D, C4, **C4D**, …) the local +SSD count is **fixed by the machine type**. gcloud therefore rejects an explicit +count — you pass `--local-nvme-ssd-block` **with NO `count=` field**, and GKE +provisions exactly the number of devices that come attached to the VM shape +(6 for `c4d-standard-64-lssd`): + +> "If you use a machine type from a third or fourth generation machine series, +> use the `--local-nvme-ssd-block` option, without a count field, to create a +> cluster." — GKE docs, *Provision and use Local SSD-backed raw block storage*. + +(By contrast, `--local-nvme-ssd-block=count=N` is only valid for 1st/2nd-gen +series such as N1/N2/N2D, where you choose N.) + +This REPLACES `--ephemeral-storage-local-ssd` (which would RAID0-stripe the +6 devices into one filesystem). Raw block leaves the 6 devices as separate +`/dev` nodes so we can put one filesystem per device. + +### Exact command (what `cluster-up.sh` now emits with the env gates set) + +```bash +SERVER_MACHINE=c4d-standard-64-lssd \ +SERVER_LOCAL_SSD_BLOCK=1 \ +SPOT_SERVER=1 \ +gcloud container clusters create bench-multilane \ + --zone europe-west4-b --project "$PROJECT" --num-nodes 1 \ + --machine-type c4d-standard-64-lssd \ + --local-nvme-ssd-block \ + --spot \ + --node-labels=role=server \ + --network benchmarking --subnetwork benchmarking \ + --enable-ip-alias --release-channel regular +``` + +(The clients pool `n2d-standard-32 ×3` is created by `cluster-up.sh` exactly as +today; only the SERVER pool gains the raw-block flag.) + +### Resulting device paths on the GKE COS node + +The COS host exposes the raw-block local NVMe devices under stable symlinks: + +- Ordinal symlinks (**use these** — stable per boot): + `/dev/disk/by-id/google-local-ssd-block0`, + `/dev/disk/by-id/google-local-ssd-block1`, … `google-local-ssd-block5` +- UUID symlinks (generated, not stable across recreate): + `/dev/disk/by-uuid/google-local-ssds-nvme-block/local-ssd-` +- Underlying raw nodes: `/dev/nvme0n1 … /dev/nvme5n1` (ordering NOT guaranteed — + do not hard-code these; resolve via the `by-id` symlink instead). + +Source: GKE docs — *Provision and use Local SSD-backed raw block storage* +(`https://cloud.google.com/kubernetes-engine/docs/how-to/persistent-volumes/local-ssd-raw`) +and *About Local SSD for GKE* +(`https://cloud.google.com/kubernetes-engine/docs/concepts/local-ssd`). + +`gke/durable-streams-multilane.yaml` iterates the `google-local-ssd-block*` +glob, so it adapts to whatever count the machine actually exposes. + +--- + +## 2. Mount approach (chosen) + +A single **privileged initContainer** (`mount-shards`) runs before the server: + +For each device `/dev/disk/by-id/google-local-ssd-blockI` (I = 0..5): + +1. `mkfs.ext4 -F -q ` — **only if the device is not already ext4** + (checked with `blkid`), so a pod restart that re-uses the same node's + devices does not needlessly reformat. +2. `mkdir -p /data/wal/` +3. `mount /data/wal/` +4. wipe the freshly-mounted dir (`rm -rf` its contents; `lost+found` is left). + +This gives **one filesystem per physical NVMe device**, mounted at the exact +shard directory the server uses: the server opens `/wal//` per shard +(confirmed in `src/wal/walset.rs`: `wal_dir = data_dir.join("wal")`, each shard +at `wal_dir.join(i.to_string())`, persisted-N file at `/wal/shards`). +So **no server code change** is needed — `--data-dir /data` stays as is; shard `i` +just happens to land on device `i`. The tiny `/wal/shards` metadata +file lives on the base `data` volume (not on any NVMe device) — that is fine, it +is written once at creation and never fsync-hot. + +**Shard count MUST be ≤ device count (6).** With `--wal-shards N`, the server +uses dirs `wal/0 … wal/N-1`; the init container mounts all 6 devices, so any +N ≤ 6 maps every shard onto its own device. `--wal-shards 7+` would put two +shards on the base volume / same device and defeat the experiment — the suite +caps the sweep at s6. + +### Mount-propagation requirement (the crux) + +A mount made inside a container is normally invisible to sibling containers. To +make the init container's `/data/wal/*` mounts visible to the server: + +- The **init container** mounts the shared `data` volume with + `mountPropagation: Bidirectional`. Bidirectional = the mount is propagated OUT + to the host mount namespace (rshared), so it **survives the init container + exiting** and is visible to later containers of the same pod. +- The **server container** mounts the same `data` volume with + `mountPropagation: HostToContainer`, so it **receives** those propagated + submounts. + +> DESIGN NOTE / deliberate deviation from the task text: the task asked for +> `Bidirectional` on BOTH containers. Kubernetes requires a container using +> `Bidirectional` to be **`privileged: true`**. We do NOT want the server +> running privileged. `HostToContainer` on the server is sufficient (it only +> needs to *receive* the mounts, not create/propagate any) and keeps the server +> unprivileged. Only the short-lived `mount-shards` init container is +> privileged. If you truly want Bidirectional on the server, you must also set +> `securityContext.privileged: true` on it. + +Kubernetes only honours these propagation modes when the kubelet's mount is +rshared (COS default: yes). + +--- + +## VERIFY AT PROVISION TIME + +1. **Device symlink name.** Docs give `/dev/disk/by-id/google-local-ssd-block0..5` + for raw block. (The task text guessed `google-local-nvme-ssd-0..5`; that is + the *ephemeral-storage* naming, NOT raw-block.) Confirm on the node: + `ls -l /dev/disk/by-id/ | grep local-ssd` and, if the names differ, update the + `DEV_GLOB` in `gke/durable-streams-multilane.yaml`. +2. **Device count = 6.** Confirm `c4d-standard-64-lssd` exposes 6 devices: + `ls /dev/disk/by-id/google-local-ssd-block*` on the node (or + `gcloud compute machine-types describe c4d-standard-64-lssd`). Keep the shard + sweep ≤ device count. +3. **mkfs.ext4 / mount / blkid availability.** The `mount-shards` init container + uses image `mirror.gcr.io/library/debian:12-slim` and `apt-get install -y + e2fsprogs util-linux` at runtime, because **the server image (`${IMG_SERVER}`) + and metrics image (`${IMG_METRICS}` = `ds-bench:dev`) are NOT known to ship + `mkfs.ext4`/`mount`/`blkid`**. VERIFY the node has egress to + `mirror.gcr.io` (GKE nodes do by default) and to the Debian apt mirrors. If + the node is egress-restricted, bake e2fsprogs+util-linux into a pinned image + in Artifact Registry and swap the `image:` field instead of apt-get at + runtime. +4. **Bidirectional mount survives init-container exit.** This relies on the + documented rshared propagation-to-host behavior. VERIFY after deploy: + `kubectl exec deploy/durable-streams -c durable-streams -- mount | grep /data/wal` + should list 6 ext4 mounts. If empty, the propagation did not survive — fall + back to doing the mkfs+mount in a **sidecar that stays alive** (not an init + container) OR set `privileged: true` + `Bidirectional` on the server too. +5. **hostPath `/dev` access.** The init container mounts hostPath `/dev` at + `/dev` and runs `privileged: true` so the raw `by-id` symlinks (which point + into `/dev/nvmeXn1`) resolve inside the container. VERIFY the COS node allows + privileged pods (GKE Standard does; Autopilot does NOT — this needs a + Standard cluster, which `cluster-up.sh` creates). +6. **Pod Security / seccomp.** The server pod already sets + `seccompProfile.type: Unconfined` (for io_uring). The privileged init + container is additionally exempt from restricted PSS. VERIFY no namespace + `PodSecurity: restricted` label blocks it (the `ds-bench` ns is unlabeled + today). +7. **Metrics sidecar diskstats.** The sidecar `df`s `/data` to find ONE backing + device for `/proc/diskstats` sectors-written. With 6 sub-mounts the top-level + `/data` still resolves to the base emptyDir device, so device-wide write bytes + now under-count the per-shard NVMe traffic. This does not affect the WAL + throughput/latency verdict (measured client-side); only the sidecar's + write-bytes column is affected. Left as-is; flagged for awareness. diff --git a/gke/durable-streams-multilane.yaml b/gke/durable-streams-multilane.yaml new file mode 100644 index 0000000..56155aa --- /dev/null +++ b/gke/durable-streams-multilane.yaml @@ -0,0 +1,251 @@ +# ───────────────────────────────────────────────────────────────────────────── +# durable-streams-multilane.yaml — MULTI-LANE local-NVMe variant of +# gke/durable-streams.yaml. Puts ONE physically-separate NVMe device under EACH +# WAL shard directory so every shard's fdatasync hits an independent device +# queue (an independent fsync lane). See MULTILANE_SETUP.md for the full rationale +# and the "VERIFY AT PROVISION TIME" checklist. +# +# REQUIRES the server node pool to be created with RAW-BLOCK local NVMe, NOT +# ephemeral-storage (which RAID0-stripes all devices into ONE fsync barrier): +# SERVER_MACHINE=c4d-standard-64-lssd SERVER_LOCAL_SSD_BLOCK=1 ... cluster-up.sh +# which emits `--local-nvme-ssd-block` (no count; count is fixed by the 4th-gen +# machine type = 6 devices on c4d-standard-64-lssd). +# +# HOW IT WORKS +# * The privileged `mount-shards` initContainer mkfs.ext4's + mounts each +# /dev/disk/by-id/google-local-ssd-blockI at /data/wal/I. +# * The server opens shard i at /wal// (see src/wal/walset.rs), so +# shard i lands on device i with NO server code change. --data-dir stays /data. +# * Shard count MUST be ≤ device count (6). --wal-shards 7+ would spill onto the +# base volume and defeat the experiment. The wal-multilane-shards suite caps at s6. +# +# MOUNT PROPAGATION (the crux — a mount made in one container is invisible to +# siblings unless propagated): +# * init container -> mountPropagation: Bidirectional (propagates the mounts +# OUT to the host mount namespace = rshared, so they SURVIVE the init +# container exiting and become visible to the server container). +# * server container -> mountPropagation: HostToContainer (RECEIVES those +# submounts). HostToContainer needs no privilege; Bidirectional would force +# the server to run privileged, which we avoid. Only the short-lived init +# container is privileged. +# ${...} placeholders are envsubst-substituted by deploy_server()/MANIFEST_VARS, +# identical to gke/durable-streams.yaml. +apiVersion: apps/v1 +kind: Deployment +metadata: + name: durable-streams + namespace: ds-bench + labels: + app: durable-streams +spec: + replicas: 1 + # Recreate (not RollingUpdate): the old pod must be FULLY torn down before the + # new pod's initContainers wipe/mount the shared NVMe lane devices. With surge + # rollouts the old pod's volume teardown (kubelet cleanup recursing through its + # stale host-side lane mounts) raced the NEW server and deleted its freshly + # created dirs on the same ext4 device (--data-dir /data/wal/0 → 500 ENOENT on + # every stream create; wal-decomp-lane0 2026-07-13). Bench cells tolerate the + # brief downtime between pods. + strategy: + type: Recreate + selector: + matchLabels: + app: durable-streams + template: + metadata: + labels: + app: durable-streams + spec: + nodeSelector: ${NODESEL_SERVER} + shareProcessNamespace: true + securityContext: + seccompProfile: + type: Unconfined + initContainers: + # ── wipe the base /data volume (emptyDir) between cells ──────────────── + # Runs FIRST (initContainers run sequentially). Only clears the base + # volume (the wal/shards metadata file + cold tier); the per-shard NVMe + # devices are wiped by mount-shards after they are mounted. + - name: wipe-data + image: ${IMG_SERVER} + imagePullPolicy: ${PULL_POLICY} + command: ["sh", "-c", "rm -rf /data/* /data/.[!.]* 2>/dev/null; true"] + volumeMounts: + - { name: data, mountPath: /data } + # ── mount ONE NVMe device per shard dir ─────────────────────────────── + # Privileged: needs mount(2) + access to the host raw block devices. + # VERIFY AT PROVISION TIME (see MULTILANE_SETUP.md): + # * image ships mkfs.ext4/mount/blkid — the server & metrics images do + # NOT, so we use debian-slim + apt-get e2fsprogs/util-linux at runtime; + # confirm the node has egress to mirror.gcr.io + the apt mirrors, else + # bake a pinned util image into Artifact Registry and swap image: here. + # * device symlink name = google-local-ssd-blockN (raw-block naming); + # adjust DEV_GLOB below if `ls /dev/disk/by-id | grep local-ssd` differs. + - name: mount-shards + image: mirror.gcr.io/library/debian:12-slim + imagePullPolicy: IfNotPresent + securityContext: + privileged: true + command: + - bash + - -c + - | + set -euo pipefail + # Raw-block local-NVMe device symlinks exposed by the COS host. + DEV_GLOB="/dev/disk/by-id/google-local-ssd-block*" + # mkfs.ext4 / mount / blkid are not in debian-slim by default. + export DEBIAN_FRONTEND=noninteractive + apt-get update -qq + apt-get install -y -qq e2fsprogs util-linux >/dev/null + shopt -s nullglob + devs=( $DEV_GLOB ) + if [ "${#devs[@]}" -eq 0 ]; then + echo "FATAL: no raw-block local-NVMe devices matched $DEV_GLOB" >&2 + echo " (was the node created with --local-nvme-ssd-block? see MULTILANE_SETUP.md)" >&2 + ls -l /dev/disk/by-id/ >&2 || true + exit 1 + fi + echo "found ${#devs[@]} raw-block NVMe device(s): ${devs[*]}" + for dev in "${devs[@]}"; do + # ordinal index from the symlink suffix -> shard dir /data/wal/ + i="${dev##*google-local-ssd-block}" + dir="/data/wal/${i}" + # Format ONLY if the device is not already ext4 (survive pod restarts + # that re-use the same node's devices without needless reformatting). + if ! blkid -o value -s TYPE "$dev" 2>/dev/null | grep -qx ext4; then + echo "mkfs.ext4 $dev (shard $i)" + mkfs.ext4 -F -q "$dev" + fi + mkdir -p "$dir" + # Idempotent: skip if something is already mounted here. + if ! mountpoint -q "$dir"; then + echo "mount $dev -> $dir" + mount "$dev" "$dir" + fi + # Wipe the shard dir contents (fresh cell). lost+found stays. + rm -rf "${dir:?}"/* "${dir:?}"/.[!.]* 2>/dev/null || true + done + echo "multi-lane mounts ready:" + mount | grep /data/wal || true + volumeMounts: + # Bidirectional so the mounts propagate to the host namespace and + # survive this container exiting (visible to the server container). + - { name: data, mountPath: /data, mountPropagation: Bidirectional } + # Host /dev so the raw by-id symlinks (-> /dev/nvmeXn1) resolve here. + - { name: dev, mountPath: /dev } + containers: + - name: durable-streams + image: ${IMG_SERVER} + imagePullPolicy: ${PULL_POLICY} + args: + - "--host" + - "0.0.0.0" + - "--port" + - "4438" + - "--data-dir" + - "/data" + - "--tier" + - "s3" + - "--tier-endpoint" + - "http://minio:9000" + - "--tier-region" + - "us-east-1" + - "--tier-bucket" + - "durable-streams" + - "--tier-allow-http" + env: + - name: DS_S3_ACCESS_KEY_ID + value: minioadmin + - name: DS_S3_SECRET_ACCESS_KEY + value: minioadmin + ports: + - containerPort: 4438 + readinessProbe: + tcpSocket: { port: 4438 } + initialDelaySeconds: 2 + periodSeconds: 2 + failureThreshold: 30 + volumeMounts: + # HostToContainer: RECEIVE the per-shard submounts the init container + # propagated to the host. (No privilege needed; Bidirectional here + # would force the server to run privileged — see MULTILANE_SETUP.md.) + - { name: data, mountPath: /data, mountPropagation: HostToContainer } + resources: + requests: + memory: "2Gi" + cpu: "500m" + limits: + memory: "${SERVER_MEM}" + cpu: "${SERVER_CPU}" + # ── metrics sidecar ───────────────────────────────────────────────────── + # Unchanged from gke/durable-streams.yaml. NOTE: it df's /data (the base + # volume) for /proc/diskstats, so its device-wide write-bytes column now + # under-counts the per-shard NVMe traffic (the shard writes go to the 6 + # sub-mounts, not the base device). Throughput/latency verdicts are + # client-side and unaffected. See MULTILANE_SETUP.md verify-item 7. + - name: metrics + image: ${IMG_METRICS} + imagePullPolicy: ${PULL_POLICY} + command: + - nice + - "-n" + - "19" + - bash + - /deploy/metrics/poller.sh + env: + - name: SERVER_PROC + value: durable-streams-server + - name: METRICS_INTERVAL_S + value: "1" + - name: METRICS_OUT + value: /metrics/samples.csv + - name: DATA_DIR + value: /data + - { name: POD_UID, valueFrom: { fieldRef: { fieldPath: metadata.uid } } } + volumeMounts: + - { name: metrics, mountPath: /metrics } + - { name: metrics-poller, mountPath: /deploy/metrics } + - { name: data, mountPath: /data, readOnly: true } + - { name: cgroup-host, mountPath: /host/cgroup, readOnly: true } + resources: + requests: + cpu: "10m" + memory: "32Mi" + limits: + cpu: "50m" + memory: "64Mi" + volumes: + # hostPath, NOT emptyDir: kubelet deletes emptyDir contents on pod + # teardown, and with the lane devices still mounted under the old pod's + # volume path that cleanup recursed INTO the shared ext4 lanes and wiped + # the next server's live data (see strategy: Recreate note above). A + # hostPath is never cleaned up by kubelet; per-cell freshness comes from + # the wipe-data + mount-shards initContainers, same as before. + - name: data + # /mnt/disks is the writable hostPath root on COS nodes (bare /mnt is + # a read-only filesystem → MountVolume.SetUp fails). + hostPath: { path: /mnt/disks/ds-bench-data, type: DirectoryOrCreate } + - name: metrics + emptyDir: {} + - name: metrics-poller + configMap: + name: metrics-poller + defaultMode: 0755 + - name: cgroup-host + hostPath: { path: /sys/fs/cgroup } + # Host /dev exposes the raw-block local-NVMe device nodes to the mount + # init container (its by-id symlinks point into /dev/nvmeXn1). + - name: dev + hostPath: { path: /dev } +--- +apiVersion: v1 +kind: Service +metadata: + name: durable-streams + namespace: ds-bench +spec: + selector: + app: durable-streams + ports: + - port: 4438 + targetPort: 4438 diff --git a/gke/durable-streams-splitlane-guaranteed.yaml b/gke/durable-streams-splitlane-guaranteed.yaml new file mode 100644 index 0000000..cc1b0ae --- /dev/null +++ b/gke/durable-streams-splitlane-guaranteed.yaml @@ -0,0 +1,271 @@ +# ───────────────────────────────────────────────────────────────────────────── +# durable-streams-splitlane-guaranteed.yaml — the SPLIT-LANE manifest with +# GUARANTEED QoS: every container (init + app) has requests == limits, and the +# server container requests an INTEGER CPU count, so on a node pool created with +# STATIC_CPU=1 (kubelet cpuManagerPolicy=static) the server gets EXCLUSIVE +# PINNED CORES (true CPU binding). Select via GUARANTEED=1 (lib-bench). +# Otherwise identical to durable-streams-splitlane.yaml (SPLIT-LANE: STREAM DATA +# FILES and WAL SHARDS on physically separate NVMe devices, so the checkpoint's +# stream-file writeback + syncfs NEVER contends with the commit path's WAL +# fdatasync. Motivated by wal-decomp-lane0 (2026-07-13): with everything on ONE +# lane, ref-3s = 55k @10k; with WAL on dedicated lanes (streams elsewhere), +# ref-3s = 270k ≈ the 260k no-fsync ceiling — the commit-fsync tax is mostly +# device contention, not fsync itself. +# +# REQUIRES the server node pool created with RAW-BLOCK local NVMe +# (SERVER_LOCAL_SSD_BLOCK=1, machine c4d-standard-64-lssd = 6 devices). +# +# LAYOUT (server runs with --data-dir /data/wal/0 and --wal-shards 5): +# * device 0 -> /data/wal/0 = the data root: streams/ (checkpoint +# writeback + syncfs domain), meta, cold staging. +# * device I (1..5) -> /data/wal/0/wal/ = WAL shard I-1's dir (walset +# opens shard i at /wal/), each an independent fsync lane. +# * Shard count MUST be ≤ 5 (devices minus the data lane). --wal-shards 6+ +# would spill shard 5 onto the data lane and re-introduce the contention. +# +# MOUNT PROPAGATION (the crux — a mount made in one container is invisible to +# siblings unless propagated): +# * init container -> mountPropagation: Bidirectional (propagates the mounts +# OUT to the host mount namespace = rshared, so they SURVIVE the init +# container exiting and become visible to the server container). +# * server container -> mountPropagation: HostToContainer (RECEIVES those +# submounts). HostToContainer needs no privilege; Bidirectional would force +# the server to run privileged, which we avoid. Only the short-lived init +# container is privileged. +# ${...} placeholders are envsubst-substituted by deploy_server()/MANIFEST_VARS, +# identical to gke/durable-streams.yaml. +apiVersion: apps/v1 +kind: Deployment +metadata: + name: durable-streams + namespace: ds-bench + labels: + app: durable-streams +spec: + replicas: 1 + # Recreate (not RollingUpdate): the old pod must be FULLY torn down before the + # new pod's initContainers wipe/mount the shared NVMe lane devices. With surge + # rollouts the old pod's volume teardown (kubelet cleanup recursing through its + # stale host-side lane mounts) raced the NEW server and deleted its freshly + # created dirs on the same ext4 device (--data-dir /data/wal/0 → 500 ENOENT on + # every stream create; wal-decomp-lane0 2026-07-13). Bench cells tolerate the + # brief downtime between pods. + strategy: + type: Recreate + selector: + matchLabels: + app: durable-streams + template: + metadata: + labels: + app: durable-streams + spec: + nodeSelector: ${NODESEL_SERVER} + shareProcessNamespace: true + securityContext: + seccompProfile: + type: Unconfined + initContainers: + # ── wipe the base /data volume (emptyDir) between cells ──────────────── + # Runs FIRST (initContainers run sequentially). Only clears the base + # volume (the wal/shards metadata file + cold tier); the per-shard NVMe + # devices are wiped by mount-shards after they are mounted. + - name: wipe-data + image: ${IMG_SERVER} + imagePullPolicy: ${PULL_POLICY} + command: ["sh", "-c", "rm -rf /data/* /data/.[!.]* 2>/dev/null; true"] + resources: + requests: { cpu: "100m", memory: "64Mi" } + limits: { cpu: "100m", memory: "64Mi" } + volumeMounts: + - { name: data, mountPath: /data } + # ── mount ONE NVMe device per shard dir ─────────────────────────────── + # Privileged: needs mount(2) + access to the host raw block devices. + # VERIFY AT PROVISION TIME (see MULTILANE_SETUP.md): + # * image ships mkfs.ext4/mount/blkid — the server & metrics images do + # NOT, so we use debian-slim + apt-get e2fsprogs/util-linux at runtime; + # confirm the node has egress to mirror.gcr.io + the apt mirrors, else + # bake a pinned util image into Artifact Registry and swap image: here. + # * device symlink name = google-local-ssd-blockN (raw-block naming); + # adjust DEV_GLOB below if `ls /dev/disk/by-id | grep local-ssd` differs. + - name: mount-shards + image: mirror.gcr.io/library/debian:12-slim + imagePullPolicy: IfNotPresent + securityContext: + privileged: true + resources: + requests: { cpu: "100m", memory: "128Mi" } + limits: { cpu: "100m", memory: "128Mi" } + command: + - bash + - -c + - | + set -euo pipefail + # Raw-block local-NVMe device symlinks exposed by the COS host. + DEV_GLOB="/dev/disk/by-id/google-local-ssd-block*" + # mkfs.ext4 / mount / blkid are not in debian-slim by default. + export DEBIAN_FRONTEND=noninteractive + apt-get update -qq + apt-get install -y -qq e2fsprogs util-linux >/dev/null + shopt -s nullglob + devs=( $DEV_GLOB ) + if [ "${#devs[@]}" -eq 0 ]; then + echo "FATAL: no raw-block local-NVMe devices matched $DEV_GLOB" >&2 + echo " (was the node created with --local-nvme-ssd-block? see MULTILANE_SETUP.md)" >&2 + ls -l /dev/disk/by-id/ >&2 || true + exit 1 + fi + echo "found ${#devs[@]} raw-block NVMe device(s): ${devs[*]}" + # split-lane topology: device 0 = data root (streams), devices 1..N + # nested INSIDE it as WAL shard dirs. Mount order matters: the data + # lane must be mounted before the shard mounts nest under it. + mount_one() { # $1=dev $2=dir + if ! blkid -o value -s TYPE "$1" 2>/dev/null | grep -qx ext4; then + echo "mkfs.ext4 $1 -> $2" + mkfs.ext4 -F -q "$1" + fi + mkdir -p "$2" + if ! mountpoint -q "$2"; then + echo "mount $1 -> $2" + mount "$1" "$2" + fi + # Wipe contents (fresh cell). + rm -rf "${2:?}"/* "${2:?}"/.[!.]* 2>/dev/null || true + } + # sort devices by ordinal suffix for a stable assignment + mapfile -t devs < <(printf '%s\n' "${devs[@]}" | sort -V) + mount_one "${devs[0]}" /data/wal/0 + mkdir -p /data/wal/0/wal + n=0 + for dev in "${devs[@]:1}"; do + mount_one "$dev" "/data/wal/0/wal/${n}" + n=$((n+1)) + done + echo "split-lane mounts ready (data=${devs[0]}, ${n} wal lanes):" + mount | grep /data/wal || true + volumeMounts: + # Bidirectional so the mounts propagate to the host namespace and + # survive this container exiting (visible to the server container). + - { name: data, mountPath: /data, mountPropagation: Bidirectional } + # Host /dev so the raw by-id symlinks (-> /dev/nvmeXn1) resolve here. + - { name: dev, mountPath: /dev } + containers: + - name: durable-streams + image: ${IMG_SERVER} + imagePullPolicy: ${PULL_POLICY} + args: + - "--host" + - "0.0.0.0" + - "--port" + - "4438" + - "--data-dir" + - "/data" + - "--tier" + - "s3" + - "--tier-endpoint" + - "http://minio:9000" + - "--tier-region" + - "us-east-1" + - "--tier-bucket" + - "durable-streams" + - "--tier-allow-http" + env: + - name: DS_S3_ACCESS_KEY_ID + value: minioadmin + - name: DS_S3_SECRET_ACCESS_KEY + value: minioadmin + ports: + - containerPort: 4438 + readinessProbe: + tcpSocket: { port: 4438 } + initialDelaySeconds: 2 + periodSeconds: 2 + failureThreshold: 30 + volumeMounts: + # HostToContainer: RECEIVE the per-shard submounts the init container + # propagated to the host. (No privilege needed; Bidirectional here + # would force the server to run privileged — see MULTILANE_SETUP.md.) + - { name: data, mountPath: /data, mountPropagation: HostToContainer } + resources: + # Guaranteed QoS: requests == limits, integer CPU -> exclusive cores + # under cpuManagerPolicy=static. + requests: + memory: "${SERVER_MEM}" + cpu: "${SERVER_CPU}" + limits: + memory: "${SERVER_MEM}" + cpu: "${SERVER_CPU}" + # ── metrics sidecar ───────────────────────────────────────────────────── + # Unchanged from gke/durable-streams.yaml. NOTE: it df's /data (the base + # volume) for /proc/diskstats, so its device-wide write-bytes column now + # under-counts the per-shard NVMe traffic (the shard writes go to the 6 + # sub-mounts, not the base device). Throughput/latency verdicts are + # client-side and unaffected. See MULTILANE_SETUP.md verify-item 7. + - name: metrics + image: ${IMG_METRICS} + imagePullPolicy: ${PULL_POLICY} + command: + - nice + - "-n" + - "19" + - bash + - /deploy/metrics/poller.sh + env: + - name: SERVER_PROC + value: durable-streams-server + - name: METRICS_INTERVAL_S + value: "1" + - name: METRICS_OUT + value: /metrics/samples.csv + - name: DATA_DIR + value: /data + - { name: POD_UID, valueFrom: { fieldRef: { fieldPath: metadata.uid } } } + volumeMounts: + - { name: metrics, mountPath: /metrics } + - { name: metrics-poller, mountPath: /deploy/metrics } + - { name: data, mountPath: /data, readOnly: true } + - { name: cgroup-host, mountPath: /host/cgroup, readOnly: true } + resources: + # requests == limits (pod-level Guaranteed QoS requires it of EVERY + # container; fractional CPU = no exclusive cores for this one, fine). + requests: + cpu: "50m" + memory: "64Mi" + limits: + cpu: "50m" + memory: "64Mi" + volumes: + # hostPath, NOT emptyDir: kubelet deletes emptyDir contents on pod + # teardown, and with the lane devices still mounted under the old pod's + # volume path that cleanup recursed INTO the shared ext4 lanes and wiped + # the next server's live data (see strategy: Recreate note above). A + # hostPath is never cleaned up by kubelet; per-cell freshness comes from + # the wipe-data + mount-shards initContainers, same as before. + - name: data + # /mnt/disks is the writable hostPath root on COS nodes (bare /mnt is + # a read-only filesystem → MountVolume.SetUp fails). + hostPath: { path: /mnt/disks/ds-bench-data, type: DirectoryOrCreate } + - name: metrics + emptyDir: {} + - name: metrics-poller + configMap: + name: metrics-poller + defaultMode: 0755 + - name: cgroup-host + hostPath: { path: /sys/fs/cgroup } + # Host /dev exposes the raw-block local-NVMe device nodes to the mount + # init container (its by-id symlinks point into /dev/nvmeXn1). + - name: dev + hostPath: { path: /dev } +--- +apiVersion: v1 +kind: Service +metadata: + name: durable-streams + namespace: ds-bench +spec: + selector: + app: durable-streams + ports: + - port: 4438 + targetPort: 4438 diff --git a/gke/durable-streams-splitlane.yaml b/gke/durable-streams-splitlane.yaml new file mode 100644 index 0000000..b4bea76 --- /dev/null +++ b/gke/durable-streams-splitlane.yaml @@ -0,0 +1,256 @@ +# ───────────────────────────────────────────────────────────────────────────── +# durable-streams-splitlane.yaml — SPLIT-LANE local-NVMe variant: STREAM DATA +# FILES and WAL SHARDS on physically separate NVMe devices, so the checkpoint's +# stream-file writeback + syncfs NEVER contends with the commit path's WAL +# fdatasync. Motivated by wal-decomp-lane0 (2026-07-13): with everything on ONE +# lane, ref-3s = 55k @10k; with WAL on dedicated lanes (streams elsewhere), +# ref-3s = 270k ≈ the 260k no-fsync ceiling — the commit-fsync tax is mostly +# device contention, not fsync itself. +# +# REQUIRES the server node pool created with RAW-BLOCK local NVMe +# (SERVER_LOCAL_SSD_BLOCK=1, machine c4d-standard-64-lssd = 6 devices). +# +# LAYOUT (server runs with --data-dir /data/wal/0 and --wal-shards 5): +# * device 0 -> /data/wal/0 = the data root: streams/ (checkpoint +# writeback + syncfs domain), meta, cold staging. +# * device I (1..5) -> /data/wal/0/wal/ = WAL shard I-1's dir (walset +# opens shard i at /wal/), each an independent fsync lane. +# * Shard count MUST be ≤ 5 (devices minus the data lane). --wal-shards 6+ +# would spill shard 5 onto the data lane and re-introduce the contention. +# +# MOUNT PROPAGATION (the crux — a mount made in one container is invisible to +# siblings unless propagated): +# * init container -> mountPropagation: Bidirectional (propagates the mounts +# OUT to the host mount namespace = rshared, so they SURVIVE the init +# container exiting and become visible to the server container). +# * server container -> mountPropagation: HostToContainer (RECEIVES those +# submounts). HostToContainer needs no privilege; Bidirectional would force +# the server to run privileged, which we avoid. Only the short-lived init +# container is privileged. +# ${...} placeholders are envsubst-substituted by deploy_server()/MANIFEST_VARS, +# identical to gke/durable-streams.yaml. +apiVersion: apps/v1 +kind: Deployment +metadata: + name: durable-streams + namespace: ds-bench + labels: + app: durable-streams +spec: + replicas: 1 + # Recreate (not RollingUpdate): the old pod must be FULLY torn down before the + # new pod's initContainers wipe/mount the shared NVMe lane devices. With surge + # rollouts the old pod's volume teardown (kubelet cleanup recursing through its + # stale host-side lane mounts) raced the NEW server and deleted its freshly + # created dirs on the same ext4 device (--data-dir /data/wal/0 → 500 ENOENT on + # every stream create; wal-decomp-lane0 2026-07-13). Bench cells tolerate the + # brief downtime between pods. + strategy: + type: Recreate + selector: + matchLabels: + app: durable-streams + template: + metadata: + labels: + app: durable-streams + spec: + nodeSelector: ${NODESEL_SERVER} + shareProcessNamespace: true + securityContext: + seccompProfile: + type: Unconfined + initContainers: + # ── wipe the base /data volume (emptyDir) between cells ──────────────── + # Runs FIRST (initContainers run sequentially). Only clears the base + # volume (the wal/shards metadata file + cold tier); the per-shard NVMe + # devices are wiped by mount-shards after they are mounted. + - name: wipe-data + image: ${IMG_SERVER} + imagePullPolicy: ${PULL_POLICY} + command: ["sh", "-c", "rm -rf /data/* /data/.[!.]* 2>/dev/null; true"] + volumeMounts: + - { name: data, mountPath: /data } + # ── mount ONE NVMe device per shard dir ─────────────────────────────── + # Privileged: needs mount(2) + access to the host raw block devices. + # VERIFY AT PROVISION TIME (see MULTILANE_SETUP.md): + # * image ships mkfs.ext4/mount/blkid — the server & metrics images do + # NOT, so we use debian-slim + apt-get e2fsprogs/util-linux at runtime; + # confirm the node has egress to mirror.gcr.io + the apt mirrors, else + # bake a pinned util image into Artifact Registry and swap image: here. + # * device symlink name = google-local-ssd-blockN (raw-block naming); + # adjust DEV_GLOB below if `ls /dev/disk/by-id | grep local-ssd` differs. + - name: mount-shards + image: mirror.gcr.io/library/debian:12-slim + imagePullPolicy: IfNotPresent + securityContext: + privileged: true + command: + - bash + - -c + - | + set -euo pipefail + # Raw-block local-NVMe device symlinks exposed by the COS host. + DEV_GLOB="/dev/disk/by-id/google-local-ssd-block*" + # mkfs.ext4 / mount / blkid are not in debian-slim by default. + export DEBIAN_FRONTEND=noninteractive + apt-get update -qq + apt-get install -y -qq e2fsprogs util-linux >/dev/null + shopt -s nullglob + devs=( $DEV_GLOB ) + if [ "${#devs[@]}" -eq 0 ]; then + echo "FATAL: no raw-block local-NVMe devices matched $DEV_GLOB" >&2 + echo " (was the node created with --local-nvme-ssd-block? see MULTILANE_SETUP.md)" >&2 + ls -l /dev/disk/by-id/ >&2 || true + exit 1 + fi + echo "found ${#devs[@]} raw-block NVMe device(s): ${devs[*]}" + # split-lane topology: device 0 = data root (streams), devices 1..N + # nested INSIDE it as WAL shard dirs. Mount order matters: the data + # lane must be mounted before the shard mounts nest under it. + mount_one() { # $1=dev $2=dir + if ! blkid -o value -s TYPE "$1" 2>/dev/null | grep -qx ext4; then + echo "mkfs.ext4 $1 -> $2" + mkfs.ext4 -F -q "$1" + fi + mkdir -p "$2" + if ! mountpoint -q "$2"; then + echo "mount $1 -> $2" + mount "$1" "$2" + fi + # Wipe contents (fresh cell). + rm -rf "${2:?}"/* "${2:?}"/.[!.]* 2>/dev/null || true + } + # sort devices by ordinal suffix for a stable assignment + mapfile -t devs < <(printf '%s\n' "${devs[@]}" | sort -V) + mount_one "${devs[0]}" /data/wal/0 + mkdir -p /data/wal/0/wal + n=0 + for dev in "${devs[@]:1}"; do + mount_one "$dev" "/data/wal/0/wal/${n}" + n=$((n+1)) + done + echo "split-lane mounts ready (data=${devs[0]}, ${n} wal lanes):" + mount | grep /data/wal || true + volumeMounts: + # Bidirectional so the mounts propagate to the host namespace and + # survive this container exiting (visible to the server container). + - { name: data, mountPath: /data, mountPropagation: Bidirectional } + # Host /dev so the raw by-id symlinks (-> /dev/nvmeXn1) resolve here. + - { name: dev, mountPath: /dev } + containers: + - name: durable-streams + image: ${IMG_SERVER} + imagePullPolicy: ${PULL_POLICY} + args: + - "--host" + - "0.0.0.0" + - "--port" + - "4438" + - "--data-dir" + - "/data" + - "--tier" + - "s3" + - "--tier-endpoint" + - "http://minio:9000" + - "--tier-region" + - "us-east-1" + - "--tier-bucket" + - "durable-streams" + - "--tier-allow-http" + env: + - name: DS_S3_ACCESS_KEY_ID + value: minioadmin + - name: DS_S3_SECRET_ACCESS_KEY + value: minioadmin + ports: + - containerPort: 4438 + readinessProbe: + tcpSocket: { port: 4438 } + initialDelaySeconds: 2 + periodSeconds: 2 + failureThreshold: 30 + volumeMounts: + # HostToContainer: RECEIVE the per-shard submounts the init container + # propagated to the host. (No privilege needed; Bidirectional here + # would force the server to run privileged — see MULTILANE_SETUP.md.) + - { name: data, mountPath: /data, mountPropagation: HostToContainer } + resources: + requests: + memory: "2Gi" + cpu: "500m" + limits: + memory: "${SERVER_MEM}" + cpu: "${SERVER_CPU}" + # ── metrics sidecar ───────────────────────────────────────────────────── + # Unchanged from gke/durable-streams.yaml. NOTE: it df's /data (the base + # volume) for /proc/diskstats, so its device-wide write-bytes column now + # under-counts the per-shard NVMe traffic (the shard writes go to the 6 + # sub-mounts, not the base device). Throughput/latency verdicts are + # client-side and unaffected. See MULTILANE_SETUP.md verify-item 7. + - name: metrics + image: ${IMG_METRICS} + imagePullPolicy: ${PULL_POLICY} + command: + - nice + - "-n" + - "19" + - bash + - /deploy/metrics/poller.sh + env: + - name: SERVER_PROC + value: durable-streams-server + - name: METRICS_INTERVAL_S + value: "1" + - name: METRICS_OUT + value: /metrics/samples.csv + - name: DATA_DIR + value: /data + - { name: POD_UID, valueFrom: { fieldRef: { fieldPath: metadata.uid } } } + volumeMounts: + - { name: metrics, mountPath: /metrics } + - { name: metrics-poller, mountPath: /deploy/metrics } + - { name: data, mountPath: /data, readOnly: true } + - { name: cgroup-host, mountPath: /host/cgroup, readOnly: true } + resources: + requests: + cpu: "10m" + memory: "32Mi" + limits: + cpu: "50m" + memory: "64Mi" + volumes: + # hostPath, NOT emptyDir: kubelet deletes emptyDir contents on pod + # teardown, and with the lane devices still mounted under the old pod's + # volume path that cleanup recursed INTO the shared ext4 lanes and wiped + # the next server's live data (see strategy: Recreate note above). A + # hostPath is never cleaned up by kubelet; per-cell freshness comes from + # the wipe-data + mount-shards initContainers, same as before. + - name: data + # /mnt/disks is the writable hostPath root on COS nodes (bare /mnt is + # a read-only filesystem → MountVolume.SetUp fails). + hostPath: { path: /mnt/disks/ds-bench-data, type: DirectoryOrCreate } + - name: metrics + emptyDir: {} + - name: metrics-poller + configMap: + name: metrics-poller + defaultMode: 0755 + - name: cgroup-host + hostPath: { path: /sys/fs/cgroup } + # Host /dev exposes the raw-block local-NVMe device nodes to the mount + # init container (its by-id symlinks point into /dev/nvmeXn1). + - name: dev + hostPath: { path: /dev } +--- +apiVersion: v1 +kind: Service +metadata: + name: durable-streams + namespace: ds-bench +spec: + selector: + app: durable-streams + ports: + - port: 4438 + targetPort: 4438 diff --git a/results/wal-batched-mw-local/aggregate.csv b/results/wal-batched-mw-local/aggregate.csv new file mode 100644 index 0000000..5bb1107 --- /dev/null +++ b/results/wal-batched-mw-local/aggregate.csv @@ -0,0 +1,3 @@ +mode,stream_count,pods,throughput,p50,p99,knee_pods,knee_throughput,knee_p50,knee_p99,pod_mem_mb,pod_mem_p50_mb,saturated,status,reason +baseline,50000,4,42749.3,7.447,177.535,2,27296.3,5.539,126.719,249,237,True,ok,plateau +batched-mw,50000,4,49211.6,6.783,145.279,2,39354.7,4.555,76.351,250,239,True,ok,plateau diff --git a/results/wal-batched-mw-local/aggregate.json b/results/wal-batched-mw-local/aggregate.json new file mode 100644 index 0000000..e6ccaa6 --- /dev/null +++ b/results/wal-batched-mw-local/aggregate.json @@ -0,0 +1,76 @@ +[ + { + "mode": "baseline", + "stream_count": 50000, + "pods": 4, + "throughput": 42749.3, + "p50": 7.447, + "p99": 177.535, + "knee_pods": 2, + "knee_throughput": 27296.3, + "knee_p50": 5.539, + "knee_p99": 126.719, + "pod_mem_mb": 249, + "pod_mem_p50_mb": 237, + "saturated": true, + "status": "ok", + "reason": "plateau", + "walk": [ + [ + 2, + 27296.3, + 5.539, + 126.719 + ], + [ + 4, + 49559.2, + 6.655, + 157.439 + ], + [ + 6, + 48453.6, + 9.727, + 186.879 + ] + ] + }, + { + "mode": "batched-mw", + "stream_count": 50000, + "pods": 4, + "throughput": 49211.6, + "p50": 6.783, + "p99": 145.279, + "knee_pods": 2, + "knee_throughput": 39354.7, + "knee_p50": 4.555, + "knee_p99": 76.351, + "pod_mem_mb": 250, + "pod_mem_p50_mb": 239, + "saturated": true, + "status": "ok", + "reason": "plateau", + "walk": [ + [ + 2, + 39354.7, + 4.555, + 76.351 + ], + [ + 4, + 49414.50000000001, + 7.011, + 133.887 + ], + [ + 6, + 53682.600000000006, + 9.303, + 151.039 + ] + ] + } +] \ No newline at end of file diff --git a/results/wal-batched-mw-local/baseline/cells.json b/results/wal-batched-mw-local/baseline/cells.json new file mode 100644 index 0000000..dc68c9a --- /dev/null +++ b/results/wal-batched-mw-local/baseline/cells.json @@ -0,0 +1,37 @@ +{ + "cells": { + "50000": { + "stream_count": 50000, + "image_digest": "1ee33d6c2b18", + "walk": [ + [ + 2, + 27296.3, + 5.539, + 126.719 + ], + [ + 4, + 49559.2, + 6.655, + 157.439 + ], + [ + 6, + 48453.6, + 9.727, + 186.879 + ] + ], + "pinned_pods": 4, + "throughput": 42749.3, + "p50": 7.447, + "p99": 177.535, + "pod_mem_mb": 249, + "saturated": true, + "status": "ok", + "reason": "plateau", + "pod_mem_p50_mb": 237 + } + } +} \ No newline at end of file diff --git a/results/wal-batched-mw-local/batched-mw/cells.json b/results/wal-batched-mw-local/batched-mw/cells.json new file mode 100644 index 0000000..1f6fffc --- /dev/null +++ b/results/wal-batched-mw-local/batched-mw/cells.json @@ -0,0 +1,37 @@ +{ + "cells": { + "50000": { + "stream_count": 50000, + "image_digest": "dbd4b6a5127d", + "walk": [ + [ + 2, + 39354.7, + 4.555, + 76.351 + ], + [ + 4, + 49414.50000000001, + 7.011, + 133.887 + ], + [ + 6, + 53682.600000000006, + 9.303, + 151.039 + ] + ], + "pinned_pods": 4, + "throughput": 49211.6, + "p50": 6.783, + "p99": 145.279, + "pod_mem_mb": 250, + "saturated": true, + "status": "ok", + "reason": "plateau", + "pod_mem_p50_mb": 239 + } + } +} \ No newline at end of file diff --git a/results/wal-batched-mw-local/report.md b/results/wal-batched-mw-local/report.md new file mode 100644 index 0000000..211fb6e --- /dev/null +++ b/results/wal-batched-mw-local/report.md @@ -0,0 +1,38 @@ +# wal-batched-mw-local — write-throughput report + +## Throughput at saturation (ops/s) + +| streams | baseline | batched-mw | +|---|---|---| +| 50000 | 43k | 49k | + +† = not saturated (ladder exhausted) — treat as a lower bound. + +## Pod memory at saturation — peak / p50 (MiB) + +| streams | baseline | batched-mw | +|---|---|---| +| 50000 | 249 / 237 | 250 / 239 | + +_Pod working set = cgroup `memory.current − inactive_file` (anon + active page cache), sampled each second at the pinned rung. **peak** = high-water (catches bursts like an in-RAM Raft log filling); **p50** = median (what the server steadily holds resident). peak ≈ p50 ⇒ steadily resident; peak ≫ p50 ⇒ transient spikes._ + +## Latency (ms, p50 / p99) + +| streams | baseline @≤80% load | baseline @saturation | batched-mw @≤80% load | batched-mw @saturation | +|---|---|---|---|---| +| 50000 | 5.5 / 126.7 (27k @2p) | 7.4 / 177.5 | 4.6 / 76.4 (39k @2p) | 6.8 / 145.3 | + +_@≤80% load = the largest ladder rung at ≤80% of peak throughput — the server's service latency with headroom. @saturation = the pinned plateau rung, where a closed-loop fleet measures its own queueing (≈ in-flight ÷ ceiling by Little's law), NOT the server's per-request cost. Compare against the unloaded single-request baseline (~1 ms for wal) before reading anything into large saturation values._ + +## Saturation walks (pods → ops/s, p50 ms) + +- **baseline 50000**: 2:27k@5.5ms → 4:50k@6.7ms → 6:48k@9.7ms (pinned 4, plateau) +- **batched-mw 50000**: 2:39k@4.6ms → 4:49k@7.0ms → 6:54k@9.3ms (pinned 4, plateau) + +## Findings + +_TODO: written by hand on top of the generated data._ + +## Caveats + +_Single-node best-case; not 3-node Raft. Throughput is a saturation ceiling per the ladder._ diff --git a/results/wal-checkpoint-fix-100k/aggregate.csv b/results/wal-checkpoint-fix-100k/aggregate.csv new file mode 100644 index 0000000..044b153 --- /dev/null +++ b/results/wal-checkpoint-fix-100k/aggregate.csv @@ -0,0 +1,3 @@ +mode,stream_count,pods,throughput,p50,p99,knee_pods,knee_throughput,knee_p50,knee_p99,pod_mem_mb,pod_mem_p50_mb,saturated,status,reason +both,100000,4,11913.5,,,4,8395.85,3.735,1650.687,0.0,,False,ok,ladder_exhausted +syncfs,100000,4,13604.349999999999,,,2,10814.7,1.742,1461.247,549.0,,False,ok,ladder_exhausted diff --git a/results/wal-checkpoint-fix-100k/aggregate.json b/results/wal-checkpoint-fix-100k/aggregate.json new file mode 100644 index 0000000..71ea777 --- /dev/null +++ b/results/wal-checkpoint-fix-100k/aggregate.json @@ -0,0 +1,64 @@ +[ + { + "mode": "both", + "stream_count": 100000, + "pods": 4, + "throughput": 11913.5, + "p50": null, + "p99": null, + "knee_pods": 4, + "knee_throughput": 8395.85, + "knee_p50": 3.735, + "knee_p99": 1650.687, + "pod_mem_mb": 0.0, + "pod_mem_p50_mb": null, + "saturated": false, + "status": "ok", + "reason": "ladder_exhausted", + "walk": [ + [ + 2, + 11913.5, + 1.574, + 983.551 + ], + [ + 4, + 8395.85, + 3.735, + 1650.687 + ] + ] + }, + { + "mode": "syncfs", + "stream_count": 100000, + "pods": 4, + "throughput": 13604.349999999999, + "p50": null, + "p99": null, + "knee_pods": 2, + "knee_throughput": 10814.7, + "knee_p50": 1.742, + "knee_p99": 1461.247, + "pod_mem_mb": 549.0, + "pod_mem_p50_mb": null, + "saturated": false, + "status": "ok", + "reason": "ladder_exhausted", + "walk": [ + [ + 2, + 10814.7, + 1.742, + 1461.247 + ], + [ + 4, + 13604.349999999999, + 3.669, + 1008.127 + ] + ] + } +] \ No newline at end of file diff --git a/results/wal-checkpoint-fix-100k/both/cells.json b/results/wal-checkpoint-fix-100k/both/cells.json new file mode 100644 index 0000000..da03a22 --- /dev/null +++ b/results/wal-checkpoint-fix-100k/both/cells.json @@ -0,0 +1,30 @@ +{ + "cells": { + "100000": { + "stream_count": 100000, + "image_digest": "b5aaa0ca74a8", + "walk": [ + [ + 2, + 11913.5, + 1.574, + 983.551 + ], + [ + 4, + 8395.85, + 3.735, + 1650.687 + ] + ], + "pinned_pods": 4, + "throughput": 8395.85, + "p50": null, + "p99": null, + "pod_mem_mb": 0.0, + "saturated": false, + "status": "ok", + "reason": "ladder_exhausted" + } + } +} \ No newline at end of file diff --git a/results/wal-checkpoint-fix-100k/report.md b/results/wal-checkpoint-fix-100k/report.md new file mode 100644 index 0000000..34925d9 --- /dev/null +++ b/results/wal-checkpoint-fix-100k/report.md @@ -0,0 +1,44 @@ +# wal-checkpoint-fix-100k — write-throughput report + +## Throughput at saturation (ops/s) + +| streams | syncfs | both | +|---|---|---| +| 100000 | 14k† | 12k† | + +† = not saturated (ladder exhausted) — treat as a lower bound. + +## Pod memory at saturation — peak / p50 (MiB) + +| streams | syncfs | both | +|---|---|---| +| 100000 | 549 | — | + +_Pod working set = cgroup `memory.current − inactive_file` (anon + active page cache), sampled each second at the pinned rung. **peak** = high-water (catches bursts like an in-RAM Raft log filling); **p50** = median (what the server steadily holds resident). peak ≈ p50 ⇒ steadily resident; peak ≫ p50 ⇒ transient spikes._ + +## Latency (ms, p50 / p99) + +| streams | syncfs @≤80% load | syncfs @saturation | both @≤80% load | both @saturation | +|---|---|---|---|---| +| 100000 | 1.7 / 1461.2 (11k @2p) | — | 3.7 / 1650.7 (8k @4p) | — | + +_@≤80% load = the largest ladder rung at ≤80% of peak throughput — the server's service latency with headroom. @saturation = the pinned plateau rung, where a closed-loop fleet measures its own queueing (≈ in-flight ÷ ceiling by Little's law), NOT the server's per-request cost. Compare against the unloaded single-request baseline (~1 ms for wal) before reading anything into large saturation values._ + +## Saturation walks (pods → ops/s, p50 ms) + +- **both 100000**: 2:12k@1.6ms → 4:8k@3.7ms (pinned 4, ladder_exhausted) +- **syncfs 100000**: 2:11k@1.7ms → 4:14k@3.7ms (pinned 4, ladder_exhausted) + +## Findings + +Reference (aborted wal-checkpoint-fix-ab run, same cluster/image family): **baseline@100k = 10.4k** ops/s at p2, degrading to 9.0k (p4) / 7.3k (p8) — throughput *falls* as load rises because the synchronized O(N)-fdatasync checkpoint wave stalls commits. + +- **syncfs wins**: 13.6k at p4 (+51% vs baseline's 9.0k at matched p4), and still climbing when the ladder ended — a lower bound. Shape flips from degrades-with-load to scales-with-load. +- **stagger adds nothing**: `both` ≤ syncfs alone. Consistent with mechanism: one shard's checkpoint pass (~13s at 100k) exceeds the 3s interval, so staggering cannot keep pace. Stagger prototype dropped from the branch. +- **Cliff softened, not eliminated**: ~110k @10k streams vs ~14k @100k, and p99 @≤80% load is still ~1.5s (checkpoint stalls remain). The residual cost is the O(N_touched) random writeback of per-stream files itself, which no barrier strategy fixes — that is the log-structured-store argument (issue #4695); cold-tier-as-durable is #4696. + +Merged: `--wal-checkpoint-syncfs` (branch perf/wal-checkpoint-syncfs → campaign branch), flag-gated, default off. + +## Caveats + +_Single-node best-case; not 3-node Raft. Throughput is a saturation ceiling per the ladder._ diff --git a/results/wal-checkpoint-fix-100k/syncfs/cells.json b/results/wal-checkpoint-fix-100k/syncfs/cells.json new file mode 100644 index 0000000..175bc10 --- /dev/null +++ b/results/wal-checkpoint-fix-100k/syncfs/cells.json @@ -0,0 +1,30 @@ +{ + "cells": { + "100000": { + "stream_count": 100000, + "image_digest": "3e4e1bf0d499", + "walk": [ + [ + 2, + 10814.7, + 1.742, + 1461.247 + ], + [ + 4, + 13604.349999999999, + 3.669, + 1008.127 + ] + ], + "pinned_pods": 4, + "throughput": 13604.349999999999, + "p50": null, + "p99": null, + "pod_mem_mb": 549.0, + "saturated": false, + "status": "ok", + "reason": "ladder_exhausted" + } + } +} \ No newline at end of file diff --git a/results/wal-checkpoint-fix-ab.run1/aggregate.csv b/results/wal-checkpoint-fix-ab.run1/aggregate.csv new file mode 100644 index 0000000..c5d3929 --- /dev/null +++ b/results/wal-checkpoint-fix-ab.run1/aggregate.csv @@ -0,0 +1,9 @@ +mode,stream_count,pods,throughput,p50,p99,knee_pods,knee_throughput,knee_p50,knee_p99,pod_mem_mb,pod_mem_p50_mb,saturated,status,reason +baseline,10000,12,111007.09999999999,,,2,106165.5,2.673,24.671,0.0,,False,ok,ladder_exhausted +both,10000,,292450.15,,,8,0.0,0.0,0.0,,,False,error,creation_choke +stagger,10000,2,107514.0,3.271,21.455,2,108458.35,3.329,22.031,0.0,,True,ok,plateau +syncfs,10000,2,285084.0,1.496,8.087,2,299792.4,1.477,7.067,0.0,,True,ok,plateau +baseline,100000,2,10901.1,36.255,215.807,8,7258.349999999999,282.367,654.335,0.0,,True,ok,plateau +both,100000,2,11012.0,1.752,1217.535,8,8989.0,187.007,1298.431,0.0,,True,ok,plateau +stagger,100000,2,11694.9,37.279,363.519,4,8609.6,104.511,722.943,0.0,,True,ok,plateau +syncfs,100000,2,15408.9,1.588,1419.263,4,9134.4,4.727,1599.487,485,434,True,ok,plateau diff --git a/results/wal-checkpoint-fix-ab.run1/aggregate.json b/results/wal-checkpoint-fix-ab.run1/aggregate.json new file mode 100644 index 0000000..278514a --- /dev/null +++ b/results/wal-checkpoint-fix-ab.run1/aggregate.json @@ -0,0 +1,304 @@ +[ + { + "mode": "baseline", + "stream_count": 10000, + "pods": 12, + "throughput": 111007.09999999999, + "p50": null, + "p99": null, + "knee_pods": 2, + "knee_throughput": 106165.5, + "knee_p50": 2.673, + "knee_p99": 24.671, + "pod_mem_mb": 0.0, + "pod_mem_p50_mb": null, + "saturated": false, + "status": "ok", + "reason": "ladder_exhausted", + "walk": [ + [ + 2, + 106165.5, + 2.673, + 24.671 + ], + [ + 4, + 96072.65, + 9.575, + 34.303 + ], + [ + 8, + 111007.09999999999, + 17.295, + 52.703 + ], + [ + 12, + 95111.55000000002, + 30.975, + 80.895 + ] + ] + }, + { + "mode": "both", + "stream_count": 10000, + "pods": null, + "throughput": 292450.15, + "p50": null, + "p99": null, + "knee_pods": 8, + "knee_throughput": 0.0, + "knee_p50": 0.0, + "knee_p99": 0.0, + "pod_mem_mb": null, + "pod_mem_p50_mb": null, + "saturated": false, + "status": "error", + "reason": "creation_choke", + "walk": [ + [ + 2, + 288573.2, + 1.468, + 8.279 + ], + [ + 4, + 292450.15, + 3.109, + 11.903 + ], + [ + 8, + 0.0, + 0.0, + 0.0 + ] + ] + }, + { + "mode": "stagger", + "stream_count": 10000, + "pods": 2, + "throughput": 107514.0, + "p50": 3.271, + "p99": 21.455, + "knee_pods": 2, + "knee_throughput": 108458.35, + "knee_p50": 3.329, + "knee_p99": 22.031, + "pod_mem_mb": 0.0, + "pod_mem_p50_mb": null, + "saturated": true, + "status": "ok", + "reason": "plateau", + "walk": [ + [ + 2, + 108458.35, + 3.329, + 22.031 + ], + [ + 4, + 100049.5, + 8.115, + 34.975 + ], + [ + 8, + 98728.04999999999, + 19.663, + 52.607 + ] + ] + }, + { + "mode": "syncfs", + "stream_count": 10000, + "pods": 2, + "throughput": 285084.0, + "p50": 1.496, + "p99": 8.087, + "knee_pods": 2, + "knee_throughput": 299792.4, + "knee_p50": 1.477, + "knee_p99": 7.067, + "pod_mem_mb": 0.0, + "pod_mem_p50_mb": null, + "saturated": true, + "status": "ok", + "reason": "plateau", + "walk": [ + [ + 2, + 299792.4, + 1.477, + 7.067 + ], + [ + 4, + 294786.80000000005, + 3.145, + 9.911 + ], + [ + 8, + 280042.3, + 6.499, + 28.127 + ] + ] + }, + { + "mode": "baseline", + "stream_count": 100000, + "pods": 2, + "throughput": 10901.1, + "p50": 36.255, + "p99": 215.807, + "knee_pods": 8, + "knee_throughput": 7258.349999999999, + "knee_p50": 282.367, + "knee_p99": 654.335, + "pod_mem_mb": 0.0, + "pod_mem_p50_mb": null, + "saturated": true, + "status": "ok", + "reason": "plateau", + "walk": [ + [ + 2, + 10405.25, + 38.335, + 212.991 + ], + [ + 4, + 8980.95, + 99.263, + 684.543 + ], + [ + 8, + 7258.349999999999, + 282.367, + 654.335 + ] + ] + }, + { + "mode": "both", + "stream_count": 100000, + "pods": 2, + "throughput": 11012.0, + "p50": 1.752, + "p99": 1217.535, + "knee_pods": 8, + "knee_throughput": 8989.0, + "knee_p50": 187.007, + "knee_p99": 1298.431, + "pod_mem_mb": 0.0, + "pod_mem_p50_mb": null, + "saturated": true, + "status": "ok", + "reason": "plateau", + "walk": [ + [ + 2, + 14105.35, + 1.621, + 1198.079 + ], + [ + 4, + 11892.1, + 3.803, + 1565.695 + ], + [ + 8, + 8989.0, + 187.007, + 1298.431 + ] + ] + }, + { + "mode": "stagger", + "stream_count": 100000, + "pods": 2, + "throughput": 11694.9, + "p50": 37.279, + "p99": 363.519, + "knee_pods": 4, + "knee_throughput": 8609.6, + "knee_p50": 104.511, + "knee_p99": 722.943, + "pod_mem_mb": 0.0, + "pod_mem_p50_mb": null, + "saturated": true, + "status": "ok", + "reason": "plateau", + "walk": [ + [ + 2, + 11781.900000000001, + 30.703, + 445.695 + ], + [ + 4, + 8609.6, + 104.511, + 722.943 + ], + [ + 8, + 6816.599999999999, + 267.775, + 1178.623 + ] + ] + }, + { + "mode": "syncfs", + "stream_count": 100000, + "pods": 2, + "throughput": 15408.9, + "p50": 1.588, + "p99": 1419.263, + "knee_pods": 4, + "knee_throughput": 9134.4, + "knee_p50": 4.727, + "knee_p99": 1599.487, + "pod_mem_mb": 485, + "pod_mem_p50_mb": 434, + "saturated": true, + "status": "ok", + "reason": "plateau", + "walk": [ + [ + 2, + 18637.4, + 1.588, + 1301.503 + ], + [ + 4, + 9134.4, + 4.727, + 1599.487 + ], + [ + 8, + 7473.8, + 242.047, + 986.623 + ] + ] + } +] \ No newline at end of file diff --git a/results/wal-checkpoint-fix-ab.run1/baseline/cells.json b/results/wal-checkpoint-fix-ab.run1/baseline/cells.json new file mode 100644 index 0000000..8cf77ff --- /dev/null +++ b/results/wal-checkpoint-fix-ab.run1/baseline/cells.json @@ -0,0 +1,74 @@ +{ + "cells": { + "10000": { + "stream_count": 10000, + "image_digest": "e4ae264d73e0", + "walk": [ + [ + 2, + 106165.5, + 2.673, + 24.671 + ], + [ + 4, + 96072.65, + 9.575, + 34.303 + ], + [ + 8, + 111007.09999999999, + 17.295, + 52.703 + ], + [ + 12, + 95111.55000000002, + 30.975, + 80.895 + ] + ], + "pinned_pods": 12, + "throughput": 95111.55000000002, + "p50": null, + "p99": null, + "pod_mem_mb": 0.0, + "saturated": false, + "status": "ok", + "reason": "ladder_exhausted" + }, + "100000": { + "stream_count": 100000, + "image_digest": "e4ae264d73e0", + "walk": [ + [ + 2, + 10405.25, + 38.335, + 212.991 + ], + [ + 4, + 8980.95, + 99.263, + 684.543 + ], + [ + 8, + 7258.349999999999, + 282.367, + 654.335 + ] + ], + "pinned_pods": 2, + "throughput": 10901.1, + "p50": 36.255, + "p99": 215.807, + "pod_mem_mb": 0.0, + "saturated": true, + "status": "ok", + "reason": "plateau" + } + } +} \ No newline at end of file diff --git a/results/wal-checkpoint-fix-ab.run1/both/cells.json b/results/wal-checkpoint-fix-ab.run1/both/cells.json new file mode 100644 index 0000000..683764b --- /dev/null +++ b/results/wal-checkpoint-fix-ab.run1/both/cells.json @@ -0,0 +1,68 @@ +{ + "cells": { + "10000": { + "stream_count": 10000, + "image_digest": "b5aaa0ca74a8", + "walk": [ + [ + 2, + 288573.2, + 1.468, + 8.279 + ], + [ + 4, + 292450.15, + 3.109, + 11.903 + ], + [ + 8, + 0.0, + 0.0, + 0.0 + ] + ], + "pinned_pods": null, + "throughput": 0.0, + "p50": null, + "p99": null, + "pod_mem_mb": null, + "saturated": false, + "status": "error", + "reason": "creation_choke" + }, + "100000": { + "stream_count": 100000, + "image_digest": "b5aaa0ca74a8", + "walk": [ + [ + 2, + 14105.35, + 1.621, + 1198.079 + ], + [ + 4, + 11892.1, + 3.803, + 1565.695 + ], + [ + 8, + 8989.0, + 187.007, + 1298.431 + ] + ], + "pinned_pods": 2, + "throughput": 11012.0, + "p50": 1.752, + "p99": 1217.535, + "pod_mem_mb": 0.0, + "saturated": true, + "status": "ok", + "reason": "plateau" + } + } +} \ No newline at end of file diff --git a/results/wal-checkpoint-fix-ab.run1/report.md b/results/wal-checkpoint-fix-ab.run1/report.md new file mode 100644 index 0000000..c7b7b39 --- /dev/null +++ b/results/wal-checkpoint-fix-ab.run1/report.md @@ -0,0 +1,47 @@ +# wal-checkpoint-fix-ab — write-throughput report + +## Throughput at saturation (ops/s) + +| streams | baseline | stagger | syncfs | both | +|---|---|---|---|---| +| 10000 | 111k† | 108k | 285k | ERROR (creation_choke) | +| 100000 | 11k | 12k | 15k | 11k | + +† = not saturated (ladder exhausted) — treat as a lower bound. + +## Pod memory at saturation — peak / p50 (MiB) + +| streams | baseline | stagger | syncfs | both | +|---|---|---|---|---| +| 10000 | — | — | — | — | +| 100000 | — | — | 485 / 434 | — | + +_Pod working set = cgroup `memory.current − inactive_file` (anon + active page cache), sampled each second at the pinned rung. **peak** = high-water (catches bursts like an in-RAM Raft log filling); **p50** = median (what the server steadily holds resident). peak ≈ p50 ⇒ steadily resident; peak ≫ p50 ⇒ transient spikes._ + +## Latency (ms, p50 / p99) + +| streams | baseline @≤80% load | baseline @saturation | stagger @≤80% load | stagger @saturation | syncfs @≤80% load | syncfs @saturation | both @≤80% load | both @saturation | +|---|---|---|---|---|---|---|---|---| +| 10000 | 2.7 / 24.7 (106k @2p) | — | 3.3 / 22.0 (108k @2p) | 3.3 / 21.5 | 1.5 / 7.1 (300k @2p) | 1.5 / 8.1 | 0.0 / 0.0 (0k @8p) | — | +| 100000 | 282.4 / 654.3 (7k @8p) | 36.3 / 215.8 | 104.5 / 722.9 (9k @4p) | 37.3 / 363.5 | 4.7 / 1599.5 (9k @4p) | 1.6 / 1419.3 | 187.0 / 1298.4 (9k @8p) | 1.8 / 1217.5 | + +_@≤80% load = the largest ladder rung at ≤80% of peak throughput — the server's service latency with headroom. @saturation = the pinned plateau rung, where a closed-loop fleet measures its own queueing (≈ in-flight ÷ ceiling by Little's law), NOT the server's per-request cost. Compare against the unloaded single-request baseline (~1 ms for wal) before reading anything into large saturation values._ + +## Saturation walks (pods → ops/s, p50 ms) + +- **baseline 10000**: 2:106k@2.7ms → 4:96k@9.6ms → 8:111k@17.3ms → 12:95k@31.0ms (pinned 12, ladder_exhausted) +- **both 10000**: 2:289k@1.5ms → 4:292k@3.1ms → 8:0k@0.0ms (pinned None, creation_choke) +- **stagger 10000**: 2:108k@3.3ms → 4:100k@8.1ms → 8:99k@19.7ms (pinned 2, plateau) +- **syncfs 10000**: 2:300k@1.5ms → 4:295k@3.1ms → 8:280k@6.5ms (pinned 2, plateau) +- **baseline 100000**: 2:10k@38.3ms → 4:9k@99.3ms → 8:7k@282.4ms (pinned 2, plateau) +- **both 100000**: 2:14k@1.6ms → 4:12k@3.8ms → 8:9k@187.0ms (pinned 2, plateau) +- **stagger 100000**: 2:12k@30.7ms → 4:9k@104.5ms → 8:7k@267.8ms (pinned 2, plateau) +- **syncfs 100000**: 2:19k@1.6ms → 4:9k@4.7ms → 8:7k@242.0ms (pinned 2, plateau) + +## Findings + +_TODO: written by hand on top of the generated data._ + +## Caveats + +_Single-node best-case; not 3-node Raft. Throughput is a saturation ceiling per the ladder._ diff --git a/results/wal-checkpoint-fix-ab.run1/stagger/cells.json b/results/wal-checkpoint-fix-ab.run1/stagger/cells.json new file mode 100644 index 0000000..95a4c03 --- /dev/null +++ b/results/wal-checkpoint-fix-ab.run1/stagger/cells.json @@ -0,0 +1,68 @@ +{ + "cells": { + "10000": { + "stream_count": 10000, + "image_digest": "2ebed436ee56", + "walk": [ + [ + 2, + 108458.35, + 3.329, + 22.031 + ], + [ + 4, + 100049.5, + 8.115, + 34.975 + ], + [ + 8, + 98728.04999999999, + 19.663, + 52.607 + ] + ], + "pinned_pods": 2, + "throughput": 107514.0, + "p50": 3.271, + "p99": 21.455, + "pod_mem_mb": 0.0, + "saturated": true, + "status": "ok", + "reason": "plateau" + }, + "100000": { + "stream_count": 100000, + "image_digest": "2ebed436ee56", + "walk": [ + [ + 2, + 11781.900000000001, + 30.703, + 445.695 + ], + [ + 4, + 8609.6, + 104.511, + 722.943 + ], + [ + 8, + 6816.599999999999, + 267.775, + 1178.623 + ] + ], + "pinned_pods": 2, + "throughput": 11694.9, + "p50": 37.279, + "p99": 363.519, + "pod_mem_mb": 0.0, + "saturated": true, + "status": "ok", + "reason": "plateau" + } + } +} \ No newline at end of file diff --git a/results/wal-checkpoint-fix-ab.run1/syncfs/cells.json b/results/wal-checkpoint-fix-ab.run1/syncfs/cells.json new file mode 100644 index 0000000..ba30868 --- /dev/null +++ b/results/wal-checkpoint-fix-ab.run1/syncfs/cells.json @@ -0,0 +1,69 @@ +{ + "cells": { + "10000": { + "stream_count": 10000, + "image_digest": "3e4e1bf0d499", + "walk": [ + [ + 2, + 299792.4, + 1.477, + 7.067 + ], + [ + 4, + 294786.80000000005, + 3.145, + 9.911 + ], + [ + 8, + 280042.3, + 6.499, + 28.127 + ] + ], + "pinned_pods": 2, + "throughput": 285084.0, + "p50": 1.496, + "p99": 8.087, + "pod_mem_mb": 0.0, + "saturated": true, + "status": "ok", + "reason": "plateau" + }, + "100000": { + "stream_count": 100000, + "image_digest": "3e4e1bf0d499", + "walk": [ + [ + 2, + 18637.4, + 1.588, + 1301.503 + ], + [ + 4, + 9134.4, + 4.727, + 1599.487 + ], + [ + 8, + 7473.8, + 242.047, + 986.623 + ] + ], + "pinned_pods": 2, + "throughput": 15408.9, + "p50": 1.588, + "p99": 1419.263, + "pod_mem_mb": 485, + "saturated": true, + "status": "ok", + "reason": "plateau", + "pod_mem_p50_mb": 434 + } + } +} \ No newline at end of file diff --git a/results/wal-checkpoint-fix-ab/baseline/cells.json b/results/wal-checkpoint-fix-ab/baseline/cells.json new file mode 100644 index 0000000..382cb81 --- /dev/null +++ b/results/wal-checkpoint-fix-ab/baseline/cells.json @@ -0,0 +1,30 @@ +{ + "cells": { + "10000": { + "stream_count": 10000, + "image_digest": "ed0fed8321da", + "walk": [ + [ + 2, + 112568.2, + 3.069, + 19.983 + ], + [ + 4, + 0.0, + 0.0, + 0.0 + ] + ], + "pinned_pods": null, + "throughput": 0.0, + "p50": null, + "p99": null, + "pod_mem_mb": null, + "saturated": false, + "status": "error", + "reason": "creation_choke" + } + } +} \ No newline at end of file diff --git a/results/wal-cpubind/aggregate.csv b/results/wal-cpubind/aggregate.csv new file mode 100644 index 0000000..a0ec00f --- /dev/null +++ b/results/wal-cpubind/aggregate.csv @@ -0,0 +1,3 @@ +mode,stream_count,pods,throughput,p50,p99,knee_pods,knee_throughput,knee_p50,knee_p99,pod_mem_mb,pod_mem_p50_mb,saturated,status,reason +bound-3s,10000,8,369978.75,,,4,369978.75,2.525,5.227,249,198,False,ok,ladder_exhausted +bound-3s,100000,2,328045.0,1.278,2.837,2,324751.9,1.274,2.835,694,596,True,ok,plateau diff --git a/results/wal-cpubind/aggregate.json b/results/wal-cpubind/aggregate.json new file mode 100644 index 0000000..9ed2599 --- /dev/null +++ b/results/wal-cpubind/aggregate.json @@ -0,0 +1,70 @@ +[ + { + "mode": "bound-3s", + "stream_count": 10000, + "pods": 8, + "throughput": 369978.75, + "p50": null, + "p99": null, + "knee_pods": 4, + "knee_throughput": 369978.75, + "knee_p50": 2.525, + "knee_p99": 5.227, + "pod_mem_mb": 249, + "pod_mem_p50_mb": 198, + "saturated": false, + "status": "ok", + "reason": "ladder_exhausted", + "walk": [ + [ + 4, + 369978.75, + 2.525, + 5.227 + ], + [ + 8, + 356028.55000000005, + 5.387, + 20.399 + ] + ] + }, + { + "mode": "bound-3s", + "stream_count": 100000, + "pods": 2, + "throughput": 328045.0, + "p50": 1.278, + "p99": 2.837, + "knee_pods": 2, + "knee_throughput": 324751.9, + "knee_p50": 1.274, + "knee_p99": 2.835, + "pod_mem_mb": 694, + "pod_mem_p50_mb": 596, + "saturated": true, + "status": "ok", + "reason": "plateau", + "walk": [ + [ + 2, + 324751.9, + 1.274, + 2.835 + ], + [ + 4, + 342369.39999999997, + 2.573, + 4.667 + ], + [ + 8, + 317598.9, + 5.539, + 9.287 + ] + ] + } +] \ No newline at end of file diff --git a/results/wal-cpubind/bound-3s/cells.json b/results/wal-cpubind/bound-3s/cells.json new file mode 100644 index 0000000..001e27f --- /dev/null +++ b/results/wal-cpubind/bound-3s/cells.json @@ -0,0 +1,64 @@ +{ + "cells": { + "10000": { + "stream_count": 10000, + "image_digest": "1bd30dc4c80f", + "walk": [ + [ + 4, + 369978.75, + 2.525, + 5.227 + ], + [ + 8, + 356028.55000000005, + 5.387, + 20.399 + ] + ], + "pinned_pods": 8, + "throughput": 356028.55000000005, + "p50": null, + "p99": null, + "pod_mem_mb": 249, + "saturated": false, + "status": "ok", + "reason": "ladder_exhausted", + "pod_mem_p50_mb": 198 + }, + "100000": { + "stream_count": 100000, + "image_digest": "1bd30dc4c80f", + "walk": [ + [ + 2, + 324751.9, + 1.274, + 2.835 + ], + [ + 4, + 342369.39999999997, + 2.573, + 4.667 + ], + [ + 8, + 317598.9, + 5.539, + 9.287 + ] + ], + "pinned_pods": 2, + "throughput": 328045.0, + "p50": 1.278, + "p99": 2.837, + "pod_mem_mb": 694, + "saturated": true, + "status": "ok", + "reason": "plateau", + "pod_mem_p50_mb": 596 + } + } +} \ No newline at end of file diff --git a/results/wal-cpubind/report.md b/results/wal-cpubind/report.md new file mode 100644 index 0000000..32f5ba6 --- /dev/null +++ b/results/wal-cpubind/report.md @@ -0,0 +1,41 @@ +# wal-cpubind — write-throughput report + +## Throughput at saturation (ops/s) + +| streams | bound-3s | +|---|---| +| 10000 | 370k† | +| 100000 | 328k | + +† = not saturated (ladder exhausted) — treat as a lower bound. + +## Pod memory at saturation — peak / p50 (MiB) + +| streams | bound-3s | +|---|---| +| 10000 | 249 / 198 | +| 100000 | 694 / 596 | + +_Pod working set = cgroup `memory.current − inactive_file` (anon + active page cache), sampled each second at the pinned rung. **peak** = high-water (catches bursts like an in-RAM Raft log filling); **p50** = median (what the server steadily holds resident). peak ≈ p50 ⇒ steadily resident; peak ≫ p50 ⇒ transient spikes._ + +## Latency (ms, p50 / p99) + +| streams | bound-3s @≤80% load | bound-3s @saturation | +|---|---|---| +| 10000 | 2.5 / 5.2 (370k @4p) | — | +| 100000 | 1.3 / 2.8 (325k @2p) | 1.3 / 2.8 | + +_@≤80% load = the largest ladder rung at ≤80% of peak throughput — the server's service latency with headroom. @saturation = the pinned plateau rung, where a closed-loop fleet measures its own queueing (≈ in-flight ÷ ceiling by Little's law), NOT the server's per-request cost. Compare against the unloaded single-request baseline (~1 ms for wal) before reading anything into large saturation values._ + +## Saturation walks (pods → ops/s, p50 ms) + +- **bound-3s 10000**: 4:370k@2.5ms → 8:356k@5.4ms (pinned 8, ladder_exhausted) +- **bound-3s 100000**: 2:325k@1.3ms → 4:342k@2.6ms → 8:318k@5.5ms (pinned 2, plateau) + +## Findings + +Exclusive pinned cores (STATIC_CPU=1 + GUARANTEED=1, cpuManagerPolicy=static, integer 8-CPU Guaranteed pod) = 356.0k @10k / 328.0k @100k vs 286.3k/271.6k on shared cores (wal-splitlane ref-3s, same layout/image/args): +24% / +21%. With wal no longer fsync-bound, CPU binding is a real lever. Caveat: cross-cluster comparison (bench-cpubind vs bench-multilane), same instance type/zone. + +## Caveats + +_Single-node best-case; not 3-node Raft. Throughput is a saturation ceiling per the ladder._ diff --git a/results/wal-decomp-lane0/aggregate.csv b/results/wal-decomp-lane0/aggregate.csv new file mode 100644 index 0000000..bd815d2 --- /dev/null +++ b/results/wal-decomp-lane0/aggregate.csv @@ -0,0 +1,9 @@ +mode,stream_count,pods,throughput,p50,p99,knee_pods,knee_throughput,knee_p50,knee_p99,pod_mem_mb,pod_mem_p50_mb,saturated,status,reason +ckpt-off,10000,8,85294.9,,,4,76193.25,11.431,15.343,176,146,False,ok,ladder_exhausted +memory,10000,8,542955.0,,,4,542955.0,1.75,4.063,274,190,False,ok,ladder_exhausted +nofsync,10000,8,259910.85,,,4,259910.85,3.123,9.071,222,177,False,ok,ladder_exhausted +ref-3s,10000,8,55409.45,,,4,53065.649999999994,13.863,58.239,162,144,False,ok,ladder_exhausted +ckpt-off,100000,4,66064.6,,,2,66064.6,6.847,8.807,586,506,False,ok,ladder_exhausted +memory,100000,4,511486.10000000003,,,2,407791.45,1.174,2.361,659,597,False,ok,ladder_exhausted +nofsync,100000,4,252852.65,,,2,252852.65,1.452,7.431,644,596,False,ok,ladder_exhausted +ref-3s,100000,4,46835.899999999994,,,2,46835.899999999994,8.935,36.575,523,509,False,ok,ladder_exhausted diff --git a/results/wal-decomp-lane0/aggregate.json b/results/wal-decomp-lane0/aggregate.json new file mode 100644 index 0000000..f69776a --- /dev/null +++ b/results/wal-decomp-lane0/aggregate.json @@ -0,0 +1,250 @@ +[ + { + "mode": "ckpt-off", + "stream_count": 10000, + "pods": 8, + "throughput": 85294.9, + "p50": null, + "p99": null, + "knee_pods": 4, + "knee_throughput": 76193.25, + "knee_p50": 11.431, + "knee_p99": 15.343, + "pod_mem_mb": 176, + "pod_mem_p50_mb": 146, + "saturated": false, + "status": "ok", + "reason": "ladder_exhausted", + "walk": [ + [ + 4, + 76193.25, + 11.431, + 15.343 + ], + [ + 8, + 85294.9, + 22.527, + 31.711 + ] + ] + }, + { + "mode": "memory", + "stream_count": 10000, + "pods": 8, + "throughput": 542955.0, + "p50": null, + "p99": null, + "knee_pods": 4, + "knee_throughput": 542955.0, + "knee_p50": 1.75, + "knee_p99": 4.063, + "pod_mem_mb": 274, + "pod_mem_p50_mb": 190, + "saturated": false, + "status": "ok", + "reason": "ladder_exhausted", + "walk": [ + [ + 4, + 542955.0, + 1.75, + 4.063 + ], + [ + 8, + 541637.95, + 2.973, + 9.799 + ] + ] + }, + { + "mode": "nofsync", + "stream_count": 10000, + "pods": 8, + "throughput": 259910.85, + "p50": null, + "p99": null, + "knee_pods": 4, + "knee_throughput": 259910.85, + "knee_p50": 3.123, + "knee_p99": 9.071, + "pod_mem_mb": 222, + "pod_mem_p50_mb": 177, + "saturated": false, + "status": "ok", + "reason": "ladder_exhausted", + "walk": [ + [ + 4, + 259910.85, + 3.123, + 9.071 + ], + [ + 8, + 259645.05, + 6.439, + 13.807 + ] + ] + }, + { + "mode": "ref-3s", + "stream_count": 10000, + "pods": 8, + "throughput": 55409.45, + "p50": null, + "p99": null, + "knee_pods": 4, + "knee_throughput": 53065.649999999994, + "knee_p50": 13.863, + "knee_p99": 58.239, + "pod_mem_mb": 162, + "pod_mem_p50_mb": 144, + "saturated": false, + "status": "ok", + "reason": "ladder_exhausted", + "walk": [ + [ + 4, + 53065.649999999994, + 13.863, + 58.239 + ], + [ + 8, + 55409.45, + 36.799, + 67.327 + ] + ] + }, + { + "mode": "ckpt-off", + "stream_count": 100000, + "pods": 4, + "throughput": 66064.6, + "p50": null, + "p99": null, + "knee_pods": 2, + "knee_throughput": 66064.6, + "knee_p50": 6.847, + "knee_p99": 8.807, + "pod_mem_mb": 586, + "pod_mem_p50_mb": 506, + "saturated": false, + "status": "ok", + "reason": "ladder_exhausted", + "walk": [ + [ + 2, + 66064.6, + 6.847, + 8.807 + ], + [ + 4, + 64858.6, + 13.847, + 142.079 + ] + ] + }, + { + "mode": "memory", + "stream_count": 100000, + "pods": 4, + "throughput": 511486.10000000003, + "p50": null, + "p99": null, + "knee_pods": 2, + "knee_throughput": 407791.45, + "knee_p50": 1.174, + "knee_p99": 2.361, + "pod_mem_mb": 659, + "pod_mem_p50_mb": 597, + "saturated": false, + "status": "ok", + "reason": "ladder_exhausted", + "walk": [ + [ + 2, + 407791.45, + 1.174, + 2.361 + ], + [ + 4, + 511486.10000000003, + 1.809, + 4.227 + ] + ] + }, + { + "mode": "nofsync", + "stream_count": 100000, + "pods": 4, + "throughput": 252852.65, + "p50": null, + "p99": null, + "knee_pods": 2, + "knee_throughput": 252852.65, + "knee_p50": 1.452, + "knee_p99": 7.431, + "pod_mem_mb": 644, + "pod_mem_p50_mb": 596, + "saturated": false, + "status": "ok", + "reason": "ladder_exhausted", + "walk": [ + [ + 2, + 252852.65, + 1.452, + 7.431 + ], + [ + 4, + 252641.40000000002, + 3.177, + 9.135 + ] + ] + }, + { + "mode": "ref-3s", + "stream_count": 100000, + "pods": 4, + "throughput": 46835.899999999994, + "p50": null, + "p99": null, + "knee_pods": 2, + "knee_throughput": 46835.899999999994, + "knee_p50": 8.935, + "knee_p99": 36.575, + "pod_mem_mb": 523, + "pod_mem_p50_mb": 509, + "saturated": false, + "status": "ok", + "reason": "ladder_exhausted", + "walk": [ + [ + 2, + 46835.899999999994, + 8.935, + 36.575 + ], + [ + 4, + 46132.9, + 21.487, + 71.935 + ] + ] + } +] \ No newline at end of file diff --git a/results/wal-decomp-lane0/ckpt-off/cells.json b/results/wal-decomp-lane0/ckpt-off/cells.json new file mode 100644 index 0000000..f800fd7 --- /dev/null +++ b/results/wal-decomp-lane0/ckpt-off/cells.json @@ -0,0 +1,58 @@ +{ + "cells": { + "10000": { + "stream_count": 10000, + "image_digest": "500e885c4956", + "walk": [ + [ + 4, + 76193.25, + 11.431, + 15.343 + ], + [ + 8, + 85294.9, + 22.527, + 31.711 + ] + ], + "pinned_pods": 8, + "throughput": 85294.9, + "p50": null, + "p99": null, + "pod_mem_mb": 176, + "saturated": false, + "status": "ok", + "reason": "ladder_exhausted", + "pod_mem_p50_mb": 146 + }, + "100000": { + "stream_count": 100000, + "image_digest": "500e885c4956", + "walk": [ + [ + 2, + 66064.6, + 6.847, + 8.807 + ], + [ + 4, + 64858.6, + 13.847, + 142.079 + ] + ], + "pinned_pods": 4, + "throughput": 64858.6, + "p50": null, + "p99": null, + "pod_mem_mb": 586, + "saturated": false, + "status": "ok", + "reason": "ladder_exhausted", + "pod_mem_p50_mb": 506 + } + } +} \ No newline at end of file diff --git a/results/wal-decomp-lane0/memory/cells.json b/results/wal-decomp-lane0/memory/cells.json new file mode 100644 index 0000000..3db70b6 --- /dev/null +++ b/results/wal-decomp-lane0/memory/cells.json @@ -0,0 +1,58 @@ +{ + "cells": { + "10000": { + "stream_count": 10000, + "image_digest": "ddd833e769ff", + "walk": [ + [ + 4, + 542955.0, + 1.75, + 4.063 + ], + [ + 8, + 541637.95, + 2.973, + 9.799 + ] + ], + "pinned_pods": 8, + "throughput": 541637.95, + "p50": null, + "p99": null, + "pod_mem_mb": 274, + "saturated": false, + "status": "ok", + "reason": "ladder_exhausted", + "pod_mem_p50_mb": 190 + }, + "100000": { + "stream_count": 100000, + "image_digest": "ddd833e769ff", + "walk": [ + [ + 2, + 407791.45, + 1.174, + 2.361 + ], + [ + 4, + 511486.10000000003, + 1.809, + 4.227 + ] + ], + "pinned_pods": 4, + "throughput": 511486.10000000003, + "p50": null, + "p99": null, + "pod_mem_mb": 659, + "saturated": false, + "status": "ok", + "reason": "ladder_exhausted", + "pod_mem_p50_mb": 597 + } + } +} \ No newline at end of file diff --git a/results/wal-decomp-lane0/nofsync/cells.json b/results/wal-decomp-lane0/nofsync/cells.json new file mode 100644 index 0000000..f35ec83 --- /dev/null +++ b/results/wal-decomp-lane0/nofsync/cells.json @@ -0,0 +1,58 @@ +{ + "cells": { + "10000": { + "stream_count": 10000, + "image_digest": "f2cd2c5a0498", + "walk": [ + [ + 4, + 259910.85, + 3.123, + 9.071 + ], + [ + 8, + 259645.05, + 6.439, + 13.807 + ] + ], + "pinned_pods": 8, + "throughput": 259645.05, + "p50": null, + "p99": null, + "pod_mem_mb": 222, + "saturated": false, + "status": "ok", + "reason": "ladder_exhausted", + "pod_mem_p50_mb": 177 + }, + "100000": { + "stream_count": 100000, + "image_digest": "f2cd2c5a0498", + "walk": [ + [ + 2, + 252852.65, + 1.452, + 7.431 + ], + [ + 4, + 252641.40000000002, + 3.177, + 9.135 + ] + ], + "pinned_pods": 4, + "throughput": 252641.40000000002, + "p50": null, + "p99": null, + "pod_mem_mb": 644, + "saturated": false, + "status": "ok", + "reason": "ladder_exhausted", + "pod_mem_p50_mb": 596 + } + } +} \ No newline at end of file diff --git a/results/wal-decomp-lane0/ref-3s/cells.json b/results/wal-decomp-lane0/ref-3s/cells.json new file mode 100644 index 0000000..1cec0e4 --- /dev/null +++ b/results/wal-decomp-lane0/ref-3s/cells.json @@ -0,0 +1,58 @@ +{ + "cells": { + "10000": { + "stream_count": 10000, + "image_digest": "6fcaf53b737e", + "walk": [ + [ + 4, + 53065.649999999994, + 13.863, + 58.239 + ], + [ + 8, + 55409.45, + 36.799, + 67.327 + ] + ], + "pinned_pods": 8, + "throughput": 55409.45, + "p50": null, + "p99": null, + "pod_mem_mb": 162, + "saturated": false, + "status": "ok", + "reason": "ladder_exhausted", + "pod_mem_p50_mb": 144 + }, + "100000": { + "stream_count": 100000, + "image_digest": "6fcaf53b737e", + "walk": [ + [ + 2, + 46835.899999999994, + 8.935, + 36.575 + ], + [ + 4, + 46132.9, + 21.487, + 71.935 + ] + ], + "pinned_pods": 4, + "throughput": 46132.9, + "p50": null, + "p99": null, + "pod_mem_mb": 523, + "saturated": false, + "status": "ok", + "reason": "ladder_exhausted", + "pod_mem_p50_mb": 509 + } + } +} \ No newline at end of file diff --git a/results/wal-decomp-lane0/report.md b/results/wal-decomp-lane0/report.md new file mode 100644 index 0000000..f6706b8 --- /dev/null +++ b/results/wal-decomp-lane0/report.md @@ -0,0 +1,47 @@ +# wal-decomp-lane0 — write-throughput report + +## Throughput at saturation (ops/s) + +| streams | memory | nofsync | ckpt-off | ref-3s | +|---|---|---|---|---| +| 10000 | 543k† | 260k† | 85k† | 55k† | +| 100000 | 511k† | 253k† | 66k† | 47k† | + +† = not saturated (ladder exhausted) — treat as a lower bound. + +## Pod memory at saturation — peak / p50 (MiB) + +| streams | memory | nofsync | ckpt-off | ref-3s | +|---|---|---|---|---| +| 10000 | 274 / 190 | 222 / 177 | 176 / 146 | 162 / 144 | +| 100000 | 659 / 597 | 644 / 596 | 586 / 506 | 523 / 509 | + +_Pod working set = cgroup `memory.current − inactive_file` (anon + active page cache), sampled each second at the pinned rung. **peak** = high-water (catches bursts like an in-RAM Raft log filling); **p50** = median (what the server steadily holds resident). peak ≈ p50 ⇒ steadily resident; peak ≫ p50 ⇒ transient spikes._ + +## Latency (ms, p50 / p99) + +| streams | memory @≤80% load | memory @saturation | nofsync @≤80% load | nofsync @saturation | ckpt-off @≤80% load | ckpt-off @saturation | ref-3s @≤80% load | ref-3s @saturation | +|---|---|---|---|---|---|---|---|---| +| 10000 | 1.8 / 4.1 (543k @4p) | — | 3.1 / 9.1 (260k @4p) | — | 11.4 / 15.3 (76k @4p) | — | 13.9 / 58.2 (53k @4p) | — | +| 100000 | 1.2 / 2.4 (408k @2p) | — | 1.5 / 7.4 (253k @2p) | — | 6.8 / 8.8 (66k @2p) | — | 8.9 / 36.6 (47k @2p) | — | + +_@≤80% load = the largest ladder rung at ≤80% of peak throughput — the server's service latency with headroom. @saturation = the pinned plateau rung, where a closed-loop fleet measures its own queueing (≈ in-flight ÷ ceiling by Little's law), NOT the server's per-request cost. Compare against the unloaded single-request baseline (~1 ms for wal) before reading anything into large saturation values._ + +## Saturation walks (pods → ops/s, p50 ms) + +- **ckpt-off 10000**: 4:76k@11.4ms → 8:85k@22.5ms (pinned 8, ladder_exhausted) +- **memory 10000**: 4:543k@1.8ms → 8:542k@3.0ms (pinned 8, ladder_exhausted) +- **nofsync 10000**: 4:260k@3.1ms → 8:260k@6.4ms (pinned 8, ladder_exhausted) +- **ref-3s 10000**: 4:53k@13.9ms → 8:55k@36.8ms (pinned 8, ladder_exhausted) +- **ckpt-off 100000**: 2:66k@6.8ms → 4:65k@13.8ms (pinned 4, ladder_exhausted) +- **memory 100000**: 2:408k@1.2ms → 4:511k@1.8ms (pinned 4, ladder_exhausted) +- **nofsync 100000**: 2:253k@1.5ms → 4:253k@3.2ms (pinned 4, ladder_exhausted) +- **ref-3s 100000**: 2:47k@8.9ms → 4:46k@21.5ms (pinned 4, ladder_exhausted) + +## Findings + +_TODO: written by hand on top of the generated data._ + +## Caveats + +_Single-node best-case; not 3-node Raft. Throughput is a saturation ceiling per the ladder._ diff --git a/results/wal-fanout-sweep/aggregate.csv b/results/wal-fanout-sweep/aggregate.csv new file mode 100644 index 0000000..49e7a4a --- /dev/null +++ b/results/wal-fanout-sweep/aggregate.csv @@ -0,0 +1,5 @@ +mode,stream_count,pods,throughput,p50,p99,knee_pods,knee_throughput,knee_p50,knee_p99,pod_mem_mb,pod_mem_p50_mb,saturated,status,reason +s4-f1,200000,8,71098.5,27.855,59.295,2,55694.9,7.967,32.463,1059,1032,True,ok,plateau +s4-f2,200000,4,67583.9,14.495,41.951,2,58915.25,8.159,26.543,1032,1004,True,ok,plateau +s4-f4,200000,16,75057.70000000001,,,4,58659.0,16.199,37.983,1109,1049,False,ok,ladder_exhausted +s4-f8,200000,16,66705.1,,,2,45221.2,10.951,23.359,1101,1039,False,ok,ladder_exhausted diff --git a/results/wal-fanout-sweep/aggregate.json b/results/wal-fanout-sweep/aggregate.json new file mode 100644 index 0000000..bc125ab --- /dev/null +++ b/results/wal-fanout-sweep/aggregate.json @@ -0,0 +1,192 @@ +[ + { + "mode": "s4-f1", + "stream_count": 200000, + "pods": 8, + "throughput": 71098.5, + "p50": 27.855, + "p99": 59.295, + "knee_pods": 2, + "knee_throughput": 55694.9, + "knee_p50": 7.967, + "knee_p99": 32.463, + "pod_mem_mb": 1059, + "pod_mem_p50_mb": 1032, + "saturated": true, + "status": "ok", + "reason": "plateau", + "walk": [ + [ + 2, + 55694.9, + 7.967, + 32.463 + ], + [ + 4, + 61040.0, + 14.527, + 52.255 + ], + [ + 8, + 72946.5, + 27.343, + 57.119 + ], + [ + 12, + 71171.20000000001, + 41.375, + 99.647 + ], + [ + 16, + 75300.9, + 53.439, + 105.471 + ] + ] + }, + { + "mode": "s4-f2", + "stream_count": 200000, + "pods": 4, + "throughput": 67583.9, + "p50": 14.495, + "p99": 41.951, + "knee_pods": 2, + "knee_throughput": 58915.25, + "knee_p50": 8.159, + "knee_p99": 26.543, + "pod_mem_mb": 1032, + "pod_mem_p50_mb": 1004, + "saturated": true, + "status": "ok", + "reason": "plateau", + "walk": [ + [ + 2, + 58915.25, + 8.159, + 26.543 + ], + [ + 4, + 68520.95, + 14.431, + 41.663 + ], + [ + 8, + 68227.55, + 28.111, + 63.231 + ], + [ + 12, + 72834.0, + 41.343, + 80.255 + ] + ] + }, + { + "mode": "s4-f4", + "stream_count": 200000, + "pods": 16, + "throughput": 75057.70000000001, + "p50": null, + "p99": null, + "knee_pods": 4, + "knee_throughput": 58659.0, + "knee_p50": 16.199, + "knee_p99": 37.983, + "pod_mem_mb": 1109, + "pod_mem_p50_mb": 1049, + "saturated": false, + "status": "ok", + "reason": "ladder_exhausted", + "walk": [ + [ + 2, + 47302.15, + 9.551, + 29.807 + ], + [ + 4, + 58659.0, + 16.199, + 37.983 + ], + [ + 8, + 65971.34999999999, + 29.007, + 66.239 + ], + [ + 12, + 74076.75, + 41.503, + 73.471 + ], + [ + 16, + 75057.70000000001, + 54.527, + 99.007 + ] + ] + }, + { + "mode": "s4-f8", + "stream_count": 200000, + "pods": 16, + "throughput": 66705.1, + "p50": null, + "p99": null, + "knee_pods": 2, + "knee_throughput": 45221.2, + "knee_p50": 10.951, + "knee_p99": 23.359, + "pod_mem_mb": 1101, + "pod_mem_p50_mb": 1039, + "saturated": false, + "status": "ok", + "reason": "ladder_exhausted", + "walk": [ + [ + 2, + 45221.2, + 10.951, + 23.359 + ], + [ + 4, + 55360.25, + 17.839, + 36.191 + ], + [ + 8, + 59471.4, + 33.119, + 63.359 + ], + [ + 12, + 65443.2, + 45.599, + 81.535 + ], + [ + 16, + 66705.1, + 59.743, + 104.191 + ] + ] + } +] \ No newline at end of file diff --git a/results/wal-fanout-sweep/report.md b/results/wal-fanout-sweep/report.md new file mode 100644 index 0000000..36a7af4 --- /dev/null +++ b/results/wal-fanout-sweep/report.md @@ -0,0 +1,40 @@ +# wal-fanout-sweep — write-throughput report + +## Throughput at saturation (ops/s) + +| streams | s4-f1 | s4-f2 | s4-f4 | s4-f8 | +|---|---|---|---|---| +| 200000 | 71k | 68k | 75k† | 67k† | + +† = not saturated (ladder exhausted) — treat as a lower bound. + +## Pod memory at saturation — peak / p50 (MiB) + +| streams | s4-f1 | s4-f2 | s4-f4 | s4-f8 | +|---|---|---|---|---| +| 200000 | 1059 / 1032 | 1032 / 1004 | 1109 / 1049 | 1101 / 1039 | + +_Pod working set = cgroup `memory.current − inactive_file` (anon + active page cache), sampled each second at the pinned rung. **peak** = high-water (catches bursts like an in-RAM Raft log filling); **p50** = median (what the server steadily holds resident). peak ≈ p50 ⇒ steadily resident; peak ≫ p50 ⇒ transient spikes._ + +## Latency (ms, p50 / p99) + +| streams | s4-f1 @≤80% load | s4-f1 @saturation | s4-f2 @≤80% load | s4-f2 @saturation | s4-f4 @≤80% load | s4-f4 @saturation | s4-f8 @≤80% load | s4-f8 @saturation | +|---|---|---|---|---|---|---|---|---| +| 200000 | 8.0 / 32.5 (56k @2p) | 27.9 / 59.3 | 8.2 / 26.5 (59k @2p) | 14.5 / 42.0 | 16.2 / 38.0 (59k @4p) | — | 11.0 / 23.4 (45k @2p) | — | + +_@≤80% load = the largest ladder rung at ≤80% of peak throughput — the server's service latency with headroom. @saturation = the pinned plateau rung, where a closed-loop fleet measures its own queueing (≈ in-flight ÷ ceiling by Little's law), NOT the server's per-request cost. Compare against the unloaded single-request baseline (~1 ms for wal) before reading anything into large saturation values._ + +## Saturation walks (pods → ops/s, p50 ms) + +- **s4-f1 200000**: 2:56k@8.0ms → 4:61k@14.5ms → 8:73k@27.3ms → 12:71k@41.4ms → 16:75k@53.4ms (pinned 8, plateau) +- **s4-f2 200000**: 2:59k@8.2ms → 4:69k@14.4ms → 8:68k@28.1ms → 12:73k@41.3ms (pinned 4, plateau) +- **s4-f4 200000**: 2:47k@9.6ms → 4:59k@16.2ms → 8:66k@29.0ms → 12:74k@41.5ms → 16:75k@54.5ms (pinned 16, ladder_exhausted) +- **s4-f8 200000**: 2:45k@11.0ms → 4:55k@17.8ms → 8:59k@33.1ms → 12:65k@45.6ms → 16:67k@59.7ms (pinned 16, ladder_exhausted) + +## Findings + +_TODO: written by hand on top of the generated data._ + +## Caveats + +_Single-node best-case; not 3-node Raft. Throughput is a saturation ceiling per the ladder._ diff --git a/results/wal-fanout-sweep/s4-f1/cells.json b/results/wal-fanout-sweep/s4-f1/cells.json new file mode 100644 index 0000000..c8af02d --- /dev/null +++ b/results/wal-fanout-sweep/s4-f1/cells.json @@ -0,0 +1,49 @@ +{ + "cells": { + "200000": { + "stream_count": 200000, + "image_digest": "f5692b252ebb", + "walk": [ + [ + 2, + 55694.9, + 7.967, + 32.463 + ], + [ + 4, + 61040.0, + 14.527, + 52.255 + ], + [ + 8, + 72946.5, + 27.343, + 57.119 + ], + [ + 12, + 71171.20000000001, + 41.375, + 99.647 + ], + [ + 16, + 75300.9, + 53.439, + 105.471 + ] + ], + "pinned_pods": 8, + "throughput": 71098.5, + "p50": 27.855, + "p99": 59.295, + "pod_mem_mb": 1059, + "saturated": true, + "status": "ok", + "reason": "plateau", + "pod_mem_p50_mb": 1032 + } + } +} \ No newline at end of file diff --git a/results/wal-fanout-sweep/s4-f2/cells.json b/results/wal-fanout-sweep/s4-f2/cells.json new file mode 100644 index 0000000..21f88ae --- /dev/null +++ b/results/wal-fanout-sweep/s4-f2/cells.json @@ -0,0 +1,43 @@ +{ + "cells": { + "200000": { + "stream_count": 200000, + "image_digest": "23b2edc0096a", + "walk": [ + [ + 2, + 58915.25, + 8.159, + 26.543 + ], + [ + 4, + 68520.95, + 14.431, + 41.663 + ], + [ + 8, + 68227.55, + 28.111, + 63.231 + ], + [ + 12, + 72834.0, + 41.343, + 80.255 + ] + ], + "pinned_pods": 4, + "throughput": 67583.9, + "p50": 14.495, + "p99": 41.951, + "pod_mem_mb": 1032, + "saturated": true, + "status": "ok", + "reason": "plateau", + "pod_mem_p50_mb": 1004 + } + } +} \ No newline at end of file diff --git a/results/wal-fanout-sweep/s4-f4/cells.json b/results/wal-fanout-sweep/s4-f4/cells.json new file mode 100644 index 0000000..bb2d6f1 --- /dev/null +++ b/results/wal-fanout-sweep/s4-f4/cells.json @@ -0,0 +1,49 @@ +{ + "cells": { + "200000": { + "stream_count": 200000, + "image_digest": "e715dfccc606", + "walk": [ + [ + 2, + 47302.15, + 9.551, + 29.807 + ], + [ + 4, + 58659.0, + 16.199, + 37.983 + ], + [ + 8, + 65971.34999999999, + 29.007, + 66.239 + ], + [ + 12, + 74076.75, + 41.503, + 73.471 + ], + [ + 16, + 75057.70000000001, + 54.527, + 99.007 + ] + ], + "pinned_pods": 16, + "throughput": 75057.70000000001, + "p50": null, + "p99": null, + "pod_mem_mb": 1109, + "saturated": false, + "status": "ok", + "reason": "ladder_exhausted", + "pod_mem_p50_mb": 1049 + } + } +} \ No newline at end of file diff --git a/results/wal-fanout-sweep/s4-f8/cells.json b/results/wal-fanout-sweep/s4-f8/cells.json new file mode 100644 index 0000000..579fb4b --- /dev/null +++ b/results/wal-fanout-sweep/s4-f8/cells.json @@ -0,0 +1,49 @@ +{ + "cells": { + "200000": { + "stream_count": 200000, + "image_digest": "73aa375b7242", + "walk": [ + [ + 2, + 45221.2, + 10.951, + 23.359 + ], + [ + 4, + 55360.25, + 17.839, + 36.191 + ], + [ + 8, + 59471.4, + 33.119, + 63.359 + ], + [ + 12, + 65443.2, + 45.599, + 81.535 + ], + [ + 16, + 66705.1, + 59.743, + 104.191 + ] + ], + "pinned_pods": 16, + "throughput": 66705.1, + "p50": null, + "p99": null, + "pod_mem_mb": 1101, + "saturated": false, + "status": "ok", + "reason": "ladder_exhausted", + "pod_mem_p50_mb": 1039 + } + } +} \ No newline at end of file diff --git a/results/wal-fsync-diag-local/aggregate.csv b/results/wal-fsync-diag-local/aggregate.csv new file mode 100644 index 0000000..e8639a2 --- /dev/null +++ b/results/wal-fsync-diag-local/aggregate.csv @@ -0,0 +1,7 @@ +mode,stream_count,pods,throughput,p50,p99,knee_pods,knee_throughput,knee_p50,knee_p99,pod_mem_mb,pod_mem_p50_mb,saturated,status,reason +memory,20000,2,165326.0,1.323,5.311,2,162666.25,1.32,5.667,121,112,True,ok,plateau +wal-nofsync,20000,2,98446.8,2.111,10.047,2,118315.625,1.874,7.443,132,118,True,ok,plateau +wal-normal,20000,4,74963.0,,,2,55980.625,3.655,14.047,128,110,False,ok,ladder_exhausted +memory,50000,2,132834.0,1.527,7.711,4,58021.875,5.515,53.471,266,231,True,ok,plateau +wal-nofsync,50000,4,103272.0,4.279,15.519,2,91121.125,2.245,10.183,290,244,True,ok,plateau +wal-normal,50000,6,68782.625,,,4,51999.0,6.247,138.879,259,244,False,ok,ladder_exhausted diff --git a/results/wal-fsync-diag-local/aggregate.json b/results/wal-fsync-diag-local/aggregate.json new file mode 100644 index 0000000..1b26c78 --- /dev/null +++ b/results/wal-fsync-diag-local/aggregate.json @@ -0,0 +1,200 @@ +[ + { + "mode": "memory", + "stream_count": 20000, + "pods": 2, + "throughput": 165326.0, + "p50": 1.323, + "p99": 5.311, + "knee_pods": 2, + "knee_throughput": 162666.25, + "knee_p50": 1.32, + "knee_p99": 5.667, + "pod_mem_mb": 121, + "pod_mem_p50_mb": 112, + "saturated": true, + "status": "ok", + "reason": "plateau", + "walk": [ + [ + 2, + 162666.25, + 1.32, + 5.667 + ], + [ + 4, + 177057.75, + 2.171, + 12.407 + ] + ] + }, + { + "mode": "wal-nofsync", + "stream_count": 20000, + "pods": 2, + "throughput": 98446.8, + "p50": 2.111, + "p99": 10.047, + "knee_pods": 2, + "knee_throughput": 118315.625, + "knee_p50": 1.874, + "knee_p99": 7.443, + "pod_mem_mb": 132, + "pod_mem_p50_mb": 118, + "saturated": true, + "status": "ok", + "reason": "plateau", + "walk": [ + [ + 2, + 118315.625, + 1.874, + 7.443 + ], + [ + 4, + 119608.375, + 3.795, + 13.327 + ] + ] + }, + { + "mode": "wal-normal", + "stream_count": 20000, + "pods": 4, + "throughput": 74963.0, + "p50": null, + "p99": null, + "knee_pods": 2, + "knee_throughput": 55980.625, + "knee_p50": 3.655, + "knee_p99": 14.047, + "pod_mem_mb": 128, + "pod_mem_p50_mb": 110, + "saturated": false, + "status": "ok", + "reason": "ladder_exhausted", + "walk": [ + [ + 2, + 55980.625, + 3.655, + 14.047 + ], + [ + 4, + 74963.0, + 5.791, + 20.207 + ] + ] + }, + { + "mode": "memory", + "stream_count": 50000, + "pods": 2, + "throughput": 132834.0, + "p50": 1.527, + "p99": 7.711, + "knee_pods": 4, + "knee_throughput": 58021.875, + "knee_p50": 5.515, + "knee_p99": 53.471, + "pod_mem_mb": 266, + "pod_mem_p50_mb": 231, + "saturated": true, + "status": "ok", + "reason": "plateau", + "walk": [ + [ + 2, + 146339.875, + 1.393, + 5.163 + ], + [ + 4, + 58021.875, + 5.515, + 53.471 + ] + ] + }, + { + "mode": "wal-nofsync", + "stream_count": 50000, + "pods": 4, + "throughput": 103272.0, + "p50": 4.279, + "p99": 15.519, + "knee_pods": 2, + "knee_throughput": 91121.125, + "knee_p50": 2.245, + "knee_p99": 10.183, + "pod_mem_mb": 290, + "pod_mem_p50_mb": 244, + "saturated": true, + "status": "ok", + "reason": "plateau", + "walk": [ + [ + 2, + 91121.125, + 2.245, + 10.183 + ], + [ + 4, + 101021.25, + 4.299, + 15.415 + ], + [ + 6, + 98758.375, + 6.579, + 21.807 + ] + ] + }, + { + "mode": "wal-normal", + "stream_count": 50000, + "pods": 6, + "throughput": 68782.625, + "p50": null, + "p99": null, + "knee_pods": 4, + "knee_throughput": 51999.0, + "knee_p50": 6.247, + "knee_p99": 138.879, + "pod_mem_mb": 259, + "pod_mem_p50_mb": 244, + "saturated": false, + "status": "ok", + "reason": "ladder_exhausted", + "walk": [ + [ + 2, + 27575.875, + 5.191, + 152.447 + ], + [ + 4, + 51999.0, + 6.247, + 138.879 + ], + [ + 6, + 68782.625, + 7.939, + 132.991 + ] + ] + } +] \ No newline at end of file diff --git a/results/wal-fsync-diag-local/memory/cells.json b/results/wal-fsync-diag-local/memory/cells.json new file mode 100644 index 0000000..bf263bb --- /dev/null +++ b/results/wal-fsync-diag-local/memory/cells.json @@ -0,0 +1,58 @@ +{ + "cells": { + "20000": { + "stream_count": 20000, + "image_digest": "aa40e13f0cf3", + "walk": [ + [ + 2, + 162666.25, + 1.32, + 5.667 + ], + [ + 4, + 177057.75, + 2.171, + 12.407 + ] + ], + "pinned_pods": 2, + "throughput": 165326.0, + "p50": 1.323, + "p99": 5.311, + "pod_mem_mb": 121, + "saturated": true, + "status": "ok", + "reason": "plateau", + "pod_mem_p50_mb": 112 + }, + "50000": { + "stream_count": 50000, + "image_digest": "aa40e13f0cf3", + "walk": [ + [ + 2, + 146339.875, + 1.393, + 5.163 + ], + [ + 4, + 58021.875, + 5.515, + 53.471 + ] + ], + "pinned_pods": 2, + "throughput": 132834.0, + "p50": 1.527, + "p99": 7.711, + "pod_mem_mb": 266, + "saturated": true, + "status": "ok", + "reason": "plateau", + "pod_mem_p50_mb": 231 + } + } +} \ No newline at end of file diff --git a/results/wal-fsync-diag-local/report.md b/results/wal-fsync-diag-local/report.md new file mode 100644 index 0000000..566ce03 --- /dev/null +++ b/results/wal-fsync-diag-local/report.md @@ -0,0 +1,45 @@ +# wal-fsync-diag-local — write-throughput report + +## Throughput at saturation (ops/s) + +| streams | wal-normal | wal-nofsync | memory | +|---|---|---|---| +| 20000 | 75k† | 98k | 165k | +| 50000 | 69k† | 103k | 133k | + +† = not saturated (ladder exhausted) — treat as a lower bound. + +## Pod memory at saturation — peak / p50 (MiB) + +| streams | wal-normal | wal-nofsync | memory | +|---|---|---|---| +| 20000 | 128 / 110 | 132 / 118 | 121 / 112 | +| 50000 | 259 / 244 | 290 / 244 | 266 / 231 | + +_Pod working set = cgroup `memory.current − inactive_file` (anon + active page cache), sampled each second at the pinned rung. **peak** = high-water (catches bursts like an in-RAM Raft log filling); **p50** = median (what the server steadily holds resident). peak ≈ p50 ⇒ steadily resident; peak ≫ p50 ⇒ transient spikes._ + +## Latency (ms, p50 / p99) + +| streams | wal-normal @≤80% load | wal-normal @saturation | wal-nofsync @≤80% load | wal-nofsync @saturation | memory @≤80% load | memory @saturation | +|---|---|---|---|---|---|---| +| 20000 | 3.7 / 14.0 (56k @2p) | — | 1.9 / 7.4 (118k @2p) | 2.1 / 10.0 | 1.3 / 5.7 (163k @2p) | 1.3 / 5.3 | +| 50000 | 6.2 / 138.9 (52k @4p) | — | 2.2 / 10.2 (91k @2p) | 4.3 / 15.5 | 5.5 / 53.5 (58k @4p) | 1.5 / 7.7 | + +_@≤80% load = the largest ladder rung at ≤80% of peak throughput — the server's service latency with headroom. @saturation = the pinned plateau rung, where a closed-loop fleet measures its own queueing (≈ in-flight ÷ ceiling by Little's law), NOT the server's per-request cost. Compare against the unloaded single-request baseline (~1 ms for wal) before reading anything into large saturation values._ + +## Saturation walks (pods → ops/s, p50 ms) + +- **memory 20000**: 2:163k@1.3ms → 4:177k@2.2ms (pinned 2, plateau) +- **wal-nofsync 20000**: 2:118k@1.9ms → 4:120k@3.8ms (pinned 2, plateau) +- **wal-normal 20000**: 2:56k@3.7ms → 4:75k@5.8ms (pinned 4, ladder_exhausted) +- **memory 50000**: 2:146k@1.4ms → 4:58k@5.5ms (pinned 2, plateau) +- **wal-nofsync 50000**: 2:91k@2.2ms → 4:101k@4.3ms → 6:99k@6.6ms (pinned 4, plateau) +- **wal-normal 50000**: 2:28k@5.2ms → 4:52k@6.2ms → 6:69k@7.9ms (pinned 6, ladder_exhausted) + +## Findings + +_TODO: written by hand on top of the generated data._ + +## Caveats + +_Single-node best-case; not 3-node Raft. Throughput is a saturation ceiling per the ladder._ diff --git a/results/wal-fsync-diag-local/wal-nofsync/cells.json b/results/wal-fsync-diag-local/wal-nofsync/cells.json new file mode 100644 index 0000000..7c07caa --- /dev/null +++ b/results/wal-fsync-diag-local/wal-nofsync/cells.json @@ -0,0 +1,64 @@ +{ + "cells": { + "20000": { + "stream_count": 20000, + "image_digest": "c6dda090c943", + "walk": [ + [ + 2, + 118315.625, + 1.874, + 7.443 + ], + [ + 4, + 119608.375, + 3.795, + 13.327 + ] + ], + "pinned_pods": 2, + "throughput": 98446.8, + "p50": 2.111, + "p99": 10.047, + "pod_mem_mb": 132, + "saturated": true, + "status": "ok", + "reason": "plateau", + "pod_mem_p50_mb": 118 + }, + "50000": { + "stream_count": 50000, + "image_digest": "c6dda090c943", + "walk": [ + [ + 2, + 91121.125, + 2.245, + 10.183 + ], + [ + 4, + 101021.25, + 4.299, + 15.415 + ], + [ + 6, + 98758.375, + 6.579, + 21.807 + ] + ], + "pinned_pods": 4, + "throughput": 103272.0, + "p50": 4.279, + "p99": 15.519, + "pod_mem_mb": 290, + "saturated": true, + "status": "ok", + "reason": "plateau", + "pod_mem_p50_mb": 244 + } + } +} \ No newline at end of file diff --git a/results/wal-fsync-diag-local/wal-normal/cells.json b/results/wal-fsync-diag-local/wal-normal/cells.json new file mode 100644 index 0000000..c6b3c48 --- /dev/null +++ b/results/wal-fsync-diag-local/wal-normal/cells.json @@ -0,0 +1,64 @@ +{ + "cells": { + "20000": { + "stream_count": 20000, + "image_digest": "b659d4d00350", + "walk": [ + [ + 2, + 55980.625, + 3.655, + 14.047 + ], + [ + 4, + 74963.0, + 5.791, + 20.207 + ] + ], + "pinned_pods": 4, + "throughput": 74963.0, + "p50": null, + "p99": null, + "pod_mem_mb": 128, + "saturated": false, + "status": "ok", + "reason": "ladder_exhausted", + "pod_mem_p50_mb": 110 + }, + "50000": { + "stream_count": 50000, + "image_digest": "b659d4d00350", + "walk": [ + [ + 2, + 27575.875, + 5.191, + 152.447 + ], + [ + 4, + 51999.0, + 6.247, + 138.879 + ], + [ + 6, + 68782.625, + 7.939, + 132.991 + ] + ], + "pinned_pods": 6, + "throughput": 68782.625, + "p50": null, + "p99": null, + "pod_mem_mb": 259, + "saturated": false, + "status": "ok", + "reason": "ladder_exhausted", + "pod_mem_p50_mb": 244 + } + } +} \ No newline at end of file diff --git a/results/wal-lazy-checkpoint-local/aggregate.csv b/results/wal-lazy-checkpoint-local/aggregate.csv new file mode 100644 index 0000000..b3b3f88 --- /dev/null +++ b/results/wal-lazy-checkpoint-local/aggregate.csv @@ -0,0 +1,9 @@ +mode,stream_count,pods,throughput,p50,p99,knee_pods,knee_throughput,knee_p50,knee_p99,pod_mem_mb,pod_mem_p50_mb,saturated,status,reason +lazy-syncfs-30s,20000,4,100066.25,,,2,68584.25,3.223,9.447,129,118,False,ok,ladder_exhausted +lazy-syncfs-60s,20000,4,98228.375,,,2,80041.5,3.133,5.231,128,118,False,ok,ladder_exhausted +ref-baseline-3s,20000,4,77092.75,,,2,54549.0,4.503,9.463,128,106,False,ok,ladder_exhausted +ref-syncfs-3s,20000,4,88036.75,,,2,60107.625,3.461,12.295,134,123,False,ok,ladder_exhausted +lazy-syncfs-30s,50000,4,72479.5,5.411,55.775,2,58209.0,4.155,9.391,278,239,True,ok,plateau +lazy-syncfs-60s,50000,6,89838.75,,,2,59949.5,3.945,11.079,281,242,False,ok,ladder_exhausted +ref-baseline-3s,50000,4,61508.1,6.503,30.639,2,21584.5,8.655,62.879,262,239,True,ok,plateau +ref-syncfs-3s,50000,4,50372.9,5.883,136.319,2,49339.875,3.859,19.711,247,238,True,ok,plateau diff --git a/results/wal-lazy-checkpoint-local/aggregate.json b/results/wal-lazy-checkpoint-local/aggregate.json new file mode 100644 index 0000000..33d7c29 --- /dev/null +++ b/results/wal-lazy-checkpoint-local/aggregate.json @@ -0,0 +1,274 @@ +[ + { + "mode": "lazy-syncfs-30s", + "stream_count": 20000, + "pods": 4, + "throughput": 100066.25, + "p50": null, + "p99": null, + "knee_pods": 2, + "knee_throughput": 68584.25, + "knee_p50": 3.223, + "knee_p99": 9.447, + "pod_mem_mb": 129, + "pod_mem_p50_mb": 118, + "saturated": false, + "status": "ok", + "reason": "ladder_exhausted", + "walk": [ + [ + 2, + 68584.25, + 3.223, + 9.447 + ], + [ + 4, + 100066.25, + 4.807, + 9.431 + ] + ] + }, + { + "mode": "lazy-syncfs-60s", + "stream_count": 20000, + "pods": 4, + "throughput": 98228.375, + "p50": null, + "p99": null, + "knee_pods": 2, + "knee_throughput": 80041.5, + "knee_p50": 3.133, + "knee_p99": 5.231, + "pod_mem_mb": 128, + "pod_mem_p50_mb": 118, + "saturated": false, + "status": "ok", + "reason": "ladder_exhausted", + "walk": [ + [ + 2, + 80041.5, + 3.133, + 5.231 + ], + [ + 4, + 98228.375, + 4.819, + 9.559 + ] + ] + }, + { + "mode": "ref-baseline-3s", + "stream_count": 20000, + "pods": 4, + "throughput": 77092.75, + "p50": null, + "p99": null, + "knee_pods": 2, + "knee_throughput": 54549.0, + "knee_p50": 4.503, + "knee_p99": 9.463, + "pod_mem_mb": 128, + "pod_mem_p50_mb": 106, + "saturated": false, + "status": "ok", + "reason": "ladder_exhausted", + "walk": [ + [ + 2, + 54549.0, + 4.503, + 9.463 + ], + [ + 4, + 77092.75, + 6.063, + 16.247 + ] + ] + }, + { + "mode": "ref-syncfs-3s", + "stream_count": 20000, + "pods": 4, + "throughput": 88036.75, + "p50": null, + "p99": null, + "knee_pods": 2, + "knee_throughput": 60107.625, + "knee_p50": 3.461, + "knee_p99": 12.295, + "pod_mem_mb": 134, + "pod_mem_p50_mb": 123, + "saturated": false, + "status": "ok", + "reason": "ladder_exhausted", + "walk": [ + [ + 2, + 60107.625, + 3.461, + 12.295 + ], + [ + 4, + 88036.75, + 4.991, + 12.447 + ] + ] + }, + { + "mode": "lazy-syncfs-30s", + "stream_count": 50000, + "pods": 4, + "throughput": 72479.5, + "p50": 5.411, + "p99": 55.775, + "knee_pods": 2, + "knee_throughput": 58209.0, + "knee_p50": 4.155, + "knee_p99": 9.391, + "pod_mem_mb": 278, + "pod_mem_p50_mb": 239, + "saturated": true, + "status": "ok", + "reason": "plateau", + "walk": [ + [ + 2, + 58209.0, + 4.155, + 9.391 + ], + [ + 4, + 80708.5, + 5.367, + 10.455 + ], + [ + 6, + 72297.625, + 7.559, + 103.231 + ] + ] + }, + { + "mode": "lazy-syncfs-60s", + "stream_count": 50000, + "pods": 6, + "throughput": 89838.75, + "p50": null, + "p99": null, + "knee_pods": 2, + "knee_throughput": 59949.5, + "knee_p50": 3.945, + "knee_p99": 11.079, + "pod_mem_mb": 281, + "pod_mem_p50_mb": 242, + "saturated": false, + "status": "ok", + "reason": "ladder_exhausted", + "walk": [ + [ + 2, + 59949.5, + 3.945, + 11.079 + ], + [ + 4, + 80318.125, + 5.359, + 25.855 + ], + [ + 6, + 89838.75, + 7.551, + 35.135 + ] + ] + }, + { + "mode": "ref-baseline-3s", + "stream_count": 50000, + "pods": 4, + "throughput": 61508.1, + "p50": 6.503, + "p99": 30.639, + "knee_pods": 2, + "knee_throughput": 21584.5, + "knee_p50": 8.655, + "knee_p99": 62.879, + "pod_mem_mb": 262, + "pod_mem_p50_mb": 239, + "saturated": true, + "status": "ok", + "reason": "plateau", + "walk": [ + [ + 2, + 21584.5, + 8.655, + 62.879 + ], + [ + 4, + 64409.25, + 7.271, + 18.607 + ], + [ + 6, + 63920.625, + 9.183, + 56.639 + ] + ] + }, + { + "mode": "ref-syncfs-3s", + "stream_count": 50000, + "pods": 4, + "throughput": 50372.9, + "p50": 5.883, + "p99": 136.319, + "knee_pods": 2, + "knee_throughput": 49339.875, + "knee_p50": 3.859, + "knee_p99": 19.711, + "pod_mem_mb": 247, + "pod_mem_p50_mb": 238, + "saturated": true, + "status": "ok", + "reason": "plateau", + "walk": [ + [ + 2, + 49339.875, + 3.859, + 19.711 + ], + [ + 4, + 67934.75, + 5.511, + 98.431 + ], + [ + 6, + 69571.25, + 7.743, + 129.279 + ] + ] + } +] \ No newline at end of file diff --git a/results/wal-lazy-checkpoint-local/lazy-syncfs-30s/cells.json b/results/wal-lazy-checkpoint-local/lazy-syncfs-30s/cells.json new file mode 100644 index 0000000..af802b5 --- /dev/null +++ b/results/wal-lazy-checkpoint-local/lazy-syncfs-30s/cells.json @@ -0,0 +1,64 @@ +{ + "cells": { + "20000": { + "stream_count": 20000, + "image_digest": "11407215416a", + "walk": [ + [ + 2, + 68584.25, + 3.223, + 9.447 + ], + [ + 4, + 100066.25, + 4.807, + 9.431 + ] + ], + "pinned_pods": 4, + "throughput": 100066.25, + "p50": null, + "p99": null, + "pod_mem_mb": 129, + "saturated": false, + "status": "ok", + "reason": "ladder_exhausted", + "pod_mem_p50_mb": 118 + }, + "50000": { + "stream_count": 50000, + "image_digest": "11407215416a", + "walk": [ + [ + 2, + 58209.0, + 4.155, + 9.391 + ], + [ + 4, + 80708.5, + 5.367, + 10.455 + ], + [ + 6, + 72297.625, + 7.559, + 103.231 + ] + ], + "pinned_pods": 4, + "throughput": 72479.5, + "p50": 5.411, + "p99": 55.775, + "pod_mem_mb": 278, + "saturated": true, + "status": "ok", + "reason": "plateau", + "pod_mem_p50_mb": 239 + } + } +} \ No newline at end of file diff --git a/results/wal-lazy-checkpoint-local/lazy-syncfs-60s/cells.json b/results/wal-lazy-checkpoint-local/lazy-syncfs-60s/cells.json new file mode 100644 index 0000000..71b6d58 --- /dev/null +++ b/results/wal-lazy-checkpoint-local/lazy-syncfs-60s/cells.json @@ -0,0 +1,64 @@ +{ + "cells": { + "20000": { + "stream_count": 20000, + "image_digest": "48c267b280e1", + "walk": [ + [ + 2, + 80041.5, + 3.133, + 5.231 + ], + [ + 4, + 98228.375, + 4.819, + 9.559 + ] + ], + "pinned_pods": 4, + "throughput": 98228.375, + "p50": null, + "p99": null, + "pod_mem_mb": 128, + "saturated": false, + "status": "ok", + "reason": "ladder_exhausted", + "pod_mem_p50_mb": 118 + }, + "50000": { + "stream_count": 50000, + "image_digest": "48c267b280e1", + "walk": [ + [ + 2, + 59949.5, + 3.945, + 11.079 + ], + [ + 4, + 80318.125, + 5.359, + 25.855 + ], + [ + 6, + 89838.75, + 7.551, + 35.135 + ] + ], + "pinned_pods": 6, + "throughput": 89838.75, + "p50": null, + "p99": null, + "pod_mem_mb": 281, + "saturated": false, + "status": "ok", + "reason": "ladder_exhausted", + "pod_mem_p50_mb": 242 + } + } +} \ No newline at end of file diff --git a/results/wal-lazy-checkpoint-local/ref-baseline-3s/cells.json b/results/wal-lazy-checkpoint-local/ref-baseline-3s/cells.json new file mode 100644 index 0000000..d4d096e --- /dev/null +++ b/results/wal-lazy-checkpoint-local/ref-baseline-3s/cells.json @@ -0,0 +1,64 @@ +{ + "cells": { + "20000": { + "stream_count": 20000, + "image_digest": "0bb2c6c89ce6", + "walk": [ + [ + 2, + 54549.0, + 4.503, + 9.463 + ], + [ + 4, + 77092.75, + 6.063, + 16.247 + ] + ], + "pinned_pods": 4, + "throughput": 77092.75, + "p50": null, + "p99": null, + "pod_mem_mb": 128, + "saturated": false, + "status": "ok", + "reason": "ladder_exhausted", + "pod_mem_p50_mb": 106 + }, + "50000": { + "stream_count": 50000, + "image_digest": "0bb2c6c89ce6", + "walk": [ + [ + 2, + 21584.5, + 8.655, + 62.879 + ], + [ + 4, + 64409.25, + 7.271, + 18.607 + ], + [ + 6, + 63920.625, + 9.183, + 56.639 + ] + ], + "pinned_pods": 4, + "throughput": 61508.1, + "p50": 6.503, + "p99": 30.639, + "pod_mem_mb": 262, + "saturated": true, + "status": "ok", + "reason": "plateau", + "pod_mem_p50_mb": 239 + } + } +} \ No newline at end of file diff --git a/results/wal-lazy-checkpoint-local/ref-syncfs-3s/cells.json b/results/wal-lazy-checkpoint-local/ref-syncfs-3s/cells.json new file mode 100644 index 0000000..02ae227 --- /dev/null +++ b/results/wal-lazy-checkpoint-local/ref-syncfs-3s/cells.json @@ -0,0 +1,64 @@ +{ + "cells": { + "20000": { + "stream_count": 20000, + "image_digest": "8c09d3514768", + "walk": [ + [ + 2, + 60107.625, + 3.461, + 12.295 + ], + [ + 4, + 88036.75, + 4.991, + 12.447 + ] + ], + "pinned_pods": 4, + "throughput": 88036.75, + "p50": null, + "p99": null, + "pod_mem_mb": 134, + "saturated": false, + "status": "ok", + "reason": "ladder_exhausted", + "pod_mem_p50_mb": 123 + }, + "50000": { + "stream_count": 50000, + "image_digest": "8c09d3514768", + "walk": [ + [ + 2, + 49339.875, + 3.859, + 19.711 + ], + [ + 4, + 67934.75, + 5.511, + 98.431 + ], + [ + 6, + 69571.25, + 7.743, + 129.279 + ] + ], + "pinned_pods": 4, + "throughput": 50372.9, + "p50": 5.883, + "p99": 136.319, + "pod_mem_mb": 247, + "saturated": true, + "status": "ok", + "reason": "plateau", + "pod_mem_p50_mb": 238 + } + } +} \ No newline at end of file diff --git a/results/wal-lazy-checkpoint-local/report.md b/results/wal-lazy-checkpoint-local/report.md new file mode 100644 index 0000000..4035795 --- /dev/null +++ b/results/wal-lazy-checkpoint-local/report.md @@ -0,0 +1,47 @@ +# wal-lazy-checkpoint-local — write-throughput report + +## Throughput at saturation (ops/s) + +| streams | ref-baseline-3s | ref-syncfs-3s | lazy-syncfs-30s | lazy-syncfs-60s | +|---|---|---|---|---| +| 20000 | 77k† | 88k† | 100k† | 98k† | +| 50000 | 62k | 50k | 72k | 90k† | + +† = not saturated (ladder exhausted) — treat as a lower bound. + +## Pod memory at saturation — peak / p50 (MiB) + +| streams | ref-baseline-3s | ref-syncfs-3s | lazy-syncfs-30s | lazy-syncfs-60s | +|---|---|---|---|---| +| 20000 | 128 / 106 | 134 / 123 | 129 / 118 | 128 / 118 | +| 50000 | 262 / 239 | 247 / 238 | 278 / 239 | 281 / 242 | + +_Pod working set = cgroup `memory.current − inactive_file` (anon + active page cache), sampled each second at the pinned rung. **peak** = high-water (catches bursts like an in-RAM Raft log filling); **p50** = median (what the server steadily holds resident). peak ≈ p50 ⇒ steadily resident; peak ≫ p50 ⇒ transient spikes._ + +## Latency (ms, p50 / p99) + +| streams | ref-baseline-3s @≤80% load | ref-baseline-3s @saturation | ref-syncfs-3s @≤80% load | ref-syncfs-3s @saturation | lazy-syncfs-30s @≤80% load | lazy-syncfs-30s @saturation | lazy-syncfs-60s @≤80% load | lazy-syncfs-60s @saturation | +|---|---|---|---|---|---|---|---|---| +| 20000 | 4.5 / 9.5 (55k @2p) | — | 3.5 / 12.3 (60k @2p) | — | 3.2 / 9.4 (69k @2p) | — | 3.1 / 5.2 (80k @2p) | — | +| 50000 | 8.7 / 62.9 (22k @2p) | 6.5 / 30.6 | 3.9 / 19.7 (49k @2p) | 5.9 / 136.3 | 4.2 / 9.4 (58k @2p) | 5.4 / 55.8 | 3.9 / 11.1 (60k @2p) | — | + +_@≤80% load = the largest ladder rung at ≤80% of peak throughput — the server's service latency with headroom. @saturation = the pinned plateau rung, where a closed-loop fleet measures its own queueing (≈ in-flight ÷ ceiling by Little's law), NOT the server's per-request cost. Compare against the unloaded single-request baseline (~1 ms for wal) before reading anything into large saturation values._ + +## Saturation walks (pods → ops/s, p50 ms) + +- **lazy-syncfs-30s 20000**: 2:69k@3.2ms → 4:100k@4.8ms (pinned 4, ladder_exhausted) +- **lazy-syncfs-60s 20000**: 2:80k@3.1ms → 4:98k@4.8ms (pinned 4, ladder_exhausted) +- **ref-baseline-3s 20000**: 2:55k@4.5ms → 4:77k@6.1ms (pinned 4, ladder_exhausted) +- **ref-syncfs-3s 20000**: 2:60k@3.5ms → 4:88k@5.0ms (pinned 4, ladder_exhausted) +- **lazy-syncfs-30s 50000**: 2:58k@4.2ms → 4:81k@5.4ms → 6:72k@7.6ms (pinned 4, plateau) +- **lazy-syncfs-60s 50000**: 2:60k@3.9ms → 4:80k@5.4ms → 6:90k@7.6ms (pinned 6, ladder_exhausted) +- **ref-baseline-3s 50000**: 2:22k@8.7ms → 4:64k@7.3ms → 6:64k@9.2ms (pinned 4, plateau) +- **ref-syncfs-3s 50000**: 2:49k@3.9ms → 4:68k@5.5ms → 6:70k@7.7ms (pinned 4, plateau) + +## Findings + +_TODO: written by hand on top of the generated data._ + +## Caveats + +_Single-node best-case; not 3-node Raft. Throughput is a saturation ceiling per the ladder._ diff --git a/results/wal-lazy-checkpoint-multilane/lazy-120s/cells.json b/results/wal-lazy-checkpoint-multilane/lazy-120s/cells.json new file mode 100644 index 0000000..f6af398 --- /dev/null +++ b/results/wal-lazy-checkpoint-multilane/lazy-120s/cells.json @@ -0,0 +1,36 @@ +{ + "cells": { + "10000": { + "stream_count": 10000, + "image_digest": "e3009dc55854", + "walk": [ + [ + 2, + 282020.05, + 1.478, + 11.111 + ], + [ + 4, + 288492.8, + 3.099, + 15.647 + ], + [ + 8, + 292573.05, + 6.259, + 28.463 + ] + ], + "pinned_pods": 2, + "throughput": 304788.0, + "p50": 1.433, + "p99": 6.447, + "pod_mem_mb": 0.0, + "saturated": true, + "status": "ok", + "reason": "plateau" + } + } +} \ No newline at end of file diff --git a/results/wal-lazy-checkpoint-multilane/lazy-30s/cells.json b/results/wal-lazy-checkpoint-multilane/lazy-30s/cells.json new file mode 100644 index 0000000..af2c198 --- /dev/null +++ b/results/wal-lazy-checkpoint-multilane/lazy-30s/cells.json @@ -0,0 +1,68 @@ +{ + "cells": { + "10000": { + "stream_count": 10000, + "image_digest": "822a4f41ce43", + "walk": [ + [ + 2, + 274731.75, + 1.477, + 11.967 + ], + [ + 4, + 289023.25, + 3.127, + 13.775 + ], + [ + 8, + 262443.75, + 6.463, + 44.543 + ] + ], + "pinned_pods": 2, + "throughput": 291903.0, + "p50": 1.47, + "p99": 6.807, + "pod_mem_mb": 282.0, + "saturated": true, + "status": "ok", + "reason": "plateau" + }, + "100000": { + "stream_count": 100000, + "image_digest": "822a4f41ce43", + "walk": [ + [ + 2, + 13614.25, + 1.437, + 2015.231 + ], + [ + 4, + 6778.85, + 6.639, + 3770.367 + ], + [ + 8, + 9256.1, + 113.791, + 1354.751 + ] + ], + "pinned_pods": 8, + "throughput": 9256.1, + "p50": null, + "p99": null, + "pod_mem_mb": 0.0, + "saturated": false, + "status": "ok", + "reason": "ladder_exhausted" + } + } +} \ No newline at end of file diff --git a/results/wal-lazy-checkpoint-multilane/lazy-60s/cells.json b/results/wal-lazy-checkpoint-multilane/lazy-60s/cells.json new file mode 100644 index 0000000..016e465 --- /dev/null +++ b/results/wal-lazy-checkpoint-multilane/lazy-60s/cells.json @@ -0,0 +1,68 @@ +{ + "cells": { + "10000": { + "stream_count": 10000, + "image_digest": "c1696ad4ac9a", + "walk": [ + [ + 2, + 277239.9, + 1.444, + 12.551 + ], + [ + 4, + 263360.7, + 3.097, + 24.831 + ], + [ + 8, + 294393.25, + 6.343, + 17.487 + ] + ], + "pinned_pods": 8, + "throughput": 294393.25, + "p50": null, + "p99": null, + "pod_mem_mb": 0.0, + "saturated": false, + "status": "ok", + "reason": "ladder_exhausted" + }, + "100000": { + "stream_count": 100000, + "image_digest": "c1696ad4ac9a", + "walk": [ + [ + 2, + 15470.5, + 1.577, + 1514.495 + ], + [ + 4, + 10707.900000000001, + 4.075, + 1467.391 + ], + [ + 8, + 6952.1, + 270.847, + 1154.047 + ] + ], + "pinned_pods": 2, + "throughput": 14787.7, + "p50": 1.744, + "p99": 1406.975, + "pod_mem_mb": 558.0, + "saturated": true, + "status": "ok", + "reason": "plateau" + } + } +} \ No newline at end of file diff --git a/results/wal-lazy-checkpoint-multilane/ref-syncfs-3s/cells.json b/results/wal-lazy-checkpoint-multilane/ref-syncfs-3s/cells.json new file mode 100644 index 0000000..ce00e9d --- /dev/null +++ b/results/wal-lazy-checkpoint-multilane/ref-syncfs-3s/cells.json @@ -0,0 +1,68 @@ +{ + "cells": { + "10000": { + "stream_count": 10000, + "image_digest": "c8d713e7f8a2", + "walk": [ + [ + 2, + 185198.9, + 1.435, + 27.375 + ], + [ + 4, + 256548.85, + 3.151, + 24.831 + ], + [ + 8, + 270540.89999999997, + 6.299, + 34.495 + ] + ], + "pinned_pods": 8, + "throughput": 270540.89999999997, + "p50": null, + "p99": null, + "pod_mem_mb": 307.0, + "saturated": false, + "status": "ok", + "reason": "ladder_exhausted" + }, + "100000": { + "stream_count": 100000, + "image_digest": "c8d713e7f8a2", + "walk": [ + [ + 2, + 8546.45, + 1.765, + 1037.823 + ], + [ + 4, + 12124.550000000001, + 4.123, + 1283.071 + ], + [ + 8, + 8495.05, + 165.119, + 1568.767 + ] + ], + "pinned_pods": 8, + "throughput": 8495.05, + "p50": null, + "p99": null, + "pod_mem_mb": 0.0, + "saturated": false, + "status": "ok", + "reason": "ladder_exhausted" + } + } +} \ No newline at end of file diff --git a/results/wal-machinery-baseline-local/aggregate.csv b/results/wal-machinery-baseline-local/aggregate.csv new file mode 100644 index 0000000..806dbe1 --- /dev/null +++ b/results/wal-machinery-baseline-local/aggregate.csv @@ -0,0 +1,2 @@ +mode,stream_count,pods,throughput,p50,p99,knee_pods,knee_throughput,knee_p50,knee_p99,pod_mem_mb,pod_mem_p50_mb,saturated,status,reason +wal-baseline,50000,6,50662.1,,,2,32697.800000000003,4.495,128.959,252,242,False,ok,ladder_exhausted diff --git a/results/wal-machinery-baseline-local/aggregate.json b/results/wal-machinery-baseline-local/aggregate.json new file mode 100644 index 0000000..cc5d074 --- /dev/null +++ b/results/wal-machinery-baseline-local/aggregate.json @@ -0,0 +1,39 @@ +[ + { + "mode": "wal-baseline", + "stream_count": 50000, + "pods": 6, + "throughput": 50662.1, + "p50": null, + "p99": null, + "knee_pods": 2, + "knee_throughput": 32697.800000000003, + "knee_p50": 4.495, + "knee_p99": 128.959, + "pod_mem_mb": 252, + "pod_mem_p50_mb": 242, + "saturated": false, + "status": "ok", + "reason": "ladder_exhausted", + "walk": [ + [ + 2, + 32697.800000000003, + 4.495, + 128.959 + ], + [ + 4, + 45841.4, + 6.711, + 187.263 + ], + [ + 6, + 50662.1, + 9.407, + 201.855 + ] + ] + } +] \ No newline at end of file diff --git a/results/wal-machinery-baseline-local/report.md b/results/wal-machinery-baseline-local/report.md new file mode 100644 index 0000000..52ca9e7 --- /dev/null +++ b/results/wal-machinery-baseline-local/report.md @@ -0,0 +1,37 @@ +# wal-machinery-baseline-local — write-throughput report + +## Throughput at saturation (ops/s) + +| streams | wal-baseline | +|---|---| +| 50000 | 51k† | + +† = not saturated (ladder exhausted) — treat as a lower bound. + +## Pod memory at saturation — peak / p50 (MiB) + +| streams | wal-baseline | +|---|---| +| 50000 | 252 / 242 | + +_Pod working set = cgroup `memory.current − inactive_file` (anon + active page cache), sampled each second at the pinned rung. **peak** = high-water (catches bursts like an in-RAM Raft log filling); **p50** = median (what the server steadily holds resident). peak ≈ p50 ⇒ steadily resident; peak ≫ p50 ⇒ transient spikes._ + +## Latency (ms, p50 / p99) + +| streams | wal-baseline @≤80% load | wal-baseline @saturation | +|---|---|---| +| 50000 | 4.5 / 129.0 (33k @2p) | — | + +_@≤80% load = the largest ladder rung at ≤80% of peak throughput — the server's service latency with headroom. @saturation = the pinned plateau rung, where a closed-loop fleet measures its own queueing (≈ in-flight ÷ ceiling by Little's law), NOT the server's per-request cost. Compare against the unloaded single-request baseline (~1 ms for wal) before reading anything into large saturation values._ + +## Saturation walks (pods → ops/s, p50 ms) + +- **wal-baseline 50000**: 2:33k@4.5ms → 4:46k@6.7ms → 6:51k@9.4ms (pinned 6, ladder_exhausted) + +## Findings + +_TODO: written by hand on top of the generated data._ + +## Caveats + +_Single-node best-case; not 3-node Raft. Throughput is a saturation ceiling per the ladder._ diff --git a/results/wal-machinery-baseline-local/wal-baseline/cells.json b/results/wal-machinery-baseline-local/wal-baseline/cells.json new file mode 100644 index 0000000..cdbc3b5 --- /dev/null +++ b/results/wal-machinery-baseline-local/wal-baseline/cells.json @@ -0,0 +1,37 @@ +{ + "cells": { + "50000": { + "stream_count": 50000, + "image_digest": "1ee33d6c2b18", + "walk": [ + [ + 2, + 32697.800000000003, + 4.495, + 128.959 + ], + [ + 4, + 45841.4, + 6.711, + 187.263 + ], + [ + 6, + 50662.1, + 9.407, + 201.855 + ] + ], + "pinned_pods": 6, + "throughput": 50662.1, + "p50": null, + "p99": null, + "pod_mem_mb": 252, + "saturated": false, + "status": "ok", + "reason": "ladder_exhausted", + "pod_mem_p50_mb": 242 + } + } +} \ No newline at end of file diff --git a/results/wal-mem-diag-local/aggregate.csv b/results/wal-mem-diag-local/aggregate.csv new file mode 100644 index 0000000..77fa8ae --- /dev/null +++ b/results/wal-mem-diag-local/aggregate.csv @@ -0,0 +1,5 @@ +mode,stream_count,pods,throughput,p50,p99,knee_pods,knee_throughput,knee_p50,knee_p99,pod_mem_mb,pod_mem_p50_mb,saturated,status,reason +memory,20000,4,170410.25,,,2,136822.375,1.517,7.163,127,117,False,ok,ladder_exhausted +wal-syncfs,20000,4,73717.25,,,2,62928.875,3.463,11.343,133,106,False,ok,ladder_exhausted +memory,50000,2,139270.0,1.44,7.987,2,145706.0,1.458,6.383,270,230,True,ok,plateau +wal-syncfs,50000,2,23309.2,6.339,146.943,2,43436.625,4.075,39.519,243,226,True,ok,plateau diff --git a/results/wal-mem-diag-local/aggregate.json b/results/wal-mem-diag-local/aggregate.json new file mode 100644 index 0000000..244d9a3 --- /dev/null +++ b/results/wal-mem-diag-local/aggregate.json @@ -0,0 +1,126 @@ +[ + { + "mode": "memory", + "stream_count": 20000, + "pods": 4, + "throughput": 170410.25, + "p50": null, + "p99": null, + "knee_pods": 2, + "knee_throughput": 136822.375, + "knee_p50": 1.517, + "knee_p99": 7.163, + "pod_mem_mb": 127, + "pod_mem_p50_mb": 117, + "saturated": false, + "status": "ok", + "reason": "ladder_exhausted", + "walk": [ + [ + 2, + 136822.375, + 1.517, + 7.163 + ], + [ + 4, + 170410.25, + 2.221, + 13.199 + ] + ] + }, + { + "mode": "wal-syncfs", + "stream_count": 20000, + "pods": 4, + "throughput": 73717.25, + "p50": null, + "p99": null, + "knee_pods": 2, + "knee_throughput": 62928.875, + "knee_p50": 3.463, + "knee_p99": 11.343, + "pod_mem_mb": 133, + "pod_mem_p50_mb": 106, + "saturated": false, + "status": "ok", + "reason": "ladder_exhausted", + "walk": [ + [ + 2, + 62928.875, + 3.463, + 11.343 + ], + [ + 4, + 73717.25, + 5.839, + 14.951 + ] + ] + }, + { + "mode": "memory", + "stream_count": 50000, + "pods": 2, + "throughput": 139270.0, + "p50": 1.44, + "p99": 7.987, + "knee_pods": 2, + "knee_throughput": 145706.0, + "knee_p50": 1.458, + "knee_p99": 6.383, + "pod_mem_mb": 270, + "pod_mem_p50_mb": 230, + "saturated": true, + "status": "ok", + "reason": "plateau", + "walk": [ + [ + 2, + 145706.0, + 1.458, + 6.383 + ], + [ + 4, + 146135.0, + 2.589, + 14.735 + ] + ] + }, + { + "mode": "wal-syncfs", + "stream_count": 50000, + "pods": 2, + "throughput": 23309.2, + "p50": 6.339, + "p99": 146.943, + "knee_pods": 2, + "knee_throughput": 43436.625, + "knee_p50": 4.075, + "knee_p99": 39.519, + "pod_mem_mb": 243, + "pod_mem_p50_mb": 226, + "saturated": true, + "status": "ok", + "reason": "plateau", + "walk": [ + [ + 2, + 43436.625, + 4.075, + 39.519 + ], + [ + 4, + 36744.125, + 8.759, + 149.375 + ] + ] + } +] \ No newline at end of file diff --git a/results/wal-mem-diag-local/memory/cells.json b/results/wal-mem-diag-local/memory/cells.json new file mode 100644 index 0000000..fc7a0a1 --- /dev/null +++ b/results/wal-mem-diag-local/memory/cells.json @@ -0,0 +1,58 @@ +{ + "cells": { + "20000": { + "stream_count": 20000, + "image_digest": "aa40e13f0cf3", + "walk": [ + [ + 2, + 136822.375, + 1.517, + 7.163 + ], + [ + 4, + 170410.25, + 2.221, + 13.199 + ] + ], + "pinned_pods": 4, + "throughput": 170410.25, + "p50": null, + "p99": null, + "pod_mem_mb": 127, + "saturated": false, + "status": "ok", + "reason": "ladder_exhausted", + "pod_mem_p50_mb": 117 + }, + "50000": { + "stream_count": 50000, + "image_digest": "aa40e13f0cf3", + "walk": [ + [ + 2, + 145706.0, + 1.458, + 6.383 + ], + [ + 4, + 146135.0, + 2.589, + 14.735 + ] + ], + "pinned_pods": 2, + "throughput": 139270.0, + "p50": 1.44, + "p99": 7.987, + "pod_mem_mb": 270, + "saturated": true, + "status": "ok", + "reason": "plateau", + "pod_mem_p50_mb": 230 + } + } +} \ No newline at end of file diff --git a/results/wal-mem-diag-local/report.md b/results/wal-mem-diag-local/report.md new file mode 100644 index 0000000..812d1b5 --- /dev/null +++ b/results/wal-mem-diag-local/report.md @@ -0,0 +1,43 @@ +# wal-mem-diag-local — write-throughput report + +## Throughput at saturation (ops/s) + +| streams | wal-syncfs | memory | +|---|---|---| +| 20000 | 74k† | 170k† | +| 50000 | 23k | 139k | + +† = not saturated (ladder exhausted) — treat as a lower bound. + +## Pod memory at saturation — peak / p50 (MiB) + +| streams | wal-syncfs | memory | +|---|---|---| +| 20000 | 133 / 106 | 127 / 117 | +| 50000 | 243 / 226 | 270 / 230 | + +_Pod working set = cgroup `memory.current − inactive_file` (anon + active page cache), sampled each second at the pinned rung. **peak** = high-water (catches bursts like an in-RAM Raft log filling); **p50** = median (what the server steadily holds resident). peak ≈ p50 ⇒ steadily resident; peak ≫ p50 ⇒ transient spikes._ + +## Latency (ms, p50 / p99) + +| streams | wal-syncfs @≤80% load | wal-syncfs @saturation | memory @≤80% load | memory @saturation | +|---|---|---|---|---| +| 20000 | 3.5 / 11.3 (63k @2p) | — | 1.5 / 7.2 (137k @2p) | — | +| 50000 | 4.1 / 39.5 (43k @2p) | 6.3 / 146.9 | 1.5 / 6.4 (146k @2p) | 1.4 / 8.0 | + +_@≤80% load = the largest ladder rung at ≤80% of peak throughput — the server's service latency with headroom. @saturation = the pinned plateau rung, where a closed-loop fleet measures its own queueing (≈ in-flight ÷ ceiling by Little's law), NOT the server's per-request cost. Compare against the unloaded single-request baseline (~1 ms for wal) before reading anything into large saturation values._ + +## Saturation walks (pods → ops/s, p50 ms) + +- **memory 20000**: 2:137k@1.5ms → 4:170k@2.2ms (pinned 4, ladder_exhausted) +- **wal-syncfs 20000**: 2:63k@3.5ms → 4:74k@5.8ms (pinned 4, ladder_exhausted) +- **memory 50000**: 2:146k@1.5ms → 4:146k@2.6ms (pinned 2, plateau) +- **wal-syncfs 50000**: 2:43k@4.1ms → 4:37k@8.8ms (pinned 2, plateau) + +## Findings + +_TODO: written by hand on top of the generated data._ + +## Caveats + +_Single-node best-case; not 3-node Raft. Throughput is a saturation ceiling per the ladder._ diff --git a/results/wal-mem-diag-local/wal-syncfs/cells.json b/results/wal-mem-diag-local/wal-syncfs/cells.json new file mode 100644 index 0000000..55ca6b7 --- /dev/null +++ b/results/wal-mem-diag-local/wal-syncfs/cells.json @@ -0,0 +1,58 @@ +{ + "cells": { + "20000": { + "stream_count": 20000, + "image_digest": "1ee33d6c2b18", + "walk": [ + [ + 2, + 62928.875, + 3.463, + 11.343 + ], + [ + 4, + 73717.25, + 5.839, + 14.951 + ] + ], + "pinned_pods": 4, + "throughput": 73717.25, + "p50": null, + "p99": null, + "pod_mem_mb": 133, + "saturated": false, + "status": "ok", + "reason": "ladder_exhausted", + "pod_mem_p50_mb": 106 + }, + "50000": { + "stream_count": 50000, + "image_digest": "1ee33d6c2b18", + "walk": [ + [ + 2, + 43436.625, + 4.075, + 39.519 + ], + [ + 4, + 36744.125, + 8.759, + 149.375 + ] + ], + "pinned_pods": 2, + "throughput": 23309.2, + "p50": 6.339, + "p99": 146.943, + "pod_mem_mb": 243, + "saturated": true, + "status": "ok", + "reason": "plateau", + "pod_mem_p50_mb": 226 + } + } +} \ No newline at end of file diff --git a/results/wal-multilane-shards/s1/cells.json b/results/wal-multilane-shards/s1/cells.json new file mode 100644 index 0000000..68eed15 --- /dev/null +++ b/results/wal-multilane-shards/s1/cells.json @@ -0,0 +1,170 @@ +{ + "cells": { + "200000": { + "stream_count": 200000, + "image_digest": "eb1a9e1707f7", + "walk": [ + [ + 2, + 13302.0, + 6.287, + 404.735 + ], + [ + 4, + 8562.849999999999, + 93.183, + 667.135 + ], + [ + 8, + 4774.500000000001, + 404.991, + 1549.311 + ] + ], + "pinned_pods": 2, + "throughput": 9914.3, + "p50": 56.799, + "p99": 255.231, + "pod_mem_mb": 0.0, + "saturated": true, + "status": "ok", + "reason": "plateau" + }, + "100": { + "stream_count": 100, + "image_digest": "eb1a9e1707f7", + "walk": [ + [ + 2, + 279181.1, + 1.505, + 5.343 + ], + [ + 4, + 267247.7, + 3.117, + 11.215 + ], + [ + 8, + 238230.0, + 6.571, + 28.543 + ] + ], + "pinned_pods": 2, + "throughput": 245088.0, + "p50": 1.515, + "p99": 6.599, + "pod_mem_mb": 1743.0, + "saturated": true, + "status": "ok", + "reason": "plateau" + }, + "1000": { + "stream_count": 1000, + "image_digest": "eb1a9e1707f7", + "walk": [ + [ + 2, + 152025.05, + 3.041, + 8.991 + ], + [ + 4, + 169909.3, + 5.587, + 15.935 + ], + [ + 8, + 151581.9, + 12.967, + 26.495 + ], + [ + 12, + 137728.9, + 21.135, + 50.015 + ] + ], + "pinned_pods": 4, + "throughput": 166667.0, + "p50": 5.863, + "p99": 15.455, + "pod_mem_mb": 0.0, + "saturated": true, + "status": "ok", + "reason": "plateau" + }, + "10000": { + "stream_count": 10000, + "image_digest": "eb1a9e1707f7", + "walk": [ + [ + 2, + 104758.45000000001, + 4.631, + 17.631 + ], + [ + 4, + 95399.15, + 9.255, + 33.343 + ], + [ + 8, + 100706.70000000001, + 19.823, + 48.543 + ] + ], + "pinned_pods": 2, + "throughput": 96294.4, + "p50": 3.627, + "p99": 27.215, + "pod_mem_mb": 0.0, + "saturated": true, + "status": "ok", + "reason": "plateau" + }, + "100000": { + "stream_count": 100000, + "image_digest": "eb1a9e1707f7", + "walk": [ + [ + 2, + 11471.400000000001, + 43.263, + 143.871 + ], + [ + 4, + 8124.65, + 124.351, + 436.991 + ], + [ + 8, + 7367.45, + 268.287, + 979.455 + ] + ], + "pinned_pods": 2, + "throughput": 12871.7, + "p50": 4.323, + "p99": 444.671, + "pod_mem_mb": 0.0, + "saturated": true, + "status": "ok", + "reason": "plateau" + } + } +} \ No newline at end of file diff --git a/results/wal-multilane-shards/s2/cells.json b/results/wal-multilane-shards/s2/cells.json new file mode 100644 index 0000000..7ebb0ca --- /dev/null +++ b/results/wal-multilane-shards/s2/cells.json @@ -0,0 +1,138 @@ +{ + "cells": { + "100": { + "stream_count": 100, + "image_digest": "aa0b738cc42c", + "walk": [ + [ + 2, + 249622.40000000002, + 1.381, + 9.367 + ], + [ + 4, + 218285.55, + 3.021, + 24.159 + ], + [ + 8, + 248984.59999999998, + 6.079, + 24.623 + ], + [ + 12, + 248463.90000000002, + 9.255, + 38.175 + ] + ], + "pinned_pods": 12, + "throughput": 248463.90000000002, + "p50": null, + "p99": null, + "pod_mem_mb": 2031.0, + "saturated": false, + "status": "ok", + "reason": "ladder_exhausted" + }, + "1000": { + "stream_count": 1000, + "image_digest": "aa0b738cc42c", + "walk": [ + [ + 2, + 215026.25, + 1.794, + 8.107 + ], + [ + 4, + 196914.45, + 4.069, + 14.991 + ], + [ + 8, + 208946.0, + 8.239, + 22.703 + ] + ], + "pinned_pods": 2, + "throughput": 217378.0, + "p50": 1.733, + "p99": 8.295, + "pod_mem_mb": 0.0, + "saturated": true, + "status": "ok", + "reason": "plateau" + }, + "10000": { + "stream_count": 10000, + "image_digest": "aa0b738cc42c", + "walk": [ + [ + 2, + 103985.20000000001, + 3.975, + 21.247 + ], + [ + 4, + 102119.7, + 8.903, + 35.935 + ], + [ + 8, + 107801.40000000001, + 17.455, + 58.175 + ] + ], + "pinned_pods": 2, + "throughput": 110716.0, + "p50": 2.729, + "p99": 23.455, + "pod_mem_mb": 0.0, + "saturated": true, + "status": "ok", + "reason": "plateau" + }, + "100000": { + "stream_count": 100000, + "image_digest": "aa0b738cc42c", + "walk": [ + [ + 2, + 10436.35, + 54.015, + 156.799 + ], + [ + 4, + 8514.15, + 96.639, + 829.951 + ], + [ + 8, + 6386.4, + 325.887, + 1096.703 + ] + ], + "pinned_pods": 2, + "throughput": 22756.7, + "p50": 2.083, + "p99": 229.119, + "pod_mem_mb": 0.0, + "saturated": true, + "status": "ok", + "reason": "plateau" + } + } +} \ No newline at end of file diff --git a/results/wal-multilane-shards/s4/cells.json b/results/wal-multilane-shards/s4/cells.json new file mode 100644 index 0000000..b13439b --- /dev/null +++ b/results/wal-multilane-shards/s4/cells.json @@ -0,0 +1,144 @@ +{ + "cells": { + "100": { + "stream_count": 100, + "image_digest": "3fde14045b8f", + "walk": [ + [ + 2, + 195818.0, + 1.395, + 15.999 + ], + [ + 4, + 254600.09999999998, + 2.915, + 22.063 + ], + [ + 8, + 222623.75, + 6.215, + 50.079 + ], + [ + 12, + 236353.5, + 9.383, + 52.799 + ] + ], + "pinned_pods": 4, + "throughput": 209048.0, + "p50": 2.957, + "p99": 28.543, + "pod_mem_mb": 0.0, + "saturated": true, + "status": "ok", + "reason": "plateau" + }, + "1000": { + "stream_count": 1000, + "image_digest": "3fde14045b8f", + "walk": [ + [ + 2, + 221033.25, + 1.608, + 11.047 + ], + [ + 4, + 226656.85, + 3.517, + 14.039 + ], + [ + 8, + 203777.15, + 7.171, + 42.911 + ] + ], + "pinned_pods": 2, + "throughput": 239381.0, + "p50": 1.576, + "p99": 8.919, + "pod_mem_mb": 0.0, + "saturated": true, + "status": "ok", + "reason": "plateau" + }, + "10000": { + "stream_count": 10000, + "image_digest": "3fde14045b8f", + "walk": [ + [ + 2, + 98871.45000000001, + 3.565, + 27.151 + ], + [ + 4, + 109948.79999999999, + 7.307, + 35.423 + ], + [ + 8, + 106981.80000000002, + 17.599, + 55.199 + ], + [ + 12, + 108550.0, + 27.999, + 69.055 + ] + ], + "pinned_pods": 4, + "throughput": 104960.0, + "p50": 8.607, + "p99": 33.407, + "pod_mem_mb": 0.0, + "saturated": true, + "status": "ok", + "reason": "plateau" + }, + "100000": { + "stream_count": 100000, + "image_digest": "3fde14045b8f", + "walk": [ + [ + 2, + 10807.3, + 44.895, + 157.951 + ], + [ + 4, + 9755.95, + 98.175, + 287.999 + ], + [ + 8, + 6471.650000000001, + 313.855, + 696.319 + ] + ], + "pinned_pods": 2, + "throughput": 13678.5, + "p50": 33.023, + "p99": 201.087, + "pod_mem_mb": 0.0, + "saturated": true, + "status": "ok", + "reason": "plateau" + } + } +} \ No newline at end of file diff --git a/results/wal-multilane-shards/s6/cells.json b/results/wal-multilane-shards/s6/cells.json new file mode 100644 index 0000000..8456f72 --- /dev/null +++ b/results/wal-multilane-shards/s6/cells.json @@ -0,0 +1,36 @@ +{ + "cells": { + "100": { + "stream_count": 100, + "image_digest": "e4ae264d73e0", + "walk": [ + [ + 2, + 223352.25, + 1.377, + 15.535 + ], + [ + 4, + 213347.15, + 2.799, + 35.999 + ], + [ + 8, + 225984.35, + 6.023, + 49.695 + ] + ], + "pinned_pods": 2, + "throughput": 211132.0, + "p50": 1.357, + "p99": 15.255, + "pod_mem_mb": 0.0, + "saturated": true, + "status": "ok", + "reason": "plateau" + } + } +} \ No newline at end of file diff --git a/results/wal-shard-sweep/aggregate.csv b/results/wal-shard-sweep/aggregate.csv new file mode 100644 index 0000000..1c88b7c --- /dev/null +++ b/results/wal-shard-sweep/aggregate.csv @@ -0,0 +1,6 @@ +mode,stream_count,pods,throughput,p50,p99,knee_pods,knee_throughput,knee_p50,knee_p99,pod_mem_mb,pod_mem_p50_mb,saturated,status,reason +wal-s1,200000,8,66183.7,27.807,96.191,1,54072.85,4.179,21.679,1088,1055,True,ok,plateau +wal-s16,200000,8,71106.2,28.575,59.103,2,48790.9,10.223,21.247,1073,1048,True,ok,plateau +wal-s24,200000,8,64311.2,30.319,66.239,4,51795.95,18.815,46.047,1100,1063,True,ok,plateau +wal-s4,200000,8,66530.6,27.647,84.735,4,60279.2,14.743,50.495,1092,1076,True,ok,plateau +wal-s8,200000,4,67487.9,14.599,40.223,2,55098.649999999994,8.367,31.855,1041,1020,True,ok,plateau diff --git a/results/wal-shard-sweep/aggregate.json b/results/wal-shard-sweep/aggregate.json new file mode 100644 index 0000000..db040ce --- /dev/null +++ b/results/wal-shard-sweep/aggregate.json @@ -0,0 +1,271 @@ +[ + { + "mode": "wal-s1", + "stream_count": 200000, + "pods": 8, + "throughput": 66183.7, + "p50": 27.807, + "p99": 96.191, + "knee_pods": 1, + "knee_throughput": 54072.85, + "knee_p50": 4.179, + "knee_p99": 21.679, + "pod_mem_mb": 1088, + "pod_mem_p50_mb": 1055, + "saturated": true, + "status": "ok", + "reason": "plateau", + "walk": [ + [ + 1, + 54072.85, + 4.179, + 21.679 + ], + [ + 2, + 58946.05, + 7.435, + 38.623 + ], + [ + 4, + 62244.65000000001, + 14.063, + 65.215 + ], + [ + 8, + 72061.7, + 27.647, + 78.591 + ], + [ + 12, + 67940.1, + 41.535, + 130.687 + ], + [ + 16, + 67103.4, + 55.039, + 155.391 + ] + ] + }, + { + "mode": "wal-s16", + "stream_count": 200000, + "pods": 8, + "throughput": 71106.2, + "p50": 28.575, + "p99": 59.103, + "knee_pods": 2, + "knee_throughput": 48790.9, + "knee_p50": 10.223, + "knee_p99": 21.247, + "pod_mem_mb": 1073, + "pod_mem_p50_mb": 1048, + "saturated": true, + "status": "ok", + "reason": "plateau", + "walk": [ + [ + 1, + 34123.9, + 7.051, + 23.791 + ], + [ + 2, + 48790.9, + 10.223, + 21.247 + ], + [ + 4, + 60155.950000000004, + 16.351, + 39.263 + ], + [ + 8, + 68474.2, + 28.831, + 67.647 + ], + [ + 12, + 71914.3, + 42.111, + 80.767 + ], + [ + 16, + 72954.3, + 55.135, + 108.735 + ] + ] + }, + { + "mode": "wal-s24", + "stream_count": 200000, + "pods": 8, + "throughput": 64311.2, + "p50": 30.319, + "p99": 66.239, + "knee_pods": 4, + "knee_throughput": 51795.95, + "knee_p50": 18.815, + "knee_p99": 46.047, + "pod_mem_mb": 1100, + "pod_mem_p50_mb": 1063, + "saturated": true, + "status": "ok", + "reason": "plateau", + "walk": [ + [ + 1, + 34123.65, + 7.123, + 23.087 + ], + [ + 2, + 42837.75, + 11.415, + 29.583 + ], + [ + 4, + 51795.95, + 18.815, + 46.047 + ], + [ + 8, + 67258.7, + 29.983, + 53.983 + ], + [ + 12, + 68259.4, + 43.615, + 85.503 + ], + [ + 16, + 71206.24999999999, + 56.351, + 102.463 + ] + ] + }, + { + "mode": "wal-s4", + "stream_count": 200000, + "pods": 8, + "throughput": 66530.6, + "p50": 27.647, + "p99": 84.735, + "knee_pods": 4, + "knee_throughput": 60279.2, + "knee_p50": 14.743, + "knee_p99": 50.495, + "pod_mem_mb": 1092, + "pod_mem_p50_mb": 1076, + "saturated": true, + "status": "ok", + "reason": "plateau", + "walk": [ + [ + 1, + 48947.0, + 4.747, + 26.319 + ], + [ + 2, + 56331.65, + 7.943, + 32.047 + ], + [ + 4, + 60279.2, + 14.743, + 50.495 + ], + [ + 8, + 69483.9, + 27.871, + 73.983 + ], + [ + 12, + 70896.50000000001, + 41.279, + 95.039 + ], + [ + 16, + 75355.6, + 53.919, + 104.447 + ] + ] + }, + { + "mode": "wal-s8", + "stream_count": 200000, + "pods": 4, + "throughput": 67487.9, + "p50": 14.599, + "p99": 40.223, + "knee_pods": 2, + "knee_throughput": 55098.649999999994, + "knee_p50": 8.367, + "knee_p99": 31.855, + "pod_mem_mb": 1041, + "pod_mem_p50_mb": 1020, + "saturated": true, + "status": "ok", + "reason": "plateau", + "walk": [ + [ + 1, + 41781.65, + 5.651, + 21.247 + ], + [ + 2, + 55098.649999999994, + 8.367, + 31.855 + ], + [ + 4, + 67617.15, + 14.575, + 37.951 + ], + [ + 8, + 72084.95, + 27.791, + 59.871 + ], + [ + 12, + 73472.9, + 41.567, + 78.143 + ] + ] + } +] \ No newline at end of file diff --git a/results/wal-shard-sweep/report.md b/results/wal-shard-sweep/report.md new file mode 100644 index 0000000..9d87394 --- /dev/null +++ b/results/wal-shard-sweep/report.md @@ -0,0 +1,41 @@ +# wal-shard-sweep — write-throughput report + +## Throughput at saturation (ops/s) + +| streams | wal-s1 | wal-s4 | wal-s8 | wal-s16 | wal-s24 | +|---|---|---|---|---|---| +| 200000 | 66k | 67k | 67k | 71k | 64k | + +† = not saturated (ladder exhausted) — treat as a lower bound. + +## Pod memory at saturation — peak / p50 (MiB) + +| streams | wal-s1 | wal-s4 | wal-s8 | wal-s16 | wal-s24 | +|---|---|---|---|---|---| +| 200000 | 1088 / 1055 | 1092 / 1076 | 1041 / 1020 | 1073 / 1048 | 1100 / 1063 | + +_Pod working set = cgroup `memory.current − inactive_file` (anon + active page cache), sampled each second at the pinned rung. **peak** = high-water (catches bursts like an in-RAM Raft log filling); **p50** = median (what the server steadily holds resident). peak ≈ p50 ⇒ steadily resident; peak ≫ p50 ⇒ transient spikes._ + +## Latency (ms, p50 / p99) + +| streams | wal-s1 @≤80% load | wal-s1 @saturation | wal-s4 @≤80% load | wal-s4 @saturation | wal-s8 @≤80% load | wal-s8 @saturation | wal-s16 @≤80% load | wal-s16 @saturation | wal-s24 @≤80% load | wal-s24 @saturation | +|---|---|---|---|---|---|---|---|---|---|---| +| 200000 | 4.2 / 21.7 (54k @1p) | 27.8 / 96.2 | 14.7 / 50.5 (60k @4p) | 27.6 / 84.7 | 8.4 / 31.9 (55k @2p) | 14.6 / 40.2 | 10.2 / 21.2 (49k @2p) | 28.6 / 59.1 | 18.8 / 46.0 (52k @4p) | 30.3 / 66.2 | + +_@≤80% load = the largest ladder rung at ≤80% of peak throughput — the server's service latency with headroom. @saturation = the pinned plateau rung, where a closed-loop fleet measures its own queueing (≈ in-flight ÷ ceiling by Little's law), NOT the server's per-request cost. Compare against the unloaded single-request baseline (~1 ms for wal) before reading anything into large saturation values._ + +## Saturation walks (pods → ops/s, p50 ms) + +- **wal-s1 200000**: 1:54k@4.2ms → 2:59k@7.4ms → 4:62k@14.1ms → 8:72k@27.6ms → 12:68k@41.5ms → 16:67k@55.0ms (pinned 8, plateau) +- **wal-s16 200000**: 1:34k@7.1ms → 2:49k@10.2ms → 4:60k@16.4ms → 8:68k@28.8ms → 12:72k@42.1ms → 16:73k@55.1ms (pinned 8, plateau) +- **wal-s24 200000**: 1:34k@7.1ms → 2:43k@11.4ms → 4:52k@18.8ms → 8:67k@30.0ms → 12:68k@43.6ms → 16:71k@56.4ms (pinned 8, plateau) +- **wal-s4 200000**: 1:49k@4.7ms → 2:56k@7.9ms → 4:60k@14.7ms → 8:69k@27.9ms → 12:71k@41.3ms → 16:75k@53.9ms (pinned 8, plateau) +- **wal-s8 200000**: 1:42k@5.7ms → 2:55k@8.4ms → 4:68k@14.6ms → 8:72k@27.8ms → 12:73k@41.6ms (pinned 4, plateau) + +## Findings + +_TODO: written by hand on top of the generated data._ + +## Caveats + +_Single-node best-case; not 3-node Raft. Throughput is a saturation ceiling per the ladder._ diff --git a/results/wal-shard-sweep/wal-s1/cells.json b/results/wal-shard-sweep/wal-s1/cells.json new file mode 100644 index 0000000..ab90d3a --- /dev/null +++ b/results/wal-shard-sweep/wal-s1/cells.json @@ -0,0 +1,55 @@ +{ + "cells": { + "200000": { + "stream_count": 200000, + "image_digest": "eb1a9e1707f7", + "walk": [ + [ + 1, + 54072.85, + 4.179, + 21.679 + ], + [ + 2, + 58946.05, + 7.435, + 38.623 + ], + [ + 4, + 62244.65000000001, + 14.063, + 65.215 + ], + [ + 8, + 72061.7, + 27.647, + 78.591 + ], + [ + 12, + 67940.1, + 41.535, + 130.687 + ], + [ + 16, + 67103.4, + 55.039, + 155.391 + ] + ], + "pinned_pods": 8, + "throughput": 66183.7, + "p50": 27.807, + "p99": 96.191, + "pod_mem_mb": 1088, + "saturated": true, + "status": "ok", + "reason": "plateau", + "pod_mem_p50_mb": 1055 + } + } +} \ No newline at end of file diff --git a/results/wal-shard-sweep/wal-s16/cells.json b/results/wal-shard-sweep/wal-s16/cells.json new file mode 100644 index 0000000..bc6fee6 --- /dev/null +++ b/results/wal-shard-sweep/wal-s16/cells.json @@ -0,0 +1,55 @@ +{ + "cells": { + "200000": { + "stream_count": 200000, + "image_digest": "ac15eab20c5a", + "walk": [ + [ + 1, + 34123.9, + 7.051, + 23.791 + ], + [ + 2, + 48790.9, + 10.223, + 21.247 + ], + [ + 4, + 60155.950000000004, + 16.351, + 39.263 + ], + [ + 8, + 68474.2, + 28.831, + 67.647 + ], + [ + 12, + 71914.3, + 42.111, + 80.767 + ], + [ + 16, + 72954.3, + 55.135, + 108.735 + ] + ], + "pinned_pods": 8, + "throughput": 71106.2, + "p50": 28.575, + "p99": 59.103, + "pod_mem_mb": 1073, + "saturated": true, + "status": "ok", + "reason": "plateau", + "pod_mem_p50_mb": 1048 + } + } +} \ No newline at end of file diff --git a/results/wal-shard-sweep/wal-s24/cells.json b/results/wal-shard-sweep/wal-s24/cells.json new file mode 100644 index 0000000..5670324 --- /dev/null +++ b/results/wal-shard-sweep/wal-s24/cells.json @@ -0,0 +1,55 @@ +{ + "cells": { + "200000": { + "stream_count": 200000, + "image_digest": "bd1859edf336", + "walk": [ + [ + 1, + 34123.65, + 7.123, + 23.087 + ], + [ + 2, + 42837.75, + 11.415, + 29.583 + ], + [ + 4, + 51795.95, + 18.815, + 46.047 + ], + [ + 8, + 67258.7, + 29.983, + 53.983 + ], + [ + 12, + 68259.4, + 43.615, + 85.503 + ], + [ + 16, + 71206.24999999999, + 56.351, + 102.463 + ] + ], + "pinned_pods": 8, + "throughput": 64311.2, + "p50": 30.319, + "p99": 66.239, + "pod_mem_mb": 1100, + "saturated": true, + "status": "ok", + "reason": "plateau", + "pod_mem_p50_mb": 1063 + } + } +} \ No newline at end of file diff --git a/results/wal-shard-sweep/wal-s4/cells.json b/results/wal-shard-sweep/wal-s4/cells.json new file mode 100644 index 0000000..f7dd372 --- /dev/null +++ b/results/wal-shard-sweep/wal-s4/cells.json @@ -0,0 +1,55 @@ +{ + "cells": { + "200000": { + "stream_count": 200000, + "image_digest": "3fde14045b8f", + "walk": [ + [ + 1, + 48947.0, + 4.747, + 26.319 + ], + [ + 2, + 56331.65, + 7.943, + 32.047 + ], + [ + 4, + 60279.2, + 14.743, + 50.495 + ], + [ + 8, + 69483.9, + 27.871, + 73.983 + ], + [ + 12, + 70896.50000000001, + 41.279, + 95.039 + ], + [ + 16, + 75355.6, + 53.919, + 104.447 + ] + ], + "pinned_pods": 8, + "throughput": 66530.6, + "p50": 27.647, + "p99": 84.735, + "pod_mem_mb": 1092, + "saturated": true, + "status": "ok", + "reason": "plateau", + "pod_mem_p50_mb": 1076 + } + } +} \ No newline at end of file diff --git a/results/wal-shard-sweep/wal-s8/cells.json b/results/wal-shard-sweep/wal-s8/cells.json new file mode 100644 index 0000000..2e0298d --- /dev/null +++ b/results/wal-shard-sweep/wal-s8/cells.json @@ -0,0 +1,49 @@ +{ + "cells": { + "200000": { + "stream_count": 200000, + "image_digest": "1c7d5a766789", + "walk": [ + [ + 1, + 41781.65, + 5.651, + 21.247 + ], + [ + 2, + 55098.649999999994, + 8.367, + 31.855 + ], + [ + 4, + 67617.15, + 14.575, + 37.951 + ], + [ + 8, + 72084.95, + 27.791, + 59.871 + ], + [ + 12, + 73472.9, + 41.567, + 78.143 + ] + ], + "pinned_pods": 4, + "throughput": 67487.9, + "p50": 14.599, + "p99": 40.223, + "pod_mem_mb": 1041, + "saturated": true, + "status": "ok", + "reason": "plateau", + "pod_mem_p50_mb": 1020 + } + } +} \ No newline at end of file diff --git a/results/wal-sizetrigger/aggregate.csv b/results/wal-sizetrigger/aggregate.csv new file mode 100644 index 0000000..614c284 --- /dev/null +++ b/results/wal-sizetrigger/aggregate.csv @@ -0,0 +1,5 @@ +mode,stream_count,pods,throughput,p50,p99,knee_pods,knee_throughput,knee_p50,knee_p99,pod_mem_mb,pod_mem_p50_mb,saturated,status,reason +ref-3s,10000,8,298060.2,,,4,298060.2,3.149,8.311,236,183,False,ok,ladder_exhausted +size-1g,10000,8,314005.9,,,4,314005.9,3.083,7.475,294,186,False,ok,ladder_exhausted +ref-3s,100000,2,275252.0,1.479,6.383,2,268921.9,1.5,6.163,663,591,True,ok,plateau +size-1g,100000,2,303152.0,1.491,6.095,2,296213.65,1.479,5.843,693,600,True,ok,plateau diff --git a/results/wal-sizetrigger/aggregate.json b/results/wal-sizetrigger/aggregate.json new file mode 100644 index 0000000..bd56349 --- /dev/null +++ b/results/wal-sizetrigger/aggregate.json @@ -0,0 +1,138 @@ +[ + { + "mode": "ref-3s", + "stream_count": 10000, + "pods": 8, + "throughput": 298060.2, + "p50": null, + "p99": null, + "knee_pods": 4, + "knee_throughput": 298060.2, + "knee_p50": 3.149, + "knee_p99": 8.311, + "pod_mem_mb": 236, + "pod_mem_p50_mb": 183, + "saturated": false, + "status": "ok", + "reason": "ladder_exhausted", + "walk": [ + [ + 4, + 298060.2, + 3.149, + 8.311 + ], + [ + 8, + 293881.35000000003, + 6.519, + 21.375 + ] + ] + }, + { + "mode": "size-1g", + "stream_count": 10000, + "pods": 8, + "throughput": 314005.9, + "p50": null, + "p99": null, + "knee_pods": 4, + "knee_throughput": 314005.9, + "knee_p50": 3.083, + "knee_p99": 7.475, + "pod_mem_mb": 294, + "pod_mem_p50_mb": 186, + "saturated": false, + "status": "ok", + "reason": "ladder_exhausted", + "walk": [ + [ + 4, + 314005.9, + 3.083, + 7.475 + ], + [ + 8, + 309625.25, + 6.359, + 11.623 + ] + ] + }, + { + "mode": "ref-3s", + "stream_count": 100000, + "pods": 2, + "throughput": 275252.0, + "p50": 1.479, + "p99": 6.383, + "knee_pods": 2, + "knee_throughput": 268921.9, + "knee_p50": 1.5, + "knee_p99": 6.163, + "pod_mem_mb": 663, + "pod_mem_p50_mb": 591, + "saturated": true, + "status": "ok", + "reason": "plateau", + "walk": [ + [ + 2, + 268921.9, + 1.5, + 6.163 + ], + [ + 4, + 272068.4, + 3.137, + 7.847 + ], + [ + 8, + 269593.05, + 6.511, + 12.583 + ] + ] + }, + { + "mode": "size-1g", + "stream_count": 100000, + "pods": 2, + "throughput": 303152.0, + "p50": 1.491, + "p99": 6.095, + "knee_pods": 2, + "knee_throughput": 296213.65, + "knee_p50": 1.479, + "knee_p99": 5.843, + "pod_mem_mb": 693, + "pod_mem_p50_mb": 600, + "saturated": true, + "status": "ok", + "reason": "plateau", + "walk": [ + [ + 2, + 296213.65, + 1.479, + 5.843 + ], + [ + 4, + 292395.45, + 3.127, + 7.535 + ], + [ + 8, + 289558.95, + 6.395, + 11.727 + ] + ] + } +] \ No newline at end of file diff --git a/results/wal-sizetrigger/ref-3s/cells.json b/results/wal-sizetrigger/ref-3s/cells.json new file mode 100644 index 0000000..e9b33c4 --- /dev/null +++ b/results/wal-sizetrigger/ref-3s/cells.json @@ -0,0 +1,64 @@ +{ + "cells": { + "10000": { + "stream_count": 10000, + "image_digest": "a2298d6c8011", + "walk": [ + [ + 4, + 298060.2, + 3.149, + 8.311 + ], + [ + 8, + 293881.35000000003, + 6.519, + 21.375 + ] + ], + "pinned_pods": 8, + "throughput": 293881.35000000003, + "p50": null, + "p99": null, + "pod_mem_mb": 236, + "saturated": false, + "status": "ok", + "reason": "ladder_exhausted", + "pod_mem_p50_mb": 183 + }, + "100000": { + "stream_count": 100000, + "image_digest": "a2298d6c8011", + "walk": [ + [ + 2, + 268921.9, + 1.5, + 6.163 + ], + [ + 4, + 272068.4, + 3.137, + 7.847 + ], + [ + 8, + 269593.05, + 6.511, + 12.583 + ] + ], + "pinned_pods": 2, + "throughput": 275252.0, + "p50": 1.479, + "p99": 6.383, + "pod_mem_mb": 663, + "saturated": true, + "status": "ok", + "reason": "plateau", + "pod_mem_p50_mb": 591 + } + } +} \ No newline at end of file diff --git a/results/wal-sizetrigger/report.md b/results/wal-sizetrigger/report.md new file mode 100644 index 0000000..be733d2 --- /dev/null +++ b/results/wal-sizetrigger/report.md @@ -0,0 +1,43 @@ +# wal-sizetrigger — write-throughput report + +## Throughput at saturation (ops/s) + +| streams | ref-3s | size-1g | +|---|---|---| +| 10000 | 298k† | 314k† | +| 100000 | 275k | 303k | + +† = not saturated (ladder exhausted) — treat as a lower bound. + +## Pod memory at saturation — peak / p50 (MiB) + +| streams | ref-3s | size-1g | +|---|---|---| +| 10000 | 236 / 183 | 294 / 186 | +| 100000 | 663 / 591 | 693 / 600 | + +_Pod working set = cgroup `memory.current − inactive_file` (anon + active page cache), sampled each second at the pinned rung. **peak** = high-water (catches bursts like an in-RAM Raft log filling); **p50** = median (what the server steadily holds resident). peak ≈ p50 ⇒ steadily resident; peak ≫ p50 ⇒ transient spikes._ + +## Latency (ms, p50 / p99) + +| streams | ref-3s @≤80% load | ref-3s @saturation | size-1g @≤80% load | size-1g @saturation | +|---|---|---|---|---| +| 10000 | 3.1 / 8.3 (298k @4p) | — | 3.1 / 7.5 (314k @4p) | — | +| 100000 | 1.5 / 6.2 (269k @2p) | 1.5 / 6.4 | 1.5 / 5.8 (296k @2p) | 1.5 / 6.1 | + +_@≤80% load = the largest ladder rung at ≤80% of peak throughput — the server's service latency with headroom. @saturation = the pinned plateau rung, where a closed-loop fleet measures its own queueing (≈ in-flight ÷ ceiling by Little's law), NOT the server's per-request cost. Compare against the unloaded single-request baseline (~1 ms for wal) before reading anything into large saturation values._ + +## Saturation walks (pods → ops/s, p50 ms) + +- **ref-3s 10000**: 4:298k@3.1ms → 8:294k@6.5ms (pinned 8, ladder_exhausted) +- **size-1g 10000**: 4:314k@3.1ms → 8:310k@6.4ms (pinned 8, ladder_exhausted) +- **ref-3s 100000**: 2:269k@1.5ms → 4:272k@3.1ms → 8:270k@6.5ms (pinned 2, plateau) +- **size-1g 100000**: 2:296k@1.5ms → 4:292k@3.1ms → 8:290k@6.4ms (pinned 2, plateau) + +## Findings + +size-1g (--wal-checkpoint-wal-bytes 1GiB, 60s fallback) reaches the checkpoint-off ceiling: 309.6k @10k / 303.2k @100k vs ckpt-off 305k/306k (wal-splitlane) and ref-3s 293.9k/275.3k. The size trigger reclaims the entire 7-11% checkpoint cost while bounding crash-replay to <=1 GiB retained WAL per shard. Shipped as PR #4704 (stacked on #4697). + +## Caveats + +_Single-node best-case; not 3-node Raft. Throughput is a saturation ceiling per the ladder._ diff --git a/results/wal-sizetrigger/size-1g/cells.json b/results/wal-sizetrigger/size-1g/cells.json new file mode 100644 index 0000000..cf73585 --- /dev/null +++ b/results/wal-sizetrigger/size-1g/cells.json @@ -0,0 +1,64 @@ +{ + "cells": { + "10000": { + "stream_count": 10000, + "image_digest": "2cfd37a9a235", + "walk": [ + [ + 4, + 314005.9, + 3.083, + 7.475 + ], + [ + 8, + 309625.25, + 6.359, + 11.623 + ] + ], + "pinned_pods": 8, + "throughput": 309625.25, + "p50": null, + "p99": null, + "pod_mem_mb": 294, + "saturated": false, + "status": "ok", + "reason": "ladder_exhausted", + "pod_mem_p50_mb": 186 + }, + "100000": { + "stream_count": 100000, + "image_digest": "2cfd37a9a235", + "walk": [ + [ + 2, + 296213.65, + 1.479, + 5.843 + ], + [ + 4, + 292395.45, + 3.127, + 7.535 + ], + [ + 8, + 289558.95, + 6.395, + 11.727 + ] + ], + "pinned_pods": 2, + "throughput": 303152.0, + "p50": 1.491, + "p99": 6.095, + "pod_mem_mb": 693, + "saturated": true, + "status": "ok", + "reason": "plateau", + "pod_mem_p50_mb": 600 + } + } +} \ No newline at end of file diff --git a/results/wal-splitlane/aggregate.csv b/results/wal-splitlane/aggregate.csv new file mode 100644 index 0000000..32a229a --- /dev/null +++ b/results/wal-splitlane/aggregate.csv @@ -0,0 +1,7 @@ +mode,stream_count,pods,throughput,p50,p99,knee_pods,knee_throughput,knee_p50,knee_p99,pod_mem_mb,pod_mem_p50_mb,saturated,status,reason +ckpt-off,10000,8,309316.80000000005,,,4,309316.80000000005,3.129,7.943,291,204,False,ok,ladder_exhausted +nofsync,10000,8,269793.2,,,4,260584.6,3.135,8.831,233,190,False,ok,ladder_exhausted +ref-3s,10000,8,293223.55,,,4,293223.55,3.135,16.447,229,176,False,ok,ladder_exhausted +ckpt-off,100000,2,305989.0,1.478,6.599,2,296232.25,1.518,6.399,689,598,True,ok,plateau +nofsync,100000,2,263776.0,1.43,7.523,2,259764.05,1.447,7.371,629,584,True,ok,plateau +ref-3s,100000,2,271574.0,1.512,6.531,2,269027.80000000005,1.513,6.671,657,580,True,ok,plateau diff --git a/results/wal-splitlane/aggregate.json b/results/wal-splitlane/aggregate.json new file mode 100644 index 0000000..5350f74 --- /dev/null +++ b/results/wal-splitlane/aggregate.json @@ -0,0 +1,206 @@ +[ + { + "mode": "ckpt-off", + "stream_count": 10000, + "pods": 8, + "throughput": 309316.80000000005, + "p50": null, + "p99": null, + "knee_pods": 4, + "knee_throughput": 309316.80000000005, + "knee_p50": 3.129, + "knee_p99": 7.943, + "pod_mem_mb": 291, + "pod_mem_p50_mb": 204, + "saturated": false, + "status": "ok", + "reason": "ladder_exhausted", + "walk": [ + [ + 4, + 309316.80000000005, + 3.129, + 7.943 + ], + [ + 8, + 305053.85, + 6.459, + 11.687 + ] + ] + }, + { + "mode": "nofsync", + "stream_count": 10000, + "pods": 8, + "throughput": 269793.2, + "p50": null, + "p99": null, + "knee_pods": 4, + "knee_throughput": 260584.6, + "knee_p50": 3.135, + "knee_p99": 8.831, + "pod_mem_mb": 233, + "pod_mem_p50_mb": 190, + "saturated": false, + "status": "ok", + "reason": "ladder_exhausted", + "walk": [ + [ + 4, + 260584.6, + 3.135, + 8.831 + ], + [ + 8, + 269793.2, + 6.555, + 13.343 + ] + ] + }, + { + "mode": "ref-3s", + "stream_count": 10000, + "pods": 8, + "throughput": 293223.55, + "p50": null, + "p99": null, + "knee_pods": 4, + "knee_throughput": 293223.55, + "knee_p50": 3.135, + "knee_p99": 16.447, + "pod_mem_mb": 229, + "pod_mem_p50_mb": 176, + "saturated": false, + "status": "ok", + "reason": "ladder_exhausted", + "walk": [ + [ + 4, + 293223.55, + 3.135, + 16.447 + ], + [ + 8, + 286282.1, + 6.523, + 27.487 + ] + ] + }, + { + "mode": "ckpt-off", + "stream_count": 100000, + "pods": 2, + "throughput": 305989.0, + "p50": 1.478, + "p99": 6.599, + "knee_pods": 2, + "knee_throughput": 296232.25, + "knee_p50": 1.518, + "knee_p99": 6.399, + "pod_mem_mb": 689, + "pod_mem_p50_mb": 598, + "saturated": true, + "status": "ok", + "reason": "plateau", + "walk": [ + [ + 2, + 296232.25, + 1.518, + 6.399 + ], + [ + 4, + 298473.25, + 3.145, + 8.399 + ], + [ + 8, + 293619.2, + 6.515, + 12.151 + ] + ] + }, + { + "mode": "nofsync", + "stream_count": 100000, + "pods": 2, + "throughput": 263776.0, + "p50": 1.43, + "p99": 7.523, + "knee_pods": 2, + "knee_throughput": 259764.05, + "knee_p50": 1.447, + "knee_p99": 7.371, + "pod_mem_mb": 629, + "pod_mem_p50_mb": 584, + "saturated": true, + "status": "ok", + "reason": "plateau", + "walk": [ + [ + 2, + 259764.05, + 1.447, + 7.371 + ], + [ + 4, + 258613.3, + 3.151, + 8.895 + ], + [ + 8, + 250507.55000000002, + 6.679, + 14.063 + ] + ] + }, + { + "mode": "ref-3s", + "stream_count": 100000, + "pods": 2, + "throughput": 271574.0, + "p50": 1.512, + "p99": 6.531, + "knee_pods": 2, + "knee_throughput": 269027.80000000005, + "knee_p50": 1.513, + "knee_p99": 6.671, + "pod_mem_mb": 657, + "pod_mem_p50_mb": 580, + "saturated": true, + "status": "ok", + "reason": "plateau", + "walk": [ + [ + 2, + 269027.80000000005, + 1.513, + 6.671 + ], + [ + 4, + 255747.45, + 3.289, + 8.639 + ], + [ + 8, + 270907.8, + 6.727, + 13.487 + ] + ] + } +] \ No newline at end of file diff --git a/results/wal-splitlane/ckpt-off/cells.json b/results/wal-splitlane/ckpt-off/cells.json new file mode 100644 index 0000000..2a8eead --- /dev/null +++ b/results/wal-splitlane/ckpt-off/cells.json @@ -0,0 +1,64 @@ +{ + "cells": { + "10000": { + "stream_count": 10000, + "image_digest": "3cf478620244", + "walk": [ + [ + 4, + 309316.80000000005, + 3.129, + 7.943 + ], + [ + 8, + 305053.85, + 6.459, + 11.687 + ] + ], + "pinned_pods": 8, + "throughput": 305053.85, + "p50": null, + "p99": null, + "pod_mem_mb": 291, + "saturated": false, + "status": "ok", + "reason": "ladder_exhausted", + "pod_mem_p50_mb": 204 + }, + "100000": { + "stream_count": 100000, + "image_digest": "3cf478620244", + "walk": [ + [ + 2, + 296232.25, + 1.518, + 6.399 + ], + [ + 4, + 298473.25, + 3.145, + 8.399 + ], + [ + 8, + 293619.2, + 6.515, + 12.151 + ] + ], + "pinned_pods": 2, + "throughput": 305989.0, + "p50": 1.478, + "p99": 6.599, + "pod_mem_mb": 689, + "saturated": true, + "status": "ok", + "reason": "plateau", + "pod_mem_p50_mb": 598 + } + } +} \ No newline at end of file diff --git a/results/wal-splitlane/nofsync/cells.json b/results/wal-splitlane/nofsync/cells.json new file mode 100644 index 0000000..711ea1c --- /dev/null +++ b/results/wal-splitlane/nofsync/cells.json @@ -0,0 +1,64 @@ +{ + "cells": { + "10000": { + "stream_count": 10000, + "image_digest": "8ecce2503136", + "walk": [ + [ + 4, + 260584.6, + 3.135, + 8.831 + ], + [ + 8, + 269793.2, + 6.555, + 13.343 + ] + ], + "pinned_pods": 8, + "throughput": 269793.2, + "p50": null, + "p99": null, + "pod_mem_mb": 233, + "saturated": false, + "status": "ok", + "reason": "ladder_exhausted", + "pod_mem_p50_mb": 190 + }, + "100000": { + "stream_count": 100000, + "image_digest": "8ecce2503136", + "walk": [ + [ + 2, + 259764.05, + 1.447, + 7.371 + ], + [ + 4, + 258613.3, + 3.151, + 8.895 + ], + [ + 8, + 250507.55000000002, + 6.679, + 14.063 + ] + ], + "pinned_pods": 2, + "throughput": 263776.0, + "p50": 1.43, + "p99": 7.523, + "pod_mem_mb": 629, + "saturated": true, + "status": "ok", + "reason": "plateau", + "pod_mem_p50_mb": 584 + } + } +} \ No newline at end of file diff --git a/results/wal-splitlane/ref-3s/cells.json b/results/wal-splitlane/ref-3s/cells.json new file mode 100644 index 0000000..0aeda68 --- /dev/null +++ b/results/wal-splitlane/ref-3s/cells.json @@ -0,0 +1,64 @@ +{ + "cells": { + "10000": { + "stream_count": 10000, + "image_digest": "1bd30dc4c80f", + "walk": [ + [ + 4, + 293223.55, + 3.135, + 16.447 + ], + [ + 8, + 286282.1, + 6.523, + 27.487 + ] + ], + "pinned_pods": 8, + "throughput": 286282.1, + "p50": null, + "p99": null, + "pod_mem_mb": 229, + "saturated": false, + "status": "ok", + "reason": "ladder_exhausted", + "pod_mem_p50_mb": 176 + }, + "100000": { + "stream_count": 100000, + "image_digest": "1bd30dc4c80f", + "walk": [ + [ + 2, + 269027.80000000005, + 1.513, + 6.671 + ], + [ + 4, + 255747.45, + 3.289, + 8.639 + ], + [ + 8, + 270907.8, + 6.727, + 13.487 + ] + ], + "pinned_pods": 2, + "throughput": 271574.0, + "p50": 1.512, + "p99": 6.531, + "pod_mem_mb": 657, + "saturated": true, + "status": "ok", + "reason": "plateau", + "pod_mem_p50_mb": 580 + } + } +} \ No newline at end of file diff --git a/results/wal-splitlane/report.md b/results/wal-splitlane/report.md new file mode 100644 index 0000000..916e6f7 --- /dev/null +++ b/results/wal-splitlane/report.md @@ -0,0 +1,53 @@ +# wal-splitlane — write-throughput report + +## Throughput at saturation (ops/s) + +| streams | ref-3s | ckpt-off | nofsync | +|---|---|---|---| +| 10000 | 293k† | 309k† | 270k† | +| 100000 | 272k | 306k | 264k | + +† = not saturated (ladder exhausted) — treat as a lower bound. + +## Pod memory at saturation — peak / p50 (MiB) + +| streams | ref-3s | ckpt-off | nofsync | +|---|---|---|---| +| 10000 | 229 / 176 | 291 / 204 | 233 / 190 | +| 100000 | 657 / 580 | 689 / 598 | 629 / 584 | + +_Pod working set = cgroup `memory.current − inactive_file` (anon + active page cache), sampled each second at the pinned rung. **peak** = high-water (catches bursts like an in-RAM Raft log filling); **p50** = median (what the server steadily holds resident). peak ≈ p50 ⇒ steadily resident; peak ≫ p50 ⇒ transient spikes._ + +## Latency (ms, p50 / p99) + +| streams | ref-3s @≤80% load | ref-3s @saturation | ckpt-off @≤80% load | ckpt-off @saturation | nofsync @≤80% load | nofsync @saturation | +|---|---|---|---|---|---|---| +| 10000 | 3.1 / 16.4 (293k @4p) | — | 3.1 / 7.9 (309k @4p) | — | 3.1 / 8.8 (261k @4p) | — | +| 100000 | 1.5 / 6.7 (269k @2p) | 1.5 / 6.5 | 1.5 / 6.4 (296k @2p) | 1.5 / 6.6 | 1.4 / 7.4 (260k @2p) | 1.4 / 7.5 | + +_@≤80% load = the largest ladder rung at ≤80% of peak throughput — the server's service latency with headroom. @saturation = the pinned plateau rung, where a closed-loop fleet measures its own queueing (≈ in-flight ÷ ceiling by Little's law), NOT the server's per-request cost. Compare against the unloaded single-request baseline (~1 ms for wal) before reading anything into large saturation values._ + +## Saturation walks (pods → ops/s, p50 ms) + +- **ckpt-off 10000**: 4:309k@3.1ms → 8:305k@6.5ms (pinned 8, ladder_exhausted) +- **nofsync 10000**: 4:261k@3.1ms → 8:270k@6.6ms (pinned 8, ladder_exhausted) +- **ref-3s 10000**: 4:293k@3.1ms → 8:286k@6.5ms (pinned 8, ladder_exhausted) +- **ckpt-off 100000**: 2:296k@1.5ms → 4:298k@3.1ms → 8:294k@6.5ms (pinned 2, plateau) +- **nofsync 100000**: 2:260k@1.4ms → 4:259k@3.2ms → 8:251k@6.7ms (pinned 2, plateau) +- **ref-3s 100000**: 2:269k@1.5ms → 4:256k@3.3ms → 8:271k@6.7ms (pinned 2, plateau) + +## Findings + +**The write cardinality cliff is eliminated.** ref-3s (syncfs checkpoint @3s, full durability) holds 286k→272k ops/s from 10k→100k streams (−5%) on the split-lane layout — vs 10.4k @100k where this investigation started (26×). + +Decomposition (with wal-decomp-lane0): +- **Storage layout is the #1 lever.** Streams on the PD boot disk → 10.4k; everything on one shared NVMe lane → 46k; streams on their own lane + WAL shards on dedicated lanes → 272k. The old "~1000 fdatasync/s device ceiling" was commit-vs-checkpoint device contention. +- **Commit fdatasync on dedicated lanes is free**: ckpt-off (306k) ≥ nofsync (264k) — group-commit amortizes better under fsync backpressure than the free-running no-fsync path. +- **Checkpoint @3s costs ~7–11%** on this layout (306k → 286k/272k). The per-shard size-trigger knob (perf/wal-checkpoint-sizetrigger) can reclaim most of it by checkpointing on a retained-WAL budget instead of a timer. +- Remaining gap to memory mode (512k) is ~1.9× = WAL machinery (staging/double-write), not fsync — future work: io_uring WAL writer seam, batched mark-written. + +Production recipe: multi-device NVMe instance, streams dir on its own device, one WAL shard per remaining device, `--wal-checkpoint-syncfs on`. + +## Caveats + +_Single-node best-case; not 3-node Raft. Throughput is a saturation ceiling per the ladder._ diff --git a/results/wal-syncfs-local/aggregate.csv b/results/wal-syncfs-local/aggregate.csv new file mode 100644 index 0000000..9934156 --- /dev/null +++ b/results/wal-syncfs-local/aggregate.csv @@ -0,0 +1,5 @@ +mode,stream_count,pods,throughput,p50,p99,knee_pods,knee_throughput,knee_p50,knee_p99,pod_mem_mb,pod_mem_p50_mb,saturated,status,reason +baseline,20000,4,87167.5,,,2,43451.75,5.119,18.319,133,119,False,ok,ladder_exhausted +syncfs,20000,4,75809.25,,,2,61261.25,3.361,11.567,129,107,False,ok,ladder_exhausted +baseline,50000,6,66145.25,,,2,34725.625,5.875,29.039,271,243,False,ok,ladder_exhausted +syncfs,50000,6,61155.875,,,2,48074.625,4.089,19.663,255,244,False,ok,ladder_exhausted diff --git a/results/wal-syncfs-local/aggregate.json b/results/wal-syncfs-local/aggregate.json new file mode 100644 index 0000000..8caff33 --- /dev/null +++ b/results/wal-syncfs-local/aggregate.json @@ -0,0 +1,138 @@ +[ + { + "mode": "baseline", + "stream_count": 20000, + "pods": 4, + "throughput": 87167.5, + "p50": null, + "p99": null, + "knee_pods": 2, + "knee_throughput": 43451.75, + "knee_p50": 5.119, + "knee_p99": 18.319, + "pod_mem_mb": 133, + "pod_mem_p50_mb": 119, + "saturated": false, + "status": "ok", + "reason": "ladder_exhausted", + "walk": [ + [ + 2, + 43451.75, + 5.119, + 18.319 + ], + [ + 4, + 87167.5, + 5.511, + 12.943 + ] + ] + }, + { + "mode": "syncfs", + "stream_count": 20000, + "pods": 4, + "throughput": 75809.25, + "p50": null, + "p99": null, + "knee_pods": 2, + "knee_throughput": 61261.25, + "knee_p50": 3.361, + "knee_p99": 11.567, + "pod_mem_mb": 129, + "pod_mem_p50_mb": 107, + "saturated": false, + "status": "ok", + "reason": "ladder_exhausted", + "walk": [ + [ + 2, + 61261.25, + 3.361, + 11.567 + ], + [ + 4, + 75809.25, + 5.071, + 20.495 + ] + ] + }, + { + "mode": "baseline", + "stream_count": 50000, + "pods": 6, + "throughput": 66145.25, + "p50": null, + "p99": null, + "knee_pods": 2, + "knee_throughput": 34725.625, + "knee_p50": 5.875, + "knee_p99": 29.039, + "pod_mem_mb": 271, + "pod_mem_p50_mb": 243, + "saturated": false, + "status": "ok", + "reason": "ladder_exhausted", + "walk": [ + [ + 2, + 34725.625, + 5.875, + 29.039 + ], + [ + 4, + 56672.125, + 8.287, + 21.071 + ], + [ + 6, + 66145.25, + 9.567, + 42.303 + ] + ] + }, + { + "mode": "syncfs", + "stream_count": 50000, + "pods": 6, + "throughput": 61155.875, + "p50": null, + "p99": null, + "knee_pods": 2, + "knee_throughput": 48074.625, + "knee_p50": 4.089, + "knee_p99": 19.663, + "pod_mem_mb": 255, + "pod_mem_p50_mb": 244, + "saturated": false, + "status": "ok", + "reason": "ladder_exhausted", + "walk": [ + [ + 2, + 48074.625, + 4.089, + 19.663 + ], + [ + 4, + 54739.875, + 6.255, + 126.335 + ], + [ + 6, + 61155.875, + 8.003, + 173.439 + ] + ] + } +] \ No newline at end of file diff --git a/results/wal-syncfs-local/baseline/cells.json b/results/wal-syncfs-local/baseline/cells.json new file mode 100644 index 0000000..090f24a --- /dev/null +++ b/results/wal-syncfs-local/baseline/cells.json @@ -0,0 +1,64 @@ +{ + "cells": { + "20000": { + "stream_count": 20000, + "image_digest": "0bb2c6c89ce6", + "walk": [ + [ + 2, + 43451.75, + 5.119, + 18.319 + ], + [ + 4, + 87167.5, + 5.511, + 12.943 + ] + ], + "pinned_pods": 4, + "throughput": 87167.5, + "p50": null, + "p99": null, + "pod_mem_mb": 133, + "saturated": false, + "status": "ok", + "reason": "ladder_exhausted", + "pod_mem_p50_mb": 119 + }, + "50000": { + "stream_count": 50000, + "image_digest": "0bb2c6c89ce6", + "walk": [ + [ + 2, + 34725.625, + 5.875, + 29.039 + ], + [ + 4, + 56672.125, + 8.287, + 21.071 + ], + [ + 6, + 66145.25, + 9.567, + 42.303 + ] + ], + "pinned_pods": 6, + "throughput": 66145.25, + "p50": null, + "p99": null, + "pod_mem_mb": 271, + "saturated": false, + "status": "ok", + "reason": "ladder_exhausted", + "pod_mem_p50_mb": 243 + } + } +} \ No newline at end of file diff --git a/results/wal-syncfs-local/report.md b/results/wal-syncfs-local/report.md new file mode 100644 index 0000000..f0022e1 --- /dev/null +++ b/results/wal-syncfs-local/report.md @@ -0,0 +1,43 @@ +# wal-syncfs-local — write-throughput report + +## Throughput at saturation (ops/s) + +| streams | baseline | syncfs | +|---|---|---| +| 20000 | 87k† | 76k† | +| 50000 | 66k† | 61k† | + +† = not saturated (ladder exhausted) — treat as a lower bound. + +## Pod memory at saturation — peak / p50 (MiB) + +| streams | baseline | syncfs | +|---|---|---| +| 20000 | 133 / 119 | 129 / 107 | +| 50000 | 271 / 243 | 255 / 244 | + +_Pod working set = cgroup `memory.current − inactive_file` (anon + active page cache), sampled each second at the pinned rung. **peak** = high-water (catches bursts like an in-RAM Raft log filling); **p50** = median (what the server steadily holds resident). peak ≈ p50 ⇒ steadily resident; peak ≫ p50 ⇒ transient spikes._ + +## Latency (ms, p50 / p99) + +| streams | baseline @≤80% load | baseline @saturation | syncfs @≤80% load | syncfs @saturation | +|---|---|---|---|---| +| 20000 | 5.1 / 18.3 (43k @2p) | — | 3.4 / 11.6 (61k @2p) | — | +| 50000 | 5.9 / 29.0 (35k @2p) | — | 4.1 / 19.7 (48k @2p) | — | + +_@≤80% load = the largest ladder rung at ≤80% of peak throughput — the server's service latency with headroom. @saturation = the pinned plateau rung, where a closed-loop fleet measures its own queueing (≈ in-flight ÷ ceiling by Little's law), NOT the server's per-request cost. Compare against the unloaded single-request baseline (~1 ms for wal) before reading anything into large saturation values._ + +## Saturation walks (pods → ops/s, p50 ms) + +- **baseline 20000**: 2:43k@5.1ms → 4:87k@5.5ms (pinned 4, ladder_exhausted) +- **syncfs 20000**: 2:61k@3.4ms → 4:76k@5.1ms (pinned 4, ladder_exhausted) +- **baseline 50000**: 2:35k@5.9ms → 4:57k@8.3ms → 6:66k@9.6ms (pinned 6, ladder_exhausted) +- **syncfs 50000**: 2:48k@4.1ms → 4:55k@6.3ms → 6:61k@8.0ms (pinned 6, ladder_exhausted) + +## Findings + +_TODO: written by hand on top of the generated data._ + +## Caveats + +_Single-node best-case; not 3-node Raft. Throughput is a saturation ceiling per the ladder._ diff --git a/results/wal-syncfs-local/syncfs/cells.json b/results/wal-syncfs-local/syncfs/cells.json new file mode 100644 index 0000000..b0cc3b9 --- /dev/null +++ b/results/wal-syncfs-local/syncfs/cells.json @@ -0,0 +1,64 @@ +{ + "cells": { + "20000": { + "stream_count": 20000, + "image_digest": "8c09d3514768", + "walk": [ + [ + 2, + 61261.25, + 3.361, + 11.567 + ], + [ + 4, + 75809.25, + 5.071, + 20.495 + ] + ], + "pinned_pods": 4, + "throughput": 75809.25, + "p50": null, + "p99": null, + "pod_mem_mb": 129, + "saturated": false, + "status": "ok", + "reason": "ladder_exhausted", + "pod_mem_p50_mb": 107 + }, + "50000": { + "stream_count": 50000, + "image_digest": "8c09d3514768", + "walk": [ + [ + 2, + 48074.625, + 4.089, + 19.663 + ], + [ + 4, + 54739.875, + 6.255, + 126.335 + ], + [ + 6, + 61155.875, + 8.003, + 173.439 + ] + ], + "pinned_pods": 6, + "throughput": 61155.875, + "p50": null, + "p99": null, + "pod_mem_mb": 255, + "saturated": false, + "status": "ok", + "reason": "ladder_exhausted", + "pod_mem_p50_mb": 244 + } + } +} \ No newline at end of file diff --git a/results/write-cliff-nvme/memory-gated/cells.json b/results/write-cliff-nvme/memory-gated/cells.json new file mode 100644 index 0000000..46295f3 --- /dev/null +++ b/results/write-cliff-nvme/memory-gated/cells.json @@ -0,0 +1,80 @@ +{ + "cells": { + "100000": { + "stream_count": 100000, + "image_digest": "159e0d8b1abd", + "walk": [ + [ + 1, + 200201.96, + 1.181, + 2.395 + ], + [ + 2, + 369682.72, + 1.294, + 2.487 + ], + [ + 4, + 364785.6, + 2.183, + 7.383 + ], + [ + 8, + 327910.68, + 2.149, + 21.807 + ] + ], + "pinned_pods": 2, + "throughput": 367876.0, + "p50": 1.308, + "p99": 2.497, + "pod_mem_mb": 689.0, + "saturated": true, + "status": "ok", + "reason": "plateau" + }, + "500000": { + "stream_count": 500000, + "image_digest": "159e0d8b1abd", + "walk": [ + [ + 1, + 186231.68, + 1.183, + 2.415 + ], + [ + 2, + 257181.8, + 1.378, + 20.927 + ], + [ + 4, + 270023.48000000004, + 2.337, + 20.127 + ], + [ + 8, + 257586.04000000004, + 2.289, + 41.151 + ] + ], + "pinned_pods": 2, + "throughput": 231658.0, + "p50": 1.363, + "p99": 16.359, + "pod_mem_mb": 2868.0, + "saturated": true, + "status": "ok", + "reason": "plateau" + } + } +} \ No newline at end of file diff --git a/results/write-cliff-nvme/memory-nogate/cells.json b/results/write-cliff-nvme/memory-nogate/cells.json new file mode 100644 index 0000000..c39fc01 --- /dev/null +++ b/results/write-cliff-nvme/memory-nogate/cells.json @@ -0,0 +1,86 @@ +{ + "cells": { + "100000": { + "stream_count": 100000, + "image_digest": "23f7f67df1e1", + "walk": [ + [ + 1, + 195085.48, + 1.216, + 2.485 + ], + [ + 2, + 317949.8, + 1.297, + 19.135 + ], + [ + 4, + 300326.43999999994, + 2.163, + 24.335 + ], + [ + 8, + 302953.44000000006, + 2.221, + 36.287 + ] + ], + "pinned_pods": 2, + "throughput": 313013.0, + "p50": 1.308, + "p99": 19.023, + "pod_mem_mb": 691.0, + "saturated": true, + "status": "ok", + "reason": "plateau" + }, + "500000": { + "stream_count": 500000, + "image_digest": "23f7f67df1e1", + "walk": [ + [ + 1, + 158198.28, + 1.215, + 2.473 + ], + [ + 2, + 216115.44, + 1.371, + 20.671 + ], + [ + 4, + 237205.72000000003, + 2.259, + 23.183 + ], + [ + 8, + 224745.24, + 2.321, + 57.727 + ], + [ + 16, + 202831.88000000003, + 2.419, + 196.607 + ] + ], + "pinned_pods": 4, + "throughput": 236564.0, + "p50": 2.241, + "p99": 23.759, + "pod_mem_mb": 3028.0, + "saturated": true, + "status": "ok", + "reason": "plateau" + } + } +} \ No newline at end of file diff --git a/results/write-cliff-nvme/wal-fsync1/cells.json b/results/write-cliff-nvme/wal-fsync1/cells.json new file mode 100644 index 0000000..ce31450 --- /dev/null +++ b/results/write-cliff-nvme/wal-fsync1/cells.json @@ -0,0 +1,104 @@ +{ + "cells": { + "100000": { + "stream_count": 100000, + "image_digest": "195884eee1f7", + "walk": [ + [ + 1, + 55295.4, + 4.055, + 29.135 + ], + [ + 2, + 60305.36, + 7.459, + 33.791 + ], + [ + 4, + 62912.44, + 14.559, + 44.703 + ], + [ + 8, + 66045.48, + 28.879, + 56.255 + ] + ], + "pinned_pods": 2, + "throughput": 59301.0, + "p50": 7.543, + "p99": 34.175, + "pod_mem_mb": 641.0, + "saturated": true, + "status": "ok", + "reason": "plateau" + }, + "500000": { + "stream_count": 500000, + "image_digest": "195884eee1f7", + "walk": [ + [ + 1, + 33970.6, + 5.391, + 31.247 + ], + [ + 2, + 37496.119999999995, + 10.647, + 42.783 + ], + [ + 4, + 37202.4, + 25.023, + 66.815 + ], + [ + 8, + 45211.96000000001, + 39.231, + 125.183 + ], + [ + 16, + 42289.96000000001, + 69.567, + 332.543 + ], + [ + 32, + 46918.16, + 164.607, + 366.847 + ], + [ + 48, + 45099.28, + 247.423, + 570.879 + ], + [ + 64, + 48463.23999999999, + 325.887, + 937.983 + ] + ], + "pinned_pods": 32, + "throughput": 46918.16, + "p50": 0.0, + "p99": 0.0, + "pod_mem_mb": 2871.0, + "saturated": true, + "status": "ok", + "reason": "plateau" + } + } +} \ No newline at end of file diff --git a/results/write-cliff-nvme/wal-fsync16/cells.json b/results/write-cliff-nvme/wal-fsync16/cells.json new file mode 100644 index 0000000..d4c5fdd --- /dev/null +++ b/results/write-cliff-nvme/wal-fsync16/cells.json @@ -0,0 +1,54 @@ +{ + "cells": { + "100000": { + "stream_count": 100000, + "image_digest": "29798c4812fb", + "walk": [ + [ + 1, + 44686.32, + 4.131, + 12.495 + ], + [ + 2, + 46483.6, + 11.095, + 23.647 + ], + [ + 4, + 54569.68, + 18.335, + 40.735 + ], + [ + 8, + 54271.64, + 35.039, + 81.471 + ], + [ + 16, + 58713.31999999999, + 64.383, + 155.775 + ], + [ + 32, + 0.0, + 0.0, + 0.0 + ] + ], + "pinned_pods": null, + "throughput": 0.0, + "p50": null, + "p99": null, + "pod_mem_mb": null, + "saturated": false, + "status": "error", + "reason": "creation_choke" + } + } +} \ No newline at end of file diff --git a/results/write-nvme-bottleneck/memory/cells.json b/results/write-nvme-bottleneck/memory/cells.json new file mode 100644 index 0000000..c63c3d3 --- /dev/null +++ b/results/write-nvme-bottleneck/memory/cells.json @@ -0,0 +1,48 @@ +{ + "cells": { + "500000": { + "stream_count": 500000, + "image_digest": "3734ed07b9db", + "walk": [ + [ + 1, + 175274.44, + 1.191, + 2.431 + ], + [ + 2, + 234427.91999999998, + 1.404, + 18.639 + ], + [ + 4, + 267114.0, + 2.321, + 22.991 + ], + [ + 8, + 259408.03999999998, + 2.335, + 32.207 + ], + [ + 16, + 218496.95999999996, + 2.407, + 124.095 + ] + ], + "pinned_pods": 4, + "throughput": 259287.0, + "p50": 2.335, + "p99": 22.207, + "pod_mem_mb": 3247.0, + "saturated": true, + "status": "ok", + "reason": "plateau" + } + } +} \ No newline at end of file diff --git a/results/write-nvme-bottleneck/wal-s4/cells.json b/results/write-nvme-bottleneck/wal-s4/cells.json new file mode 100644 index 0000000..2592611 --- /dev/null +++ b/results/write-nvme-bottleneck/wal-s4/cells.json @@ -0,0 +1,36 @@ +{ + "cells": { + "500000": { + "stream_count": 500000, + "image_digest": "ce452f9cfacb", + "walk": [ + [ + 1, + 43001.4, + 5.043, + 24.207 + ], + [ + 2, + 41748.16, + 10.103, + 38.623 + ], + [ + 4, + 37944.8, + 24.495, + 67.135 + ] + ], + "pinned_pods": 1, + "throughput": 39937.7, + "p50": 4.879, + "p99": 29.407, + "pod_mem_mb": 2524.0, + "saturated": true, + "status": "ok", + "reason": "plateau" + } + } +} \ No newline at end of file diff --git a/scripts/cluster-up.sh b/scripts/cluster-up.sh index 1cbb18e..2047f46 100755 --- a/scripts/cluster-up.sh +++ b/scripts/cluster-up.sh @@ -45,18 +45,46 @@ else # 4th-gen Titanium "-lssd" machines bundle a fixed Local SSD (the count is set # by the machine type — gcloud rejects an explicit count). Older N2D-style # types let you stripe N×375 GB devices via LOCAL_SSD_COUNT. - case "$SERVER_MACHINE" in - *-lssd) LSSD_FLAG=(--ephemeral-storage-local-ssd) ;; - *) LSSD_FLAG=(--ephemeral-storage-local-ssd "count=${LOCAL_SSD_COUNT:-1}") ;; - esac + # + # SERVER_LOCAL_SSD_BLOCK=1 (default OFF) switches the server pool to RAW-BLOCK + # local NVMe (--local-nvme-ssd-block) INSTEAD of ephemeral-storage. Rationale: + # --ephemeral-storage-local-ssd RAID0-stripes ALL local SSDs into ONE + # filesystem = ONE fsync barrier (single fsync lane), so per-shard fdatasync + # can't scale. Raw block keeps each physical NVMe device a separate /dev node, + # and gke/durable-streams-multilane.yaml (MULTILANE=1) mkfs+mounts one device + # per WAL shard dir = one independent fsync lane per shard. On 3rd/4th-gen + # (C3/C4/C4D "-lssd") the count is FIXED by the machine type, so the raw-block + # flag takes NO count (see MULTILANE_SETUP.md). Existing behavior is preserved + # when the env is unset. + if [ "${SERVER_LOCAL_SSD_BLOCK:-0}" = "1" ]; then + case "$SERVER_MACHINE" in + *-lssd) LSSD_FLAG=(--local-nvme-ssd-block) ;; + *) LSSD_FLAG=(--local-nvme-ssd-block "count=${LOCAL_SSD_COUNT:-1}") ;; + esac + else + case "$SERVER_MACHINE" in + *-lssd) LSSD_FLAG=(--ephemeral-storage-local-ssd) ;; + *) LSSD_FLAG=(--ephemeral-storage-local-ssd "count=${LOCAL_SSD_COUNT:-1}") ;; + esac + fi # The server pool holds state, so it is on-demand by DEFAULT (a Spot # preemption mid-run kills the stateful server and invalidates that cell). # SPOT_SERVER=1 opts the server node into Spot too (cheapest; accept that a # preemption forces a re-run of the affected cells — the suite is resumable). SPOT_SERVER_FLAG=() [ "${SPOT_SERVER:-0}" = "1" ] && SPOT_SERVER_FLAG=(--spot) + # STATIC_CPU=1: server pool kubelet runs cpuManagerPolicy=static, so a + # GUARANTEED pod (integer CPU, requests==limits) gets EXCLUSIVE pinned cores + # (true CPU binding; pairs with gke/durable-streams-splitlane-guaranteed.yaml). + # Default off = shared cores (existing behavior preserved). + STATIC_CPU_FLAG=() + if [ "${STATIC_CPU:-0}" = "1" ]; then + _SYSCFG="$(mktemp /tmp/ds-syscfg-XXXXXX.yaml)" + printf 'kubeletConfig:\n cpuManagerPolicy: static\n' > "$_SYSCFG" + STATIC_CPU_FLAG=(--system-config-from-file "$_SYSCFG") + fi gcloud container clusters create "$CLUSTER" --zone "$ZONE" --project "$PROJECT" --num-nodes 1 \ - --machine-type "$SERVER_MACHINE" "${LSSD_FLAG[@]}" "${SPOT_SERVER_FLAG[@]}" \ + --machine-type "$SERVER_MACHINE" "${LSSD_FLAG[@]}" "${SPOT_SERVER_FLAG[@]}" "${STATIC_CPU_FLAG[@]}" \ --node-labels=role=server --network benchmarking --subnetwork benchmarking \ --enable-ip-alias --release-channel regular # The client fleet is disposable + fault-tolerant (the bench tolerates pod diff --git a/scripts/lib-bench.sh b/scripts/lib-bench.sh index c1b0f16..3dbcb8b 100755 --- a/scripts/lib-bench.sh +++ b/scripts/lib-bench.sh @@ -83,8 +83,24 @@ deploy_server() { echo " deploying durable-streams server: cpu=${cpu} extra='${extra_args}' (${DS_TARGET})..." + # MULTILANE=1 deploys the multi-lane variant (one NVMe device per WAL shard dir; + # see gke/durable-streams-multilane.yaml + MULTILANE_SETUP.md). It REQUIRES a + # server pool created with SERVER_LOCAL_SSD_BLOCK=1 (raw-block local NVMe). + # Default = the single-filesystem manifest (existing behavior preserved). The + # variant is arg-compatible, so the same tier/injection sed paths below apply. + local server_manifest="gke/durable-streams.yaml" + [ "${MULTILANE:-0}" = "1" ] && server_manifest="gke/durable-streams-multilane.yaml" + # SPLITLANE=1: device 0 = stream-data lane, devices 1..5 = WAL shard lanes + # (see gke/durable-streams-splitlane.yaml). Server args must use + # --data-dir /data/wal/0 and --wal-shards <= 5. Takes precedence over MULTILANE. + [ "${SPLITLANE:-0}" = "1" ] && server_manifest="gke/durable-streams-splitlane.yaml" + # GUARANTEED=1 (with SPLITLANE=1): Guaranteed-QoS variant — requests==limits on + # every container, integer server CPU. On a STATIC_CPU=1 node pool the server + # gets exclusive pinned cores (CPU-binding experiment). + [ "${GUARANTEED:-0}" = "1" ] && server_manifest="gke/durable-streams-splitlane-guaranteed.yaml" + if [ -z "$extra_args" ]; then - envsubst "${MANIFEST_VARS} \${SERVER_CPU}" < gke/durable-streams.yaml | K apply -f - + envsubst "${MANIFEST_VARS} \${SERVER_CPU}" < "$server_manifest" | K apply -f - elif echo "$extra_args" | grep -q -- "--tier local"; then local tmp_tier @@ -95,7 +111,7 @@ deploy_server() { printf ' - "/data/cold"\n' >> "$tmp_tier" printf ' - "--tier-segment-bytes"\n' >> "$tmp_tier" printf ' - "1048576"\n' >> "$tmp_tier" - envsubst "${MANIFEST_VARS} \${SERVER_CPU}" < gke/durable-streams.yaml \ + envsubst "${MANIFEST_VARS} \${SERVER_CPU}" < "$server_manifest" \ | sed \ -e '/- "--tier"$/,/- "--tier-allow-http"$/d' \ -e "/- \"\/data\"/r ${tmp_tier}" \ @@ -108,7 +124,7 @@ deploy_server() { for flag in $extra_args; do printf ' - "%s"\n' "$flag" >> "$tmp_inject" done - envsubst "${MANIFEST_VARS} \${SERVER_CPU}" < gke/durable-streams.yaml \ + envsubst "${MANIFEST_VARS} \${SERVER_CPU}" < "$server_manifest" \ | sed "/--tier-allow-http/r ${tmp_inject}" \ | K apply -f - rm -f "$tmp_inject" diff --git a/suites/wal-batched-mw-local.json b/suites/wal-batched-mw-local.json new file mode 100644 index 0000000..4842d19 --- /dev/null +++ b/suites/wal-batched-mw-local.json @@ -0,0 +1,42 @@ +{ + "suite": "wal-batched-mw-local", + "_doc": "A/B for the WAL-machinery optimization --wal-batched-markwritten. Baseline takes the inner shard lock twice per append (reserve + mark_written); batched pushes the completed lsn to a side buffer and lets the committer fold the batch into the watermark in one inner acquisition, cutting appender inner traffic 2->1. Hypothesis: less inner-lock contention -> faster commit cycle -> lower durwait -> higher throughput, most visible at high inflight (50k streams, 6 pods x 128 conns). Both arms syncfs on. Capture SRV_STATS (durwait/stage) from the server pod. Local kind: durwait is dominated by slow Docker fsync, so if batched helps HERE it's via reduced contention; confirm the machinery win cleanly on remote NVMe after.", + "cluster": { + "server_cpus": 4 + }, + "saturation": { + "plateau_pct": 10, + "fleet_cpu": 0.15, + "connections": 128, + "repeats": 1, + "warmup_secs": 4, + "measure_secs": 10, + "setup_concurrency": 8, + "payload_bytes": 256 + }, + "modes": [ + "wal" + ], + "server_configs": { + "wal": [ + { + "label": "baseline", + "args": "--wal-shards 4 --worker-threads 4 --wal-checkpoint-syncfs on --wal-stats 2 --server-stats 2" + }, + { + "label": "batched-mw", + "args": "--wal-shards 4 --worker-threads 4 --wal-checkpoint-syncfs on --wal-batched-markwritten on --wal-stats 2 --server-stats 2" + } + ] + }, + "stream_counts": [ + 50000 + ], + "pod_ladder": { + "50000": [ + 2, + 4, + 6 + ] + } +} diff --git a/suites/wal-checkpoint-fix-100k.json b/suites/wal-checkpoint-fix-100k.json new file mode 100644 index 0000000..3ce1c16 --- /dev/null +++ b/suites/wal-checkpoint-fix-100k.json @@ -0,0 +1,48 @@ +{ + "suite": "wal-checkpoint-fix-100k", + "_doc": "MINIMAL decisive rerun of the checkpoint-fix A/B (the full wal-checkpoint-fix-ab was aborted before the fix arms reached 100k). Only the missing measurement: syncfs and both at 100k streams, ladder [2,4]. Known reference points from the aborted run: baseline@100k = 10.4k ops/s at p2 degrading to 7.3k at p8 (the cliff); baseline/stagger@10k ~107-111k (fixes don't regress below the cliff). Verdict rule: if syncfs/both peak >> 10.4k the fix works. Requires MULTILANE=1 SERVER_LOCAL_SSD_BLOCK=1 SPOT_SERVER=1. Image: europe-west1-docker.pkg.dev/vaxine/ds-bench/durable-streams:dev (digest 2ebed436* = perf/wal-syncfs @ 394375168, both flags compiled in, default off).", + "cluster": { + "server_machine": "c4d-standard-64-lssd", + "server_cpus": 8, + "client_machine": "n2d-standard-32", + "client_nodes": 3, + "cluster_name": "bench-multilane", + "zone": "europe-west4-b", + "region": "europe-west4" + }, + "saturation": { + "plateau_pct": 8, + "patience": 2, + "fleet_cpu": 2, + "connections": 256, + "repeats": 1, + "warmup_secs": 12, + "measure_secs": 20, + "setup_concurrency": 16, + "payload_bytes": 256 + }, + "modes": [ + "wal" + ], + "server_configs": { + "wal": [ + { + "label": "syncfs", + "args": "--wal-shards 6 --wal-checkpoint-syncfs on --worker-threads 8 --wal-stats 3 --server-stats 3" + }, + { + "label": "both", + "args": "--wal-shards 6 --wal-checkpoint-stagger on --wal-checkpoint-syncfs on --worker-threads 8 --wal-stats 3 --server-stats 3" + } + ] + }, + "stream_counts": [ + 100000 + ], + "pod_ladder": { + "100000": [ + 2, + 4 + ] + } +} diff --git a/suites/wal-checkpoint-fix-ab.json b/suites/wal-checkpoint-fix-ab.json new file mode 100644 index 0000000..08bc0f8 --- /dev/null +++ b/suites/wal-checkpoint-fix-ab.json @@ -0,0 +1,63 @@ +{ + "suite": "wal-checkpoint-fix-ab", + "_doc": "A/B of the cardinality-cliff checkpoint fixes on the 6-lane NVMe cluster (bench-multilane, one physical NVMe per shard, MULTILANE=1). All configs use --wal-shards 6 (6 lanes). Compares: baseline (synchronized checkpoint wave = current), stagger (--wal-checkpoint-stagger on: round-robin per-shard checkpoints so only one device storms at a time), syncfs (--wal-checkpoint-syncfs on: one syncfs barrier per shard-device instead of O(N) per-stream fdatasync), and both. The baseline arm is also the s6@100k point the shard sweep was heading toward. Hypothesis: baseline stays ~10-11k at 100k (all lanes stall together); stagger and/or syncfs recover throughput by de-conflicting or shrinking the checkpoint storm. Requires MULTILANE=1 SERVER_LOCAL_SSD_BLOCK=1 SPOT_SERVER=1.", + "cluster": { + "server_machine": "c4d-standard-64-lssd", + "server_cpus": 8, + "client_machine": "n2d-standard-32", + "client_nodes": 3, + "cluster_name": "bench-multilane", + "zone": "europe-west4-b", + "region": "europe-west4" + }, + "saturation": { + "plateau_pct": 8, + "patience": 2, + "fleet_cpu": 2, + "connections": 256, + "repeats": 1, + "warmup_secs": 12, + "measure_secs": 20, + "setup_concurrency": 16, + "payload_bytes": 256 + }, + "modes": [ + "wal" + ], + "server_configs": { + "wal": [ + { + "label": "baseline", + "args": "--wal-shards 6 --worker-threads 8 --wal-stats 3 --server-stats 3" + }, + { + "label": "stagger", + "args": "--wal-shards 6 --wal-checkpoint-stagger on --worker-threads 8 --wal-stats 3 --server-stats 3" + }, + { + "label": "syncfs", + "args": "--wal-shards 6 --wal-checkpoint-syncfs on --worker-threads 8 --wal-stats 3 --server-stats 3" + }, + { + "label": "both", + "args": "--wal-shards 6 --wal-checkpoint-stagger on --wal-checkpoint-syncfs on --worker-threads 8 --wal-stats 3 --server-stats 3" + } + ] + }, + "stream_counts": [ + 10000, + 100000 + ], + "pod_ladder": { + "10000": [ + 2, + 4, + 8 + ], + "100000": [ + 2, + 4, + 8 + ] + } +} diff --git a/suites/wal-cpubind.json b/suites/wal-cpubind.json new file mode 100644 index 0000000..4721860 --- /dev/null +++ b/suites/wal-cpubind.json @@ -0,0 +1,50 @@ +{ + "suite": "wal-cpubind", + "_doc": "CPU-binding experiment (task: do exclusive pinned cores help now that wal is no longer fsync-bound at 270k+ ops/s?). Own cluster bench-cpubind created with STATIC_CPU=1 (kubelet cpuManagerPolicy=static on the server pool) and deployed with GUARANTEED=1 (gke/durable-streams-splitlane-guaranteed.yaml: every container requests==limits, server cpu integer 8 -> EXCLUSIVE pinned cores). Same split-lane layout + image (:ckptoff) + args as wal-splitlane's ref-3s, so the control is wal-splitlane ref-3s = 286k @10k / 272k @100k on shared cores. Success = bound arm meaningfully above 286k/272k (>= +5%); parity means CPU binding is not a lever at this throughput and the task closes. VERIFY at deploy: kubectl exec ... cat /sys/fs/cgroup/cpuset.cpus.effective in the server container should show a dedicated core set, not all cores. Requires STATIC_CPU=1 SPLITLANE=1 GUARANTEED=1 SERVER_LOCAL_SSD_BLOCK=1 SPOT_SERVER=1 IMG_SERVER=...:ckptoff.", + "cluster": { + "server_machine": "c4d-standard-64-lssd", + "server_cpus": 8, + "client_machine": "n2d-standard-32", + "client_nodes": 3, + "cluster_name": "bench-cpubind", + "zone": "europe-west4-b", + "region": "europe-west4" + }, + "saturation": { + "plateau_pct": 8, + "patience": 2, + "fleet_cpu": 2, + "connections": 256, + "repeats": 1, + "warmup_secs": 12, + "measure_secs": 20, + "setup_concurrency": 16, + "payload_bytes": 256 + }, + "modes": [ + "wal" + ], + "server_configs": { + "wal": [ + { + "label": "bound-3s", + "args": "--wal-shards 5 --wal-checkpoint-syncfs on --worker-threads 8 --wal-stats 3 --server-stats 3 --data-dir /data/wal/0" + } + ] + }, + "stream_counts": [ + 10000, + 100000 + ], + "pod_ladder": { + "10000": [ + 4, + 8 + ], + "100000": [ + 2, + 4, + 8 + ] + } +} diff --git a/suites/wal-decomp-lane0.json b/suites/wal-decomp-lane0.json new file mode 100644 index 0000000..a96bc44 --- /dev/null +++ b/suites/wal-decomp-lane0.json @@ -0,0 +1,61 @@ +{ + "suite": "wal-decomp-lane0", + "_doc": "Cardinality-cliff DECOMPOSITION on true NVMe, killing a newly-found confound: stream data files live at /streams/ = the BASE /data volume, which on the raw-block multilane cluster is a plain emptyDir on the PD BOOT DISK — so every prior multilane run had the WAL commit path on NVMe but the checkpoint's O(N)-file writeback on a network PD. Here every arm appends '--data-dir /data/wal/0' (last --data-dir wins in main.rs arg parsing) so streams/ AND wal/ all sit on NVMe lane 0 (one fs; shard count is concurrency only, not lanes). Arms decompose the wal-vs-memory gap: memory (no wal, no fsync ever) | nofsync (full wal machinery, --unsafe-disable-fsync: no fsync at all) | ckpt-off (wal + commit fdatasync, checkpoint deferred 1h = off within a bench window) | ref-3s (normal syncfs @3s = PR #4697 config, re-measured on real NVMe). Verdicts: nofsync ~= memory => wal machinery is free, gap is all fsync. ckpt-off ~= nofsync => commit fdatasync is cheap (group-committed), gap is all checkpoint. ref-3s @100k >> the ~10-18k measured with streams-on-PD => the 'residual cliff' in PR #4697 was substantially the boot-disk artifact. Image :ckptoff is built from the perf/wal-lazy-checkpoint worktree (has --wal-checkpoint-interval-ms, --unsafe-disable-fsync, syncfs). Run with IMG_SERVER=europe-west1-docker.pkg.dev/vaxine/ds-bench/durable-streams:ckptoff MULTILANE=1 SERVER_LOCAL_SSD_BLOCK=1 SPOT_SERVER=1.", + "cluster": { + "server_machine": "c4d-standard-64-lssd", + "server_cpus": 8, + "client_machine": "n2d-standard-32", + "client_nodes": 3, + "cluster_name": "bench-multilane", + "zone": "europe-west4-b", + "region": "europe-west4" + }, + "saturation": { + "plateau_pct": 8, + "patience": 2, + "fleet_cpu": 2, + "connections": 256, + "repeats": 1, + "warmup_secs": 12, + "measure_secs": 20, + "setup_concurrency": 16, + "payload_bytes": 256 + }, + "modes": [ + "wal" + ], + "server_configs": { + "wal": [ + { + "label": "memory", + "args": "--durability memory --worker-threads 8 --server-stats 3 --data-dir /data/wal/0" + }, + { + "label": "nofsync", + "args": "--wal-shards 6 --unsafe-disable-fsync on --worker-threads 8 --wal-stats 3 --server-stats 3 --data-dir /data/wal/0" + }, + { + "label": "ckpt-off", + "args": "--wal-shards 6 --wal-checkpoint-syncfs on --wal-checkpoint-interval-ms 3600000 --worker-threads 8 --wal-stats 3 --server-stats 3 --data-dir /data/wal/0" + }, + { + "label": "ref-3s", + "args": "--wal-shards 6 --wal-checkpoint-syncfs on --worker-threads 8 --wal-stats 3 --server-stats 3 --data-dir /data/wal/0" + } + ] + }, + "stream_counts": [ + 10000, + 100000 + ], + "pod_ladder": { + "10000": [ + 4, + 8 + ], + "100000": [ + 2, + 4 + ] + } +} diff --git a/suites/wal-fanout-sweep.json b/suites/wal-fanout-sweep.json new file mode 100644 index 0000000..3fdd6a4 --- /dev/null +++ b/suites/wal-fanout-sweep.json @@ -0,0 +1,59 @@ +{ + "suite": "wal-fanout-sweep", + "_doc": "Controlled --wal-fsync-parallel (checkpoint per-stream fdatasync fan-out) ladder at the winning shard count (4), on the SAME NVMe cluster as wal-shard-sweep (cluster_name bench-walshard, idempotent reuse). The shard sweep showed throughput is device-flush-bound and shard-insensitive (~72-75k); the remaining question is whether parallelizing the checkpoint's O(N_touched) per-stream fdatasync storm (which competes with commit fsyncs for the device flush budget) helps or hurts. Prior data only had fanout 1 vs 16 (16 regressed); this sweeps 1/2/4/8 to properly characterize whether a SMALL fanout helps before device contention dominates. Same 200k streams / 8 vCPU pin / 256 conns so it's directly comparable to wal-s4.", + "cluster": { + "server_machine": "c4d-standard-16-lssd", + "server_cpus": 8, + "client_machine": "n2d-standard-32", + "client_nodes": 3, + "cluster_name": "bench-walshard", + "zone": "europe-west4-b", + "region": "europe-west4" + }, + "saturation": { + "plateau_pct": 8, + "patience": 2, + "fleet_cpu": 2, + "connections": 256, + "repeats": 1, + "warmup_secs": 12, + "measure_secs": 20, + "setup_concurrency": 16, + "payload_bytes": 256 + }, + "modes": [ + "wal" + ], + "server_configs": { + "wal": [ + { + "label": "s4-f1", + "args": "--wal-shards 4 --wal-fsync-parallel 1 --worker-threads 8 --wal-stats 3 --server-stats 3" + }, + { + "label": "s4-f2", + "args": "--wal-shards 4 --wal-fsync-parallel 2 --worker-threads 8 --wal-stats 3 --server-stats 3" + }, + { + "label": "s4-f4", + "args": "--wal-shards 4 --wal-fsync-parallel 4 --worker-threads 8 --wal-stats 3 --server-stats 3" + }, + { + "label": "s4-f8", + "args": "--wal-shards 4 --wal-fsync-parallel 8 --worker-threads 8 --wal-stats 3 --server-stats 3" + } + ] + }, + "stream_counts": [ + 200000 + ], + "pod_ladder": { + "200000": [ + 2, + 4, + 8, + 12, + 16 + ] + } +} diff --git a/suites/wal-fsync-diag-local.json b/suites/wal-fsync-diag-local.json new file mode 100644 index 0000000..788fad8 --- /dev/null +++ b/suites/wal-fsync-diag-local.json @@ -0,0 +1,51 @@ +{ + "suite": "wal-fsync-diag-local", + "_doc": "Decompose the WAL-path cardinality cliff: is it the fsync BARRIERS or the WAL write machinery? Three arms at the same cardinality: (1) wal-normal = syncfs+fsync on (the cliff), (2) wal-nofsync = --unsafe-disable-fsync on (keeps the full WAL write path — staging, register_dirty, group-commit, per-stream data-file writes — but no-ops every device barrier), (3) memory = no WAL at all. If wal-nofsync jumps to ~memory throughput and flattens, the cliff is the fsync barriers. If wal-nofsync stays cliffy like wal-normal, it's the WAL write machinery (register_dirty / staging / per-stream writes), not the barrier. SRV_STATS (--server-stats 2) now splits append time into applock_us / wrwire_us (per-stream data-file write) / durwait_us (fsync wait) — capture it from the server pod during the 50k cells to see exactly where the time goes. Local kind, relative signal.", + "cluster": { + "server_cpus": 4 + }, + "saturation": { + "plateau_pct": 10, + "fleet_cpu": 0.15, + "connections": 128, + "repeats": 1, + "warmup_secs": 4, + "measure_secs": 8, + "setup_concurrency": 8, + "payload_bytes": 256 + }, + "modes": [ + "wal" + ], + "server_configs": { + "wal": [ + { + "label": "wal-normal", + "args": "--wal-shards 4 --worker-threads 4 --wal-checkpoint-syncfs on --server-stats 2" + }, + { + "label": "wal-nofsync", + "args": "--wal-shards 4 --worker-threads 4 --wal-checkpoint-syncfs on --unsafe-disable-fsync on --server-stats 2" + }, + { + "label": "memory", + "args": "--durability memory --worker-threads 4 --server-stats 2" + } + ] + }, + "stream_counts": [ + 20000, + 50000 + ], + "pod_ladder": { + "20000": [ + 2, + 4 + ], + "50000": [ + 2, + 4, + 6 + ] + } +} diff --git a/suites/wal-lazy-checkpoint-local.json b/suites/wal-lazy-checkpoint-local.json new file mode 100644 index 0000000..a20a3f8 --- /dev/null +++ b/suites/wal-lazy-checkpoint-local.json @@ -0,0 +1,55 @@ +{ + "suite": "wal-lazy-checkpoint-local", + "_doc": "Local-kind value-finder for the lazy-checkpoint approach (cardinality cliff). The checkpoint's per-stream fsync/syncfs is not a correctness barrier (ack = WAL fdatasync; recovery replays the retained WAL) — it only bounds WAL size for recycling. So the checkpoint interval is a pure amortization knob: a LARGE interval fires the O(shards) syncfs barrier far less often and coalesces many appends to the same stream into one metadata/journal update, trading a larger retained WAL + longer crash-replay for cheaper steady-state writes. This sweeps the interval (--wal-checkpoint-interval-ms) with syncfs on, against two reference arms (baseline = per-stream fdatasync @3s; syncfs @3s = PR #4697). Local kind is a relative value-finder only (Docker VM fsync != NVMe); if larger intervals lift throughput here, re-tune the actual values on the remote NVMe cluster. Watch WAL_CKPT fsync_us + cadence via --wal-stats 2.", + "cluster": { + "server_cpus": 4 + }, + "saturation": { + "plateau_pct": 10, + "fleet_cpu": 0.15, + "connections": 128, + "repeats": 1, + "warmup_secs": 4, + "measure_secs": 8, + "setup_concurrency": 8, + "payload_bytes": 256 + }, + "modes": [ + "wal" + ], + "server_configs": { + "wal": [ + { + "label": "ref-baseline-3s", + "args": "--wal-shards 4 --worker-threads 4 --wal-stats 2" + }, + { + "label": "ref-syncfs-3s", + "args": "--wal-shards 4 --worker-threads 4 --wal-checkpoint-syncfs on --wal-stats 2" + }, + { + "label": "lazy-syncfs-30s", + "args": "--wal-shards 4 --worker-threads 4 --wal-checkpoint-syncfs on --wal-checkpoint-interval-ms 30000 --wal-stats 2" + }, + { + "label": "lazy-syncfs-60s", + "args": "--wal-shards 4 --worker-threads 4 --wal-checkpoint-syncfs on --wal-checkpoint-interval-ms 60000 --wal-stats 2" + } + ] + }, + "stream_counts": [ + 20000, + 50000 + ], + "pod_ladder": { + "20000": [ + 2, + 4 + ], + "50000": [ + 2, + 4, + 6 + ] + } +} diff --git a/suites/wal-lazy-checkpoint-multilane.json b/suites/wal-lazy-checkpoint-multilane.json new file mode 100644 index 0000000..6e51d67 --- /dev/null +++ b/suites/wal-lazy-checkpoint-multilane.json @@ -0,0 +1,63 @@ +{ + "suite": "wal-lazy-checkpoint-multilane", + "_doc": "Remote NVMe tuning of the lazy-checkpoint interval (cardinality cliff). Runs on the 6-lane c4d-standard-64-lssd multilane cluster (one physical NVMe per WAL shard, MULTILANE=1 SERVER_LOCAL_SSD_BLOCK=1 SPOT_SERVER=1). All arms use --wal-shards 6 + --wal-checkpoint-syncfs on; the ONLY variable is --wal-checkpoint-interval-ms. ref-syncfs-3s is the reference (PR #4697 cadence on the SAME binary); 30s/60s/120s sweep the amortization knob. Local kind showed monotonic gains 3s->30s->60s at 50k with 60s still climbing, so this pushes to 120s to find the knee. Success = higher throughput at 100k as the interval grows; the winning value + the WAL_CKPT fsync cadence/cost (--wal-stats 3) sets the production default. Watch retained-WAL growth (larger interval = larger WAL + longer crash-replay) as the ceiling.", + "cluster": { + "server_machine": "c4d-standard-64-lssd", + "server_cpus": 8, + "client_machine": "n2d-standard-32", + "client_nodes": 3, + "cluster_name": "bench-multilane", + "zone": "europe-west4-b", + "region": "europe-west4" + }, + "saturation": { + "plateau_pct": 8, + "patience": 2, + "fleet_cpu": 2, + "connections": 256, + "repeats": 1, + "warmup_secs": 12, + "measure_secs": 20, + "setup_concurrency": 16, + "payload_bytes": 256 + }, + "modes": [ + "wal" + ], + "server_configs": { + "wal": [ + { + "label": "ref-syncfs-3s", + "args": "--wal-shards 6 --wal-checkpoint-syncfs on --worker-threads 8 --wal-stats 3 --server-stats 3" + }, + { + "label": "lazy-30s", + "args": "--wal-shards 6 --wal-checkpoint-syncfs on --wal-checkpoint-interval-ms 30000 --worker-threads 8 --wal-stats 3 --server-stats 3" + }, + { + "label": "lazy-60s", + "args": "--wal-shards 6 --wal-checkpoint-syncfs on --wal-checkpoint-interval-ms 60000 --worker-threads 8 --wal-stats 3 --server-stats 3" + }, + { + "label": "lazy-120s", + "args": "--wal-shards 6 --wal-checkpoint-syncfs on --wal-checkpoint-interval-ms 120000 --worker-threads 8 --wal-stats 3 --server-stats 3" + } + ] + }, + "stream_counts": [ + 10000, + 100000 + ], + "pod_ladder": { + "10000": [ + 2, + 4, + 8 + ], + "100000": [ + 2, + 4, + 8 + ] + } +} diff --git a/suites/wal-machinery-baseline-local.json b/suites/wal-machinery-baseline-local.json new file mode 100644 index 0000000..e6c5810 --- /dev/null +++ b/suites/wal-machinery-baseline-local.json @@ -0,0 +1,38 @@ +{ + "suite": "wal-machinery-baseline-local", + "_doc": "Baseline for WAL-machinery optimization #1 (lock-free LSN/offset reservation). Runs wal with --wal-stats 2 so WAL_STATS exposes inner_lock_wait — the per-shard write-serialization signal (every stream on a shard funnels through the inner mutex twice per append: reserve + mark_written). If inner_lock_wait is a large fraction of the append time at 50k, the shard lock is the machinery tax and #1 is the right fix. Also captures SRV_STATS (svc/applock/wrwire/durwait). Local kind, relative. 50k only to be fast.", + "cluster": { + "server_cpus": 4 + }, + "saturation": { + "plateau_pct": 10, + "fleet_cpu": 0.15, + "connections": 128, + "repeats": 1, + "warmup_secs": 4, + "measure_secs": 10, + "setup_concurrency": 8, + "payload_bytes": 256 + }, + "modes": [ + "wal" + ], + "server_configs": { + "wal": [ + { + "label": "wal-baseline", + "args": "--wal-shards 4 --worker-threads 4 --wal-checkpoint-syncfs on --wal-stats 2 --server-stats 2" + } + ] + }, + "stream_counts": [ + 50000 + ], + "pod_ladder": { + "50000": [ + 2, + 4, + 6 + ] + } +} diff --git a/suites/wal-mem-diag-local.json b/suites/wal-mem-diag-local.json new file mode 100644 index 0000000..bd93cbb --- /dev/null +++ b/suites/wal-mem-diag-local.json @@ -0,0 +1,47 @@ +{ + "suite": "wal-mem-diag-local", + "_doc": "Diagnostic: does the cardinality cliff exist in MEMORY durability mode too? Memory mode (--durability memory) skips the WAL + all per-append fsync, but STILL writes each append into the per-stream data file (page cache) for reads. So if memory mode ALSO cliffs at high cardinality, the cause is the per-stream data files / N-inode writeback, not the WAL/fsync path — which points at the log-structured store. If memory mode is FLAT while wal cliffs, the cause is WAL durability. Local kind, relative signal. wal arm uses syncfs (best current wal). Capture SRV_STATS (svc_us/applock_us/durwait_us) from the server pod during the 50k cells for the time breakdown.", + "cluster": { + "server_cpus": 4 + }, + "saturation": { + "plateau_pct": 10, + "fleet_cpu": 0.15, + "connections": 128, + "repeats": 1, + "warmup_secs": 4, + "measure_secs": 8, + "setup_concurrency": 8, + "payload_bytes": 256 + }, + "modes": [ + "wal" + ], + "server_configs": { + "wal": [ + { + "label": "wal-syncfs", + "args": "--wal-shards 4 --worker-threads 4 --wal-checkpoint-syncfs on --wal-stats 2 --server-stats 2" + }, + { + "label": "memory", + "args": "--durability memory --worker-threads 4 --server-stats 2" + } + ] + }, + "stream_counts": [ + 20000, + 50000 + ], + "pod_ladder": { + "20000": [ + 2, + 4 + ], + "50000": [ + 2, + 4, + 6 + ] + } +} diff --git a/suites/wal-multilane-shards.json b/suites/wal-multilane-shards.json new file mode 100644 index 0000000..8b1e23f --- /dev/null +++ b/suites/wal-multilane-shards.json @@ -0,0 +1,79 @@ +{ + "suite": "wal-multilane-shards", + "_doc": "MULTI-LANE re-test of WAL shard scaling. On the single-device c4d-standard-16-lssd (and the RAID0-striped -lssd ephemeral-storage layout) --wal-shards did NOT scale write throughput: every shard's fdatasync serialised behind ONE filesystem = ONE fsync barrier (a single-lane artifact, not a real ceiling). This suite instead runs on c4d-standard-64-lssd, whose 6 physically-attached Titanium NVMe devices are attached as RAW BLOCK and mkfs+mounted ONE device per WAL shard dir, so each shard's fdatasync hits an independent device queue (an independent fsync lane). It sweeps shards 1/2/4/6 (capped at the 6-device count — s7+ would spill onto the base volume and defeat the test) with the server pinned to 8 vCPU headroom (prior telemetry: wal is fsync-bound + CPU-idle) and worker-threads fixed at 8. Each config emits SRV_STATS (cpu_cores/inflight/durwait) + WAL_CKPT so fsync-lane saturation is visible directly. If throughput now rises ~linearly s1->s6, the earlier flatness was the single-lane disk, not the WAL design. REQUIRES these env vars when running (they are NOT encoded in this JSON): MULTILANE=1 (apply gke/durable-streams-multilane.yaml = one device per shard dir), SERVER_LOCAL_SSD_BLOCK=1 (create the server pool with raw-block --local-nvme-ssd-block instead of RAID0 ephemeral-storage), SPOT_SERVER=1 (cheap; a preemption just re-runs the affected cells). See MULTILANE_SETUP.md for the provisioning command + the VERIFY-AT-PROVISION-TIME checklist.", + "cluster": { + "server_machine": "c4d-standard-64-lssd", + "server_cpus": 8, + "client_machine": "n2d-standard-32", + "client_nodes": 3, + "cluster_name": "bench-multilane", + "zone": "europe-west4-b", + "region": "europe-west4" + }, + "saturation": { + "plateau_pct": 8, + "patience": 2, + "fleet_cpu": 2, + "connections": 256, + "repeats": 1, + "warmup_secs": 12, + "measure_secs": 20, + "setup_concurrency": 16, + "payload_bytes": 256 + }, + "modes": [ + "wal" + ], + "server_configs": { + "wal": [ + { + "label": "s1", + "args": "--wal-shards 1 --worker-threads 8 --wal-stats 3 --server-stats 3" + }, + { + "label": "s2", + "args": "--wal-shards 2 --worker-threads 8 --wal-stats 3 --server-stats 3" + }, + { + "label": "s4", + "args": "--wal-shards 4 --worker-threads 8 --wal-stats 3 --server-stats 3" + }, + { + "label": "s6", + "args": "--wal-shards 6 --worker-threads 8 --wal-stats 3 --server-stats 3" + } + ] + }, + "stream_counts": [ + 100, + 1000, + 10000, + 100000 + ], + "pod_ladder": { + "100": [ + 2, + 4, + 8, + 12 + ], + "1000": [ + 2, + 4, + 8, + 12 + ], + "10000": [ + 2, + 4, + 8, + 12 + ], + "100000": [ + 2, + 4, + 8, + 12 + ] + } +} diff --git a/suites/wal-shard-sweep.json b/suites/wal-shard-sweep.json new file mode 100644 index 0000000..cb15a40 --- /dev/null +++ b/suites/wal-shard-sweep.json @@ -0,0 +1,64 @@ +{ + "suite": "wal-shard-sweep", + "_doc": "Controlled --wal-shards ladder to find the optimal wal config on real NVMe (c4d-standard-16-lssd, Titanium local SSD). Isolates shard count as the ONLY variable: server pinned to 8 vCPU (headroom so CPU is never the limit — prior telemetry showed wal at ~1.3 busy cores, fsync-bound + CPU-idle) and worker-threads fixed at 8. Sweeps shards 1/4/8/16/24 at 200k streams (matches the documented s16 reference point). Each config emits SRV_STATS (cpu_cores/inflight/durwait) + WAL_CKPT so we can see fsync-lane saturation directly. The pod ladder over-provisions offered load so even the highest-ceiling shard config saturates; the barrier-aligned walker auto-stops each config at its plateau (plateau_pct 8, patience 2). Resumable per cell (cells.json), so an interrupted run resumes without recompute.", + "cluster": { + "server_machine": "c4d-standard-16-lssd", + "server_cpus": 8, + "client_machine": "n2d-standard-32", + "client_nodes": 3, + "cluster_name": "bench-walshard", + "zone": "europe-west4-b", + "region": "europe-west4" + }, + "saturation": { + "plateau_pct": 8, + "patience": 2, + "fleet_cpu": 2, + "connections": 256, + "repeats": 1, + "warmup_secs": 12, + "measure_secs": 20, + "setup_concurrency": 16, + "payload_bytes": 256 + }, + "modes": [ + "wal" + ], + "server_configs": { + "wal": [ + { + "label": "wal-s1", + "args": "--wal-shards 1 --worker-threads 8 --wal-stats 3 --server-stats 3" + }, + { + "label": "wal-s4", + "args": "--wal-shards 4 --worker-threads 8 --wal-stats 3 --server-stats 3" + }, + { + "label": "wal-s8", + "args": "--wal-shards 8 --worker-threads 8 --wal-stats 3 --server-stats 3" + }, + { + "label": "wal-s16", + "args": "--wal-shards 16 --worker-threads 8 --wal-stats 3 --server-stats 3" + }, + { + "label": "wal-s24", + "args": "--wal-shards 24 --worker-threads 8 --wal-stats 3 --server-stats 3" + } + ] + }, + "stream_counts": [ + 200000 + ], + "pod_ladder": { + "200000": [ + 1, + 2, + 4, + 8, + 12, + 16 + ] + } +} diff --git a/suites/wal-sizetrigger.json b/suites/wal-sizetrigger.json new file mode 100644 index 0000000..e5f1f82 --- /dev/null +++ b/suites/wal-sizetrigger.json @@ -0,0 +1,54 @@ +{ + "suite": "wal-sizetrigger", + "_doc": "Validate the per-shard checkpoint size-trigger (branch perf/wal-checkpoint-sizetrigger, image :sizetrigger) on the split-lane layout. wal-splitlane measured checkpoint @3s costing 7-11% vs checkpoint-off (ref-3s 286k/272k vs ckpt-off 305k/306k). Hypothesis: --wal-checkpoint-wal-bytes 1GiB (+60s interval fallback) amortizes the checkpoint into a rare event and recovers most of that gap, with bounded replay (<=1GiB retained WAL per shard). Arms share the SAME image; ref-3s is the control at defaults (3s timer). Success = size-1g within ~2% of the 306k ckpt-off ceiling at both stream counts. Requires SPLITLANE=1 SERVER_LOCAL_SSD_BLOCK=1 SPOT_SERVER=1 IMG_SERVER=...:sizetrigger.", + "cluster": { + "server_machine": "c4d-standard-64-lssd", + "server_cpus": 8, + "client_machine": "n2d-standard-32", + "client_nodes": 3, + "cluster_name": "bench-multilane", + "zone": "europe-west4-b", + "region": "europe-west4" + }, + "saturation": { + "plateau_pct": 8, + "patience": 2, + "fleet_cpu": 2, + "connections": 256, + "repeats": 1, + "warmup_secs": 12, + "measure_secs": 20, + "setup_concurrency": 16, + "payload_bytes": 256 + }, + "modes": [ + "wal" + ], + "server_configs": { + "wal": [ + { + "label": "ref-3s", + "args": "--wal-shards 5 --wal-checkpoint-syncfs on --worker-threads 8 --wal-stats 3 --server-stats 3 --data-dir /data/wal/0" + }, + { + "label": "size-1g", + "args": "--wal-shards 5 --wal-checkpoint-syncfs on --wal-checkpoint-wal-bytes 1073741824 --wal-checkpoint-interval-ms 60000 --worker-threads 8 --wal-stats 3 --server-stats 3 --data-dir /data/wal/0" + } + ] + }, + "stream_counts": [ + 10000, + 100000 + ], + "pod_ladder": { + "10000": [ + 4, + 8 + ], + "100000": [ + 2, + 4, + 8 + ] + } +} diff --git a/suites/wal-splitlane.json b/suites/wal-splitlane.json new file mode 100644 index 0000000..3e027b2 --- /dev/null +++ b/suites/wal-splitlane.json @@ -0,0 +1,58 @@ +{ + "suite": "wal-splitlane", + "_doc": "SPLIT-LANE validation of the storage-layout hypothesis from wal-decomp-lane0: with WAL shards on dedicated NVMe lanes AND stream files on their own separate NVMe lane, wal+syncfs-checkpoint should reach ~250k ops/s FLAT across 10k->100k streams (ref-3s hit 270k @10k when WAL had dedicated lanes, ~= the 260k nofsync machinery ceiling; the old 8-18k @100k collapse was streams sitting on the PD boot disk). Layout via gke/durable-streams-splitlane.yaml (SPLITLANE=1): device 0 = /data/wal/0 (data root: streams/, the checkpoint syncfs domain), devices 1-5 = /data/wal/0/wal/<0..4> (5 WAL shard lanes). All arms: --data-dir /data/wal/0 --wal-shards 5. Arms: ref-3s (syncfs @3s = production candidate), ckpt-off (checkpoint deferred 1h — isolates commit-fsync cost on this layout), nofsync (machinery ceiling control). Success = ref-3s ~= ckpt-off ~= nofsync at BOTH stream counts, proving checkpoint + commit fsync are both ~free on the right layout. Requires SPLITLANE=1 SERVER_LOCAL_SSD_BLOCK=1 SPOT_SERVER=1 IMG_SERVER=...:ckptoff (image has --wal-checkpoint-interval-ms + --unsafe-disable-fsync).", + "cluster": { + "server_machine": "c4d-standard-64-lssd", + "server_cpus": 8, + "client_machine": "n2d-standard-32", + "client_nodes": 3, + "cluster_name": "bench-multilane", + "zone": "europe-west4-b", + "region": "europe-west4" + }, + "saturation": { + "plateau_pct": 8, + "patience": 2, + "fleet_cpu": 2, + "connections": 256, + "repeats": 1, + "warmup_secs": 12, + "measure_secs": 20, + "setup_concurrency": 16, + "payload_bytes": 256 + }, + "modes": [ + "wal" + ], + "server_configs": { + "wal": [ + { + "label": "ref-3s", + "args": "--wal-shards 5 --wal-checkpoint-syncfs on --worker-threads 8 --wal-stats 3 --server-stats 3 --data-dir /data/wal/0" + }, + { + "label": "ckpt-off", + "args": "--wal-shards 5 --wal-checkpoint-syncfs on --wal-checkpoint-interval-ms 3600000 --worker-threads 8 --wal-stats 3 --server-stats 3 --data-dir /data/wal/0" + }, + { + "label": "nofsync", + "args": "--wal-shards 5 --unsafe-disable-fsync on --worker-threads 8 --wal-stats 3 --server-stats 3 --data-dir /data/wal/0" + } + ] + }, + "stream_counts": [ + 10000, + 100000 + ], + "pod_ladder": { + "10000": [ + 4, + 8 + ], + "100000": [ + 2, + 4, + 8 + ] + } +} diff --git a/suites/wal-syncfs-local.json b/suites/wal-syncfs-local.json new file mode 100644 index 0000000..4691a28 --- /dev/null +++ b/suites/wal-syncfs-local.json @@ -0,0 +1,47 @@ +{ + "suite": "wal-syncfs-local", + "_doc": "Local-kind A/B for cardinality-cliff #1: checkpoint durability via one syncfs() barrier (--wal-checkpoint-syncfs on) vs the per-stream fdatasync loop (baseline). Same cluster, two labeled wal configs, high-enough cardinality (20k/50k) that the checkpoint's O(N_touched) fdatasync phase is a real cost. Compare throughput + WAL_CKPT fsync_us (--wal-stats 2) between labels. This is the local gate: if syncfs is faster (or clearly lower checkpoint fsync cost) here, promote to a remote NVMe run.", + "cluster": { + "server_cpus": 4 + }, + "saturation": { + "plateau_pct": 10, + "fleet_cpu": 0.15, + "connections": 128, + "repeats": 1, + "warmup_secs": 4, + "measure_secs": 8, + "setup_concurrency": 8, + "payload_bytes": 256 + }, + "modes": [ + "wal" + ], + "server_configs": { + "wal": [ + { + "label": "baseline", + "args": "--wal-shards 4 --worker-threads 4 --wal-stats 2" + }, + { + "label": "syncfs", + "args": "--wal-shards 4 --worker-threads 4 --wal-checkpoint-syncfs on --wal-stats 2" + } + ] + }, + "stream_counts": [ + 20000, + 50000 + ], + "pod_ladder": { + "20000": [ + 2, + 4 + ], + "50000": [ + 2, + 4, + 6 + ] + } +} diff --git a/suites/wal-syncfs-nvme.json b/suites/wal-syncfs-nvme.json new file mode 100644 index 0000000..c3c6531 --- /dev/null +++ b/suites/wal-syncfs-nvme.json @@ -0,0 +1,57 @@ +{ + "suite": "wal-syncfs-nvme", + "_doc": "Remote NVMe A/B for cardinality-cliff #1: checkpoint durability via one syncfs() barrier (--wal-checkpoint-syncfs on) vs the per-stream fdatasync loop (baseline), on Titanium NVMe (c4d-standard-16-lssd, 8 vCPU). This is the environment where the optimization's premise holds: per-file fdatasync is an expensive ~1ms hardware barrier, so the checkpoint's O(N_touched) storm (measured ~1.4s/shard at 200k) steals device budget from commits. syncfs collapses that to ONE barrier. Tests at 200k AND 500k streams (the storm is worst at 500k). Same shards(4)/cpu(8)/conns(256) as the wal-shard-sweep baseline so numbers are comparable. If syncfs raises throughput here, merge the flag.", + "cluster": { + "server_machine": "c4d-standard-16-lssd", + "server_cpus": 8, + "client_machine": "n2d-standard-32", + "client_nodes": 3, + "cluster_name": "bench-syncfs", + "zone": "europe-west4-b", + "region": "europe-west4" + }, + "saturation": { + "plateau_pct": 8, + "patience": 2, + "fleet_cpu": 2, + "connections": 256, + "repeats": 1, + "warmup_secs": 12, + "measure_secs": 20, + "setup_concurrency": 16, + "payload_bytes": 256 + }, + "modes": [ + "wal" + ], + "server_configs": { + "wal": [ + { + "label": "baseline", + "args": "--wal-shards 4 --worker-threads 8 --wal-stats 3 --server-stats 3" + }, + { + "label": "syncfs", + "args": "--wal-shards 4 --worker-threads 8 --wal-checkpoint-syncfs on --wal-stats 3 --server-stats 3" + } + ] + }, + "stream_counts": [ + 200000, + 500000 + ], + "pod_ladder": { + "200000": [ + 2, + 4, + 8, + 12 + ], + "500000": [ + 2, + 4, + 8, + 12 + ] + } +} diff --git a/suites/write-cliff-nvme.json b/suites/write-cliff-nvme.json new file mode 100644 index 0000000..0b58b0c --- /dev/null +++ b/suites/write-cliff-nvme.json @@ -0,0 +1,55 @@ +{ + "suite": "write-cliff-nvme", + "_doc": "Remote real-NVMe validation of the cardinality-cliff work on c4d-standard-16-lssd (physically-attached Titanium NVMe local SSD), server pinned to 4 vCPU / 4 shards (production write shape, AGENTS.md §3). Four configs, same image, same-binary A/Bs: (1) memory-gated vs memory-nogate validates the #1 memory sidecar fix on real NVMe; (2) wal-fsync1 vs wal-fsync16 tests whether parallelizing the checkpoint fdatasync scales on real NVMe (it regressed on 2-vCPU/virtiofs locally). Pool client: disjoint per-pod slices, plain appends, pre-created before the fleet barrier. connections=256 @ fleet_cpu=2 is the ~80% single-pod calibration (AGENTS.md §7).", + "cluster": { + "server_machine": "c4d-standard-16-lssd", + "server_cpus": 4, + "client_machine": "n2d-standard-32", + "client_nodes": 5, + "cluster_name": "bench-cliff-nvme", + "zone": "europe-west4-a", + "region": "europe-west4" + }, + "saturation": { + "plateau_pct": 8, + "patience": 2, + "fleet_cpu": 2, + "connections": 256, + "repeats": 1, + "warmup_secs": 15, + "measure_secs": 25, + "setup_concurrency": 16, + "payload_bytes": 256 + }, + "modes": [ + "wal" + ], + "server_configs": { + "wal": [ + { + "label": "memory-gated", + "args": "--durability memory --worker-threads 4" + }, + { + "label": "memory-nogate", + "args": "--durability memory --worker-threads 4 --mem-meta-gate off" + }, + { + "label": "wal-fsync1", + "args": "--wal-shards 4 --worker-threads 4 --wal-fsync-parallel 1 --wal-stats 3" + }, + { + "label": "wal-fsync16", + "args": "--wal-shards 4 --worker-threads 4 --wal-fsync-parallel 16 --wal-stats 3" + } + ] + }, + "stream_counts": [ + 100000, + 500000 + ], + "pod_ladder": { + "100000": [1, 2, 4, 8, 16, 32, 48, 64], + "500000": [1, 2, 4, 8, 16, 32, 48, 64] + } +} diff --git a/suites/write-nvme-bottleneck.json b/suites/write-nvme-bottleneck.json new file mode 100644 index 0000000..6b6dd18 --- /dev/null +++ b/suites/write-nvme-bottleneck.json @@ -0,0 +1,60 @@ +{ + "suite": "write-nvme-bottleneck", + "_doc": "Real-NVMe bottleneck confirmation + wal shard-scaling on c4d-standard-16-lssd (Titanium NVMe), 4 vCPU pin. Uses --server-stats (SRV_STATS: cpu_cores/inflight/durwait) to confirm on NVMe what the local Docker probe showed: memory is CPU-bound, wal is fsync-bound (CPU-idle). Sweeps wal shards (4/8/16) to test whether wal throughput scales with parallel fsync lanes (the documented lever) rather than CPU. 500k streams (heaviest checkpoint/fsync pressure); shorter ladder since wal plateaus early on fsync and memory saturates cores by ~a few pods.", + "cluster": { + "server_machine": "c4d-standard-16-lssd", + "server_cpus": 4, + "client_machine": "n2d-standard-32", + "client_nodes": 3, + "cluster_name": "bench-nvme-bneck", + "zone": "europe-west4-b", + "region": "europe-west4" + }, + "saturation": { + "plateau_pct": 8, + "patience": 2, + "fleet_cpu": 2, + "connections": 256, + "repeats": 1, + "warmup_secs": 15, + "measure_secs": 25, + "setup_concurrency": 16, + "payload_bytes": 256 + }, + "modes": [ + "wal" + ], + "server_configs": { + "wal": [ + { + "label": "memory", + "args": "--durability memory --worker-threads 4 --server-stats 3" + }, + { + "label": "wal-s4", + "args": "--wal-shards 4 --worker-threads 4 --wal-stats 3 --server-stats 3" + }, + { + "label": "wal-s8", + "args": "--wal-shards 8 --worker-threads 4 --wal-stats 3 --server-stats 3" + }, + { + "label": "wal-s16", + "args": "--wal-shards 16 --worker-threads 4 --wal-stats 3 --server-stats 3" + } + ] + }, + "stream_counts": [ + 500000 + ], + "pod_ladder": { + "500000": [ + 1, + 2, + 4, + 8, + 16, + 24 + ] + } +} \ No newline at end of file From 98ab4edc01cd000ff45bffde167ed61bc346536c Mon Sep 17 00:00:00 2001 From: Valter Balegas Date: Mon, 13 Jul 2026 20:08:17 +0100 Subject: [PATCH 2/8] bench: wal-stacked-1m results + cluster-up hardening (mktemp trailing-X, fail-hard create) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Stacked ideal config (split-lane + syncfs + 1GiB size-trigger + pinned cores): 383k @100k (37x campaign total), 244k @500k, 56k @1M — a NEW mechanism appears near 1M streams (fd count / dir scale / working set), next investigation. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01Y3x7bcT9vLGiT4tXZeQpnk --- gke/bench-job.yaml | 1 + results/wal-stacked-1m/aggregate.csv | 4 ++ results/wal-stacked-1m/aggregate.json | 83 +++++++++++++++++++++++ results/wal-stacked-1m/report.md | 47 +++++++++++++ results/wal-stacked-1m/stacked/cells.json | 73 ++++++++++++++++++++ scripts/cluster-up.sh | 11 ++- suites/wal-stacked-1m.json | 52 ++++++++++++++ 7 files changed, 269 insertions(+), 2 deletions(-) create mode 100644 results/wal-stacked-1m/aggregate.csv create mode 100644 results/wal-stacked-1m/aggregate.json create mode 100644 results/wal-stacked-1m/report.md create mode 100644 results/wal-stacked-1m/stacked/cells.json create mode 100644 suites/wal-stacked-1m.json diff --git a/gke/bench-job.yaml b/gke/bench-job.yaml index de42ed7..3857416 100644 --- a/gke/bench-job.yaml +++ b/gke/bench-job.yaml @@ -59,6 +59,7 @@ spec: DS_BENCH_INSTANCE="$JOB_COMPLETION_INDEX" DS_BENCH_SHARDS="${PARALLELISM}" DS_BENCH_BARRIER_DIR="${BARRIER_DIR}" + DS_BENCH_BARRIER_TIMEOUT_SECS=3600 ds-bench ${BENCH_CMD} > /out/${OUT_PREFIX}-${JOB_COMPLETION_INDEX}.json && up=0; diff --git a/results/wal-stacked-1m/aggregate.csv b/results/wal-stacked-1m/aggregate.csv new file mode 100644 index 0000000..08a6660 --- /dev/null +++ b/results/wal-stacked-1m/aggregate.csv @@ -0,0 +1,4 @@ +mode,stream_count,pods,throughput,p50,p99,knee_pods,knee_throughput,knee_p50,knee_p99,pod_mem_mb,pod_mem_p50_mb,saturated,status,reason +stacked,100000,8,382727.89999999997,,,4,382727.89999999997,2.537,4.539,774,668,False,ok,ladder_exhausted +stacked,500000,8,244203.8,,,8,244203.8,5.763,26.543,2925,2551,False,ok,ladder_exhausted +stacked,1000000,8,56003.35,,,8,56003.35,13.919,1186.815,4939,3852,False,ok,ladder_exhausted diff --git a/results/wal-stacked-1m/aggregate.json b/results/wal-stacked-1m/aggregate.json new file mode 100644 index 0000000..40ad068 --- /dev/null +++ b/results/wal-stacked-1m/aggregate.json @@ -0,0 +1,83 @@ +[ + { + "mode": "stacked", + "stream_count": 100000, + "pods": 8, + "throughput": 382727.89999999997, + "p50": null, + "p99": null, + "knee_pods": 4, + "knee_throughput": 382727.89999999997, + "knee_p50": 2.537, + "knee_p99": 4.539, + "pod_mem_mb": 774, + "pod_mem_p50_mb": 668, + "saturated": false, + "status": "ok", + "reason": "ladder_exhausted", + "walk": [ + [ + 4, + 382727.89999999997, + 2.537, + 4.539 + ], + [ + 8, + 362346.35000000003, + 5.439, + 8.751 + ] + ] + }, + { + "mode": "stacked", + "stream_count": 500000, + "pods": 8, + "throughput": 244203.8, + "p50": null, + "p99": null, + "knee_pods": 8, + "knee_throughput": 244203.8, + "knee_p50": 5.763, + "knee_p99": 26.543, + "pod_mem_mb": 2925, + "pod_mem_p50_mb": 2551, + "saturated": false, + "status": "ok", + "reason": "ladder_exhausted", + "walk": [ + [ + 8, + 244203.8, + 5.763, + 26.543 + ] + ] + }, + { + "mode": "stacked", + "stream_count": 1000000, + "pods": 8, + "throughput": 56003.35, + "p50": null, + "p99": null, + "knee_pods": 8, + "knee_throughput": 56003.35, + "knee_p50": 13.919, + "knee_p99": 1186.815, + "pod_mem_mb": 4939, + "pod_mem_p50_mb": 3852, + "saturated": false, + "status": "ok", + "reason": "ladder_exhausted", + "walk": [ + [ + 8, + 56003.35, + 13.919, + 1186.815 + ] + ] + } +] \ No newline at end of file diff --git a/results/wal-stacked-1m/report.md b/results/wal-stacked-1m/report.md new file mode 100644 index 0000000..141b811 --- /dev/null +++ b/results/wal-stacked-1m/report.md @@ -0,0 +1,47 @@ +# wal-stacked-1m — write-throughput report + +## Throughput at saturation (ops/s) + +| streams | stacked | +|---|---| +| 100000 | 383k† | +| 500000 | 244k† | +| 1000000 | 56k† | + +† = not saturated (ladder exhausted) — treat as a lower bound. + +## Pod memory at saturation — peak / p50 (MiB) + +| streams | stacked | +|---|---| +| 100000 | 774 / 668 | +| 500000 | 2925 / 2551 | +| 1000000 | 4939 / 3852 | + +_Pod working set = cgroup `memory.current − inactive_file` (anon + active page cache), sampled each second at the pinned rung. **peak** = high-water (catches bursts like an in-RAM Raft log filling); **p50** = median (what the server steadily holds resident). peak ≈ p50 ⇒ steadily resident; peak ≫ p50 ⇒ transient spikes._ + +## Latency (ms, p50 / p99) + +| streams | stacked @≤80% load | stacked @saturation | +|---|---|---| +| 100000 | 2.5 / 4.5 (383k @4p) | — | +| 500000 | 5.8 / 26.5 (244k @8p) | — | +| 1000000 | 13.9 / 1186.8 (56k @8p) | — | + +_@≤80% load = the largest ladder rung at ≤80% of peak throughput — the server's service latency with headroom. @saturation = the pinned plateau rung, where a closed-loop fleet measures its own queueing (≈ in-flight ÷ ceiling by Little's law), NOT the server's per-request cost. Compare against the unloaded single-request baseline (~1 ms for wal) before reading anything into large saturation values._ + +## Saturation walks (pods → ops/s, p50 ms) + +- **stacked 100000**: 4:383k@2.5ms → 8:362k@5.4ms (pinned 8, ladder_exhausted) +- **stacked 500000**: 8:244k@5.8ms (pinned 8, ladder_exhausted) +- **stacked 1000000**: 8:56k@13.9ms (pinned 8, ladder_exhausted) + +## Findings + +**Stacked ideal config validated**: 382.7k ops/s @100k streams (p4; 362.3k pinned @p8) — matches the ~360k projection from the separately-measured size-trigger (+10%) and CPU-binding (+21-24%) effects. Campaign total: 10.4k -> 383k = 37x, cardinality-flat through 100k. + +**A NEW wall near 1M streams**: 500k holds 244k (-36%), 1M collapses to 56k (-85%). This is not the old checkpoint storm (fixed, flat to 100k). Candidate mechanisms for the next investigation: 1M open fds (one per live stream ~= container nofile ceiling), ext4 directory with 1M files, stream-map/tails/meta working set, page-cache pressure from 1M dirty files. Needs a dedicated profiling pass with SRV_STATS/WAL_CKPT telemetry review at 500k/1M. + +## Caveats + +_Single-node best-case; not 3-node Raft. Throughput is a saturation ceiling per the ladder._ diff --git a/results/wal-stacked-1m/stacked/cells.json b/results/wal-stacked-1m/stacked/cells.json new file mode 100644 index 0000000..f25df2c --- /dev/null +++ b/results/wal-stacked-1m/stacked/cells.json @@ -0,0 +1,73 @@ +{ + "cells": { + "100000": { + "stream_count": 100000, + "image_digest": "2cfd37a9a235", + "walk": [ + [ + 4, + 382727.89999999997, + 2.537, + 4.539 + ], + [ + 8, + 362346.35000000003, + 5.439, + 8.751 + ] + ], + "pinned_pods": 8, + "throughput": 362346.35000000003, + "p50": null, + "p99": null, + "pod_mem_mb": 774, + "saturated": false, + "status": "ok", + "reason": "ladder_exhausted", + "pod_mem_p50_mb": 668 + }, + "500000": { + "stream_count": 500000, + "image_digest": "2cfd37a9a235", + "walk": [ + [ + 8, + 244203.8, + 5.763, + 26.543 + ] + ], + "pinned_pods": 8, + "throughput": 244203.8, + "p50": null, + "p99": null, + "pod_mem_mb": 2925, + "saturated": false, + "status": "ok", + "reason": "ladder_exhausted", + "pod_mem_p50_mb": 2551 + }, + "1000000": { + "stream_count": 1000000, + "image_digest": "2cfd37a9a235", + "walk": [ + [ + 8, + 56003.35, + 13.919, + 1186.815 + ] + ], + "pinned_pods": 8, + "throughput": 56003.35, + "p50": null, + "p99": null, + "pod_mem_mb": 4939, + "saturated": false, + "status": "ok", + "reason": "ladder_exhausted", + "pod_mem_p50_mb": 3852 + } + } +} \ No newline at end of file diff --git a/scripts/cluster-up.sh b/scripts/cluster-up.sh index 2047f46..3e519b0 100755 --- a/scripts/cluster-up.sh +++ b/scripts/cluster-up.sh @@ -79,14 +79,21 @@ else # Default off = shared cores (existing behavior preserved). STATIC_CPU_FLAG=() if [ "${STATIC_CPU:-0}" = "1" ]; then - _SYSCFG="$(mktemp /tmp/ds-syscfg-XXXXXX.yaml)" + # NOTE: X's must be TRAILING — BSD/macOS mktemp doesn't substitute a + # template with a suffix ("ds-syscfg-XXXXXX.yaml" is taken literally, so a + # second run collides with "File exists" and the create never happens). + _SYSCFG="$(mktemp /tmp/ds-syscfg-XXXXXX)" || { echo "FATAL: mktemp for kubelet system config failed" >&2; exit 1; } printf 'kubeletConfig:\n cpuManagerPolicy: static\n' > "$_SYSCFG" STATIC_CPU_FLAG=(--system-config-from-file "$_SYSCFG") fi + # Fail HARD if the create fails: continuing hands every later kubectl a + # stale kubeconfig from a previous same-name cluster (dead IP), and the + # harness's transient-error tolerance then burns a whole run against it. gcloud container clusters create "$CLUSTER" --zone "$ZONE" --project "$PROJECT" --num-nodes 1 \ --machine-type "$SERVER_MACHINE" "${LSSD_FLAG[@]}" "${SPOT_SERVER_FLAG[@]}" "${STATIC_CPU_FLAG[@]}" \ --node-labels=role=server --network benchmarking --subnetwork benchmarking \ - --enable-ip-alias --release-channel regular + --enable-ip-alias --release-channel regular \ + || { echo "FATAL: cluster create failed for $CLUSTER" >&2; exit 1; } # The client fleet is disposable + fault-tolerant (the bench tolerates pod # failures), so run it on Spot VMs by default (~60-80% cheaper). The SERVER # pool stays on-demand (it holds state). SPOT_CLIENTS=0 forces on-demand. diff --git a/suites/wal-stacked-1m.json b/suites/wal-stacked-1m.json new file mode 100644 index 0000000..e187826 --- /dev/null +++ b/suites/wal-stacked-1m.json @@ -0,0 +1,52 @@ +{ + "suite": "wal-stacked-1m", + "_doc": "Closes the cardinality-cliff campaign: (a) measure the STACKED ideal config (split-lane + syncfs + 1GiB size-trigger checkpoint [PR #4704] + exclusive pinned cores [STATIC_CPU=1 + GUARANTEED=1]) which was previously measured only separately (size-1g 303k @100k on shared cores; bound-3s 328k @100k with 3s timer; stacked projection ~360k); (b) extreme-cardinality check at 500k and 1M streams — the mechanisms that caused the old cliff are gone, this verifies no NEW mechanism appears (candidates: 1 fd per live stream vs the container nofile limit ~1M, DashMap/meta working set, ext4 dir with 1M files, tails-map rewrite cost). An EMFILE/collapse at 1M is itself a finding, not a broken run. Single arm. 500k/1M use a single p8 rung (each rung recreates all streams after the server reset — setup dominates; BARRIER_SETUP_TIMEOUT_SECS=3600 host-side and DS_BENCH_BARRIER_TIMEOUT_SECS=3600 pod-side cover the long creation phase, setup_concurrency 32). Requires STATIC_CPU=1 SPLITLANE=1 GUARANTEED=1 SERVER_LOCAL_SSD_BLOCK=1 SPOT_SERVER=1 IMG_SERVER=...:sizetrigger BARRIER_SETUP_TIMEOUT_SECS=3600.", + "cluster": { + "server_machine": "c4d-standard-64-lssd", + "server_cpus": 8, + "client_machine": "n2d-standard-32", + "client_nodes": 3, + "cluster_name": "bench-cpubind", + "zone": "europe-west4-b", + "region": "europe-west4" + }, + "saturation": { + "plateau_pct": 8, + "patience": 2, + "fleet_cpu": 2, + "connections": 256, + "repeats": 1, + "warmup_secs": 12, + "measure_secs": 20, + "setup_concurrency": 32, + "payload_bytes": 256 + }, + "modes": [ + "wal" + ], + "server_configs": { + "wal": [ + { + "label": "stacked", + "args": "--wal-shards 5 --wal-checkpoint-syncfs on --wal-checkpoint-wal-bytes 1073741824 --wal-checkpoint-interval-ms 60000 --worker-threads 8 --wal-stats 3 --server-stats 3 --data-dir /data/wal/0" + } + ] + }, + "stream_counts": [ + 100000, + 500000, + 1000000 + ], + "pod_ladder": { + "100000": [ + 4, + 8 + ], + "500000": [ + 8 + ], + "1000000": [ + 8 + ] + } +} From 5e4648993072db0bb1cb8f035c249a2affec13c3 Mon Sep 17 00:00:00 2001 From: Valter Balegas Date: Mon, 13 Jul 2026 21:55:27 +0100 Subject: [PATCH 3/8] bench: stream-lanes 3x3 validation (1M wall broken: 68k -> 212k) + 1m-diag telemetry harness MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit wal-1m-diag: root cause = checkpoint syncfs writeback of ~200k dirty files on ONE data device (60-74s per barrier, ~40x metadata amplification of small appends); fds peaked at 1,005,724 (96% of ceiling — adjacent risk, not the cause). wal-streamlanes-1m: 3 stream lanes + 3 WAL shards -> 374k/285k/212k @100k/500k/1M, syncfs 5.7-11.2s. New: splitlane3x3-guaranteed manifest, SERVER_MANIFEST override, DS_BENCH_BARRIER_TIMEOUT_SECS=3600 for 1M setups. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01Y3x7bcT9vLGiT4tXZeQpnk --- ...rable-streams-splitlane3x3-guaranteed.yaml | 277 ++++++++++++++++++ results/wal-1m-diag/aggregate.csv | 3 + results/wal-1m-diag/aggregate.json | 52 ++++ results/wal-1m-diag/report.md | 41 +++ results/wal-1m-diag/stacked/cells.json | 46 +++ results/wal-streamlanes-1m/aggregate.csv | 4 + results/wal-streamlanes-1m/aggregate.json | 77 +++++ .../wal-streamlanes-1m/lanes3x3/cells.json | 67 +++++ results/wal-streamlanes-1m/report.md | 45 +++ scripts/lib-bench.sh | 3 + suites/wal-1m-diag.json | 47 +++ suites/wal-streamlanes-1m.json | 51 ++++ 12 files changed, 713 insertions(+) create mode 100644 gke/durable-streams-splitlane3x3-guaranteed.yaml create mode 100644 results/wal-1m-diag/aggregate.csv create mode 100644 results/wal-1m-diag/aggregate.json create mode 100644 results/wal-1m-diag/report.md create mode 100644 results/wal-1m-diag/stacked/cells.json create mode 100644 results/wal-streamlanes-1m/aggregate.csv create mode 100644 results/wal-streamlanes-1m/aggregate.json create mode 100644 results/wal-streamlanes-1m/lanes3x3/cells.json create mode 100644 results/wal-streamlanes-1m/report.md create mode 100644 suites/wal-1m-diag.json create mode 100644 suites/wal-streamlanes-1m.json diff --git a/gke/durable-streams-splitlane3x3-guaranteed.yaml b/gke/durable-streams-splitlane3x3-guaranteed.yaml new file mode 100644 index 0000000..32f0aae --- /dev/null +++ b/gke/durable-streams-splitlane3x3-guaranteed.yaml @@ -0,0 +1,277 @@ +# ───────────────────────────────────────────────────────────────────────────── +# durable-streams-splitlane3x3-guaranteed.yaml — 3 STREAM LANES + 3 WAL LANES +# variant of the guaranteed split-lane manifest (~1M-stream wall fix, pairs with +# --stream-lanes 3 --wal-shards 3): device 0 = data root (stream lane 0 lives on +# it), devices 1-2 = stream lanes 1-2 (streams/ mounts), devices 3-5 = WAL +# shards 0-2. Checkpoint writeback spreads over 3 devices with 3 parallel syncfs +# barriers instead of storming one. Base: the SPLIT-LANE manifest with +# GUARANTEED QoS: every container (init + app) has requests == limits, and the +# server container requests an INTEGER CPU count, so on a node pool created with +# STATIC_CPU=1 (kubelet cpuManagerPolicy=static) the server gets EXCLUSIVE +# PINNED CORES (true CPU binding). Select via GUARANTEED=1 (lib-bench). +# Otherwise identical to durable-streams-splitlane.yaml (SPLIT-LANE: STREAM DATA +# FILES and WAL SHARDS on physically separate NVMe devices, so the checkpoint's +# stream-file writeback + syncfs NEVER contends with the commit path's WAL +# fdatasync. Motivated by wal-decomp-lane0 (2026-07-13): with everything on ONE +# lane, ref-3s = 55k @10k; with WAL on dedicated lanes (streams elsewhere), +# ref-3s = 270k ≈ the 260k no-fsync ceiling — the commit-fsync tax is mostly +# device contention, not fsync itself. +# +# REQUIRES the server node pool created with RAW-BLOCK local NVMe +# (SERVER_LOCAL_SSD_BLOCK=1, machine c4d-standard-64-lssd = 6 devices). +# +# LAYOUT (server runs with --data-dir /data/wal/0 and --wal-shards 5): +# * device 0 -> /data/wal/0 = the data root: streams/ (checkpoint +# writeback + syncfs domain), meta, cold staging. +# * device I (1..5) -> /data/wal/0/wal/ = WAL shard I-1's dir (walset +# opens shard i at /wal/), each an independent fsync lane. +# * Shard count MUST be ≤ 5 (devices minus the data lane). --wal-shards 6+ +# would spill shard 5 onto the data lane and re-introduce the contention. +# +# MOUNT PROPAGATION (the crux — a mount made in one container is invisible to +# siblings unless propagated): +# * init container -> mountPropagation: Bidirectional (propagates the mounts +# OUT to the host mount namespace = rshared, so they SURVIVE the init +# container exiting and become visible to the server container). +# * server container -> mountPropagation: HostToContainer (RECEIVES those +# submounts). HostToContainer needs no privilege; Bidirectional would force +# the server to run privileged, which we avoid. Only the short-lived init +# container is privileged. +# ${...} placeholders are envsubst-substituted by deploy_server()/MANIFEST_VARS, +# identical to gke/durable-streams.yaml. +apiVersion: apps/v1 +kind: Deployment +metadata: + name: durable-streams + namespace: ds-bench + labels: + app: durable-streams +spec: + replicas: 1 + # Recreate (not RollingUpdate): the old pod must be FULLY torn down before the + # new pod's initContainers wipe/mount the shared NVMe lane devices. With surge + # rollouts the old pod's volume teardown (kubelet cleanup recursing through its + # stale host-side lane mounts) raced the NEW server and deleted its freshly + # created dirs on the same ext4 device (--data-dir /data/wal/0 → 500 ENOENT on + # every stream create; wal-decomp-lane0 2026-07-13). Bench cells tolerate the + # brief downtime between pods. + strategy: + type: Recreate + selector: + matchLabels: + app: durable-streams + template: + metadata: + labels: + app: durable-streams + spec: + nodeSelector: ${NODESEL_SERVER} + shareProcessNamespace: true + securityContext: + seccompProfile: + type: Unconfined + initContainers: + # ── wipe the base /data volume (emptyDir) between cells ──────────────── + # Runs FIRST (initContainers run sequentially). Only clears the base + # volume (the wal/shards metadata file + cold tier); the per-shard NVMe + # devices are wiped by mount-shards after they are mounted. + - name: wipe-data + image: ${IMG_SERVER} + imagePullPolicy: ${PULL_POLICY} + command: ["sh", "-c", "rm -rf /data/* /data/.[!.]* 2>/dev/null; true"] + resources: + requests: { cpu: "100m", memory: "64Mi" } + limits: { cpu: "100m", memory: "64Mi" } + volumeMounts: + - { name: data, mountPath: /data } + # ── mount ONE NVMe device per shard dir ─────────────────────────────── + # Privileged: needs mount(2) + access to the host raw block devices. + # VERIFY AT PROVISION TIME (see MULTILANE_SETUP.md): + # * image ships mkfs.ext4/mount/blkid — the server & metrics images do + # NOT, so we use debian-slim + apt-get e2fsprogs/util-linux at runtime; + # confirm the node has egress to mirror.gcr.io + the apt mirrors, else + # bake a pinned util image into Artifact Registry and swap image: here. + # * device symlink name = google-local-ssd-blockN (raw-block naming); + # adjust DEV_GLOB below if `ls /dev/disk/by-id | grep local-ssd` differs. + - name: mount-shards + image: mirror.gcr.io/library/debian:12-slim + imagePullPolicy: IfNotPresent + securityContext: + privileged: true + resources: + requests: { cpu: "100m", memory: "128Mi" } + limits: { cpu: "100m", memory: "128Mi" } + command: + - bash + - -c + - | + set -euo pipefail + # Raw-block local-NVMe device symlinks exposed by the COS host. + DEV_GLOB="/dev/disk/by-id/google-local-ssd-block*" + # mkfs.ext4 / mount / blkid are not in debian-slim by default. + export DEBIAN_FRONTEND=noninteractive + apt-get update -qq + apt-get install -y -qq e2fsprogs util-linux >/dev/null + shopt -s nullglob + devs=( $DEV_GLOB ) + if [ "${#devs[@]}" -eq 0 ]; then + echo "FATAL: no raw-block local-NVMe devices matched $DEV_GLOB" >&2 + echo " (was the node created with --local-nvme-ssd-block? see MULTILANE_SETUP.md)" >&2 + ls -l /dev/disk/by-id/ >&2 || true + exit 1 + fi + echo "found ${#devs[@]} raw-block NVMe device(s): ${devs[*]}" + # split-lane 3x3 topology: dev0 = data root (stream lane 0), + # dev1-2 = stream lanes 1-2 (nested at streams/), dev3-5 = WAL + # shards 0-2 (nested at wal/). Mount order matters: the data + # root must be mounted before anything nests under it. + mount_one() { # $1=dev $2=dir + if ! blkid -o value -s TYPE "$1" 2>/dev/null | grep -qx ext4; then + echo "mkfs.ext4 $1 -> $2" + mkfs.ext4 -F -q "$1" + fi + mkdir -p "$2" + if ! mountpoint -q "$2"; then + echo "mount $1 -> $2" + mount "$1" "$2" + fi + # Wipe contents (fresh cell). + rm -rf "${2:?}"/* "${2:?}"/.[!.]* 2>/dev/null || true + } + # sort devices by ordinal suffix for a stable assignment + mapfile -t devs < <(printf '%s\n' "${devs[@]}" | sort -V) + mount_one "${devs[0]}" /data/wal/0 + mkdir -p /data/wal/0/streams /data/wal/0/wal + mount_one "${devs[1]}" /data/wal/0/streams/1 + mount_one "${devs[2]}" /data/wal/0/streams/2 + mount_one "${devs[3]}" /data/wal/0/wal/0 + mount_one "${devs[4]}" /data/wal/0/wal/1 + mount_one "${devs[5]}" /data/wal/0/wal/2 + echo "split-lane 3x3 mounts ready:" + mount | grep /data/wal || true + volumeMounts: + # Bidirectional so the mounts propagate to the host namespace and + # survive this container exiting (visible to the server container). + - { name: data, mountPath: /data, mountPropagation: Bidirectional } + # Host /dev so the raw by-id symlinks (-> /dev/nvmeXn1) resolve here. + - { name: dev, mountPath: /dev } + containers: + - name: durable-streams + image: ${IMG_SERVER} + imagePullPolicy: ${PULL_POLICY} + args: + - "--host" + - "0.0.0.0" + - "--port" + - "4438" + - "--data-dir" + - "/data" + - "--tier" + - "s3" + - "--tier-endpoint" + - "http://minio:9000" + - "--tier-region" + - "us-east-1" + - "--tier-bucket" + - "durable-streams" + - "--tier-allow-http" + env: + - name: DS_S3_ACCESS_KEY_ID + value: minioadmin + - name: DS_S3_SECRET_ACCESS_KEY + value: minioadmin + ports: + - containerPort: 4438 + readinessProbe: + tcpSocket: { port: 4438 } + initialDelaySeconds: 2 + periodSeconds: 2 + failureThreshold: 30 + volumeMounts: + # HostToContainer: RECEIVE the per-shard submounts the init container + # propagated to the host. (No privilege needed; Bidirectional here + # would force the server to run privileged — see MULTILANE_SETUP.md.) + - { name: data, mountPath: /data, mountPropagation: HostToContainer } + resources: + # Guaranteed QoS: requests == limits, integer CPU -> exclusive cores + # under cpuManagerPolicy=static. + requests: + memory: "${SERVER_MEM}" + cpu: "${SERVER_CPU}" + limits: + memory: "${SERVER_MEM}" + cpu: "${SERVER_CPU}" + # ── metrics sidecar ───────────────────────────────────────────────────── + # Unchanged from gke/durable-streams.yaml. NOTE: it df's /data (the base + # volume) for /proc/diskstats, so its device-wide write-bytes column now + # under-counts the per-shard NVMe traffic (the shard writes go to the 6 + # sub-mounts, not the base device). Throughput/latency verdicts are + # client-side and unaffected. See MULTILANE_SETUP.md verify-item 7. + - name: metrics + image: ${IMG_METRICS} + imagePullPolicy: ${PULL_POLICY} + command: + - nice + - "-n" + - "19" + - bash + - /deploy/metrics/poller.sh + env: + - name: SERVER_PROC + value: durable-streams-server + - name: METRICS_INTERVAL_S + value: "1" + - name: METRICS_OUT + value: /metrics/samples.csv + - name: DATA_DIR + value: /data + - { name: POD_UID, valueFrom: { fieldRef: { fieldPath: metadata.uid } } } + volumeMounts: + - { name: metrics, mountPath: /metrics } + - { name: metrics-poller, mountPath: /deploy/metrics } + - { name: data, mountPath: /data, readOnly: true } + - { name: cgroup-host, mountPath: /host/cgroup, readOnly: true } + resources: + # requests == limits (pod-level Guaranteed QoS requires it of EVERY + # container; fractional CPU = no exclusive cores for this one, fine). + requests: + cpu: "50m" + memory: "64Mi" + limits: + cpu: "50m" + memory: "64Mi" + volumes: + # hostPath, NOT emptyDir: kubelet deletes emptyDir contents on pod + # teardown, and with the lane devices still mounted under the old pod's + # volume path that cleanup recursed INTO the shared ext4 lanes and wiped + # the next server's live data (see strategy: Recreate note above). A + # hostPath is never cleaned up by kubelet; per-cell freshness comes from + # the wipe-data + mount-shards initContainers, same as before. + - name: data + # /mnt/disks is the writable hostPath root on COS nodes (bare /mnt is + # a read-only filesystem → MountVolume.SetUp fails). + hostPath: { path: /mnt/disks/ds-bench-data, type: DirectoryOrCreate } + - name: metrics + emptyDir: {} + - name: metrics-poller + configMap: + name: metrics-poller + defaultMode: 0755 + - name: cgroup-host + hostPath: { path: /sys/fs/cgroup } + # Host /dev exposes the raw-block local-NVMe device nodes to the mount + # init container (its by-id symlinks point into /dev/nvmeXn1). + - name: dev + hostPath: { path: /dev } +--- +apiVersion: v1 +kind: Service +metadata: + name: durable-streams + namespace: ds-bench +spec: + selector: + app: durable-streams + ports: + - port: 4438 + targetPort: 4438 diff --git a/results/wal-1m-diag/aggregate.csv b/results/wal-1m-diag/aggregate.csv new file mode 100644 index 0000000..da5767b --- /dev/null +++ b/results/wal-1m-diag/aggregate.csv @@ -0,0 +1,3 @@ +mode,stream_count,pods,throughput,p50,p99,knee_pods,knee_throughput,knee_p50,knee_p99,pod_mem_mb,pod_mem_p50_mb,saturated,status,reason +stacked,500000,8,251909.06666666668,,,8,251909.06666666668,5.867,13.343,3727,3144,False,ok,ladder_exhausted +stacked,1000000,8,68271.79999999999,,,8,68271.79999999999,8.495,1102.847,6081,5774,False,ok,ladder_exhausted diff --git a/results/wal-1m-diag/aggregate.json b/results/wal-1m-diag/aggregate.json new file mode 100644 index 0000000..0b5d38a --- /dev/null +++ b/results/wal-1m-diag/aggregate.json @@ -0,0 +1,52 @@ +[ + { + "mode": "stacked", + "stream_count": 500000, + "pods": 8, + "throughput": 251909.06666666668, + "p50": null, + "p99": null, + "knee_pods": 8, + "knee_throughput": 251909.06666666668, + "knee_p50": 5.867, + "knee_p99": 13.343, + "pod_mem_mb": 3727, + "pod_mem_p50_mb": 3144, + "saturated": false, + "status": "ok", + "reason": "ladder_exhausted", + "walk": [ + [ + 8, + 251909.06666666668, + 5.867, + 13.343 + ] + ] + }, + { + "mode": "stacked", + "stream_count": 1000000, + "pods": 8, + "throughput": 68271.79999999999, + "p50": null, + "p99": null, + "knee_pods": 8, + "knee_throughput": 68271.79999999999, + "knee_p50": 8.495, + "knee_p99": 1102.847, + "pod_mem_mb": 6081, + "pod_mem_p50_mb": 5774, + "saturated": false, + "status": "ok", + "reason": "ladder_exhausted", + "walk": [ + [ + 8, + 68271.79999999999, + 8.495, + 1102.847 + ] + ] + } +] \ No newline at end of file diff --git a/results/wal-1m-diag/report.md b/results/wal-1m-diag/report.md new file mode 100644 index 0000000..a1a21e2 --- /dev/null +++ b/results/wal-1m-diag/report.md @@ -0,0 +1,41 @@ +# wal-1m-diag — write-throughput report + +## Throughput at saturation (ops/s) + +| streams | stacked | +|---|---| +| 500000 | 252k† | +| 1000000 | 68k† | + +† = not saturated (ladder exhausted) — treat as a lower bound. + +## Pod memory at saturation — peak / p50 (MiB) + +| streams | stacked | +|---|---| +| 500000 | 3727 / 3144 | +| 1000000 | 6081 / 5774 | + +_Pod working set = cgroup `memory.current − inactive_file` (anon + active page cache), sampled each second at the pinned rung. **peak** = high-water (catches bursts like an in-RAM Raft log filling); **p50** = median (what the server steadily holds resident). peak ≈ p50 ⇒ steadily resident; peak ≫ p50 ⇒ transient spikes._ + +## Latency (ms, p50 / p99) + +| streams | stacked @≤80% load | stacked @saturation | +|---|---|---| +| 500000 | 5.9 / 13.3 (252k @8p) | — | +| 1000000 | 8.5 / 1102.8 (68k @8p) | — | + +_@≤80% load = the largest ladder rung at ≤80% of peak throughput — the server's service latency with headroom. @saturation = the pinned plateau rung, where a closed-loop fleet measures its own queueing (≈ in-flight ÷ ceiling by Little's law), NOT the server's per-request cost. Compare against the unloaded single-request baseline (~1 ms for wal) before reading anything into large saturation values._ + +## Saturation walks (pods → ops/s, p50 ms) + +- **stacked 500000**: 8:252k@5.9ms (pinned 8, ladder_exhausted) +- **stacked 1000000**: 8:68k@8.5ms (pinned 8, ladder_exhausted) + +## Findings + +_TODO: written by hand on top of the generated data._ + +## Caveats + +_Single-node best-case; not 3-node Raft. Throughput is a saturation ceiling per the ladder._ diff --git a/results/wal-1m-diag/stacked/cells.json b/results/wal-1m-diag/stacked/cells.json new file mode 100644 index 0000000..43b37f2 --- /dev/null +++ b/results/wal-1m-diag/stacked/cells.json @@ -0,0 +1,46 @@ +{ + "cells": { + "500000": { + "stream_count": 500000, + "image_digest": "2cfd37a9a235", + "walk": [ + [ + 8, + 251909.06666666668, + 5.867, + 13.343 + ] + ], + "pinned_pods": 8, + "throughput": 251909.06666666668, + "p50": null, + "p99": null, + "pod_mem_mb": 3727, + "saturated": false, + "status": "ok", + "reason": "ladder_exhausted", + "pod_mem_p50_mb": 3144 + }, + "1000000": { + "stream_count": 1000000, + "image_digest": "2cfd37a9a235", + "walk": [ + [ + 8, + 68271.79999999999, + 8.495, + 1102.847 + ] + ], + "pinned_pods": 8, + "throughput": 68271.79999999999, + "p50": null, + "p99": null, + "pod_mem_mb": 6081, + "saturated": false, + "status": "ok", + "reason": "ladder_exhausted", + "pod_mem_p50_mb": 5774 + } + } +} \ No newline at end of file diff --git a/results/wal-streamlanes-1m/aggregate.csv b/results/wal-streamlanes-1m/aggregate.csv new file mode 100644 index 0000000..d6ad0a5 --- /dev/null +++ b/results/wal-streamlanes-1m/aggregate.csv @@ -0,0 +1,4 @@ +mode,stream_count,pods,throughput,p50,p99,knee_pods,knee_throughput,knee_p50,knee_p99,pod_mem_mb,pod_mem_p50_mb,saturated,status,reason +lanes3x3,100000,8,374115.79999999993,,,8,374115.79999999993,5.291,8.671,1053,859,False,ok,ladder_exhausted +lanes3x3,500000,8,284627.3333333333,,,8,284627.3333333333,5.731,11.031,3160,3026,False,ok,ladder_exhausted +lanes3x3,1000000,8,211678.4333333333,,,8,211678.4333333333,6.011,29.727,5588,4968,False,ok,ladder_exhausted diff --git a/results/wal-streamlanes-1m/aggregate.json b/results/wal-streamlanes-1m/aggregate.json new file mode 100644 index 0000000..6344344 --- /dev/null +++ b/results/wal-streamlanes-1m/aggregate.json @@ -0,0 +1,77 @@ +[ + { + "mode": "lanes3x3", + "stream_count": 100000, + "pods": 8, + "throughput": 374115.79999999993, + "p50": null, + "p99": null, + "knee_pods": 8, + "knee_throughput": 374115.79999999993, + "knee_p50": 5.291, + "knee_p99": 8.671, + "pod_mem_mb": 1053, + "pod_mem_p50_mb": 859, + "saturated": false, + "status": "ok", + "reason": "ladder_exhausted", + "walk": [ + [ + 8, + 374115.79999999993, + 5.291, + 8.671 + ] + ] + }, + { + "mode": "lanes3x3", + "stream_count": 500000, + "pods": 8, + "throughput": 284627.3333333333, + "p50": null, + "p99": null, + "knee_pods": 8, + "knee_throughput": 284627.3333333333, + "knee_p50": 5.731, + "knee_p99": 11.031, + "pod_mem_mb": 3160, + "pod_mem_p50_mb": 3026, + "saturated": false, + "status": "ok", + "reason": "ladder_exhausted", + "walk": [ + [ + 8, + 284627.3333333333, + 5.731, + 11.031 + ] + ] + }, + { + "mode": "lanes3x3", + "stream_count": 1000000, + "pods": 8, + "throughput": 211678.4333333333, + "p50": null, + "p99": null, + "knee_pods": 8, + "knee_throughput": 211678.4333333333, + "knee_p50": 6.011, + "knee_p99": 29.727, + "pod_mem_mb": 5588, + "pod_mem_p50_mb": 4968, + "saturated": false, + "status": "ok", + "reason": "ladder_exhausted", + "walk": [ + [ + 8, + 211678.4333333333, + 6.011, + 29.727 + ] + ] + } +] \ No newline at end of file diff --git a/results/wal-streamlanes-1m/lanes3x3/cells.json b/results/wal-streamlanes-1m/lanes3x3/cells.json new file mode 100644 index 0000000..d06b384 --- /dev/null +++ b/results/wal-streamlanes-1m/lanes3x3/cells.json @@ -0,0 +1,67 @@ +{ + "cells": { + "100000": { + "stream_count": 100000, + "image_digest": "79a7f7d5f756", + "walk": [ + [ + 8, + 374115.79999999993, + 5.291, + 8.671 + ] + ], + "pinned_pods": 8, + "throughput": 374115.79999999993, + "p50": null, + "p99": null, + "pod_mem_mb": 1053, + "saturated": false, + "status": "ok", + "reason": "ladder_exhausted", + "pod_mem_p50_mb": 859 + }, + "500000": { + "stream_count": 500000, + "image_digest": "79a7f7d5f756", + "walk": [ + [ + 8, + 284627.3333333333, + 5.731, + 11.031 + ] + ], + "pinned_pods": 8, + "throughput": 284627.3333333333, + "p50": null, + "p99": null, + "pod_mem_mb": 3160, + "saturated": false, + "status": "ok", + "reason": "ladder_exhausted", + "pod_mem_p50_mb": 3026 + }, + "1000000": { + "stream_count": 1000000, + "image_digest": "79a7f7d5f756", + "walk": [ + [ + 8, + 211678.4333333333, + 6.011, + 29.727 + ] + ], + "pinned_pods": 8, + "throughput": 211678.4333333333, + "p50": null, + "p99": null, + "pod_mem_mb": 5588, + "saturated": false, + "status": "ok", + "reason": "ladder_exhausted", + "pod_mem_p50_mb": 4968 + } + } +} \ No newline at end of file diff --git a/results/wal-streamlanes-1m/report.md b/results/wal-streamlanes-1m/report.md new file mode 100644 index 0000000..b7ec241 --- /dev/null +++ b/results/wal-streamlanes-1m/report.md @@ -0,0 +1,45 @@ +# wal-streamlanes-1m — write-throughput report + +## Throughput at saturation (ops/s) + +| streams | lanes3x3 | +|---|---| +| 100000 | 374k† | +| 500000 | 285k† | +| 1000000 | 212k† | + +† = not saturated (ladder exhausted) — treat as a lower bound. + +## Pod memory at saturation — peak / p50 (MiB) + +| streams | lanes3x3 | +|---|---| +| 100000 | 1053 / 859 | +| 500000 | 3160 / 3026 | +| 1000000 | 5588 / 4968 | + +_Pod working set = cgroup `memory.current − inactive_file` (anon + active page cache), sampled each second at the pinned rung. **peak** = high-water (catches bursts like an in-RAM Raft log filling); **p50** = median (what the server steadily holds resident). peak ≈ p50 ⇒ steadily resident; peak ≫ p50 ⇒ transient spikes._ + +## Latency (ms, p50 / p99) + +| streams | lanes3x3 @≤80% load | lanes3x3 @saturation | +|---|---|---| +| 100000 | 5.3 / 8.7 (374k @8p) | — | +| 500000 | 5.7 / 11.0 (285k @8p) | — | +| 1000000 | 6.0 / 29.7 (212k @8p) | — | + +_@≤80% load = the largest ladder rung at ≤80% of peak throughput — the server's service latency with headroom. @saturation = the pinned plateau rung, where a closed-loop fleet measures its own queueing (≈ in-flight ÷ ceiling by Little's law), NOT the server's per-request cost. Compare against the unloaded single-request baseline (~1 ms for wal) before reading anything into large saturation values._ + +## Saturation walks (pods → ops/s, p50 ms) + +- **lanes3x3 100000**: 8:374k@5.3ms (pinned 8, ladder_exhausted) +- **lanes3x3 500000**: 8:285k@5.7ms (pinned 8, ladder_exhausted) +- **lanes3x3 1000000**: 8:212k@6.0ms (pinned 8, ladder_exhausted) + +## Findings + +_TODO: written by hand on top of the generated data._ + +## Caveats + +_Single-node best-case; not 3-node Raft. Throughput is a saturation ceiling per the ladder._ diff --git a/scripts/lib-bench.sh b/scripts/lib-bench.sh index 3dbcb8b..5ba7ac0 100755 --- a/scripts/lib-bench.sh +++ b/scripts/lib-bench.sh @@ -98,6 +98,9 @@ deploy_server() { # every container, integer server CPU. On a STATIC_CPU=1 node pool the server # gets exclusive pinned cores (CPU-binding experiment). [ "${GUARANTEED:-0}" = "1" ] && server_manifest="gke/durable-streams-splitlane-guaranteed.yaml" + # SERVER_MANIFEST: explicit override for one-off topologies (e.g. the 3x3 + # stream-lane/WAL-lane variant). Wins over all the flags above. + [ -n "${SERVER_MANIFEST:-}" ] && server_manifest="$SERVER_MANIFEST" if [ -z "$extra_args" ]; then envsubst "${MANIFEST_VARS} \${SERVER_CPU}" < "$server_manifest" | K apply -f - diff --git a/suites/wal-1m-diag.json b/suites/wal-1m-diag.json new file mode 100644 index 0000000..369aab7 --- /dev/null +++ b/suites/wal-1m-diag.json @@ -0,0 +1,47 @@ +{ + "suite": "wal-1m-diag", + "_doc": "Diagnose the NEW ~1M-stream wall found by wal-stacked-1m (stacked ideal config: 383k @100k, 244k @500k, 56k @1M). Candidates: (a) one open fd per live stream vs the container nofile ceiling (~1.05M) — 1M streams sits right at it, the accept loop sheds on EMFILE; (b) ext4 streams/ directory with 1M files; (c) tails-map full rewrite per checkpoint (cumulative map = O(N_total) write); (d) post-setup settle: creating 1M streams leaves 1M dirty files + meta sidecars mid-writeback and the 12s warmup measured the storm, not steady state. This suite separates (d) from the rest via warmup_secs 90 (vs 12), and the launcher runs a telemetry capture loop (SRV_STATS / WAL_CKPT / WAL_CONT + live fd count of the server process every 15s -> results/wal-1m-diag/telemetry-.log) to identify (a)/(b)/(c) directly. Same stacked config/image as wal-stacked-1m. If 1M with a long warmup recovers to ~200k+, the wall is mostly (d); if fd count plateaus at ~1.05M with appends collapsing, it is (a); WAL_CKPT fsync_us/cadence exposes (b)/(c). Requires STATIC_CPU=1 SPLITLANE=1 GUARANTEED=1 SERVER_LOCAL_SSD_BLOCK=1 SPOT_SERVER=1 BARRIER_SETUP_TIMEOUT_SECS=3600 IMG_SERVER=...:sizetrigger.", + "cluster": { + "server_machine": "c4d-standard-64-lssd", + "server_cpus": 8, + "client_machine": "n2d-standard-32", + "client_nodes": 3, + "cluster_name": "bench-cpubind", + "zone": "europe-west4-b", + "region": "europe-west4" + }, + "saturation": { + "plateau_pct": 8, + "patience": 2, + "fleet_cpu": 2, + "connections": 256, + "repeats": 1, + "warmup_secs": 90, + "measure_secs": 30, + "setup_concurrency": 32, + "payload_bytes": 256 + }, + "modes": [ + "wal" + ], + "server_configs": { + "wal": [ + { + "label": "stacked", + "args": "--wal-shards 5 --wal-checkpoint-syncfs on --wal-checkpoint-wal-bytes 1073741824 --wal-checkpoint-interval-ms 60000 --worker-threads 8 --wal-stats 3 --server-stats 3 --data-dir /data/wal/0" + } + ] + }, + "stream_counts": [ + 500000, + 1000000 + ], + "pod_ladder": { + "500000": [ + 8 + ], + "1000000": [ + 8 + ] + } +} diff --git a/suites/wal-streamlanes-1m.json b/suites/wal-streamlanes-1m.json new file mode 100644 index 0000000..e62f9ae --- /dev/null +++ b/suites/wal-streamlanes-1m.json @@ -0,0 +1,51 @@ +{ + "suite": "wal-streamlanes-1m", + "_doc": "Validate the --stream-lanes fix for the ~1M-stream writeback wall (wal-1m-diag: one shared data device -> checkpoint syncfs of ~200k dirty files = 60-74s per shard, append staging starved, 68k @1M). Topology 3x3 via SERVER_MANIFEST=gke/durable-streams-splitlane3x3-guaranteed.yaml: dev0 = data root + stream lane 0, dev1-2 = stream lanes 1-2, dev3-5 = WAL shards 0-2; args --stream-lanes 3 --wal-shards 3. Controls (from wal-1m-diag, 5 WAL lanes + 1 data lane, same image family/QoS): 500k = 252k, 1M = 68k. Success = 1M meaningfully above 68k (expect 150k+ if writeback capacity was the binding constraint); 500k should hold >= 250k (3 WAL lanes must not regress the commit path - WAL fsync was measured far from saturation). 100k anchors against the stacked 383k. Requires STATIC_CPU=1 SPLITLANE=1 GUARANTEED=1 SERVER_MANIFEST=gke/durable-streams-splitlane3x3-guaranteed.yaml SERVER_LOCAL_SSD_BLOCK=1 SPOT_SERVER=1 BARRIER_SETUP_TIMEOUT_SECS=3600 IMG_SERVER=...:streamlanes.", + "cluster": { + "server_machine": "c4d-standard-64-lssd", + "server_cpus": 8, + "client_machine": "n2d-standard-32", + "client_nodes": 3, + "cluster_name": "bench-cpubind", + "zone": "europe-west4-b", + "region": "europe-west4" + }, + "saturation": { + "plateau_pct": 8, + "patience": 2, + "fleet_cpu": 2, + "connections": 256, + "repeats": 1, + "warmup_secs": 90, + "measure_secs": 30, + "setup_concurrency": 32, + "payload_bytes": 256 + }, + "modes": [ + "wal" + ], + "server_configs": { + "wal": [ + { + "label": "lanes3x3", + "args": "--wal-shards 3 --stream-lanes 3 --wal-checkpoint-syncfs on --wal-checkpoint-wal-bytes 1073741824 --wal-checkpoint-interval-ms 60000 --worker-threads 8 --wal-stats 3 --server-stats 3 --data-dir /data/wal/0" + } + ] + }, + "stream_counts": [ + 100000, + 500000, + 1000000 + ], + "pod_ladder": { + "100000": [ + 8 + ], + "500000": [ + 8 + ], + "1000000": [ + 8 + ] + } +} From 73642cc6283c095ad4794d3acfe5d2b327f76088 Mon Sep 17 00:00:00 2001 From: Valter Balegas Date: Mon, 13 Jul 2026 21:58:50 +0100 Subject: [PATCH 4/8] bench: AGENTS.md stream-lanes guidance + wal-streamlanes-1m findings Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01Y3x7bcT9vLGiT4tXZeQpnk --- AGENTS.md | 7 +++++++ results/wal-streamlanes-1m/report.md | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/AGENTS.md b/AGENTS.md index e6f3227..20fdfef 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -411,6 +411,13 @@ NVMe box**, so you size the fleet to the bottleneck instead of guessing. GUARANTEED=1 = requests==limits everywhere + integer server CPU) measured 356k @10k / 328k @100k vs 286k/272k on shared cores, same layout/image/args. Now that wal isn't fsync-bound, bind the server's cores for wal benches. + - **≥500k streams: add STREAM lanes (wal-streamlanes-1m, PR #4705).** On one + data lane the checkpoint's dirty-file writeback saturates the device + (syncfs 60-74s at 1M; 68k ops/s). 3 data lanes + 3 WAL lanes + (SERVER_MANIFEST=gke/durable-streams-splitlane3x3-guaranteed.yaml, + --stream-lanes 3 --wal-shards 3) → 374k/285k/212k @100k/500k/1M. + Split the 6 devices by cardinality: writes-per-file amplification means the + DATA side needs the lanes at high stream counts, not the WAL side. - **Checkpoint size trigger ≈ free checkpointing (wal-sizetrigger, PR #4704).** `--wal-checkpoint-wal-bytes 1073741824` (+60s fallback interval) hits the checkpoint-off ceiling (303k vs 306k @100k) while bounding replay to ≤1 GiB diff --git a/results/wal-streamlanes-1m/report.md b/results/wal-streamlanes-1m/report.md index b7ec241..da6641f 100644 --- a/results/wal-streamlanes-1m/report.md +++ b/results/wal-streamlanes-1m/report.md @@ -38,7 +38,7 @@ _@≤80% load = the largest ladder rung at ≤80% of peak throughput — the ser ## Findings -_TODO: written by hand on top of the generated data._ +3 stream lanes + 3 WAL lanes (--stream-lanes 3, PR #4705) breaks the 1M writeback wall: 374.1k @100k (parity with the 5-WAL-lane stacked config), 284.6k @500k (+13% vs 1 data lane), 211.7k @1M (3.1x vs 68k). Checkpoint syncfs fell 60-74s -> 5.7-11.2s with MORE touched files per shard. Residual 100k->1M slope (-43%) = per-file writeback amplification vs total data-lane capacity; add lanes or see #4695. fd note: ~1.006M fds at 1M streams (96% of ceiling) -> #4706. ## Caveats From 13a2d5f54caf3f613313e05c327a92aee58d20f1 Mon Sep 17 00:00:00 2001 From: Valter Balegas Date: Mon, 13 Jul 2026 23:35:17 +0100 Subject: [PATCH 5/8] =?UTF-8?q?bench:=20canonical=20suite=20set=20+=20idea?= =?UTF-8?q?l-configuration=20invariants=20(AGENTS.md=20=C2=A70)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Six canonical suites replace the ad-hoc set as the maintained benchmarks: canonical-write (durable-streams wal-ideal per WAL_TUNING.md + memory), canonical-write-ursula (upstream v0.2.0), canonical-sustained, canonical-reads-{catchup,live,sse}. Each _doc states reference numbers and the regression gate (wal-ideal@100k < 250k / memory@100k < 400k / >30% cardinality drop = the cliff is back). AGENTS.md §0 documents the do-not-break invariants: stream data never on the boot PD, WAL lanes != data lanes, Guaranteed+static CPU, size-triggered checkpoints, memory arms need --tier off (server refuses memory+tier), removed-flag list, persisted layout knobs. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01Y3x7bcT9vLGiT4tXZeQpnk --- AGENTS.md | 55 +++++++++++++++++++++++++++ suites/canonical-reads-catchup.json | 37 ++++++++++++++++++ suites/canonical-reads-live.json | 39 +++++++++++++++++++ suites/canonical-reads-sse.json | 38 +++++++++++++++++++ suites/canonical-sustained.json | 44 ++++++++++++++++++++++ suites/canonical-write-ursula.json | 58 +++++++++++++++++++++++++++++ suites/canonical-write.json | 53 ++++++++++++++++++++++++++ 7 files changed, 324 insertions(+) create mode 100644 suites/canonical-reads-catchup.json create mode 100644 suites/canonical-reads-live.json create mode 100644 suites/canonical-reads-sse.json create mode 100644 suites/canonical-sustained.json create mode 100644 suites/canonical-write-ursula.json create mode 100644 suites/canonical-write.json diff --git a/AGENTS.md b/AGENTS.md index 20fdfef..4f6f1da 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -11,6 +11,61 @@ covers *how to run it*, the typical deployment we use, and the gotchas that bite --- +## 0. Canonical suites — the maintained benchmark set + +These are THE benchmarks we maintain and rerun; prefer them over the historical +suites (which remain for provenance). Each is one `scripts/bench suites/.json +run` away, self-tears-down on clean completion, and states its reference numbers ++ regression gate in its `_doc`. + +| suite | workload | systems / configs | env (beyond `DS_TARGET=remote`) | +|---|---|---|---| +| `canonical-write` | write saturation (10k/100k streams) | durable-streams `wal-ideal` (the WAL_TUNING.md ideal config) + `memory` | `STATIC_CPU=1 SPLITLANE=1 GUARANTEED=1 SERVER_LOCAL_SSD_BLOCK=1 SERVER_MANIFEST=gke/durable-streams-splitlane3x3-guaranteed.yaml SPOT_SERVER=1` | +| `canonical-write-ursula` | write saturation (100/1k/10k) | ursula v0.2.0 (`ghcr.io/tonbo-io/ursula:v0.2.0`) memory + disk | `SPOT_SERVER=1` | +| `canonical-sustained` | long-window latency + memory stability | durable-streams wal + memory | — | +| `canonical-reads-catchup` | historical replay reads | durable-streams wal | — | +| `canonical-reads-live` | live-tail long-poll delivery | durable-streams wal | — | +| `canonical-reads-sse` | SSE fan-out delivery | durable-streams wal | — | + +Reference numbers (c4d-standard-64-lssd, 2026-07-13): `wal-ideal` ≈ 385k @10k / +374k @100k; `memory` ≈ 540k @10k / 512k @100k. **Regression gate: wal-ideal@100k +< 250k, memory@100k < 400k, or a >30% drop from 10k→100k = the cliff is back — +stop, diagnose (WAL_CKPT/SRV_STATS), fix before publishing numbers.** + +### The ideal configuration — invariants (DO NOT break these again) + +The 37× write-throughput recovery (10.4k → 383k @100k streams; see +`durable-streams-rust/WAL_TUNING.md` for the full ladder) depends on ALL of: + +1. **Stream data files on local NVMe, never the boot PD.** On raw-block node + pools (`SERVER_LOCAL_SSD_BLOCK=1`) the base emptyDir/hostPath sits on the PD + boot disk — server args MUST route `--data-dir` onto a lane + (`--data-dir /data/wal/0`, the splitlane manifests mount device 0 there). + Getting this wrong mismeasures wal by 5–26× and looks exactly like a + "cardinality cliff". +2. **WAL lanes and data lanes on separate devices.** Commit fdatasync vs + checkpoint writeback on one device queue costs 5×. The 3×3 split + (`--wal-shards 3 --stream-lanes 3` + `durable-streams-splitlane3x3-guaranteed.yaml`) + is the general-purpose layout; ≥500k streams is exactly where 1 data lane + collapses (syncfs 60–74 s) — do not "simplify" back to it. +3. **Guaranteed QoS + static CPU manager** (`GUARANTEED=1` + `STATIC_CPU=1`): + exclusive pinned cores are +21–24% now that wal isn't fsync-bound. +4. **Size-triggered checkpoints** (`--wal-checkpoint-wal-bytes 1073741824 + --wal-checkpoint-interval-ms 60000`): reclaims the checkpoint's 7–11%; the + 1 GiB budget bounds crash-replay. +5. **memory arms MUST pass `--tier off`.** The manifests bake in `--tier s3` + (MinIO) and the server now REFUSES `--durability memory` + tier (non-durable + acks must not feed a "durable" cold tier). A memory config without + `--tier off` crash-loops. +6. **Removed server flags — never pass them** (the server exits 2 on unknown + args): `--wal-checkpoint-syncfs` (syncfs is unconditional on Linux), + `--wal-fsync-parallel`, `--wal-meta-gate`, `--mem-meta-gate`, + `--meta-sweep-disable`, `--meta-sweep-stats`, `--tier local`, + `--tier-local-dir`. +7. **`--stream-lanes` / `--wal-shards` are persisted on-disk layout choices** — + the server refuses a mismatch on an existing data dir. Fresh bench cells wipe + the dirs, so suites just need each config to be internally consistent. + ## 1. What it does `ds-bench` is a single-node, server-agnostic benchmark harness for durable-stream diff --git a/suites/canonical-reads-catchup.json b/suites/canonical-reads-catchup.json new file mode 100644 index 0000000..b07b94a --- /dev/null +++ b/suites/canonical-reads-catchup.json @@ -0,0 +1,37 @@ +{ + "suite": "canonical-reads-catchup", + "workload": "reads", + "cluster": { + "server_machine": "c4d-standard-16-lssd", + "client_machine": "n2d-standard-32", + "client_nodes": 2, + "region": "europe-west4", + "cluster_name": "bench-canon-reads", + "zone": "europe-west4-a" + }, + "reads": { + "mode": "catchup", + "read_size_bytes": 4096, + "seed_bytes": 16777216, + "duration_secs": 15, + "warmup_secs": 5, + "settle_secs": 2, + "connection_levels": [ + 8, + 32, + 128, + 512 + ], + "pods": 1, + "repeats": 1 + }, + "modes": [ + "wal", + "ursula" + ], + "stream_counts": [ + 10, + 100 + ], + "_doc": "CANONICAL catch-up read benchmark (historical replay throughput/latency vs connections)." +} \ No newline at end of file diff --git a/suites/canonical-reads-live.json b/suites/canonical-reads-live.json new file mode 100644 index 0000000..dd89bd2 --- /dev/null +++ b/suites/canonical-reads-live.json @@ -0,0 +1,39 @@ +{ + "suite": "canonical-reads-live", + "workload": "reads", + "cluster": { + "server_machine": "c4d-standard-16-lssd", + "client_machine": "n2d-standard-32", + "client_nodes": 2, + "region": "europe-west4", + "cluster_name": "bench-canon-reads", + "zone": "europe-west4-a" + }, + "reads": { + "mode": "long-poll", + "read_size_bytes": 4096, + "append_rate_per_sec": 50, + "seed_bytes": 0, + "duration_secs": 15, + "warmup_secs": 5, + "settle_secs": 2, + "connection_levels": [ + 32, + 128, + 512, + 1024, + 2048 + ], + "pods": 1, + "repeats": 1 + }, + "modes": [ + "wal", + "ursula" + ], + "stream_counts": [ + 100, + 1000 + ], + "_doc": "CANONICAL live-tail delivery benchmark (long-poll) \u2014 delivery latency vs connections." +} \ No newline at end of file diff --git a/suites/canonical-reads-sse.json b/suites/canonical-reads-sse.json new file mode 100644 index 0000000..10a9c42 --- /dev/null +++ b/suites/canonical-reads-sse.json @@ -0,0 +1,38 @@ +{ + "suite": "canonical-reads-sse", + "workload": "reads", + "cluster": { + "server_machine": "c4d-standard-16-lssd", + "client_machine": "n2d-standard-32", + "client_nodes": 2, + "region": "europe-west4", + "cluster_name": "bench-canon-reads", + "zone": "europe-west4-a" + }, + "reads": { + "mode": "sse", + "read_size_bytes": 256, + "append_rate_per_sec": 50, + "seed_bytes": 0, + "duration_secs": 15, + "warmup_secs": 5, + "settle_secs": 2, + "connection_levels": [ + 64, + 256, + 1024, + 2048 + ], + "pods": 1, + "repeats": 1 + }, + "modes": [ + "wal", + "ursula" + ], + "stream_counts": [ + 10, + 100 + ], + "_doc": "CANONICAL SSE fan-out delivery benchmark \u2014 per-event delivery latency vs subscribers." +} \ No newline at end of file diff --git a/suites/canonical-sustained.json b/suites/canonical-sustained.json new file mode 100644 index 0000000..ae82844 --- /dev/null +++ b/suites/canonical-sustained.json @@ -0,0 +1,44 @@ +{ + "suite": "canonical-sustained", + "workload": "sustained", + "cluster": { + "server_machine": "c4d-standard-16-lssd", + "client_machine": "n2d-standard-32", + "client_nodes": 2, + "region": "europe-west4", + "cluster_name": "bench-canon", + "zone": "europe-west4-b" + }, + "sustained": { + "rate_per_stream": 10, + "duration_secs": 90, + "snapshot_secs": 5, + "pods": 1, + "fleet_cpu": 0.5, + "payload_bytes": 256, + "setup_concurrency": 64, + "repeats": 1 + }, + "modes": [ + "wal" + ], + "stream_counts": [ + 10, + 50, + 100, + 150 + ], + "server_configs": { + "wal": [ + { + "label": "wal", + "args": "--durability wal --wal-shards 4" + }, + { + "label": "memory", + "args": "--durability memory --tier off" + } + ] + }, + "_doc": "CANONICAL sustained-load stability benchmark (latency + server-memory over a long window). Standard single-fs manifest is fine here (stability, not throughput). memory arm passes --tier off (server refuses memory+tier)." +} \ No newline at end of file diff --git a/suites/canonical-write-ursula.json b/suites/canonical-write-ursula.json new file mode 100644 index 0000000..29110c7 --- /dev/null +++ b/suites/canonical-write-ursula.json @@ -0,0 +1,58 @@ +{ + "suite": "canonical-write-ursula", + "cluster": { + "server_machine": "c4d-standard-16-lssd", + "client_machine": "n2d-standard-32", + "client_nodes": 2, + "region": "europe-west4", + "cluster_name": "bench-ursula", + "zone": "europe-west4-c" + }, + "saturation": { + "plateau_pct": 8, + "fleet_cpu": 0.5, + "repeats": 1, + "warmup_secs": 8, + "measure_secs": 8, + "setup_concurrency": 16, + "payload_bytes": 256 + }, + "modes": [ + "ursula" + ], + "server_configs": { + "ursula": [ + { + "label": "ursula-memory", + "args": "memory" + }, + { + "label": "ursula-disk", + "args": "disk" + } + ] + }, + "stream_counts": [ + 100, + 1000, + 10000 + ], + "pod_ladder": { + "100": [ + 4, + 8, + 16 + ], + "1000": [ + 4, + 8, + 16 + ], + "10000": [ + 8, + 16, + 24 + ] + }, + "_doc": "CANONICAL write-saturation benchmark for ursula v0.2.0 (upstream image ghcr.io/tonbo-io/ursula:v0.2.0, target-env.sh default). Same workload as canonical-write, ursula's historic cardinality range. Runs on its own cluster (parallel-safe with canonical-write)." +} \ No newline at end of file diff --git a/suites/canonical-write.json b/suites/canonical-write.json new file mode 100644 index 0000000..89da6ac --- /dev/null +++ b/suites/canonical-write.json @@ -0,0 +1,53 @@ +{ + "suite": "canonical-write", + "_doc": "CANONICAL write-saturation benchmark for durable-streams (see AGENTS.md 'Canonical suites'). Two configs on ONE cluster: wal-ideal = the validated ideal disk configuration (WAL_TUNING.md: 3 stream lanes + 3 WAL shard lanes on 6 raw-block NVMe devices, syncfs checkpoints [unconditional on Linux], 1GiB retained-WAL checkpoint budget + 60s fallback, exclusive pinned cores) and memory = the no-fsync ceiling. References (2026-07-13): wal-ideal 374k @100k (no cliff: -3% vs 10k... 385k @10k), memory 512k @100k (-6% vs 10k). REGRESSION GATE: wal-ideal@100k < 250k or memory@100k < 400k or a >30% 10k->100k drop = cliff regression -> stop, diagnose, fix. RUN WITH: DS_TARGET=remote STATIC_CPU=1 SPLITLANE=1 GUARANTEED=1 SERVER_LOCAL_SSD_BLOCK=1 SERVER_MANIFEST=gke/durable-streams-splitlane3x3-guaranteed.yaml SPOT_SERVER=1. NOTE: memory arm passes --tier off (the manifest bakes in --tier s3 and the server refuses memory+tier).", + "cluster": { + "server_machine": "c4d-standard-64-lssd", + "server_cpus": 8, + "client_machine": "n2d-standard-32", + "client_nodes": 3, + "cluster_name": "bench-canon", + "zone": "europe-west4-b", + "region": "europe-west4" + }, + "saturation": { + "plateau_pct": 8, + "patience": 2, + "fleet_cpu": 2, + "connections": 256, + "repeats": 1, + "warmup_secs": 12, + "measure_secs": 20, + "setup_concurrency": 32, + "payload_bytes": 256 + }, + "modes": [ + "wal" + ], + "server_configs": { + "wal": [ + { + "label": "wal-ideal", + "args": "--wal-shards 3 --stream-lanes 3 --wal-checkpoint-wal-bytes 1073741824 --wal-checkpoint-interval-ms 60000 --worker-threads 8 --wal-stats 3 --server-stats 3 --data-dir /data/wal/0" + }, + { + "label": "memory", + "args": "--durability memory --worker-threads 8 --server-stats 3 --data-dir /data/wal/0 --tier off" + } + ] + }, + "stream_counts": [ + 10000, + 100000 + ], + "pod_ladder": { + "10000": [ + 4, + 8 + ], + "100000": [ + 4, + 8 + ] + } +} \ No newline at end of file From 1c67ad0ac6fbdec32c230045f76dce9fa347b12f Mon Sep 17 00:00:00 2001 From: Valter Balegas Date: Mon, 13 Jul 2026 23:39:04 +0100 Subject: [PATCH 6/8] =?UTF-8?q?bench:=20canonical-only=20=E2=80=94=20delet?= =?UTF-8?q?e=20legacy=20suites=20+=20all=20previous=20results;=20REPORT=5F?= =?UTF-8?q?TEMPLATE.md?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - suites/: only the 7 canonical suites remain (write, write-ursula, reads-catchup, reads-sse, mixed-{cal,writes,delivery}), matching exactly the workloads we report (template: the 2026-07-02 report structure). Long-poll and sustained are not in the reported set and were dropped. - results/ + results-2026-07-02/: deleted (git history keeps them). The 2026-07-02 write numbers were later found inflated (physics-impossible 2.05M @500k on 4 CPUs, wal > memory) — superseded by canonical references. - REPORT_TEMPLATE.md: the report skeleton + a pre-publication caveat checklist encoding every past numbers-inflation failure (storage layout, window alignment, ladder-vs-plateau, digest provenance, physics sanity, cardinality shape, client-bound cells). - AGENTS.md: §0/§1 aligned to the canonical set. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01Y3x7bcT9vLGiT4tXZeQpnk --- AGENTS.md | 23 +- REPORT_TEMPLATE.md | 72 +++ results-2026-07-02/PROVENANCE.md | 47 -- results-2026-07-02/REPORT.md | 149 ------ results-2026-07-02/mixed-cal/aggregate.csv | 2 - results-2026-07-02/mixed-cal/aggregate.json | 28 -- results-2026-07-02/mixed-cal/report.md | 13 - results-2026-07-02/mixed-cal/wal/cells.json | 40 -- .../mixed-delivery/aggregate.csv | 11 - .../mixed-delivery/aggregate.json | 262 ----------- .../mixed-delivery/memory/cells.json | 156 ------- results-2026-07-02/mixed-delivery/report.md | 27 -- .../mixed-delivery/wal/cells.json | 156 ------- .../mixed-writes-hot/aggregate.csv | 4 - .../mixed-writes-hot/aggregate.json | 80 ---- results-2026-07-02/mixed-writes-hot/report.md | 15 - .../mixed-writes-hot/wal/cells.json | 98 ---- results-2026-07-02/mixed-writes/aggregate.csv | 5 - .../mixed-writes/aggregate.json | 106 ----- results-2026-07-02/mixed-writes/report.md | 16 - .../mixed-writes/wal/cells.json | 127 ----- .../reads-catchup/aggregate.csv | 17 - .../reads-catchup/aggregate.json | 226 --------- results-2026-07-02/reads-catchup/report.md | 29 -- .../reads-catchup/ursula/cells.json | 106 ----- .../reads-catchup/wal/cells.json | 106 ----- .../reads-sse-remote/aggregate.csv | 17 - .../reads-sse-remote/aggregate.json | 226 --------- results-2026-07-02/reads-sse-remote/report.md | 29 -- .../reads-sse-remote/ursula/cells.json | 106 ----- .../reads-sse-remote/wal/cells.json | 106 ----- results-2026-07-02/run-durable/aggregate.csv | 19 - results-2026-07-02/run-durable/aggregate.json | 432 ------------------ .../run-durable/memory/cells.json | 146 ------ results-2026-07-02/run-durable/report.md | 56 --- .../run-durable/wal-tailcache/cells.json | 146 ------ results-2026-07-02/run-durable/wal/cells.json | 150 ------ results-2026-07-02/run-node/aggregate.csv | 4 - results-2026-07-02/run-node/aggregate.json | 75 --- results-2026-07-02/run-node/node/cells.json | 77 ---- results-2026-07-02/run-node/report.md | 35 -- results-2026-07-02/run-s2/aggregate.csv | 3 - results-2026-07-02/run-s2/aggregate.json | 48 -- results-2026-07-02/run-s2/report.md | 30 -- results-2026-07-02/run-s2/s2/cells.json | 49 -- results-2026-07-02/run-ursula/aggregate.csv | 7 - results-2026-07-02/run-ursula/aggregate.json | 152 ------ results-2026-07-02/run-ursula/report.md | 38 -- .../run-ursula/ursula-disk/cells.json | 77 ---- .../run-ursula/ursula-memory/cells.json | 81 ---- results-2026-07-02/sse-comparison.csv | 13 - results-2026-07-02/sse-comparison.md | 42 -- results-2026-07-02/sse-raw/sse-comparison.csv | 13 - results-2026-07-02/sse-raw/sse-comparison.md | 42 -- results/mixed-cal-local/aggregate.csv | 3 - results/mixed-cal-local/aggregate.json | 54 --- results/mixed-cal-local/memory/cells.json | 40 -- results/mixed-cal-local/report.md | 19 - results/mixed-cal-local/wal/cells.json | 40 -- results/mixed-cal/aggregate.csv | 2 - results/mixed-cal/aggregate.json | 28 -- results/mixed-cal/report.md | 13 - results/mixed-cal/wal/cells.json | 40 -- results/mixed-delivery-local/aggregate.csv | 11 - results/mixed-delivery-local/aggregate.json | 262 ----------- .../mixed-delivery-local/memory/cells.json | 156 ------- results/mixed-delivery-local/report.md | 27 -- results/mixed-delivery-local/wal/cells.json | 156 ------- results/mixed-delivery/aggregate.csv | 11 - results/mixed-delivery/aggregate.json | 262 ----------- results/mixed-delivery/memory/cells.json | 156 ------- results/mixed-delivery/report.md | 27 -- results/mixed-delivery/wal/cells.json | 156 ------- results/mixed-writes-hot/aggregate.csv | 4 - results/mixed-writes-hot/aggregate.json | 80 ---- results/mixed-writes-hot/report.md | 15 - results/mixed-writes-hot/wal/cells.json | 98 ---- results/mixed-writes-local/FINDINGS.md | 60 --- results/mixed-writes-local/aggregate.csv | 6 - results/mixed-writes-local/aggregate.json | 132 ------ results/mixed-writes-local/report.md | 17 - results/mixed-writes-local/wal/cells.json | 156 ------- results/mixed-writes/aggregate.csv | 5 - results/mixed-writes/aggregate.json | 106 ----- results/mixed-writes/report.md | 16 - results/mixed-writes/wal/cells.json | 127 ----- results/reads-catchup/aggregate.csv | 17 - results/reads-catchup/aggregate.json | 226 --------- results/reads-catchup/report.md | 29 -- results/reads-catchup/ursula/cells.json | 106 ----- results/reads-catchup/wal/cells.json | 106 ----- results/reads-sse-remote/aggregate.csv | 17 - results/reads-sse-remote/aggregate.json | 226 --------- results/reads-sse-remote/report.md | 29 -- results/reads-sse-remote/ursula/cells.json | 106 ----- results/reads-sse-remote/wal/cells.json | 106 ----- results/run-durable/aggregate.csv | 19 - results/run-durable/aggregate.json | 432 ------------------ results/run-durable/memory/cells.json | 146 ------ results/run-durable/report.md | 56 --- results/run-durable/wal-tailcache/cells.json | 146 ------ results/run-durable/wal/cells.json | 150 ------ results/run-node/aggregate.csv | 4 - results/run-node/aggregate.json | 75 --- results/run-node/node/cells.json | 77 ---- results/run-node/report.md | 35 -- results/run-s2/aggregate.csv | 3 - results/run-s2/aggregate.json | 48 -- results/run-s2/report.md | 30 -- results/run-s2/s2/cells.json | 49 -- results/run-ursula/aggregate.csv | 7 - results/run-ursula/aggregate.json | 152 ------ results/run-ursula/report.md | 38 -- results/run-ursula/ursula-disk/cells.json | 77 ---- results/run-ursula/ursula-memory/cells.json | 81 ---- results/sse-comparison.csv | 13 - results/sse-comparison.md | 42 -- results/wal-1m-diag/aggregate.csv | 3 - results/wal-1m-diag/aggregate.json | 52 --- results/wal-1m-diag/report.md | 41 -- results/wal-1m-diag/stacked/cells.json | 46 -- results/wal-batched-mw-local/aggregate.csv | 3 - results/wal-batched-mw-local/aggregate.json | 76 --- .../wal-batched-mw-local/baseline/cells.json | 37 -- .../batched-mw/cells.json | 37 -- results/wal-batched-mw-local/report.md | 38 -- results/wal-checkpoint-fix-100k/aggregate.csv | 3 - .../wal-checkpoint-fix-100k/aggregate.json | 64 --- .../wal-checkpoint-fix-100k/both/cells.json | 30 -- results/wal-checkpoint-fix-100k/report.md | 44 -- .../wal-checkpoint-fix-100k/syncfs/cells.json | 30 -- .../wal-checkpoint-fix-ab.run1/aggregate.csv | 9 - .../wal-checkpoint-fix-ab.run1/aggregate.json | 304 ------------ .../baseline/cells.json | 74 --- .../both/cells.json | 68 --- results/wal-checkpoint-fix-ab.run1/report.md | 47 -- .../stagger/cells.json | 68 --- .../syncfs/cells.json | 69 --- .../wal-checkpoint-fix-ab/baseline/cells.json | 30 -- results/wal-cpubind/aggregate.csv | 3 - results/wal-cpubind/aggregate.json | 70 --- results/wal-cpubind/bound-3s/cells.json | 64 --- results/wal-cpubind/report.md | 41 -- results/wal-decomp-lane0/aggregate.csv | 9 - results/wal-decomp-lane0/aggregate.json | 250 ---------- results/wal-decomp-lane0/ckpt-off/cells.json | 58 --- results/wal-decomp-lane0/memory/cells.json | 58 --- results/wal-decomp-lane0/nofsync/cells.json | 58 --- results/wal-decomp-lane0/ref-3s/cells.json | 58 --- results/wal-decomp-lane0/report.md | 47 -- results/wal-fanout-sweep/aggregate.csv | 5 - results/wal-fanout-sweep/aggregate.json | 192 -------- results/wal-fanout-sweep/report.md | 40 -- results/wal-fanout-sweep/s4-f1/cells.json | 49 -- results/wal-fanout-sweep/s4-f2/cells.json | 43 -- results/wal-fanout-sweep/s4-f4/cells.json | 49 -- results/wal-fanout-sweep/s4-f8/cells.json | 49 -- results/wal-fsync-diag-local/aggregate.csv | 7 - results/wal-fsync-diag-local/aggregate.json | 200 -------- .../wal-fsync-diag-local/memory/cells.json | 58 --- results/wal-fsync-diag-local/report.md | 45 -- .../wal-nofsync/cells.json | 64 --- .../wal-normal/cells.json | 64 --- .../wal-lazy-checkpoint-local/aggregate.csv | 9 - .../wal-lazy-checkpoint-local/aggregate.json | 274 ----------- .../lazy-syncfs-30s/cells.json | 64 --- .../lazy-syncfs-60s/cells.json | 64 --- .../ref-baseline-3s/cells.json | 64 --- .../ref-syncfs-3s/cells.json | 64 --- results/wal-lazy-checkpoint-local/report.md | 47 -- .../lazy-120s/cells.json | 36 -- .../lazy-30s/cells.json | 68 --- .../lazy-60s/cells.json | 68 --- .../ref-syncfs-3s/cells.json | 68 --- .../aggregate.csv | 2 - .../aggregate.json | 39 -- .../wal-machinery-baseline-local/report.md | 37 -- .../wal-baseline/cells.json | 37 -- results/wal-mem-diag-local/aggregate.csv | 5 - results/wal-mem-diag-local/aggregate.json | 126 ----- results/wal-mem-diag-local/memory/cells.json | 58 --- results/wal-mem-diag-local/report.md | 43 -- .../wal-mem-diag-local/wal-syncfs/cells.json | 58 --- results/wal-multilane-shards/s1/cells.json | 170 ------- results/wal-multilane-shards/s2/cells.json | 138 ------ results/wal-multilane-shards/s4/cells.json | 144 ------ results/wal-multilane-shards/s6/cells.json | 36 -- results/wal-shard-sweep/aggregate.csv | 6 - results/wal-shard-sweep/aggregate.json | 271 ----------- results/wal-shard-sweep/report.md | 41 -- results/wal-shard-sweep/wal-s1/cells.json | 55 --- results/wal-shard-sweep/wal-s16/cells.json | 55 --- results/wal-shard-sweep/wal-s24/cells.json | 55 --- results/wal-shard-sweep/wal-s4/cells.json | 55 --- results/wal-shard-sweep/wal-s8/cells.json | 49 -- results/wal-sizetrigger/aggregate.csv | 5 - results/wal-sizetrigger/aggregate.json | 138 ------ results/wal-sizetrigger/ref-3s/cells.json | 64 --- results/wal-sizetrigger/report.md | 43 -- results/wal-sizetrigger/size-1g/cells.json | 64 --- results/wal-splitlane/aggregate.csv | 7 - results/wal-splitlane/aggregate.json | 206 --------- results/wal-splitlane/ckpt-off/cells.json | 64 --- results/wal-splitlane/nofsync/cells.json | 64 --- results/wal-splitlane/ref-3s/cells.json | 64 --- results/wal-splitlane/report.md | 53 --- results/wal-stacked-1m/aggregate.csv | 4 - results/wal-stacked-1m/aggregate.json | 83 ---- results/wal-stacked-1m/report.md | 47 -- results/wal-stacked-1m/stacked/cells.json | 73 --- results/wal-streamlanes-1m/aggregate.csv | 4 - results/wal-streamlanes-1m/aggregate.json | 77 ---- .../wal-streamlanes-1m/lanes3x3/cells.json | 67 --- results/wal-streamlanes-1m/report.md | 45 -- results/wal-syncfs-local/aggregate.csv | 5 - results/wal-syncfs-local/aggregate.json | 138 ------ results/wal-syncfs-local/baseline/cells.json | 64 --- results/wal-syncfs-local/report.md | 43 -- results/wal-syncfs-local/syncfs/cells.json | 64 --- results/write-accuracy-local/aggregate.csv | 3 - results/write-accuracy-local/aggregate.json | 48 -- .../write-accuracy-local/memory/cells.json | 27 -- results/write-accuracy-local/report.md | 30 -- results/write-accuracy-local/wal/cells.json | 27 -- results/write-cliff-local-cpu4/aggregate.csv | 3 - results/write-cliff-local-cpu4/aggregate.json | 76 --- results/write-cliff-local-cpu4/report.md | 41 -- .../wal-cpu4/cells.json | 70 --- results/write-cliff-local/FINDINGS.md | 63 --- results/write-cliff-local/aggregate.csv | 10 - results/write-cliff-local/aggregate.json | 317 ------------- results/write-cliff-local/memory/cells.json | 118 ----- results/write-cliff-local/report.md | 51 --- results/write-cliff-local/wal/cells.json | 169 ------- .../write-cliff-nvme/memory-gated/cells.json | 80 ---- .../write-cliff-nvme/memory-nogate/cells.json | 86 ---- .../write-cliff-nvme/wal-fsync1/cells.json | 104 ----- .../write-cliff-nvme/wal-fsync16/cells.json | 54 --- .../write-nvme-bottleneck/memory/cells.json | 48 -- .../write-nvme-bottleneck/wal-s4/cells.json | 36 -- results/write-wal-vs-mem-cpu4/FINDINGS.md | 82 ---- results/write-wal-vs-mem-cpu4/aggregate.csv | 5 - results/write-wal-vs-mem-cpu4/aggregate.json | 144 ------ .../write-wal-vs-mem-cpu4/memory/cells.json | 70 --- results/write-wal-vs-mem-cpu4/report.md | 43 -- results/write-wal-vs-mem-cpu4/wal/cells.json | 64 --- results/write-wal-vs-mem-cpu8/FINDINGS.md | 10 - results/write-wal-vs-mem-cpu8/aggregate.csv | 5 - results/write-wal-vs-mem-cpu8/aggregate.json | 150 ------ .../write-wal-vs-mem-cpu8/memory/cells.json | 82 ---- results/write-wal-vs-mem-cpu8/report.md | 43 -- results/write-wal-vs-mem-cpu8/wal/cells.json | 58 --- ...ites-hot.json => canonical-mixed-cal.json} | 18 +- ...ery.json => canonical-mixed-delivery.json} | 11 +- ...rites.json => canonical-mixed-writes.json} | 9 +- suites/canonical-reads-live.json | 39 -- suites/canonical-sustained.json | 44 -- suites/catchup-durable.json | 16 - suites/catchup-local.json | 8 - suites/catchup-node.json | 15 - suites/catchup-s2.json | 15 - suites/catchup-ursula.json | 15 - suites/mixed-cal-local.json | 36 -- suites/mixed-cal.json | 24 - suites/mixed-delivery-local.json | 40 -- suites/mixed-writes-local.json | 29 -- suites/reads-catchup.json | 23 - suites/reads-local-ursula.json | 17 - suites/reads-local.json | 17 - suites/reads-longpoll.json | 24 - suites/reads-sse-local.json | 24 - suites/reads-sse-remote.json | 24 - suites/reads.json | 24 - suites/run-durable-500k-sat.json | 28 -- suites/run-durable-500k-sat2.json | 28 -- suites/run-durable-cal.json | 30 -- suites/run-durable-cpu16-1m-card-stats.json | 39 -- suites/run-durable-cpu16-1m-card.json | 43 -- suites/run-durable-cpu16-1m-p99-combined.json | 39 -- suites/run-durable-cpu16-clean.json | 29 -- suites/run-durable-cpu16.json | 29 -- suites/run-durable-cpu4-clean.json | 29 -- suites/run-durable-cpu4-pool-base.json | 49 -- suites/run-durable-cpu4-pool-combined.json | 49 -- suites/run-durable-cpu4-sat-base.json | 51 --- suites/run-durable-cpu4-sat-combined.json | 51 --- suites/run-durable-cpu4-stats-base.json | 43 -- suites/run-durable-cpu4-stats-combined.json | 43 -- suites/run-durable-cpu8-1m-p99-combined.json | 39 -- suites/run-durable-cpu8-clean.json | 29 -- suites/run-durable-cpu8.json | 29 -- suites/run-durable-dense-local.json | 13 - suites/run-durable-dense.json | 29 -- suites/run-durable-pool-barrier.json | 36 -- suites/run-durable-pool-opt.json | 32 -- suites/run-durable-pool.json | 31 -- suites/run-durable-pool2-combined.json | 47 -- suites/run-durable-pool2.json | 31 -- suites/run-durable-tune.json | 31 -- suites/run-durable.json | 77 ---- suites/run-node.json | 25 - suites/run-s2.json | 31 -- suites/run-ursula.json | 31 -- suites/sustained-local-base.json | 23 - suites/sustained-local-fix.json | 23 - suites/sustained.json | 30 -- suites/wal-1m-diag.json | 47 -- suites/wal-batched-mw-local.json | 42 -- suites/wal-checkpoint-fix-100k.json | 48 -- suites/wal-checkpoint-fix-ab.json | 63 --- suites/wal-cpubind.json | 50 -- suites/wal-decomp-lane0.json | 61 --- suites/wal-fanout-sweep.json | 59 --- suites/wal-fsync-diag-local.json | 51 --- suites/wal-lazy-checkpoint-local.json | 55 --- suites/wal-lazy-checkpoint-multilane.json | 63 --- suites/wal-machinery-baseline-local.json | 38 -- suites/wal-mem-diag-local.json | 47 -- suites/wal-multilane-shards.json | 79 ---- suites/wal-shard-sweep.json | 64 --- suites/wal-sizetrigger.json | 54 --- suites/wal-splitlane.json | 58 --- suites/wal-stacked-1m.json | 52 --- suites/wal-streamlanes-1m.json | 51 --- suites/wal-syncfs-local.json | 47 -- suites/wal-syncfs-nvme.json | 57 --- suites/write-accuracy-local.json | 24 - suites/write-cliff-local-cpu4.json | 44 -- suites/write-cliff-local.json | 55 --- suites/write-cliff-nvme.json | 55 --- suites/write-nvme-bottleneck.json | 60 --- suites/write-throughput-local.json | 8 - suites/write-wal-vs-mem-cpu4.json | 67 --- suites/write-wal-vs-mem-cpu8.json | 69 --- 334 files changed, 106 insertions(+), 20837 deletions(-) create mode 100644 REPORT_TEMPLATE.md delete mode 100644 results-2026-07-02/PROVENANCE.md delete mode 100644 results-2026-07-02/REPORT.md delete mode 100644 results-2026-07-02/mixed-cal/aggregate.csv delete mode 100644 results-2026-07-02/mixed-cal/aggregate.json delete mode 100644 results-2026-07-02/mixed-cal/report.md delete mode 100644 results-2026-07-02/mixed-cal/wal/cells.json delete mode 100644 results-2026-07-02/mixed-delivery/aggregate.csv delete mode 100644 results-2026-07-02/mixed-delivery/aggregate.json delete mode 100644 results-2026-07-02/mixed-delivery/memory/cells.json delete mode 100644 results-2026-07-02/mixed-delivery/report.md delete mode 100644 results-2026-07-02/mixed-delivery/wal/cells.json delete mode 100644 results-2026-07-02/mixed-writes-hot/aggregate.csv delete mode 100644 results-2026-07-02/mixed-writes-hot/aggregate.json delete mode 100644 results-2026-07-02/mixed-writes-hot/report.md delete mode 100644 results-2026-07-02/mixed-writes-hot/wal/cells.json delete mode 100644 results-2026-07-02/mixed-writes/aggregate.csv delete mode 100644 results-2026-07-02/mixed-writes/aggregate.json delete mode 100644 results-2026-07-02/mixed-writes/report.md delete mode 100644 results-2026-07-02/mixed-writes/wal/cells.json delete mode 100644 results-2026-07-02/reads-catchup/aggregate.csv delete mode 100644 results-2026-07-02/reads-catchup/aggregate.json delete mode 100644 results-2026-07-02/reads-catchup/report.md delete mode 100644 results-2026-07-02/reads-catchup/ursula/cells.json delete mode 100644 results-2026-07-02/reads-catchup/wal/cells.json delete mode 100644 results-2026-07-02/reads-sse-remote/aggregate.csv delete mode 100644 results-2026-07-02/reads-sse-remote/aggregate.json delete mode 100644 results-2026-07-02/reads-sse-remote/report.md delete mode 100644 results-2026-07-02/reads-sse-remote/ursula/cells.json delete mode 100644 results-2026-07-02/reads-sse-remote/wal/cells.json delete mode 100644 results-2026-07-02/run-durable/aggregate.csv delete mode 100644 results-2026-07-02/run-durable/aggregate.json delete mode 100644 results-2026-07-02/run-durable/memory/cells.json delete mode 100644 results-2026-07-02/run-durable/report.md delete mode 100644 results-2026-07-02/run-durable/wal-tailcache/cells.json delete mode 100644 results-2026-07-02/run-durable/wal/cells.json delete mode 100644 results-2026-07-02/run-node/aggregate.csv delete mode 100644 results-2026-07-02/run-node/aggregate.json delete mode 100644 results-2026-07-02/run-node/node/cells.json delete mode 100644 results-2026-07-02/run-node/report.md delete mode 100644 results-2026-07-02/run-s2/aggregate.csv delete mode 100644 results-2026-07-02/run-s2/aggregate.json delete mode 100644 results-2026-07-02/run-s2/report.md delete mode 100644 results-2026-07-02/run-s2/s2/cells.json delete mode 100644 results-2026-07-02/run-ursula/aggregate.csv delete mode 100644 results-2026-07-02/run-ursula/aggregate.json delete mode 100644 results-2026-07-02/run-ursula/report.md delete mode 100644 results-2026-07-02/run-ursula/ursula-disk/cells.json delete mode 100644 results-2026-07-02/run-ursula/ursula-memory/cells.json delete mode 100644 results-2026-07-02/sse-comparison.csv delete mode 100644 results-2026-07-02/sse-comparison.md delete mode 100644 results-2026-07-02/sse-raw/sse-comparison.csv delete mode 100644 results-2026-07-02/sse-raw/sse-comparison.md delete mode 100644 results/mixed-cal-local/aggregate.csv delete mode 100644 results/mixed-cal-local/aggregate.json delete mode 100644 results/mixed-cal-local/memory/cells.json delete mode 100644 results/mixed-cal-local/report.md delete mode 100644 results/mixed-cal-local/wal/cells.json delete mode 100644 results/mixed-cal/aggregate.csv delete mode 100644 results/mixed-cal/aggregate.json delete mode 100644 results/mixed-cal/report.md delete mode 100644 results/mixed-cal/wal/cells.json delete mode 100644 results/mixed-delivery-local/aggregate.csv delete mode 100644 results/mixed-delivery-local/aggregate.json delete mode 100644 results/mixed-delivery-local/memory/cells.json delete mode 100644 results/mixed-delivery-local/report.md delete mode 100644 results/mixed-delivery-local/wal/cells.json delete mode 100644 results/mixed-delivery/aggregate.csv delete mode 100644 results/mixed-delivery/aggregate.json delete mode 100644 results/mixed-delivery/memory/cells.json delete mode 100644 results/mixed-delivery/report.md delete mode 100644 results/mixed-delivery/wal/cells.json delete mode 100644 results/mixed-writes-hot/aggregate.csv delete mode 100644 results/mixed-writes-hot/aggregate.json delete mode 100644 results/mixed-writes-hot/report.md delete mode 100644 results/mixed-writes-hot/wal/cells.json delete mode 100644 results/mixed-writes-local/FINDINGS.md delete mode 100644 results/mixed-writes-local/aggregate.csv delete mode 100644 results/mixed-writes-local/aggregate.json delete mode 100644 results/mixed-writes-local/report.md delete mode 100644 results/mixed-writes-local/wal/cells.json delete mode 100644 results/mixed-writes/aggregate.csv delete mode 100644 results/mixed-writes/aggregate.json delete mode 100644 results/mixed-writes/report.md delete mode 100644 results/mixed-writes/wal/cells.json delete mode 100644 results/reads-catchup/aggregate.csv delete mode 100644 results/reads-catchup/aggregate.json delete mode 100644 results/reads-catchup/report.md delete mode 100644 results/reads-catchup/ursula/cells.json delete mode 100644 results/reads-catchup/wal/cells.json delete mode 100644 results/reads-sse-remote/aggregate.csv delete mode 100644 results/reads-sse-remote/aggregate.json delete mode 100644 results/reads-sse-remote/report.md delete mode 100644 results/reads-sse-remote/ursula/cells.json delete mode 100644 results/reads-sse-remote/wal/cells.json delete mode 100644 results/run-durable/aggregate.csv delete mode 100644 results/run-durable/aggregate.json delete mode 100644 results/run-durable/memory/cells.json delete mode 100644 results/run-durable/report.md delete mode 100644 results/run-durable/wal-tailcache/cells.json delete mode 100644 results/run-durable/wal/cells.json delete mode 100644 results/run-node/aggregate.csv delete mode 100644 results/run-node/aggregate.json delete mode 100644 results/run-node/node/cells.json delete mode 100644 results/run-node/report.md delete mode 100644 results/run-s2/aggregate.csv delete mode 100644 results/run-s2/aggregate.json delete mode 100644 results/run-s2/report.md delete mode 100644 results/run-s2/s2/cells.json delete mode 100644 results/run-ursula/aggregate.csv delete mode 100644 results/run-ursula/aggregate.json delete mode 100644 results/run-ursula/report.md delete mode 100644 results/run-ursula/ursula-disk/cells.json delete mode 100644 results/run-ursula/ursula-memory/cells.json delete mode 100644 results/sse-comparison.csv delete mode 100644 results/sse-comparison.md delete mode 100644 results/wal-1m-diag/aggregate.csv delete mode 100644 results/wal-1m-diag/aggregate.json delete mode 100644 results/wal-1m-diag/report.md delete mode 100644 results/wal-1m-diag/stacked/cells.json delete mode 100644 results/wal-batched-mw-local/aggregate.csv delete mode 100644 results/wal-batched-mw-local/aggregate.json delete mode 100644 results/wal-batched-mw-local/baseline/cells.json delete mode 100644 results/wal-batched-mw-local/batched-mw/cells.json delete mode 100644 results/wal-batched-mw-local/report.md delete mode 100644 results/wal-checkpoint-fix-100k/aggregate.csv delete mode 100644 results/wal-checkpoint-fix-100k/aggregate.json delete mode 100644 results/wal-checkpoint-fix-100k/both/cells.json delete mode 100644 results/wal-checkpoint-fix-100k/report.md delete mode 100644 results/wal-checkpoint-fix-100k/syncfs/cells.json delete mode 100644 results/wal-checkpoint-fix-ab.run1/aggregate.csv delete mode 100644 results/wal-checkpoint-fix-ab.run1/aggregate.json delete mode 100644 results/wal-checkpoint-fix-ab.run1/baseline/cells.json delete mode 100644 results/wal-checkpoint-fix-ab.run1/both/cells.json delete mode 100644 results/wal-checkpoint-fix-ab.run1/report.md delete mode 100644 results/wal-checkpoint-fix-ab.run1/stagger/cells.json delete mode 100644 results/wal-checkpoint-fix-ab.run1/syncfs/cells.json delete mode 100644 results/wal-checkpoint-fix-ab/baseline/cells.json delete mode 100644 results/wal-cpubind/aggregate.csv delete mode 100644 results/wal-cpubind/aggregate.json delete mode 100644 results/wal-cpubind/bound-3s/cells.json delete mode 100644 results/wal-cpubind/report.md delete mode 100644 results/wal-decomp-lane0/aggregate.csv delete mode 100644 results/wal-decomp-lane0/aggregate.json delete mode 100644 results/wal-decomp-lane0/ckpt-off/cells.json delete mode 100644 results/wal-decomp-lane0/memory/cells.json delete mode 100644 results/wal-decomp-lane0/nofsync/cells.json delete mode 100644 results/wal-decomp-lane0/ref-3s/cells.json delete mode 100644 results/wal-decomp-lane0/report.md delete mode 100644 results/wal-fanout-sweep/aggregate.csv delete mode 100644 results/wal-fanout-sweep/aggregate.json delete mode 100644 results/wal-fanout-sweep/report.md delete mode 100644 results/wal-fanout-sweep/s4-f1/cells.json delete mode 100644 results/wal-fanout-sweep/s4-f2/cells.json delete mode 100644 results/wal-fanout-sweep/s4-f4/cells.json delete mode 100644 results/wal-fanout-sweep/s4-f8/cells.json delete mode 100644 results/wal-fsync-diag-local/aggregate.csv delete mode 100644 results/wal-fsync-diag-local/aggregate.json delete mode 100644 results/wal-fsync-diag-local/memory/cells.json delete mode 100644 results/wal-fsync-diag-local/report.md delete mode 100644 results/wal-fsync-diag-local/wal-nofsync/cells.json delete mode 100644 results/wal-fsync-diag-local/wal-normal/cells.json delete mode 100644 results/wal-lazy-checkpoint-local/aggregate.csv delete mode 100644 results/wal-lazy-checkpoint-local/aggregate.json delete mode 100644 results/wal-lazy-checkpoint-local/lazy-syncfs-30s/cells.json delete mode 100644 results/wal-lazy-checkpoint-local/lazy-syncfs-60s/cells.json delete mode 100644 results/wal-lazy-checkpoint-local/ref-baseline-3s/cells.json delete mode 100644 results/wal-lazy-checkpoint-local/ref-syncfs-3s/cells.json delete mode 100644 results/wal-lazy-checkpoint-local/report.md delete mode 100644 results/wal-lazy-checkpoint-multilane/lazy-120s/cells.json delete mode 100644 results/wal-lazy-checkpoint-multilane/lazy-30s/cells.json delete mode 100644 results/wal-lazy-checkpoint-multilane/lazy-60s/cells.json delete mode 100644 results/wal-lazy-checkpoint-multilane/ref-syncfs-3s/cells.json delete mode 100644 results/wal-machinery-baseline-local/aggregate.csv delete mode 100644 results/wal-machinery-baseline-local/aggregate.json delete mode 100644 results/wal-machinery-baseline-local/report.md delete mode 100644 results/wal-machinery-baseline-local/wal-baseline/cells.json delete mode 100644 results/wal-mem-diag-local/aggregate.csv delete mode 100644 results/wal-mem-diag-local/aggregate.json delete mode 100644 results/wal-mem-diag-local/memory/cells.json delete mode 100644 results/wal-mem-diag-local/report.md delete mode 100644 results/wal-mem-diag-local/wal-syncfs/cells.json delete mode 100644 results/wal-multilane-shards/s1/cells.json delete mode 100644 results/wal-multilane-shards/s2/cells.json delete mode 100644 results/wal-multilane-shards/s4/cells.json delete mode 100644 results/wal-multilane-shards/s6/cells.json delete mode 100644 results/wal-shard-sweep/aggregate.csv delete mode 100644 results/wal-shard-sweep/aggregate.json delete mode 100644 results/wal-shard-sweep/report.md delete mode 100644 results/wal-shard-sweep/wal-s1/cells.json delete mode 100644 results/wal-shard-sweep/wal-s16/cells.json delete mode 100644 results/wal-shard-sweep/wal-s24/cells.json delete mode 100644 results/wal-shard-sweep/wal-s4/cells.json delete mode 100644 results/wal-shard-sweep/wal-s8/cells.json delete mode 100644 results/wal-sizetrigger/aggregate.csv delete mode 100644 results/wal-sizetrigger/aggregate.json delete mode 100644 results/wal-sizetrigger/ref-3s/cells.json delete mode 100644 results/wal-sizetrigger/report.md delete mode 100644 results/wal-sizetrigger/size-1g/cells.json delete mode 100644 results/wal-splitlane/aggregate.csv delete mode 100644 results/wal-splitlane/aggregate.json delete mode 100644 results/wal-splitlane/ckpt-off/cells.json delete mode 100644 results/wal-splitlane/nofsync/cells.json delete mode 100644 results/wal-splitlane/ref-3s/cells.json delete mode 100644 results/wal-splitlane/report.md delete mode 100644 results/wal-stacked-1m/aggregate.csv delete mode 100644 results/wal-stacked-1m/aggregate.json delete mode 100644 results/wal-stacked-1m/report.md delete mode 100644 results/wal-stacked-1m/stacked/cells.json delete mode 100644 results/wal-streamlanes-1m/aggregate.csv delete mode 100644 results/wal-streamlanes-1m/aggregate.json delete mode 100644 results/wal-streamlanes-1m/lanes3x3/cells.json delete mode 100644 results/wal-streamlanes-1m/report.md delete mode 100644 results/wal-syncfs-local/aggregate.csv delete mode 100644 results/wal-syncfs-local/aggregate.json delete mode 100644 results/wal-syncfs-local/baseline/cells.json delete mode 100644 results/wal-syncfs-local/report.md delete mode 100644 results/wal-syncfs-local/syncfs/cells.json delete mode 100644 results/write-accuracy-local/aggregate.csv delete mode 100644 results/write-accuracy-local/aggregate.json delete mode 100644 results/write-accuracy-local/memory/cells.json delete mode 100644 results/write-accuracy-local/report.md delete mode 100644 results/write-accuracy-local/wal/cells.json delete mode 100644 results/write-cliff-local-cpu4/aggregate.csv delete mode 100644 results/write-cliff-local-cpu4/aggregate.json delete mode 100644 results/write-cliff-local-cpu4/report.md delete mode 100644 results/write-cliff-local-cpu4/wal-cpu4/cells.json delete mode 100644 results/write-cliff-local/FINDINGS.md delete mode 100644 results/write-cliff-local/aggregate.csv delete mode 100644 results/write-cliff-local/aggregate.json delete mode 100644 results/write-cliff-local/memory/cells.json delete mode 100644 results/write-cliff-local/report.md delete mode 100644 results/write-cliff-local/wal/cells.json delete mode 100644 results/write-cliff-nvme/memory-gated/cells.json delete mode 100644 results/write-cliff-nvme/memory-nogate/cells.json delete mode 100644 results/write-cliff-nvme/wal-fsync1/cells.json delete mode 100644 results/write-cliff-nvme/wal-fsync16/cells.json delete mode 100644 results/write-nvme-bottleneck/memory/cells.json delete mode 100644 results/write-nvme-bottleneck/wal-s4/cells.json delete mode 100644 results/write-wal-vs-mem-cpu4/FINDINGS.md delete mode 100644 results/write-wal-vs-mem-cpu4/aggregate.csv delete mode 100644 results/write-wal-vs-mem-cpu4/aggregate.json delete mode 100644 results/write-wal-vs-mem-cpu4/memory/cells.json delete mode 100644 results/write-wal-vs-mem-cpu4/report.md delete mode 100644 results/write-wal-vs-mem-cpu4/wal/cells.json delete mode 100644 results/write-wal-vs-mem-cpu8/FINDINGS.md delete mode 100644 results/write-wal-vs-mem-cpu8/aggregate.csv delete mode 100644 results/write-wal-vs-mem-cpu8/aggregate.json delete mode 100644 results/write-wal-vs-mem-cpu8/memory/cells.json delete mode 100644 results/write-wal-vs-mem-cpu8/report.md delete mode 100644 results/write-wal-vs-mem-cpu8/wal/cells.json rename suites/{mixed-writes-hot.json => canonical-mixed-cal.json} (58%) rename suites/{mixed-delivery.json => canonical-mixed-delivery.json} (71%) rename suites/{mixed-writes.json => canonical-mixed-writes.json} (72%) delete mode 100644 suites/canonical-reads-live.json delete mode 100644 suites/canonical-sustained.json delete mode 100644 suites/catchup-durable.json delete mode 100644 suites/catchup-local.json delete mode 100644 suites/catchup-node.json delete mode 100644 suites/catchup-s2.json delete mode 100644 suites/catchup-ursula.json delete mode 100644 suites/mixed-cal-local.json delete mode 100644 suites/mixed-cal.json delete mode 100644 suites/mixed-delivery-local.json delete mode 100644 suites/mixed-writes-local.json delete mode 100644 suites/reads-catchup.json delete mode 100644 suites/reads-local-ursula.json delete mode 100644 suites/reads-local.json delete mode 100644 suites/reads-longpoll.json delete mode 100644 suites/reads-sse-local.json delete mode 100644 suites/reads-sse-remote.json delete mode 100644 suites/reads.json delete mode 100644 suites/run-durable-500k-sat.json delete mode 100644 suites/run-durable-500k-sat2.json delete mode 100644 suites/run-durable-cal.json delete mode 100644 suites/run-durable-cpu16-1m-card-stats.json delete mode 100644 suites/run-durable-cpu16-1m-card.json delete mode 100644 suites/run-durable-cpu16-1m-p99-combined.json delete mode 100644 suites/run-durable-cpu16-clean.json delete mode 100644 suites/run-durable-cpu16.json delete mode 100644 suites/run-durable-cpu4-clean.json delete mode 100644 suites/run-durable-cpu4-pool-base.json delete mode 100644 suites/run-durable-cpu4-pool-combined.json delete mode 100644 suites/run-durable-cpu4-sat-base.json delete mode 100644 suites/run-durable-cpu4-sat-combined.json delete mode 100644 suites/run-durable-cpu4-stats-base.json delete mode 100644 suites/run-durable-cpu4-stats-combined.json delete mode 100644 suites/run-durable-cpu8-1m-p99-combined.json delete mode 100644 suites/run-durable-cpu8-clean.json delete mode 100644 suites/run-durable-cpu8.json delete mode 100644 suites/run-durable-dense-local.json delete mode 100644 suites/run-durable-dense.json delete mode 100644 suites/run-durable-pool-barrier.json delete mode 100644 suites/run-durable-pool-opt.json delete mode 100644 suites/run-durable-pool.json delete mode 100644 suites/run-durable-pool2-combined.json delete mode 100644 suites/run-durable-pool2.json delete mode 100644 suites/run-durable-tune.json delete mode 100644 suites/run-durable.json delete mode 100644 suites/run-node.json delete mode 100644 suites/run-s2.json delete mode 100644 suites/run-ursula.json delete mode 100644 suites/sustained-local-base.json delete mode 100644 suites/sustained-local-fix.json delete mode 100644 suites/sustained.json delete mode 100644 suites/wal-1m-diag.json delete mode 100644 suites/wal-batched-mw-local.json delete mode 100644 suites/wal-checkpoint-fix-100k.json delete mode 100644 suites/wal-checkpoint-fix-ab.json delete mode 100644 suites/wal-cpubind.json delete mode 100644 suites/wal-decomp-lane0.json delete mode 100644 suites/wal-fanout-sweep.json delete mode 100644 suites/wal-fsync-diag-local.json delete mode 100644 suites/wal-lazy-checkpoint-local.json delete mode 100644 suites/wal-lazy-checkpoint-multilane.json delete mode 100644 suites/wal-machinery-baseline-local.json delete mode 100644 suites/wal-mem-diag-local.json delete mode 100644 suites/wal-multilane-shards.json delete mode 100644 suites/wal-shard-sweep.json delete mode 100644 suites/wal-sizetrigger.json delete mode 100644 suites/wal-splitlane.json delete mode 100644 suites/wal-stacked-1m.json delete mode 100644 suites/wal-streamlanes-1m.json delete mode 100644 suites/wal-syncfs-local.json delete mode 100644 suites/wal-syncfs-nvme.json delete mode 100644 suites/write-accuracy-local.json delete mode 100644 suites/write-cliff-local-cpu4.json delete mode 100644 suites/write-cliff-local.json delete mode 100644 suites/write-cliff-nvme.json delete mode 100644 suites/write-nvme-bottleneck.json delete mode 100644 suites/write-throughput-local.json delete mode 100644 suites/write-wal-vs-mem-cpu4.json delete mode 100644 suites/write-wal-vs-mem-cpu8.json diff --git a/AGENTS.md b/AGENTS.md index 4f6f1da..a4561ae 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -22,10 +22,18 @@ run` away, self-tears-down on clean completion, and states its reference numbers |---|---|---|---| | `canonical-write` | write saturation (10k/100k streams) | durable-streams `wal-ideal` (the WAL_TUNING.md ideal config) + `memory` | `STATIC_CPU=1 SPLITLANE=1 GUARANTEED=1 SERVER_LOCAL_SSD_BLOCK=1 SERVER_MANIFEST=gke/durable-streams-splitlane3x3-guaranteed.yaml SPOT_SERVER=1` | | `canonical-write-ursula` | write saturation (100/1k/10k) | ursula v0.2.0 (`ghcr.io/tonbo-io/ursula:v0.2.0`) memory + disk | `SPOT_SERVER=1` | -| `canonical-sustained` | long-window latency + memory stability | durable-streams wal + memory | — | -| `canonical-reads-catchup` | historical replay reads | durable-streams wal | — | -| `canonical-reads-live` | live-tail long-poll delivery | durable-streams wal | — | -| `canonical-reads-sse` | SSE fan-out delivery | durable-streams wal | — | +| `canonical-reads-catchup` | historical replay reads | durable-streams wal + ursula | — | +| `canonical-reads-sse` | SSE tail delivery vs connections | durable-streams wal + ursula | — | +| `canonical-mixed-cal` | mixed-shape single-pod ceiling (anchor) | durable-streams wal | — | +| `canonical-mixed-writes` | paced readers vs pinned writes (interference) | durable-streams wal | — | +| `canonical-mixed-delivery` | SSE delivery under a write ladder | durable-streams wal + memory | — | + +SSE single-stream fan-out (subscriber ladder) is script-driven: `scripts/run-sse.sh`. +Report structure + pre-publication caveats: `REPORT_TEMPLATE.md`. Historical +suites and results were deleted (2026-07-14) — they live in git history; the +2026-07-02 campaign's write numbers were later found inflated (see the +template's physics-sanity caveat) and are superseded by the canonical references +above. Reference numbers (c4d-standard-64-lssd, 2026-07-13): `wal-ideal` ≈ 385k @10k / 374k @100k; `memory` ≈ 540k @10k / 512k @100k. **Regression gate: wal-ideal@100k @@ -76,10 +84,9 @@ percentiles, and writes per-cell results. | Workload | Measures | Driver | |---|---|---| -| **Write** (saturation) | append/s at saturation + tail latency + pod memory | `suites/run-{durable,ursula,s2,node}.json` | -| **Sustained** | latency + server-memory stability over a long window | `suites/sustained.json` | -| **Catch-up** | per-client replay latency + body size | `suites/catchup-{durable,ursula,s2}.json` | -| **Reads** (`catchup` / `long-poll` / `sse`) | live-tail delivery latency vs connections | `suites/reads-{catchup,longpoll,sse-remote}.json` | +| **Write** (saturation) | append/s at saturation + tail latency + pod memory | `suites/canonical-write.json`, `suites/canonical-write-ursula.json` | +| **Reads** (catch-up / SSE tail) | replay + live delivery vs connections | `suites/canonical-reads-{catchup,sse}.json` | +| **Mixed interference** | reads vs pinned writes; delivery under write load | `suites/canonical-mixed-{cal,writes,delivery}.json` | | **SSE fan-out** | per-event delivery latency + memory vs subscriber count | `scripts/run-sse.sh` | Systems under test: **durable-streams** (Rust; `wal` / `wal-tailcache` / `memory` diff --git a/REPORT_TEMPLATE.md b/REPORT_TEMPLATE.md new file mode 100644 index 0000000..87192f6 --- /dev/null +++ b/REPORT_TEMPLATE.md @@ -0,0 +1,72 @@ +# ds-bench campaign report — TEMPLATE + +Copy into `results/REPORT.md` for a campaign. Structure follows the workloads we +report (canonical suites cover exactly these); the caveat checklist at the bottom +exists because past reports shipped inflated or mis-attributed numbers — run it +before publishing. + +> Header: date, server build (branch @ commit / PR), hardware (server machine + +> CPU pin + storage layout, client fleet), zone. Per-suite provenance in +> `PROVENANCE.md`; per-suite grids in each subdirectory. + +## 1. Write saturation (`canonical-write`, `canonical-write-ursula`) + +Peak append/s at saturation (256 B payloads, saturation pod-ladder per cardinality): + +| streams | wal-ideal | memory | ursula-mem | ursula-disk | +|---|---|---|---|---| +| 10k | | | | | +| 100k | | | — | — | + +- State which cells are true plateaus vs ladder-bounded lower bounds (`reason` + field: `plateau` vs `ladder_exhausted`) — never present a ladder ceiling as a + server ceiling. +- Compare against the reference numbers in the suite `_doc`; a breached + regression gate is a finding, not a footnote. + +## 2. SSE fan-out (`scripts/run-sse.sh` — 1 stream, subscriber ladder) + +Delivery p50/p99 (ms) vs subscriber count {1, 10, 100, 1000}, per config. + +## 3. Read scalability (`canonical-reads-catchup`, `canonical-reads-sse`) + +- SSE tail: ops/s @ p99 ms per connection level. +- Catch-up: MiB/s @ p99 ms per connection level. Mark client-pod OOM cells as + client-bound gaps, not server data. + +## 4. Mixed read/write interference (`canonical-mixed-{cal,writes,delivery}`) + +- Anchor: single-pod mixed-shape ceiling from `canonical-mixed-cal`; sweeps pin + writers at ~60% of it. +- Paced readers vs pinned writes: write ops/s must hold flat across reader + levels; report replays/s + read MiB/s + read p50/p99. +- Delivery under write load: del/s must track writes/s; call out any collapse + (the 2026-07-02 memory-mode delivery collapse pattern). + +## Known gaps & artifacts + +List every dropped cell, re-run, resume, and client-bound ceiling explicitly. + +--- + +## Pre-publication caveat checklist (numbers-inflation guards) + +Every one of these has produced a wrong or inflated headline in a past campaign: + +1. **Storage layout verified?** Stream data files MUST be on local NVMe lanes, + never the boot PD, and WAL lanes ≠ data lanes (AGENTS.md §0 invariants). A + mis-mounted layout mis-states wal throughput by 5–26× in either direction. +2. **Windows aligned?** Only barrier-aligned fleet windows (`windows_aligned`) + count; an unaligned cell reports partial-fleet throughput. +3. **Plateau vs ladder?** `ladder_exhausted` cells are lower bounds — extend the + ladder or label them; comparing a plateau to a ladder ceiling fabricates a win. +4. **Same image + digest for every arm being compared?** Record digests in + PROVENANCE.md; the resume digest is tag-based and does NOT prove content. +5. **Sanity-check against physics.** Appends/s × payload vs device writeback; + fsync/s vs device barrier rate; a "durable" number beating the no-fsync + ceiling (e.g. wal > memory) means the measurement or the durability is broken + — treat as a bug until explained (2026-07-02's 2.05M @500k on 4 CPUs was this). +6. **Cardinality shape.** Report 10k AND 100k; a >30% drop is the cliff + regression gate firing, not a shrug. +7. **Client-bound cells** (fleet OOM, creation choke) are gaps — never zeros, + never averaged in. diff --git a/results-2026-07-02/PROVENANCE.md b/results-2026-07-02/PROVENANCE.md deleted file mode 100644 index 337ec96..0000000 --- a/results-2026-07-02/PROVENANCE.md +++ /dev/null @@ -1,47 +0,0 @@ -# Benchmark provenance — 2026-07-02 - -Full matrix on the **perf-branch** durable build, from scratch. First campaign to -include the mixed read/write interference workload (PR electric#4679 has the local -validation this extends). - -## Versions (commit hashes) -- **durable-streams**: `330ccd1b09a2a2429e10ec3b6eeb0dacbf6e60b5` - (branch `bench/mixed-interference-validation` = `perf/combined-t1a-t1c-t2a` head, PR #4679). - Image `europe-west1-docker.pkg.dev/vaxine/ds-bench/durable-streams:dev`, built 2026-07-02 via Cloud Build from the crate checkout. -- **ds-bench**: `60ab813c26c7165ad3b9f6451d2ae94dbbc15fad` (mixed workload harness). -- **ursula**: `ghcr.io/tonbo-io/ursula:v0.1.5` · **Node.js reference**: `durable-node:dev` · **S2**: `ghcr.io/s2-streamstore/s2` - -## Workloads -- **Write** saturation: `run-durable` (wal, wal-tailcache, memory — streams up to - **500k**), `run-ursula` (memory, disk), `run-node`, `run-s2`. -- **SSE fan-out**: `run-sse.sh` — subscribers 1/10/100/1000, **no tailcache variant** - (single-stream fan-out is a micro-benchmark; spread fan-out is mixed-delivery's job). -- **Reads**: `reads-catchup`, `reads-sse-remote` (wal + ursula; long-poll dropped this run). -- **Mixed interference** (NEW): `mixed-cal` (ceiling anchor), `mixed-writes` - (readers 0→**100k**, one staggered replay/30s each, vs a 60%-pinned write load over - 10k streams), `mixed-writes-hot` (unpaced adversarial), `mixed-delivery` - (2000 SSE subscribers over 2000 streams vs write-rate ladder, wal + memory). - -## Hardware -Server `c4d-standard-16-lssd` pinned to 4 CPUs; client fleet `n2d-standard-32` Spot. europe-west4. - -## Run notes (2026-07-02) - -- **Server binary source**: branch head `330ccd1b09a2a2429e10ec3b6eeb0dacbf6e60b5` - at build time; code identical to `perf/combined-t1a-t1c-t2a` head `06a8a37c5` - (the extra commit is docs-only: MIXED_WORKLOAD_VALIDATION.md). -- **s2**: the 1000-stream write cell creation-choked 3x (incl. once from a fully - clean slate: bucket wiped with s2lite stopped). Scoped run-s2 to 100 streams; - the 1000 cell is a gap, not a zero. -- **mixed-writes retries**: readers=1000 failed once on Spot-fleet scheduling; - readers=100000 OOMKilled the fleet pod at the old 4Gi cap (~110k task futures + - per-reader HDR histograms). Fixed by raising the fleet memory cap to 24Gi - (gke/bench-job.yaml); both levels re-ran clean. -- **200k write cells** are ladder-bounded lower bounds (200k ladder tops at 250 - pods vs 500k's 625; every config reads higher at 500k than 200k). -- **Interruptions**: the harness kills background tasks at 2h; the campaign was - SIGTERMed twice (trap tore clusters down cleanly both times) and resumed with - RESUME=1 — no measured cells lost. Final relaunch ran detached (nohup). -- **reads-catchup** high-connection error cells are the documented client-pod OOM - ceiling (AGENTS.md §8), not server data: wal n10/n100 conn>=128, ursula n100 all. -- Headline synthesis: see REPORT.md alongside this file. diff --git a/results-2026-07-02/REPORT.md b/results-2026-07-02/REPORT.md deleted file mode 100644 index 60393cc..0000000 --- a/results-2026-07-02/REPORT.md +++ /dev/null @@ -1,149 +0,0 @@ -# ds-bench full matrix — 2026-07-02 (perf-branch durable build) - -Complete campaign on the **perf branch** durable-streams build -(`bench/mixed-interference-validation` @ `330ccd1b0`, code identical to -`perf/combined-t1a-t1c-t2a` head `06a8a37c5` — the delta is a docs-only commit; -PR electric#4679). First campaign to include the **mixed read/write interference -workload**. Hardware: server `c4d-standard-16-lssd` pinned to 4 CPUs, client fleet -`n2d-standard-32` Spot, europe-west4. Versions, images and per-suite details in -`PROVENANCE.md`; per-suite grids in each subdirectory. - -## 1. Write saturation (`run-durable`, `run-ursula`, `run-s2`, `run-node`) - -Peak append/s at saturation (256 B payloads, saturation pod-ladder per cardinality): - -| streams | wal | wal-tailcache | memory | ursula-mem | ursula-disk | node | -|---|---|---|---|---|---|---| -| 100 | 457k | 488k | 436k | 64k | 2.5k | 55k | -| 1k | 655k | 565k | 479k | 111k | 7.0k | 60k | -| 10k | 816k | 794k | 575k | 150k | 11.7k | 151k | -| 100k | 1.56M | 1.73M | 732k | — | — | — | -| 200k | 1.50M* | 1.41M* | 534k* | — | — | — | -| 500k | **2.05M** | 1.89M | 1.33M | — | — | — | - -- **The wal path clears 2M appends/s at 500k streams on 4 CPUs** — cardinality is - no longer the limiter it was in the 2026-06-30 baseline (1.48M @ 200k on more - server cores). -- *200k cells are ladder-shaped, not server ceilings: the 200k ladder tops at 250 - pods vs 500k's 625, and every config reads higher at 500k than 200k. Treat 200k - as a lower bound; re-walk with a taller ladder if the point matters. -- **`--durability memory` is SLOWER than wal everywhere** (e.g. 575k vs 816k @ 10k - streams; 1.33M vs 2.05M @ 500k). The wal path's sharded committer outperforms - the memory path on this branch — worth a look server-side (see also the memory - delivery collapse in §4). -- s2: 2.0k @ 100 streams; its 1000-stream cell **creation-choked 3× (once from a - clean slate)** and was dropped from scope — recorded as a gap, not a zero. - -## 2. SSE fan-out (`sse-comparison.md` — 1 stream, 50 ev/s, no cache variant) - -Delivery latency vs subscriber count (1 stream, 1 writer @ 50 ev/s, writer-paced): - -**Delivery p50 (ms)** - -| config \ subscribers | 1 | 10 | 100 | 1000 | -|---|---|---|---|---| -| wal (cache off) | 0.359 | 0.482 | 0.905 | 3.645 | -| ursula in-memory | 0.396 | 0.504 | 0.835 | 2.979 | -| ursula disk | 1.307 | 1.822 | 2.135 | 4.247 | - -**Delivery p99 (ms)** - -| config \ subscribers | 1 | 10 | 100 | 1000 | -|---|---|---|---|---| -| wal (cache off) | 0.577 | 0.717 | 1.276 | 5.079 | -| ursula in-memory | 0.589 | 0.748 | 1.192 | 4.511 | -| ursula disk | 1.528 | 2.261 | 2.761 | 5.867 | - -Durable wal stays sub-ms through 100 subscribers — ~2× better than ursula-disk, -on par with ursula-memory. (Single-stream fan-out is a micro-benchmark; the -spread-subscriber story is §4.) - -## 3. Read scalability (`reads-catchup`, `reads-sse-remote`; long-poll dropped) - -**SSE tail** (`reads-sse-remote`; cell = ops/s @ p99 ms per connection level): - -| system, streams | 64 conns | 256 | 1024 | 2048 | -|---|---|---|---|---| -| wal, 10 | 3.2k @ 1.4 | 12.8k @ 2.1 | 51.3k @ 2.6 | 102.5k @ 2.9 | -| wal, 100 | 3.2k @ 1.2 | 12.8k @ 2.3 | 51.3k @ 2.4 | 102.5k @ 2.8 | -| ursula, 10 | 3.2k @ 1.5 | 12.8k @ 1.9 | 51.3k @ 2.7 | 102.5k @ 2.8 | -| ursula, 100 | 3.1k @ 42.2 | 11.8k @ 47.3 | 44.1k @ 56.7 | 80.6k @ 62.9 | - -Durable wal is flat across 10 → 100 streams all the way to 2048 concurrent -connections; ursula matches at 10 streams but degrades to p99 ~63 ms at 100 -streams. - -**Catch-up (hot resident re-scan)** (`reads-catchup`; cell = MiB/s @ p99 ms): - -| system, streams | 8 conns | 32 | 128 | 512 | -|---|---|---|---|---| -| wal, 10 | 1345 @ 110.8 | 2381 @ 237.3 | OOM | OOM | -| wal, 100 | 1333 @ 111.7 | 2382 @ 238.2 | OOM | OOM | -| ursula, 10 | 2351 @ 85.9 | 2375 @ 516.6 | OOM | OOM | -| ursula, 100 | OOM | OOM | OOM | OOM | - -_OOM cells are the documented client-pod OOM ceiling (AGENTS.md §8) — client-bound -cells, not server data._ - -Both wal and ursula plateau ~2.4 GiB/s at 32 connections at 10 streams, with -durable holding a ~2× better p99 there (237 ms vs 517 ms); ursula @ 100 streams -OOMs at every level. - -## 4. Mixed read/write interference (NEW — `mixed-*`) - -Anchor (`mixed-cal`): 50 unthrottled writers → **81.4k ops/s** single-pod mixed-shape -ceiling; sweeps pin writers at ~60% of it. - -**Paced readers vs pinned writes** (`mixed-writes`: 10k streams, 10k writers @ 5/s -= 50k ops/s pinned, readers replay once/30 s, staggered, 60 s windows): - -| readers | write ops/s | replays/s | read MiB/s | read p50/p99 ms | -|---|---|---|---|---| -| 0 | 49,964 | — | — | — | -| 1,000 | 50,026 | 50 | 3 | 0.49 / 419 | -| 10,000 | 49,826 | 499 | 30 | 0.56 / 404 | -| **100,000** | **50,041** | **4,987** | **304** | 0.52 / 880 | - -**The premise holds: 100k concurrent catch-up readers cost the write path -nothing** — throughput is identical to the zero-reader baseline while the server -also serves ~5k replays/s at 304 MiB/s with sub-ms median reads and zero errors. -(Write p99 sits at 350–450 ms at *every* level including zero readers — that is -the single-pod 10k-writer client shape, not reader interference.) - -**Unpaced (adversarial) readers** (`mixed-writes-hot`: 50 streams, writers pinned -48.9k ops/s): 16–64 hot readers pull an enormous **~2.3 GiB/s** of replay bandwidth -while writes hold at 48–49k (−2%); at 256 hot readers writes collapse to 7.7k -(−84%). Same cliff as the local validation: bounded read load is free, -*saturating* read load fair-shares everything down — and the server never sheds -(zero 429/503 anywhere). - -**Delivery under write load** (`mixed-delivery`: 2000 SSE subscribers spread 1-per- -stream over 2000 streams, write ladder ≈5→100% of ceiling): - -| writes/s offered | wal del/s | wal deliv p50/p99 ms | memory del/s | memory deliv p50/p99 ms | -|---|---|---|---|---| -| 4k | 3.3k | 0.42 / 163 | 3.9k | 0.41 / 133 | -| 16k | 13.3k | 0.53 / 15 | 14.7k | 0.63 / 335 | -| 40k | 33.2k | 1.8 / 22 | 15.9k | **227 / 362** | -| 66k | 65.7k | 10.2 / 45 | 20.6k | 207 / 299 | -| max | 65.5k @ 86k writes | 37.9 / 57 | 19.0k @ 62k writes | 201 / 299 | - -- **wal: delivery keeps pace to ~66k writes/s** (del/s ≈ writes, p99 ≤ 45 ms); - at full saturation (86k writes/s) delivery caps at ~65k/s, p99 57 ms. -- **memory: delivery collapses above ~16k writes/s** — subscribers receive only - 16–20k del/s against 40–62k writes with p50 200+ ms. Not the client (the same - pod did 65k del/s under wal minutes earlier): without WAL commit pacing, the - fan-out path appears starved. **Server-side finding for the perf branch.** -- The low-rate p99 spike (163 ms at 4k writes/s) tracks the write path's own - low-rate tail (write p99 138 ms at that level) — commit batching at near-idle, - not an SSE effect; delivery p99 ≈ write p99 + a few ms throughout. - -## Known gaps & artifacts - -- s2 @ 1000 streams: dropped (creation_choke ×3, once from clean slate). -- 200k write cells: ladder-bounded lower bounds (see §1). -- `mixed-writes` 1k/100k levels were re-run after Spot-scheduling and fleet-pod - OOM failures; the OOM fix (fleet memory cap 4 → 24 GiB for ~110k reader tasks + - per-reader HDR histograms) is in `gke/bench-job.yaml`. -- The campaign was interrupted twice by a 2-hour harness cap on background tasks - and resumed via `RESUME=1` (no cells lost; ~50 min of cluster-recreation churn). diff --git a/results-2026-07-02/mixed-cal/aggregate.csv b/results-2026-07-02/mixed-cal/aggregate.csv deleted file mode 100644 index bc53f67..0000000 --- a/results-2026-07-02/mixed-cal/aggregate.csv +++ /dev/null @@ -1,2 +0,0 @@ -mode,stream_count,sweep,level,readers,subscribers,writer_rate,read_rate,write_ops_per_sec,write_p50,write_p99,read_ops_per_sec,read_mib_per_sec,read_p50,read_p99,events_per_sec,delivery_p50,delivery_p99,write_bp,write_err,read_bp,read_err,status,reason -wal,50,writer_rate,0,0,0,0,0,81435.95,0.494,1.131,0.0,0.0,,,0.0,,,0,0,0,0,ok,complete diff --git a/results-2026-07-02/mixed-cal/aggregate.json b/results-2026-07-02/mixed-cal/aggregate.json deleted file mode 100644 index 3a35ffa..0000000 --- a/results-2026-07-02/mixed-cal/aggregate.json +++ /dev/null @@ -1,28 +0,0 @@ -[ - { - "mode": "wal", - "stream_count": 50, - "sweep": "writer_rate", - "level": 0, - "readers": 0, - "subscribers": 0, - "writer_rate": 0, - "read_rate": 0, - "write_ops_per_sec": 81435.95, - "write_p50": 0.494, - "write_p99": 1.131, - "read_ops_per_sec": 0.0, - "read_mib_per_sec": 0.0, - "read_p50": null, - "read_p99": null, - "events_per_sec": 0.0, - "delivery_p50": null, - "delivery_p99": null, - "write_bp": 0, - "write_err": 0, - "read_bp": 0, - "read_err": 0, - "status": "ok", - "reason": "complete" - } -] \ No newline at end of file diff --git a/results-2026-07-02/mixed-cal/report.md b/results-2026-07-02/mixed-cal/report.md deleted file mode 100644 index 4427b71..0000000 --- a/results-2026-07-02/mixed-cal/report.md +++ /dev/null @@ -1,13 +0,0 @@ -# mixed-cal — mixed read/write interference report - -Sweep axis: **writer_rate**. Latency cells are p50/p99 ms. ‡ = backpressure (503/429) observed in that class. - -## wal — 50 streams - -| rate/writer | write ops/s | write ms | read ops/s | read MiB/s | read ms | deliv rec/s | deliv ms | status | -|---|---|---|---|---|---|---|---|---| -| max | 81436 | 0.5/1.1 | 0 | 0.0 | — | 0 | — | ok | - -## Findings - -_TODO: written by hand on top of the generated data._ diff --git a/results-2026-07-02/mixed-cal/wal/cells.json b/results-2026-07-02/mixed-cal/wal/cells.json deleted file mode 100644 index eb87aab..0000000 --- a/results-2026-07-02/mixed-cal/wal/cells.json +++ /dev/null @@ -1,40 +0,0 @@ -{ - "cells": { - "50": { - "stream_count": 50, - "image_digest": "92f4a6387253", - "complete": true, - "levels": { - "0": { - "write_ops_per_sec": 81435.95, - "write_p50": 0.494, - "write_p99": 1.131, - "read_ops_per_sec": 0.0, - "read_mib_per_sec": 0.0, - "read_p50": null, - "read_p99": null, - "events_per_sec": 0.0, - "delivery_p50": null, - "delivery_p99": null, - "write_ok": 1628719, - "write_bp": 0, - "write_err": 0, - "read_ok": 0, - "read_bp": 0, - "read_err": 0, - "events_received": 0, - "control_events_received": 0, - "elapsed_secs": 20.042467107, - "drive_secs": 20.0, - "level": 0, - "readers": 0, - "subscribers": 0, - "writer_rate": 0, - "read_rate": 0, - "status": "ok", - "reason": "complete" - } - } - } - } -} \ No newline at end of file diff --git a/results-2026-07-02/mixed-delivery/aggregate.csv b/results-2026-07-02/mixed-delivery/aggregate.csv deleted file mode 100644 index cd6a694..0000000 --- a/results-2026-07-02/mixed-delivery/aggregate.csv +++ /dev/null @@ -1,11 +0,0 @@ -mode,stream_count,sweep,level,readers,subscribers,writer_rate,read_rate,write_ops_per_sec,write_p50,write_p99,read_ops_per_sec,read_mib_per_sec,read_p50,read_p99,events_per_sec,delivery_p50,delivery_p99,write_bp,write_err,read_bp,read_err,status,reason -memory,2000,writer_rate,0,0,2000,0,0,61626.9,9.191,166.911,0.0,0.0,,,19046.066666666666,200.575,299.263,0,0,0,0,ok,complete -memory,2000,writer_rate,2,0,2000,2,0,4066.6666666666665,0.378,127.167,0.0,0.0,,,3866.6666666666665,0.406,132.607,0,0,0,0,ok,complete -memory,2000,writer_rate,8,0,2000,8,0,16050.933333333332,0.509,58.879,0.0,0.0,,,14676.966666666667,0.629,334.591,0,0,0,0,ok,complete -memory,2000,writer_rate,20,0,2000,20,0,39931.433333333334,11.591,82.431,0.0,0.0,,,15902.333333333334,226.687,362.239,0,0,0,0,ok,complete -memory,2000,writer_rate,33,0,2000,33,0,60841.96666666667,7.231,166.527,0.0,0.0,,,20597.4,206.591,299.263,0,0,0,0,ok,complete -wal,2000,writer_rate,0,0,2000,0,0,86268.53333333334,22.831,41.119,0.0,0.0,,,65537.13333333333,37.919,57.311,0,0,0,0,ok,complete -wal,2000,writer_rate,2,0,2000,2,0,4066.6666666666665,0.39,138.239,0.0,0.0,,,3339.6,0.422,162.559,0,0,0,0,ok,complete -wal,2000,writer_rate,8,0,2000,8,0,16050.666666666666,0.484,13.823,0.0,0.0,,,13323.9,0.525,15.447,0,0,0,0,ok,complete -wal,2000,writer_rate,20,0,2000,20,0,39981.433333333334,1.319,19.039,0.0,0.0,,,33210.566666666666,1.838,21.599,0,0,0,0,ok,complete -wal,2000,writer_rate,33,0,2000,33,0,65910.46666666666,9.239,29.183,0.0,0.0,,,65673.86666666667,10.167,44.735,0,0,0,0,ok,complete diff --git a/results-2026-07-02/mixed-delivery/aggregate.json b/results-2026-07-02/mixed-delivery/aggregate.json deleted file mode 100644 index 2ed2c16..0000000 --- a/results-2026-07-02/mixed-delivery/aggregate.json +++ /dev/null @@ -1,262 +0,0 @@ -[ - { - "mode": "memory", - "stream_count": 2000, - "sweep": "writer_rate", - "level": 0, - "readers": 0, - "subscribers": 2000, - "writer_rate": 0, - "read_rate": 0, - "write_ops_per_sec": 61626.9, - "write_p50": 9.191, - "write_p99": 166.911, - "read_ops_per_sec": 0.0, - "read_mib_per_sec": 0.0, - "read_p50": null, - "read_p99": null, - "events_per_sec": 19046.066666666666, - "delivery_p50": 200.575, - "delivery_p99": 299.263, - "write_bp": 0, - "write_err": 0, - "read_bp": 0, - "read_err": 0, - "status": "ok", - "reason": "complete" - }, - { - "mode": "memory", - "stream_count": 2000, - "sweep": "writer_rate", - "level": 2, - "readers": 0, - "subscribers": 2000, - "writer_rate": 2, - "read_rate": 0, - "write_ops_per_sec": 4066.6666666666665, - "write_p50": 0.378, - "write_p99": 127.167, - "read_ops_per_sec": 0.0, - "read_mib_per_sec": 0.0, - "read_p50": null, - "read_p99": null, - "events_per_sec": 3866.6666666666665, - "delivery_p50": 0.406, - "delivery_p99": 132.607, - "write_bp": 0, - "write_err": 0, - "read_bp": 0, - "read_err": 0, - "status": "ok", - "reason": "complete" - }, - { - "mode": "memory", - "stream_count": 2000, - "sweep": "writer_rate", - "level": 8, - "readers": 0, - "subscribers": 2000, - "writer_rate": 8, - "read_rate": 0, - "write_ops_per_sec": 16050.933333333332, - "write_p50": 0.509, - "write_p99": 58.879, - "read_ops_per_sec": 0.0, - "read_mib_per_sec": 0.0, - "read_p50": null, - "read_p99": null, - "events_per_sec": 14676.966666666667, - "delivery_p50": 0.629, - "delivery_p99": 334.591, - "write_bp": 0, - "write_err": 0, - "read_bp": 0, - "read_err": 0, - "status": "ok", - "reason": "complete" - }, - { - "mode": "memory", - "stream_count": 2000, - "sweep": "writer_rate", - "level": 20, - "readers": 0, - "subscribers": 2000, - "writer_rate": 20, - "read_rate": 0, - "write_ops_per_sec": 39931.433333333334, - "write_p50": 11.591, - "write_p99": 82.431, - "read_ops_per_sec": 0.0, - "read_mib_per_sec": 0.0, - "read_p50": null, - "read_p99": null, - "events_per_sec": 15902.333333333334, - "delivery_p50": 226.687, - "delivery_p99": 362.239, - "write_bp": 0, - "write_err": 0, - "read_bp": 0, - "read_err": 0, - "status": "ok", - "reason": "complete" - }, - { - "mode": "memory", - "stream_count": 2000, - "sweep": "writer_rate", - "level": 33, - "readers": 0, - "subscribers": 2000, - "writer_rate": 33, - "read_rate": 0, - "write_ops_per_sec": 60841.96666666667, - "write_p50": 7.231, - "write_p99": 166.527, - "read_ops_per_sec": 0.0, - "read_mib_per_sec": 0.0, - "read_p50": null, - "read_p99": null, - "events_per_sec": 20597.4, - "delivery_p50": 206.591, - "delivery_p99": 299.263, - "write_bp": 0, - "write_err": 0, - "read_bp": 0, - "read_err": 0, - "status": "ok", - "reason": "complete" - }, - { - "mode": "wal", - "stream_count": 2000, - "sweep": "writer_rate", - "level": 0, - "readers": 0, - "subscribers": 2000, - "writer_rate": 0, - "read_rate": 0, - "write_ops_per_sec": 86268.53333333334, - "write_p50": 22.831, - "write_p99": 41.119, - "read_ops_per_sec": 0.0, - "read_mib_per_sec": 0.0, - "read_p50": null, - "read_p99": null, - "events_per_sec": 65537.13333333333, - "delivery_p50": 37.919, - "delivery_p99": 57.311, - "write_bp": 0, - "write_err": 0, - "read_bp": 0, - "read_err": 0, - "status": "ok", - "reason": "complete" - }, - { - "mode": "wal", - "stream_count": 2000, - "sweep": "writer_rate", - "level": 2, - "readers": 0, - "subscribers": 2000, - "writer_rate": 2, - "read_rate": 0, - "write_ops_per_sec": 4066.6666666666665, - "write_p50": 0.39, - "write_p99": 138.239, - "read_ops_per_sec": 0.0, - "read_mib_per_sec": 0.0, - "read_p50": null, - "read_p99": null, - "events_per_sec": 3339.6, - "delivery_p50": 0.422, - "delivery_p99": 162.559, - "write_bp": 0, - "write_err": 0, - "read_bp": 0, - "read_err": 0, - "status": "ok", - "reason": "complete" - }, - { - "mode": "wal", - "stream_count": 2000, - "sweep": "writer_rate", - "level": 8, - "readers": 0, - "subscribers": 2000, - "writer_rate": 8, - "read_rate": 0, - "write_ops_per_sec": 16050.666666666666, - "write_p50": 0.484, - "write_p99": 13.823, - "read_ops_per_sec": 0.0, - "read_mib_per_sec": 0.0, - "read_p50": null, - "read_p99": null, - "events_per_sec": 13323.9, - "delivery_p50": 0.525, - "delivery_p99": 15.447, - "write_bp": 0, - "write_err": 0, - "read_bp": 0, - "read_err": 0, - "status": "ok", - "reason": "complete" - }, - { - "mode": "wal", - "stream_count": 2000, - "sweep": "writer_rate", - "level": 20, - "readers": 0, - "subscribers": 2000, - "writer_rate": 20, - "read_rate": 0, - "write_ops_per_sec": 39981.433333333334, - "write_p50": 1.319, - "write_p99": 19.039, - "read_ops_per_sec": 0.0, - "read_mib_per_sec": 0.0, - "read_p50": null, - "read_p99": null, - "events_per_sec": 33210.566666666666, - "delivery_p50": 1.838, - "delivery_p99": 21.599, - "write_bp": 0, - "write_err": 0, - "read_bp": 0, - "read_err": 0, - "status": "ok", - "reason": "complete" - }, - { - "mode": "wal", - "stream_count": 2000, - "sweep": "writer_rate", - "level": 33, - "readers": 0, - "subscribers": 2000, - "writer_rate": 33, - "read_rate": 0, - "write_ops_per_sec": 65910.46666666666, - "write_p50": 9.239, - "write_p99": 29.183, - "read_ops_per_sec": 0.0, - "read_mib_per_sec": 0.0, - "read_p50": null, - "read_p99": null, - "events_per_sec": 65673.86666666667, - "delivery_p50": 10.167, - "delivery_p99": 44.735, - "write_bp": 0, - "write_err": 0, - "read_bp": 0, - "read_err": 0, - "status": "ok", - "reason": "complete" - } -] \ No newline at end of file diff --git a/results-2026-07-02/mixed-delivery/memory/cells.json b/results-2026-07-02/mixed-delivery/memory/cells.json deleted file mode 100644 index 7999e86..0000000 --- a/results-2026-07-02/mixed-delivery/memory/cells.json +++ /dev/null @@ -1,156 +0,0 @@ -{ - "cells": { - "2000": { - "stream_count": 2000, - "image_digest": "b7d78084e886", - "complete": true, - "levels": { - "2": { - "write_ops_per_sec": 4066.6666666666665, - "write_p50": 0.378, - "write_p99": 127.167, - "read_ops_per_sec": 0.0, - "read_mib_per_sec": 0.0, - "read_p50": null, - "read_p99": null, - "events_per_sec": 3866.6666666666665, - "delivery_p50": 0.406, - "delivery_p99": 132.607, - "write_ok": 122000, - "write_bp": 0, - "write_err": 0, - "read_ok": 0, - "read_bp": 0, - "read_err": 0, - "events_received": 116000, - "control_events_received": 118000, - "elapsed_secs": 31.400304374, - "drive_secs": 30.0, - "level": 2, - "readers": 0, - "subscribers": 2000, - "writer_rate": 2, - "read_rate": 0, - "status": "ok", - "reason": "complete" - }, - "8": { - "write_ops_per_sec": 16050.933333333332, - "write_p50": 0.509, - "write_p99": 58.879, - "read_ops_per_sec": 0.0, - "read_mib_per_sec": 0.0, - "read_p50": null, - "read_p99": null, - "events_per_sec": 14676.966666666667, - "delivery_p50": 0.629, - "delivery_p99": 334.591, - "write_ok": 481528, - "write_bp": 0, - "write_err": 0, - "read_ok": 0, - "read_bp": 0, - "read_err": 0, - "events_received": 440309, - "control_events_received": 442309, - "elapsed_secs": 32.226361259, - "drive_secs": 30.0, - "level": 8, - "readers": 0, - "subscribers": 2000, - "writer_rate": 8, - "read_rate": 0, - "status": "ok", - "reason": "complete" - }, - "20": { - "write_ops_per_sec": 39931.433333333334, - "write_p50": 11.591, - "write_p99": 82.431, - "read_ops_per_sec": 0.0, - "read_mib_per_sec": 0.0, - "read_p50": null, - "read_p99": null, - "events_per_sec": 15902.333333333334, - "delivery_p50": 226.687, - "delivery_p99": 362.239, - "write_ok": 1197943, - "write_bp": 0, - "write_err": 0, - "read_ok": 0, - "read_bp": 0, - "read_err": 0, - "events_received": 477070, - "control_events_received": 479070, - "elapsed_secs": 32.293532483999996, - "drive_secs": 30.0, - "level": 20, - "readers": 0, - "subscribers": 2000, - "writer_rate": 20, - "read_rate": 0, - "status": "ok", - "reason": "complete" - }, - "33": { - "write_ops_per_sec": 60841.96666666667, - "write_p50": 7.231, - "write_p99": 166.527, - "read_ops_per_sec": 0.0, - "read_mib_per_sec": 0.0, - "read_p50": null, - "read_p99": null, - "events_per_sec": 20597.4, - "delivery_p50": 206.591, - "delivery_p99": 299.263, - "write_ok": 1825259, - "write_bp": 0, - "write_err": 0, - "read_ok": 0, - "read_bp": 0, - "read_err": 0, - "events_received": 617922, - "control_events_received": 619922, - "elapsed_secs": 32.215251699, - "drive_secs": 30.0, - "level": 33, - "readers": 0, - "subscribers": 2000, - "writer_rate": 33, - "read_rate": 0, - "status": "ok", - "reason": "complete" - }, - "0": { - "write_ops_per_sec": 61626.9, - "write_p50": 9.191, - "write_p99": 166.911, - "read_ops_per_sec": 0.0, - "read_mib_per_sec": 0.0, - "read_p50": null, - "read_p99": null, - "events_per_sec": 19046.066666666666, - "delivery_p50": 200.575, - "delivery_p99": 299.263, - "write_ok": 1848807, - "write_bp": 0, - "write_err": 0, - "read_ok": 0, - "read_bp": 0, - "read_err": 0, - "events_received": 571382, - "control_events_received": 573382, - "elapsed_secs": 35.225584119, - "drive_secs": 30.0, - "level": 0, - "readers": 0, - "subscribers": 2000, - "writer_rate": 0, - "read_rate": 0, - "status": "ok", - "reason": "complete" - } - } - } - } -} \ No newline at end of file diff --git a/results-2026-07-02/mixed-delivery/report.md b/results-2026-07-02/mixed-delivery/report.md deleted file mode 100644 index 8a4814d..0000000 --- a/results-2026-07-02/mixed-delivery/report.md +++ /dev/null @@ -1,27 +0,0 @@ -# mixed-delivery — mixed read/write interference report - -Sweep axis: **writer_rate**. Latency cells are p50/p99 ms. ‡ = backpressure (503/429) observed in that class. - -## wal — 2000 streams - -| rate/writer | write ops/s | write ms | read ops/s | read MiB/s | read ms | deliv rec/s | deliv ms | status | -|---|---|---|---|---|---|---|---|---| -| 2 | 4067 | 0.4/138.2 | 0 | 0.0 | — | 3340 | 0.4/162.6 | ok | -| 8 | 16051 | 0.5/13.8 | 0 | 0.0 | — | 13324 | 0.5/15.4 | ok | -| 20 | 39981 | 1.3/19.0 | 0 | 0.0 | — | 33211 | 1.8/21.6 | ok | -| 33 | 65910 | 9.2/29.2 | 0 | 0.0 | — | 65674 | 10.2/44.7 | ok | -| max | 86269 | 22.8/41.1 | 0 | 0.0 | — | 65537 | 37.9/57.3 | ok | - -## memory — 2000 streams - -| rate/writer | write ops/s | write ms | read ops/s | read MiB/s | read ms | deliv rec/s | deliv ms | status | -|---|---|---|---|---|---|---|---|---| -| 2 | 4067 | 0.4/127.2 | 0 | 0.0 | — | 3867 | 0.4/132.6 | ok | -| 8 | 16051 | 0.5/58.9 | 0 | 0.0 | — | 14677 | 0.6/334.6 | ok | -| 20 | 39931 | 11.6/82.4 | 0 | 0.0 | — | 15902 | 226.7/362.2 | ok | -| 33 | 60842 | 7.2/166.5 | 0 | 0.0 | — | 20597 | 206.6/299.3 | ok | -| max | 61627 | 9.2/166.9 | 0 | 0.0 | — | 19046 | 200.6/299.3 | ok | - -## Findings - -_TODO: written by hand on top of the generated data._ diff --git a/results-2026-07-02/mixed-delivery/wal/cells.json b/results-2026-07-02/mixed-delivery/wal/cells.json deleted file mode 100644 index 11ee9e9..0000000 --- a/results-2026-07-02/mixed-delivery/wal/cells.json +++ /dev/null @@ -1,156 +0,0 @@ -{ - "cells": { - "2000": { - "stream_count": 2000, - "image_digest": "92f4a6387253", - "complete": true, - "levels": { - "2": { - "write_ops_per_sec": 4066.6666666666665, - "write_p50": 0.39, - "write_p99": 138.239, - "read_ops_per_sec": 0.0, - "read_mib_per_sec": 0.0, - "read_p50": null, - "read_p99": null, - "events_per_sec": 3339.6, - "delivery_p50": 0.422, - "delivery_p99": 162.559, - "write_ok": 122000, - "write_bp": 0, - "write_err": 0, - "read_ok": 0, - "read_bp": 0, - "read_err": 0, - "events_received": 100188, - "control_events_received": 102188, - "elapsed_secs": 35.300253942, - "drive_secs": 30.0, - "level": 2, - "readers": 0, - "subscribers": 2000, - "writer_rate": 2, - "read_rate": 0, - "status": "ok", - "reason": "complete" - }, - "8": { - "write_ops_per_sec": 16050.666666666666, - "write_p50": 0.484, - "write_p99": 13.823, - "read_ops_per_sec": 0.0, - "read_mib_per_sec": 0.0, - "read_p50": null, - "read_p99": null, - "events_per_sec": 13323.9, - "delivery_p50": 0.525, - "delivery_p99": 15.447, - "write_ok": 481520, - "write_bp": 0, - "write_err": 0, - "read_ok": 0, - "read_bp": 0, - "read_err": 0, - "events_received": 399717, - "control_events_received": 401717, - "elapsed_secs": 35.164210628, - "drive_secs": 30.0, - "level": 8, - "readers": 0, - "subscribers": 2000, - "writer_rate": 8, - "read_rate": 0, - "status": "ok", - "reason": "complete" - }, - "20": { - "write_ops_per_sec": 39981.433333333334, - "write_p50": 1.319, - "write_p99": 19.039, - "read_ops_per_sec": 0.0, - "read_mib_per_sec": 0.0, - "read_p50": null, - "read_p99": null, - "events_per_sec": 33210.566666666666, - "delivery_p50": 1.838, - "delivery_p99": 21.599, - "write_ok": 1199443, - "write_bp": 0, - "write_err": 0, - "read_ok": 0, - "read_bp": 0, - "read_err": 0, - "events_received": 996317, - "control_events_received": 998317, - "elapsed_secs": 35.172896945, - "drive_secs": 30.0, - "level": 20, - "readers": 0, - "subscribers": 2000, - "writer_rate": 20, - "read_rate": 0, - "status": "ok", - "reason": "complete" - }, - "33": { - "write_ops_per_sec": 65910.46666666666, - "write_p50": 9.239, - "write_p99": 29.183, - "read_ops_per_sec": 0.0, - "read_mib_per_sec": 0.0, - "read_p50": null, - "read_p99": null, - "events_per_sec": 65673.86666666667, - "delivery_p50": 10.167, - "delivery_p99": 44.735, - "write_ok": 1977314, - "write_bp": 0, - "write_err": 0, - "read_ok": 0, - "read_bp": 0, - "read_err": 0, - "events_received": 1970216, - "control_events_received": 1972216, - "elapsed_secs": 30.238965801, - "drive_secs": 30.0, - "level": 33, - "readers": 0, - "subscribers": 2000, - "writer_rate": 33, - "read_rate": 0, - "status": "ok", - "reason": "complete" - }, - "0": { - "write_ops_per_sec": 86268.53333333334, - "write_p50": 22.831, - "write_p99": 41.119, - "read_ops_per_sec": 0.0, - "read_mib_per_sec": 0.0, - "read_p50": null, - "read_p99": null, - "events_per_sec": 65537.13333333333, - "delivery_p50": 37.919, - "delivery_p99": 57.311, - "write_ok": 2588056, - "write_bp": 0, - "write_err": 0, - "read_ok": 0, - "read_bp": 0, - "read_err": 0, - "events_received": 1966114, - "control_events_received": 1968114, - "elapsed_secs": 35.134834253, - "drive_secs": 30.0, - "level": 0, - "readers": 0, - "subscribers": 2000, - "writer_rate": 0, - "read_rate": 0, - "status": "ok", - "reason": "complete" - } - } - } - } -} \ No newline at end of file diff --git a/results-2026-07-02/mixed-writes-hot/aggregate.csv b/results-2026-07-02/mixed-writes-hot/aggregate.csv deleted file mode 100644 index 919d8f2..0000000 --- a/results-2026-07-02/mixed-writes-hot/aggregate.csv +++ /dev/null @@ -1,4 +0,0 @@ -mode,stream_count,sweep,level,readers,subscribers,writer_rate,read_rate,write_ops_per_sec,write_p50,write_p99,read_ops_per_sec,read_mib_per_sec,read_p50,read_p99,events_per_sec,delivery_p50,delivery_p99,write_bp,write_err,read_bp,read_err,status,reason -wal,50,readers,16,16,0,977,0,48874.45,0.413,1.653,2138.8,2342.6493408203123,3.577,38.559,0.0,,,0,0,0,0,ok,complete -wal,50,readers,64,64,0,977,0,47714.75,0.546,9.607,2251.75,2321.505236816406,11.055,223.743,0.0,,,0,0,0,0,ok,complete -wal,50,readers,256,256,0,977,0,7654.65,3.599,35.359,6133.05,2282.3429321289063,30.655,183.039,0.0,,,0,0,0,0,ok,complete diff --git a/results-2026-07-02/mixed-writes-hot/aggregate.json b/results-2026-07-02/mixed-writes-hot/aggregate.json deleted file mode 100644 index b4152f7..0000000 --- a/results-2026-07-02/mixed-writes-hot/aggregate.json +++ /dev/null @@ -1,80 +0,0 @@ -[ - { - "mode": "wal", - "stream_count": 50, - "sweep": "readers", - "level": 16, - "readers": 16, - "subscribers": 0, - "writer_rate": 977, - "read_rate": 0, - "write_ops_per_sec": 48874.45, - "write_p50": 0.413, - "write_p99": 1.653, - "read_ops_per_sec": 2138.8, - "read_mib_per_sec": 2342.6493408203123, - "read_p50": 3.577, - "read_p99": 38.559, - "events_per_sec": 0.0, - "delivery_p50": null, - "delivery_p99": null, - "write_bp": 0, - "write_err": 0, - "read_bp": 0, - "read_err": 0, - "status": "ok", - "reason": "complete" - }, - { - "mode": "wal", - "stream_count": 50, - "sweep": "readers", - "level": 64, - "readers": 64, - "subscribers": 0, - "writer_rate": 977, - "read_rate": 0, - "write_ops_per_sec": 47714.75, - "write_p50": 0.546, - "write_p99": 9.607, - "read_ops_per_sec": 2251.75, - "read_mib_per_sec": 2321.505236816406, - "read_p50": 11.055, - "read_p99": 223.743, - "events_per_sec": 0.0, - "delivery_p50": null, - "delivery_p99": null, - "write_bp": 0, - "write_err": 0, - "read_bp": 0, - "read_err": 0, - "status": "ok", - "reason": "complete" - }, - { - "mode": "wal", - "stream_count": 50, - "sweep": "readers", - "level": 256, - "readers": 256, - "subscribers": 0, - "writer_rate": 977, - "read_rate": 0, - "write_ops_per_sec": 7654.65, - "write_p50": 3.599, - "write_p99": 35.359, - "read_ops_per_sec": 6133.05, - "read_mib_per_sec": 2282.3429321289063, - "read_p50": 30.655, - "read_p99": 183.039, - "events_per_sec": 0.0, - "delivery_p50": null, - "delivery_p99": null, - "write_bp": 0, - "write_err": 0, - "read_bp": 0, - "read_err": 0, - "status": "ok", - "reason": "complete" - } -] \ No newline at end of file diff --git a/results-2026-07-02/mixed-writes-hot/report.md b/results-2026-07-02/mixed-writes-hot/report.md deleted file mode 100644 index 4fd6545..0000000 --- a/results-2026-07-02/mixed-writes-hot/report.md +++ /dev/null @@ -1,15 +0,0 @@ -# mixed-writes-hot — mixed read/write interference report - -Sweep axis: **readers**. Latency cells are p50/p99 ms. ‡ = backpressure (503/429) observed in that class. - -## wal — 50 streams - -| readers | write ops/s | write ms | read ops/s | read MiB/s | read ms | deliv rec/s | deliv ms | status | -|---|---|---|---|---|---|---|---|---| -| 16 | 48874 | 0.4/1.7 | 2139 | 2342.6 | 3.6/38.6 | 0 | — | ok | -| 64 | 47715 | 0.5/9.6 | 2252 | 2321.5 | 11.1/223.7 | 0 | — | ok | -| 256 | 7655 | 3.6/35.4 | 6133 | 2282.3 | 30.7/183.0 | 0 | — | ok | - -## Findings - -_TODO: written by hand on top of the generated data._ diff --git a/results-2026-07-02/mixed-writes-hot/wal/cells.json b/results-2026-07-02/mixed-writes-hot/wal/cells.json deleted file mode 100644 index eb05124..0000000 --- a/results-2026-07-02/mixed-writes-hot/wal/cells.json +++ /dev/null @@ -1,98 +0,0 @@ -{ - "cells": { - "50": { - "stream_count": 50, - "image_digest": "92f4a6387253", - "complete": true, - "levels": { - "16": { - "write_ops_per_sec": 48874.45, - "write_p50": 0.413, - "write_p99": 1.653, - "read_ops_per_sec": 2138.8, - "read_mib_per_sec": 2342.6493408203123, - "read_p50": 3.577, - "read_p99": 38.559, - "events_per_sec": 0.0, - "delivery_p50": null, - "delivery_p99": null, - "write_ok": 977489, - "write_bp": 0, - "write_err": 0, - "read_ok": 42776, - "read_bp": 0, - "read_err": 0, - "events_received": 0, - "control_events_received": 0, - "elapsed_secs": 20.021721405, - "drive_secs": 20.0, - "level": 16, - "readers": 16, - "subscribers": 0, - "writer_rate": 977, - "read_rate": 0, - "status": "ok", - "reason": "complete" - }, - "64": { - "write_ops_per_sec": 47714.75, - "write_p50": 0.546, - "write_p99": 9.607, - "read_ops_per_sec": 2251.75, - "read_mib_per_sec": 2321.505236816406, - "read_p50": 11.055, - "read_p99": 223.743, - "events_per_sec": 0.0, - "delivery_p50": null, - "delivery_p99": null, - "write_ok": 954295, - "write_bp": 0, - "write_err": 0, - "read_ok": 45035, - "read_bp": 0, - "read_err": 0, - "events_received": 0, - "control_events_received": 0, - "elapsed_secs": 20.078124411, - "drive_secs": 20.0, - "level": 64, - "readers": 64, - "subscribers": 0, - "writer_rate": 977, - "read_rate": 0, - "status": "ok", - "reason": "complete" - }, - "256": { - "write_ops_per_sec": 7654.65, - "write_p50": 3.599, - "write_p99": 35.359, - "read_ops_per_sec": 6133.05, - "read_mib_per_sec": 2282.3429321289063, - "read_p50": 30.655, - "read_p99": 183.039, - "events_per_sec": 0.0, - "delivery_p50": null, - "delivery_p99": null, - "write_ok": 153093, - "write_bp": 0, - "write_err": 0, - "read_ok": 122661, - "read_bp": 0, - "read_err": 0, - "events_received": 0, - "control_events_received": 0, - "elapsed_secs": 20.052673988, - "drive_secs": 20.0, - "level": 256, - "readers": 256, - "subscribers": 0, - "writer_rate": 977, - "read_rate": 0, - "status": "ok", - "reason": "complete" - } - } - } - } -} \ No newline at end of file diff --git a/results-2026-07-02/mixed-writes/aggregate.csv b/results-2026-07-02/mixed-writes/aggregate.csv deleted file mode 100644 index 61c8261..0000000 --- a/results-2026-07-02/mixed-writes/aggregate.csv +++ /dev/null @@ -1,5 +0,0 @@ -mode,stream_count,sweep,level,readers,subscribers,writer_rate,read_rate,write_ops_per_sec,write_p50,write_p99,read_ops_per_sec,read_mib_per_sec,read_p50,read_p99,events_per_sec,delivery_p50,delivery_p99,write_bp,write_err,read_bp,read_err,status,reason -wal,10000,readers,0,0,0,5,0,49964.416666666664,10.055,422.143,0.0,0.0,,,0.0,,,0,0,0,0,ok,complete -wal,10000,readers,1000,1000,0,5,0,50026.05,11.415,441.087,49.916666666666664,3.044083658854167,0.488,418.559,0.0,,,0,0,0,0,ok,complete -wal,10000,readers,10000,10000,0,5,0,49825.63333333333,5.927,353.791,498.93333333333334,30.363899739583335,0.561,403.967,0.0,,,0,0,0,0,ok,complete -wal,10000,readers,100000,100000,0,5,0,50041.48333333333,14.863,454.655,4987.333333333333,303.7662068684896,0.524,879.615,0.0,,,0,0,0,0,ok,complete diff --git a/results-2026-07-02/mixed-writes/aggregate.json b/results-2026-07-02/mixed-writes/aggregate.json deleted file mode 100644 index 117a63d..0000000 --- a/results-2026-07-02/mixed-writes/aggregate.json +++ /dev/null @@ -1,106 +0,0 @@ -[ - { - "mode": "wal", - "stream_count": 10000, - "sweep": "readers", - "level": 0, - "readers": 0, - "subscribers": 0, - "writer_rate": 5, - "read_rate": 0, - "write_ops_per_sec": 49964.416666666664, - "write_p50": 10.055, - "write_p99": 422.143, - "read_ops_per_sec": 0.0, - "read_mib_per_sec": 0.0, - "read_p50": null, - "read_p99": null, - "events_per_sec": 0.0, - "delivery_p50": null, - "delivery_p99": null, - "write_bp": 0, - "write_err": 0, - "read_bp": 0, - "read_err": 0, - "status": "ok", - "reason": "complete" - }, - { - "mode": "wal", - "stream_count": 10000, - "sweep": "readers", - "level": 1000, - "readers": 1000, - "subscribers": 0, - "writer_rate": 5, - "read_rate": 0, - "write_ops_per_sec": 50026.05, - "write_p50": 11.415, - "write_p99": 441.087, - "read_ops_per_sec": 49.916666666666664, - "read_mib_per_sec": 3.044083658854167, - "read_p50": 0.488, - "read_p99": 418.559, - "events_per_sec": 0.0, - "delivery_p50": null, - "delivery_p99": null, - "write_bp": 0, - "write_err": 0, - "read_bp": 0, - "read_err": 0, - "status": "ok", - "reason": "complete" - }, - { - "mode": "wal", - "stream_count": 10000, - "sweep": "readers", - "level": 10000, - "readers": 10000, - "subscribers": 0, - "writer_rate": 5, - "read_rate": 0, - "write_ops_per_sec": 49825.63333333333, - "write_p50": 5.927, - "write_p99": 353.791, - "read_ops_per_sec": 498.93333333333334, - "read_mib_per_sec": 30.363899739583335, - "read_p50": 0.561, - "read_p99": 403.967, - "events_per_sec": 0.0, - "delivery_p50": null, - "delivery_p99": null, - "write_bp": 0, - "write_err": 0, - "read_bp": 0, - "read_err": 0, - "status": "ok", - "reason": "complete" - }, - { - "mode": "wal", - "stream_count": 10000, - "sweep": "readers", - "level": 100000, - "readers": 100000, - "subscribers": 0, - "writer_rate": 5, - "read_rate": 0, - "write_ops_per_sec": 50041.48333333333, - "write_p50": 14.863, - "write_p99": 454.655, - "read_ops_per_sec": 4987.333333333333, - "read_mib_per_sec": 303.7662068684896, - "read_p50": 0.524, - "read_p99": 879.615, - "events_per_sec": 0.0, - "delivery_p50": null, - "delivery_p99": null, - "write_bp": 0, - "write_err": 0, - "read_bp": 0, - "read_err": 0, - "status": "ok", - "reason": "complete" - } -] \ No newline at end of file diff --git a/results-2026-07-02/mixed-writes/report.md b/results-2026-07-02/mixed-writes/report.md deleted file mode 100644 index 7901a8b..0000000 --- a/results-2026-07-02/mixed-writes/report.md +++ /dev/null @@ -1,16 +0,0 @@ -# mixed-writes — mixed read/write interference report - -Sweep axis: **readers**. Latency cells are p50/p99 ms. ‡ = backpressure (503/429) observed in that class. - -## wal — 10000 streams - -| readers | write ops/s | write ms | read ops/s | read MiB/s | read ms | deliv rec/s | deliv ms | status | -|---|---|---|---|---|---|---|---|---| -| 0 | 49964 | 10.1/422.1 | 0 | 0.0 | — | 0 | — | ok | -| 1000 | 50026 | 11.4/441.1 | 50 | 3.0 | 0.5/418.6 | 0 | — | ok | -| 10000 | 49826 | 5.9/353.8 | 499 | 30.4 | 0.6/404.0 | 0 | — | ok | -| 100000 | 50041 | 14.9/454.7 | 4987 | 303.8 | 0.5/879.6 | 0 | — | ok | - -## Findings - -_TODO: written by hand on top of the generated data._ diff --git a/results-2026-07-02/mixed-writes/wal/cells.json b/results-2026-07-02/mixed-writes/wal/cells.json deleted file mode 100644 index 6fc6d13..0000000 --- a/results-2026-07-02/mixed-writes/wal/cells.json +++ /dev/null @@ -1,127 +0,0 @@ -{ - "cells": { - "10000": { - "stream_count": 10000, - "image_digest": "92f4a6387253", - "complete": true, - "levels": { - "0": { - "write_ops_per_sec": 49964.416666666664, - "write_p50": 10.055, - "write_p99": 422.143, - "read_ops_per_sec": 0.0, - "read_mib_per_sec": 0.0, - "read_p50": null, - "read_p99": null, - "events_per_sec": 0.0, - "delivery_p50": null, - "delivery_p99": null, - "write_ok": 2997865, - "write_bp": 0, - "write_err": 0, - "read_ok": 0, - "read_bp": 0, - "read_err": 0, - "events_received": 0, - "control_events_received": 0, - "elapsed_secs": 60.482057524, - "drive_secs": 60.0, - "level": 0, - "readers": 0, - "subscribers": 0, - "writer_rate": 5, - "read_rate": 0, - "status": "ok", - "reason": "complete" - }, - "1000": { - "write_ops_per_sec": 50026.05, - "write_p50": 11.415, - "write_p99": 441.087, - "read_ops_per_sec": 49.916666666666664, - "read_mib_per_sec": 3.044083658854167, - "read_p50": 0.488, - "read_p99": 418.559, - "events_per_sec": 0.0, - "delivery_p50": null, - "delivery_p99": null, - "write_ok": 3001563, - "write_bp": 0, - "write_err": 0, - "read_ok": 2995, - "read_bp": 0, - "read_err": 0, - "events_received": 0, - "control_events_received": 0, - "elapsed_secs": 90.000492785, - "drive_secs": 60.0, - "level": 1000, - "readers": 1000, - "subscribers": 0, - "writer_rate": 5, - "read_rate": 0, - "status": "ok", - "reason": "complete" - }, - "10000": { - "write_ops_per_sec": 49825.63333333333, - "write_p50": 5.927, - "write_p99": 353.791, - "read_ops_per_sec": 498.93333333333334, - "read_mib_per_sec": 30.363899739583335, - "read_p50": 0.561, - "read_p99": 403.967, - "events_per_sec": 0.0, - "delivery_p50": null, - "delivery_p99": null, - "write_ok": 2989538, - "write_bp": 0, - "write_err": 0, - "read_ok": 29936, - "read_bp": 0, - "read_err": 0, - "events_received": 0, - "control_events_received": 0, - "elapsed_secs": 90.027624664, - "drive_secs": 60.0, - "level": 10000, - "readers": 10000, - "subscribers": 0, - "writer_rate": 5, - "read_rate": 0, - "status": "ok", - "reason": "complete" - }, - "100000": { - "write_ops_per_sec": 50041.48333333333, - "write_p50": 14.863, - "write_p99": 454.655, - "read_ops_per_sec": 4987.333333333333, - "read_mib_per_sec": 303.7662068684896, - "read_p50": 0.524, - "read_p99": 879.615, - "events_per_sec": 0.0, - "delivery_p50": null, - "delivery_p99": null, - "write_ok": 3002489, - "write_bp": 0, - "write_err": 0, - "read_ok": 299240, - "read_bp": 0, - "read_err": 0, - "events_received": 0, - "control_events_received": 0, - "elapsed_secs": 90.200862867, - "drive_secs": 60.0, - "level": 100000, - "readers": 100000, - "subscribers": 0, - "writer_rate": 5, - "read_rate": 0, - "status": "ok", - "reason": "complete" - } - } - } - } -} \ No newline at end of file diff --git a/results-2026-07-02/reads-catchup/aggregate.csv b/results-2026-07-02/reads-catchup/aggregate.csv deleted file mode 100644 index ce6188f..0000000 --- a/results-2026-07-02/reads-catchup/aggregate.csv +++ /dev/null @@ -1,17 +0,0 @@ -mode,stream_count,connections,ops_per_sec,bytes_per_sec,mib_per_sec,p50,p99,backpressure,other_err,status,is_peak -ursula,10,8,146.93333333333334,2465132270.9333334,2350.9,52.159,85.887,0,0,ok,False -ursula,10,32,148.46666666666667,2490857335.4666667,2375.5,202.239,516.607,0,0,ok,True -ursula,10,128,0.0,0.0,0.0,,,0,0,error,False -ursula,10,512,0.0,0.0,0.0,,,0,0,error,False -ursula,100,8,0.0,0.0,0.0,,,0,0,error,True -ursula,100,32,0.0,0.0,0.0,,,0,0,error,False -ursula,100,128,0.0,0.0,0.0,,,0,0,error,False -ursula,100,512,0.0,0.0,0.0,,,0,0,error,False -wal,10,8,84.06666666666666,1410404625.0666666,1345.1,94.719,110.847,0,0,ok,False -wal,10,32,148.8,2496449740.8,2380.8,217.727,237.311,0,0,ok,True -wal,10,128,0.0,0.0,0.0,,,0,0,error,False -wal,10,512,0.0,0.0,0.0,,,0,0,error,False -wal,100,8,83.33333333333333,1398101333.3333333,1333.3,95.551,111.743,0,0,ok,False -wal,100,32,148.86666666666667,2497568221.866667,2381.9,218.495,238.207,0,0,ok,True -wal,100,128,0.0,0.0,0.0,,,0,0,error,False -wal,100,512,0.0,0.0,0.0,,,0,0,error,False diff --git a/results-2026-07-02/reads-catchup/aggregate.json b/results-2026-07-02/reads-catchup/aggregate.json deleted file mode 100644 index 460524e..0000000 --- a/results-2026-07-02/reads-catchup/aggregate.json +++ /dev/null @@ -1,226 +0,0 @@ -[ - { - "mode": "ursula", - "stream_count": 10, - "connections": 8, - "ops_per_sec": 146.93333333333334, - "bytes_per_sec": 2465132270.9333334, - "mib_per_sec": 2350.9, - "p50": 52.159, - "p99": 85.887, - "backpressure": 0, - "other_err": 0, - "status": "ok", - "is_peak": false - }, - { - "mode": "ursula", - "stream_count": 10, - "connections": 32, - "ops_per_sec": 148.46666666666667, - "bytes_per_sec": 2490857335.4666667, - "mib_per_sec": 2375.5, - "p50": 202.239, - "p99": 516.607, - "backpressure": 0, - "other_err": 0, - "status": "ok", - "is_peak": true - }, - { - "mode": "ursula", - "stream_count": 10, - "connections": 128, - "ops_per_sec": 0.0, - "bytes_per_sec": 0.0, - "mib_per_sec": 0.0, - "p50": null, - "p99": null, - "backpressure": 0, - "other_err": 0, - "status": "error", - "is_peak": false - }, - { - "mode": "ursula", - "stream_count": 10, - "connections": 512, - "ops_per_sec": 0.0, - "bytes_per_sec": 0.0, - "mib_per_sec": 0.0, - "p50": null, - "p99": null, - "backpressure": 0, - "other_err": 0, - "status": "error", - "is_peak": false - }, - { - "mode": "ursula", - "stream_count": 100, - "connections": 8, - "ops_per_sec": 0.0, - "bytes_per_sec": 0.0, - "mib_per_sec": 0.0, - "p50": null, - "p99": null, - "backpressure": 0, - "other_err": 0, - "status": "error", - "is_peak": true - }, - { - "mode": "ursula", - "stream_count": 100, - "connections": 32, - "ops_per_sec": 0.0, - "bytes_per_sec": 0.0, - "mib_per_sec": 0.0, - "p50": null, - "p99": null, - "backpressure": 0, - "other_err": 0, - "status": "error", - "is_peak": false - }, - { - "mode": "ursula", - "stream_count": 100, - "connections": 128, - "ops_per_sec": 0.0, - "bytes_per_sec": 0.0, - "mib_per_sec": 0.0, - "p50": null, - "p99": null, - "backpressure": 0, - "other_err": 0, - "status": "error", - "is_peak": false - }, - { - "mode": "ursula", - "stream_count": 100, - "connections": 512, - "ops_per_sec": 0.0, - "bytes_per_sec": 0.0, - "mib_per_sec": 0.0, - "p50": null, - "p99": null, - "backpressure": 0, - "other_err": 0, - "status": "error", - "is_peak": false - }, - { - "mode": "wal", - "stream_count": 10, - "connections": 8, - "ops_per_sec": 84.06666666666666, - "bytes_per_sec": 1410404625.0666666, - "mib_per_sec": 1345.1, - "p50": 94.719, - "p99": 110.847, - "backpressure": 0, - "other_err": 0, - "status": "ok", - "is_peak": false - }, - { - "mode": "wal", - "stream_count": 10, - "connections": 32, - "ops_per_sec": 148.8, - "bytes_per_sec": 2496449740.8, - "mib_per_sec": 2380.8, - "p50": 217.727, - "p99": 237.311, - "backpressure": 0, - "other_err": 0, - "status": "ok", - "is_peak": true - }, - { - "mode": "wal", - "stream_count": 10, - "connections": 128, - "ops_per_sec": 0.0, - "bytes_per_sec": 0.0, - "mib_per_sec": 0.0, - "p50": null, - "p99": null, - "backpressure": 0, - "other_err": 0, - "status": "error", - "is_peak": false - }, - { - "mode": "wal", - "stream_count": 10, - "connections": 512, - "ops_per_sec": 0.0, - "bytes_per_sec": 0.0, - "mib_per_sec": 0.0, - "p50": null, - "p99": null, - "backpressure": 0, - "other_err": 0, - "status": "error", - "is_peak": false - }, - { - "mode": "wal", - "stream_count": 100, - "connections": 8, - "ops_per_sec": 83.33333333333333, - "bytes_per_sec": 1398101333.3333333, - "mib_per_sec": 1333.3, - "p50": 95.551, - "p99": 111.743, - "backpressure": 0, - "other_err": 0, - "status": "ok", - "is_peak": false - }, - { - "mode": "wal", - "stream_count": 100, - "connections": 32, - "ops_per_sec": 148.86666666666667, - "bytes_per_sec": 2497568221.866667, - "mib_per_sec": 2381.9, - "p50": 218.495, - "p99": 238.207, - "backpressure": 0, - "other_err": 0, - "status": "ok", - "is_peak": true - }, - { - "mode": "wal", - "stream_count": 100, - "connections": 128, - "ops_per_sec": 0.0, - "bytes_per_sec": 0.0, - "mib_per_sec": 0.0, - "p50": null, - "p99": null, - "backpressure": 0, - "other_err": 0, - "status": "error", - "is_peak": false - }, - { - "mode": "wal", - "stream_count": 100, - "connections": 512, - "ops_per_sec": 0.0, - "bytes_per_sec": 0.0, - "mib_per_sec": 0.0, - "p50": null, - "p99": null, - "backpressure": 0, - "other_err": 0, - "status": "error", - "is_peak": false - } -] \ No newline at end of file diff --git a/results-2026-07-02/reads-catchup/report.md b/results-2026-07-02/reads-catchup/report.md deleted file mode 100644 index 7693c38..0000000 --- a/results-2026-07-02/reads-catchup/report.md +++ /dev/null @@ -1,29 +0,0 @@ -# reads-catchup — read-scalability report - -Each cell: aggregate read throughput (MiB/s) @ p99 latency (ms). ‡ = backpressure (503/429) observed at this load. - -## wal — throughput @ p99 over stream_count × connections - -| streams | 8 | 32 | 128 | 512 | -|---|---|---|---|---| -| 10 | 1345MiB/s@111ms | 2381MiB/s@237ms | ERR(0) | ERR(0) | -| 100 | 1333MiB/s@112ms | 2382MiB/s@238ms | ERR(0) | ERR(0) | - -Peak read throughput per cardinality: -- streams=10: 2381 MiB/s at 32 connections -- streams=100: 2382 MiB/s at 32 connections - -## ursula — throughput @ p99 over stream_count × connections - -| streams | 8 | 32 | 128 | 512 | -|---|---|---|---|---| -| 10 | 2351MiB/s@86ms | 2375MiB/s@517ms | ERR(0) | ERR(0) | -| 100 | ERR(0) | ERR(0) | ERR(0) | ERR(0) | - -Peak read throughput per cardinality: -- streams=10: 2375 MiB/s at 32 connections -- streams=100: 0 MiB/s at 8 connections - -## Findings - -_TODO: written by hand on top of the generated data._ diff --git a/results-2026-07-02/reads-catchup/ursula/cells.json b/results-2026-07-02/reads-catchup/ursula/cells.json deleted file mode 100644 index 3373b64..0000000 --- a/results-2026-07-02/reads-catchup/ursula/cells.json +++ /dev/null @@ -1,106 +0,0 @@ -{ - "cells": { - "10": { - "stream_count": 10, - "image_digest": "5669f4bbeed1", - "complete": true, - "connections": { - "8": { - "connections": 8, - "ops_per_sec": 146.93333333333334, - "bytes_per_sec": 2465132270.9333334, - "p50": 52.159, - "p99": 85.887, - "backpressure": 0, - "other_err": 0, - "status": "ok", - "reason": "complete" - }, - "32": { - "connections": 32, - "ops_per_sec": 148.46666666666667, - "bytes_per_sec": 2490857335.4666667, - "p50": 202.239, - "p99": 516.607, - "backpressure": 0, - "other_err": 0, - "status": "ok", - "reason": "complete" - }, - "128": { - "connections": 128, - "ops_per_sec": 0.0, - "bytes_per_sec": 0.0, - "p50": null, - "p99": null, - "backpressure": 0, - "other_err": 0, - "status": "error", - "reason": "no_reads" - }, - "512": { - "connections": 512, - "ops_per_sec": 0.0, - "bytes_per_sec": 0.0, - "p50": null, - "p99": null, - "backpressure": 0, - "other_err": 0, - "status": "error", - "reason": "no_reads" - } - } - }, - "100": { - "stream_count": 100, - "image_digest": "5669f4bbeed1", - "complete": true, - "connections": { - "8": { - "connections": 8, - "ops_per_sec": 0.0, - "bytes_per_sec": 0.0, - "p50": null, - "p99": null, - "backpressure": 0, - "other_err": 0, - "status": "error", - "reason": "no_reads" - }, - "32": { - "connections": 32, - "ops_per_sec": 0.0, - "bytes_per_sec": 0.0, - "p50": null, - "p99": null, - "backpressure": 0, - "other_err": 0, - "status": "error", - "reason": "no_reads" - }, - "128": { - "connections": 128, - "ops_per_sec": 0.0, - "bytes_per_sec": 0.0, - "p50": null, - "p99": null, - "backpressure": 0, - "other_err": 0, - "status": "error", - "reason": "no_reads" - }, - "512": { - "connections": 512, - "ops_per_sec": 0.0, - "bytes_per_sec": 0.0, - "p50": null, - "p99": null, - "backpressure": 0, - "other_err": 0, - "status": "error", - "reason": "no_reads" - } - } - } - } -} \ No newline at end of file diff --git a/results-2026-07-02/reads-catchup/wal/cells.json b/results-2026-07-02/reads-catchup/wal/cells.json deleted file mode 100644 index 25697d8..0000000 --- a/results-2026-07-02/reads-catchup/wal/cells.json +++ /dev/null @@ -1,106 +0,0 @@ -{ - "cells": { - "10": { - "stream_count": 10, - "image_digest": "92f4a6387253", - "complete": true, - "connections": { - "8": { - "connections": 8, - "ops_per_sec": 84.06666666666666, - "bytes_per_sec": 1410404625.0666666, - "p50": 94.719, - "p99": 110.847, - "backpressure": 0, - "other_err": 0, - "status": "ok", - "reason": "complete" - }, - "32": { - "connections": 32, - "ops_per_sec": 148.8, - "bytes_per_sec": 2496449740.8, - "p50": 217.727, - "p99": 237.311, - "backpressure": 0, - "other_err": 0, - "status": "ok", - "reason": "complete" - }, - "128": { - "connections": 128, - "ops_per_sec": 0.0, - "bytes_per_sec": 0.0, - "p50": null, - "p99": null, - "backpressure": 0, - "other_err": 0, - "status": "error", - "reason": "no_reads" - }, - "512": { - "connections": 512, - "ops_per_sec": 0.0, - "bytes_per_sec": 0.0, - "p50": null, - "p99": null, - "backpressure": 0, - "other_err": 0, - "status": "error", - "reason": "no_reads" - } - } - }, - "100": { - "stream_count": 100, - "image_digest": "92f4a6387253", - "complete": true, - "connections": { - "8": { - "connections": 8, - "ops_per_sec": 83.33333333333333, - "bytes_per_sec": 1398101333.3333333, - "p50": 95.551, - "p99": 111.743, - "backpressure": 0, - "other_err": 0, - "status": "ok", - "reason": "complete" - }, - "32": { - "connections": 32, - "ops_per_sec": 148.86666666666667, - "bytes_per_sec": 2497568221.866667, - "p50": 218.495, - "p99": 238.207, - "backpressure": 0, - "other_err": 0, - "status": "ok", - "reason": "complete" - }, - "128": { - "connections": 128, - "ops_per_sec": 0.0, - "bytes_per_sec": 0.0, - "p50": null, - "p99": null, - "backpressure": 0, - "other_err": 0, - "status": "error", - "reason": "no_reads" - }, - "512": { - "connections": 512, - "ops_per_sec": 0.0, - "bytes_per_sec": 0.0, - "p50": null, - "p99": null, - "backpressure": 0, - "other_err": 0, - "status": "error", - "reason": "no_reads" - } - } - } - } -} \ No newline at end of file diff --git a/results-2026-07-02/reads-sse-remote/aggregate.csv b/results-2026-07-02/reads-sse-remote/aggregate.csv deleted file mode 100644 index cfb810c..0000000 --- a/results-2026-07-02/reads-sse-remote/aggregate.csv +++ /dev/null @@ -1,17 +0,0 @@ -mode,stream_count,connections,ops_per_sec,bytes_per_sec,mib_per_sec,p50,p99,backpressure,other_err,status,is_peak -ursula,10,64,3204.266666666667,820292.2666666667,0.8,0.959,1.482,0,0,ok,False -ursula,10,256,12817.066666666668,3281169.066666667,3.1,1.309,1.89,0,0,ok,False -ursula,10,1024,51261.4,13122918.4,12.5,1.73,2.659,0,0,ok,False -ursula,10,2048,102532.8,26248396.8,25.0,1.867,2.765,0,0,ok,True -ursula,100,64,3068.9333333333334,785646.9333333333,0.7,1.238,42.175,0,0,ok,False -ursula,100,256,11841.533333333333,3031432.533333333,2.9,1.923,47.295,0,0,ok,False -ursula,100,1024,44130.13333333333,11297314.133333333,10.8,1.976,56.735,0,0,ok,False -ursula,100,2048,80552.0,20621312.0,19.7,1.872,62.879,0,0,ok,True -wal,10,64,3204.266666666667,820292.2666666667,0.8,1.07,1.415,0,0,ok,False -wal,10,256,12817.066666666668,3281169.066666667,3.1,1.533,2.097,0,0,ok,False -wal,10,1024,51268.26666666667,13124676.266666668,12.5,1.45,2.593,0,0,ok,False -wal,10,2048,102536.53333333334,26249352.533333335,25.0,1.664,2.883,0,0,ok,True -wal,100,64,3204.266666666667,820292.2666666667,0.8,0.93,1.237,0,0,ok,False -wal,100,256,12817.066666666668,3281169.066666667,3.1,1.43,2.323,0,0,ok,False -wal,100,1024,51268.26666666667,13124676.266666668,12.5,1.329,2.407,0,0,ok,False -wal,100,2048,102537.13333333332,26249506.133333333,25.0,1.375,2.755,0,0,ok,True diff --git a/results-2026-07-02/reads-sse-remote/aggregate.json b/results-2026-07-02/reads-sse-remote/aggregate.json deleted file mode 100644 index d969776..0000000 --- a/results-2026-07-02/reads-sse-remote/aggregate.json +++ /dev/null @@ -1,226 +0,0 @@ -[ - { - "mode": "ursula", - "stream_count": 10, - "connections": 64, - "ops_per_sec": 3204.266666666667, - "bytes_per_sec": 820292.2666666667, - "mib_per_sec": 0.8, - "p50": 0.959, - "p99": 1.482, - "backpressure": 0, - "other_err": 0, - "status": "ok", - "is_peak": false - }, - { - "mode": "ursula", - "stream_count": 10, - "connections": 256, - "ops_per_sec": 12817.066666666668, - "bytes_per_sec": 3281169.066666667, - "mib_per_sec": 3.1, - "p50": 1.309, - "p99": 1.89, - "backpressure": 0, - "other_err": 0, - "status": "ok", - "is_peak": false - }, - { - "mode": "ursula", - "stream_count": 10, - "connections": 1024, - "ops_per_sec": 51261.4, - "bytes_per_sec": 13122918.4, - "mib_per_sec": 12.5, - "p50": 1.73, - "p99": 2.659, - "backpressure": 0, - "other_err": 0, - "status": "ok", - "is_peak": false - }, - { - "mode": "ursula", - "stream_count": 10, - "connections": 2048, - "ops_per_sec": 102532.8, - "bytes_per_sec": 26248396.8, - "mib_per_sec": 25.0, - "p50": 1.867, - "p99": 2.765, - "backpressure": 0, - "other_err": 0, - "status": "ok", - "is_peak": true - }, - { - "mode": "ursula", - "stream_count": 100, - "connections": 64, - "ops_per_sec": 3068.9333333333334, - "bytes_per_sec": 785646.9333333333, - "mib_per_sec": 0.7, - "p50": 1.238, - "p99": 42.175, - "backpressure": 0, - "other_err": 0, - "status": "ok", - "is_peak": false - }, - { - "mode": "ursula", - "stream_count": 100, - "connections": 256, - "ops_per_sec": 11841.533333333333, - "bytes_per_sec": 3031432.533333333, - "mib_per_sec": 2.9, - "p50": 1.923, - "p99": 47.295, - "backpressure": 0, - "other_err": 0, - "status": "ok", - "is_peak": false - }, - { - "mode": "ursula", - "stream_count": 100, - "connections": 1024, - "ops_per_sec": 44130.13333333333, - "bytes_per_sec": 11297314.133333333, - "mib_per_sec": 10.8, - "p50": 1.976, - "p99": 56.735, - "backpressure": 0, - "other_err": 0, - "status": "ok", - "is_peak": false - }, - { - "mode": "ursula", - "stream_count": 100, - "connections": 2048, - "ops_per_sec": 80552.0, - "bytes_per_sec": 20621312.0, - "mib_per_sec": 19.7, - "p50": 1.872, - "p99": 62.879, - "backpressure": 0, - "other_err": 0, - "status": "ok", - "is_peak": true - }, - { - "mode": "wal", - "stream_count": 10, - "connections": 64, - "ops_per_sec": 3204.266666666667, - "bytes_per_sec": 820292.2666666667, - "mib_per_sec": 0.8, - "p50": 1.07, - "p99": 1.415, - "backpressure": 0, - "other_err": 0, - "status": "ok", - "is_peak": false - }, - { - "mode": "wal", - "stream_count": 10, - "connections": 256, - "ops_per_sec": 12817.066666666668, - "bytes_per_sec": 3281169.066666667, - "mib_per_sec": 3.1, - "p50": 1.533, - "p99": 2.097, - "backpressure": 0, - "other_err": 0, - "status": "ok", - "is_peak": false - }, - { - "mode": "wal", - "stream_count": 10, - "connections": 1024, - "ops_per_sec": 51268.26666666667, - "bytes_per_sec": 13124676.266666668, - "mib_per_sec": 12.5, - "p50": 1.45, - "p99": 2.593, - "backpressure": 0, - "other_err": 0, - "status": "ok", - "is_peak": false - }, - { - "mode": "wal", - "stream_count": 10, - "connections": 2048, - "ops_per_sec": 102536.53333333334, - "bytes_per_sec": 26249352.533333335, - "mib_per_sec": 25.0, - "p50": 1.664, - "p99": 2.883, - "backpressure": 0, - "other_err": 0, - "status": "ok", - "is_peak": true - }, - { - "mode": "wal", - "stream_count": 100, - "connections": 64, - "ops_per_sec": 3204.266666666667, - "bytes_per_sec": 820292.2666666667, - "mib_per_sec": 0.8, - "p50": 0.93, - "p99": 1.237, - "backpressure": 0, - "other_err": 0, - "status": "ok", - "is_peak": false - }, - { - "mode": "wal", - "stream_count": 100, - "connections": 256, - "ops_per_sec": 12817.066666666668, - "bytes_per_sec": 3281169.066666667, - "mib_per_sec": 3.1, - "p50": 1.43, - "p99": 2.323, - "backpressure": 0, - "other_err": 0, - "status": "ok", - "is_peak": false - }, - { - "mode": "wal", - "stream_count": 100, - "connections": 1024, - "ops_per_sec": 51268.26666666667, - "bytes_per_sec": 13124676.266666668, - "mib_per_sec": 12.5, - "p50": 1.329, - "p99": 2.407, - "backpressure": 0, - "other_err": 0, - "status": "ok", - "is_peak": false - }, - { - "mode": "wal", - "stream_count": 100, - "connections": 2048, - "ops_per_sec": 102537.13333333332, - "bytes_per_sec": 26249506.133333333, - "mib_per_sec": 25.0, - "p50": 1.375, - "p99": 2.755, - "backpressure": 0, - "other_err": 0, - "status": "ok", - "is_peak": true - } -] \ No newline at end of file diff --git a/results-2026-07-02/reads-sse-remote/report.md b/results-2026-07-02/reads-sse-remote/report.md deleted file mode 100644 index f1dc2ea..0000000 --- a/results-2026-07-02/reads-sse-remote/report.md +++ /dev/null @@ -1,29 +0,0 @@ -# reads-sse-remote — read-scalability report - -Each cell: aggregate read throughput (MiB/s) @ p99 latency (ms). ‡ = backpressure (503/429) observed at this load. - -## wal — throughput @ p99 over stream_count × connections - -| streams | 64 | 256 | 1024 | 2048 | -|---|---|---|---|---| -| 10 | 1MiB/s@1ms | 3MiB/s@2ms | 13MiB/s@3ms | 25MiB/s@3ms | -| 100 | 1MiB/s@1ms | 3MiB/s@2ms | 13MiB/s@2ms | 25MiB/s@3ms | - -Peak read throughput per cardinality: -- streams=10: 25 MiB/s at 2048 connections -- streams=100: 25 MiB/s at 2048 connections - -## ursula — throughput @ p99 over stream_count × connections - -| streams | 64 | 256 | 1024 | 2048 | -|---|---|---|---|---| -| 10 | 1MiB/s@1ms | 3MiB/s@2ms | 13MiB/s@3ms | 25MiB/s@3ms | -| 100 | 1MiB/s@42ms | 3MiB/s@47ms | 11MiB/s@57ms | 20MiB/s@63ms | - -Peak read throughput per cardinality: -- streams=10: 25 MiB/s at 2048 connections -- streams=100: 20 MiB/s at 2048 connections - -## Findings - -_TODO: written by hand on top of the generated data._ diff --git a/results-2026-07-02/reads-sse-remote/ursula/cells.json b/results-2026-07-02/reads-sse-remote/ursula/cells.json deleted file mode 100644 index 9380e1f..0000000 --- a/results-2026-07-02/reads-sse-remote/ursula/cells.json +++ /dev/null @@ -1,106 +0,0 @@ -{ - "cells": { - "10": { - "stream_count": 10, - "image_digest": "5669f4bbeed1", - "complete": true, - "connections": { - "64": { - "connections": 64, - "ops_per_sec": 3204.266666666667, - "bytes_per_sec": 820292.2666666667, - "p50": 0.959, - "p99": 1.482, - "backpressure": 0, - "other_err": 0, - "status": "ok", - "reason": "complete" - }, - "256": { - "connections": 256, - "ops_per_sec": 12817.066666666668, - "bytes_per_sec": 3281169.066666667, - "p50": 1.309, - "p99": 1.89, - "backpressure": 0, - "other_err": 0, - "status": "ok", - "reason": "complete" - }, - "1024": { - "connections": 1024, - "ops_per_sec": 51261.4, - "bytes_per_sec": 13122918.4, - "p50": 1.73, - "p99": 2.659, - "backpressure": 0, - "other_err": 0, - "status": "ok", - "reason": "complete" - }, - "2048": { - "connections": 2048, - "ops_per_sec": 102532.8, - "bytes_per_sec": 26248396.8, - "p50": 1.867, - "p99": 2.765, - "backpressure": 0, - "other_err": 0, - "status": "ok", - "reason": "complete" - } - } - }, - "100": { - "stream_count": 100, - "image_digest": "5669f4bbeed1", - "complete": true, - "connections": { - "64": { - "connections": 64, - "ops_per_sec": 3068.9333333333334, - "bytes_per_sec": 785646.9333333333, - "p50": 1.238, - "p99": 42.175, - "backpressure": 0, - "other_err": 0, - "status": "ok", - "reason": "complete" - }, - "256": { - "connections": 256, - "ops_per_sec": 11841.533333333333, - "bytes_per_sec": 3031432.533333333, - "p50": 1.923, - "p99": 47.295, - "backpressure": 0, - "other_err": 0, - "status": "ok", - "reason": "complete" - }, - "1024": { - "connections": 1024, - "ops_per_sec": 44130.13333333333, - "bytes_per_sec": 11297314.133333333, - "p50": 1.976, - "p99": 56.735, - "backpressure": 0, - "other_err": 0, - "status": "ok", - "reason": "complete" - }, - "2048": { - "connections": 2048, - "ops_per_sec": 80552.0, - "bytes_per_sec": 20621312.0, - "p50": 1.872, - "p99": 62.879, - "backpressure": 0, - "other_err": 0, - "status": "ok", - "reason": "complete" - } - } - } - } -} \ No newline at end of file diff --git a/results-2026-07-02/reads-sse-remote/wal/cells.json b/results-2026-07-02/reads-sse-remote/wal/cells.json deleted file mode 100644 index 80a1aaf..0000000 --- a/results-2026-07-02/reads-sse-remote/wal/cells.json +++ /dev/null @@ -1,106 +0,0 @@ -{ - "cells": { - "10": { - "stream_count": 10, - "image_digest": "92f4a6387253", - "complete": true, - "connections": { - "64": { - "connections": 64, - "ops_per_sec": 3204.266666666667, - "bytes_per_sec": 820292.2666666667, - "p50": 1.07, - "p99": 1.415, - "backpressure": 0, - "other_err": 0, - "status": "ok", - "reason": "complete" - }, - "256": { - "connections": 256, - "ops_per_sec": 12817.066666666668, - "bytes_per_sec": 3281169.066666667, - "p50": 1.533, - "p99": 2.097, - "backpressure": 0, - "other_err": 0, - "status": "ok", - "reason": "complete" - }, - "1024": { - "connections": 1024, - "ops_per_sec": 51268.26666666667, - "bytes_per_sec": 13124676.266666668, - "p50": 1.45, - "p99": 2.593, - "backpressure": 0, - "other_err": 0, - "status": "ok", - "reason": "complete" - }, - "2048": { - "connections": 2048, - "ops_per_sec": 102536.53333333334, - "bytes_per_sec": 26249352.533333335, - "p50": 1.664, - "p99": 2.883, - "backpressure": 0, - "other_err": 0, - "status": "ok", - "reason": "complete" - } - } - }, - "100": { - "stream_count": 100, - "image_digest": "92f4a6387253", - "complete": true, - "connections": { - "64": { - "connections": 64, - "ops_per_sec": 3204.266666666667, - "bytes_per_sec": 820292.2666666667, - "p50": 0.93, - "p99": 1.237, - "backpressure": 0, - "other_err": 0, - "status": "ok", - "reason": "complete" - }, - "256": { - "connections": 256, - "ops_per_sec": 12817.066666666668, - "bytes_per_sec": 3281169.066666667, - "p50": 1.43, - "p99": 2.323, - "backpressure": 0, - "other_err": 0, - "status": "ok", - "reason": "complete" - }, - "1024": { - "connections": 1024, - "ops_per_sec": 51268.26666666667, - "bytes_per_sec": 13124676.266666668, - "p50": 1.329, - "p99": 2.407, - "backpressure": 0, - "other_err": 0, - "status": "ok", - "reason": "complete" - }, - "2048": { - "connections": 2048, - "ops_per_sec": 102537.13333333332, - "bytes_per_sec": 26249506.133333333, - "p50": 1.375, - "p99": 2.755, - "backpressure": 0, - "other_err": 0, - "status": "ok", - "reason": "complete" - } - } - } - } -} \ No newline at end of file diff --git a/results-2026-07-02/run-durable/aggregate.csv b/results-2026-07-02/run-durable/aggregate.csv deleted file mode 100644 index f06ba79..0000000 --- a/results-2026-07-02/run-durable/aggregate.csv +++ /dev/null @@ -1,19 +0,0 @@ -mode,stream_count,pods,throughput,p50,p99,pod_mem_mb,pod_mem_p50_mb,saturated,status,reason -memory,100,16,436185.375,0.249,0.397,60,58,True,ok,plateau -wal,100,16,457133.875,0.235,0.38,95,14,True,ok,plateau -wal-tailcache,100,24,488139.625,0.232,0.377,69,68,True,ok,plateau -memory,1000,16,479132.875,1.846,6.659,46,37,True,ok,plateau -wal,1000,24,655482.875,1.429,5.087,37,27,True,ok,plateau -wal-tailcache,1000,16,565306.375,1.516,5.471,56,46,True,ok,plateau -memory,10000,32,575104.875,2.139,122.239,151,86,True,ok,plateau -wal,10000,32,815869.75,1.81,87.039,175,75,True,ok,plateau -wal-tailcache,10000,32,794364.625,1.823,90.623,170,115,True,ok,plateau -memory,100000,80,732216.125,3.047,867.327,736,374,True,ok,plateau -wal,100000,100,1560267.0,3.227,719.359,915,726,True,ok,plateau -wal-tailcache,100000,80,1734866.375,3.169,626.687,908,706,True,ok,plateau -memory,200000,100,534176.625,25.983,1686.527,781,311,True,ok,plateau -wal,200000,100,1503496.125,80.575,973.311,987,743,True,ok,plateau -wal-tailcache,200000,100,1414943.125,50.879,1158.143,775,489,True,ok,plateau -memory,500000,400,1329370.0,33.247,2099.199,752,491,True,ok,plateau -wal,500000,250,2045868.875,151.551,2609.151,857,513,True,ok,plateau -wal-tailcache,500000,250,1892514.75,127.359,1221.631,960,716,True,ok,plateau diff --git a/results-2026-07-02/run-durable/aggregate.json b/results-2026-07-02/run-durable/aggregate.json deleted file mode 100644 index 5bb6a04..0000000 --- a/results-2026-07-02/run-durable/aggregate.json +++ /dev/null @@ -1,432 +0,0 @@ -[ - { - "mode": "memory", - "stream_count": 100, - "pods": 16, - "throughput": 436185.375, - "p50": 0.249, - "p99": 0.397, - "pod_mem_mb": 60, - "pod_mem_p50_mb": 58, - "saturated": true, - "status": "ok", - "reason": "plateau", - "walk": [ - [ - 16, - 436185.375 - ], - [ - 24, - 435389.75 - ] - ] - }, - { - "mode": "wal", - "stream_count": 100, - "pods": 16, - "throughput": 457133.875, - "p50": 0.235, - "p99": 0.38, - "pod_mem_mb": 95, - "pod_mem_p50_mb": 14, - "saturated": true, - "status": "ok", - "reason": "plateau", - "walk": [ - [ - 16, - 457133.875 - ], - [ - 24, - 482137.625 - ] - ] - }, - { - "mode": "wal-tailcache", - "stream_count": 100, - "pods": 24, - "throughput": 488139.625, - "p50": 0.232, - "p99": 0.377, - "pod_mem_mb": 69, - "pod_mem_p50_mb": 68, - "saturated": true, - "status": "ok", - "reason": "plateau", - "walk": [ - [ - 16, - 450994.75 - ], - [ - 24, - 488139.625 - ], - [ - 32, - 518751.75 - ] - ] - }, - { - "mode": "memory", - "stream_count": 1000, - "pods": 16, - "throughput": 479132.875, - "p50": 1.846, - "p99": 6.659, - "pod_mem_mb": 46, - "pod_mem_p50_mb": 37, - "saturated": true, - "status": "ok", - "reason": "plateau", - "walk": [ - [ - 16, - 479132.875 - ], - [ - 24, - 470912.25 - ] - ] - }, - { - "mode": "wal", - "stream_count": 1000, - "pods": 24, - "throughput": 655482.875, - "p50": 1.429, - "p99": 5.087, - "pod_mem_mb": 37, - "pod_mem_p50_mb": 27, - "saturated": true, - "status": "ok", - "reason": "plateau", - "walk": [ - [ - 16, - 586828.125 - ], - [ - 24, - 655482.875 - ], - [ - 32, - 690208.25 - ] - ] - }, - { - "mode": "wal-tailcache", - "stream_count": 1000, - "pods": 16, - "throughput": 565306.375, - "p50": 1.516, - "p99": 5.471, - "pod_mem_mb": 56, - "pod_mem_p50_mb": 46, - "saturated": true, - "status": "ok", - "reason": "plateau", - "walk": [ - [ - 16, - 565306.375 - ], - [ - 24, - 603933.125 - ] - ] - }, - { - "mode": "memory", - "stream_count": 10000, - "pods": 32, - "throughput": 575104.875, - "p50": 2.139, - "p99": 122.239, - "pod_mem_mb": 151, - "pod_mem_p50_mb": 86, - "saturated": true, - "status": "ok", - "reason": "plateau", - "walk": [ - [ - 32, - 575104.875 - ], - [ - 48, - 616271.875 - ] - ] - }, - { - "mode": "wal", - "stream_count": 10000, - "pods": 32, - "throughput": 815869.75, - "p50": 1.81, - "p99": 87.039, - "pod_mem_mb": 175, - "pod_mem_p50_mb": 75, - "saturated": true, - "status": "ok", - "reason": "plateau", - "walk": [ - [ - 32, - 815869.75 - ], - [ - 48, - 863481.875 - ] - ] - }, - { - "mode": "wal-tailcache", - "stream_count": 10000, - "pods": 32, - "throughput": 794364.625, - "p50": 1.823, - "p99": 90.623, - "pod_mem_mb": 170, - "pod_mem_p50_mb": 115, - "saturated": true, - "status": "ok", - "reason": "plateau", - "walk": [ - [ - 32, - 794364.625 - ], - [ - 48, - 752539.625 - ] - ] - }, - { - "mode": "memory", - "stream_count": 100000, - "pods": 80, - "throughput": 732216.125, - "p50": 3.047, - "p99": 867.327, - "pod_mem_mb": 736, - "pod_mem_p50_mb": 374, - "saturated": true, - "status": "ok", - "reason": "plateau", - "walk": [ - [ - 80, - 732216.125 - ], - [ - 100, - 407044.0 - ] - ] - }, - { - "mode": "wal", - "stream_count": 100000, - "pods": 100, - "throughput": 1560267.0, - "p50": 3.227, - "p99": 719.359, - "pod_mem_mb": 915, - "pod_mem_p50_mb": 726, - "saturated": true, - "status": "ok", - "reason": "plateau", - "walk": [ - [ - 80, - 810519.875 - ], - [ - 100, - 1560267.0 - ], - [ - 110, - 503403.625 - ] - ] - }, - { - "mode": "wal-tailcache", - "stream_count": 100000, - "pods": 80, - "throughput": 1734866.375, - "p50": 3.169, - "p99": 626.687, - "pod_mem_mb": 908, - "pod_mem_p50_mb": 706, - "saturated": true, - "status": "ok", - "reason": "plateau", - "walk": [ - [ - 80, - 1734866.375 - ], - [ - 100, - 516286.0 - ] - ] - }, - { - "mode": "memory", - "stream_count": 200000, - "pods": 100, - "throughput": 534176.625, - "p50": 25.983, - "p99": 1686.527, - "pod_mem_mb": 781, - "pod_mem_p50_mb": 311, - "saturated": true, - "status": "ok", - "reason": "plateau", - "walk": [ - [ - 100, - 534176.625 - ], - [ - 160, - 377864.25 - ] - ] - }, - { - "mode": "wal", - "stream_count": 200000, - "pods": 100, - "throughput": 1503496.125, - "p50": 80.575, - "p99": 973.311, - "pod_mem_mb": 987, - "pod_mem_p50_mb": 743, - "saturated": true, - "status": "ok", - "reason": "plateau", - "walk": [ - [ - 100, - 1503496.125 - ], - [ - 160, - 1602824.375 - ] - ] - }, - { - "mode": "wal-tailcache", - "stream_count": 200000, - "pods": 100, - "throughput": 1414943.125, - "p50": 50.879, - "p99": 1158.143, - "pod_mem_mb": 775, - "pod_mem_p50_mb": 489, - "saturated": true, - "status": "ok", - "reason": "plateau", - "walk": [ - [ - 100, - 1414943.125 - ], - [ - 160, - 1191656.375 - ] - ] - }, - { - "mode": "memory", - "stream_count": 500000, - "pods": 400, - "throughput": 1329370.0, - "p50": 33.247, - "p99": 2099.199, - "pod_mem_mb": 752, - "pod_mem_p50_mb": 491, - "saturated": true, - "status": "ok", - "reason": "plateau", - "walk": [ - [ - 250, - 1224462.625 - ], - [ - 400, - 1329370.0 - ], - [ - 625, - 361267.75 - ] - ] - }, - { - "mode": "wal", - "stream_count": 500000, - "pods": 250, - "throughput": 2045868.875, - "p50": 151.551, - "p99": 2609.151, - "pod_mem_mb": 857, - "pod_mem_p50_mb": 513, - "saturated": true, - "status": "ok", - "reason": "plateau", - "walk": [ - [ - 250, - 2045868.875 - ], - [ - 400, - 1828700.75 - ] - ] - }, - { - "mode": "wal-tailcache", - "stream_count": 500000, - "pods": 250, - "throughput": 1892514.75, - "p50": 127.359, - "p99": 1221.631, - "pod_mem_mb": 960, - "pod_mem_p50_mb": 716, - "saturated": true, - "status": "ok", - "reason": "plateau", - "walk": [ - [ - 250, - 1892514.75 - ], - [ - 400, - 1589852.0 - ] - ] - } -] \ No newline at end of file diff --git a/results-2026-07-02/run-durable/memory/cells.json b/results-2026-07-02/run-durable/memory/cells.json deleted file mode 100644 index 1db56c1..0000000 --- a/results-2026-07-02/run-durable/memory/cells.json +++ /dev/null @@ -1,146 +0,0 @@ -{ - "cells": { - "100": { - "stream_count": 100, - "image_digest": "b7d78084e886", - "walk": [ - [ - 16, - 436185.375 - ], - [ - 24, - 435389.75 - ] - ], - "pinned_pods": 16, - "throughput": 436185.375, - "p50": 0.249, - "p99": 0.397, - "pod_mem_mb": 60, - "saturated": true, - "status": "ok", - "reason": "plateau", - "pod_mem_p50_mb": 58 - }, - "1000": { - "stream_count": 1000, - "image_digest": "b7d78084e886", - "walk": [ - [ - 16, - 479132.875 - ], - [ - 24, - 470912.25 - ] - ], - "pinned_pods": 16, - "throughput": 479132.875, - "p50": 1.846, - "p99": 6.659, - "pod_mem_mb": 46, - "saturated": true, - "status": "ok", - "reason": "plateau", - "pod_mem_p50_mb": 37 - }, - "10000": { - "stream_count": 10000, - "image_digest": "b7d78084e886", - "walk": [ - [ - 32, - 575104.875 - ], - [ - 48, - 616271.875 - ] - ], - "pinned_pods": 32, - "throughput": 575104.875, - "p50": 2.139, - "p99": 122.239, - "pod_mem_mb": 151, - "saturated": true, - "status": "ok", - "reason": "plateau", - "pod_mem_p50_mb": 86 - }, - "100000": { - "stream_count": 100000, - "image_digest": "b7d78084e886", - "walk": [ - [ - 80, - 732216.125 - ], - [ - 100, - 407044.0 - ] - ], - "pinned_pods": 80, - "throughput": 732216.125, - "p50": 3.047, - "p99": 867.327, - "pod_mem_mb": 736, - "saturated": true, - "status": "ok", - "reason": "plateau", - "pod_mem_p50_mb": 374 - }, - "200000": { - "stream_count": 200000, - "image_digest": "b7d78084e886", - "walk": [ - [ - 100, - 534176.625 - ], - [ - 160, - 377864.25 - ] - ], - "pinned_pods": 100, - "throughput": 534176.625, - "p50": 25.983, - "p99": 1686.527, - "pod_mem_mb": 781, - "saturated": true, - "status": "ok", - "reason": "plateau", - "pod_mem_p50_mb": 311 - }, - "500000": { - "stream_count": 500000, - "image_digest": "b7d78084e886", - "walk": [ - [ - 250, - 1224462.625 - ], - [ - 400, - 1329370.0 - ], - [ - 625, - 361267.75 - ] - ], - "pinned_pods": 400, - "throughput": 1329370.0, - "p50": 33.247, - "p99": 2099.199, - "pod_mem_mb": 752, - "saturated": true, - "status": "ok", - "reason": "plateau", - "pod_mem_p50_mb": 491 - } - } -} \ No newline at end of file diff --git a/results-2026-07-02/run-durable/report.md b/results-2026-07-02/run-durable/report.md deleted file mode 100644 index f5cfebb..0000000 --- a/results-2026-07-02/run-durable/report.md +++ /dev/null @@ -1,56 +0,0 @@ -# run-durable — write-throughput report - -## Throughput at saturation (ops/s) - -| streams | wal | wal-tailcache | memory | -|---|---|---|---| -| 100 | 457k | 488k | 436k | -| 1000 | 655k | 565k | 479k | -| 10000 | 816k | 794k | 575k | -| 100000 | 1560k | 1735k | 732k | -| 200000 | 1503k | 1415k | 534k | -| 500000 | 2046k | 1893k | 1329k | - -† = not saturated (ladder exhausted) — treat as a lower bound. - -## Pod memory at saturation — peak / p50 (MiB) - -| streams | wal | wal-tailcache | memory | -|---|---|---|---| -| 100 | 95 / 14 | 69 / 68 | 60 / 58 | -| 1000 | 37 / 27 | 56 / 46 | 46 / 37 | -| 10000 | 175 / 75 | 170 / 115 | 151 / 86 | -| 100000 | 915 / 726 | 908 / 706 | 736 / 374 | -| 200000 | 987 / 743 | 775 / 489 | 781 / 311 | -| 500000 | 857 / 513 | 960 / 716 | 752 / 491 | - -_Pod working set = cgroup `memory.current − inactive_file` (anon + active page cache), sampled each second at the pinned rung. **peak** = high-water (catches bursts like an in-RAM Raft log filling); **p50** = median (what the server steadily holds resident). peak ≈ p50 ⇒ steadily resident; peak ≫ p50 ⇒ transient spikes._ - -## Saturation walks (pods → ops/s) - -- **memory 100**: 16:436k → 24:435k (pinned 16, plateau) -- **wal 100**: 16:457k → 24:482k (pinned 16, plateau) -- **wal-tailcache 100**: 16:451k → 24:488k → 32:519k (pinned 24, plateau) -- **memory 1000**: 16:479k → 24:471k (pinned 16, plateau) -- **wal 1000**: 16:587k → 24:655k → 32:690k (pinned 24, plateau) -- **wal-tailcache 1000**: 16:565k → 24:604k (pinned 16, plateau) -- **memory 10000**: 32:575k → 48:616k (pinned 32, plateau) -- **wal 10000**: 32:816k → 48:863k (pinned 32, plateau) -- **wal-tailcache 10000**: 32:794k → 48:753k (pinned 32, plateau) -- **memory 100000**: 80:732k → 100:407k (pinned 80, plateau) -- **wal 100000**: 80:811k → 100:1560k → 110:503k (pinned 100, plateau) -- **wal-tailcache 100000**: 80:1735k → 100:516k (pinned 80, plateau) -- **memory 200000**: 100:534k → 160:378k (pinned 100, plateau) -- **wal 200000**: 100:1503k → 160:1603k (pinned 100, plateau) -- **wal-tailcache 200000**: 100:1415k → 160:1192k (pinned 100, plateau) -- **memory 500000**: 250:1224k → 400:1329k → 625:361k (pinned 400, plateau) -- **wal 500000**: 250:2046k → 400:1829k (pinned 250, plateau) -- **wal-tailcache 500000**: 250:1893k → 400:1590k (pinned 250, plateau) - -## Findings - -_TODO: written by hand on top of the generated data._ - -## Caveats - -_Single-node best-case; not 3-node Raft. Throughput is a saturation ceiling per the ladder._ diff --git a/results-2026-07-02/run-durable/wal-tailcache/cells.json b/results-2026-07-02/run-durable/wal-tailcache/cells.json deleted file mode 100644 index 3e30caa..0000000 --- a/results-2026-07-02/run-durable/wal-tailcache/cells.json +++ /dev/null @@ -1,146 +0,0 @@ -{ - "cells": { - "100": { - "stream_count": 100, - "image_digest": "a3af0b74d273", - "walk": [ - [ - 16, - 450994.75 - ], - [ - 24, - 488139.625 - ], - [ - 32, - 518751.75 - ] - ], - "pinned_pods": 24, - "throughput": 488139.625, - "p50": 0.232, - "p99": 0.377, - "pod_mem_mb": 69, - "saturated": true, - "status": "ok", - "reason": "plateau", - "pod_mem_p50_mb": 68 - }, - "1000": { - "stream_count": 1000, - "image_digest": "a3af0b74d273", - "walk": [ - [ - 16, - 565306.375 - ], - [ - 24, - 603933.125 - ] - ], - "pinned_pods": 16, - "throughput": 565306.375, - "p50": 1.516, - "p99": 5.471, - "pod_mem_mb": 56, - "saturated": true, - "status": "ok", - "reason": "plateau", - "pod_mem_p50_mb": 46 - }, - "10000": { - "stream_count": 10000, - "image_digest": "a3af0b74d273", - "walk": [ - [ - 32, - 794364.625 - ], - [ - 48, - 752539.625 - ] - ], - "pinned_pods": 32, - "throughput": 794364.625, - "p50": 1.823, - "p99": 90.623, - "pod_mem_mb": 170, - "saturated": true, - "status": "ok", - "reason": "plateau", - "pod_mem_p50_mb": 115 - }, - "100000": { - "stream_count": 100000, - "image_digest": "a3af0b74d273", - "walk": [ - [ - 80, - 1734866.375 - ], - [ - 100, - 516286.0 - ] - ], - "pinned_pods": 80, - "throughput": 1734866.375, - "p50": 3.169, - "p99": 626.687, - "pod_mem_mb": 908, - "saturated": true, - "status": "ok", - "reason": "plateau", - "pod_mem_p50_mb": 706 - }, - "200000": { - "stream_count": 200000, - "image_digest": "a3af0b74d273", - "walk": [ - [ - 100, - 1414943.125 - ], - [ - 160, - 1191656.375 - ] - ], - "pinned_pods": 100, - "throughput": 1414943.125, - "p50": 50.879, - "p99": 1158.143, - "pod_mem_mb": 775, - "saturated": true, - "status": "ok", - "reason": "plateau", - "pod_mem_p50_mb": 489 - }, - "500000": { - "stream_count": 500000, - "image_digest": "a3af0b74d273", - "walk": [ - [ - 250, - 1892514.75 - ], - [ - 400, - 1589852.0 - ] - ], - "pinned_pods": 250, - "throughput": 1892514.75, - "p50": 127.359, - "p99": 1221.631, - "pod_mem_mb": 960, - "saturated": true, - "status": "ok", - "reason": "plateau", - "pod_mem_p50_mb": 716 - } - } -} \ No newline at end of file diff --git a/results-2026-07-02/run-durable/wal/cells.json b/results-2026-07-02/run-durable/wal/cells.json deleted file mode 100644 index d39c489..0000000 --- a/results-2026-07-02/run-durable/wal/cells.json +++ /dev/null @@ -1,150 +0,0 @@ -{ - "cells": { - "100": { - "stream_count": 100, - "image_digest": "92f4a6387253", - "walk": [ - [ - 16, - 457133.875 - ], - [ - 24, - 482137.625 - ] - ], - "pinned_pods": 16, - "throughput": 457133.875, - "p50": 0.235, - "p99": 0.38, - "pod_mem_mb": 95, - "saturated": true, - "status": "ok", - "reason": "plateau", - "pod_mem_p50_mb": 14 - }, - "1000": { - "stream_count": 1000, - "image_digest": "92f4a6387253", - "walk": [ - [ - 16, - 586828.125 - ], - [ - 24, - 655482.875 - ], - [ - 32, - 690208.25 - ] - ], - "pinned_pods": 24, - "throughput": 655482.875, - "p50": 1.429, - "p99": 5.087, - "pod_mem_mb": 37, - "saturated": true, - "status": "ok", - "reason": "plateau", - "pod_mem_p50_mb": 27 - }, - "10000": { - "stream_count": 10000, - "image_digest": "92f4a6387253", - "walk": [ - [ - 32, - 815869.75 - ], - [ - 48, - 863481.875 - ] - ], - "pinned_pods": 32, - "throughput": 815869.75, - "p50": 1.81, - "p99": 87.039, - "pod_mem_mb": 175, - "saturated": true, - "status": "ok", - "reason": "plateau", - "pod_mem_p50_mb": 75 - }, - "100000": { - "stream_count": 100000, - "image_digest": "92f4a6387253", - "walk": [ - [ - 80, - 810519.875 - ], - [ - 100, - 1560267.0 - ], - [ - 110, - 503403.625 - ] - ], - "pinned_pods": 100, - "throughput": 1560267.0, - "p50": 3.227, - "p99": 719.359, - "pod_mem_mb": 915, - "saturated": true, - "status": "ok", - "reason": "plateau", - "pod_mem_p50_mb": 726 - }, - "200000": { - "stream_count": 200000, - "image_digest": "92f4a6387253", - "walk": [ - [ - 100, - 1503496.125 - ], - [ - 160, - 1602824.375 - ] - ], - "pinned_pods": 100, - "throughput": 1503496.125, - "p50": 80.575, - "p99": 973.311, - "pod_mem_mb": 987, - "saturated": true, - "status": "ok", - "reason": "plateau", - "pod_mem_p50_mb": 743 - }, - "500000": { - "stream_count": 500000, - "image_digest": "92f4a6387253", - "walk": [ - [ - 250, - 2045868.875 - ], - [ - 400, - 1828700.75 - ] - ], - "pinned_pods": 250, - "throughput": 2045868.875, - "p50": 151.551, - "p99": 2609.151, - "pod_mem_mb": 857, - "saturated": true, - "status": "ok", - "reason": "plateau", - "pod_mem_p50_mb": 513 - } - } -} \ No newline at end of file diff --git a/results-2026-07-02/run-node/aggregate.csv b/results-2026-07-02/run-node/aggregate.csv deleted file mode 100644 index a3998dc..0000000 --- a/results-2026-07-02/run-node/aggregate.csv +++ /dev/null @@ -1,4 +0,0 @@ -mode,stream_count,pods,throughput,p50,p99,pod_mem_mb,pod_mem_p50_mb,saturated,status,reason -node,100,4,55147.75,1.671,3.829,245,171,True,ok,plateau -node,1000,4,60056.75,19.375,29.391,393,290,True,ok,plateau -node,10000,16,150917.0,21.791,43.487,803,651,True,ok,plateau diff --git a/results-2026-07-02/run-node/aggregate.json b/results-2026-07-02/run-node/aggregate.json deleted file mode 100644 index b321c15..0000000 --- a/results-2026-07-02/run-node/aggregate.json +++ /dev/null @@ -1,75 +0,0 @@ -[ - { - "mode": "node", - "stream_count": 100, - "pods": 4, - "throughput": 55147.75, - "p50": 1.671, - "p99": 3.829, - "pod_mem_mb": 245, - "pod_mem_p50_mb": 171, - "saturated": true, - "status": "ok", - "reason": "plateau", - "walk": [ - [ - 4, - 55147.75 - ], - [ - 8, - 57874.375 - ] - ] - }, - { - "mode": "node", - "stream_count": 1000, - "pods": 4, - "throughput": 60056.75, - "p50": 19.375, - "p99": 29.391, - "pod_mem_mb": 393, - "pod_mem_p50_mb": 290, - "saturated": true, - "status": "ok", - "reason": "plateau", - "walk": [ - [ - 4, - 60056.75 - ], - [ - 8, - 62403.0 - ] - ] - }, - { - "mode": "node", - "stream_count": 10000, - "pods": 16, - "throughput": 150917.0, - "p50": 21.791, - "p99": 43.487, - "pod_mem_mb": 803, - "pod_mem_p50_mb": 651, - "saturated": true, - "status": "ok", - "reason": "plateau", - "walk": [ - [ - 8, - 116379.875 - ], - [ - 16, - 150917.0 - ], - [ - 24, - 87496.375 - ] - ] - } -] \ No newline at end of file diff --git a/results-2026-07-02/run-node/node/cells.json b/results-2026-07-02/run-node/node/cells.json deleted file mode 100644 index 1437097..0000000 --- a/results-2026-07-02/run-node/node/cells.json +++ /dev/null @@ -1,77 +0,0 @@ -{ - "cells": { - "100": { - "stream_count": 100, - "image_digest": "75a11da44c80", - "walk": [ - [ - 4, - 55147.75 - ], - [ - 8, - 57874.375 - ] - ], - "pinned_pods": 4, - "throughput": 55147.75, - "p50": 1.671, - "p99": 3.829, - "pod_mem_mb": 245, - "saturated": true, - "status": "ok", - "reason": "plateau", - "pod_mem_p50_mb": 171 - }, - "1000": { - "stream_count": 1000, - "image_digest": "75a11da44c80", - "walk": [ - [ - 4, - 60056.75 - ], - [ - 8, - 62403.0 - ] - ], - "pinned_pods": 4, - "throughput": 60056.75, - "p50": 19.375, - "p99": 29.391, - "pod_mem_mb": 393, - "saturated": true, - "status": "ok", - "reason": "plateau", - "pod_mem_p50_mb": 290 - }, - "10000": { - "stream_count": 10000, - "image_digest": "75a11da44c80", - "walk": [ - [ - 8, - 116379.875 - ], - [ - 16, - 150917.0 - ], - [ - 24, - 87496.375 - ] - ], - "pinned_pods": 16, - "throughput": 150917.0, - "p50": 21.791, - "p99": 43.487, - "pod_mem_mb": 803, - "saturated": true, - "status": "ok", - "reason": "plateau", - "pod_mem_p50_mb": 651 - } - } -} \ No newline at end of file diff --git a/results-2026-07-02/run-node/report.md b/results-2026-07-02/run-node/report.md deleted file mode 100644 index 5ce68a5..0000000 --- a/results-2026-07-02/run-node/report.md +++ /dev/null @@ -1,35 +0,0 @@ -# run-node — write-throughput report - -## Throughput at saturation (ops/s) - -| streams | node | -|---|---| -| 100 | 55k | -| 1000 | 60k | -| 10000 | 151k | - -† = not saturated (ladder exhausted) — treat as a lower bound. - -## Pod memory at saturation — peak / p50 (MiB) - -| streams | node | -|---|---| -| 100 | 245 / 171 | -| 1000 | 393 / 290 | -| 10000 | 803 / 651 | - -_Pod working set = cgroup `memory.current − inactive_file` (anon + active page cache), sampled each second at the pinned rung. **peak** = high-water (catches bursts like an in-RAM Raft log filling); **p50** = median (what the server steadily holds resident). peak ≈ p50 ⇒ steadily resident; peak ≫ p50 ⇒ transient spikes._ - -## Saturation walks (pods → ops/s) - -- **node 100**: 4:55k → 8:58k (pinned 4, plateau) -- **node 1000**: 4:60k → 8:62k (pinned 4, plateau) -- **node 10000**: 8:116k → 16:151k → 24:87k (pinned 16, plateau) - -## Findings - -_TODO: written by hand on top of the generated data._ - -## Caveats - -_Single-node best-case; not 3-node Raft. Throughput is a saturation ceiling per the ladder._ diff --git a/results-2026-07-02/run-s2/aggregate.csv b/results-2026-07-02/run-s2/aggregate.csv deleted file mode 100644 index b479777..0000000 --- a/results-2026-07-02/run-s2/aggregate.csv +++ /dev/null @@ -1,3 +0,0 @@ -mode,stream_count,pods,throughput,p50,p99,pod_mem_mb,pod_mem_p50_mb,saturated,status,reason -s2,100,2,1975.0,51.135,52.351,64,50,True,ok,plateau -s2,1000,,0.0,,,,,False,error,creation_choke diff --git a/results-2026-07-02/run-s2/aggregate.json b/results-2026-07-02/run-s2/aggregate.json deleted file mode 100644 index beeae13..0000000 --- a/results-2026-07-02/run-s2/aggregate.json +++ /dev/null @@ -1,48 +0,0 @@ -[ - { - "mode": "s2", - "stream_count": 100, - "pods": 2, - "throughput": 1975.0, - "p50": 51.135, - "p99": 52.351, - "pod_mem_mb": 64, - "pod_mem_p50_mb": 50, - "saturated": true, - "status": "ok", - "reason": "plateau", - "walk": [ - [ - 2, - 1975.0 - ], - [ - 4, - 1965.625 - ] - ] - }, - { - "mode": "s2", - "stream_count": 1000, - "pods": null, - "throughput": 0.0, - "p50": null, - "p99": null, - "pod_mem_mb": null, - "pod_mem_p50_mb": null, - "saturated": false, - "status": "error", - "reason": "creation_choke", - "walk": [ - [ - 2, - 19000.0 - ], - [ - 4, - 0.0 - ] - ] - } -] \ No newline at end of file diff --git a/results-2026-07-02/run-s2/report.md b/results-2026-07-02/run-s2/report.md deleted file mode 100644 index 0e10237..0000000 --- a/results-2026-07-02/run-s2/report.md +++ /dev/null @@ -1,30 +0,0 @@ -# run-s2 — write-throughput report - -## Throughput at saturation (ops/s) - -| streams | s2 | -|---|---| -| 100 | 2k | - -† = not saturated (ladder exhausted) — treat as a lower bound. - -## Pod memory at saturation — peak / p50 (MiB) - -| streams | s2 | -|---|---| -| 100 | 64 / 50 | - -_Pod working set = cgroup `memory.current − inactive_file` (anon + active page cache), sampled each second at the pinned rung. **peak** = high-water (catches bursts like an in-RAM Raft log filling); **p50** = median (what the server steadily holds resident). peak ≈ p50 ⇒ steadily resident; peak ≫ p50 ⇒ transient spikes._ - -## Saturation walks (pods → ops/s) - -- **s2 100**: 2:2k → 4:2k (pinned 2, plateau) -- **s2 1000**: 2:19k → 4:0k (pinned None, creation_choke) - -## Findings - -_TODO: written by hand on top of the generated data._ - -## Caveats - -_Single-node best-case; not 3-node Raft. Throughput is a saturation ceiling per the ladder._ diff --git a/results-2026-07-02/run-s2/s2/cells.json b/results-2026-07-02/run-s2/s2/cells.json deleted file mode 100644 index 7ba71be..0000000 --- a/results-2026-07-02/run-s2/s2/cells.json +++ /dev/null @@ -1,49 +0,0 @@ -{ - "cells": { - "100": { - "stream_count": 100, - "image_digest": "b6341d566e3d", - "walk": [ - [ - 2, - 1975.0 - ], - [ - 4, - 1965.625 - ] - ], - "pinned_pods": 2, - "throughput": 1975.0, - "p50": 51.135, - "p99": 52.351, - "pod_mem_mb": 64, - "saturated": true, - "status": "ok", - "reason": "plateau", - "pod_mem_p50_mb": 50 - }, - "1000": { - "stream_count": 1000, - "image_digest": "b6341d566e3d", - "walk": [ - [ - 2, - 19000.0 - ], - [ - 4, - 0.0 - ] - ], - "pinned_pods": null, - "throughput": 0.0, - "p50": null, - "p99": null, - "pod_mem_mb": null, - "saturated": false, - "status": "error", - "reason": "creation_choke" - } - } -} \ No newline at end of file diff --git a/results-2026-07-02/run-ursula/aggregate.csv b/results-2026-07-02/run-ursula/aggregate.csv deleted file mode 100644 index 67b2177..0000000 --- a/results-2026-07-02/run-ursula/aggregate.csv +++ /dev/null @@ -1,7 +0,0 @@ -mode,stream_count,pods,throughput,p50,p99,pod_mem_mb,pod_mem_p50_mb,saturated,status,reason -ursula-disk,100,4,2451.625,14.215,88.191,1098,1053,True,ok,plateau -ursula-memory,100,4,64287.125,0.534,35.839,2613,2076,True,ok,plateau -ursula-disk,1000,4,6974.5,111.999,387.583,1655,1515,True,ok,plateau -ursula-memory,1000,16,112295.625,,,2481,2054,False,ok,ladder_exhausted -ursula-disk,10000,16,11683.625,405.503,2598.911,2487,2259,True,ok,plateau -ursula-memory,10000,16,149503.5,64.223,298.751,3482,2899,True,ok,plateau diff --git a/results-2026-07-02/run-ursula/aggregate.json b/results-2026-07-02/run-ursula/aggregate.json deleted file mode 100644 index 2acc2f5..0000000 --- a/results-2026-07-02/run-ursula/aggregate.json +++ /dev/null @@ -1,152 +0,0 @@ -[ - { - "mode": "ursula-disk", - "stream_count": 100, - "pods": 4, - "throughput": 2451.625, - "p50": 14.215, - "p99": 88.191, - "pod_mem_mb": 1098, - "pod_mem_p50_mb": 1053, - "saturated": true, - "status": "ok", - "reason": "plateau", - "walk": [ - [ - 4, - 2451.625 - ], - [ - 8, - 1946.625 - ] - ] - }, - { - "mode": "ursula-memory", - "stream_count": 100, - "pods": 4, - "throughput": 64287.125, - "p50": 0.534, - "p99": 35.839, - "pod_mem_mb": 2613, - "pod_mem_p50_mb": 2076, - "saturated": true, - "status": "ok", - "reason": "plateau", - "walk": [ - [ - 4, - 64287.125 - ], - [ - 8, - 66007.5 - ] - ] - }, - { - "mode": "ursula-disk", - "stream_count": 1000, - "pods": 4, - "throughput": 6974.5, - "p50": 111.999, - "p99": 387.583, - "pod_mem_mb": 1655, - "pod_mem_p50_mb": 1515, - "saturated": true, - "status": "ok", - "reason": "plateau", - "walk": [ - [ - 4, - 6974.5 - ], - [ - 8, - 5255.625 - ] - ] - }, - { - "mode": "ursula-memory", - "stream_count": 1000, - "pods": 16, - "throughput": 112295.625, - "p50": null, - "p99": null, - "pod_mem_mb": 2481, - "pod_mem_p50_mb": 2054, - "saturated": false, - "status": "ok", - "reason": "ladder_exhausted", - "walk": [ - [ - 4, - 79092.625 - ], - [ - 8, - 103785.25 - ], - [ - 16, - 112295.625 - ] - ] - }, - { - "mode": "ursula-disk", - "stream_count": 10000, - "pods": 16, - "throughput": 11683.625, - "p50": 405.503, - "p99": 2598.911, - "pod_mem_mb": 2487, - "pod_mem_p50_mb": 2259, - "saturated": true, - "status": "ok", - "reason": "plateau", - "walk": [ - [ - 8, - 8921.625 - ], - [ - 16, - 11683.625 - ], - [ - 24, - 10545.75 - ] - ] - }, - { - "mode": "ursula-memory", - "stream_count": 10000, - "pods": 16, - "throughput": 149503.5, - "p50": 64.223, - "p99": 298.751, - "pod_mem_mb": 3482, - "pod_mem_p50_mb": 2899, - "saturated": true, - "status": "ok", - "reason": "plateau", - "walk": [ - [ - 8, - 119871.25 - ], - [ - 16, - 149503.5 - ], - [ - 24, - 130657.75 - ] - ] - } -] \ No newline at end of file diff --git a/results-2026-07-02/run-ursula/report.md b/results-2026-07-02/run-ursula/report.md deleted file mode 100644 index 1e3766a..0000000 --- a/results-2026-07-02/run-ursula/report.md +++ /dev/null @@ -1,38 +0,0 @@ -# run-ursula — write-throughput report - -## Throughput at saturation (ops/s) - -| streams | ursula-memory | ursula-disk | -|---|---|---| -| 100 | 64k | 2k | -| 1000 | 112k† | 7k | -| 10000 | 150k | 12k | - -† = not saturated (ladder exhausted) — treat as a lower bound. - -## Pod memory at saturation — peak / p50 (MiB) - -| streams | ursula-memory | ursula-disk | -|---|---|---| -| 100 | 2613 / 2076 | 1098 / 1053 | -| 1000 | 2481 / 2054 | 1655 / 1515 | -| 10000 | 3482 / 2899 | 2487 / 2259 | - -_Pod working set = cgroup `memory.current − inactive_file` (anon + active page cache), sampled each second at the pinned rung. **peak** = high-water (catches bursts like an in-RAM Raft log filling); **p50** = median (what the server steadily holds resident). peak ≈ p50 ⇒ steadily resident; peak ≫ p50 ⇒ transient spikes._ - -## Saturation walks (pods → ops/s) - -- **ursula-disk 100**: 4:2k → 8:2k (pinned 4, plateau) -- **ursula-memory 100**: 4:64k → 8:66k (pinned 4, plateau) -- **ursula-disk 1000**: 4:7k → 8:5k (pinned 4, plateau) -- **ursula-memory 1000**: 4:79k → 8:104k → 16:112k (pinned 16, ladder_exhausted) -- **ursula-disk 10000**: 8:9k → 16:12k → 24:11k (pinned 16, plateau) -- **ursula-memory 10000**: 8:120k → 16:150k → 24:131k (pinned 16, plateau) - -## Findings - -_TODO: written by hand on top of the generated data._ - -## Caveats - -_Single-node best-case; not 3-node Raft. Throughput is a saturation ceiling per the ladder._ diff --git a/results-2026-07-02/run-ursula/ursula-disk/cells.json b/results-2026-07-02/run-ursula/ursula-disk/cells.json deleted file mode 100644 index e98b13e..0000000 --- a/results-2026-07-02/run-ursula/ursula-disk/cells.json +++ /dev/null @@ -1,77 +0,0 @@ -{ - "cells": { - "100": { - "stream_count": 100, - "image_digest": "51db2ad52ee4", - "walk": [ - [ - 4, - 2451.625 - ], - [ - 8, - 1946.625 - ] - ], - "pinned_pods": 4, - "throughput": 2451.625, - "p50": 14.215, - "p99": 88.191, - "pod_mem_mb": 1098, - "saturated": true, - "status": "ok", - "reason": "plateau", - "pod_mem_p50_mb": 1053 - }, - "1000": { - "stream_count": 1000, - "image_digest": "51db2ad52ee4", - "walk": [ - [ - 4, - 6974.5 - ], - [ - 8, - 5255.625 - ] - ], - "pinned_pods": 4, - "throughput": 6974.5, - "p50": 111.999, - "p99": 387.583, - "pod_mem_mb": 1655, - "saturated": true, - "status": "ok", - "reason": "plateau", - "pod_mem_p50_mb": 1515 - }, - "10000": { - "stream_count": 10000, - "image_digest": "51db2ad52ee4", - "walk": [ - [ - 8, - 8921.625 - ], - [ - 16, - 11683.625 - ], - [ - 24, - 10545.75 - ] - ], - "pinned_pods": 16, - "throughput": 11683.625, - "p50": 405.503, - "p99": 2598.911, - "pod_mem_mb": 2487, - "saturated": true, - "status": "ok", - "reason": "plateau", - "pod_mem_p50_mb": 2259 - } - } -} \ No newline at end of file diff --git a/results-2026-07-02/run-ursula/ursula-memory/cells.json b/results-2026-07-02/run-ursula/ursula-memory/cells.json deleted file mode 100644 index 0da06fc..0000000 --- a/results-2026-07-02/run-ursula/ursula-memory/cells.json +++ /dev/null @@ -1,81 +0,0 @@ -{ - "cells": { - "100": { - "stream_count": 100, - "image_digest": "eeeb6d3de518", - "walk": [ - [ - 4, - 64287.125 - ], - [ - 8, - 66007.5 - ] - ], - "pinned_pods": 4, - "throughput": 64287.125, - "p50": 0.534, - "p99": 35.839, - "pod_mem_mb": 2613, - "saturated": true, - "status": "ok", - "reason": "plateau", - "pod_mem_p50_mb": 2076 - }, - "1000": { - "stream_count": 1000, - "image_digest": "eeeb6d3de518", - "walk": [ - [ - 4, - 79092.625 - ], - [ - 8, - 103785.25 - ], - [ - 16, - 112295.625 - ] - ], - "pinned_pods": 16, - "throughput": 112295.625, - "p50": null, - "p99": null, - "pod_mem_mb": 2481, - "saturated": false, - "status": "ok", - "reason": "ladder_exhausted", - "pod_mem_p50_mb": 2054 - }, - "10000": { - "stream_count": 10000, - "image_digest": "eeeb6d3de518", - "walk": [ - [ - 8, - 119871.25 - ], - [ - 16, - 149503.5 - ], - [ - 24, - 130657.75 - ] - ], - "pinned_pods": 16, - "throughput": 149503.5, - "p50": 64.223, - "p99": 298.751, - "pod_mem_mb": 3482, - "saturated": true, - "status": "ok", - "reason": "plateau", - "pod_mem_p50_mb": 2899 - } - } -} \ No newline at end of file diff --git a/results-2026-07-02/sse-comparison.csv b/results-2026-07-02/sse-comparison.csv deleted file mode 100644 index d353d3d..0000000 --- a/results-2026-07-02/sse-comparison.csv +++ /dev/null @@ -1,13 +0,0 @@ -config,subs,p50_ms,p90_ms,p99_ms,p999_ms,max_ms,pod_mem_mb,pod_mem_p50_mb,events_per_sec -ursula disk,1,1.307,1.41,1.528,2.315,2.315,15,14,85.7 -ursula disk,10,1.822,1.976,2.261,41.215,41.247,14,14,857.3 -ursula disk,100,2.135,2.391,2.761,3.373,3.605,14,13,8560.5 -ursula disk,1000,4.247,5.199,5.867,6.503,7.611,14,13,75080.2 -ursula in-memory,1,0.396,0.487,0.589,0.745,0.745,14,13,85.7 -ursula in-memory,10,0.504,0.621,0.748,42.623,42.719,14,13,857.3 -ursula in-memory,100,0.835,1.036,1.192,1.968,2.191,14,13,8564.2 -ursula in-memory,1000,2.979,3.899,4.511,4.831,5.427,14,13,75206.5 -wal (cache off),1,0.359,0.449,0.577,0.904,0.904,6,5,85.7 -wal (cache off),10,0.482,0.598,0.717,0.855,0.931,7,6,857.3 -wal (cache off),100,0.905,1.113,1.276,1.408,1.685,10,9,8560.9 -wal (cache off),1000,3.645,4.531,5.079,5.387,5.743,19,18,75008.5 diff --git a/results-2026-07-02/sse-comparison.md b/results-2026-07-02/sse-comparison.md deleted file mode 100644 index dd01ee9..0000000 --- a/results-2026-07-02/sse-comparison.md +++ /dev/null @@ -1,42 +0,0 @@ -# SSE Fan-out — delivery latency - -1 stream, 1 writer @ 50 ev/s, swept total subscribers; one well-provisioned -client pod (single wall clock). Writer-paced → metric is delivery latency. - -## Median (p50, ms) - -| config \ subscribers | 1 | 10 | 100 | 1000 | -|---|---|---|---|---| -| wal (cache off) | 0.359 | 0.482 | 0.905 | 3.645 | -| ursula in-memory | 0.396 | 0.504 | 0.835 | 2.979 | -| ursula disk | 1.307 | 1.822 | 2.135 | 4.247 | - -## Full spread (p50 / p99 / max, ms) - -| config | subs | p50 | p90 | p99 | p999 | max | -|---|---|---|---|---|---|---| -| wal (cache off) | 1 | 0.359 | 0.449 | 0.577 | 0.904 | 0.904 | -| wal (cache off) | 10 | 0.482 | 0.598 | 0.717 | 0.855 | 0.931 | -| wal (cache off) | 100 | 0.905 | 1.113 | 1.276 | 1.408 | 1.685 | -| wal (cache off) | 1000 | 3.645 | 4.531 | 5.079 | 5.387 | 5.743 | -| ursula in-memory | 1 | 0.396 | 0.487 | 0.589 | 0.745 | 0.745 | -| ursula in-memory | 10 | 0.504 | 0.621 | 0.748 | 42.623 | 42.719 | -| ursula in-memory | 100 | 0.835 | 1.036 | 1.192 | 1.968 | 2.191 | -| ursula in-memory | 1000 | 2.979 | 3.899 | 4.511 | 4.831 | 5.427 | -| ursula disk | 1 | 1.307 | 1.41 | 1.528 | 2.315 | 2.315 | -| ursula disk | 10 | 1.822 | 1.976 | 2.261 | 41.215 | 41.247 | -| ursula disk | 100 | 2.135 | 2.391 | 2.761 | 3.373 | 3.605 | -| ursula disk | 1000 | 4.247 | 5.199 | 5.867 | 6.503 | 7.611 | - -## Pod memory vs subscribers — peak / p50 (MiB) - -| config \ subscribers | 1 | 10 | 100 | 1000 | -|---|---|---|---|---| -| wal (cache off) | 6 / 5 | 7 / 6 | 10 / 9 | 19 / 18 | -| ursula in-memory | 14 / 13 | 14 / 13 | 14 / 13 | 14 / 13 | -| ursula disk | 15 / 14 | 14 / 14 | 14 / 13 | 14 / 13 | - -_Pod working set (cgroup `memory.current − inactive_file`) during each subscriber-count cell. **Flat across the row ⇒ a shared fan-out buffer** (one resident tail served to all subscribers); growth ⇒ per-subscriber buffering._ - - -_p50 = median; lower is better. — = not measured._ diff --git a/results-2026-07-02/sse-raw/sse-comparison.csv b/results-2026-07-02/sse-raw/sse-comparison.csv deleted file mode 100644 index d353d3d..0000000 --- a/results-2026-07-02/sse-raw/sse-comparison.csv +++ /dev/null @@ -1,13 +0,0 @@ -config,subs,p50_ms,p90_ms,p99_ms,p999_ms,max_ms,pod_mem_mb,pod_mem_p50_mb,events_per_sec -ursula disk,1,1.307,1.41,1.528,2.315,2.315,15,14,85.7 -ursula disk,10,1.822,1.976,2.261,41.215,41.247,14,14,857.3 -ursula disk,100,2.135,2.391,2.761,3.373,3.605,14,13,8560.5 -ursula disk,1000,4.247,5.199,5.867,6.503,7.611,14,13,75080.2 -ursula in-memory,1,0.396,0.487,0.589,0.745,0.745,14,13,85.7 -ursula in-memory,10,0.504,0.621,0.748,42.623,42.719,14,13,857.3 -ursula in-memory,100,0.835,1.036,1.192,1.968,2.191,14,13,8564.2 -ursula in-memory,1000,2.979,3.899,4.511,4.831,5.427,14,13,75206.5 -wal (cache off),1,0.359,0.449,0.577,0.904,0.904,6,5,85.7 -wal (cache off),10,0.482,0.598,0.717,0.855,0.931,7,6,857.3 -wal (cache off),100,0.905,1.113,1.276,1.408,1.685,10,9,8560.9 -wal (cache off),1000,3.645,4.531,5.079,5.387,5.743,19,18,75008.5 diff --git a/results-2026-07-02/sse-raw/sse-comparison.md b/results-2026-07-02/sse-raw/sse-comparison.md deleted file mode 100644 index dd01ee9..0000000 --- a/results-2026-07-02/sse-raw/sse-comparison.md +++ /dev/null @@ -1,42 +0,0 @@ -# SSE Fan-out — delivery latency - -1 stream, 1 writer @ 50 ev/s, swept total subscribers; one well-provisioned -client pod (single wall clock). Writer-paced → metric is delivery latency. - -## Median (p50, ms) - -| config \ subscribers | 1 | 10 | 100 | 1000 | -|---|---|---|---|---| -| wal (cache off) | 0.359 | 0.482 | 0.905 | 3.645 | -| ursula in-memory | 0.396 | 0.504 | 0.835 | 2.979 | -| ursula disk | 1.307 | 1.822 | 2.135 | 4.247 | - -## Full spread (p50 / p99 / max, ms) - -| config | subs | p50 | p90 | p99 | p999 | max | -|---|---|---|---|---|---|---| -| wal (cache off) | 1 | 0.359 | 0.449 | 0.577 | 0.904 | 0.904 | -| wal (cache off) | 10 | 0.482 | 0.598 | 0.717 | 0.855 | 0.931 | -| wal (cache off) | 100 | 0.905 | 1.113 | 1.276 | 1.408 | 1.685 | -| wal (cache off) | 1000 | 3.645 | 4.531 | 5.079 | 5.387 | 5.743 | -| ursula in-memory | 1 | 0.396 | 0.487 | 0.589 | 0.745 | 0.745 | -| ursula in-memory | 10 | 0.504 | 0.621 | 0.748 | 42.623 | 42.719 | -| ursula in-memory | 100 | 0.835 | 1.036 | 1.192 | 1.968 | 2.191 | -| ursula in-memory | 1000 | 2.979 | 3.899 | 4.511 | 4.831 | 5.427 | -| ursula disk | 1 | 1.307 | 1.41 | 1.528 | 2.315 | 2.315 | -| ursula disk | 10 | 1.822 | 1.976 | 2.261 | 41.215 | 41.247 | -| ursula disk | 100 | 2.135 | 2.391 | 2.761 | 3.373 | 3.605 | -| ursula disk | 1000 | 4.247 | 5.199 | 5.867 | 6.503 | 7.611 | - -## Pod memory vs subscribers — peak / p50 (MiB) - -| config \ subscribers | 1 | 10 | 100 | 1000 | -|---|---|---|---|---| -| wal (cache off) | 6 / 5 | 7 / 6 | 10 / 9 | 19 / 18 | -| ursula in-memory | 14 / 13 | 14 / 13 | 14 / 13 | 14 / 13 | -| ursula disk | 15 / 14 | 14 / 14 | 14 / 13 | 14 / 13 | - -_Pod working set (cgroup `memory.current − inactive_file`) during each subscriber-count cell. **Flat across the row ⇒ a shared fan-out buffer** (one resident tail served to all subscribers); growth ⇒ per-subscriber buffering._ - - -_p50 = median; lower is better. — = not measured._ diff --git a/results/mixed-cal-local/aggregate.csv b/results/mixed-cal-local/aggregate.csv deleted file mode 100644 index 5e2dbc5..0000000 --- a/results/mixed-cal-local/aggregate.csv +++ /dev/null @@ -1,3 +0,0 @@ -mode,stream_count,sweep,level,readers,subscribers,writer_rate,read_rate,write_ops_per_sec,write_p50,write_p99,read_ops_per_sec,read_mib_per_sec,read_p50,read_p99,events_per_sec,delivery_p50,delivery_p99,write_bp,write_err,read_bp,read_err,status,reason -memory,50,writer_rate,0,0,0,0,0,96344.2,0.45,1.633,0.0,0.0,,,0.0,,,0,0,0,0,ok,complete -wal,50,writer_rate,0,0,0,0,0,23286.0,2.013,6.471,0.0,0.0,,,0.0,,,0,0,0,0,ok,complete diff --git a/results/mixed-cal-local/aggregate.json b/results/mixed-cal-local/aggregate.json deleted file mode 100644 index 993aa34..0000000 --- a/results/mixed-cal-local/aggregate.json +++ /dev/null @@ -1,54 +0,0 @@ -[ - { - "mode": "memory", - "stream_count": 50, - "sweep": "writer_rate", - "level": 0, - "readers": 0, - "subscribers": 0, - "writer_rate": 0, - "read_rate": 0, - "write_ops_per_sec": 96344.2, - "write_p50": 0.45, - "write_p99": 1.633, - "read_ops_per_sec": 0.0, - "read_mib_per_sec": 0.0, - "read_p50": null, - "read_p99": null, - "events_per_sec": 0.0, - "delivery_p50": null, - "delivery_p99": null, - "write_bp": 0, - "write_err": 0, - "read_bp": 0, - "read_err": 0, - "status": "ok", - "reason": "complete" - }, - { - "mode": "wal", - "stream_count": 50, - "sweep": "writer_rate", - "level": 0, - "readers": 0, - "subscribers": 0, - "writer_rate": 0, - "read_rate": 0, - "write_ops_per_sec": 23286.0, - "write_p50": 2.013, - "write_p99": 6.471, - "read_ops_per_sec": 0.0, - "read_mib_per_sec": 0.0, - "read_p50": null, - "read_p99": null, - "events_per_sec": 0.0, - "delivery_p50": null, - "delivery_p99": null, - "write_bp": 0, - "write_err": 0, - "read_bp": 0, - "read_err": 0, - "status": "ok", - "reason": "complete" - } -] \ No newline at end of file diff --git a/results/mixed-cal-local/memory/cells.json b/results/mixed-cal-local/memory/cells.json deleted file mode 100644 index 3541030..0000000 --- a/results/mixed-cal-local/memory/cells.json +++ /dev/null @@ -1,40 +0,0 @@ -{ - "cells": { - "50": { - "stream_count": 50, - "image_digest": "f60d91fcfa07", - "complete": true, - "levels": { - "0": { - "write_ops_per_sec": 96344.2, - "write_p50": 0.45, - "write_p99": 1.633, - "read_ops_per_sec": 0.0, - "read_mib_per_sec": 0.0, - "read_p50": null, - "read_p99": null, - "events_per_sec": 0.0, - "delivery_p50": null, - "delivery_p99": null, - "write_ok": 1445163, - "write_bp": 0, - "write_err": 0, - "read_ok": 0, - "read_bp": 0, - "read_err": 0, - "events_received": 0, - "control_events_received": 0, - "elapsed_secs": 15.020538798, - "drive_secs": 15.0, - "level": 0, - "readers": 0, - "subscribers": 0, - "writer_rate": 0, - "read_rate": 0, - "status": "ok", - "reason": "complete" - } - } - } - } -} \ No newline at end of file diff --git a/results/mixed-cal-local/report.md b/results/mixed-cal-local/report.md deleted file mode 100644 index 0bd0121..0000000 --- a/results/mixed-cal-local/report.md +++ /dev/null @@ -1,19 +0,0 @@ -# mixed-cal-local — mixed read/write interference report - -Sweep axis: **writer_rate**. Latency cells are p50/p99 ms. ‡ = backpressure (503/429) observed in that class. - -## wal — 50 streams - -| rate/writer | write ops/s | write ms | read ops/s | read MiB/s | read ms | deliv rec/s | deliv ms | status | -|---|---|---|---|---|---|---|---|---| -| max | 23286 | 2.0/6.5 | 0 | 0.0 | — | 0 | — | ok | - -## memory — 50 streams - -| rate/writer | write ops/s | write ms | read ops/s | read MiB/s | read ms | deliv rec/s | deliv ms | status | -|---|---|---|---|---|---|---|---|---| -| max | 96344 | 0.5/1.6 | 0 | 0.0 | — | 0 | — | ok | - -## Findings - -_TODO: written by hand on top of the generated data._ diff --git a/results/mixed-cal-local/wal/cells.json b/results/mixed-cal-local/wal/cells.json deleted file mode 100644 index 37df5bb..0000000 --- a/results/mixed-cal-local/wal/cells.json +++ /dev/null @@ -1,40 +0,0 @@ -{ - "cells": { - "50": { - "stream_count": 50, - "image_digest": "c7df7a0880e0", - "complete": true, - "levels": { - "0": { - "write_ops_per_sec": 23286.0, - "write_p50": 2.013, - "write_p99": 6.471, - "read_ops_per_sec": 0.0, - "read_mib_per_sec": 0.0, - "read_p50": null, - "read_p99": null, - "events_per_sec": 0.0, - "delivery_p50": null, - "delivery_p99": null, - "write_ok": 349290, - "write_bp": 0, - "write_err": 0, - "read_ok": 0, - "read_bp": 0, - "read_err": 0, - "events_received": 0, - "control_events_received": 0, - "elapsed_secs": 15.01055959, - "drive_secs": 15.0, - "level": 0, - "readers": 0, - "subscribers": 0, - "writer_rate": 0, - "read_rate": 0, - "status": "ok", - "reason": "complete" - } - } - } - } -} \ No newline at end of file diff --git a/results/mixed-cal/aggregate.csv b/results/mixed-cal/aggregate.csv deleted file mode 100644 index bc53f67..0000000 --- a/results/mixed-cal/aggregate.csv +++ /dev/null @@ -1,2 +0,0 @@ -mode,stream_count,sweep,level,readers,subscribers,writer_rate,read_rate,write_ops_per_sec,write_p50,write_p99,read_ops_per_sec,read_mib_per_sec,read_p50,read_p99,events_per_sec,delivery_p50,delivery_p99,write_bp,write_err,read_bp,read_err,status,reason -wal,50,writer_rate,0,0,0,0,0,81435.95,0.494,1.131,0.0,0.0,,,0.0,,,0,0,0,0,ok,complete diff --git a/results/mixed-cal/aggregate.json b/results/mixed-cal/aggregate.json deleted file mode 100644 index 3a35ffa..0000000 --- a/results/mixed-cal/aggregate.json +++ /dev/null @@ -1,28 +0,0 @@ -[ - { - "mode": "wal", - "stream_count": 50, - "sweep": "writer_rate", - "level": 0, - "readers": 0, - "subscribers": 0, - "writer_rate": 0, - "read_rate": 0, - "write_ops_per_sec": 81435.95, - "write_p50": 0.494, - "write_p99": 1.131, - "read_ops_per_sec": 0.0, - "read_mib_per_sec": 0.0, - "read_p50": null, - "read_p99": null, - "events_per_sec": 0.0, - "delivery_p50": null, - "delivery_p99": null, - "write_bp": 0, - "write_err": 0, - "read_bp": 0, - "read_err": 0, - "status": "ok", - "reason": "complete" - } -] \ No newline at end of file diff --git a/results/mixed-cal/report.md b/results/mixed-cal/report.md deleted file mode 100644 index 4427b71..0000000 --- a/results/mixed-cal/report.md +++ /dev/null @@ -1,13 +0,0 @@ -# mixed-cal — mixed read/write interference report - -Sweep axis: **writer_rate**. Latency cells are p50/p99 ms. ‡ = backpressure (503/429) observed in that class. - -## wal — 50 streams - -| rate/writer | write ops/s | write ms | read ops/s | read MiB/s | read ms | deliv rec/s | deliv ms | status | -|---|---|---|---|---|---|---|---|---| -| max | 81436 | 0.5/1.1 | 0 | 0.0 | — | 0 | — | ok | - -## Findings - -_TODO: written by hand on top of the generated data._ diff --git a/results/mixed-cal/wal/cells.json b/results/mixed-cal/wal/cells.json deleted file mode 100644 index eb87aab..0000000 --- a/results/mixed-cal/wal/cells.json +++ /dev/null @@ -1,40 +0,0 @@ -{ - "cells": { - "50": { - "stream_count": 50, - "image_digest": "92f4a6387253", - "complete": true, - "levels": { - "0": { - "write_ops_per_sec": 81435.95, - "write_p50": 0.494, - "write_p99": 1.131, - "read_ops_per_sec": 0.0, - "read_mib_per_sec": 0.0, - "read_p50": null, - "read_p99": null, - "events_per_sec": 0.0, - "delivery_p50": null, - "delivery_p99": null, - "write_ok": 1628719, - "write_bp": 0, - "write_err": 0, - "read_ok": 0, - "read_bp": 0, - "read_err": 0, - "events_received": 0, - "control_events_received": 0, - "elapsed_secs": 20.042467107, - "drive_secs": 20.0, - "level": 0, - "readers": 0, - "subscribers": 0, - "writer_rate": 0, - "read_rate": 0, - "status": "ok", - "reason": "complete" - } - } - } - } -} \ No newline at end of file diff --git a/results/mixed-delivery-local/aggregate.csv b/results/mixed-delivery-local/aggregate.csv deleted file mode 100644 index 806e369..0000000 --- a/results/mixed-delivery-local/aggregate.csv +++ /dev/null @@ -1,11 +0,0 @@ -mode,stream_count,sweep,level,readers,subscribers,writer_rate,read_rate,write_ops_per_sec,write_p50,write_p99,read_ops_per_sec,read_mib_per_sec,read_p50,read_p99,events_per_sec,delivery_p50,delivery_p99,write_bp,write_err,read_bp,read_err,status,reason -memory,50,writer_rate,0,0,100,0,0,20964.4,1.594,30.175,0.0,0.0,,,32717.1,4.379,36.351,0,0,0,0,ok,complete -memory,50,writer_rate,30,0,100,30,0,1502.5,0.502,4.359,0.0,0.0,,,3005.0,0.928,6.919,0,0,0,0,ok,complete -memory,50,writer_rate,120,0,100,120,0,6001.35,0.476,4.487,0.0,0.0,,,11980.4,0.976,7.791,0,0,0,0,ok,complete -memory,50,writer_rate,300,0,100,300,0,15000.3,0.549,7.131,0.0,0.0,,,29118.45,1.14,13.215,0,0,0,0,ok,complete -memory,50,writer_rate,475,0,100,475,0,21425.6,1.569,30.319,0.0,0.0,,,33124.95,4.363,35.679,0,0,0,0,ok,complete -wal,50,writer_rate,0,0,100,0,0,12152.05,3.389,14.767,0.0,0.0,,,24290.9,4.155,17.055,0,0,0,0,ok,complete -wal,50,writer_rate,30,0,100,30,0,1502.5,0.893,5.955,0.0,0.0,,,3005.0,1.13,6.975,0,0,0,0,ok,complete -wal,50,writer_rate,120,0,100,120,0,6001.9,0.963,11.583,0.0,0.0,,,12000.8,1.158,15.231,0,0,0,0,ok,complete -wal,50,writer_rate,300,0,100,300,0,13923.55,3.027,16.031,0.0,0.0,,,27831.85,3.671,18.175,0,0,0,0,ok,complete -wal,50,writer_rate,475,0,100,475,0,13816.55,3.067,16.383,0.0,0.0,,,27608.75,3.741,19.039,0,0,0,0,ok,complete diff --git a/results/mixed-delivery-local/aggregate.json b/results/mixed-delivery-local/aggregate.json deleted file mode 100644 index bdd0f3b..0000000 --- a/results/mixed-delivery-local/aggregate.json +++ /dev/null @@ -1,262 +0,0 @@ -[ - { - "mode": "memory", - "stream_count": 50, - "sweep": "writer_rate", - "level": 0, - "readers": 0, - "subscribers": 100, - "writer_rate": 0, - "read_rate": 0, - "write_ops_per_sec": 20964.4, - "write_p50": 1.594, - "write_p99": 30.175, - "read_ops_per_sec": 0.0, - "read_mib_per_sec": 0.0, - "read_p50": null, - "read_p99": null, - "events_per_sec": 32717.1, - "delivery_p50": 4.379, - "delivery_p99": 36.351, - "write_bp": 0, - "write_err": 0, - "read_bp": 0, - "read_err": 0, - "status": "ok", - "reason": "complete" - }, - { - "mode": "memory", - "stream_count": 50, - "sweep": "writer_rate", - "level": 30, - "readers": 0, - "subscribers": 100, - "writer_rate": 30, - "read_rate": 0, - "write_ops_per_sec": 1502.5, - "write_p50": 0.502, - "write_p99": 4.359, - "read_ops_per_sec": 0.0, - "read_mib_per_sec": 0.0, - "read_p50": null, - "read_p99": null, - "events_per_sec": 3005.0, - "delivery_p50": 0.928, - "delivery_p99": 6.919, - "write_bp": 0, - "write_err": 0, - "read_bp": 0, - "read_err": 0, - "status": "ok", - "reason": "complete" - }, - { - "mode": "memory", - "stream_count": 50, - "sweep": "writer_rate", - "level": 120, - "readers": 0, - "subscribers": 100, - "writer_rate": 120, - "read_rate": 0, - "write_ops_per_sec": 6001.35, - "write_p50": 0.476, - "write_p99": 4.487, - "read_ops_per_sec": 0.0, - "read_mib_per_sec": 0.0, - "read_p50": null, - "read_p99": null, - "events_per_sec": 11980.4, - "delivery_p50": 0.976, - "delivery_p99": 7.791, - "write_bp": 0, - "write_err": 0, - "read_bp": 0, - "read_err": 0, - "status": "ok", - "reason": "complete" - }, - { - "mode": "memory", - "stream_count": 50, - "sweep": "writer_rate", - "level": 300, - "readers": 0, - "subscribers": 100, - "writer_rate": 300, - "read_rate": 0, - "write_ops_per_sec": 15000.3, - "write_p50": 0.549, - "write_p99": 7.131, - "read_ops_per_sec": 0.0, - "read_mib_per_sec": 0.0, - "read_p50": null, - "read_p99": null, - "events_per_sec": 29118.45, - "delivery_p50": 1.14, - "delivery_p99": 13.215, - "write_bp": 0, - "write_err": 0, - "read_bp": 0, - "read_err": 0, - "status": "ok", - "reason": "complete" - }, - { - "mode": "memory", - "stream_count": 50, - "sweep": "writer_rate", - "level": 475, - "readers": 0, - "subscribers": 100, - "writer_rate": 475, - "read_rate": 0, - "write_ops_per_sec": 21425.6, - "write_p50": 1.569, - "write_p99": 30.319, - "read_ops_per_sec": 0.0, - "read_mib_per_sec": 0.0, - "read_p50": null, - "read_p99": null, - "events_per_sec": 33124.95, - "delivery_p50": 4.363, - "delivery_p99": 35.679, - "write_bp": 0, - "write_err": 0, - "read_bp": 0, - "read_err": 0, - "status": "ok", - "reason": "complete" - }, - { - "mode": "wal", - "stream_count": 50, - "sweep": "writer_rate", - "level": 0, - "readers": 0, - "subscribers": 100, - "writer_rate": 0, - "read_rate": 0, - "write_ops_per_sec": 12152.05, - "write_p50": 3.389, - "write_p99": 14.767, - "read_ops_per_sec": 0.0, - "read_mib_per_sec": 0.0, - "read_p50": null, - "read_p99": null, - "events_per_sec": 24290.9, - "delivery_p50": 4.155, - "delivery_p99": 17.055, - "write_bp": 0, - "write_err": 0, - "read_bp": 0, - "read_err": 0, - "status": "ok", - "reason": "complete" - }, - { - "mode": "wal", - "stream_count": 50, - "sweep": "writer_rate", - "level": 30, - "readers": 0, - "subscribers": 100, - "writer_rate": 30, - "read_rate": 0, - "write_ops_per_sec": 1502.5, - "write_p50": 0.893, - "write_p99": 5.955, - "read_ops_per_sec": 0.0, - "read_mib_per_sec": 0.0, - "read_p50": null, - "read_p99": null, - "events_per_sec": 3005.0, - "delivery_p50": 1.13, - "delivery_p99": 6.975, - "write_bp": 0, - "write_err": 0, - "read_bp": 0, - "read_err": 0, - "status": "ok", - "reason": "complete" - }, - { - "mode": "wal", - "stream_count": 50, - "sweep": "writer_rate", - "level": 120, - "readers": 0, - "subscribers": 100, - "writer_rate": 120, - "read_rate": 0, - "write_ops_per_sec": 6001.9, - "write_p50": 0.963, - "write_p99": 11.583, - "read_ops_per_sec": 0.0, - "read_mib_per_sec": 0.0, - "read_p50": null, - "read_p99": null, - "events_per_sec": 12000.8, - "delivery_p50": 1.158, - "delivery_p99": 15.231, - "write_bp": 0, - "write_err": 0, - "read_bp": 0, - "read_err": 0, - "status": "ok", - "reason": "complete" - }, - { - "mode": "wal", - "stream_count": 50, - "sweep": "writer_rate", - "level": 300, - "readers": 0, - "subscribers": 100, - "writer_rate": 300, - "read_rate": 0, - "write_ops_per_sec": 13923.55, - "write_p50": 3.027, - "write_p99": 16.031, - "read_ops_per_sec": 0.0, - "read_mib_per_sec": 0.0, - "read_p50": null, - "read_p99": null, - "events_per_sec": 27831.85, - "delivery_p50": 3.671, - "delivery_p99": 18.175, - "write_bp": 0, - "write_err": 0, - "read_bp": 0, - "read_err": 0, - "status": "ok", - "reason": "complete" - }, - { - "mode": "wal", - "stream_count": 50, - "sweep": "writer_rate", - "level": 475, - "readers": 0, - "subscribers": 100, - "writer_rate": 475, - "read_rate": 0, - "write_ops_per_sec": 13816.55, - "write_p50": 3.067, - "write_p99": 16.383, - "read_ops_per_sec": 0.0, - "read_mib_per_sec": 0.0, - "read_p50": null, - "read_p99": null, - "events_per_sec": 27608.75, - "delivery_p50": 3.741, - "delivery_p99": 19.039, - "write_bp": 0, - "write_err": 0, - "read_bp": 0, - "read_err": 0, - "status": "ok", - "reason": "complete" - } -] \ No newline at end of file diff --git a/results/mixed-delivery-local/memory/cells.json b/results/mixed-delivery-local/memory/cells.json deleted file mode 100644 index 0c5495e..0000000 --- a/results/mixed-delivery-local/memory/cells.json +++ /dev/null @@ -1,156 +0,0 @@ -{ - "cells": { - "50": { - "stream_count": 50, - "image_digest": "f60d91fcfa07", - "complete": true, - "levels": { - "30": { - "write_ops_per_sec": 1502.5, - "write_p50": 0.502, - "write_p99": 4.359, - "read_ops_per_sec": 0.0, - "read_mib_per_sec": 0.0, - "read_p50": null, - "read_p99": null, - "events_per_sec": 3005.0, - "delivery_p50": 0.928, - "delivery_p99": 6.919, - "write_ok": 30050, - "write_bp": 0, - "write_err": 0, - "read_ok": 0, - "read_bp": 0, - "read_err": 0, - "events_received": 60100, - "control_events_received": 60200, - "elapsed_secs": 22.09005351, - "drive_secs": 20.0, - "level": 30, - "readers": 0, - "subscribers": 100, - "writer_rate": 30, - "read_rate": 0, - "status": "ok", - "reason": "complete" - }, - "120": { - "write_ops_per_sec": 6001.35, - "write_p50": 0.476, - "write_p99": 4.487, - "read_ops_per_sec": 0.0, - "read_mib_per_sec": 0.0, - "read_p50": null, - "read_p99": null, - "events_per_sec": 11980.4, - "delivery_p50": 0.976, - "delivery_p99": 7.791, - "write_ok": 120027, - "write_bp": 0, - "write_err": 0, - "read_ok": 0, - "read_bp": 0, - "read_err": 0, - "events_received": 239608, - "control_events_received": 239708, - "elapsed_secs": 22.065323886, - "drive_secs": 20.0, - "level": 120, - "readers": 0, - "subscribers": 100, - "writer_rate": 120, - "read_rate": 0, - "status": "ok", - "reason": "complete" - }, - "300": { - "write_ops_per_sec": 15000.3, - "write_p50": 0.549, - "write_p99": 7.131, - "read_ops_per_sec": 0.0, - "read_mib_per_sec": 0.0, - "read_p50": null, - "read_p99": null, - "events_per_sec": 29118.45, - "delivery_p50": 1.14, - "delivery_p99": 13.215, - "write_ok": 300006, - "write_bp": 0, - "write_err": 0, - "read_ok": 0, - "read_bp": 0, - "read_err": 0, - "events_received": 582369, - "control_events_received": 582469, - "elapsed_secs": 22.088860801, - "drive_secs": 20.0, - "level": 300, - "readers": 0, - "subscribers": 100, - "writer_rate": 300, - "read_rate": 0, - "status": "ok", - "reason": "complete" - }, - "475": { - "write_ops_per_sec": 21425.6, - "write_p50": 1.569, - "write_p99": 30.319, - "read_ops_per_sec": 0.0, - "read_mib_per_sec": 0.0, - "read_p50": null, - "read_p99": null, - "events_per_sec": 33124.95, - "delivery_p50": 4.363, - "delivery_p99": 35.679, - "write_ok": 428512, - "write_bp": 0, - "write_err": 0, - "read_ok": 0, - "read_bp": 0, - "read_err": 0, - "events_received": 662499, - "control_events_received": 662599, - "elapsed_secs": 22.101281801, - "drive_secs": 20.0, - "level": 475, - "readers": 0, - "subscribers": 100, - "writer_rate": 475, - "read_rate": 0, - "status": "ok", - "reason": "complete" - }, - "0": { - "write_ops_per_sec": 20964.4, - "write_p50": 1.594, - "write_p99": 30.175, - "read_ops_per_sec": 0.0, - "read_mib_per_sec": 0.0, - "read_p50": null, - "read_p99": null, - "events_per_sec": 32717.1, - "delivery_p50": 4.379, - "delivery_p99": 36.351, - "write_ok": 419288, - "write_bp": 0, - "write_err": 0, - "read_ok": 0, - "read_bp": 0, - "read_err": 0, - "events_received": 654342, - "control_events_received": 654442, - "elapsed_secs": 22.104042427, - "drive_secs": 20.0, - "level": 0, - "readers": 0, - "subscribers": 100, - "writer_rate": 0, - "read_rate": 0, - "status": "ok", - "reason": "complete" - } - } - } - } -} \ No newline at end of file diff --git a/results/mixed-delivery-local/report.md b/results/mixed-delivery-local/report.md deleted file mode 100644 index d824a59..0000000 --- a/results/mixed-delivery-local/report.md +++ /dev/null @@ -1,27 +0,0 @@ -# mixed-delivery-local — mixed read/write interference report - -Sweep axis: **writer_rate**. Latency cells are p50/p99 ms. ‡ = backpressure (503/429) observed in that class. - -## wal — 50 streams - -| rate/writer | write ops/s | write ms | read ops/s | read MiB/s | read ms | deliv rec/s | deliv ms | status | -|---|---|---|---|---|---|---|---|---| -| 30 | 1502 | 0.9/6.0 | 0 | 0.0 | — | 3005 | 1.1/7.0 | ok | -| 120 | 6002 | 1.0/11.6 | 0 | 0.0 | — | 12001 | 1.2/15.2 | ok | -| 300 | 13924 | 3.0/16.0 | 0 | 0.0 | — | 27832 | 3.7/18.2 | ok | -| 475 | 13817 | 3.1/16.4 | 0 | 0.0 | — | 27609 | 3.7/19.0 | ok | -| max | 12152 | 3.4/14.8 | 0 | 0.0 | — | 24291 | 4.2/17.1 | ok | - -## memory — 50 streams - -| rate/writer | write ops/s | write ms | read ops/s | read MiB/s | read ms | deliv rec/s | deliv ms | status | -|---|---|---|---|---|---|---|---|---| -| 30 | 1502 | 0.5/4.4 | 0 | 0.0 | — | 3005 | 0.9/6.9 | ok | -| 120 | 6001 | 0.5/4.5 | 0 | 0.0 | — | 11980 | 1.0/7.8 | ok | -| 300 | 15000 | 0.5/7.1 | 0 | 0.0 | — | 29118 | 1.1/13.2 | ok | -| 475 | 21426 | 1.6/30.3 | 0 | 0.0 | — | 33125 | 4.4/35.7 | ok | -| max | 20964 | 1.6/30.2 | 0 | 0.0 | — | 32717 | 4.4/36.4 | ok | - -## Findings - -_TODO: written by hand on top of the generated data._ diff --git a/results/mixed-delivery-local/wal/cells.json b/results/mixed-delivery-local/wal/cells.json deleted file mode 100644 index 9966918..0000000 --- a/results/mixed-delivery-local/wal/cells.json +++ /dev/null @@ -1,156 +0,0 @@ -{ - "cells": { - "50": { - "stream_count": 50, - "image_digest": "c7df7a0880e0", - "complete": true, - "levels": { - "30": { - "write_ops_per_sec": 1502.5, - "write_p50": 0.893, - "write_p99": 5.955, - "read_ops_per_sec": 0.0, - "read_mib_per_sec": 0.0, - "read_p50": null, - "read_p99": null, - "events_per_sec": 3005.0, - "delivery_p50": 1.13, - "delivery_p99": 6.975, - "write_ok": 30050, - "write_bp": 0, - "write_err": 0, - "read_ok": 0, - "read_bp": 0, - "read_err": 0, - "events_received": 60100, - "control_events_received": 60200, - "elapsed_secs": 22.08244351, - "drive_secs": 20.0, - "level": 30, - "readers": 0, - "subscribers": 100, - "writer_rate": 30, - "read_rate": 0, - "status": "ok", - "reason": "complete" - }, - "120": { - "write_ops_per_sec": 6001.9, - "write_p50": 0.963, - "write_p99": 11.583, - "read_ops_per_sec": 0.0, - "read_mib_per_sec": 0.0, - "read_p50": null, - "read_p99": null, - "events_per_sec": 12000.8, - "delivery_p50": 1.158, - "delivery_p99": 15.231, - "write_ok": 120038, - "write_bp": 0, - "write_err": 0, - "read_ok": 0, - "read_bp": 0, - "read_err": 0, - "events_received": 240016, - "control_events_received": 240116, - "elapsed_secs": 22.075566594, - "drive_secs": 20.0, - "level": 120, - "readers": 0, - "subscribers": 100, - "writer_rate": 120, - "read_rate": 0, - "status": "ok", - "reason": "complete" - }, - "300": { - "write_ops_per_sec": 13923.55, - "write_p50": 3.027, - "write_p99": 16.031, - "read_ops_per_sec": 0.0, - "read_mib_per_sec": 0.0, - "read_p50": null, - "read_p99": null, - "events_per_sec": 27831.85, - "delivery_p50": 3.671, - "delivery_p99": 18.175, - "write_ok": 278471, - "write_bp": 0, - "write_err": 0, - "read_ok": 0, - "read_bp": 0, - "read_err": 0, - "events_received": 556637, - "control_events_received": 556737, - "elapsed_secs": 22.089519135, - "drive_secs": 20.0, - "level": 300, - "readers": 0, - "subscribers": 100, - "writer_rate": 300, - "read_rate": 0, - "status": "ok", - "reason": "complete" - }, - "475": { - "write_ops_per_sec": 13816.55, - "write_p50": 3.067, - "write_p99": 16.383, - "read_ops_per_sec": 0.0, - "read_mib_per_sec": 0.0, - "read_p50": null, - "read_p99": null, - "events_per_sec": 27608.75, - "delivery_p50": 3.741, - "delivery_p99": 19.039, - "write_ok": 276331, - "write_bp": 0, - "write_err": 0, - "read_ok": 0, - "read_bp": 0, - "read_err": 0, - "events_received": 552175, - "control_events_received": 552275, - "elapsed_secs": 22.098459886, - "drive_secs": 20.0, - "level": 475, - "readers": 0, - "subscribers": 100, - "writer_rate": 475, - "read_rate": 0, - "status": "ok", - "reason": "complete" - }, - "0": { - "write_ops_per_sec": 12152.05, - "write_p50": 3.389, - "write_p99": 14.767, - "read_ops_per_sec": 0.0, - "read_mib_per_sec": 0.0, - "read_p50": null, - "read_p99": null, - "events_per_sec": 24290.9, - "delivery_p50": 4.155, - "delivery_p99": 17.055, - "write_ok": 243041, - "write_bp": 0, - "write_err": 0, - "read_ok": 0, - "read_bp": 0, - "read_err": 0, - "events_received": 485818, - "control_events_received": 485918, - "elapsed_secs": 22.074549803, - "drive_secs": 20.0, - "level": 0, - "readers": 0, - "subscribers": 100, - "writer_rate": 0, - "read_rate": 0, - "status": "ok", - "reason": "complete" - } - } - } - } -} \ No newline at end of file diff --git a/results/mixed-delivery/aggregate.csv b/results/mixed-delivery/aggregate.csv deleted file mode 100644 index cd6a694..0000000 --- a/results/mixed-delivery/aggregate.csv +++ /dev/null @@ -1,11 +0,0 @@ -mode,stream_count,sweep,level,readers,subscribers,writer_rate,read_rate,write_ops_per_sec,write_p50,write_p99,read_ops_per_sec,read_mib_per_sec,read_p50,read_p99,events_per_sec,delivery_p50,delivery_p99,write_bp,write_err,read_bp,read_err,status,reason -memory,2000,writer_rate,0,0,2000,0,0,61626.9,9.191,166.911,0.0,0.0,,,19046.066666666666,200.575,299.263,0,0,0,0,ok,complete -memory,2000,writer_rate,2,0,2000,2,0,4066.6666666666665,0.378,127.167,0.0,0.0,,,3866.6666666666665,0.406,132.607,0,0,0,0,ok,complete -memory,2000,writer_rate,8,0,2000,8,0,16050.933333333332,0.509,58.879,0.0,0.0,,,14676.966666666667,0.629,334.591,0,0,0,0,ok,complete -memory,2000,writer_rate,20,0,2000,20,0,39931.433333333334,11.591,82.431,0.0,0.0,,,15902.333333333334,226.687,362.239,0,0,0,0,ok,complete -memory,2000,writer_rate,33,0,2000,33,0,60841.96666666667,7.231,166.527,0.0,0.0,,,20597.4,206.591,299.263,0,0,0,0,ok,complete -wal,2000,writer_rate,0,0,2000,0,0,86268.53333333334,22.831,41.119,0.0,0.0,,,65537.13333333333,37.919,57.311,0,0,0,0,ok,complete -wal,2000,writer_rate,2,0,2000,2,0,4066.6666666666665,0.39,138.239,0.0,0.0,,,3339.6,0.422,162.559,0,0,0,0,ok,complete -wal,2000,writer_rate,8,0,2000,8,0,16050.666666666666,0.484,13.823,0.0,0.0,,,13323.9,0.525,15.447,0,0,0,0,ok,complete -wal,2000,writer_rate,20,0,2000,20,0,39981.433333333334,1.319,19.039,0.0,0.0,,,33210.566666666666,1.838,21.599,0,0,0,0,ok,complete -wal,2000,writer_rate,33,0,2000,33,0,65910.46666666666,9.239,29.183,0.0,0.0,,,65673.86666666667,10.167,44.735,0,0,0,0,ok,complete diff --git a/results/mixed-delivery/aggregate.json b/results/mixed-delivery/aggregate.json deleted file mode 100644 index 2ed2c16..0000000 --- a/results/mixed-delivery/aggregate.json +++ /dev/null @@ -1,262 +0,0 @@ -[ - { - "mode": "memory", - "stream_count": 2000, - "sweep": "writer_rate", - "level": 0, - "readers": 0, - "subscribers": 2000, - "writer_rate": 0, - "read_rate": 0, - "write_ops_per_sec": 61626.9, - "write_p50": 9.191, - "write_p99": 166.911, - "read_ops_per_sec": 0.0, - "read_mib_per_sec": 0.0, - "read_p50": null, - "read_p99": null, - "events_per_sec": 19046.066666666666, - "delivery_p50": 200.575, - "delivery_p99": 299.263, - "write_bp": 0, - "write_err": 0, - "read_bp": 0, - "read_err": 0, - "status": "ok", - "reason": "complete" - }, - { - "mode": "memory", - "stream_count": 2000, - "sweep": "writer_rate", - "level": 2, - "readers": 0, - "subscribers": 2000, - "writer_rate": 2, - "read_rate": 0, - "write_ops_per_sec": 4066.6666666666665, - "write_p50": 0.378, - "write_p99": 127.167, - "read_ops_per_sec": 0.0, - "read_mib_per_sec": 0.0, - "read_p50": null, - "read_p99": null, - "events_per_sec": 3866.6666666666665, - "delivery_p50": 0.406, - "delivery_p99": 132.607, - "write_bp": 0, - "write_err": 0, - "read_bp": 0, - "read_err": 0, - "status": "ok", - "reason": "complete" - }, - { - "mode": "memory", - "stream_count": 2000, - "sweep": "writer_rate", - "level": 8, - "readers": 0, - "subscribers": 2000, - "writer_rate": 8, - "read_rate": 0, - "write_ops_per_sec": 16050.933333333332, - "write_p50": 0.509, - "write_p99": 58.879, - "read_ops_per_sec": 0.0, - "read_mib_per_sec": 0.0, - "read_p50": null, - "read_p99": null, - "events_per_sec": 14676.966666666667, - "delivery_p50": 0.629, - "delivery_p99": 334.591, - "write_bp": 0, - "write_err": 0, - "read_bp": 0, - "read_err": 0, - "status": "ok", - "reason": "complete" - }, - { - "mode": "memory", - "stream_count": 2000, - "sweep": "writer_rate", - "level": 20, - "readers": 0, - "subscribers": 2000, - "writer_rate": 20, - "read_rate": 0, - "write_ops_per_sec": 39931.433333333334, - "write_p50": 11.591, - "write_p99": 82.431, - "read_ops_per_sec": 0.0, - "read_mib_per_sec": 0.0, - "read_p50": null, - "read_p99": null, - "events_per_sec": 15902.333333333334, - "delivery_p50": 226.687, - "delivery_p99": 362.239, - "write_bp": 0, - "write_err": 0, - "read_bp": 0, - "read_err": 0, - "status": "ok", - "reason": "complete" - }, - { - "mode": "memory", - "stream_count": 2000, - "sweep": "writer_rate", - "level": 33, - "readers": 0, - "subscribers": 2000, - "writer_rate": 33, - "read_rate": 0, - "write_ops_per_sec": 60841.96666666667, - "write_p50": 7.231, - "write_p99": 166.527, - "read_ops_per_sec": 0.0, - "read_mib_per_sec": 0.0, - "read_p50": null, - "read_p99": null, - "events_per_sec": 20597.4, - "delivery_p50": 206.591, - "delivery_p99": 299.263, - "write_bp": 0, - "write_err": 0, - "read_bp": 0, - "read_err": 0, - "status": "ok", - "reason": "complete" - }, - { - "mode": "wal", - "stream_count": 2000, - "sweep": "writer_rate", - "level": 0, - "readers": 0, - "subscribers": 2000, - "writer_rate": 0, - "read_rate": 0, - "write_ops_per_sec": 86268.53333333334, - "write_p50": 22.831, - "write_p99": 41.119, - "read_ops_per_sec": 0.0, - "read_mib_per_sec": 0.0, - "read_p50": null, - "read_p99": null, - "events_per_sec": 65537.13333333333, - "delivery_p50": 37.919, - "delivery_p99": 57.311, - "write_bp": 0, - "write_err": 0, - "read_bp": 0, - "read_err": 0, - "status": "ok", - "reason": "complete" - }, - { - "mode": "wal", - "stream_count": 2000, - "sweep": "writer_rate", - "level": 2, - "readers": 0, - "subscribers": 2000, - "writer_rate": 2, - "read_rate": 0, - "write_ops_per_sec": 4066.6666666666665, - "write_p50": 0.39, - "write_p99": 138.239, - "read_ops_per_sec": 0.0, - "read_mib_per_sec": 0.0, - "read_p50": null, - "read_p99": null, - "events_per_sec": 3339.6, - "delivery_p50": 0.422, - "delivery_p99": 162.559, - "write_bp": 0, - "write_err": 0, - "read_bp": 0, - "read_err": 0, - "status": "ok", - "reason": "complete" - }, - { - "mode": "wal", - "stream_count": 2000, - "sweep": "writer_rate", - "level": 8, - "readers": 0, - "subscribers": 2000, - "writer_rate": 8, - "read_rate": 0, - "write_ops_per_sec": 16050.666666666666, - "write_p50": 0.484, - "write_p99": 13.823, - "read_ops_per_sec": 0.0, - "read_mib_per_sec": 0.0, - "read_p50": null, - "read_p99": null, - "events_per_sec": 13323.9, - "delivery_p50": 0.525, - "delivery_p99": 15.447, - "write_bp": 0, - "write_err": 0, - "read_bp": 0, - "read_err": 0, - "status": "ok", - "reason": "complete" - }, - { - "mode": "wal", - "stream_count": 2000, - "sweep": "writer_rate", - "level": 20, - "readers": 0, - "subscribers": 2000, - "writer_rate": 20, - "read_rate": 0, - "write_ops_per_sec": 39981.433333333334, - "write_p50": 1.319, - "write_p99": 19.039, - "read_ops_per_sec": 0.0, - "read_mib_per_sec": 0.0, - "read_p50": null, - "read_p99": null, - "events_per_sec": 33210.566666666666, - "delivery_p50": 1.838, - "delivery_p99": 21.599, - "write_bp": 0, - "write_err": 0, - "read_bp": 0, - "read_err": 0, - "status": "ok", - "reason": "complete" - }, - { - "mode": "wal", - "stream_count": 2000, - "sweep": "writer_rate", - "level": 33, - "readers": 0, - "subscribers": 2000, - "writer_rate": 33, - "read_rate": 0, - "write_ops_per_sec": 65910.46666666666, - "write_p50": 9.239, - "write_p99": 29.183, - "read_ops_per_sec": 0.0, - "read_mib_per_sec": 0.0, - "read_p50": null, - "read_p99": null, - "events_per_sec": 65673.86666666667, - "delivery_p50": 10.167, - "delivery_p99": 44.735, - "write_bp": 0, - "write_err": 0, - "read_bp": 0, - "read_err": 0, - "status": "ok", - "reason": "complete" - } -] \ No newline at end of file diff --git a/results/mixed-delivery/memory/cells.json b/results/mixed-delivery/memory/cells.json deleted file mode 100644 index 7999e86..0000000 --- a/results/mixed-delivery/memory/cells.json +++ /dev/null @@ -1,156 +0,0 @@ -{ - "cells": { - "2000": { - "stream_count": 2000, - "image_digest": "b7d78084e886", - "complete": true, - "levels": { - "2": { - "write_ops_per_sec": 4066.6666666666665, - "write_p50": 0.378, - "write_p99": 127.167, - "read_ops_per_sec": 0.0, - "read_mib_per_sec": 0.0, - "read_p50": null, - "read_p99": null, - "events_per_sec": 3866.6666666666665, - "delivery_p50": 0.406, - "delivery_p99": 132.607, - "write_ok": 122000, - "write_bp": 0, - "write_err": 0, - "read_ok": 0, - "read_bp": 0, - "read_err": 0, - "events_received": 116000, - "control_events_received": 118000, - "elapsed_secs": 31.400304374, - "drive_secs": 30.0, - "level": 2, - "readers": 0, - "subscribers": 2000, - "writer_rate": 2, - "read_rate": 0, - "status": "ok", - "reason": "complete" - }, - "8": { - "write_ops_per_sec": 16050.933333333332, - "write_p50": 0.509, - "write_p99": 58.879, - "read_ops_per_sec": 0.0, - "read_mib_per_sec": 0.0, - "read_p50": null, - "read_p99": null, - "events_per_sec": 14676.966666666667, - "delivery_p50": 0.629, - "delivery_p99": 334.591, - "write_ok": 481528, - "write_bp": 0, - "write_err": 0, - "read_ok": 0, - "read_bp": 0, - "read_err": 0, - "events_received": 440309, - "control_events_received": 442309, - "elapsed_secs": 32.226361259, - "drive_secs": 30.0, - "level": 8, - "readers": 0, - "subscribers": 2000, - "writer_rate": 8, - "read_rate": 0, - "status": "ok", - "reason": "complete" - }, - "20": { - "write_ops_per_sec": 39931.433333333334, - "write_p50": 11.591, - "write_p99": 82.431, - "read_ops_per_sec": 0.0, - "read_mib_per_sec": 0.0, - "read_p50": null, - "read_p99": null, - "events_per_sec": 15902.333333333334, - "delivery_p50": 226.687, - "delivery_p99": 362.239, - "write_ok": 1197943, - "write_bp": 0, - "write_err": 0, - "read_ok": 0, - "read_bp": 0, - "read_err": 0, - "events_received": 477070, - "control_events_received": 479070, - "elapsed_secs": 32.293532483999996, - "drive_secs": 30.0, - "level": 20, - "readers": 0, - "subscribers": 2000, - "writer_rate": 20, - "read_rate": 0, - "status": "ok", - "reason": "complete" - }, - "33": { - "write_ops_per_sec": 60841.96666666667, - "write_p50": 7.231, - "write_p99": 166.527, - "read_ops_per_sec": 0.0, - "read_mib_per_sec": 0.0, - "read_p50": null, - "read_p99": null, - "events_per_sec": 20597.4, - "delivery_p50": 206.591, - "delivery_p99": 299.263, - "write_ok": 1825259, - "write_bp": 0, - "write_err": 0, - "read_ok": 0, - "read_bp": 0, - "read_err": 0, - "events_received": 617922, - "control_events_received": 619922, - "elapsed_secs": 32.215251699, - "drive_secs": 30.0, - "level": 33, - "readers": 0, - "subscribers": 2000, - "writer_rate": 33, - "read_rate": 0, - "status": "ok", - "reason": "complete" - }, - "0": { - "write_ops_per_sec": 61626.9, - "write_p50": 9.191, - "write_p99": 166.911, - "read_ops_per_sec": 0.0, - "read_mib_per_sec": 0.0, - "read_p50": null, - "read_p99": null, - "events_per_sec": 19046.066666666666, - "delivery_p50": 200.575, - "delivery_p99": 299.263, - "write_ok": 1848807, - "write_bp": 0, - "write_err": 0, - "read_ok": 0, - "read_bp": 0, - "read_err": 0, - "events_received": 571382, - "control_events_received": 573382, - "elapsed_secs": 35.225584119, - "drive_secs": 30.0, - "level": 0, - "readers": 0, - "subscribers": 2000, - "writer_rate": 0, - "read_rate": 0, - "status": "ok", - "reason": "complete" - } - } - } - } -} \ No newline at end of file diff --git a/results/mixed-delivery/report.md b/results/mixed-delivery/report.md deleted file mode 100644 index 8a4814d..0000000 --- a/results/mixed-delivery/report.md +++ /dev/null @@ -1,27 +0,0 @@ -# mixed-delivery — mixed read/write interference report - -Sweep axis: **writer_rate**. Latency cells are p50/p99 ms. ‡ = backpressure (503/429) observed in that class. - -## wal — 2000 streams - -| rate/writer | write ops/s | write ms | read ops/s | read MiB/s | read ms | deliv rec/s | deliv ms | status | -|---|---|---|---|---|---|---|---|---| -| 2 | 4067 | 0.4/138.2 | 0 | 0.0 | — | 3340 | 0.4/162.6 | ok | -| 8 | 16051 | 0.5/13.8 | 0 | 0.0 | — | 13324 | 0.5/15.4 | ok | -| 20 | 39981 | 1.3/19.0 | 0 | 0.0 | — | 33211 | 1.8/21.6 | ok | -| 33 | 65910 | 9.2/29.2 | 0 | 0.0 | — | 65674 | 10.2/44.7 | ok | -| max | 86269 | 22.8/41.1 | 0 | 0.0 | — | 65537 | 37.9/57.3 | ok | - -## memory — 2000 streams - -| rate/writer | write ops/s | write ms | read ops/s | read MiB/s | read ms | deliv rec/s | deliv ms | status | -|---|---|---|---|---|---|---|---|---| -| 2 | 4067 | 0.4/127.2 | 0 | 0.0 | — | 3867 | 0.4/132.6 | ok | -| 8 | 16051 | 0.5/58.9 | 0 | 0.0 | — | 14677 | 0.6/334.6 | ok | -| 20 | 39931 | 11.6/82.4 | 0 | 0.0 | — | 15902 | 226.7/362.2 | ok | -| 33 | 60842 | 7.2/166.5 | 0 | 0.0 | — | 20597 | 206.6/299.3 | ok | -| max | 61627 | 9.2/166.9 | 0 | 0.0 | — | 19046 | 200.6/299.3 | ok | - -## Findings - -_TODO: written by hand on top of the generated data._ diff --git a/results/mixed-delivery/wal/cells.json b/results/mixed-delivery/wal/cells.json deleted file mode 100644 index 11ee9e9..0000000 --- a/results/mixed-delivery/wal/cells.json +++ /dev/null @@ -1,156 +0,0 @@ -{ - "cells": { - "2000": { - "stream_count": 2000, - "image_digest": "92f4a6387253", - "complete": true, - "levels": { - "2": { - "write_ops_per_sec": 4066.6666666666665, - "write_p50": 0.39, - "write_p99": 138.239, - "read_ops_per_sec": 0.0, - "read_mib_per_sec": 0.0, - "read_p50": null, - "read_p99": null, - "events_per_sec": 3339.6, - "delivery_p50": 0.422, - "delivery_p99": 162.559, - "write_ok": 122000, - "write_bp": 0, - "write_err": 0, - "read_ok": 0, - "read_bp": 0, - "read_err": 0, - "events_received": 100188, - "control_events_received": 102188, - "elapsed_secs": 35.300253942, - "drive_secs": 30.0, - "level": 2, - "readers": 0, - "subscribers": 2000, - "writer_rate": 2, - "read_rate": 0, - "status": "ok", - "reason": "complete" - }, - "8": { - "write_ops_per_sec": 16050.666666666666, - "write_p50": 0.484, - "write_p99": 13.823, - "read_ops_per_sec": 0.0, - "read_mib_per_sec": 0.0, - "read_p50": null, - "read_p99": null, - "events_per_sec": 13323.9, - "delivery_p50": 0.525, - "delivery_p99": 15.447, - "write_ok": 481520, - "write_bp": 0, - "write_err": 0, - "read_ok": 0, - "read_bp": 0, - "read_err": 0, - "events_received": 399717, - "control_events_received": 401717, - "elapsed_secs": 35.164210628, - "drive_secs": 30.0, - "level": 8, - "readers": 0, - "subscribers": 2000, - "writer_rate": 8, - "read_rate": 0, - "status": "ok", - "reason": "complete" - }, - "20": { - "write_ops_per_sec": 39981.433333333334, - "write_p50": 1.319, - "write_p99": 19.039, - "read_ops_per_sec": 0.0, - "read_mib_per_sec": 0.0, - "read_p50": null, - "read_p99": null, - "events_per_sec": 33210.566666666666, - "delivery_p50": 1.838, - "delivery_p99": 21.599, - "write_ok": 1199443, - "write_bp": 0, - "write_err": 0, - "read_ok": 0, - "read_bp": 0, - "read_err": 0, - "events_received": 996317, - "control_events_received": 998317, - "elapsed_secs": 35.172896945, - "drive_secs": 30.0, - "level": 20, - "readers": 0, - "subscribers": 2000, - "writer_rate": 20, - "read_rate": 0, - "status": "ok", - "reason": "complete" - }, - "33": { - "write_ops_per_sec": 65910.46666666666, - "write_p50": 9.239, - "write_p99": 29.183, - "read_ops_per_sec": 0.0, - "read_mib_per_sec": 0.0, - "read_p50": null, - "read_p99": null, - "events_per_sec": 65673.86666666667, - "delivery_p50": 10.167, - "delivery_p99": 44.735, - "write_ok": 1977314, - "write_bp": 0, - "write_err": 0, - "read_ok": 0, - "read_bp": 0, - "read_err": 0, - "events_received": 1970216, - "control_events_received": 1972216, - "elapsed_secs": 30.238965801, - "drive_secs": 30.0, - "level": 33, - "readers": 0, - "subscribers": 2000, - "writer_rate": 33, - "read_rate": 0, - "status": "ok", - "reason": "complete" - }, - "0": { - "write_ops_per_sec": 86268.53333333334, - "write_p50": 22.831, - "write_p99": 41.119, - "read_ops_per_sec": 0.0, - "read_mib_per_sec": 0.0, - "read_p50": null, - "read_p99": null, - "events_per_sec": 65537.13333333333, - "delivery_p50": 37.919, - "delivery_p99": 57.311, - "write_ok": 2588056, - "write_bp": 0, - "write_err": 0, - "read_ok": 0, - "read_bp": 0, - "read_err": 0, - "events_received": 1966114, - "control_events_received": 1968114, - "elapsed_secs": 35.134834253, - "drive_secs": 30.0, - "level": 0, - "readers": 0, - "subscribers": 2000, - "writer_rate": 0, - "read_rate": 0, - "status": "ok", - "reason": "complete" - } - } - } - } -} \ No newline at end of file diff --git a/results/mixed-writes-hot/aggregate.csv b/results/mixed-writes-hot/aggregate.csv deleted file mode 100644 index 919d8f2..0000000 --- a/results/mixed-writes-hot/aggregate.csv +++ /dev/null @@ -1,4 +0,0 @@ -mode,stream_count,sweep,level,readers,subscribers,writer_rate,read_rate,write_ops_per_sec,write_p50,write_p99,read_ops_per_sec,read_mib_per_sec,read_p50,read_p99,events_per_sec,delivery_p50,delivery_p99,write_bp,write_err,read_bp,read_err,status,reason -wal,50,readers,16,16,0,977,0,48874.45,0.413,1.653,2138.8,2342.6493408203123,3.577,38.559,0.0,,,0,0,0,0,ok,complete -wal,50,readers,64,64,0,977,0,47714.75,0.546,9.607,2251.75,2321.505236816406,11.055,223.743,0.0,,,0,0,0,0,ok,complete -wal,50,readers,256,256,0,977,0,7654.65,3.599,35.359,6133.05,2282.3429321289063,30.655,183.039,0.0,,,0,0,0,0,ok,complete diff --git a/results/mixed-writes-hot/aggregate.json b/results/mixed-writes-hot/aggregate.json deleted file mode 100644 index b4152f7..0000000 --- a/results/mixed-writes-hot/aggregate.json +++ /dev/null @@ -1,80 +0,0 @@ -[ - { - "mode": "wal", - "stream_count": 50, - "sweep": "readers", - "level": 16, - "readers": 16, - "subscribers": 0, - "writer_rate": 977, - "read_rate": 0, - "write_ops_per_sec": 48874.45, - "write_p50": 0.413, - "write_p99": 1.653, - "read_ops_per_sec": 2138.8, - "read_mib_per_sec": 2342.6493408203123, - "read_p50": 3.577, - "read_p99": 38.559, - "events_per_sec": 0.0, - "delivery_p50": null, - "delivery_p99": null, - "write_bp": 0, - "write_err": 0, - "read_bp": 0, - "read_err": 0, - "status": "ok", - "reason": "complete" - }, - { - "mode": "wal", - "stream_count": 50, - "sweep": "readers", - "level": 64, - "readers": 64, - "subscribers": 0, - "writer_rate": 977, - "read_rate": 0, - "write_ops_per_sec": 47714.75, - "write_p50": 0.546, - "write_p99": 9.607, - "read_ops_per_sec": 2251.75, - "read_mib_per_sec": 2321.505236816406, - "read_p50": 11.055, - "read_p99": 223.743, - "events_per_sec": 0.0, - "delivery_p50": null, - "delivery_p99": null, - "write_bp": 0, - "write_err": 0, - "read_bp": 0, - "read_err": 0, - "status": "ok", - "reason": "complete" - }, - { - "mode": "wal", - "stream_count": 50, - "sweep": "readers", - "level": 256, - "readers": 256, - "subscribers": 0, - "writer_rate": 977, - "read_rate": 0, - "write_ops_per_sec": 7654.65, - "write_p50": 3.599, - "write_p99": 35.359, - "read_ops_per_sec": 6133.05, - "read_mib_per_sec": 2282.3429321289063, - "read_p50": 30.655, - "read_p99": 183.039, - "events_per_sec": 0.0, - "delivery_p50": null, - "delivery_p99": null, - "write_bp": 0, - "write_err": 0, - "read_bp": 0, - "read_err": 0, - "status": "ok", - "reason": "complete" - } -] \ No newline at end of file diff --git a/results/mixed-writes-hot/report.md b/results/mixed-writes-hot/report.md deleted file mode 100644 index 4fd6545..0000000 --- a/results/mixed-writes-hot/report.md +++ /dev/null @@ -1,15 +0,0 @@ -# mixed-writes-hot — mixed read/write interference report - -Sweep axis: **readers**. Latency cells are p50/p99 ms. ‡ = backpressure (503/429) observed in that class. - -## wal — 50 streams - -| readers | write ops/s | write ms | read ops/s | read MiB/s | read ms | deliv rec/s | deliv ms | status | -|---|---|---|---|---|---|---|---|---| -| 16 | 48874 | 0.4/1.7 | 2139 | 2342.6 | 3.6/38.6 | 0 | — | ok | -| 64 | 47715 | 0.5/9.6 | 2252 | 2321.5 | 11.1/223.7 | 0 | — | ok | -| 256 | 7655 | 3.6/35.4 | 6133 | 2282.3 | 30.7/183.0 | 0 | — | ok | - -## Findings - -_TODO: written by hand on top of the generated data._ diff --git a/results/mixed-writes-hot/wal/cells.json b/results/mixed-writes-hot/wal/cells.json deleted file mode 100644 index eb05124..0000000 --- a/results/mixed-writes-hot/wal/cells.json +++ /dev/null @@ -1,98 +0,0 @@ -{ - "cells": { - "50": { - "stream_count": 50, - "image_digest": "92f4a6387253", - "complete": true, - "levels": { - "16": { - "write_ops_per_sec": 48874.45, - "write_p50": 0.413, - "write_p99": 1.653, - "read_ops_per_sec": 2138.8, - "read_mib_per_sec": 2342.6493408203123, - "read_p50": 3.577, - "read_p99": 38.559, - "events_per_sec": 0.0, - "delivery_p50": null, - "delivery_p99": null, - "write_ok": 977489, - "write_bp": 0, - "write_err": 0, - "read_ok": 42776, - "read_bp": 0, - "read_err": 0, - "events_received": 0, - "control_events_received": 0, - "elapsed_secs": 20.021721405, - "drive_secs": 20.0, - "level": 16, - "readers": 16, - "subscribers": 0, - "writer_rate": 977, - "read_rate": 0, - "status": "ok", - "reason": "complete" - }, - "64": { - "write_ops_per_sec": 47714.75, - "write_p50": 0.546, - "write_p99": 9.607, - "read_ops_per_sec": 2251.75, - "read_mib_per_sec": 2321.505236816406, - "read_p50": 11.055, - "read_p99": 223.743, - "events_per_sec": 0.0, - "delivery_p50": null, - "delivery_p99": null, - "write_ok": 954295, - "write_bp": 0, - "write_err": 0, - "read_ok": 45035, - "read_bp": 0, - "read_err": 0, - "events_received": 0, - "control_events_received": 0, - "elapsed_secs": 20.078124411, - "drive_secs": 20.0, - "level": 64, - "readers": 64, - "subscribers": 0, - "writer_rate": 977, - "read_rate": 0, - "status": "ok", - "reason": "complete" - }, - "256": { - "write_ops_per_sec": 7654.65, - "write_p50": 3.599, - "write_p99": 35.359, - "read_ops_per_sec": 6133.05, - "read_mib_per_sec": 2282.3429321289063, - "read_p50": 30.655, - "read_p99": 183.039, - "events_per_sec": 0.0, - "delivery_p50": null, - "delivery_p99": null, - "write_ok": 153093, - "write_bp": 0, - "write_err": 0, - "read_ok": 122661, - "read_bp": 0, - "read_err": 0, - "events_received": 0, - "control_events_received": 0, - "elapsed_secs": 20.052673988, - "drive_secs": 20.0, - "level": 256, - "readers": 256, - "subscribers": 0, - "writer_rate": 977, - "read_rate": 0, - "status": "ok", - "reason": "complete" - } - } - } - } -} \ No newline at end of file diff --git a/results/mixed-writes-local/FINDINGS.md b/results/mixed-writes-local/FINDINGS.md deleted file mode 100644 index 682577e..0000000 --- a/results/mixed-writes-local/FINDINGS.md +++ /dev/null @@ -1,60 +0,0 @@ -# mixed-writes-local — do catch-up readers hurt a pinned write load? (local kind validation) - -**Setup (v2, 2026-07-02).** durable `wal` built from `electric-ds-rust` -`bench/mixed-interference-validation` (off `perf/combined-t1a-t1c-t2a`), local kind -(server 2 CPU / 2 Gi), 50 shared streams, 1 writer/stream pinned at 355 appends/s each -= **17.75k ops/s offered ≈ 60% of the measured local ceiling** (29.7k ops/s from -`mixed-cal-local`, p50 1.5 ms / p99 6.4 ms). Readers are **paced at 1 replay/s each** -(`read_rate: 1`), so the sweep axis is a bounded offered read load; all three classes -now share one barrier→deadline window and rates divide by it. 20 s windows, 256 B -payloads, fresh server per level. - -| readers (≈replays/s offered) | write ops/s | write p50/p99 ms | read replays/s | read MiB/s | read p50/p99 ms | -|---|---|---|---|---|---| -| 0 | 17303 | 1.2 / 9.8 | — | — | — | -| 4 | 17755 | 1.0 / 5.7 | 4 | 3.8 | 1.7 / 10.5 | -| 16 | 17756 | 0.9 / 5.3 | 17 | 15.4 | 3.9 / 17.1 | -| 64 | 17756 | 1.0 / 6.5 | 67 | 61.5 | 5.5 / 65.5 | -| 128 | 17750 | 1.1 / 52.9 | 134 | 123.0 | 4.9 / 50.9 | - -## Conclusions vs the premise (v2 — bounded read load) - -1. **Write throughput is unharmed by a bounded catch-up read load.** The pinned 60%-of- - ceiling write rate is delivered to within noise at every level, up to 128 readers - pulling **123 MiB/s** of replay bandwidth off the same 2-CPU server. Every reader also - achieved its offered pace (134 ≈ 128 replays/s). -2. **Interference shows up in the tails, and only past ~60 MiB/s of read bandwidth.** - Write p99 sits at 5–10 ms through 64 readers, then jumps to **53 ms at 128** while - write p50 stays ~1 ms — tail coupling, not capacity loss. Read p99 degrades in step - (65 ms at 64 readers, when each replay is already ~1 MB). -3. **Still no load shedding** — zero 429/503 anywhere; the coupling is silent. - -## The v1 contrast: unpaced readers (adversarial reference, 2026-07-02 morning run) - -The first validation ran the same sweep with **unpaced hot-loop readers** (and writers -pinned at 12k = 61% of the then-19.6k ceiling, older server build, rates over full -elapsed): write throughput collapsed −39% at 16 readers, −65% at 64, −82% at 128. That -mode measures *read-saturated coexistence* — worst-case, and the reason the paced knob -exists. Keep `read_rate: 0` as the adversarial variant; with pacing the same suite -measures realistic bounded interference. The two together bracket the behaviour: -**bounded read load costs ~nothing in write throughput; unbounded read load fair-shares -everything down.** - -## Validity caveats - -- Single-node kind co-location (client fleet + server + MinIO on one Docker VM): the - 128-reader p99 spike may be partly node-level; magnitudes need a remote run with - separate node pools. -- Replay bodies grow during the window (~200 backfill + up to ~7k appended events by the - end), so per-replay cost rises within a run — read MiB/s is the steadier axis. - -## Status for promotion - -The v1 blockers are fixed in `ds-bench mixed`: paced readers (`--read-rate`), read -bytes/s recorded, barrier-aligned drive window, delivery counters split (data vs -control frames). Remaining before adding to the full remote suite: pick per-system -%-of-ceiling anchors from the `run-*` saturation results, and decide the paced level -grid (readers × read_rate) per cardinality. - -Companion: `results/mixed-delivery-local/FINDINGS.md`. Raw grid: `report.md` / -`aggregate.{csv,json}` here; suite: `suites/mixed-writes-local.json`. diff --git a/results/mixed-writes-local/aggregate.csv b/results/mixed-writes-local/aggregate.csv deleted file mode 100644 index bb3a00b..0000000 --- a/results/mixed-writes-local/aggregate.csv +++ /dev/null @@ -1,6 +0,0 @@ -mode,stream_count,sweep,level,readers,subscribers,writer_rate,read_rate,write_ops_per_sec,write_p50,write_p99,read_ops_per_sec,read_mib_per_sec,read_p50,read_p99,events_per_sec,delivery_p50,delivery_p99,write_bp,write_err,read_bp,read_err,status,reason -wal,50,readers,0,0,0,355,1,17303.35,1.177,9.767,0.0,0.0,,,0.0,,,0,0,0,0,ok,complete -wal,50,readers,4,4,0,355,1,17755.4,0.966,5.723,4.2,3.84564208984375,1.742,10.535,0.0,,,0,0,0,0,ok,complete -wal,50,readers,16,16,0,355,1,17756.0,0.946,5.287,16.8,15.37923583984375,3.895,17.103,0.0,,,0,0,0,0,ok,complete -wal,50,readers,64,64,0,355,1,17755.7,0.997,6.483,67.2,61.54200439453125,5.463,65.535,0.0,,,0,0,0,0,ok,complete -wal,50,readers,128,128,0,355,1,17749.55,1.087,52.927,134.4,123.02186279296875,4.911,50.943,0.0,,,0,0,0,0,ok,complete diff --git a/results/mixed-writes-local/aggregate.json b/results/mixed-writes-local/aggregate.json deleted file mode 100644 index 7ce0019..0000000 --- a/results/mixed-writes-local/aggregate.json +++ /dev/null @@ -1,132 +0,0 @@ -[ - { - "mode": "wal", - "stream_count": 50, - "sweep": "readers", - "level": 0, - "readers": 0, - "subscribers": 0, - "writer_rate": 355, - "read_rate": 1, - "write_ops_per_sec": 17303.35, - "write_p50": 1.177, - "write_p99": 9.767, - "read_ops_per_sec": 0.0, - "read_mib_per_sec": 0.0, - "read_p50": null, - "read_p99": null, - "events_per_sec": 0.0, - "delivery_p50": null, - "delivery_p99": null, - "write_bp": 0, - "write_err": 0, - "read_bp": 0, - "read_err": 0, - "status": "ok", - "reason": "complete" - }, - { - "mode": "wal", - "stream_count": 50, - "sweep": "readers", - "level": 4, - "readers": 4, - "subscribers": 0, - "writer_rate": 355, - "read_rate": 1, - "write_ops_per_sec": 17755.4, - "write_p50": 0.966, - "write_p99": 5.723, - "read_ops_per_sec": 4.2, - "read_mib_per_sec": 3.84564208984375, - "read_p50": 1.742, - "read_p99": 10.535, - "events_per_sec": 0.0, - "delivery_p50": null, - "delivery_p99": null, - "write_bp": 0, - "write_err": 0, - "read_bp": 0, - "read_err": 0, - "status": "ok", - "reason": "complete" - }, - { - "mode": "wal", - "stream_count": 50, - "sweep": "readers", - "level": 16, - "readers": 16, - "subscribers": 0, - "writer_rate": 355, - "read_rate": 1, - "write_ops_per_sec": 17756.0, - "write_p50": 0.946, - "write_p99": 5.287, - "read_ops_per_sec": 16.8, - "read_mib_per_sec": 15.37923583984375, - "read_p50": 3.895, - "read_p99": 17.103, - "events_per_sec": 0.0, - "delivery_p50": null, - "delivery_p99": null, - "write_bp": 0, - "write_err": 0, - "read_bp": 0, - "read_err": 0, - "status": "ok", - "reason": "complete" - }, - { - "mode": "wal", - "stream_count": 50, - "sweep": "readers", - "level": 64, - "readers": 64, - "subscribers": 0, - "writer_rate": 355, - "read_rate": 1, - "write_ops_per_sec": 17755.7, - "write_p50": 0.997, - "write_p99": 6.483, - "read_ops_per_sec": 67.2, - "read_mib_per_sec": 61.54200439453125, - "read_p50": 5.463, - "read_p99": 65.535, - "events_per_sec": 0.0, - "delivery_p50": null, - "delivery_p99": null, - "write_bp": 0, - "write_err": 0, - "read_bp": 0, - "read_err": 0, - "status": "ok", - "reason": "complete" - }, - { - "mode": "wal", - "stream_count": 50, - "sweep": "readers", - "level": 128, - "readers": 128, - "subscribers": 0, - "writer_rate": 355, - "read_rate": 1, - "write_ops_per_sec": 17749.55, - "write_p50": 1.087, - "write_p99": 52.927, - "read_ops_per_sec": 134.4, - "read_mib_per_sec": 123.02186279296875, - "read_p50": 4.911, - "read_p99": 50.943, - "events_per_sec": 0.0, - "delivery_p50": null, - "delivery_p99": null, - "write_bp": 0, - "write_err": 0, - "read_bp": 0, - "read_err": 0, - "status": "ok", - "reason": "complete" - } -] \ No newline at end of file diff --git a/results/mixed-writes-local/report.md b/results/mixed-writes-local/report.md deleted file mode 100644 index d5ce6f1..0000000 --- a/results/mixed-writes-local/report.md +++ /dev/null @@ -1,17 +0,0 @@ -# mixed-writes-local — mixed read/write interference report - -Sweep axis: **readers**. Latency cells are p50/p99 ms. ‡ = backpressure (503/429) observed in that class. - -## wal — 50 streams - -| readers | write ops/s | write ms | read ops/s | read MiB/s | read ms | deliv rec/s | deliv ms | status | -|---|---|---|---|---|---|---|---|---| -| 0 | 17303 | 1.2/9.8 | 0 | 0.0 | — | 0 | — | ok | -| 4 | 17755 | 1.0/5.7 | 4 | 3.8 | 1.7/10.5 | 0 | — | ok | -| 16 | 17756 | 0.9/5.3 | 17 | 15.4 | 3.9/17.1 | 0 | — | ok | -| 64 | 17756 | 1.0/6.5 | 67 | 61.5 | 5.5/65.5 | 0 | — | ok | -| 128 | 17750 | 1.1/52.9 | 134 | 123.0 | 4.9/50.9 | 0 | — | ok | - -## Findings - -_See `FINDINGS.md` in this directory for the premise-by-premise conclusions and validity caveats._ diff --git a/results/mixed-writes-local/wal/cells.json b/results/mixed-writes-local/wal/cells.json deleted file mode 100644 index e87f011..0000000 --- a/results/mixed-writes-local/wal/cells.json +++ /dev/null @@ -1,156 +0,0 @@ -{ - "cells": { - "50": { - "stream_count": 50, - "image_digest": "c7df7a0880e0", - "complete": true, - "levels": { - "0": { - "write_ops_per_sec": 17303.35, - "write_p50": 1.177, - "write_p99": 9.767, - "read_ops_per_sec": 0.0, - "read_mib_per_sec": 0.0, - "read_p50": null, - "read_p99": null, - "events_per_sec": 0.0, - "delivery_p50": null, - "delivery_p99": null, - "write_ok": 346067, - "write_bp": 0, - "write_err": 0, - "read_ok": 0, - "read_bp": 0, - "read_err": 0, - "events_received": 0, - "control_events_received": 0, - "elapsed_secs": 20.025420676, - "drive_secs": 20.0, - "level": 0, - "readers": 0, - "subscribers": 0, - "writer_rate": 355, - "read_rate": 1, - "status": "ok", - "reason": "complete" - }, - "4": { - "write_ops_per_sec": 17755.4, - "write_p50": 0.966, - "write_p99": 5.723, - "read_ops_per_sec": 4.2, - "read_mib_per_sec": 3.84564208984375, - "read_p50": 1.742, - "read_p99": 10.535, - "events_per_sec": 0.0, - "delivery_p50": null, - "delivery_p99": null, - "write_ok": 355108, - "write_bp": 0, - "write_err": 0, - "read_ok": 84, - "read_bp": 0, - "read_err": 0, - "events_received": 0, - "control_events_received": 0, - "elapsed_secs": 20.013428426, - "drive_secs": 20.0, - "level": 4, - "readers": 4, - "subscribers": 0, - "writer_rate": 355, - "read_rate": 1, - "status": "ok", - "reason": "complete" - }, - "16": { - "write_ops_per_sec": 17756.0, - "write_p50": 0.946, - "write_p99": 5.287, - "read_ops_per_sec": 16.8, - "read_mib_per_sec": 15.37923583984375, - "read_p50": 3.895, - "read_p99": 17.103, - "events_per_sec": 0.0, - "delivery_p50": null, - "delivery_p99": null, - "write_ok": 355120, - "write_bp": 0, - "write_err": 0, - "read_ok": 336, - "read_bp": 0, - "read_err": 0, - "events_received": 0, - "control_events_received": 0, - "elapsed_secs": 20.024328176, - "drive_secs": 20.0, - "level": 16, - "readers": 16, - "subscribers": 0, - "writer_rate": 355, - "read_rate": 1, - "status": "ok", - "reason": "complete" - }, - "64": { - "write_ops_per_sec": 17755.7, - "write_p50": 0.997, - "write_p99": 6.483, - "read_ops_per_sec": 67.2, - "read_mib_per_sec": 61.54200439453125, - "read_p50": 5.463, - "read_p99": 65.535, - "events_per_sec": 0.0, - "delivery_p50": null, - "delivery_p99": null, - "write_ok": 355114, - "write_bp": 0, - "write_err": 0, - "read_ok": 1344, - "read_bp": 0, - "read_err": 0, - "events_received": 0, - "control_events_received": 0, - "elapsed_secs": 20.031429217, - "drive_secs": 20.0, - "level": 64, - "readers": 64, - "subscribers": 0, - "writer_rate": 355, - "read_rate": 1, - "status": "ok", - "reason": "complete" - }, - "128": { - "write_ops_per_sec": 17749.55, - "write_p50": 1.087, - "write_p99": 52.927, - "read_ops_per_sec": 134.4, - "read_mib_per_sec": 123.02186279296875, - "read_p50": 4.911, - "read_p99": 50.943, - "events_per_sec": 0.0, - "delivery_p50": null, - "delivery_p99": null, - "write_ok": 354991, - "write_bp": 0, - "write_err": 0, - "read_ok": 2688, - "read_bp": 0, - "read_err": 0, - "events_received": 0, - "control_events_received": 0, - "elapsed_secs": 20.045879885, - "drive_secs": 20.0, - "level": 128, - "readers": 128, - "subscribers": 0, - "writer_rate": 355, - "read_rate": 1, - "status": "ok", - "reason": "complete" - } - } - } - } -} \ No newline at end of file diff --git a/results/mixed-writes/aggregate.csv b/results/mixed-writes/aggregate.csv deleted file mode 100644 index 61c8261..0000000 --- a/results/mixed-writes/aggregate.csv +++ /dev/null @@ -1,5 +0,0 @@ -mode,stream_count,sweep,level,readers,subscribers,writer_rate,read_rate,write_ops_per_sec,write_p50,write_p99,read_ops_per_sec,read_mib_per_sec,read_p50,read_p99,events_per_sec,delivery_p50,delivery_p99,write_bp,write_err,read_bp,read_err,status,reason -wal,10000,readers,0,0,0,5,0,49964.416666666664,10.055,422.143,0.0,0.0,,,0.0,,,0,0,0,0,ok,complete -wal,10000,readers,1000,1000,0,5,0,50026.05,11.415,441.087,49.916666666666664,3.044083658854167,0.488,418.559,0.0,,,0,0,0,0,ok,complete -wal,10000,readers,10000,10000,0,5,0,49825.63333333333,5.927,353.791,498.93333333333334,30.363899739583335,0.561,403.967,0.0,,,0,0,0,0,ok,complete -wal,10000,readers,100000,100000,0,5,0,50041.48333333333,14.863,454.655,4987.333333333333,303.7662068684896,0.524,879.615,0.0,,,0,0,0,0,ok,complete diff --git a/results/mixed-writes/aggregate.json b/results/mixed-writes/aggregate.json deleted file mode 100644 index 117a63d..0000000 --- a/results/mixed-writes/aggregate.json +++ /dev/null @@ -1,106 +0,0 @@ -[ - { - "mode": "wal", - "stream_count": 10000, - "sweep": "readers", - "level": 0, - "readers": 0, - "subscribers": 0, - "writer_rate": 5, - "read_rate": 0, - "write_ops_per_sec": 49964.416666666664, - "write_p50": 10.055, - "write_p99": 422.143, - "read_ops_per_sec": 0.0, - "read_mib_per_sec": 0.0, - "read_p50": null, - "read_p99": null, - "events_per_sec": 0.0, - "delivery_p50": null, - "delivery_p99": null, - "write_bp": 0, - "write_err": 0, - "read_bp": 0, - "read_err": 0, - "status": "ok", - "reason": "complete" - }, - { - "mode": "wal", - "stream_count": 10000, - "sweep": "readers", - "level": 1000, - "readers": 1000, - "subscribers": 0, - "writer_rate": 5, - "read_rate": 0, - "write_ops_per_sec": 50026.05, - "write_p50": 11.415, - "write_p99": 441.087, - "read_ops_per_sec": 49.916666666666664, - "read_mib_per_sec": 3.044083658854167, - "read_p50": 0.488, - "read_p99": 418.559, - "events_per_sec": 0.0, - "delivery_p50": null, - "delivery_p99": null, - "write_bp": 0, - "write_err": 0, - "read_bp": 0, - "read_err": 0, - "status": "ok", - "reason": "complete" - }, - { - "mode": "wal", - "stream_count": 10000, - "sweep": "readers", - "level": 10000, - "readers": 10000, - "subscribers": 0, - "writer_rate": 5, - "read_rate": 0, - "write_ops_per_sec": 49825.63333333333, - "write_p50": 5.927, - "write_p99": 353.791, - "read_ops_per_sec": 498.93333333333334, - "read_mib_per_sec": 30.363899739583335, - "read_p50": 0.561, - "read_p99": 403.967, - "events_per_sec": 0.0, - "delivery_p50": null, - "delivery_p99": null, - "write_bp": 0, - "write_err": 0, - "read_bp": 0, - "read_err": 0, - "status": "ok", - "reason": "complete" - }, - { - "mode": "wal", - "stream_count": 10000, - "sweep": "readers", - "level": 100000, - "readers": 100000, - "subscribers": 0, - "writer_rate": 5, - "read_rate": 0, - "write_ops_per_sec": 50041.48333333333, - "write_p50": 14.863, - "write_p99": 454.655, - "read_ops_per_sec": 4987.333333333333, - "read_mib_per_sec": 303.7662068684896, - "read_p50": 0.524, - "read_p99": 879.615, - "events_per_sec": 0.0, - "delivery_p50": null, - "delivery_p99": null, - "write_bp": 0, - "write_err": 0, - "read_bp": 0, - "read_err": 0, - "status": "ok", - "reason": "complete" - } -] \ No newline at end of file diff --git a/results/mixed-writes/report.md b/results/mixed-writes/report.md deleted file mode 100644 index 7901a8b..0000000 --- a/results/mixed-writes/report.md +++ /dev/null @@ -1,16 +0,0 @@ -# mixed-writes — mixed read/write interference report - -Sweep axis: **readers**. Latency cells are p50/p99 ms. ‡ = backpressure (503/429) observed in that class. - -## wal — 10000 streams - -| readers | write ops/s | write ms | read ops/s | read MiB/s | read ms | deliv rec/s | deliv ms | status | -|---|---|---|---|---|---|---|---|---| -| 0 | 49964 | 10.1/422.1 | 0 | 0.0 | — | 0 | — | ok | -| 1000 | 50026 | 11.4/441.1 | 50 | 3.0 | 0.5/418.6 | 0 | — | ok | -| 10000 | 49826 | 5.9/353.8 | 499 | 30.4 | 0.6/404.0 | 0 | — | ok | -| 100000 | 50041 | 14.9/454.7 | 4987 | 303.8 | 0.5/879.6 | 0 | — | ok | - -## Findings - -_TODO: written by hand on top of the generated data._ diff --git a/results/mixed-writes/wal/cells.json b/results/mixed-writes/wal/cells.json deleted file mode 100644 index 6fc6d13..0000000 --- a/results/mixed-writes/wal/cells.json +++ /dev/null @@ -1,127 +0,0 @@ -{ - "cells": { - "10000": { - "stream_count": 10000, - "image_digest": "92f4a6387253", - "complete": true, - "levels": { - "0": { - "write_ops_per_sec": 49964.416666666664, - "write_p50": 10.055, - "write_p99": 422.143, - "read_ops_per_sec": 0.0, - "read_mib_per_sec": 0.0, - "read_p50": null, - "read_p99": null, - "events_per_sec": 0.0, - "delivery_p50": null, - "delivery_p99": null, - "write_ok": 2997865, - "write_bp": 0, - "write_err": 0, - "read_ok": 0, - "read_bp": 0, - "read_err": 0, - "events_received": 0, - "control_events_received": 0, - "elapsed_secs": 60.482057524, - "drive_secs": 60.0, - "level": 0, - "readers": 0, - "subscribers": 0, - "writer_rate": 5, - "read_rate": 0, - "status": "ok", - "reason": "complete" - }, - "1000": { - "write_ops_per_sec": 50026.05, - "write_p50": 11.415, - "write_p99": 441.087, - "read_ops_per_sec": 49.916666666666664, - "read_mib_per_sec": 3.044083658854167, - "read_p50": 0.488, - "read_p99": 418.559, - "events_per_sec": 0.0, - "delivery_p50": null, - "delivery_p99": null, - "write_ok": 3001563, - "write_bp": 0, - "write_err": 0, - "read_ok": 2995, - "read_bp": 0, - "read_err": 0, - "events_received": 0, - "control_events_received": 0, - "elapsed_secs": 90.000492785, - "drive_secs": 60.0, - "level": 1000, - "readers": 1000, - "subscribers": 0, - "writer_rate": 5, - "read_rate": 0, - "status": "ok", - "reason": "complete" - }, - "10000": { - "write_ops_per_sec": 49825.63333333333, - "write_p50": 5.927, - "write_p99": 353.791, - "read_ops_per_sec": 498.93333333333334, - "read_mib_per_sec": 30.363899739583335, - "read_p50": 0.561, - "read_p99": 403.967, - "events_per_sec": 0.0, - "delivery_p50": null, - "delivery_p99": null, - "write_ok": 2989538, - "write_bp": 0, - "write_err": 0, - "read_ok": 29936, - "read_bp": 0, - "read_err": 0, - "events_received": 0, - "control_events_received": 0, - "elapsed_secs": 90.027624664, - "drive_secs": 60.0, - "level": 10000, - "readers": 10000, - "subscribers": 0, - "writer_rate": 5, - "read_rate": 0, - "status": "ok", - "reason": "complete" - }, - "100000": { - "write_ops_per_sec": 50041.48333333333, - "write_p50": 14.863, - "write_p99": 454.655, - "read_ops_per_sec": 4987.333333333333, - "read_mib_per_sec": 303.7662068684896, - "read_p50": 0.524, - "read_p99": 879.615, - "events_per_sec": 0.0, - "delivery_p50": null, - "delivery_p99": null, - "write_ok": 3002489, - "write_bp": 0, - "write_err": 0, - "read_ok": 299240, - "read_bp": 0, - "read_err": 0, - "events_received": 0, - "control_events_received": 0, - "elapsed_secs": 90.200862867, - "drive_secs": 60.0, - "level": 100000, - "readers": 100000, - "subscribers": 0, - "writer_rate": 5, - "read_rate": 0, - "status": "ok", - "reason": "complete" - } - } - } - } -} \ No newline at end of file diff --git a/results/reads-catchup/aggregate.csv b/results/reads-catchup/aggregate.csv deleted file mode 100644 index ce6188f..0000000 --- a/results/reads-catchup/aggregate.csv +++ /dev/null @@ -1,17 +0,0 @@ -mode,stream_count,connections,ops_per_sec,bytes_per_sec,mib_per_sec,p50,p99,backpressure,other_err,status,is_peak -ursula,10,8,146.93333333333334,2465132270.9333334,2350.9,52.159,85.887,0,0,ok,False -ursula,10,32,148.46666666666667,2490857335.4666667,2375.5,202.239,516.607,0,0,ok,True -ursula,10,128,0.0,0.0,0.0,,,0,0,error,False -ursula,10,512,0.0,0.0,0.0,,,0,0,error,False -ursula,100,8,0.0,0.0,0.0,,,0,0,error,True -ursula,100,32,0.0,0.0,0.0,,,0,0,error,False -ursula,100,128,0.0,0.0,0.0,,,0,0,error,False -ursula,100,512,0.0,0.0,0.0,,,0,0,error,False -wal,10,8,84.06666666666666,1410404625.0666666,1345.1,94.719,110.847,0,0,ok,False -wal,10,32,148.8,2496449740.8,2380.8,217.727,237.311,0,0,ok,True -wal,10,128,0.0,0.0,0.0,,,0,0,error,False -wal,10,512,0.0,0.0,0.0,,,0,0,error,False -wal,100,8,83.33333333333333,1398101333.3333333,1333.3,95.551,111.743,0,0,ok,False -wal,100,32,148.86666666666667,2497568221.866667,2381.9,218.495,238.207,0,0,ok,True -wal,100,128,0.0,0.0,0.0,,,0,0,error,False -wal,100,512,0.0,0.0,0.0,,,0,0,error,False diff --git a/results/reads-catchup/aggregate.json b/results/reads-catchup/aggregate.json deleted file mode 100644 index 460524e..0000000 --- a/results/reads-catchup/aggregate.json +++ /dev/null @@ -1,226 +0,0 @@ -[ - { - "mode": "ursula", - "stream_count": 10, - "connections": 8, - "ops_per_sec": 146.93333333333334, - "bytes_per_sec": 2465132270.9333334, - "mib_per_sec": 2350.9, - "p50": 52.159, - "p99": 85.887, - "backpressure": 0, - "other_err": 0, - "status": "ok", - "is_peak": false - }, - { - "mode": "ursula", - "stream_count": 10, - "connections": 32, - "ops_per_sec": 148.46666666666667, - "bytes_per_sec": 2490857335.4666667, - "mib_per_sec": 2375.5, - "p50": 202.239, - "p99": 516.607, - "backpressure": 0, - "other_err": 0, - "status": "ok", - "is_peak": true - }, - { - "mode": "ursula", - "stream_count": 10, - "connections": 128, - "ops_per_sec": 0.0, - "bytes_per_sec": 0.0, - "mib_per_sec": 0.0, - "p50": null, - "p99": null, - "backpressure": 0, - "other_err": 0, - "status": "error", - "is_peak": false - }, - { - "mode": "ursula", - "stream_count": 10, - "connections": 512, - "ops_per_sec": 0.0, - "bytes_per_sec": 0.0, - "mib_per_sec": 0.0, - "p50": null, - "p99": null, - "backpressure": 0, - "other_err": 0, - "status": "error", - "is_peak": false - }, - { - "mode": "ursula", - "stream_count": 100, - "connections": 8, - "ops_per_sec": 0.0, - "bytes_per_sec": 0.0, - "mib_per_sec": 0.0, - "p50": null, - "p99": null, - "backpressure": 0, - "other_err": 0, - "status": "error", - "is_peak": true - }, - { - "mode": "ursula", - "stream_count": 100, - "connections": 32, - "ops_per_sec": 0.0, - "bytes_per_sec": 0.0, - "mib_per_sec": 0.0, - "p50": null, - "p99": null, - "backpressure": 0, - "other_err": 0, - "status": "error", - "is_peak": false - }, - { - "mode": "ursula", - "stream_count": 100, - "connections": 128, - "ops_per_sec": 0.0, - "bytes_per_sec": 0.0, - "mib_per_sec": 0.0, - "p50": null, - "p99": null, - "backpressure": 0, - "other_err": 0, - "status": "error", - "is_peak": false - }, - { - "mode": "ursula", - "stream_count": 100, - "connections": 512, - "ops_per_sec": 0.0, - "bytes_per_sec": 0.0, - "mib_per_sec": 0.0, - "p50": null, - "p99": null, - "backpressure": 0, - "other_err": 0, - "status": "error", - "is_peak": false - }, - { - "mode": "wal", - "stream_count": 10, - "connections": 8, - "ops_per_sec": 84.06666666666666, - "bytes_per_sec": 1410404625.0666666, - "mib_per_sec": 1345.1, - "p50": 94.719, - "p99": 110.847, - "backpressure": 0, - "other_err": 0, - "status": "ok", - "is_peak": false - }, - { - "mode": "wal", - "stream_count": 10, - "connections": 32, - "ops_per_sec": 148.8, - "bytes_per_sec": 2496449740.8, - "mib_per_sec": 2380.8, - "p50": 217.727, - "p99": 237.311, - "backpressure": 0, - "other_err": 0, - "status": "ok", - "is_peak": true - }, - { - "mode": "wal", - "stream_count": 10, - "connections": 128, - "ops_per_sec": 0.0, - "bytes_per_sec": 0.0, - "mib_per_sec": 0.0, - "p50": null, - "p99": null, - "backpressure": 0, - "other_err": 0, - "status": "error", - "is_peak": false - }, - { - "mode": "wal", - "stream_count": 10, - "connections": 512, - "ops_per_sec": 0.0, - "bytes_per_sec": 0.0, - "mib_per_sec": 0.0, - "p50": null, - "p99": null, - "backpressure": 0, - "other_err": 0, - "status": "error", - "is_peak": false - }, - { - "mode": "wal", - "stream_count": 100, - "connections": 8, - "ops_per_sec": 83.33333333333333, - "bytes_per_sec": 1398101333.3333333, - "mib_per_sec": 1333.3, - "p50": 95.551, - "p99": 111.743, - "backpressure": 0, - "other_err": 0, - "status": "ok", - "is_peak": false - }, - { - "mode": "wal", - "stream_count": 100, - "connections": 32, - "ops_per_sec": 148.86666666666667, - "bytes_per_sec": 2497568221.866667, - "mib_per_sec": 2381.9, - "p50": 218.495, - "p99": 238.207, - "backpressure": 0, - "other_err": 0, - "status": "ok", - "is_peak": true - }, - { - "mode": "wal", - "stream_count": 100, - "connections": 128, - "ops_per_sec": 0.0, - "bytes_per_sec": 0.0, - "mib_per_sec": 0.0, - "p50": null, - "p99": null, - "backpressure": 0, - "other_err": 0, - "status": "error", - "is_peak": false - }, - { - "mode": "wal", - "stream_count": 100, - "connections": 512, - "ops_per_sec": 0.0, - "bytes_per_sec": 0.0, - "mib_per_sec": 0.0, - "p50": null, - "p99": null, - "backpressure": 0, - "other_err": 0, - "status": "error", - "is_peak": false - } -] \ No newline at end of file diff --git a/results/reads-catchup/report.md b/results/reads-catchup/report.md deleted file mode 100644 index 7693c38..0000000 --- a/results/reads-catchup/report.md +++ /dev/null @@ -1,29 +0,0 @@ -# reads-catchup — read-scalability report - -Each cell: aggregate read throughput (MiB/s) @ p99 latency (ms). ‡ = backpressure (503/429) observed at this load. - -## wal — throughput @ p99 over stream_count × connections - -| streams | 8 | 32 | 128 | 512 | -|---|---|---|---|---| -| 10 | 1345MiB/s@111ms | 2381MiB/s@237ms | ERR(0) | ERR(0) | -| 100 | 1333MiB/s@112ms | 2382MiB/s@238ms | ERR(0) | ERR(0) | - -Peak read throughput per cardinality: -- streams=10: 2381 MiB/s at 32 connections -- streams=100: 2382 MiB/s at 32 connections - -## ursula — throughput @ p99 over stream_count × connections - -| streams | 8 | 32 | 128 | 512 | -|---|---|---|---|---| -| 10 | 2351MiB/s@86ms | 2375MiB/s@517ms | ERR(0) | ERR(0) | -| 100 | ERR(0) | ERR(0) | ERR(0) | ERR(0) | - -Peak read throughput per cardinality: -- streams=10: 2375 MiB/s at 32 connections -- streams=100: 0 MiB/s at 8 connections - -## Findings - -_TODO: written by hand on top of the generated data._ diff --git a/results/reads-catchup/ursula/cells.json b/results/reads-catchup/ursula/cells.json deleted file mode 100644 index 3373b64..0000000 --- a/results/reads-catchup/ursula/cells.json +++ /dev/null @@ -1,106 +0,0 @@ -{ - "cells": { - "10": { - "stream_count": 10, - "image_digest": "5669f4bbeed1", - "complete": true, - "connections": { - "8": { - "connections": 8, - "ops_per_sec": 146.93333333333334, - "bytes_per_sec": 2465132270.9333334, - "p50": 52.159, - "p99": 85.887, - "backpressure": 0, - "other_err": 0, - "status": "ok", - "reason": "complete" - }, - "32": { - "connections": 32, - "ops_per_sec": 148.46666666666667, - "bytes_per_sec": 2490857335.4666667, - "p50": 202.239, - "p99": 516.607, - "backpressure": 0, - "other_err": 0, - "status": "ok", - "reason": "complete" - }, - "128": { - "connections": 128, - "ops_per_sec": 0.0, - "bytes_per_sec": 0.0, - "p50": null, - "p99": null, - "backpressure": 0, - "other_err": 0, - "status": "error", - "reason": "no_reads" - }, - "512": { - "connections": 512, - "ops_per_sec": 0.0, - "bytes_per_sec": 0.0, - "p50": null, - "p99": null, - "backpressure": 0, - "other_err": 0, - "status": "error", - "reason": "no_reads" - } - } - }, - "100": { - "stream_count": 100, - "image_digest": "5669f4bbeed1", - "complete": true, - "connections": { - "8": { - "connections": 8, - "ops_per_sec": 0.0, - "bytes_per_sec": 0.0, - "p50": null, - "p99": null, - "backpressure": 0, - "other_err": 0, - "status": "error", - "reason": "no_reads" - }, - "32": { - "connections": 32, - "ops_per_sec": 0.0, - "bytes_per_sec": 0.0, - "p50": null, - "p99": null, - "backpressure": 0, - "other_err": 0, - "status": "error", - "reason": "no_reads" - }, - "128": { - "connections": 128, - "ops_per_sec": 0.0, - "bytes_per_sec": 0.0, - "p50": null, - "p99": null, - "backpressure": 0, - "other_err": 0, - "status": "error", - "reason": "no_reads" - }, - "512": { - "connections": 512, - "ops_per_sec": 0.0, - "bytes_per_sec": 0.0, - "p50": null, - "p99": null, - "backpressure": 0, - "other_err": 0, - "status": "error", - "reason": "no_reads" - } - } - } - } -} \ No newline at end of file diff --git a/results/reads-catchup/wal/cells.json b/results/reads-catchup/wal/cells.json deleted file mode 100644 index 25697d8..0000000 --- a/results/reads-catchup/wal/cells.json +++ /dev/null @@ -1,106 +0,0 @@ -{ - "cells": { - "10": { - "stream_count": 10, - "image_digest": "92f4a6387253", - "complete": true, - "connections": { - "8": { - "connections": 8, - "ops_per_sec": 84.06666666666666, - "bytes_per_sec": 1410404625.0666666, - "p50": 94.719, - "p99": 110.847, - "backpressure": 0, - "other_err": 0, - "status": "ok", - "reason": "complete" - }, - "32": { - "connections": 32, - "ops_per_sec": 148.8, - "bytes_per_sec": 2496449740.8, - "p50": 217.727, - "p99": 237.311, - "backpressure": 0, - "other_err": 0, - "status": "ok", - "reason": "complete" - }, - "128": { - "connections": 128, - "ops_per_sec": 0.0, - "bytes_per_sec": 0.0, - "p50": null, - "p99": null, - "backpressure": 0, - "other_err": 0, - "status": "error", - "reason": "no_reads" - }, - "512": { - "connections": 512, - "ops_per_sec": 0.0, - "bytes_per_sec": 0.0, - "p50": null, - "p99": null, - "backpressure": 0, - "other_err": 0, - "status": "error", - "reason": "no_reads" - } - } - }, - "100": { - "stream_count": 100, - "image_digest": "92f4a6387253", - "complete": true, - "connections": { - "8": { - "connections": 8, - "ops_per_sec": 83.33333333333333, - "bytes_per_sec": 1398101333.3333333, - "p50": 95.551, - "p99": 111.743, - "backpressure": 0, - "other_err": 0, - "status": "ok", - "reason": "complete" - }, - "32": { - "connections": 32, - "ops_per_sec": 148.86666666666667, - "bytes_per_sec": 2497568221.866667, - "p50": 218.495, - "p99": 238.207, - "backpressure": 0, - "other_err": 0, - "status": "ok", - "reason": "complete" - }, - "128": { - "connections": 128, - "ops_per_sec": 0.0, - "bytes_per_sec": 0.0, - "p50": null, - "p99": null, - "backpressure": 0, - "other_err": 0, - "status": "error", - "reason": "no_reads" - }, - "512": { - "connections": 512, - "ops_per_sec": 0.0, - "bytes_per_sec": 0.0, - "p50": null, - "p99": null, - "backpressure": 0, - "other_err": 0, - "status": "error", - "reason": "no_reads" - } - } - } - } -} \ No newline at end of file diff --git a/results/reads-sse-remote/aggregate.csv b/results/reads-sse-remote/aggregate.csv deleted file mode 100644 index cfb810c..0000000 --- a/results/reads-sse-remote/aggregate.csv +++ /dev/null @@ -1,17 +0,0 @@ -mode,stream_count,connections,ops_per_sec,bytes_per_sec,mib_per_sec,p50,p99,backpressure,other_err,status,is_peak -ursula,10,64,3204.266666666667,820292.2666666667,0.8,0.959,1.482,0,0,ok,False -ursula,10,256,12817.066666666668,3281169.066666667,3.1,1.309,1.89,0,0,ok,False -ursula,10,1024,51261.4,13122918.4,12.5,1.73,2.659,0,0,ok,False -ursula,10,2048,102532.8,26248396.8,25.0,1.867,2.765,0,0,ok,True -ursula,100,64,3068.9333333333334,785646.9333333333,0.7,1.238,42.175,0,0,ok,False -ursula,100,256,11841.533333333333,3031432.533333333,2.9,1.923,47.295,0,0,ok,False -ursula,100,1024,44130.13333333333,11297314.133333333,10.8,1.976,56.735,0,0,ok,False -ursula,100,2048,80552.0,20621312.0,19.7,1.872,62.879,0,0,ok,True -wal,10,64,3204.266666666667,820292.2666666667,0.8,1.07,1.415,0,0,ok,False -wal,10,256,12817.066666666668,3281169.066666667,3.1,1.533,2.097,0,0,ok,False -wal,10,1024,51268.26666666667,13124676.266666668,12.5,1.45,2.593,0,0,ok,False -wal,10,2048,102536.53333333334,26249352.533333335,25.0,1.664,2.883,0,0,ok,True -wal,100,64,3204.266666666667,820292.2666666667,0.8,0.93,1.237,0,0,ok,False -wal,100,256,12817.066666666668,3281169.066666667,3.1,1.43,2.323,0,0,ok,False -wal,100,1024,51268.26666666667,13124676.266666668,12.5,1.329,2.407,0,0,ok,False -wal,100,2048,102537.13333333332,26249506.133333333,25.0,1.375,2.755,0,0,ok,True diff --git a/results/reads-sse-remote/aggregate.json b/results/reads-sse-remote/aggregate.json deleted file mode 100644 index d969776..0000000 --- a/results/reads-sse-remote/aggregate.json +++ /dev/null @@ -1,226 +0,0 @@ -[ - { - "mode": "ursula", - "stream_count": 10, - "connections": 64, - "ops_per_sec": 3204.266666666667, - "bytes_per_sec": 820292.2666666667, - "mib_per_sec": 0.8, - "p50": 0.959, - "p99": 1.482, - "backpressure": 0, - "other_err": 0, - "status": "ok", - "is_peak": false - }, - { - "mode": "ursula", - "stream_count": 10, - "connections": 256, - "ops_per_sec": 12817.066666666668, - "bytes_per_sec": 3281169.066666667, - "mib_per_sec": 3.1, - "p50": 1.309, - "p99": 1.89, - "backpressure": 0, - "other_err": 0, - "status": "ok", - "is_peak": false - }, - { - "mode": "ursula", - "stream_count": 10, - "connections": 1024, - "ops_per_sec": 51261.4, - "bytes_per_sec": 13122918.4, - "mib_per_sec": 12.5, - "p50": 1.73, - "p99": 2.659, - "backpressure": 0, - "other_err": 0, - "status": "ok", - "is_peak": false - }, - { - "mode": "ursula", - "stream_count": 10, - "connections": 2048, - "ops_per_sec": 102532.8, - "bytes_per_sec": 26248396.8, - "mib_per_sec": 25.0, - "p50": 1.867, - "p99": 2.765, - "backpressure": 0, - "other_err": 0, - "status": "ok", - "is_peak": true - }, - { - "mode": "ursula", - "stream_count": 100, - "connections": 64, - "ops_per_sec": 3068.9333333333334, - "bytes_per_sec": 785646.9333333333, - "mib_per_sec": 0.7, - "p50": 1.238, - "p99": 42.175, - "backpressure": 0, - "other_err": 0, - "status": "ok", - "is_peak": false - }, - { - "mode": "ursula", - "stream_count": 100, - "connections": 256, - "ops_per_sec": 11841.533333333333, - "bytes_per_sec": 3031432.533333333, - "mib_per_sec": 2.9, - "p50": 1.923, - "p99": 47.295, - "backpressure": 0, - "other_err": 0, - "status": "ok", - "is_peak": false - }, - { - "mode": "ursula", - "stream_count": 100, - "connections": 1024, - "ops_per_sec": 44130.13333333333, - "bytes_per_sec": 11297314.133333333, - "mib_per_sec": 10.8, - "p50": 1.976, - "p99": 56.735, - "backpressure": 0, - "other_err": 0, - "status": "ok", - "is_peak": false - }, - { - "mode": "ursula", - "stream_count": 100, - "connections": 2048, - "ops_per_sec": 80552.0, - "bytes_per_sec": 20621312.0, - "mib_per_sec": 19.7, - "p50": 1.872, - "p99": 62.879, - "backpressure": 0, - "other_err": 0, - "status": "ok", - "is_peak": true - }, - { - "mode": "wal", - "stream_count": 10, - "connections": 64, - "ops_per_sec": 3204.266666666667, - "bytes_per_sec": 820292.2666666667, - "mib_per_sec": 0.8, - "p50": 1.07, - "p99": 1.415, - "backpressure": 0, - "other_err": 0, - "status": "ok", - "is_peak": false - }, - { - "mode": "wal", - "stream_count": 10, - "connections": 256, - "ops_per_sec": 12817.066666666668, - "bytes_per_sec": 3281169.066666667, - "mib_per_sec": 3.1, - "p50": 1.533, - "p99": 2.097, - "backpressure": 0, - "other_err": 0, - "status": "ok", - "is_peak": false - }, - { - "mode": "wal", - "stream_count": 10, - "connections": 1024, - "ops_per_sec": 51268.26666666667, - "bytes_per_sec": 13124676.266666668, - "mib_per_sec": 12.5, - "p50": 1.45, - "p99": 2.593, - "backpressure": 0, - "other_err": 0, - "status": "ok", - "is_peak": false - }, - { - "mode": "wal", - "stream_count": 10, - "connections": 2048, - "ops_per_sec": 102536.53333333334, - "bytes_per_sec": 26249352.533333335, - "mib_per_sec": 25.0, - "p50": 1.664, - "p99": 2.883, - "backpressure": 0, - "other_err": 0, - "status": "ok", - "is_peak": true - }, - { - "mode": "wal", - "stream_count": 100, - "connections": 64, - "ops_per_sec": 3204.266666666667, - "bytes_per_sec": 820292.2666666667, - "mib_per_sec": 0.8, - "p50": 0.93, - "p99": 1.237, - "backpressure": 0, - "other_err": 0, - "status": "ok", - "is_peak": false - }, - { - "mode": "wal", - "stream_count": 100, - "connections": 256, - "ops_per_sec": 12817.066666666668, - "bytes_per_sec": 3281169.066666667, - "mib_per_sec": 3.1, - "p50": 1.43, - "p99": 2.323, - "backpressure": 0, - "other_err": 0, - "status": "ok", - "is_peak": false - }, - { - "mode": "wal", - "stream_count": 100, - "connections": 1024, - "ops_per_sec": 51268.26666666667, - "bytes_per_sec": 13124676.266666668, - "mib_per_sec": 12.5, - "p50": 1.329, - "p99": 2.407, - "backpressure": 0, - "other_err": 0, - "status": "ok", - "is_peak": false - }, - { - "mode": "wal", - "stream_count": 100, - "connections": 2048, - "ops_per_sec": 102537.13333333332, - "bytes_per_sec": 26249506.133333333, - "mib_per_sec": 25.0, - "p50": 1.375, - "p99": 2.755, - "backpressure": 0, - "other_err": 0, - "status": "ok", - "is_peak": true - } -] \ No newline at end of file diff --git a/results/reads-sse-remote/report.md b/results/reads-sse-remote/report.md deleted file mode 100644 index f1dc2ea..0000000 --- a/results/reads-sse-remote/report.md +++ /dev/null @@ -1,29 +0,0 @@ -# reads-sse-remote — read-scalability report - -Each cell: aggregate read throughput (MiB/s) @ p99 latency (ms). ‡ = backpressure (503/429) observed at this load. - -## wal — throughput @ p99 over stream_count × connections - -| streams | 64 | 256 | 1024 | 2048 | -|---|---|---|---|---| -| 10 | 1MiB/s@1ms | 3MiB/s@2ms | 13MiB/s@3ms | 25MiB/s@3ms | -| 100 | 1MiB/s@1ms | 3MiB/s@2ms | 13MiB/s@2ms | 25MiB/s@3ms | - -Peak read throughput per cardinality: -- streams=10: 25 MiB/s at 2048 connections -- streams=100: 25 MiB/s at 2048 connections - -## ursula — throughput @ p99 over stream_count × connections - -| streams | 64 | 256 | 1024 | 2048 | -|---|---|---|---|---| -| 10 | 1MiB/s@1ms | 3MiB/s@2ms | 13MiB/s@3ms | 25MiB/s@3ms | -| 100 | 1MiB/s@42ms | 3MiB/s@47ms | 11MiB/s@57ms | 20MiB/s@63ms | - -Peak read throughput per cardinality: -- streams=10: 25 MiB/s at 2048 connections -- streams=100: 20 MiB/s at 2048 connections - -## Findings - -_TODO: written by hand on top of the generated data._ diff --git a/results/reads-sse-remote/ursula/cells.json b/results/reads-sse-remote/ursula/cells.json deleted file mode 100644 index 9380e1f..0000000 --- a/results/reads-sse-remote/ursula/cells.json +++ /dev/null @@ -1,106 +0,0 @@ -{ - "cells": { - "10": { - "stream_count": 10, - "image_digest": "5669f4bbeed1", - "complete": true, - "connections": { - "64": { - "connections": 64, - "ops_per_sec": 3204.266666666667, - "bytes_per_sec": 820292.2666666667, - "p50": 0.959, - "p99": 1.482, - "backpressure": 0, - "other_err": 0, - "status": "ok", - "reason": "complete" - }, - "256": { - "connections": 256, - "ops_per_sec": 12817.066666666668, - "bytes_per_sec": 3281169.066666667, - "p50": 1.309, - "p99": 1.89, - "backpressure": 0, - "other_err": 0, - "status": "ok", - "reason": "complete" - }, - "1024": { - "connections": 1024, - "ops_per_sec": 51261.4, - "bytes_per_sec": 13122918.4, - "p50": 1.73, - "p99": 2.659, - "backpressure": 0, - "other_err": 0, - "status": "ok", - "reason": "complete" - }, - "2048": { - "connections": 2048, - "ops_per_sec": 102532.8, - "bytes_per_sec": 26248396.8, - "p50": 1.867, - "p99": 2.765, - "backpressure": 0, - "other_err": 0, - "status": "ok", - "reason": "complete" - } - } - }, - "100": { - "stream_count": 100, - "image_digest": "5669f4bbeed1", - "complete": true, - "connections": { - "64": { - "connections": 64, - "ops_per_sec": 3068.9333333333334, - "bytes_per_sec": 785646.9333333333, - "p50": 1.238, - "p99": 42.175, - "backpressure": 0, - "other_err": 0, - "status": "ok", - "reason": "complete" - }, - "256": { - "connections": 256, - "ops_per_sec": 11841.533333333333, - "bytes_per_sec": 3031432.533333333, - "p50": 1.923, - "p99": 47.295, - "backpressure": 0, - "other_err": 0, - "status": "ok", - "reason": "complete" - }, - "1024": { - "connections": 1024, - "ops_per_sec": 44130.13333333333, - "bytes_per_sec": 11297314.133333333, - "p50": 1.976, - "p99": 56.735, - "backpressure": 0, - "other_err": 0, - "status": "ok", - "reason": "complete" - }, - "2048": { - "connections": 2048, - "ops_per_sec": 80552.0, - "bytes_per_sec": 20621312.0, - "p50": 1.872, - "p99": 62.879, - "backpressure": 0, - "other_err": 0, - "status": "ok", - "reason": "complete" - } - } - } - } -} \ No newline at end of file diff --git a/results/reads-sse-remote/wal/cells.json b/results/reads-sse-remote/wal/cells.json deleted file mode 100644 index 80a1aaf..0000000 --- a/results/reads-sse-remote/wal/cells.json +++ /dev/null @@ -1,106 +0,0 @@ -{ - "cells": { - "10": { - "stream_count": 10, - "image_digest": "92f4a6387253", - "complete": true, - "connections": { - "64": { - "connections": 64, - "ops_per_sec": 3204.266666666667, - "bytes_per_sec": 820292.2666666667, - "p50": 1.07, - "p99": 1.415, - "backpressure": 0, - "other_err": 0, - "status": "ok", - "reason": "complete" - }, - "256": { - "connections": 256, - "ops_per_sec": 12817.066666666668, - "bytes_per_sec": 3281169.066666667, - "p50": 1.533, - "p99": 2.097, - "backpressure": 0, - "other_err": 0, - "status": "ok", - "reason": "complete" - }, - "1024": { - "connections": 1024, - "ops_per_sec": 51268.26666666667, - "bytes_per_sec": 13124676.266666668, - "p50": 1.45, - "p99": 2.593, - "backpressure": 0, - "other_err": 0, - "status": "ok", - "reason": "complete" - }, - "2048": { - "connections": 2048, - "ops_per_sec": 102536.53333333334, - "bytes_per_sec": 26249352.533333335, - "p50": 1.664, - "p99": 2.883, - "backpressure": 0, - "other_err": 0, - "status": "ok", - "reason": "complete" - } - } - }, - "100": { - "stream_count": 100, - "image_digest": "92f4a6387253", - "complete": true, - "connections": { - "64": { - "connections": 64, - "ops_per_sec": 3204.266666666667, - "bytes_per_sec": 820292.2666666667, - "p50": 0.93, - "p99": 1.237, - "backpressure": 0, - "other_err": 0, - "status": "ok", - "reason": "complete" - }, - "256": { - "connections": 256, - "ops_per_sec": 12817.066666666668, - "bytes_per_sec": 3281169.066666667, - "p50": 1.43, - "p99": 2.323, - "backpressure": 0, - "other_err": 0, - "status": "ok", - "reason": "complete" - }, - "1024": { - "connections": 1024, - "ops_per_sec": 51268.26666666667, - "bytes_per_sec": 13124676.266666668, - "p50": 1.329, - "p99": 2.407, - "backpressure": 0, - "other_err": 0, - "status": "ok", - "reason": "complete" - }, - "2048": { - "connections": 2048, - "ops_per_sec": 102537.13333333332, - "bytes_per_sec": 26249506.133333333, - "p50": 1.375, - "p99": 2.755, - "backpressure": 0, - "other_err": 0, - "status": "ok", - "reason": "complete" - } - } - } - } -} \ No newline at end of file diff --git a/results/run-durable/aggregate.csv b/results/run-durable/aggregate.csv deleted file mode 100644 index f06ba79..0000000 --- a/results/run-durable/aggregate.csv +++ /dev/null @@ -1,19 +0,0 @@ -mode,stream_count,pods,throughput,p50,p99,pod_mem_mb,pod_mem_p50_mb,saturated,status,reason -memory,100,16,436185.375,0.249,0.397,60,58,True,ok,plateau -wal,100,16,457133.875,0.235,0.38,95,14,True,ok,plateau -wal-tailcache,100,24,488139.625,0.232,0.377,69,68,True,ok,plateau -memory,1000,16,479132.875,1.846,6.659,46,37,True,ok,plateau -wal,1000,24,655482.875,1.429,5.087,37,27,True,ok,plateau -wal-tailcache,1000,16,565306.375,1.516,5.471,56,46,True,ok,plateau -memory,10000,32,575104.875,2.139,122.239,151,86,True,ok,plateau -wal,10000,32,815869.75,1.81,87.039,175,75,True,ok,plateau -wal-tailcache,10000,32,794364.625,1.823,90.623,170,115,True,ok,plateau -memory,100000,80,732216.125,3.047,867.327,736,374,True,ok,plateau -wal,100000,100,1560267.0,3.227,719.359,915,726,True,ok,plateau -wal-tailcache,100000,80,1734866.375,3.169,626.687,908,706,True,ok,plateau -memory,200000,100,534176.625,25.983,1686.527,781,311,True,ok,plateau -wal,200000,100,1503496.125,80.575,973.311,987,743,True,ok,plateau -wal-tailcache,200000,100,1414943.125,50.879,1158.143,775,489,True,ok,plateau -memory,500000,400,1329370.0,33.247,2099.199,752,491,True,ok,plateau -wal,500000,250,2045868.875,151.551,2609.151,857,513,True,ok,plateau -wal-tailcache,500000,250,1892514.75,127.359,1221.631,960,716,True,ok,plateau diff --git a/results/run-durable/aggregate.json b/results/run-durable/aggregate.json deleted file mode 100644 index 5bb6a04..0000000 --- a/results/run-durable/aggregate.json +++ /dev/null @@ -1,432 +0,0 @@ -[ - { - "mode": "memory", - "stream_count": 100, - "pods": 16, - "throughput": 436185.375, - "p50": 0.249, - "p99": 0.397, - "pod_mem_mb": 60, - "pod_mem_p50_mb": 58, - "saturated": true, - "status": "ok", - "reason": "plateau", - "walk": [ - [ - 16, - 436185.375 - ], - [ - 24, - 435389.75 - ] - ] - }, - { - "mode": "wal", - "stream_count": 100, - "pods": 16, - "throughput": 457133.875, - "p50": 0.235, - "p99": 0.38, - "pod_mem_mb": 95, - "pod_mem_p50_mb": 14, - "saturated": true, - "status": "ok", - "reason": "plateau", - "walk": [ - [ - 16, - 457133.875 - ], - [ - 24, - 482137.625 - ] - ] - }, - { - "mode": "wal-tailcache", - "stream_count": 100, - "pods": 24, - "throughput": 488139.625, - "p50": 0.232, - "p99": 0.377, - "pod_mem_mb": 69, - "pod_mem_p50_mb": 68, - "saturated": true, - "status": "ok", - "reason": "plateau", - "walk": [ - [ - 16, - 450994.75 - ], - [ - 24, - 488139.625 - ], - [ - 32, - 518751.75 - ] - ] - }, - { - "mode": "memory", - "stream_count": 1000, - "pods": 16, - "throughput": 479132.875, - "p50": 1.846, - "p99": 6.659, - "pod_mem_mb": 46, - "pod_mem_p50_mb": 37, - "saturated": true, - "status": "ok", - "reason": "plateau", - "walk": [ - [ - 16, - 479132.875 - ], - [ - 24, - 470912.25 - ] - ] - }, - { - "mode": "wal", - "stream_count": 1000, - "pods": 24, - "throughput": 655482.875, - "p50": 1.429, - "p99": 5.087, - "pod_mem_mb": 37, - "pod_mem_p50_mb": 27, - "saturated": true, - "status": "ok", - "reason": "plateau", - "walk": [ - [ - 16, - 586828.125 - ], - [ - 24, - 655482.875 - ], - [ - 32, - 690208.25 - ] - ] - }, - { - "mode": "wal-tailcache", - "stream_count": 1000, - "pods": 16, - "throughput": 565306.375, - "p50": 1.516, - "p99": 5.471, - "pod_mem_mb": 56, - "pod_mem_p50_mb": 46, - "saturated": true, - "status": "ok", - "reason": "plateau", - "walk": [ - [ - 16, - 565306.375 - ], - [ - 24, - 603933.125 - ] - ] - }, - { - "mode": "memory", - "stream_count": 10000, - "pods": 32, - "throughput": 575104.875, - "p50": 2.139, - "p99": 122.239, - "pod_mem_mb": 151, - "pod_mem_p50_mb": 86, - "saturated": true, - "status": "ok", - "reason": "plateau", - "walk": [ - [ - 32, - 575104.875 - ], - [ - 48, - 616271.875 - ] - ] - }, - { - "mode": "wal", - "stream_count": 10000, - "pods": 32, - "throughput": 815869.75, - "p50": 1.81, - "p99": 87.039, - "pod_mem_mb": 175, - "pod_mem_p50_mb": 75, - "saturated": true, - "status": "ok", - "reason": "plateau", - "walk": [ - [ - 32, - 815869.75 - ], - [ - 48, - 863481.875 - ] - ] - }, - { - "mode": "wal-tailcache", - "stream_count": 10000, - "pods": 32, - "throughput": 794364.625, - "p50": 1.823, - "p99": 90.623, - "pod_mem_mb": 170, - "pod_mem_p50_mb": 115, - "saturated": true, - "status": "ok", - "reason": "plateau", - "walk": [ - [ - 32, - 794364.625 - ], - [ - 48, - 752539.625 - ] - ] - }, - { - "mode": "memory", - "stream_count": 100000, - "pods": 80, - "throughput": 732216.125, - "p50": 3.047, - "p99": 867.327, - "pod_mem_mb": 736, - "pod_mem_p50_mb": 374, - "saturated": true, - "status": "ok", - "reason": "plateau", - "walk": [ - [ - 80, - 732216.125 - ], - [ - 100, - 407044.0 - ] - ] - }, - { - "mode": "wal", - "stream_count": 100000, - "pods": 100, - "throughput": 1560267.0, - "p50": 3.227, - "p99": 719.359, - "pod_mem_mb": 915, - "pod_mem_p50_mb": 726, - "saturated": true, - "status": "ok", - "reason": "plateau", - "walk": [ - [ - 80, - 810519.875 - ], - [ - 100, - 1560267.0 - ], - [ - 110, - 503403.625 - ] - ] - }, - { - "mode": "wal-tailcache", - "stream_count": 100000, - "pods": 80, - "throughput": 1734866.375, - "p50": 3.169, - "p99": 626.687, - "pod_mem_mb": 908, - "pod_mem_p50_mb": 706, - "saturated": true, - "status": "ok", - "reason": "plateau", - "walk": [ - [ - 80, - 1734866.375 - ], - [ - 100, - 516286.0 - ] - ] - }, - { - "mode": "memory", - "stream_count": 200000, - "pods": 100, - "throughput": 534176.625, - "p50": 25.983, - "p99": 1686.527, - "pod_mem_mb": 781, - "pod_mem_p50_mb": 311, - "saturated": true, - "status": "ok", - "reason": "plateau", - "walk": [ - [ - 100, - 534176.625 - ], - [ - 160, - 377864.25 - ] - ] - }, - { - "mode": "wal", - "stream_count": 200000, - "pods": 100, - "throughput": 1503496.125, - "p50": 80.575, - "p99": 973.311, - "pod_mem_mb": 987, - "pod_mem_p50_mb": 743, - "saturated": true, - "status": "ok", - "reason": "plateau", - "walk": [ - [ - 100, - 1503496.125 - ], - [ - 160, - 1602824.375 - ] - ] - }, - { - "mode": "wal-tailcache", - "stream_count": 200000, - "pods": 100, - "throughput": 1414943.125, - "p50": 50.879, - "p99": 1158.143, - "pod_mem_mb": 775, - "pod_mem_p50_mb": 489, - "saturated": true, - "status": "ok", - "reason": "plateau", - "walk": [ - [ - 100, - 1414943.125 - ], - [ - 160, - 1191656.375 - ] - ] - }, - { - "mode": "memory", - "stream_count": 500000, - "pods": 400, - "throughput": 1329370.0, - "p50": 33.247, - "p99": 2099.199, - "pod_mem_mb": 752, - "pod_mem_p50_mb": 491, - "saturated": true, - "status": "ok", - "reason": "plateau", - "walk": [ - [ - 250, - 1224462.625 - ], - [ - 400, - 1329370.0 - ], - [ - 625, - 361267.75 - ] - ] - }, - { - "mode": "wal", - "stream_count": 500000, - "pods": 250, - "throughput": 2045868.875, - "p50": 151.551, - "p99": 2609.151, - "pod_mem_mb": 857, - "pod_mem_p50_mb": 513, - "saturated": true, - "status": "ok", - "reason": "plateau", - "walk": [ - [ - 250, - 2045868.875 - ], - [ - 400, - 1828700.75 - ] - ] - }, - { - "mode": "wal-tailcache", - "stream_count": 500000, - "pods": 250, - "throughput": 1892514.75, - "p50": 127.359, - "p99": 1221.631, - "pod_mem_mb": 960, - "pod_mem_p50_mb": 716, - "saturated": true, - "status": "ok", - "reason": "plateau", - "walk": [ - [ - 250, - 1892514.75 - ], - [ - 400, - 1589852.0 - ] - ] - } -] \ No newline at end of file diff --git a/results/run-durable/memory/cells.json b/results/run-durable/memory/cells.json deleted file mode 100644 index 1db56c1..0000000 --- a/results/run-durable/memory/cells.json +++ /dev/null @@ -1,146 +0,0 @@ -{ - "cells": { - "100": { - "stream_count": 100, - "image_digest": "b7d78084e886", - "walk": [ - [ - 16, - 436185.375 - ], - [ - 24, - 435389.75 - ] - ], - "pinned_pods": 16, - "throughput": 436185.375, - "p50": 0.249, - "p99": 0.397, - "pod_mem_mb": 60, - "saturated": true, - "status": "ok", - "reason": "plateau", - "pod_mem_p50_mb": 58 - }, - "1000": { - "stream_count": 1000, - "image_digest": "b7d78084e886", - "walk": [ - [ - 16, - 479132.875 - ], - [ - 24, - 470912.25 - ] - ], - "pinned_pods": 16, - "throughput": 479132.875, - "p50": 1.846, - "p99": 6.659, - "pod_mem_mb": 46, - "saturated": true, - "status": "ok", - "reason": "plateau", - "pod_mem_p50_mb": 37 - }, - "10000": { - "stream_count": 10000, - "image_digest": "b7d78084e886", - "walk": [ - [ - 32, - 575104.875 - ], - [ - 48, - 616271.875 - ] - ], - "pinned_pods": 32, - "throughput": 575104.875, - "p50": 2.139, - "p99": 122.239, - "pod_mem_mb": 151, - "saturated": true, - "status": "ok", - "reason": "plateau", - "pod_mem_p50_mb": 86 - }, - "100000": { - "stream_count": 100000, - "image_digest": "b7d78084e886", - "walk": [ - [ - 80, - 732216.125 - ], - [ - 100, - 407044.0 - ] - ], - "pinned_pods": 80, - "throughput": 732216.125, - "p50": 3.047, - "p99": 867.327, - "pod_mem_mb": 736, - "saturated": true, - "status": "ok", - "reason": "plateau", - "pod_mem_p50_mb": 374 - }, - "200000": { - "stream_count": 200000, - "image_digest": "b7d78084e886", - "walk": [ - [ - 100, - 534176.625 - ], - [ - 160, - 377864.25 - ] - ], - "pinned_pods": 100, - "throughput": 534176.625, - "p50": 25.983, - "p99": 1686.527, - "pod_mem_mb": 781, - "saturated": true, - "status": "ok", - "reason": "plateau", - "pod_mem_p50_mb": 311 - }, - "500000": { - "stream_count": 500000, - "image_digest": "b7d78084e886", - "walk": [ - [ - 250, - 1224462.625 - ], - [ - 400, - 1329370.0 - ], - [ - 625, - 361267.75 - ] - ], - "pinned_pods": 400, - "throughput": 1329370.0, - "p50": 33.247, - "p99": 2099.199, - "pod_mem_mb": 752, - "saturated": true, - "status": "ok", - "reason": "plateau", - "pod_mem_p50_mb": 491 - } - } -} \ No newline at end of file diff --git a/results/run-durable/report.md b/results/run-durable/report.md deleted file mode 100644 index f5cfebb..0000000 --- a/results/run-durable/report.md +++ /dev/null @@ -1,56 +0,0 @@ -# run-durable — write-throughput report - -## Throughput at saturation (ops/s) - -| streams | wal | wal-tailcache | memory | -|---|---|---|---| -| 100 | 457k | 488k | 436k | -| 1000 | 655k | 565k | 479k | -| 10000 | 816k | 794k | 575k | -| 100000 | 1560k | 1735k | 732k | -| 200000 | 1503k | 1415k | 534k | -| 500000 | 2046k | 1893k | 1329k | - -† = not saturated (ladder exhausted) — treat as a lower bound. - -## Pod memory at saturation — peak / p50 (MiB) - -| streams | wal | wal-tailcache | memory | -|---|---|---|---| -| 100 | 95 / 14 | 69 / 68 | 60 / 58 | -| 1000 | 37 / 27 | 56 / 46 | 46 / 37 | -| 10000 | 175 / 75 | 170 / 115 | 151 / 86 | -| 100000 | 915 / 726 | 908 / 706 | 736 / 374 | -| 200000 | 987 / 743 | 775 / 489 | 781 / 311 | -| 500000 | 857 / 513 | 960 / 716 | 752 / 491 | - -_Pod working set = cgroup `memory.current − inactive_file` (anon + active page cache), sampled each second at the pinned rung. **peak** = high-water (catches bursts like an in-RAM Raft log filling); **p50** = median (what the server steadily holds resident). peak ≈ p50 ⇒ steadily resident; peak ≫ p50 ⇒ transient spikes._ - -## Saturation walks (pods → ops/s) - -- **memory 100**: 16:436k → 24:435k (pinned 16, plateau) -- **wal 100**: 16:457k → 24:482k (pinned 16, plateau) -- **wal-tailcache 100**: 16:451k → 24:488k → 32:519k (pinned 24, plateau) -- **memory 1000**: 16:479k → 24:471k (pinned 16, plateau) -- **wal 1000**: 16:587k → 24:655k → 32:690k (pinned 24, plateau) -- **wal-tailcache 1000**: 16:565k → 24:604k (pinned 16, plateau) -- **memory 10000**: 32:575k → 48:616k (pinned 32, plateau) -- **wal 10000**: 32:816k → 48:863k (pinned 32, plateau) -- **wal-tailcache 10000**: 32:794k → 48:753k (pinned 32, plateau) -- **memory 100000**: 80:732k → 100:407k (pinned 80, plateau) -- **wal 100000**: 80:811k → 100:1560k → 110:503k (pinned 100, plateau) -- **wal-tailcache 100000**: 80:1735k → 100:516k (pinned 80, plateau) -- **memory 200000**: 100:534k → 160:378k (pinned 100, plateau) -- **wal 200000**: 100:1503k → 160:1603k (pinned 100, plateau) -- **wal-tailcache 200000**: 100:1415k → 160:1192k (pinned 100, plateau) -- **memory 500000**: 250:1224k → 400:1329k → 625:361k (pinned 400, plateau) -- **wal 500000**: 250:2046k → 400:1829k (pinned 250, plateau) -- **wal-tailcache 500000**: 250:1893k → 400:1590k (pinned 250, plateau) - -## Findings - -_TODO: written by hand on top of the generated data._ - -## Caveats - -_Single-node best-case; not 3-node Raft. Throughput is a saturation ceiling per the ladder._ diff --git a/results/run-durable/wal-tailcache/cells.json b/results/run-durable/wal-tailcache/cells.json deleted file mode 100644 index 3e30caa..0000000 --- a/results/run-durable/wal-tailcache/cells.json +++ /dev/null @@ -1,146 +0,0 @@ -{ - "cells": { - "100": { - "stream_count": 100, - "image_digest": "a3af0b74d273", - "walk": [ - [ - 16, - 450994.75 - ], - [ - 24, - 488139.625 - ], - [ - 32, - 518751.75 - ] - ], - "pinned_pods": 24, - "throughput": 488139.625, - "p50": 0.232, - "p99": 0.377, - "pod_mem_mb": 69, - "saturated": true, - "status": "ok", - "reason": "plateau", - "pod_mem_p50_mb": 68 - }, - "1000": { - "stream_count": 1000, - "image_digest": "a3af0b74d273", - "walk": [ - [ - 16, - 565306.375 - ], - [ - 24, - 603933.125 - ] - ], - "pinned_pods": 16, - "throughput": 565306.375, - "p50": 1.516, - "p99": 5.471, - "pod_mem_mb": 56, - "saturated": true, - "status": "ok", - "reason": "plateau", - "pod_mem_p50_mb": 46 - }, - "10000": { - "stream_count": 10000, - "image_digest": "a3af0b74d273", - "walk": [ - [ - 32, - 794364.625 - ], - [ - 48, - 752539.625 - ] - ], - "pinned_pods": 32, - "throughput": 794364.625, - "p50": 1.823, - "p99": 90.623, - "pod_mem_mb": 170, - "saturated": true, - "status": "ok", - "reason": "plateau", - "pod_mem_p50_mb": 115 - }, - "100000": { - "stream_count": 100000, - "image_digest": "a3af0b74d273", - "walk": [ - [ - 80, - 1734866.375 - ], - [ - 100, - 516286.0 - ] - ], - "pinned_pods": 80, - "throughput": 1734866.375, - "p50": 3.169, - "p99": 626.687, - "pod_mem_mb": 908, - "saturated": true, - "status": "ok", - "reason": "plateau", - "pod_mem_p50_mb": 706 - }, - "200000": { - "stream_count": 200000, - "image_digest": "a3af0b74d273", - "walk": [ - [ - 100, - 1414943.125 - ], - [ - 160, - 1191656.375 - ] - ], - "pinned_pods": 100, - "throughput": 1414943.125, - "p50": 50.879, - "p99": 1158.143, - "pod_mem_mb": 775, - "saturated": true, - "status": "ok", - "reason": "plateau", - "pod_mem_p50_mb": 489 - }, - "500000": { - "stream_count": 500000, - "image_digest": "a3af0b74d273", - "walk": [ - [ - 250, - 1892514.75 - ], - [ - 400, - 1589852.0 - ] - ], - "pinned_pods": 250, - "throughput": 1892514.75, - "p50": 127.359, - "p99": 1221.631, - "pod_mem_mb": 960, - "saturated": true, - "status": "ok", - "reason": "plateau", - "pod_mem_p50_mb": 716 - } - } -} \ No newline at end of file diff --git a/results/run-durable/wal/cells.json b/results/run-durable/wal/cells.json deleted file mode 100644 index d39c489..0000000 --- a/results/run-durable/wal/cells.json +++ /dev/null @@ -1,150 +0,0 @@ -{ - "cells": { - "100": { - "stream_count": 100, - "image_digest": "92f4a6387253", - "walk": [ - [ - 16, - 457133.875 - ], - [ - 24, - 482137.625 - ] - ], - "pinned_pods": 16, - "throughput": 457133.875, - "p50": 0.235, - "p99": 0.38, - "pod_mem_mb": 95, - "saturated": true, - "status": "ok", - "reason": "plateau", - "pod_mem_p50_mb": 14 - }, - "1000": { - "stream_count": 1000, - "image_digest": "92f4a6387253", - "walk": [ - [ - 16, - 586828.125 - ], - [ - 24, - 655482.875 - ], - [ - 32, - 690208.25 - ] - ], - "pinned_pods": 24, - "throughput": 655482.875, - "p50": 1.429, - "p99": 5.087, - "pod_mem_mb": 37, - "saturated": true, - "status": "ok", - "reason": "plateau", - "pod_mem_p50_mb": 27 - }, - "10000": { - "stream_count": 10000, - "image_digest": "92f4a6387253", - "walk": [ - [ - 32, - 815869.75 - ], - [ - 48, - 863481.875 - ] - ], - "pinned_pods": 32, - "throughput": 815869.75, - "p50": 1.81, - "p99": 87.039, - "pod_mem_mb": 175, - "saturated": true, - "status": "ok", - "reason": "plateau", - "pod_mem_p50_mb": 75 - }, - "100000": { - "stream_count": 100000, - "image_digest": "92f4a6387253", - "walk": [ - [ - 80, - 810519.875 - ], - [ - 100, - 1560267.0 - ], - [ - 110, - 503403.625 - ] - ], - "pinned_pods": 100, - "throughput": 1560267.0, - "p50": 3.227, - "p99": 719.359, - "pod_mem_mb": 915, - "saturated": true, - "status": "ok", - "reason": "plateau", - "pod_mem_p50_mb": 726 - }, - "200000": { - "stream_count": 200000, - "image_digest": "92f4a6387253", - "walk": [ - [ - 100, - 1503496.125 - ], - [ - 160, - 1602824.375 - ] - ], - "pinned_pods": 100, - "throughput": 1503496.125, - "p50": 80.575, - "p99": 973.311, - "pod_mem_mb": 987, - "saturated": true, - "status": "ok", - "reason": "plateau", - "pod_mem_p50_mb": 743 - }, - "500000": { - "stream_count": 500000, - "image_digest": "92f4a6387253", - "walk": [ - [ - 250, - 2045868.875 - ], - [ - 400, - 1828700.75 - ] - ], - "pinned_pods": 250, - "throughput": 2045868.875, - "p50": 151.551, - "p99": 2609.151, - "pod_mem_mb": 857, - "saturated": true, - "status": "ok", - "reason": "plateau", - "pod_mem_p50_mb": 513 - } - } -} \ No newline at end of file diff --git a/results/run-node/aggregate.csv b/results/run-node/aggregate.csv deleted file mode 100644 index a3998dc..0000000 --- a/results/run-node/aggregate.csv +++ /dev/null @@ -1,4 +0,0 @@ -mode,stream_count,pods,throughput,p50,p99,pod_mem_mb,pod_mem_p50_mb,saturated,status,reason -node,100,4,55147.75,1.671,3.829,245,171,True,ok,plateau -node,1000,4,60056.75,19.375,29.391,393,290,True,ok,plateau -node,10000,16,150917.0,21.791,43.487,803,651,True,ok,plateau diff --git a/results/run-node/aggregate.json b/results/run-node/aggregate.json deleted file mode 100644 index b321c15..0000000 --- a/results/run-node/aggregate.json +++ /dev/null @@ -1,75 +0,0 @@ -[ - { - "mode": "node", - "stream_count": 100, - "pods": 4, - "throughput": 55147.75, - "p50": 1.671, - "p99": 3.829, - "pod_mem_mb": 245, - "pod_mem_p50_mb": 171, - "saturated": true, - "status": "ok", - "reason": "plateau", - "walk": [ - [ - 4, - 55147.75 - ], - [ - 8, - 57874.375 - ] - ] - }, - { - "mode": "node", - "stream_count": 1000, - "pods": 4, - "throughput": 60056.75, - "p50": 19.375, - "p99": 29.391, - "pod_mem_mb": 393, - "pod_mem_p50_mb": 290, - "saturated": true, - "status": "ok", - "reason": "plateau", - "walk": [ - [ - 4, - 60056.75 - ], - [ - 8, - 62403.0 - ] - ] - }, - { - "mode": "node", - "stream_count": 10000, - "pods": 16, - "throughput": 150917.0, - "p50": 21.791, - "p99": 43.487, - "pod_mem_mb": 803, - "pod_mem_p50_mb": 651, - "saturated": true, - "status": "ok", - "reason": "plateau", - "walk": [ - [ - 8, - 116379.875 - ], - [ - 16, - 150917.0 - ], - [ - 24, - 87496.375 - ] - ] - } -] \ No newline at end of file diff --git a/results/run-node/node/cells.json b/results/run-node/node/cells.json deleted file mode 100644 index 1437097..0000000 --- a/results/run-node/node/cells.json +++ /dev/null @@ -1,77 +0,0 @@ -{ - "cells": { - "100": { - "stream_count": 100, - "image_digest": "75a11da44c80", - "walk": [ - [ - 4, - 55147.75 - ], - [ - 8, - 57874.375 - ] - ], - "pinned_pods": 4, - "throughput": 55147.75, - "p50": 1.671, - "p99": 3.829, - "pod_mem_mb": 245, - "saturated": true, - "status": "ok", - "reason": "plateau", - "pod_mem_p50_mb": 171 - }, - "1000": { - "stream_count": 1000, - "image_digest": "75a11da44c80", - "walk": [ - [ - 4, - 60056.75 - ], - [ - 8, - 62403.0 - ] - ], - "pinned_pods": 4, - "throughput": 60056.75, - "p50": 19.375, - "p99": 29.391, - "pod_mem_mb": 393, - "saturated": true, - "status": "ok", - "reason": "plateau", - "pod_mem_p50_mb": 290 - }, - "10000": { - "stream_count": 10000, - "image_digest": "75a11da44c80", - "walk": [ - [ - 8, - 116379.875 - ], - [ - 16, - 150917.0 - ], - [ - 24, - 87496.375 - ] - ], - "pinned_pods": 16, - "throughput": 150917.0, - "p50": 21.791, - "p99": 43.487, - "pod_mem_mb": 803, - "saturated": true, - "status": "ok", - "reason": "plateau", - "pod_mem_p50_mb": 651 - } - } -} \ No newline at end of file diff --git a/results/run-node/report.md b/results/run-node/report.md deleted file mode 100644 index 5ce68a5..0000000 --- a/results/run-node/report.md +++ /dev/null @@ -1,35 +0,0 @@ -# run-node — write-throughput report - -## Throughput at saturation (ops/s) - -| streams | node | -|---|---| -| 100 | 55k | -| 1000 | 60k | -| 10000 | 151k | - -† = not saturated (ladder exhausted) — treat as a lower bound. - -## Pod memory at saturation — peak / p50 (MiB) - -| streams | node | -|---|---| -| 100 | 245 / 171 | -| 1000 | 393 / 290 | -| 10000 | 803 / 651 | - -_Pod working set = cgroup `memory.current − inactive_file` (anon + active page cache), sampled each second at the pinned rung. **peak** = high-water (catches bursts like an in-RAM Raft log filling); **p50** = median (what the server steadily holds resident). peak ≈ p50 ⇒ steadily resident; peak ≫ p50 ⇒ transient spikes._ - -## Saturation walks (pods → ops/s) - -- **node 100**: 4:55k → 8:58k (pinned 4, plateau) -- **node 1000**: 4:60k → 8:62k (pinned 4, plateau) -- **node 10000**: 8:116k → 16:151k → 24:87k (pinned 16, plateau) - -## Findings - -_TODO: written by hand on top of the generated data._ - -## Caveats - -_Single-node best-case; not 3-node Raft. Throughput is a saturation ceiling per the ladder._ diff --git a/results/run-s2/aggregate.csv b/results/run-s2/aggregate.csv deleted file mode 100644 index b479777..0000000 --- a/results/run-s2/aggregate.csv +++ /dev/null @@ -1,3 +0,0 @@ -mode,stream_count,pods,throughput,p50,p99,pod_mem_mb,pod_mem_p50_mb,saturated,status,reason -s2,100,2,1975.0,51.135,52.351,64,50,True,ok,plateau -s2,1000,,0.0,,,,,False,error,creation_choke diff --git a/results/run-s2/aggregate.json b/results/run-s2/aggregate.json deleted file mode 100644 index beeae13..0000000 --- a/results/run-s2/aggregate.json +++ /dev/null @@ -1,48 +0,0 @@ -[ - { - "mode": "s2", - "stream_count": 100, - "pods": 2, - "throughput": 1975.0, - "p50": 51.135, - "p99": 52.351, - "pod_mem_mb": 64, - "pod_mem_p50_mb": 50, - "saturated": true, - "status": "ok", - "reason": "plateau", - "walk": [ - [ - 2, - 1975.0 - ], - [ - 4, - 1965.625 - ] - ] - }, - { - "mode": "s2", - "stream_count": 1000, - "pods": null, - "throughput": 0.0, - "p50": null, - "p99": null, - "pod_mem_mb": null, - "pod_mem_p50_mb": null, - "saturated": false, - "status": "error", - "reason": "creation_choke", - "walk": [ - [ - 2, - 19000.0 - ], - [ - 4, - 0.0 - ] - ] - } -] \ No newline at end of file diff --git a/results/run-s2/report.md b/results/run-s2/report.md deleted file mode 100644 index 0e10237..0000000 --- a/results/run-s2/report.md +++ /dev/null @@ -1,30 +0,0 @@ -# run-s2 — write-throughput report - -## Throughput at saturation (ops/s) - -| streams | s2 | -|---|---| -| 100 | 2k | - -† = not saturated (ladder exhausted) — treat as a lower bound. - -## Pod memory at saturation — peak / p50 (MiB) - -| streams | s2 | -|---|---| -| 100 | 64 / 50 | - -_Pod working set = cgroup `memory.current − inactive_file` (anon + active page cache), sampled each second at the pinned rung. **peak** = high-water (catches bursts like an in-RAM Raft log filling); **p50** = median (what the server steadily holds resident). peak ≈ p50 ⇒ steadily resident; peak ≫ p50 ⇒ transient spikes._ - -## Saturation walks (pods → ops/s) - -- **s2 100**: 2:2k → 4:2k (pinned 2, plateau) -- **s2 1000**: 2:19k → 4:0k (pinned None, creation_choke) - -## Findings - -_TODO: written by hand on top of the generated data._ - -## Caveats - -_Single-node best-case; not 3-node Raft. Throughput is a saturation ceiling per the ladder._ diff --git a/results/run-s2/s2/cells.json b/results/run-s2/s2/cells.json deleted file mode 100644 index 7ba71be..0000000 --- a/results/run-s2/s2/cells.json +++ /dev/null @@ -1,49 +0,0 @@ -{ - "cells": { - "100": { - "stream_count": 100, - "image_digest": "b6341d566e3d", - "walk": [ - [ - 2, - 1975.0 - ], - [ - 4, - 1965.625 - ] - ], - "pinned_pods": 2, - "throughput": 1975.0, - "p50": 51.135, - "p99": 52.351, - "pod_mem_mb": 64, - "saturated": true, - "status": "ok", - "reason": "plateau", - "pod_mem_p50_mb": 50 - }, - "1000": { - "stream_count": 1000, - "image_digest": "b6341d566e3d", - "walk": [ - [ - 2, - 19000.0 - ], - [ - 4, - 0.0 - ] - ], - "pinned_pods": null, - "throughput": 0.0, - "p50": null, - "p99": null, - "pod_mem_mb": null, - "saturated": false, - "status": "error", - "reason": "creation_choke" - } - } -} \ No newline at end of file diff --git a/results/run-ursula/aggregate.csv b/results/run-ursula/aggregate.csv deleted file mode 100644 index 67b2177..0000000 --- a/results/run-ursula/aggregate.csv +++ /dev/null @@ -1,7 +0,0 @@ -mode,stream_count,pods,throughput,p50,p99,pod_mem_mb,pod_mem_p50_mb,saturated,status,reason -ursula-disk,100,4,2451.625,14.215,88.191,1098,1053,True,ok,plateau -ursula-memory,100,4,64287.125,0.534,35.839,2613,2076,True,ok,plateau -ursula-disk,1000,4,6974.5,111.999,387.583,1655,1515,True,ok,plateau -ursula-memory,1000,16,112295.625,,,2481,2054,False,ok,ladder_exhausted -ursula-disk,10000,16,11683.625,405.503,2598.911,2487,2259,True,ok,plateau -ursula-memory,10000,16,149503.5,64.223,298.751,3482,2899,True,ok,plateau diff --git a/results/run-ursula/aggregate.json b/results/run-ursula/aggregate.json deleted file mode 100644 index 2acc2f5..0000000 --- a/results/run-ursula/aggregate.json +++ /dev/null @@ -1,152 +0,0 @@ -[ - { - "mode": "ursula-disk", - "stream_count": 100, - "pods": 4, - "throughput": 2451.625, - "p50": 14.215, - "p99": 88.191, - "pod_mem_mb": 1098, - "pod_mem_p50_mb": 1053, - "saturated": true, - "status": "ok", - "reason": "plateau", - "walk": [ - [ - 4, - 2451.625 - ], - [ - 8, - 1946.625 - ] - ] - }, - { - "mode": "ursula-memory", - "stream_count": 100, - "pods": 4, - "throughput": 64287.125, - "p50": 0.534, - "p99": 35.839, - "pod_mem_mb": 2613, - "pod_mem_p50_mb": 2076, - "saturated": true, - "status": "ok", - "reason": "plateau", - "walk": [ - [ - 4, - 64287.125 - ], - [ - 8, - 66007.5 - ] - ] - }, - { - "mode": "ursula-disk", - "stream_count": 1000, - "pods": 4, - "throughput": 6974.5, - "p50": 111.999, - "p99": 387.583, - "pod_mem_mb": 1655, - "pod_mem_p50_mb": 1515, - "saturated": true, - "status": "ok", - "reason": "plateau", - "walk": [ - [ - 4, - 6974.5 - ], - [ - 8, - 5255.625 - ] - ] - }, - { - "mode": "ursula-memory", - "stream_count": 1000, - "pods": 16, - "throughput": 112295.625, - "p50": null, - "p99": null, - "pod_mem_mb": 2481, - "pod_mem_p50_mb": 2054, - "saturated": false, - "status": "ok", - "reason": "ladder_exhausted", - "walk": [ - [ - 4, - 79092.625 - ], - [ - 8, - 103785.25 - ], - [ - 16, - 112295.625 - ] - ] - }, - { - "mode": "ursula-disk", - "stream_count": 10000, - "pods": 16, - "throughput": 11683.625, - "p50": 405.503, - "p99": 2598.911, - "pod_mem_mb": 2487, - "pod_mem_p50_mb": 2259, - "saturated": true, - "status": "ok", - "reason": "plateau", - "walk": [ - [ - 8, - 8921.625 - ], - [ - 16, - 11683.625 - ], - [ - 24, - 10545.75 - ] - ] - }, - { - "mode": "ursula-memory", - "stream_count": 10000, - "pods": 16, - "throughput": 149503.5, - "p50": 64.223, - "p99": 298.751, - "pod_mem_mb": 3482, - "pod_mem_p50_mb": 2899, - "saturated": true, - "status": "ok", - "reason": "plateau", - "walk": [ - [ - 8, - 119871.25 - ], - [ - 16, - 149503.5 - ], - [ - 24, - 130657.75 - ] - ] - } -] \ No newline at end of file diff --git a/results/run-ursula/report.md b/results/run-ursula/report.md deleted file mode 100644 index 1e3766a..0000000 --- a/results/run-ursula/report.md +++ /dev/null @@ -1,38 +0,0 @@ -# run-ursula — write-throughput report - -## Throughput at saturation (ops/s) - -| streams | ursula-memory | ursula-disk | -|---|---|---| -| 100 | 64k | 2k | -| 1000 | 112k† | 7k | -| 10000 | 150k | 12k | - -† = not saturated (ladder exhausted) — treat as a lower bound. - -## Pod memory at saturation — peak / p50 (MiB) - -| streams | ursula-memory | ursula-disk | -|---|---|---| -| 100 | 2613 / 2076 | 1098 / 1053 | -| 1000 | 2481 / 2054 | 1655 / 1515 | -| 10000 | 3482 / 2899 | 2487 / 2259 | - -_Pod working set = cgroup `memory.current − inactive_file` (anon + active page cache), sampled each second at the pinned rung. **peak** = high-water (catches bursts like an in-RAM Raft log filling); **p50** = median (what the server steadily holds resident). peak ≈ p50 ⇒ steadily resident; peak ≫ p50 ⇒ transient spikes._ - -## Saturation walks (pods → ops/s) - -- **ursula-disk 100**: 4:2k → 8:2k (pinned 4, plateau) -- **ursula-memory 100**: 4:64k → 8:66k (pinned 4, plateau) -- **ursula-disk 1000**: 4:7k → 8:5k (pinned 4, plateau) -- **ursula-memory 1000**: 4:79k → 8:104k → 16:112k (pinned 16, ladder_exhausted) -- **ursula-disk 10000**: 8:9k → 16:12k → 24:11k (pinned 16, plateau) -- **ursula-memory 10000**: 8:120k → 16:150k → 24:131k (pinned 16, plateau) - -## Findings - -_TODO: written by hand on top of the generated data._ - -## Caveats - -_Single-node best-case; not 3-node Raft. Throughput is a saturation ceiling per the ladder._ diff --git a/results/run-ursula/ursula-disk/cells.json b/results/run-ursula/ursula-disk/cells.json deleted file mode 100644 index e98b13e..0000000 --- a/results/run-ursula/ursula-disk/cells.json +++ /dev/null @@ -1,77 +0,0 @@ -{ - "cells": { - "100": { - "stream_count": 100, - "image_digest": "51db2ad52ee4", - "walk": [ - [ - 4, - 2451.625 - ], - [ - 8, - 1946.625 - ] - ], - "pinned_pods": 4, - "throughput": 2451.625, - "p50": 14.215, - "p99": 88.191, - "pod_mem_mb": 1098, - "saturated": true, - "status": "ok", - "reason": "plateau", - "pod_mem_p50_mb": 1053 - }, - "1000": { - "stream_count": 1000, - "image_digest": "51db2ad52ee4", - "walk": [ - [ - 4, - 6974.5 - ], - [ - 8, - 5255.625 - ] - ], - "pinned_pods": 4, - "throughput": 6974.5, - "p50": 111.999, - "p99": 387.583, - "pod_mem_mb": 1655, - "saturated": true, - "status": "ok", - "reason": "plateau", - "pod_mem_p50_mb": 1515 - }, - "10000": { - "stream_count": 10000, - "image_digest": "51db2ad52ee4", - "walk": [ - [ - 8, - 8921.625 - ], - [ - 16, - 11683.625 - ], - [ - 24, - 10545.75 - ] - ], - "pinned_pods": 16, - "throughput": 11683.625, - "p50": 405.503, - "p99": 2598.911, - "pod_mem_mb": 2487, - "saturated": true, - "status": "ok", - "reason": "plateau", - "pod_mem_p50_mb": 2259 - } - } -} \ No newline at end of file diff --git a/results/run-ursula/ursula-memory/cells.json b/results/run-ursula/ursula-memory/cells.json deleted file mode 100644 index 0da06fc..0000000 --- a/results/run-ursula/ursula-memory/cells.json +++ /dev/null @@ -1,81 +0,0 @@ -{ - "cells": { - "100": { - "stream_count": 100, - "image_digest": "eeeb6d3de518", - "walk": [ - [ - 4, - 64287.125 - ], - [ - 8, - 66007.5 - ] - ], - "pinned_pods": 4, - "throughput": 64287.125, - "p50": 0.534, - "p99": 35.839, - "pod_mem_mb": 2613, - "saturated": true, - "status": "ok", - "reason": "plateau", - "pod_mem_p50_mb": 2076 - }, - "1000": { - "stream_count": 1000, - "image_digest": "eeeb6d3de518", - "walk": [ - [ - 4, - 79092.625 - ], - [ - 8, - 103785.25 - ], - [ - 16, - 112295.625 - ] - ], - "pinned_pods": 16, - "throughput": 112295.625, - "p50": null, - "p99": null, - "pod_mem_mb": 2481, - "saturated": false, - "status": "ok", - "reason": "ladder_exhausted", - "pod_mem_p50_mb": 2054 - }, - "10000": { - "stream_count": 10000, - "image_digest": "eeeb6d3de518", - "walk": [ - [ - 8, - 119871.25 - ], - [ - 16, - 149503.5 - ], - [ - 24, - 130657.75 - ] - ], - "pinned_pods": 16, - "throughput": 149503.5, - "p50": 64.223, - "p99": 298.751, - "pod_mem_mb": 3482, - "saturated": true, - "status": "ok", - "reason": "plateau", - "pod_mem_p50_mb": 2899 - } - } -} \ No newline at end of file diff --git a/results/sse-comparison.csv b/results/sse-comparison.csv deleted file mode 100644 index d353d3d..0000000 --- a/results/sse-comparison.csv +++ /dev/null @@ -1,13 +0,0 @@ -config,subs,p50_ms,p90_ms,p99_ms,p999_ms,max_ms,pod_mem_mb,pod_mem_p50_mb,events_per_sec -ursula disk,1,1.307,1.41,1.528,2.315,2.315,15,14,85.7 -ursula disk,10,1.822,1.976,2.261,41.215,41.247,14,14,857.3 -ursula disk,100,2.135,2.391,2.761,3.373,3.605,14,13,8560.5 -ursula disk,1000,4.247,5.199,5.867,6.503,7.611,14,13,75080.2 -ursula in-memory,1,0.396,0.487,0.589,0.745,0.745,14,13,85.7 -ursula in-memory,10,0.504,0.621,0.748,42.623,42.719,14,13,857.3 -ursula in-memory,100,0.835,1.036,1.192,1.968,2.191,14,13,8564.2 -ursula in-memory,1000,2.979,3.899,4.511,4.831,5.427,14,13,75206.5 -wal (cache off),1,0.359,0.449,0.577,0.904,0.904,6,5,85.7 -wal (cache off),10,0.482,0.598,0.717,0.855,0.931,7,6,857.3 -wal (cache off),100,0.905,1.113,1.276,1.408,1.685,10,9,8560.9 -wal (cache off),1000,3.645,4.531,5.079,5.387,5.743,19,18,75008.5 diff --git a/results/sse-comparison.md b/results/sse-comparison.md deleted file mode 100644 index dd01ee9..0000000 --- a/results/sse-comparison.md +++ /dev/null @@ -1,42 +0,0 @@ -# SSE Fan-out — delivery latency - -1 stream, 1 writer @ 50 ev/s, swept total subscribers; one well-provisioned -client pod (single wall clock). Writer-paced → metric is delivery latency. - -## Median (p50, ms) - -| config \ subscribers | 1 | 10 | 100 | 1000 | -|---|---|---|---|---| -| wal (cache off) | 0.359 | 0.482 | 0.905 | 3.645 | -| ursula in-memory | 0.396 | 0.504 | 0.835 | 2.979 | -| ursula disk | 1.307 | 1.822 | 2.135 | 4.247 | - -## Full spread (p50 / p99 / max, ms) - -| config | subs | p50 | p90 | p99 | p999 | max | -|---|---|---|---|---|---|---| -| wal (cache off) | 1 | 0.359 | 0.449 | 0.577 | 0.904 | 0.904 | -| wal (cache off) | 10 | 0.482 | 0.598 | 0.717 | 0.855 | 0.931 | -| wal (cache off) | 100 | 0.905 | 1.113 | 1.276 | 1.408 | 1.685 | -| wal (cache off) | 1000 | 3.645 | 4.531 | 5.079 | 5.387 | 5.743 | -| ursula in-memory | 1 | 0.396 | 0.487 | 0.589 | 0.745 | 0.745 | -| ursula in-memory | 10 | 0.504 | 0.621 | 0.748 | 42.623 | 42.719 | -| ursula in-memory | 100 | 0.835 | 1.036 | 1.192 | 1.968 | 2.191 | -| ursula in-memory | 1000 | 2.979 | 3.899 | 4.511 | 4.831 | 5.427 | -| ursula disk | 1 | 1.307 | 1.41 | 1.528 | 2.315 | 2.315 | -| ursula disk | 10 | 1.822 | 1.976 | 2.261 | 41.215 | 41.247 | -| ursula disk | 100 | 2.135 | 2.391 | 2.761 | 3.373 | 3.605 | -| ursula disk | 1000 | 4.247 | 5.199 | 5.867 | 6.503 | 7.611 | - -## Pod memory vs subscribers — peak / p50 (MiB) - -| config \ subscribers | 1 | 10 | 100 | 1000 | -|---|---|---|---|---| -| wal (cache off) | 6 / 5 | 7 / 6 | 10 / 9 | 19 / 18 | -| ursula in-memory | 14 / 13 | 14 / 13 | 14 / 13 | 14 / 13 | -| ursula disk | 15 / 14 | 14 / 14 | 14 / 13 | 14 / 13 | - -_Pod working set (cgroup `memory.current − inactive_file`) during each subscriber-count cell. **Flat across the row ⇒ a shared fan-out buffer** (one resident tail served to all subscribers); growth ⇒ per-subscriber buffering._ - - -_p50 = median; lower is better. — = not measured._ diff --git a/results/wal-1m-diag/aggregate.csv b/results/wal-1m-diag/aggregate.csv deleted file mode 100644 index da5767b..0000000 --- a/results/wal-1m-diag/aggregate.csv +++ /dev/null @@ -1,3 +0,0 @@ -mode,stream_count,pods,throughput,p50,p99,knee_pods,knee_throughput,knee_p50,knee_p99,pod_mem_mb,pod_mem_p50_mb,saturated,status,reason -stacked,500000,8,251909.06666666668,,,8,251909.06666666668,5.867,13.343,3727,3144,False,ok,ladder_exhausted -stacked,1000000,8,68271.79999999999,,,8,68271.79999999999,8.495,1102.847,6081,5774,False,ok,ladder_exhausted diff --git a/results/wal-1m-diag/aggregate.json b/results/wal-1m-diag/aggregate.json deleted file mode 100644 index 0b5d38a..0000000 --- a/results/wal-1m-diag/aggregate.json +++ /dev/null @@ -1,52 +0,0 @@ -[ - { - "mode": "stacked", - "stream_count": 500000, - "pods": 8, - "throughput": 251909.06666666668, - "p50": null, - "p99": null, - "knee_pods": 8, - "knee_throughput": 251909.06666666668, - "knee_p50": 5.867, - "knee_p99": 13.343, - "pod_mem_mb": 3727, - "pod_mem_p50_mb": 3144, - "saturated": false, - "status": "ok", - "reason": "ladder_exhausted", - "walk": [ - [ - 8, - 251909.06666666668, - 5.867, - 13.343 - ] - ] - }, - { - "mode": "stacked", - "stream_count": 1000000, - "pods": 8, - "throughput": 68271.79999999999, - "p50": null, - "p99": null, - "knee_pods": 8, - "knee_throughput": 68271.79999999999, - "knee_p50": 8.495, - "knee_p99": 1102.847, - "pod_mem_mb": 6081, - "pod_mem_p50_mb": 5774, - "saturated": false, - "status": "ok", - "reason": "ladder_exhausted", - "walk": [ - [ - 8, - 68271.79999999999, - 8.495, - 1102.847 - ] - ] - } -] \ No newline at end of file diff --git a/results/wal-1m-diag/report.md b/results/wal-1m-diag/report.md deleted file mode 100644 index a1a21e2..0000000 --- a/results/wal-1m-diag/report.md +++ /dev/null @@ -1,41 +0,0 @@ -# wal-1m-diag — write-throughput report - -## Throughput at saturation (ops/s) - -| streams | stacked | -|---|---| -| 500000 | 252k† | -| 1000000 | 68k† | - -† = not saturated (ladder exhausted) — treat as a lower bound. - -## Pod memory at saturation — peak / p50 (MiB) - -| streams | stacked | -|---|---| -| 500000 | 3727 / 3144 | -| 1000000 | 6081 / 5774 | - -_Pod working set = cgroup `memory.current − inactive_file` (anon + active page cache), sampled each second at the pinned rung. **peak** = high-water (catches bursts like an in-RAM Raft log filling); **p50** = median (what the server steadily holds resident). peak ≈ p50 ⇒ steadily resident; peak ≫ p50 ⇒ transient spikes._ - -## Latency (ms, p50 / p99) - -| streams | stacked @≤80% load | stacked @saturation | -|---|---|---| -| 500000 | 5.9 / 13.3 (252k @8p) | — | -| 1000000 | 8.5 / 1102.8 (68k @8p) | — | - -_@≤80% load = the largest ladder rung at ≤80% of peak throughput — the server's service latency with headroom. @saturation = the pinned plateau rung, where a closed-loop fleet measures its own queueing (≈ in-flight ÷ ceiling by Little's law), NOT the server's per-request cost. Compare against the unloaded single-request baseline (~1 ms for wal) before reading anything into large saturation values._ - -## Saturation walks (pods → ops/s, p50 ms) - -- **stacked 500000**: 8:252k@5.9ms (pinned 8, ladder_exhausted) -- **stacked 1000000**: 8:68k@8.5ms (pinned 8, ladder_exhausted) - -## Findings - -_TODO: written by hand on top of the generated data._ - -## Caveats - -_Single-node best-case; not 3-node Raft. Throughput is a saturation ceiling per the ladder._ diff --git a/results/wal-1m-diag/stacked/cells.json b/results/wal-1m-diag/stacked/cells.json deleted file mode 100644 index 43b37f2..0000000 --- a/results/wal-1m-diag/stacked/cells.json +++ /dev/null @@ -1,46 +0,0 @@ -{ - "cells": { - "500000": { - "stream_count": 500000, - "image_digest": "2cfd37a9a235", - "walk": [ - [ - 8, - 251909.06666666668, - 5.867, - 13.343 - ] - ], - "pinned_pods": 8, - "throughput": 251909.06666666668, - "p50": null, - "p99": null, - "pod_mem_mb": 3727, - "saturated": false, - "status": "ok", - "reason": "ladder_exhausted", - "pod_mem_p50_mb": 3144 - }, - "1000000": { - "stream_count": 1000000, - "image_digest": "2cfd37a9a235", - "walk": [ - [ - 8, - 68271.79999999999, - 8.495, - 1102.847 - ] - ], - "pinned_pods": 8, - "throughput": 68271.79999999999, - "p50": null, - "p99": null, - "pod_mem_mb": 6081, - "saturated": false, - "status": "ok", - "reason": "ladder_exhausted", - "pod_mem_p50_mb": 5774 - } - } -} \ No newline at end of file diff --git a/results/wal-batched-mw-local/aggregate.csv b/results/wal-batched-mw-local/aggregate.csv deleted file mode 100644 index 5bb1107..0000000 --- a/results/wal-batched-mw-local/aggregate.csv +++ /dev/null @@ -1,3 +0,0 @@ -mode,stream_count,pods,throughput,p50,p99,knee_pods,knee_throughput,knee_p50,knee_p99,pod_mem_mb,pod_mem_p50_mb,saturated,status,reason -baseline,50000,4,42749.3,7.447,177.535,2,27296.3,5.539,126.719,249,237,True,ok,plateau -batched-mw,50000,4,49211.6,6.783,145.279,2,39354.7,4.555,76.351,250,239,True,ok,plateau diff --git a/results/wal-batched-mw-local/aggregate.json b/results/wal-batched-mw-local/aggregate.json deleted file mode 100644 index e6ccaa6..0000000 --- a/results/wal-batched-mw-local/aggregate.json +++ /dev/null @@ -1,76 +0,0 @@ -[ - { - "mode": "baseline", - "stream_count": 50000, - "pods": 4, - "throughput": 42749.3, - "p50": 7.447, - "p99": 177.535, - "knee_pods": 2, - "knee_throughput": 27296.3, - "knee_p50": 5.539, - "knee_p99": 126.719, - "pod_mem_mb": 249, - "pod_mem_p50_mb": 237, - "saturated": true, - "status": "ok", - "reason": "plateau", - "walk": [ - [ - 2, - 27296.3, - 5.539, - 126.719 - ], - [ - 4, - 49559.2, - 6.655, - 157.439 - ], - [ - 6, - 48453.6, - 9.727, - 186.879 - ] - ] - }, - { - "mode": "batched-mw", - "stream_count": 50000, - "pods": 4, - "throughput": 49211.6, - "p50": 6.783, - "p99": 145.279, - "knee_pods": 2, - "knee_throughput": 39354.7, - "knee_p50": 4.555, - "knee_p99": 76.351, - "pod_mem_mb": 250, - "pod_mem_p50_mb": 239, - "saturated": true, - "status": "ok", - "reason": "plateau", - "walk": [ - [ - 2, - 39354.7, - 4.555, - 76.351 - ], - [ - 4, - 49414.50000000001, - 7.011, - 133.887 - ], - [ - 6, - 53682.600000000006, - 9.303, - 151.039 - ] - ] - } -] \ No newline at end of file diff --git a/results/wal-batched-mw-local/baseline/cells.json b/results/wal-batched-mw-local/baseline/cells.json deleted file mode 100644 index dc68c9a..0000000 --- a/results/wal-batched-mw-local/baseline/cells.json +++ /dev/null @@ -1,37 +0,0 @@ -{ - "cells": { - "50000": { - "stream_count": 50000, - "image_digest": "1ee33d6c2b18", - "walk": [ - [ - 2, - 27296.3, - 5.539, - 126.719 - ], - [ - 4, - 49559.2, - 6.655, - 157.439 - ], - [ - 6, - 48453.6, - 9.727, - 186.879 - ] - ], - "pinned_pods": 4, - "throughput": 42749.3, - "p50": 7.447, - "p99": 177.535, - "pod_mem_mb": 249, - "saturated": true, - "status": "ok", - "reason": "plateau", - "pod_mem_p50_mb": 237 - } - } -} \ No newline at end of file diff --git a/results/wal-batched-mw-local/batched-mw/cells.json b/results/wal-batched-mw-local/batched-mw/cells.json deleted file mode 100644 index 1f6fffc..0000000 --- a/results/wal-batched-mw-local/batched-mw/cells.json +++ /dev/null @@ -1,37 +0,0 @@ -{ - "cells": { - "50000": { - "stream_count": 50000, - "image_digest": "dbd4b6a5127d", - "walk": [ - [ - 2, - 39354.7, - 4.555, - 76.351 - ], - [ - 4, - 49414.50000000001, - 7.011, - 133.887 - ], - [ - 6, - 53682.600000000006, - 9.303, - 151.039 - ] - ], - "pinned_pods": 4, - "throughput": 49211.6, - "p50": 6.783, - "p99": 145.279, - "pod_mem_mb": 250, - "saturated": true, - "status": "ok", - "reason": "plateau", - "pod_mem_p50_mb": 239 - } - } -} \ No newline at end of file diff --git a/results/wal-batched-mw-local/report.md b/results/wal-batched-mw-local/report.md deleted file mode 100644 index 211fb6e..0000000 --- a/results/wal-batched-mw-local/report.md +++ /dev/null @@ -1,38 +0,0 @@ -# wal-batched-mw-local — write-throughput report - -## Throughput at saturation (ops/s) - -| streams | baseline | batched-mw | -|---|---|---| -| 50000 | 43k | 49k | - -† = not saturated (ladder exhausted) — treat as a lower bound. - -## Pod memory at saturation — peak / p50 (MiB) - -| streams | baseline | batched-mw | -|---|---|---| -| 50000 | 249 / 237 | 250 / 239 | - -_Pod working set = cgroup `memory.current − inactive_file` (anon + active page cache), sampled each second at the pinned rung. **peak** = high-water (catches bursts like an in-RAM Raft log filling); **p50** = median (what the server steadily holds resident). peak ≈ p50 ⇒ steadily resident; peak ≫ p50 ⇒ transient spikes._ - -## Latency (ms, p50 / p99) - -| streams | baseline @≤80% load | baseline @saturation | batched-mw @≤80% load | batched-mw @saturation | -|---|---|---|---|---| -| 50000 | 5.5 / 126.7 (27k @2p) | 7.4 / 177.5 | 4.6 / 76.4 (39k @2p) | 6.8 / 145.3 | - -_@≤80% load = the largest ladder rung at ≤80% of peak throughput — the server's service latency with headroom. @saturation = the pinned plateau rung, where a closed-loop fleet measures its own queueing (≈ in-flight ÷ ceiling by Little's law), NOT the server's per-request cost. Compare against the unloaded single-request baseline (~1 ms for wal) before reading anything into large saturation values._ - -## Saturation walks (pods → ops/s, p50 ms) - -- **baseline 50000**: 2:27k@5.5ms → 4:50k@6.7ms → 6:48k@9.7ms (pinned 4, plateau) -- **batched-mw 50000**: 2:39k@4.6ms → 4:49k@7.0ms → 6:54k@9.3ms (pinned 4, plateau) - -## Findings - -_TODO: written by hand on top of the generated data._ - -## Caveats - -_Single-node best-case; not 3-node Raft. Throughput is a saturation ceiling per the ladder._ diff --git a/results/wal-checkpoint-fix-100k/aggregate.csv b/results/wal-checkpoint-fix-100k/aggregate.csv deleted file mode 100644 index 044b153..0000000 --- a/results/wal-checkpoint-fix-100k/aggregate.csv +++ /dev/null @@ -1,3 +0,0 @@ -mode,stream_count,pods,throughput,p50,p99,knee_pods,knee_throughput,knee_p50,knee_p99,pod_mem_mb,pod_mem_p50_mb,saturated,status,reason -both,100000,4,11913.5,,,4,8395.85,3.735,1650.687,0.0,,False,ok,ladder_exhausted -syncfs,100000,4,13604.349999999999,,,2,10814.7,1.742,1461.247,549.0,,False,ok,ladder_exhausted diff --git a/results/wal-checkpoint-fix-100k/aggregate.json b/results/wal-checkpoint-fix-100k/aggregate.json deleted file mode 100644 index 71ea777..0000000 --- a/results/wal-checkpoint-fix-100k/aggregate.json +++ /dev/null @@ -1,64 +0,0 @@ -[ - { - "mode": "both", - "stream_count": 100000, - "pods": 4, - "throughput": 11913.5, - "p50": null, - "p99": null, - "knee_pods": 4, - "knee_throughput": 8395.85, - "knee_p50": 3.735, - "knee_p99": 1650.687, - "pod_mem_mb": 0.0, - "pod_mem_p50_mb": null, - "saturated": false, - "status": "ok", - "reason": "ladder_exhausted", - "walk": [ - [ - 2, - 11913.5, - 1.574, - 983.551 - ], - [ - 4, - 8395.85, - 3.735, - 1650.687 - ] - ] - }, - { - "mode": "syncfs", - "stream_count": 100000, - "pods": 4, - "throughput": 13604.349999999999, - "p50": null, - "p99": null, - "knee_pods": 2, - "knee_throughput": 10814.7, - "knee_p50": 1.742, - "knee_p99": 1461.247, - "pod_mem_mb": 549.0, - "pod_mem_p50_mb": null, - "saturated": false, - "status": "ok", - "reason": "ladder_exhausted", - "walk": [ - [ - 2, - 10814.7, - 1.742, - 1461.247 - ], - [ - 4, - 13604.349999999999, - 3.669, - 1008.127 - ] - ] - } -] \ No newline at end of file diff --git a/results/wal-checkpoint-fix-100k/both/cells.json b/results/wal-checkpoint-fix-100k/both/cells.json deleted file mode 100644 index da03a22..0000000 --- a/results/wal-checkpoint-fix-100k/both/cells.json +++ /dev/null @@ -1,30 +0,0 @@ -{ - "cells": { - "100000": { - "stream_count": 100000, - "image_digest": "b5aaa0ca74a8", - "walk": [ - [ - 2, - 11913.5, - 1.574, - 983.551 - ], - [ - 4, - 8395.85, - 3.735, - 1650.687 - ] - ], - "pinned_pods": 4, - "throughput": 8395.85, - "p50": null, - "p99": null, - "pod_mem_mb": 0.0, - "saturated": false, - "status": "ok", - "reason": "ladder_exhausted" - } - } -} \ No newline at end of file diff --git a/results/wal-checkpoint-fix-100k/report.md b/results/wal-checkpoint-fix-100k/report.md deleted file mode 100644 index 34925d9..0000000 --- a/results/wal-checkpoint-fix-100k/report.md +++ /dev/null @@ -1,44 +0,0 @@ -# wal-checkpoint-fix-100k — write-throughput report - -## Throughput at saturation (ops/s) - -| streams | syncfs | both | -|---|---|---| -| 100000 | 14k† | 12k† | - -† = not saturated (ladder exhausted) — treat as a lower bound. - -## Pod memory at saturation — peak / p50 (MiB) - -| streams | syncfs | both | -|---|---|---| -| 100000 | 549 | — | - -_Pod working set = cgroup `memory.current − inactive_file` (anon + active page cache), sampled each second at the pinned rung. **peak** = high-water (catches bursts like an in-RAM Raft log filling); **p50** = median (what the server steadily holds resident). peak ≈ p50 ⇒ steadily resident; peak ≫ p50 ⇒ transient spikes._ - -## Latency (ms, p50 / p99) - -| streams | syncfs @≤80% load | syncfs @saturation | both @≤80% load | both @saturation | -|---|---|---|---|---| -| 100000 | 1.7 / 1461.2 (11k @2p) | — | 3.7 / 1650.7 (8k @4p) | — | - -_@≤80% load = the largest ladder rung at ≤80% of peak throughput — the server's service latency with headroom. @saturation = the pinned plateau rung, where a closed-loop fleet measures its own queueing (≈ in-flight ÷ ceiling by Little's law), NOT the server's per-request cost. Compare against the unloaded single-request baseline (~1 ms for wal) before reading anything into large saturation values._ - -## Saturation walks (pods → ops/s, p50 ms) - -- **both 100000**: 2:12k@1.6ms → 4:8k@3.7ms (pinned 4, ladder_exhausted) -- **syncfs 100000**: 2:11k@1.7ms → 4:14k@3.7ms (pinned 4, ladder_exhausted) - -## Findings - -Reference (aborted wal-checkpoint-fix-ab run, same cluster/image family): **baseline@100k = 10.4k** ops/s at p2, degrading to 9.0k (p4) / 7.3k (p8) — throughput *falls* as load rises because the synchronized O(N)-fdatasync checkpoint wave stalls commits. - -- **syncfs wins**: 13.6k at p4 (+51% vs baseline's 9.0k at matched p4), and still climbing when the ladder ended — a lower bound. Shape flips from degrades-with-load to scales-with-load. -- **stagger adds nothing**: `both` ≤ syncfs alone. Consistent with mechanism: one shard's checkpoint pass (~13s at 100k) exceeds the 3s interval, so staggering cannot keep pace. Stagger prototype dropped from the branch. -- **Cliff softened, not eliminated**: ~110k @10k streams vs ~14k @100k, and p99 @≤80% load is still ~1.5s (checkpoint stalls remain). The residual cost is the O(N_touched) random writeback of per-stream files itself, which no barrier strategy fixes — that is the log-structured-store argument (issue #4695); cold-tier-as-durable is #4696. - -Merged: `--wal-checkpoint-syncfs` (branch perf/wal-checkpoint-syncfs → campaign branch), flag-gated, default off. - -## Caveats - -_Single-node best-case; not 3-node Raft. Throughput is a saturation ceiling per the ladder._ diff --git a/results/wal-checkpoint-fix-100k/syncfs/cells.json b/results/wal-checkpoint-fix-100k/syncfs/cells.json deleted file mode 100644 index 175bc10..0000000 --- a/results/wal-checkpoint-fix-100k/syncfs/cells.json +++ /dev/null @@ -1,30 +0,0 @@ -{ - "cells": { - "100000": { - "stream_count": 100000, - "image_digest": "3e4e1bf0d499", - "walk": [ - [ - 2, - 10814.7, - 1.742, - 1461.247 - ], - [ - 4, - 13604.349999999999, - 3.669, - 1008.127 - ] - ], - "pinned_pods": 4, - "throughput": 13604.349999999999, - "p50": null, - "p99": null, - "pod_mem_mb": 549.0, - "saturated": false, - "status": "ok", - "reason": "ladder_exhausted" - } - } -} \ No newline at end of file diff --git a/results/wal-checkpoint-fix-ab.run1/aggregate.csv b/results/wal-checkpoint-fix-ab.run1/aggregate.csv deleted file mode 100644 index c5d3929..0000000 --- a/results/wal-checkpoint-fix-ab.run1/aggregate.csv +++ /dev/null @@ -1,9 +0,0 @@ -mode,stream_count,pods,throughput,p50,p99,knee_pods,knee_throughput,knee_p50,knee_p99,pod_mem_mb,pod_mem_p50_mb,saturated,status,reason -baseline,10000,12,111007.09999999999,,,2,106165.5,2.673,24.671,0.0,,False,ok,ladder_exhausted -both,10000,,292450.15,,,8,0.0,0.0,0.0,,,False,error,creation_choke -stagger,10000,2,107514.0,3.271,21.455,2,108458.35,3.329,22.031,0.0,,True,ok,plateau -syncfs,10000,2,285084.0,1.496,8.087,2,299792.4,1.477,7.067,0.0,,True,ok,plateau -baseline,100000,2,10901.1,36.255,215.807,8,7258.349999999999,282.367,654.335,0.0,,True,ok,plateau -both,100000,2,11012.0,1.752,1217.535,8,8989.0,187.007,1298.431,0.0,,True,ok,plateau -stagger,100000,2,11694.9,37.279,363.519,4,8609.6,104.511,722.943,0.0,,True,ok,plateau -syncfs,100000,2,15408.9,1.588,1419.263,4,9134.4,4.727,1599.487,485,434,True,ok,plateau diff --git a/results/wal-checkpoint-fix-ab.run1/aggregate.json b/results/wal-checkpoint-fix-ab.run1/aggregate.json deleted file mode 100644 index 278514a..0000000 --- a/results/wal-checkpoint-fix-ab.run1/aggregate.json +++ /dev/null @@ -1,304 +0,0 @@ -[ - { - "mode": "baseline", - "stream_count": 10000, - "pods": 12, - "throughput": 111007.09999999999, - "p50": null, - "p99": null, - "knee_pods": 2, - "knee_throughput": 106165.5, - "knee_p50": 2.673, - "knee_p99": 24.671, - "pod_mem_mb": 0.0, - "pod_mem_p50_mb": null, - "saturated": false, - "status": "ok", - "reason": "ladder_exhausted", - "walk": [ - [ - 2, - 106165.5, - 2.673, - 24.671 - ], - [ - 4, - 96072.65, - 9.575, - 34.303 - ], - [ - 8, - 111007.09999999999, - 17.295, - 52.703 - ], - [ - 12, - 95111.55000000002, - 30.975, - 80.895 - ] - ] - }, - { - "mode": "both", - "stream_count": 10000, - "pods": null, - "throughput": 292450.15, - "p50": null, - "p99": null, - "knee_pods": 8, - "knee_throughput": 0.0, - "knee_p50": 0.0, - "knee_p99": 0.0, - "pod_mem_mb": null, - "pod_mem_p50_mb": null, - "saturated": false, - "status": "error", - "reason": "creation_choke", - "walk": [ - [ - 2, - 288573.2, - 1.468, - 8.279 - ], - [ - 4, - 292450.15, - 3.109, - 11.903 - ], - [ - 8, - 0.0, - 0.0, - 0.0 - ] - ] - }, - { - "mode": "stagger", - "stream_count": 10000, - "pods": 2, - "throughput": 107514.0, - "p50": 3.271, - "p99": 21.455, - "knee_pods": 2, - "knee_throughput": 108458.35, - "knee_p50": 3.329, - "knee_p99": 22.031, - "pod_mem_mb": 0.0, - "pod_mem_p50_mb": null, - "saturated": true, - "status": "ok", - "reason": "plateau", - "walk": [ - [ - 2, - 108458.35, - 3.329, - 22.031 - ], - [ - 4, - 100049.5, - 8.115, - 34.975 - ], - [ - 8, - 98728.04999999999, - 19.663, - 52.607 - ] - ] - }, - { - "mode": "syncfs", - "stream_count": 10000, - "pods": 2, - "throughput": 285084.0, - "p50": 1.496, - "p99": 8.087, - "knee_pods": 2, - "knee_throughput": 299792.4, - "knee_p50": 1.477, - "knee_p99": 7.067, - "pod_mem_mb": 0.0, - "pod_mem_p50_mb": null, - "saturated": true, - "status": "ok", - "reason": "plateau", - "walk": [ - [ - 2, - 299792.4, - 1.477, - 7.067 - ], - [ - 4, - 294786.80000000005, - 3.145, - 9.911 - ], - [ - 8, - 280042.3, - 6.499, - 28.127 - ] - ] - }, - { - "mode": "baseline", - "stream_count": 100000, - "pods": 2, - "throughput": 10901.1, - "p50": 36.255, - "p99": 215.807, - "knee_pods": 8, - "knee_throughput": 7258.349999999999, - "knee_p50": 282.367, - "knee_p99": 654.335, - "pod_mem_mb": 0.0, - "pod_mem_p50_mb": null, - "saturated": true, - "status": "ok", - "reason": "plateau", - "walk": [ - [ - 2, - 10405.25, - 38.335, - 212.991 - ], - [ - 4, - 8980.95, - 99.263, - 684.543 - ], - [ - 8, - 7258.349999999999, - 282.367, - 654.335 - ] - ] - }, - { - "mode": "both", - "stream_count": 100000, - "pods": 2, - "throughput": 11012.0, - "p50": 1.752, - "p99": 1217.535, - "knee_pods": 8, - "knee_throughput": 8989.0, - "knee_p50": 187.007, - "knee_p99": 1298.431, - "pod_mem_mb": 0.0, - "pod_mem_p50_mb": null, - "saturated": true, - "status": "ok", - "reason": "plateau", - "walk": [ - [ - 2, - 14105.35, - 1.621, - 1198.079 - ], - [ - 4, - 11892.1, - 3.803, - 1565.695 - ], - [ - 8, - 8989.0, - 187.007, - 1298.431 - ] - ] - }, - { - "mode": "stagger", - "stream_count": 100000, - "pods": 2, - "throughput": 11694.9, - "p50": 37.279, - "p99": 363.519, - "knee_pods": 4, - "knee_throughput": 8609.6, - "knee_p50": 104.511, - "knee_p99": 722.943, - "pod_mem_mb": 0.0, - "pod_mem_p50_mb": null, - "saturated": true, - "status": "ok", - "reason": "plateau", - "walk": [ - [ - 2, - 11781.900000000001, - 30.703, - 445.695 - ], - [ - 4, - 8609.6, - 104.511, - 722.943 - ], - [ - 8, - 6816.599999999999, - 267.775, - 1178.623 - ] - ] - }, - { - "mode": "syncfs", - "stream_count": 100000, - "pods": 2, - "throughput": 15408.9, - "p50": 1.588, - "p99": 1419.263, - "knee_pods": 4, - "knee_throughput": 9134.4, - "knee_p50": 4.727, - "knee_p99": 1599.487, - "pod_mem_mb": 485, - "pod_mem_p50_mb": 434, - "saturated": true, - "status": "ok", - "reason": "plateau", - "walk": [ - [ - 2, - 18637.4, - 1.588, - 1301.503 - ], - [ - 4, - 9134.4, - 4.727, - 1599.487 - ], - [ - 8, - 7473.8, - 242.047, - 986.623 - ] - ] - } -] \ No newline at end of file diff --git a/results/wal-checkpoint-fix-ab.run1/baseline/cells.json b/results/wal-checkpoint-fix-ab.run1/baseline/cells.json deleted file mode 100644 index 8cf77ff..0000000 --- a/results/wal-checkpoint-fix-ab.run1/baseline/cells.json +++ /dev/null @@ -1,74 +0,0 @@ -{ - "cells": { - "10000": { - "stream_count": 10000, - "image_digest": "e4ae264d73e0", - "walk": [ - [ - 2, - 106165.5, - 2.673, - 24.671 - ], - [ - 4, - 96072.65, - 9.575, - 34.303 - ], - [ - 8, - 111007.09999999999, - 17.295, - 52.703 - ], - [ - 12, - 95111.55000000002, - 30.975, - 80.895 - ] - ], - "pinned_pods": 12, - "throughput": 95111.55000000002, - "p50": null, - "p99": null, - "pod_mem_mb": 0.0, - "saturated": false, - "status": "ok", - "reason": "ladder_exhausted" - }, - "100000": { - "stream_count": 100000, - "image_digest": "e4ae264d73e0", - "walk": [ - [ - 2, - 10405.25, - 38.335, - 212.991 - ], - [ - 4, - 8980.95, - 99.263, - 684.543 - ], - [ - 8, - 7258.349999999999, - 282.367, - 654.335 - ] - ], - "pinned_pods": 2, - "throughput": 10901.1, - "p50": 36.255, - "p99": 215.807, - "pod_mem_mb": 0.0, - "saturated": true, - "status": "ok", - "reason": "plateau" - } - } -} \ No newline at end of file diff --git a/results/wal-checkpoint-fix-ab.run1/both/cells.json b/results/wal-checkpoint-fix-ab.run1/both/cells.json deleted file mode 100644 index 683764b..0000000 --- a/results/wal-checkpoint-fix-ab.run1/both/cells.json +++ /dev/null @@ -1,68 +0,0 @@ -{ - "cells": { - "10000": { - "stream_count": 10000, - "image_digest": "b5aaa0ca74a8", - "walk": [ - [ - 2, - 288573.2, - 1.468, - 8.279 - ], - [ - 4, - 292450.15, - 3.109, - 11.903 - ], - [ - 8, - 0.0, - 0.0, - 0.0 - ] - ], - "pinned_pods": null, - "throughput": 0.0, - "p50": null, - "p99": null, - "pod_mem_mb": null, - "saturated": false, - "status": "error", - "reason": "creation_choke" - }, - "100000": { - "stream_count": 100000, - "image_digest": "b5aaa0ca74a8", - "walk": [ - [ - 2, - 14105.35, - 1.621, - 1198.079 - ], - [ - 4, - 11892.1, - 3.803, - 1565.695 - ], - [ - 8, - 8989.0, - 187.007, - 1298.431 - ] - ], - "pinned_pods": 2, - "throughput": 11012.0, - "p50": 1.752, - "p99": 1217.535, - "pod_mem_mb": 0.0, - "saturated": true, - "status": "ok", - "reason": "plateau" - } - } -} \ No newline at end of file diff --git a/results/wal-checkpoint-fix-ab.run1/report.md b/results/wal-checkpoint-fix-ab.run1/report.md deleted file mode 100644 index c7b7b39..0000000 --- a/results/wal-checkpoint-fix-ab.run1/report.md +++ /dev/null @@ -1,47 +0,0 @@ -# wal-checkpoint-fix-ab — write-throughput report - -## Throughput at saturation (ops/s) - -| streams | baseline | stagger | syncfs | both | -|---|---|---|---|---| -| 10000 | 111k† | 108k | 285k | ERROR (creation_choke) | -| 100000 | 11k | 12k | 15k | 11k | - -† = not saturated (ladder exhausted) — treat as a lower bound. - -## Pod memory at saturation — peak / p50 (MiB) - -| streams | baseline | stagger | syncfs | both | -|---|---|---|---|---| -| 10000 | — | — | — | — | -| 100000 | — | — | 485 / 434 | — | - -_Pod working set = cgroup `memory.current − inactive_file` (anon + active page cache), sampled each second at the pinned rung. **peak** = high-water (catches bursts like an in-RAM Raft log filling); **p50** = median (what the server steadily holds resident). peak ≈ p50 ⇒ steadily resident; peak ≫ p50 ⇒ transient spikes._ - -## Latency (ms, p50 / p99) - -| streams | baseline @≤80% load | baseline @saturation | stagger @≤80% load | stagger @saturation | syncfs @≤80% load | syncfs @saturation | both @≤80% load | both @saturation | -|---|---|---|---|---|---|---|---|---| -| 10000 | 2.7 / 24.7 (106k @2p) | — | 3.3 / 22.0 (108k @2p) | 3.3 / 21.5 | 1.5 / 7.1 (300k @2p) | 1.5 / 8.1 | 0.0 / 0.0 (0k @8p) | — | -| 100000 | 282.4 / 654.3 (7k @8p) | 36.3 / 215.8 | 104.5 / 722.9 (9k @4p) | 37.3 / 363.5 | 4.7 / 1599.5 (9k @4p) | 1.6 / 1419.3 | 187.0 / 1298.4 (9k @8p) | 1.8 / 1217.5 | - -_@≤80% load = the largest ladder rung at ≤80% of peak throughput — the server's service latency with headroom. @saturation = the pinned plateau rung, where a closed-loop fleet measures its own queueing (≈ in-flight ÷ ceiling by Little's law), NOT the server's per-request cost. Compare against the unloaded single-request baseline (~1 ms for wal) before reading anything into large saturation values._ - -## Saturation walks (pods → ops/s, p50 ms) - -- **baseline 10000**: 2:106k@2.7ms → 4:96k@9.6ms → 8:111k@17.3ms → 12:95k@31.0ms (pinned 12, ladder_exhausted) -- **both 10000**: 2:289k@1.5ms → 4:292k@3.1ms → 8:0k@0.0ms (pinned None, creation_choke) -- **stagger 10000**: 2:108k@3.3ms → 4:100k@8.1ms → 8:99k@19.7ms (pinned 2, plateau) -- **syncfs 10000**: 2:300k@1.5ms → 4:295k@3.1ms → 8:280k@6.5ms (pinned 2, plateau) -- **baseline 100000**: 2:10k@38.3ms → 4:9k@99.3ms → 8:7k@282.4ms (pinned 2, plateau) -- **both 100000**: 2:14k@1.6ms → 4:12k@3.8ms → 8:9k@187.0ms (pinned 2, plateau) -- **stagger 100000**: 2:12k@30.7ms → 4:9k@104.5ms → 8:7k@267.8ms (pinned 2, plateau) -- **syncfs 100000**: 2:19k@1.6ms → 4:9k@4.7ms → 8:7k@242.0ms (pinned 2, plateau) - -## Findings - -_TODO: written by hand on top of the generated data._ - -## Caveats - -_Single-node best-case; not 3-node Raft. Throughput is a saturation ceiling per the ladder._ diff --git a/results/wal-checkpoint-fix-ab.run1/stagger/cells.json b/results/wal-checkpoint-fix-ab.run1/stagger/cells.json deleted file mode 100644 index 95a4c03..0000000 --- a/results/wal-checkpoint-fix-ab.run1/stagger/cells.json +++ /dev/null @@ -1,68 +0,0 @@ -{ - "cells": { - "10000": { - "stream_count": 10000, - "image_digest": "2ebed436ee56", - "walk": [ - [ - 2, - 108458.35, - 3.329, - 22.031 - ], - [ - 4, - 100049.5, - 8.115, - 34.975 - ], - [ - 8, - 98728.04999999999, - 19.663, - 52.607 - ] - ], - "pinned_pods": 2, - "throughput": 107514.0, - "p50": 3.271, - "p99": 21.455, - "pod_mem_mb": 0.0, - "saturated": true, - "status": "ok", - "reason": "plateau" - }, - "100000": { - "stream_count": 100000, - "image_digest": "2ebed436ee56", - "walk": [ - [ - 2, - 11781.900000000001, - 30.703, - 445.695 - ], - [ - 4, - 8609.6, - 104.511, - 722.943 - ], - [ - 8, - 6816.599999999999, - 267.775, - 1178.623 - ] - ], - "pinned_pods": 2, - "throughput": 11694.9, - "p50": 37.279, - "p99": 363.519, - "pod_mem_mb": 0.0, - "saturated": true, - "status": "ok", - "reason": "plateau" - } - } -} \ No newline at end of file diff --git a/results/wal-checkpoint-fix-ab.run1/syncfs/cells.json b/results/wal-checkpoint-fix-ab.run1/syncfs/cells.json deleted file mode 100644 index ba30868..0000000 --- a/results/wal-checkpoint-fix-ab.run1/syncfs/cells.json +++ /dev/null @@ -1,69 +0,0 @@ -{ - "cells": { - "10000": { - "stream_count": 10000, - "image_digest": "3e4e1bf0d499", - "walk": [ - [ - 2, - 299792.4, - 1.477, - 7.067 - ], - [ - 4, - 294786.80000000005, - 3.145, - 9.911 - ], - [ - 8, - 280042.3, - 6.499, - 28.127 - ] - ], - "pinned_pods": 2, - "throughput": 285084.0, - "p50": 1.496, - "p99": 8.087, - "pod_mem_mb": 0.0, - "saturated": true, - "status": "ok", - "reason": "plateau" - }, - "100000": { - "stream_count": 100000, - "image_digest": "3e4e1bf0d499", - "walk": [ - [ - 2, - 18637.4, - 1.588, - 1301.503 - ], - [ - 4, - 9134.4, - 4.727, - 1599.487 - ], - [ - 8, - 7473.8, - 242.047, - 986.623 - ] - ], - "pinned_pods": 2, - "throughput": 15408.9, - "p50": 1.588, - "p99": 1419.263, - "pod_mem_mb": 485, - "saturated": true, - "status": "ok", - "reason": "plateau", - "pod_mem_p50_mb": 434 - } - } -} \ No newline at end of file diff --git a/results/wal-checkpoint-fix-ab/baseline/cells.json b/results/wal-checkpoint-fix-ab/baseline/cells.json deleted file mode 100644 index 382cb81..0000000 --- a/results/wal-checkpoint-fix-ab/baseline/cells.json +++ /dev/null @@ -1,30 +0,0 @@ -{ - "cells": { - "10000": { - "stream_count": 10000, - "image_digest": "ed0fed8321da", - "walk": [ - [ - 2, - 112568.2, - 3.069, - 19.983 - ], - [ - 4, - 0.0, - 0.0, - 0.0 - ] - ], - "pinned_pods": null, - "throughput": 0.0, - "p50": null, - "p99": null, - "pod_mem_mb": null, - "saturated": false, - "status": "error", - "reason": "creation_choke" - } - } -} \ No newline at end of file diff --git a/results/wal-cpubind/aggregate.csv b/results/wal-cpubind/aggregate.csv deleted file mode 100644 index a0ec00f..0000000 --- a/results/wal-cpubind/aggregate.csv +++ /dev/null @@ -1,3 +0,0 @@ -mode,stream_count,pods,throughput,p50,p99,knee_pods,knee_throughput,knee_p50,knee_p99,pod_mem_mb,pod_mem_p50_mb,saturated,status,reason -bound-3s,10000,8,369978.75,,,4,369978.75,2.525,5.227,249,198,False,ok,ladder_exhausted -bound-3s,100000,2,328045.0,1.278,2.837,2,324751.9,1.274,2.835,694,596,True,ok,plateau diff --git a/results/wal-cpubind/aggregate.json b/results/wal-cpubind/aggregate.json deleted file mode 100644 index 9ed2599..0000000 --- a/results/wal-cpubind/aggregate.json +++ /dev/null @@ -1,70 +0,0 @@ -[ - { - "mode": "bound-3s", - "stream_count": 10000, - "pods": 8, - "throughput": 369978.75, - "p50": null, - "p99": null, - "knee_pods": 4, - "knee_throughput": 369978.75, - "knee_p50": 2.525, - "knee_p99": 5.227, - "pod_mem_mb": 249, - "pod_mem_p50_mb": 198, - "saturated": false, - "status": "ok", - "reason": "ladder_exhausted", - "walk": [ - [ - 4, - 369978.75, - 2.525, - 5.227 - ], - [ - 8, - 356028.55000000005, - 5.387, - 20.399 - ] - ] - }, - { - "mode": "bound-3s", - "stream_count": 100000, - "pods": 2, - "throughput": 328045.0, - "p50": 1.278, - "p99": 2.837, - "knee_pods": 2, - "knee_throughput": 324751.9, - "knee_p50": 1.274, - "knee_p99": 2.835, - "pod_mem_mb": 694, - "pod_mem_p50_mb": 596, - "saturated": true, - "status": "ok", - "reason": "plateau", - "walk": [ - [ - 2, - 324751.9, - 1.274, - 2.835 - ], - [ - 4, - 342369.39999999997, - 2.573, - 4.667 - ], - [ - 8, - 317598.9, - 5.539, - 9.287 - ] - ] - } -] \ No newline at end of file diff --git a/results/wal-cpubind/bound-3s/cells.json b/results/wal-cpubind/bound-3s/cells.json deleted file mode 100644 index 001e27f..0000000 --- a/results/wal-cpubind/bound-3s/cells.json +++ /dev/null @@ -1,64 +0,0 @@ -{ - "cells": { - "10000": { - "stream_count": 10000, - "image_digest": "1bd30dc4c80f", - "walk": [ - [ - 4, - 369978.75, - 2.525, - 5.227 - ], - [ - 8, - 356028.55000000005, - 5.387, - 20.399 - ] - ], - "pinned_pods": 8, - "throughput": 356028.55000000005, - "p50": null, - "p99": null, - "pod_mem_mb": 249, - "saturated": false, - "status": "ok", - "reason": "ladder_exhausted", - "pod_mem_p50_mb": 198 - }, - "100000": { - "stream_count": 100000, - "image_digest": "1bd30dc4c80f", - "walk": [ - [ - 2, - 324751.9, - 1.274, - 2.835 - ], - [ - 4, - 342369.39999999997, - 2.573, - 4.667 - ], - [ - 8, - 317598.9, - 5.539, - 9.287 - ] - ], - "pinned_pods": 2, - "throughput": 328045.0, - "p50": 1.278, - "p99": 2.837, - "pod_mem_mb": 694, - "saturated": true, - "status": "ok", - "reason": "plateau", - "pod_mem_p50_mb": 596 - } - } -} \ No newline at end of file diff --git a/results/wal-cpubind/report.md b/results/wal-cpubind/report.md deleted file mode 100644 index 32f5ba6..0000000 --- a/results/wal-cpubind/report.md +++ /dev/null @@ -1,41 +0,0 @@ -# wal-cpubind — write-throughput report - -## Throughput at saturation (ops/s) - -| streams | bound-3s | -|---|---| -| 10000 | 370k† | -| 100000 | 328k | - -† = not saturated (ladder exhausted) — treat as a lower bound. - -## Pod memory at saturation — peak / p50 (MiB) - -| streams | bound-3s | -|---|---| -| 10000 | 249 / 198 | -| 100000 | 694 / 596 | - -_Pod working set = cgroup `memory.current − inactive_file` (anon + active page cache), sampled each second at the pinned rung. **peak** = high-water (catches bursts like an in-RAM Raft log filling); **p50** = median (what the server steadily holds resident). peak ≈ p50 ⇒ steadily resident; peak ≫ p50 ⇒ transient spikes._ - -## Latency (ms, p50 / p99) - -| streams | bound-3s @≤80% load | bound-3s @saturation | -|---|---|---| -| 10000 | 2.5 / 5.2 (370k @4p) | — | -| 100000 | 1.3 / 2.8 (325k @2p) | 1.3 / 2.8 | - -_@≤80% load = the largest ladder rung at ≤80% of peak throughput — the server's service latency with headroom. @saturation = the pinned plateau rung, where a closed-loop fleet measures its own queueing (≈ in-flight ÷ ceiling by Little's law), NOT the server's per-request cost. Compare against the unloaded single-request baseline (~1 ms for wal) before reading anything into large saturation values._ - -## Saturation walks (pods → ops/s, p50 ms) - -- **bound-3s 10000**: 4:370k@2.5ms → 8:356k@5.4ms (pinned 8, ladder_exhausted) -- **bound-3s 100000**: 2:325k@1.3ms → 4:342k@2.6ms → 8:318k@5.5ms (pinned 2, plateau) - -## Findings - -Exclusive pinned cores (STATIC_CPU=1 + GUARANTEED=1, cpuManagerPolicy=static, integer 8-CPU Guaranteed pod) = 356.0k @10k / 328.0k @100k vs 286.3k/271.6k on shared cores (wal-splitlane ref-3s, same layout/image/args): +24% / +21%. With wal no longer fsync-bound, CPU binding is a real lever. Caveat: cross-cluster comparison (bench-cpubind vs bench-multilane), same instance type/zone. - -## Caveats - -_Single-node best-case; not 3-node Raft. Throughput is a saturation ceiling per the ladder._ diff --git a/results/wal-decomp-lane0/aggregate.csv b/results/wal-decomp-lane0/aggregate.csv deleted file mode 100644 index bd815d2..0000000 --- a/results/wal-decomp-lane0/aggregate.csv +++ /dev/null @@ -1,9 +0,0 @@ -mode,stream_count,pods,throughput,p50,p99,knee_pods,knee_throughput,knee_p50,knee_p99,pod_mem_mb,pod_mem_p50_mb,saturated,status,reason -ckpt-off,10000,8,85294.9,,,4,76193.25,11.431,15.343,176,146,False,ok,ladder_exhausted -memory,10000,8,542955.0,,,4,542955.0,1.75,4.063,274,190,False,ok,ladder_exhausted -nofsync,10000,8,259910.85,,,4,259910.85,3.123,9.071,222,177,False,ok,ladder_exhausted -ref-3s,10000,8,55409.45,,,4,53065.649999999994,13.863,58.239,162,144,False,ok,ladder_exhausted -ckpt-off,100000,4,66064.6,,,2,66064.6,6.847,8.807,586,506,False,ok,ladder_exhausted -memory,100000,4,511486.10000000003,,,2,407791.45,1.174,2.361,659,597,False,ok,ladder_exhausted -nofsync,100000,4,252852.65,,,2,252852.65,1.452,7.431,644,596,False,ok,ladder_exhausted -ref-3s,100000,4,46835.899999999994,,,2,46835.899999999994,8.935,36.575,523,509,False,ok,ladder_exhausted diff --git a/results/wal-decomp-lane0/aggregate.json b/results/wal-decomp-lane0/aggregate.json deleted file mode 100644 index f69776a..0000000 --- a/results/wal-decomp-lane0/aggregate.json +++ /dev/null @@ -1,250 +0,0 @@ -[ - { - "mode": "ckpt-off", - "stream_count": 10000, - "pods": 8, - "throughput": 85294.9, - "p50": null, - "p99": null, - "knee_pods": 4, - "knee_throughput": 76193.25, - "knee_p50": 11.431, - "knee_p99": 15.343, - "pod_mem_mb": 176, - "pod_mem_p50_mb": 146, - "saturated": false, - "status": "ok", - "reason": "ladder_exhausted", - "walk": [ - [ - 4, - 76193.25, - 11.431, - 15.343 - ], - [ - 8, - 85294.9, - 22.527, - 31.711 - ] - ] - }, - { - "mode": "memory", - "stream_count": 10000, - "pods": 8, - "throughput": 542955.0, - "p50": null, - "p99": null, - "knee_pods": 4, - "knee_throughput": 542955.0, - "knee_p50": 1.75, - "knee_p99": 4.063, - "pod_mem_mb": 274, - "pod_mem_p50_mb": 190, - "saturated": false, - "status": "ok", - "reason": "ladder_exhausted", - "walk": [ - [ - 4, - 542955.0, - 1.75, - 4.063 - ], - [ - 8, - 541637.95, - 2.973, - 9.799 - ] - ] - }, - { - "mode": "nofsync", - "stream_count": 10000, - "pods": 8, - "throughput": 259910.85, - "p50": null, - "p99": null, - "knee_pods": 4, - "knee_throughput": 259910.85, - "knee_p50": 3.123, - "knee_p99": 9.071, - "pod_mem_mb": 222, - "pod_mem_p50_mb": 177, - "saturated": false, - "status": "ok", - "reason": "ladder_exhausted", - "walk": [ - [ - 4, - 259910.85, - 3.123, - 9.071 - ], - [ - 8, - 259645.05, - 6.439, - 13.807 - ] - ] - }, - { - "mode": "ref-3s", - "stream_count": 10000, - "pods": 8, - "throughput": 55409.45, - "p50": null, - "p99": null, - "knee_pods": 4, - "knee_throughput": 53065.649999999994, - "knee_p50": 13.863, - "knee_p99": 58.239, - "pod_mem_mb": 162, - "pod_mem_p50_mb": 144, - "saturated": false, - "status": "ok", - "reason": "ladder_exhausted", - "walk": [ - [ - 4, - 53065.649999999994, - 13.863, - 58.239 - ], - [ - 8, - 55409.45, - 36.799, - 67.327 - ] - ] - }, - { - "mode": "ckpt-off", - "stream_count": 100000, - "pods": 4, - "throughput": 66064.6, - "p50": null, - "p99": null, - "knee_pods": 2, - "knee_throughput": 66064.6, - "knee_p50": 6.847, - "knee_p99": 8.807, - "pod_mem_mb": 586, - "pod_mem_p50_mb": 506, - "saturated": false, - "status": "ok", - "reason": "ladder_exhausted", - "walk": [ - [ - 2, - 66064.6, - 6.847, - 8.807 - ], - [ - 4, - 64858.6, - 13.847, - 142.079 - ] - ] - }, - { - "mode": "memory", - "stream_count": 100000, - "pods": 4, - "throughput": 511486.10000000003, - "p50": null, - "p99": null, - "knee_pods": 2, - "knee_throughput": 407791.45, - "knee_p50": 1.174, - "knee_p99": 2.361, - "pod_mem_mb": 659, - "pod_mem_p50_mb": 597, - "saturated": false, - "status": "ok", - "reason": "ladder_exhausted", - "walk": [ - [ - 2, - 407791.45, - 1.174, - 2.361 - ], - [ - 4, - 511486.10000000003, - 1.809, - 4.227 - ] - ] - }, - { - "mode": "nofsync", - "stream_count": 100000, - "pods": 4, - "throughput": 252852.65, - "p50": null, - "p99": null, - "knee_pods": 2, - "knee_throughput": 252852.65, - "knee_p50": 1.452, - "knee_p99": 7.431, - "pod_mem_mb": 644, - "pod_mem_p50_mb": 596, - "saturated": false, - "status": "ok", - "reason": "ladder_exhausted", - "walk": [ - [ - 2, - 252852.65, - 1.452, - 7.431 - ], - [ - 4, - 252641.40000000002, - 3.177, - 9.135 - ] - ] - }, - { - "mode": "ref-3s", - "stream_count": 100000, - "pods": 4, - "throughput": 46835.899999999994, - "p50": null, - "p99": null, - "knee_pods": 2, - "knee_throughput": 46835.899999999994, - "knee_p50": 8.935, - "knee_p99": 36.575, - "pod_mem_mb": 523, - "pod_mem_p50_mb": 509, - "saturated": false, - "status": "ok", - "reason": "ladder_exhausted", - "walk": [ - [ - 2, - 46835.899999999994, - 8.935, - 36.575 - ], - [ - 4, - 46132.9, - 21.487, - 71.935 - ] - ] - } -] \ No newline at end of file diff --git a/results/wal-decomp-lane0/ckpt-off/cells.json b/results/wal-decomp-lane0/ckpt-off/cells.json deleted file mode 100644 index f800fd7..0000000 --- a/results/wal-decomp-lane0/ckpt-off/cells.json +++ /dev/null @@ -1,58 +0,0 @@ -{ - "cells": { - "10000": { - "stream_count": 10000, - "image_digest": "500e885c4956", - "walk": [ - [ - 4, - 76193.25, - 11.431, - 15.343 - ], - [ - 8, - 85294.9, - 22.527, - 31.711 - ] - ], - "pinned_pods": 8, - "throughput": 85294.9, - "p50": null, - "p99": null, - "pod_mem_mb": 176, - "saturated": false, - "status": "ok", - "reason": "ladder_exhausted", - "pod_mem_p50_mb": 146 - }, - "100000": { - "stream_count": 100000, - "image_digest": "500e885c4956", - "walk": [ - [ - 2, - 66064.6, - 6.847, - 8.807 - ], - [ - 4, - 64858.6, - 13.847, - 142.079 - ] - ], - "pinned_pods": 4, - "throughput": 64858.6, - "p50": null, - "p99": null, - "pod_mem_mb": 586, - "saturated": false, - "status": "ok", - "reason": "ladder_exhausted", - "pod_mem_p50_mb": 506 - } - } -} \ No newline at end of file diff --git a/results/wal-decomp-lane0/memory/cells.json b/results/wal-decomp-lane0/memory/cells.json deleted file mode 100644 index 3db70b6..0000000 --- a/results/wal-decomp-lane0/memory/cells.json +++ /dev/null @@ -1,58 +0,0 @@ -{ - "cells": { - "10000": { - "stream_count": 10000, - "image_digest": "ddd833e769ff", - "walk": [ - [ - 4, - 542955.0, - 1.75, - 4.063 - ], - [ - 8, - 541637.95, - 2.973, - 9.799 - ] - ], - "pinned_pods": 8, - "throughput": 541637.95, - "p50": null, - "p99": null, - "pod_mem_mb": 274, - "saturated": false, - "status": "ok", - "reason": "ladder_exhausted", - "pod_mem_p50_mb": 190 - }, - "100000": { - "stream_count": 100000, - "image_digest": "ddd833e769ff", - "walk": [ - [ - 2, - 407791.45, - 1.174, - 2.361 - ], - [ - 4, - 511486.10000000003, - 1.809, - 4.227 - ] - ], - "pinned_pods": 4, - "throughput": 511486.10000000003, - "p50": null, - "p99": null, - "pod_mem_mb": 659, - "saturated": false, - "status": "ok", - "reason": "ladder_exhausted", - "pod_mem_p50_mb": 597 - } - } -} \ No newline at end of file diff --git a/results/wal-decomp-lane0/nofsync/cells.json b/results/wal-decomp-lane0/nofsync/cells.json deleted file mode 100644 index f35ec83..0000000 --- a/results/wal-decomp-lane0/nofsync/cells.json +++ /dev/null @@ -1,58 +0,0 @@ -{ - "cells": { - "10000": { - "stream_count": 10000, - "image_digest": "f2cd2c5a0498", - "walk": [ - [ - 4, - 259910.85, - 3.123, - 9.071 - ], - [ - 8, - 259645.05, - 6.439, - 13.807 - ] - ], - "pinned_pods": 8, - "throughput": 259645.05, - "p50": null, - "p99": null, - "pod_mem_mb": 222, - "saturated": false, - "status": "ok", - "reason": "ladder_exhausted", - "pod_mem_p50_mb": 177 - }, - "100000": { - "stream_count": 100000, - "image_digest": "f2cd2c5a0498", - "walk": [ - [ - 2, - 252852.65, - 1.452, - 7.431 - ], - [ - 4, - 252641.40000000002, - 3.177, - 9.135 - ] - ], - "pinned_pods": 4, - "throughput": 252641.40000000002, - "p50": null, - "p99": null, - "pod_mem_mb": 644, - "saturated": false, - "status": "ok", - "reason": "ladder_exhausted", - "pod_mem_p50_mb": 596 - } - } -} \ No newline at end of file diff --git a/results/wal-decomp-lane0/ref-3s/cells.json b/results/wal-decomp-lane0/ref-3s/cells.json deleted file mode 100644 index 1cec0e4..0000000 --- a/results/wal-decomp-lane0/ref-3s/cells.json +++ /dev/null @@ -1,58 +0,0 @@ -{ - "cells": { - "10000": { - "stream_count": 10000, - "image_digest": "6fcaf53b737e", - "walk": [ - [ - 4, - 53065.649999999994, - 13.863, - 58.239 - ], - [ - 8, - 55409.45, - 36.799, - 67.327 - ] - ], - "pinned_pods": 8, - "throughput": 55409.45, - "p50": null, - "p99": null, - "pod_mem_mb": 162, - "saturated": false, - "status": "ok", - "reason": "ladder_exhausted", - "pod_mem_p50_mb": 144 - }, - "100000": { - "stream_count": 100000, - "image_digest": "6fcaf53b737e", - "walk": [ - [ - 2, - 46835.899999999994, - 8.935, - 36.575 - ], - [ - 4, - 46132.9, - 21.487, - 71.935 - ] - ], - "pinned_pods": 4, - "throughput": 46132.9, - "p50": null, - "p99": null, - "pod_mem_mb": 523, - "saturated": false, - "status": "ok", - "reason": "ladder_exhausted", - "pod_mem_p50_mb": 509 - } - } -} \ No newline at end of file diff --git a/results/wal-decomp-lane0/report.md b/results/wal-decomp-lane0/report.md deleted file mode 100644 index f6706b8..0000000 --- a/results/wal-decomp-lane0/report.md +++ /dev/null @@ -1,47 +0,0 @@ -# wal-decomp-lane0 — write-throughput report - -## Throughput at saturation (ops/s) - -| streams | memory | nofsync | ckpt-off | ref-3s | -|---|---|---|---|---| -| 10000 | 543k† | 260k† | 85k† | 55k† | -| 100000 | 511k† | 253k† | 66k† | 47k† | - -† = not saturated (ladder exhausted) — treat as a lower bound. - -## Pod memory at saturation — peak / p50 (MiB) - -| streams | memory | nofsync | ckpt-off | ref-3s | -|---|---|---|---|---| -| 10000 | 274 / 190 | 222 / 177 | 176 / 146 | 162 / 144 | -| 100000 | 659 / 597 | 644 / 596 | 586 / 506 | 523 / 509 | - -_Pod working set = cgroup `memory.current − inactive_file` (anon + active page cache), sampled each second at the pinned rung. **peak** = high-water (catches bursts like an in-RAM Raft log filling); **p50** = median (what the server steadily holds resident). peak ≈ p50 ⇒ steadily resident; peak ≫ p50 ⇒ transient spikes._ - -## Latency (ms, p50 / p99) - -| streams | memory @≤80% load | memory @saturation | nofsync @≤80% load | nofsync @saturation | ckpt-off @≤80% load | ckpt-off @saturation | ref-3s @≤80% load | ref-3s @saturation | -|---|---|---|---|---|---|---|---|---| -| 10000 | 1.8 / 4.1 (543k @4p) | — | 3.1 / 9.1 (260k @4p) | — | 11.4 / 15.3 (76k @4p) | — | 13.9 / 58.2 (53k @4p) | — | -| 100000 | 1.2 / 2.4 (408k @2p) | — | 1.5 / 7.4 (253k @2p) | — | 6.8 / 8.8 (66k @2p) | — | 8.9 / 36.6 (47k @2p) | — | - -_@≤80% load = the largest ladder rung at ≤80% of peak throughput — the server's service latency with headroom. @saturation = the pinned plateau rung, where a closed-loop fleet measures its own queueing (≈ in-flight ÷ ceiling by Little's law), NOT the server's per-request cost. Compare against the unloaded single-request baseline (~1 ms for wal) before reading anything into large saturation values._ - -## Saturation walks (pods → ops/s, p50 ms) - -- **ckpt-off 10000**: 4:76k@11.4ms → 8:85k@22.5ms (pinned 8, ladder_exhausted) -- **memory 10000**: 4:543k@1.8ms → 8:542k@3.0ms (pinned 8, ladder_exhausted) -- **nofsync 10000**: 4:260k@3.1ms → 8:260k@6.4ms (pinned 8, ladder_exhausted) -- **ref-3s 10000**: 4:53k@13.9ms → 8:55k@36.8ms (pinned 8, ladder_exhausted) -- **ckpt-off 100000**: 2:66k@6.8ms → 4:65k@13.8ms (pinned 4, ladder_exhausted) -- **memory 100000**: 2:408k@1.2ms → 4:511k@1.8ms (pinned 4, ladder_exhausted) -- **nofsync 100000**: 2:253k@1.5ms → 4:253k@3.2ms (pinned 4, ladder_exhausted) -- **ref-3s 100000**: 2:47k@8.9ms → 4:46k@21.5ms (pinned 4, ladder_exhausted) - -## Findings - -_TODO: written by hand on top of the generated data._ - -## Caveats - -_Single-node best-case; not 3-node Raft. Throughput is a saturation ceiling per the ladder._ diff --git a/results/wal-fanout-sweep/aggregate.csv b/results/wal-fanout-sweep/aggregate.csv deleted file mode 100644 index 49e7a4a..0000000 --- a/results/wal-fanout-sweep/aggregate.csv +++ /dev/null @@ -1,5 +0,0 @@ -mode,stream_count,pods,throughput,p50,p99,knee_pods,knee_throughput,knee_p50,knee_p99,pod_mem_mb,pod_mem_p50_mb,saturated,status,reason -s4-f1,200000,8,71098.5,27.855,59.295,2,55694.9,7.967,32.463,1059,1032,True,ok,plateau -s4-f2,200000,4,67583.9,14.495,41.951,2,58915.25,8.159,26.543,1032,1004,True,ok,plateau -s4-f4,200000,16,75057.70000000001,,,4,58659.0,16.199,37.983,1109,1049,False,ok,ladder_exhausted -s4-f8,200000,16,66705.1,,,2,45221.2,10.951,23.359,1101,1039,False,ok,ladder_exhausted diff --git a/results/wal-fanout-sweep/aggregate.json b/results/wal-fanout-sweep/aggregate.json deleted file mode 100644 index bc125ab..0000000 --- a/results/wal-fanout-sweep/aggregate.json +++ /dev/null @@ -1,192 +0,0 @@ -[ - { - "mode": "s4-f1", - "stream_count": 200000, - "pods": 8, - "throughput": 71098.5, - "p50": 27.855, - "p99": 59.295, - "knee_pods": 2, - "knee_throughput": 55694.9, - "knee_p50": 7.967, - "knee_p99": 32.463, - "pod_mem_mb": 1059, - "pod_mem_p50_mb": 1032, - "saturated": true, - "status": "ok", - "reason": "plateau", - "walk": [ - [ - 2, - 55694.9, - 7.967, - 32.463 - ], - [ - 4, - 61040.0, - 14.527, - 52.255 - ], - [ - 8, - 72946.5, - 27.343, - 57.119 - ], - [ - 12, - 71171.20000000001, - 41.375, - 99.647 - ], - [ - 16, - 75300.9, - 53.439, - 105.471 - ] - ] - }, - { - "mode": "s4-f2", - "stream_count": 200000, - "pods": 4, - "throughput": 67583.9, - "p50": 14.495, - "p99": 41.951, - "knee_pods": 2, - "knee_throughput": 58915.25, - "knee_p50": 8.159, - "knee_p99": 26.543, - "pod_mem_mb": 1032, - "pod_mem_p50_mb": 1004, - "saturated": true, - "status": "ok", - "reason": "plateau", - "walk": [ - [ - 2, - 58915.25, - 8.159, - 26.543 - ], - [ - 4, - 68520.95, - 14.431, - 41.663 - ], - [ - 8, - 68227.55, - 28.111, - 63.231 - ], - [ - 12, - 72834.0, - 41.343, - 80.255 - ] - ] - }, - { - "mode": "s4-f4", - "stream_count": 200000, - "pods": 16, - "throughput": 75057.70000000001, - "p50": null, - "p99": null, - "knee_pods": 4, - "knee_throughput": 58659.0, - "knee_p50": 16.199, - "knee_p99": 37.983, - "pod_mem_mb": 1109, - "pod_mem_p50_mb": 1049, - "saturated": false, - "status": "ok", - "reason": "ladder_exhausted", - "walk": [ - [ - 2, - 47302.15, - 9.551, - 29.807 - ], - [ - 4, - 58659.0, - 16.199, - 37.983 - ], - [ - 8, - 65971.34999999999, - 29.007, - 66.239 - ], - [ - 12, - 74076.75, - 41.503, - 73.471 - ], - [ - 16, - 75057.70000000001, - 54.527, - 99.007 - ] - ] - }, - { - "mode": "s4-f8", - "stream_count": 200000, - "pods": 16, - "throughput": 66705.1, - "p50": null, - "p99": null, - "knee_pods": 2, - "knee_throughput": 45221.2, - "knee_p50": 10.951, - "knee_p99": 23.359, - "pod_mem_mb": 1101, - "pod_mem_p50_mb": 1039, - "saturated": false, - "status": "ok", - "reason": "ladder_exhausted", - "walk": [ - [ - 2, - 45221.2, - 10.951, - 23.359 - ], - [ - 4, - 55360.25, - 17.839, - 36.191 - ], - [ - 8, - 59471.4, - 33.119, - 63.359 - ], - [ - 12, - 65443.2, - 45.599, - 81.535 - ], - [ - 16, - 66705.1, - 59.743, - 104.191 - ] - ] - } -] \ No newline at end of file diff --git a/results/wal-fanout-sweep/report.md b/results/wal-fanout-sweep/report.md deleted file mode 100644 index 36a7af4..0000000 --- a/results/wal-fanout-sweep/report.md +++ /dev/null @@ -1,40 +0,0 @@ -# wal-fanout-sweep — write-throughput report - -## Throughput at saturation (ops/s) - -| streams | s4-f1 | s4-f2 | s4-f4 | s4-f8 | -|---|---|---|---|---| -| 200000 | 71k | 68k | 75k† | 67k† | - -† = not saturated (ladder exhausted) — treat as a lower bound. - -## Pod memory at saturation — peak / p50 (MiB) - -| streams | s4-f1 | s4-f2 | s4-f4 | s4-f8 | -|---|---|---|---|---| -| 200000 | 1059 / 1032 | 1032 / 1004 | 1109 / 1049 | 1101 / 1039 | - -_Pod working set = cgroup `memory.current − inactive_file` (anon + active page cache), sampled each second at the pinned rung. **peak** = high-water (catches bursts like an in-RAM Raft log filling); **p50** = median (what the server steadily holds resident). peak ≈ p50 ⇒ steadily resident; peak ≫ p50 ⇒ transient spikes._ - -## Latency (ms, p50 / p99) - -| streams | s4-f1 @≤80% load | s4-f1 @saturation | s4-f2 @≤80% load | s4-f2 @saturation | s4-f4 @≤80% load | s4-f4 @saturation | s4-f8 @≤80% load | s4-f8 @saturation | -|---|---|---|---|---|---|---|---|---| -| 200000 | 8.0 / 32.5 (56k @2p) | 27.9 / 59.3 | 8.2 / 26.5 (59k @2p) | 14.5 / 42.0 | 16.2 / 38.0 (59k @4p) | — | 11.0 / 23.4 (45k @2p) | — | - -_@≤80% load = the largest ladder rung at ≤80% of peak throughput — the server's service latency with headroom. @saturation = the pinned plateau rung, where a closed-loop fleet measures its own queueing (≈ in-flight ÷ ceiling by Little's law), NOT the server's per-request cost. Compare against the unloaded single-request baseline (~1 ms for wal) before reading anything into large saturation values._ - -## Saturation walks (pods → ops/s, p50 ms) - -- **s4-f1 200000**: 2:56k@8.0ms → 4:61k@14.5ms → 8:73k@27.3ms → 12:71k@41.4ms → 16:75k@53.4ms (pinned 8, plateau) -- **s4-f2 200000**: 2:59k@8.2ms → 4:69k@14.4ms → 8:68k@28.1ms → 12:73k@41.3ms (pinned 4, plateau) -- **s4-f4 200000**: 2:47k@9.6ms → 4:59k@16.2ms → 8:66k@29.0ms → 12:74k@41.5ms → 16:75k@54.5ms (pinned 16, ladder_exhausted) -- **s4-f8 200000**: 2:45k@11.0ms → 4:55k@17.8ms → 8:59k@33.1ms → 12:65k@45.6ms → 16:67k@59.7ms (pinned 16, ladder_exhausted) - -## Findings - -_TODO: written by hand on top of the generated data._ - -## Caveats - -_Single-node best-case; not 3-node Raft. Throughput is a saturation ceiling per the ladder._ diff --git a/results/wal-fanout-sweep/s4-f1/cells.json b/results/wal-fanout-sweep/s4-f1/cells.json deleted file mode 100644 index c8af02d..0000000 --- a/results/wal-fanout-sweep/s4-f1/cells.json +++ /dev/null @@ -1,49 +0,0 @@ -{ - "cells": { - "200000": { - "stream_count": 200000, - "image_digest": "f5692b252ebb", - "walk": [ - [ - 2, - 55694.9, - 7.967, - 32.463 - ], - [ - 4, - 61040.0, - 14.527, - 52.255 - ], - [ - 8, - 72946.5, - 27.343, - 57.119 - ], - [ - 12, - 71171.20000000001, - 41.375, - 99.647 - ], - [ - 16, - 75300.9, - 53.439, - 105.471 - ] - ], - "pinned_pods": 8, - "throughput": 71098.5, - "p50": 27.855, - "p99": 59.295, - "pod_mem_mb": 1059, - "saturated": true, - "status": "ok", - "reason": "plateau", - "pod_mem_p50_mb": 1032 - } - } -} \ No newline at end of file diff --git a/results/wal-fanout-sweep/s4-f2/cells.json b/results/wal-fanout-sweep/s4-f2/cells.json deleted file mode 100644 index 21f88ae..0000000 --- a/results/wal-fanout-sweep/s4-f2/cells.json +++ /dev/null @@ -1,43 +0,0 @@ -{ - "cells": { - "200000": { - "stream_count": 200000, - "image_digest": "23b2edc0096a", - "walk": [ - [ - 2, - 58915.25, - 8.159, - 26.543 - ], - [ - 4, - 68520.95, - 14.431, - 41.663 - ], - [ - 8, - 68227.55, - 28.111, - 63.231 - ], - [ - 12, - 72834.0, - 41.343, - 80.255 - ] - ], - "pinned_pods": 4, - "throughput": 67583.9, - "p50": 14.495, - "p99": 41.951, - "pod_mem_mb": 1032, - "saturated": true, - "status": "ok", - "reason": "plateau", - "pod_mem_p50_mb": 1004 - } - } -} \ No newline at end of file diff --git a/results/wal-fanout-sweep/s4-f4/cells.json b/results/wal-fanout-sweep/s4-f4/cells.json deleted file mode 100644 index bb2d6f1..0000000 --- a/results/wal-fanout-sweep/s4-f4/cells.json +++ /dev/null @@ -1,49 +0,0 @@ -{ - "cells": { - "200000": { - "stream_count": 200000, - "image_digest": "e715dfccc606", - "walk": [ - [ - 2, - 47302.15, - 9.551, - 29.807 - ], - [ - 4, - 58659.0, - 16.199, - 37.983 - ], - [ - 8, - 65971.34999999999, - 29.007, - 66.239 - ], - [ - 12, - 74076.75, - 41.503, - 73.471 - ], - [ - 16, - 75057.70000000001, - 54.527, - 99.007 - ] - ], - "pinned_pods": 16, - "throughput": 75057.70000000001, - "p50": null, - "p99": null, - "pod_mem_mb": 1109, - "saturated": false, - "status": "ok", - "reason": "ladder_exhausted", - "pod_mem_p50_mb": 1049 - } - } -} \ No newline at end of file diff --git a/results/wal-fanout-sweep/s4-f8/cells.json b/results/wal-fanout-sweep/s4-f8/cells.json deleted file mode 100644 index 579fb4b..0000000 --- a/results/wal-fanout-sweep/s4-f8/cells.json +++ /dev/null @@ -1,49 +0,0 @@ -{ - "cells": { - "200000": { - "stream_count": 200000, - "image_digest": "73aa375b7242", - "walk": [ - [ - 2, - 45221.2, - 10.951, - 23.359 - ], - [ - 4, - 55360.25, - 17.839, - 36.191 - ], - [ - 8, - 59471.4, - 33.119, - 63.359 - ], - [ - 12, - 65443.2, - 45.599, - 81.535 - ], - [ - 16, - 66705.1, - 59.743, - 104.191 - ] - ], - "pinned_pods": 16, - "throughput": 66705.1, - "p50": null, - "p99": null, - "pod_mem_mb": 1101, - "saturated": false, - "status": "ok", - "reason": "ladder_exhausted", - "pod_mem_p50_mb": 1039 - } - } -} \ No newline at end of file diff --git a/results/wal-fsync-diag-local/aggregate.csv b/results/wal-fsync-diag-local/aggregate.csv deleted file mode 100644 index e8639a2..0000000 --- a/results/wal-fsync-diag-local/aggregate.csv +++ /dev/null @@ -1,7 +0,0 @@ -mode,stream_count,pods,throughput,p50,p99,knee_pods,knee_throughput,knee_p50,knee_p99,pod_mem_mb,pod_mem_p50_mb,saturated,status,reason -memory,20000,2,165326.0,1.323,5.311,2,162666.25,1.32,5.667,121,112,True,ok,plateau -wal-nofsync,20000,2,98446.8,2.111,10.047,2,118315.625,1.874,7.443,132,118,True,ok,plateau -wal-normal,20000,4,74963.0,,,2,55980.625,3.655,14.047,128,110,False,ok,ladder_exhausted -memory,50000,2,132834.0,1.527,7.711,4,58021.875,5.515,53.471,266,231,True,ok,plateau -wal-nofsync,50000,4,103272.0,4.279,15.519,2,91121.125,2.245,10.183,290,244,True,ok,plateau -wal-normal,50000,6,68782.625,,,4,51999.0,6.247,138.879,259,244,False,ok,ladder_exhausted diff --git a/results/wal-fsync-diag-local/aggregate.json b/results/wal-fsync-diag-local/aggregate.json deleted file mode 100644 index 1b26c78..0000000 --- a/results/wal-fsync-diag-local/aggregate.json +++ /dev/null @@ -1,200 +0,0 @@ -[ - { - "mode": "memory", - "stream_count": 20000, - "pods": 2, - "throughput": 165326.0, - "p50": 1.323, - "p99": 5.311, - "knee_pods": 2, - "knee_throughput": 162666.25, - "knee_p50": 1.32, - "knee_p99": 5.667, - "pod_mem_mb": 121, - "pod_mem_p50_mb": 112, - "saturated": true, - "status": "ok", - "reason": "plateau", - "walk": [ - [ - 2, - 162666.25, - 1.32, - 5.667 - ], - [ - 4, - 177057.75, - 2.171, - 12.407 - ] - ] - }, - { - "mode": "wal-nofsync", - "stream_count": 20000, - "pods": 2, - "throughput": 98446.8, - "p50": 2.111, - "p99": 10.047, - "knee_pods": 2, - "knee_throughput": 118315.625, - "knee_p50": 1.874, - "knee_p99": 7.443, - "pod_mem_mb": 132, - "pod_mem_p50_mb": 118, - "saturated": true, - "status": "ok", - "reason": "plateau", - "walk": [ - [ - 2, - 118315.625, - 1.874, - 7.443 - ], - [ - 4, - 119608.375, - 3.795, - 13.327 - ] - ] - }, - { - "mode": "wal-normal", - "stream_count": 20000, - "pods": 4, - "throughput": 74963.0, - "p50": null, - "p99": null, - "knee_pods": 2, - "knee_throughput": 55980.625, - "knee_p50": 3.655, - "knee_p99": 14.047, - "pod_mem_mb": 128, - "pod_mem_p50_mb": 110, - "saturated": false, - "status": "ok", - "reason": "ladder_exhausted", - "walk": [ - [ - 2, - 55980.625, - 3.655, - 14.047 - ], - [ - 4, - 74963.0, - 5.791, - 20.207 - ] - ] - }, - { - "mode": "memory", - "stream_count": 50000, - "pods": 2, - "throughput": 132834.0, - "p50": 1.527, - "p99": 7.711, - "knee_pods": 4, - "knee_throughput": 58021.875, - "knee_p50": 5.515, - "knee_p99": 53.471, - "pod_mem_mb": 266, - "pod_mem_p50_mb": 231, - "saturated": true, - "status": "ok", - "reason": "plateau", - "walk": [ - [ - 2, - 146339.875, - 1.393, - 5.163 - ], - [ - 4, - 58021.875, - 5.515, - 53.471 - ] - ] - }, - { - "mode": "wal-nofsync", - "stream_count": 50000, - "pods": 4, - "throughput": 103272.0, - "p50": 4.279, - "p99": 15.519, - "knee_pods": 2, - "knee_throughput": 91121.125, - "knee_p50": 2.245, - "knee_p99": 10.183, - "pod_mem_mb": 290, - "pod_mem_p50_mb": 244, - "saturated": true, - "status": "ok", - "reason": "plateau", - "walk": [ - [ - 2, - 91121.125, - 2.245, - 10.183 - ], - [ - 4, - 101021.25, - 4.299, - 15.415 - ], - [ - 6, - 98758.375, - 6.579, - 21.807 - ] - ] - }, - { - "mode": "wal-normal", - "stream_count": 50000, - "pods": 6, - "throughput": 68782.625, - "p50": null, - "p99": null, - "knee_pods": 4, - "knee_throughput": 51999.0, - "knee_p50": 6.247, - "knee_p99": 138.879, - "pod_mem_mb": 259, - "pod_mem_p50_mb": 244, - "saturated": false, - "status": "ok", - "reason": "ladder_exhausted", - "walk": [ - [ - 2, - 27575.875, - 5.191, - 152.447 - ], - [ - 4, - 51999.0, - 6.247, - 138.879 - ], - [ - 6, - 68782.625, - 7.939, - 132.991 - ] - ] - } -] \ No newline at end of file diff --git a/results/wal-fsync-diag-local/memory/cells.json b/results/wal-fsync-diag-local/memory/cells.json deleted file mode 100644 index bf263bb..0000000 --- a/results/wal-fsync-diag-local/memory/cells.json +++ /dev/null @@ -1,58 +0,0 @@ -{ - "cells": { - "20000": { - "stream_count": 20000, - "image_digest": "aa40e13f0cf3", - "walk": [ - [ - 2, - 162666.25, - 1.32, - 5.667 - ], - [ - 4, - 177057.75, - 2.171, - 12.407 - ] - ], - "pinned_pods": 2, - "throughput": 165326.0, - "p50": 1.323, - "p99": 5.311, - "pod_mem_mb": 121, - "saturated": true, - "status": "ok", - "reason": "plateau", - "pod_mem_p50_mb": 112 - }, - "50000": { - "stream_count": 50000, - "image_digest": "aa40e13f0cf3", - "walk": [ - [ - 2, - 146339.875, - 1.393, - 5.163 - ], - [ - 4, - 58021.875, - 5.515, - 53.471 - ] - ], - "pinned_pods": 2, - "throughput": 132834.0, - "p50": 1.527, - "p99": 7.711, - "pod_mem_mb": 266, - "saturated": true, - "status": "ok", - "reason": "plateau", - "pod_mem_p50_mb": 231 - } - } -} \ No newline at end of file diff --git a/results/wal-fsync-diag-local/report.md b/results/wal-fsync-diag-local/report.md deleted file mode 100644 index 566ce03..0000000 --- a/results/wal-fsync-diag-local/report.md +++ /dev/null @@ -1,45 +0,0 @@ -# wal-fsync-diag-local — write-throughput report - -## Throughput at saturation (ops/s) - -| streams | wal-normal | wal-nofsync | memory | -|---|---|---|---| -| 20000 | 75k† | 98k | 165k | -| 50000 | 69k† | 103k | 133k | - -† = not saturated (ladder exhausted) — treat as a lower bound. - -## Pod memory at saturation — peak / p50 (MiB) - -| streams | wal-normal | wal-nofsync | memory | -|---|---|---|---| -| 20000 | 128 / 110 | 132 / 118 | 121 / 112 | -| 50000 | 259 / 244 | 290 / 244 | 266 / 231 | - -_Pod working set = cgroup `memory.current − inactive_file` (anon + active page cache), sampled each second at the pinned rung. **peak** = high-water (catches bursts like an in-RAM Raft log filling); **p50** = median (what the server steadily holds resident). peak ≈ p50 ⇒ steadily resident; peak ≫ p50 ⇒ transient spikes._ - -## Latency (ms, p50 / p99) - -| streams | wal-normal @≤80% load | wal-normal @saturation | wal-nofsync @≤80% load | wal-nofsync @saturation | memory @≤80% load | memory @saturation | -|---|---|---|---|---|---|---| -| 20000 | 3.7 / 14.0 (56k @2p) | — | 1.9 / 7.4 (118k @2p) | 2.1 / 10.0 | 1.3 / 5.7 (163k @2p) | 1.3 / 5.3 | -| 50000 | 6.2 / 138.9 (52k @4p) | — | 2.2 / 10.2 (91k @2p) | 4.3 / 15.5 | 5.5 / 53.5 (58k @4p) | 1.5 / 7.7 | - -_@≤80% load = the largest ladder rung at ≤80% of peak throughput — the server's service latency with headroom. @saturation = the pinned plateau rung, where a closed-loop fleet measures its own queueing (≈ in-flight ÷ ceiling by Little's law), NOT the server's per-request cost. Compare against the unloaded single-request baseline (~1 ms for wal) before reading anything into large saturation values._ - -## Saturation walks (pods → ops/s, p50 ms) - -- **memory 20000**: 2:163k@1.3ms → 4:177k@2.2ms (pinned 2, plateau) -- **wal-nofsync 20000**: 2:118k@1.9ms → 4:120k@3.8ms (pinned 2, plateau) -- **wal-normal 20000**: 2:56k@3.7ms → 4:75k@5.8ms (pinned 4, ladder_exhausted) -- **memory 50000**: 2:146k@1.4ms → 4:58k@5.5ms (pinned 2, plateau) -- **wal-nofsync 50000**: 2:91k@2.2ms → 4:101k@4.3ms → 6:99k@6.6ms (pinned 4, plateau) -- **wal-normal 50000**: 2:28k@5.2ms → 4:52k@6.2ms → 6:69k@7.9ms (pinned 6, ladder_exhausted) - -## Findings - -_TODO: written by hand on top of the generated data._ - -## Caveats - -_Single-node best-case; not 3-node Raft. Throughput is a saturation ceiling per the ladder._ diff --git a/results/wal-fsync-diag-local/wal-nofsync/cells.json b/results/wal-fsync-diag-local/wal-nofsync/cells.json deleted file mode 100644 index 7c07caa..0000000 --- a/results/wal-fsync-diag-local/wal-nofsync/cells.json +++ /dev/null @@ -1,64 +0,0 @@ -{ - "cells": { - "20000": { - "stream_count": 20000, - "image_digest": "c6dda090c943", - "walk": [ - [ - 2, - 118315.625, - 1.874, - 7.443 - ], - [ - 4, - 119608.375, - 3.795, - 13.327 - ] - ], - "pinned_pods": 2, - "throughput": 98446.8, - "p50": 2.111, - "p99": 10.047, - "pod_mem_mb": 132, - "saturated": true, - "status": "ok", - "reason": "plateau", - "pod_mem_p50_mb": 118 - }, - "50000": { - "stream_count": 50000, - "image_digest": "c6dda090c943", - "walk": [ - [ - 2, - 91121.125, - 2.245, - 10.183 - ], - [ - 4, - 101021.25, - 4.299, - 15.415 - ], - [ - 6, - 98758.375, - 6.579, - 21.807 - ] - ], - "pinned_pods": 4, - "throughput": 103272.0, - "p50": 4.279, - "p99": 15.519, - "pod_mem_mb": 290, - "saturated": true, - "status": "ok", - "reason": "plateau", - "pod_mem_p50_mb": 244 - } - } -} \ No newline at end of file diff --git a/results/wal-fsync-diag-local/wal-normal/cells.json b/results/wal-fsync-diag-local/wal-normal/cells.json deleted file mode 100644 index c6b3c48..0000000 --- a/results/wal-fsync-diag-local/wal-normal/cells.json +++ /dev/null @@ -1,64 +0,0 @@ -{ - "cells": { - "20000": { - "stream_count": 20000, - "image_digest": "b659d4d00350", - "walk": [ - [ - 2, - 55980.625, - 3.655, - 14.047 - ], - [ - 4, - 74963.0, - 5.791, - 20.207 - ] - ], - "pinned_pods": 4, - "throughput": 74963.0, - "p50": null, - "p99": null, - "pod_mem_mb": 128, - "saturated": false, - "status": "ok", - "reason": "ladder_exhausted", - "pod_mem_p50_mb": 110 - }, - "50000": { - "stream_count": 50000, - "image_digest": "b659d4d00350", - "walk": [ - [ - 2, - 27575.875, - 5.191, - 152.447 - ], - [ - 4, - 51999.0, - 6.247, - 138.879 - ], - [ - 6, - 68782.625, - 7.939, - 132.991 - ] - ], - "pinned_pods": 6, - "throughput": 68782.625, - "p50": null, - "p99": null, - "pod_mem_mb": 259, - "saturated": false, - "status": "ok", - "reason": "ladder_exhausted", - "pod_mem_p50_mb": 244 - } - } -} \ No newline at end of file diff --git a/results/wal-lazy-checkpoint-local/aggregate.csv b/results/wal-lazy-checkpoint-local/aggregate.csv deleted file mode 100644 index b3b3f88..0000000 --- a/results/wal-lazy-checkpoint-local/aggregate.csv +++ /dev/null @@ -1,9 +0,0 @@ -mode,stream_count,pods,throughput,p50,p99,knee_pods,knee_throughput,knee_p50,knee_p99,pod_mem_mb,pod_mem_p50_mb,saturated,status,reason -lazy-syncfs-30s,20000,4,100066.25,,,2,68584.25,3.223,9.447,129,118,False,ok,ladder_exhausted -lazy-syncfs-60s,20000,4,98228.375,,,2,80041.5,3.133,5.231,128,118,False,ok,ladder_exhausted -ref-baseline-3s,20000,4,77092.75,,,2,54549.0,4.503,9.463,128,106,False,ok,ladder_exhausted -ref-syncfs-3s,20000,4,88036.75,,,2,60107.625,3.461,12.295,134,123,False,ok,ladder_exhausted -lazy-syncfs-30s,50000,4,72479.5,5.411,55.775,2,58209.0,4.155,9.391,278,239,True,ok,plateau -lazy-syncfs-60s,50000,6,89838.75,,,2,59949.5,3.945,11.079,281,242,False,ok,ladder_exhausted -ref-baseline-3s,50000,4,61508.1,6.503,30.639,2,21584.5,8.655,62.879,262,239,True,ok,plateau -ref-syncfs-3s,50000,4,50372.9,5.883,136.319,2,49339.875,3.859,19.711,247,238,True,ok,plateau diff --git a/results/wal-lazy-checkpoint-local/aggregate.json b/results/wal-lazy-checkpoint-local/aggregate.json deleted file mode 100644 index 33d7c29..0000000 --- a/results/wal-lazy-checkpoint-local/aggregate.json +++ /dev/null @@ -1,274 +0,0 @@ -[ - { - "mode": "lazy-syncfs-30s", - "stream_count": 20000, - "pods": 4, - "throughput": 100066.25, - "p50": null, - "p99": null, - "knee_pods": 2, - "knee_throughput": 68584.25, - "knee_p50": 3.223, - "knee_p99": 9.447, - "pod_mem_mb": 129, - "pod_mem_p50_mb": 118, - "saturated": false, - "status": "ok", - "reason": "ladder_exhausted", - "walk": [ - [ - 2, - 68584.25, - 3.223, - 9.447 - ], - [ - 4, - 100066.25, - 4.807, - 9.431 - ] - ] - }, - { - "mode": "lazy-syncfs-60s", - "stream_count": 20000, - "pods": 4, - "throughput": 98228.375, - "p50": null, - "p99": null, - "knee_pods": 2, - "knee_throughput": 80041.5, - "knee_p50": 3.133, - "knee_p99": 5.231, - "pod_mem_mb": 128, - "pod_mem_p50_mb": 118, - "saturated": false, - "status": "ok", - "reason": "ladder_exhausted", - "walk": [ - [ - 2, - 80041.5, - 3.133, - 5.231 - ], - [ - 4, - 98228.375, - 4.819, - 9.559 - ] - ] - }, - { - "mode": "ref-baseline-3s", - "stream_count": 20000, - "pods": 4, - "throughput": 77092.75, - "p50": null, - "p99": null, - "knee_pods": 2, - "knee_throughput": 54549.0, - "knee_p50": 4.503, - "knee_p99": 9.463, - "pod_mem_mb": 128, - "pod_mem_p50_mb": 106, - "saturated": false, - "status": "ok", - "reason": "ladder_exhausted", - "walk": [ - [ - 2, - 54549.0, - 4.503, - 9.463 - ], - [ - 4, - 77092.75, - 6.063, - 16.247 - ] - ] - }, - { - "mode": "ref-syncfs-3s", - "stream_count": 20000, - "pods": 4, - "throughput": 88036.75, - "p50": null, - "p99": null, - "knee_pods": 2, - "knee_throughput": 60107.625, - "knee_p50": 3.461, - "knee_p99": 12.295, - "pod_mem_mb": 134, - "pod_mem_p50_mb": 123, - "saturated": false, - "status": "ok", - "reason": "ladder_exhausted", - "walk": [ - [ - 2, - 60107.625, - 3.461, - 12.295 - ], - [ - 4, - 88036.75, - 4.991, - 12.447 - ] - ] - }, - { - "mode": "lazy-syncfs-30s", - "stream_count": 50000, - "pods": 4, - "throughput": 72479.5, - "p50": 5.411, - "p99": 55.775, - "knee_pods": 2, - "knee_throughput": 58209.0, - "knee_p50": 4.155, - "knee_p99": 9.391, - "pod_mem_mb": 278, - "pod_mem_p50_mb": 239, - "saturated": true, - "status": "ok", - "reason": "plateau", - "walk": [ - [ - 2, - 58209.0, - 4.155, - 9.391 - ], - [ - 4, - 80708.5, - 5.367, - 10.455 - ], - [ - 6, - 72297.625, - 7.559, - 103.231 - ] - ] - }, - { - "mode": "lazy-syncfs-60s", - "stream_count": 50000, - "pods": 6, - "throughput": 89838.75, - "p50": null, - "p99": null, - "knee_pods": 2, - "knee_throughput": 59949.5, - "knee_p50": 3.945, - "knee_p99": 11.079, - "pod_mem_mb": 281, - "pod_mem_p50_mb": 242, - "saturated": false, - "status": "ok", - "reason": "ladder_exhausted", - "walk": [ - [ - 2, - 59949.5, - 3.945, - 11.079 - ], - [ - 4, - 80318.125, - 5.359, - 25.855 - ], - [ - 6, - 89838.75, - 7.551, - 35.135 - ] - ] - }, - { - "mode": "ref-baseline-3s", - "stream_count": 50000, - "pods": 4, - "throughput": 61508.1, - "p50": 6.503, - "p99": 30.639, - "knee_pods": 2, - "knee_throughput": 21584.5, - "knee_p50": 8.655, - "knee_p99": 62.879, - "pod_mem_mb": 262, - "pod_mem_p50_mb": 239, - "saturated": true, - "status": "ok", - "reason": "plateau", - "walk": [ - [ - 2, - 21584.5, - 8.655, - 62.879 - ], - [ - 4, - 64409.25, - 7.271, - 18.607 - ], - [ - 6, - 63920.625, - 9.183, - 56.639 - ] - ] - }, - { - "mode": "ref-syncfs-3s", - "stream_count": 50000, - "pods": 4, - "throughput": 50372.9, - "p50": 5.883, - "p99": 136.319, - "knee_pods": 2, - "knee_throughput": 49339.875, - "knee_p50": 3.859, - "knee_p99": 19.711, - "pod_mem_mb": 247, - "pod_mem_p50_mb": 238, - "saturated": true, - "status": "ok", - "reason": "plateau", - "walk": [ - [ - 2, - 49339.875, - 3.859, - 19.711 - ], - [ - 4, - 67934.75, - 5.511, - 98.431 - ], - [ - 6, - 69571.25, - 7.743, - 129.279 - ] - ] - } -] \ No newline at end of file diff --git a/results/wal-lazy-checkpoint-local/lazy-syncfs-30s/cells.json b/results/wal-lazy-checkpoint-local/lazy-syncfs-30s/cells.json deleted file mode 100644 index af802b5..0000000 --- a/results/wal-lazy-checkpoint-local/lazy-syncfs-30s/cells.json +++ /dev/null @@ -1,64 +0,0 @@ -{ - "cells": { - "20000": { - "stream_count": 20000, - "image_digest": "11407215416a", - "walk": [ - [ - 2, - 68584.25, - 3.223, - 9.447 - ], - [ - 4, - 100066.25, - 4.807, - 9.431 - ] - ], - "pinned_pods": 4, - "throughput": 100066.25, - "p50": null, - "p99": null, - "pod_mem_mb": 129, - "saturated": false, - "status": "ok", - "reason": "ladder_exhausted", - "pod_mem_p50_mb": 118 - }, - "50000": { - "stream_count": 50000, - "image_digest": "11407215416a", - "walk": [ - [ - 2, - 58209.0, - 4.155, - 9.391 - ], - [ - 4, - 80708.5, - 5.367, - 10.455 - ], - [ - 6, - 72297.625, - 7.559, - 103.231 - ] - ], - "pinned_pods": 4, - "throughput": 72479.5, - "p50": 5.411, - "p99": 55.775, - "pod_mem_mb": 278, - "saturated": true, - "status": "ok", - "reason": "plateau", - "pod_mem_p50_mb": 239 - } - } -} \ No newline at end of file diff --git a/results/wal-lazy-checkpoint-local/lazy-syncfs-60s/cells.json b/results/wal-lazy-checkpoint-local/lazy-syncfs-60s/cells.json deleted file mode 100644 index 71b6d58..0000000 --- a/results/wal-lazy-checkpoint-local/lazy-syncfs-60s/cells.json +++ /dev/null @@ -1,64 +0,0 @@ -{ - "cells": { - "20000": { - "stream_count": 20000, - "image_digest": "48c267b280e1", - "walk": [ - [ - 2, - 80041.5, - 3.133, - 5.231 - ], - [ - 4, - 98228.375, - 4.819, - 9.559 - ] - ], - "pinned_pods": 4, - "throughput": 98228.375, - "p50": null, - "p99": null, - "pod_mem_mb": 128, - "saturated": false, - "status": "ok", - "reason": "ladder_exhausted", - "pod_mem_p50_mb": 118 - }, - "50000": { - "stream_count": 50000, - "image_digest": "48c267b280e1", - "walk": [ - [ - 2, - 59949.5, - 3.945, - 11.079 - ], - [ - 4, - 80318.125, - 5.359, - 25.855 - ], - [ - 6, - 89838.75, - 7.551, - 35.135 - ] - ], - "pinned_pods": 6, - "throughput": 89838.75, - "p50": null, - "p99": null, - "pod_mem_mb": 281, - "saturated": false, - "status": "ok", - "reason": "ladder_exhausted", - "pod_mem_p50_mb": 242 - } - } -} \ No newline at end of file diff --git a/results/wal-lazy-checkpoint-local/ref-baseline-3s/cells.json b/results/wal-lazy-checkpoint-local/ref-baseline-3s/cells.json deleted file mode 100644 index d4d096e..0000000 --- a/results/wal-lazy-checkpoint-local/ref-baseline-3s/cells.json +++ /dev/null @@ -1,64 +0,0 @@ -{ - "cells": { - "20000": { - "stream_count": 20000, - "image_digest": "0bb2c6c89ce6", - "walk": [ - [ - 2, - 54549.0, - 4.503, - 9.463 - ], - [ - 4, - 77092.75, - 6.063, - 16.247 - ] - ], - "pinned_pods": 4, - "throughput": 77092.75, - "p50": null, - "p99": null, - "pod_mem_mb": 128, - "saturated": false, - "status": "ok", - "reason": "ladder_exhausted", - "pod_mem_p50_mb": 106 - }, - "50000": { - "stream_count": 50000, - "image_digest": "0bb2c6c89ce6", - "walk": [ - [ - 2, - 21584.5, - 8.655, - 62.879 - ], - [ - 4, - 64409.25, - 7.271, - 18.607 - ], - [ - 6, - 63920.625, - 9.183, - 56.639 - ] - ], - "pinned_pods": 4, - "throughput": 61508.1, - "p50": 6.503, - "p99": 30.639, - "pod_mem_mb": 262, - "saturated": true, - "status": "ok", - "reason": "plateau", - "pod_mem_p50_mb": 239 - } - } -} \ No newline at end of file diff --git a/results/wal-lazy-checkpoint-local/ref-syncfs-3s/cells.json b/results/wal-lazy-checkpoint-local/ref-syncfs-3s/cells.json deleted file mode 100644 index 02ae227..0000000 --- a/results/wal-lazy-checkpoint-local/ref-syncfs-3s/cells.json +++ /dev/null @@ -1,64 +0,0 @@ -{ - "cells": { - "20000": { - "stream_count": 20000, - "image_digest": "8c09d3514768", - "walk": [ - [ - 2, - 60107.625, - 3.461, - 12.295 - ], - [ - 4, - 88036.75, - 4.991, - 12.447 - ] - ], - "pinned_pods": 4, - "throughput": 88036.75, - "p50": null, - "p99": null, - "pod_mem_mb": 134, - "saturated": false, - "status": "ok", - "reason": "ladder_exhausted", - "pod_mem_p50_mb": 123 - }, - "50000": { - "stream_count": 50000, - "image_digest": "8c09d3514768", - "walk": [ - [ - 2, - 49339.875, - 3.859, - 19.711 - ], - [ - 4, - 67934.75, - 5.511, - 98.431 - ], - [ - 6, - 69571.25, - 7.743, - 129.279 - ] - ], - "pinned_pods": 4, - "throughput": 50372.9, - "p50": 5.883, - "p99": 136.319, - "pod_mem_mb": 247, - "saturated": true, - "status": "ok", - "reason": "plateau", - "pod_mem_p50_mb": 238 - } - } -} \ No newline at end of file diff --git a/results/wal-lazy-checkpoint-local/report.md b/results/wal-lazy-checkpoint-local/report.md deleted file mode 100644 index 4035795..0000000 --- a/results/wal-lazy-checkpoint-local/report.md +++ /dev/null @@ -1,47 +0,0 @@ -# wal-lazy-checkpoint-local — write-throughput report - -## Throughput at saturation (ops/s) - -| streams | ref-baseline-3s | ref-syncfs-3s | lazy-syncfs-30s | lazy-syncfs-60s | -|---|---|---|---|---| -| 20000 | 77k† | 88k† | 100k† | 98k† | -| 50000 | 62k | 50k | 72k | 90k† | - -† = not saturated (ladder exhausted) — treat as a lower bound. - -## Pod memory at saturation — peak / p50 (MiB) - -| streams | ref-baseline-3s | ref-syncfs-3s | lazy-syncfs-30s | lazy-syncfs-60s | -|---|---|---|---|---| -| 20000 | 128 / 106 | 134 / 123 | 129 / 118 | 128 / 118 | -| 50000 | 262 / 239 | 247 / 238 | 278 / 239 | 281 / 242 | - -_Pod working set = cgroup `memory.current − inactive_file` (anon + active page cache), sampled each second at the pinned rung. **peak** = high-water (catches bursts like an in-RAM Raft log filling); **p50** = median (what the server steadily holds resident). peak ≈ p50 ⇒ steadily resident; peak ≫ p50 ⇒ transient spikes._ - -## Latency (ms, p50 / p99) - -| streams | ref-baseline-3s @≤80% load | ref-baseline-3s @saturation | ref-syncfs-3s @≤80% load | ref-syncfs-3s @saturation | lazy-syncfs-30s @≤80% load | lazy-syncfs-30s @saturation | lazy-syncfs-60s @≤80% load | lazy-syncfs-60s @saturation | -|---|---|---|---|---|---|---|---|---| -| 20000 | 4.5 / 9.5 (55k @2p) | — | 3.5 / 12.3 (60k @2p) | — | 3.2 / 9.4 (69k @2p) | — | 3.1 / 5.2 (80k @2p) | — | -| 50000 | 8.7 / 62.9 (22k @2p) | 6.5 / 30.6 | 3.9 / 19.7 (49k @2p) | 5.9 / 136.3 | 4.2 / 9.4 (58k @2p) | 5.4 / 55.8 | 3.9 / 11.1 (60k @2p) | — | - -_@≤80% load = the largest ladder rung at ≤80% of peak throughput — the server's service latency with headroom. @saturation = the pinned plateau rung, where a closed-loop fleet measures its own queueing (≈ in-flight ÷ ceiling by Little's law), NOT the server's per-request cost. Compare against the unloaded single-request baseline (~1 ms for wal) before reading anything into large saturation values._ - -## Saturation walks (pods → ops/s, p50 ms) - -- **lazy-syncfs-30s 20000**: 2:69k@3.2ms → 4:100k@4.8ms (pinned 4, ladder_exhausted) -- **lazy-syncfs-60s 20000**: 2:80k@3.1ms → 4:98k@4.8ms (pinned 4, ladder_exhausted) -- **ref-baseline-3s 20000**: 2:55k@4.5ms → 4:77k@6.1ms (pinned 4, ladder_exhausted) -- **ref-syncfs-3s 20000**: 2:60k@3.5ms → 4:88k@5.0ms (pinned 4, ladder_exhausted) -- **lazy-syncfs-30s 50000**: 2:58k@4.2ms → 4:81k@5.4ms → 6:72k@7.6ms (pinned 4, plateau) -- **lazy-syncfs-60s 50000**: 2:60k@3.9ms → 4:80k@5.4ms → 6:90k@7.6ms (pinned 6, ladder_exhausted) -- **ref-baseline-3s 50000**: 2:22k@8.7ms → 4:64k@7.3ms → 6:64k@9.2ms (pinned 4, plateau) -- **ref-syncfs-3s 50000**: 2:49k@3.9ms → 4:68k@5.5ms → 6:70k@7.7ms (pinned 4, plateau) - -## Findings - -_TODO: written by hand on top of the generated data._ - -## Caveats - -_Single-node best-case; not 3-node Raft. Throughput is a saturation ceiling per the ladder._ diff --git a/results/wal-lazy-checkpoint-multilane/lazy-120s/cells.json b/results/wal-lazy-checkpoint-multilane/lazy-120s/cells.json deleted file mode 100644 index f6af398..0000000 --- a/results/wal-lazy-checkpoint-multilane/lazy-120s/cells.json +++ /dev/null @@ -1,36 +0,0 @@ -{ - "cells": { - "10000": { - "stream_count": 10000, - "image_digest": "e3009dc55854", - "walk": [ - [ - 2, - 282020.05, - 1.478, - 11.111 - ], - [ - 4, - 288492.8, - 3.099, - 15.647 - ], - [ - 8, - 292573.05, - 6.259, - 28.463 - ] - ], - "pinned_pods": 2, - "throughput": 304788.0, - "p50": 1.433, - "p99": 6.447, - "pod_mem_mb": 0.0, - "saturated": true, - "status": "ok", - "reason": "plateau" - } - } -} \ No newline at end of file diff --git a/results/wal-lazy-checkpoint-multilane/lazy-30s/cells.json b/results/wal-lazy-checkpoint-multilane/lazy-30s/cells.json deleted file mode 100644 index af2c198..0000000 --- a/results/wal-lazy-checkpoint-multilane/lazy-30s/cells.json +++ /dev/null @@ -1,68 +0,0 @@ -{ - "cells": { - "10000": { - "stream_count": 10000, - "image_digest": "822a4f41ce43", - "walk": [ - [ - 2, - 274731.75, - 1.477, - 11.967 - ], - [ - 4, - 289023.25, - 3.127, - 13.775 - ], - [ - 8, - 262443.75, - 6.463, - 44.543 - ] - ], - "pinned_pods": 2, - "throughput": 291903.0, - "p50": 1.47, - "p99": 6.807, - "pod_mem_mb": 282.0, - "saturated": true, - "status": "ok", - "reason": "plateau" - }, - "100000": { - "stream_count": 100000, - "image_digest": "822a4f41ce43", - "walk": [ - [ - 2, - 13614.25, - 1.437, - 2015.231 - ], - [ - 4, - 6778.85, - 6.639, - 3770.367 - ], - [ - 8, - 9256.1, - 113.791, - 1354.751 - ] - ], - "pinned_pods": 8, - "throughput": 9256.1, - "p50": null, - "p99": null, - "pod_mem_mb": 0.0, - "saturated": false, - "status": "ok", - "reason": "ladder_exhausted" - } - } -} \ No newline at end of file diff --git a/results/wal-lazy-checkpoint-multilane/lazy-60s/cells.json b/results/wal-lazy-checkpoint-multilane/lazy-60s/cells.json deleted file mode 100644 index 016e465..0000000 --- a/results/wal-lazy-checkpoint-multilane/lazy-60s/cells.json +++ /dev/null @@ -1,68 +0,0 @@ -{ - "cells": { - "10000": { - "stream_count": 10000, - "image_digest": "c1696ad4ac9a", - "walk": [ - [ - 2, - 277239.9, - 1.444, - 12.551 - ], - [ - 4, - 263360.7, - 3.097, - 24.831 - ], - [ - 8, - 294393.25, - 6.343, - 17.487 - ] - ], - "pinned_pods": 8, - "throughput": 294393.25, - "p50": null, - "p99": null, - "pod_mem_mb": 0.0, - "saturated": false, - "status": "ok", - "reason": "ladder_exhausted" - }, - "100000": { - "stream_count": 100000, - "image_digest": "c1696ad4ac9a", - "walk": [ - [ - 2, - 15470.5, - 1.577, - 1514.495 - ], - [ - 4, - 10707.900000000001, - 4.075, - 1467.391 - ], - [ - 8, - 6952.1, - 270.847, - 1154.047 - ] - ], - "pinned_pods": 2, - "throughput": 14787.7, - "p50": 1.744, - "p99": 1406.975, - "pod_mem_mb": 558.0, - "saturated": true, - "status": "ok", - "reason": "plateau" - } - } -} \ No newline at end of file diff --git a/results/wal-lazy-checkpoint-multilane/ref-syncfs-3s/cells.json b/results/wal-lazy-checkpoint-multilane/ref-syncfs-3s/cells.json deleted file mode 100644 index ce00e9d..0000000 --- a/results/wal-lazy-checkpoint-multilane/ref-syncfs-3s/cells.json +++ /dev/null @@ -1,68 +0,0 @@ -{ - "cells": { - "10000": { - "stream_count": 10000, - "image_digest": "c8d713e7f8a2", - "walk": [ - [ - 2, - 185198.9, - 1.435, - 27.375 - ], - [ - 4, - 256548.85, - 3.151, - 24.831 - ], - [ - 8, - 270540.89999999997, - 6.299, - 34.495 - ] - ], - "pinned_pods": 8, - "throughput": 270540.89999999997, - "p50": null, - "p99": null, - "pod_mem_mb": 307.0, - "saturated": false, - "status": "ok", - "reason": "ladder_exhausted" - }, - "100000": { - "stream_count": 100000, - "image_digest": "c8d713e7f8a2", - "walk": [ - [ - 2, - 8546.45, - 1.765, - 1037.823 - ], - [ - 4, - 12124.550000000001, - 4.123, - 1283.071 - ], - [ - 8, - 8495.05, - 165.119, - 1568.767 - ] - ], - "pinned_pods": 8, - "throughput": 8495.05, - "p50": null, - "p99": null, - "pod_mem_mb": 0.0, - "saturated": false, - "status": "ok", - "reason": "ladder_exhausted" - } - } -} \ No newline at end of file diff --git a/results/wal-machinery-baseline-local/aggregate.csv b/results/wal-machinery-baseline-local/aggregate.csv deleted file mode 100644 index 806dbe1..0000000 --- a/results/wal-machinery-baseline-local/aggregate.csv +++ /dev/null @@ -1,2 +0,0 @@ -mode,stream_count,pods,throughput,p50,p99,knee_pods,knee_throughput,knee_p50,knee_p99,pod_mem_mb,pod_mem_p50_mb,saturated,status,reason -wal-baseline,50000,6,50662.1,,,2,32697.800000000003,4.495,128.959,252,242,False,ok,ladder_exhausted diff --git a/results/wal-machinery-baseline-local/aggregate.json b/results/wal-machinery-baseline-local/aggregate.json deleted file mode 100644 index cc5d074..0000000 --- a/results/wal-machinery-baseline-local/aggregate.json +++ /dev/null @@ -1,39 +0,0 @@ -[ - { - "mode": "wal-baseline", - "stream_count": 50000, - "pods": 6, - "throughput": 50662.1, - "p50": null, - "p99": null, - "knee_pods": 2, - "knee_throughput": 32697.800000000003, - "knee_p50": 4.495, - "knee_p99": 128.959, - "pod_mem_mb": 252, - "pod_mem_p50_mb": 242, - "saturated": false, - "status": "ok", - "reason": "ladder_exhausted", - "walk": [ - [ - 2, - 32697.800000000003, - 4.495, - 128.959 - ], - [ - 4, - 45841.4, - 6.711, - 187.263 - ], - [ - 6, - 50662.1, - 9.407, - 201.855 - ] - ] - } -] \ No newline at end of file diff --git a/results/wal-machinery-baseline-local/report.md b/results/wal-machinery-baseline-local/report.md deleted file mode 100644 index 52ca9e7..0000000 --- a/results/wal-machinery-baseline-local/report.md +++ /dev/null @@ -1,37 +0,0 @@ -# wal-machinery-baseline-local — write-throughput report - -## Throughput at saturation (ops/s) - -| streams | wal-baseline | -|---|---| -| 50000 | 51k† | - -† = not saturated (ladder exhausted) — treat as a lower bound. - -## Pod memory at saturation — peak / p50 (MiB) - -| streams | wal-baseline | -|---|---| -| 50000 | 252 / 242 | - -_Pod working set = cgroup `memory.current − inactive_file` (anon + active page cache), sampled each second at the pinned rung. **peak** = high-water (catches bursts like an in-RAM Raft log filling); **p50** = median (what the server steadily holds resident). peak ≈ p50 ⇒ steadily resident; peak ≫ p50 ⇒ transient spikes._ - -## Latency (ms, p50 / p99) - -| streams | wal-baseline @≤80% load | wal-baseline @saturation | -|---|---|---| -| 50000 | 4.5 / 129.0 (33k @2p) | — | - -_@≤80% load = the largest ladder rung at ≤80% of peak throughput — the server's service latency with headroom. @saturation = the pinned plateau rung, where a closed-loop fleet measures its own queueing (≈ in-flight ÷ ceiling by Little's law), NOT the server's per-request cost. Compare against the unloaded single-request baseline (~1 ms for wal) before reading anything into large saturation values._ - -## Saturation walks (pods → ops/s, p50 ms) - -- **wal-baseline 50000**: 2:33k@4.5ms → 4:46k@6.7ms → 6:51k@9.4ms (pinned 6, ladder_exhausted) - -## Findings - -_TODO: written by hand on top of the generated data._ - -## Caveats - -_Single-node best-case; not 3-node Raft. Throughput is a saturation ceiling per the ladder._ diff --git a/results/wal-machinery-baseline-local/wal-baseline/cells.json b/results/wal-machinery-baseline-local/wal-baseline/cells.json deleted file mode 100644 index cdbc3b5..0000000 --- a/results/wal-machinery-baseline-local/wal-baseline/cells.json +++ /dev/null @@ -1,37 +0,0 @@ -{ - "cells": { - "50000": { - "stream_count": 50000, - "image_digest": "1ee33d6c2b18", - "walk": [ - [ - 2, - 32697.800000000003, - 4.495, - 128.959 - ], - [ - 4, - 45841.4, - 6.711, - 187.263 - ], - [ - 6, - 50662.1, - 9.407, - 201.855 - ] - ], - "pinned_pods": 6, - "throughput": 50662.1, - "p50": null, - "p99": null, - "pod_mem_mb": 252, - "saturated": false, - "status": "ok", - "reason": "ladder_exhausted", - "pod_mem_p50_mb": 242 - } - } -} \ No newline at end of file diff --git a/results/wal-mem-diag-local/aggregate.csv b/results/wal-mem-diag-local/aggregate.csv deleted file mode 100644 index 77fa8ae..0000000 --- a/results/wal-mem-diag-local/aggregate.csv +++ /dev/null @@ -1,5 +0,0 @@ -mode,stream_count,pods,throughput,p50,p99,knee_pods,knee_throughput,knee_p50,knee_p99,pod_mem_mb,pod_mem_p50_mb,saturated,status,reason -memory,20000,4,170410.25,,,2,136822.375,1.517,7.163,127,117,False,ok,ladder_exhausted -wal-syncfs,20000,4,73717.25,,,2,62928.875,3.463,11.343,133,106,False,ok,ladder_exhausted -memory,50000,2,139270.0,1.44,7.987,2,145706.0,1.458,6.383,270,230,True,ok,plateau -wal-syncfs,50000,2,23309.2,6.339,146.943,2,43436.625,4.075,39.519,243,226,True,ok,plateau diff --git a/results/wal-mem-diag-local/aggregate.json b/results/wal-mem-diag-local/aggregate.json deleted file mode 100644 index 244d9a3..0000000 --- a/results/wal-mem-diag-local/aggregate.json +++ /dev/null @@ -1,126 +0,0 @@ -[ - { - "mode": "memory", - "stream_count": 20000, - "pods": 4, - "throughput": 170410.25, - "p50": null, - "p99": null, - "knee_pods": 2, - "knee_throughput": 136822.375, - "knee_p50": 1.517, - "knee_p99": 7.163, - "pod_mem_mb": 127, - "pod_mem_p50_mb": 117, - "saturated": false, - "status": "ok", - "reason": "ladder_exhausted", - "walk": [ - [ - 2, - 136822.375, - 1.517, - 7.163 - ], - [ - 4, - 170410.25, - 2.221, - 13.199 - ] - ] - }, - { - "mode": "wal-syncfs", - "stream_count": 20000, - "pods": 4, - "throughput": 73717.25, - "p50": null, - "p99": null, - "knee_pods": 2, - "knee_throughput": 62928.875, - "knee_p50": 3.463, - "knee_p99": 11.343, - "pod_mem_mb": 133, - "pod_mem_p50_mb": 106, - "saturated": false, - "status": "ok", - "reason": "ladder_exhausted", - "walk": [ - [ - 2, - 62928.875, - 3.463, - 11.343 - ], - [ - 4, - 73717.25, - 5.839, - 14.951 - ] - ] - }, - { - "mode": "memory", - "stream_count": 50000, - "pods": 2, - "throughput": 139270.0, - "p50": 1.44, - "p99": 7.987, - "knee_pods": 2, - "knee_throughput": 145706.0, - "knee_p50": 1.458, - "knee_p99": 6.383, - "pod_mem_mb": 270, - "pod_mem_p50_mb": 230, - "saturated": true, - "status": "ok", - "reason": "plateau", - "walk": [ - [ - 2, - 145706.0, - 1.458, - 6.383 - ], - [ - 4, - 146135.0, - 2.589, - 14.735 - ] - ] - }, - { - "mode": "wal-syncfs", - "stream_count": 50000, - "pods": 2, - "throughput": 23309.2, - "p50": 6.339, - "p99": 146.943, - "knee_pods": 2, - "knee_throughput": 43436.625, - "knee_p50": 4.075, - "knee_p99": 39.519, - "pod_mem_mb": 243, - "pod_mem_p50_mb": 226, - "saturated": true, - "status": "ok", - "reason": "plateau", - "walk": [ - [ - 2, - 43436.625, - 4.075, - 39.519 - ], - [ - 4, - 36744.125, - 8.759, - 149.375 - ] - ] - } -] \ No newline at end of file diff --git a/results/wal-mem-diag-local/memory/cells.json b/results/wal-mem-diag-local/memory/cells.json deleted file mode 100644 index fc7a0a1..0000000 --- a/results/wal-mem-diag-local/memory/cells.json +++ /dev/null @@ -1,58 +0,0 @@ -{ - "cells": { - "20000": { - "stream_count": 20000, - "image_digest": "aa40e13f0cf3", - "walk": [ - [ - 2, - 136822.375, - 1.517, - 7.163 - ], - [ - 4, - 170410.25, - 2.221, - 13.199 - ] - ], - "pinned_pods": 4, - "throughput": 170410.25, - "p50": null, - "p99": null, - "pod_mem_mb": 127, - "saturated": false, - "status": "ok", - "reason": "ladder_exhausted", - "pod_mem_p50_mb": 117 - }, - "50000": { - "stream_count": 50000, - "image_digest": "aa40e13f0cf3", - "walk": [ - [ - 2, - 145706.0, - 1.458, - 6.383 - ], - [ - 4, - 146135.0, - 2.589, - 14.735 - ] - ], - "pinned_pods": 2, - "throughput": 139270.0, - "p50": 1.44, - "p99": 7.987, - "pod_mem_mb": 270, - "saturated": true, - "status": "ok", - "reason": "plateau", - "pod_mem_p50_mb": 230 - } - } -} \ No newline at end of file diff --git a/results/wal-mem-diag-local/report.md b/results/wal-mem-diag-local/report.md deleted file mode 100644 index 812d1b5..0000000 --- a/results/wal-mem-diag-local/report.md +++ /dev/null @@ -1,43 +0,0 @@ -# wal-mem-diag-local — write-throughput report - -## Throughput at saturation (ops/s) - -| streams | wal-syncfs | memory | -|---|---|---| -| 20000 | 74k† | 170k† | -| 50000 | 23k | 139k | - -† = not saturated (ladder exhausted) — treat as a lower bound. - -## Pod memory at saturation — peak / p50 (MiB) - -| streams | wal-syncfs | memory | -|---|---|---| -| 20000 | 133 / 106 | 127 / 117 | -| 50000 | 243 / 226 | 270 / 230 | - -_Pod working set = cgroup `memory.current − inactive_file` (anon + active page cache), sampled each second at the pinned rung. **peak** = high-water (catches bursts like an in-RAM Raft log filling); **p50** = median (what the server steadily holds resident). peak ≈ p50 ⇒ steadily resident; peak ≫ p50 ⇒ transient spikes._ - -## Latency (ms, p50 / p99) - -| streams | wal-syncfs @≤80% load | wal-syncfs @saturation | memory @≤80% load | memory @saturation | -|---|---|---|---|---| -| 20000 | 3.5 / 11.3 (63k @2p) | — | 1.5 / 7.2 (137k @2p) | — | -| 50000 | 4.1 / 39.5 (43k @2p) | 6.3 / 146.9 | 1.5 / 6.4 (146k @2p) | 1.4 / 8.0 | - -_@≤80% load = the largest ladder rung at ≤80% of peak throughput — the server's service latency with headroom. @saturation = the pinned plateau rung, where a closed-loop fleet measures its own queueing (≈ in-flight ÷ ceiling by Little's law), NOT the server's per-request cost. Compare against the unloaded single-request baseline (~1 ms for wal) before reading anything into large saturation values._ - -## Saturation walks (pods → ops/s, p50 ms) - -- **memory 20000**: 2:137k@1.5ms → 4:170k@2.2ms (pinned 4, ladder_exhausted) -- **wal-syncfs 20000**: 2:63k@3.5ms → 4:74k@5.8ms (pinned 4, ladder_exhausted) -- **memory 50000**: 2:146k@1.5ms → 4:146k@2.6ms (pinned 2, plateau) -- **wal-syncfs 50000**: 2:43k@4.1ms → 4:37k@8.8ms (pinned 2, plateau) - -## Findings - -_TODO: written by hand on top of the generated data._ - -## Caveats - -_Single-node best-case; not 3-node Raft. Throughput is a saturation ceiling per the ladder._ diff --git a/results/wal-mem-diag-local/wal-syncfs/cells.json b/results/wal-mem-diag-local/wal-syncfs/cells.json deleted file mode 100644 index 55ca6b7..0000000 --- a/results/wal-mem-diag-local/wal-syncfs/cells.json +++ /dev/null @@ -1,58 +0,0 @@ -{ - "cells": { - "20000": { - "stream_count": 20000, - "image_digest": "1ee33d6c2b18", - "walk": [ - [ - 2, - 62928.875, - 3.463, - 11.343 - ], - [ - 4, - 73717.25, - 5.839, - 14.951 - ] - ], - "pinned_pods": 4, - "throughput": 73717.25, - "p50": null, - "p99": null, - "pod_mem_mb": 133, - "saturated": false, - "status": "ok", - "reason": "ladder_exhausted", - "pod_mem_p50_mb": 106 - }, - "50000": { - "stream_count": 50000, - "image_digest": "1ee33d6c2b18", - "walk": [ - [ - 2, - 43436.625, - 4.075, - 39.519 - ], - [ - 4, - 36744.125, - 8.759, - 149.375 - ] - ], - "pinned_pods": 2, - "throughput": 23309.2, - "p50": 6.339, - "p99": 146.943, - "pod_mem_mb": 243, - "saturated": true, - "status": "ok", - "reason": "plateau", - "pod_mem_p50_mb": 226 - } - } -} \ No newline at end of file diff --git a/results/wal-multilane-shards/s1/cells.json b/results/wal-multilane-shards/s1/cells.json deleted file mode 100644 index 68eed15..0000000 --- a/results/wal-multilane-shards/s1/cells.json +++ /dev/null @@ -1,170 +0,0 @@ -{ - "cells": { - "200000": { - "stream_count": 200000, - "image_digest": "eb1a9e1707f7", - "walk": [ - [ - 2, - 13302.0, - 6.287, - 404.735 - ], - [ - 4, - 8562.849999999999, - 93.183, - 667.135 - ], - [ - 8, - 4774.500000000001, - 404.991, - 1549.311 - ] - ], - "pinned_pods": 2, - "throughput": 9914.3, - "p50": 56.799, - "p99": 255.231, - "pod_mem_mb": 0.0, - "saturated": true, - "status": "ok", - "reason": "plateau" - }, - "100": { - "stream_count": 100, - "image_digest": "eb1a9e1707f7", - "walk": [ - [ - 2, - 279181.1, - 1.505, - 5.343 - ], - [ - 4, - 267247.7, - 3.117, - 11.215 - ], - [ - 8, - 238230.0, - 6.571, - 28.543 - ] - ], - "pinned_pods": 2, - "throughput": 245088.0, - "p50": 1.515, - "p99": 6.599, - "pod_mem_mb": 1743.0, - "saturated": true, - "status": "ok", - "reason": "plateau" - }, - "1000": { - "stream_count": 1000, - "image_digest": "eb1a9e1707f7", - "walk": [ - [ - 2, - 152025.05, - 3.041, - 8.991 - ], - [ - 4, - 169909.3, - 5.587, - 15.935 - ], - [ - 8, - 151581.9, - 12.967, - 26.495 - ], - [ - 12, - 137728.9, - 21.135, - 50.015 - ] - ], - "pinned_pods": 4, - "throughput": 166667.0, - "p50": 5.863, - "p99": 15.455, - "pod_mem_mb": 0.0, - "saturated": true, - "status": "ok", - "reason": "plateau" - }, - "10000": { - "stream_count": 10000, - "image_digest": "eb1a9e1707f7", - "walk": [ - [ - 2, - 104758.45000000001, - 4.631, - 17.631 - ], - [ - 4, - 95399.15, - 9.255, - 33.343 - ], - [ - 8, - 100706.70000000001, - 19.823, - 48.543 - ] - ], - "pinned_pods": 2, - "throughput": 96294.4, - "p50": 3.627, - "p99": 27.215, - "pod_mem_mb": 0.0, - "saturated": true, - "status": "ok", - "reason": "plateau" - }, - "100000": { - "stream_count": 100000, - "image_digest": "eb1a9e1707f7", - "walk": [ - [ - 2, - 11471.400000000001, - 43.263, - 143.871 - ], - [ - 4, - 8124.65, - 124.351, - 436.991 - ], - [ - 8, - 7367.45, - 268.287, - 979.455 - ] - ], - "pinned_pods": 2, - "throughput": 12871.7, - "p50": 4.323, - "p99": 444.671, - "pod_mem_mb": 0.0, - "saturated": true, - "status": "ok", - "reason": "plateau" - } - } -} \ No newline at end of file diff --git a/results/wal-multilane-shards/s2/cells.json b/results/wal-multilane-shards/s2/cells.json deleted file mode 100644 index 7ebb0ca..0000000 --- a/results/wal-multilane-shards/s2/cells.json +++ /dev/null @@ -1,138 +0,0 @@ -{ - "cells": { - "100": { - "stream_count": 100, - "image_digest": "aa0b738cc42c", - "walk": [ - [ - 2, - 249622.40000000002, - 1.381, - 9.367 - ], - [ - 4, - 218285.55, - 3.021, - 24.159 - ], - [ - 8, - 248984.59999999998, - 6.079, - 24.623 - ], - [ - 12, - 248463.90000000002, - 9.255, - 38.175 - ] - ], - "pinned_pods": 12, - "throughput": 248463.90000000002, - "p50": null, - "p99": null, - "pod_mem_mb": 2031.0, - "saturated": false, - "status": "ok", - "reason": "ladder_exhausted" - }, - "1000": { - "stream_count": 1000, - "image_digest": "aa0b738cc42c", - "walk": [ - [ - 2, - 215026.25, - 1.794, - 8.107 - ], - [ - 4, - 196914.45, - 4.069, - 14.991 - ], - [ - 8, - 208946.0, - 8.239, - 22.703 - ] - ], - "pinned_pods": 2, - "throughput": 217378.0, - "p50": 1.733, - "p99": 8.295, - "pod_mem_mb": 0.0, - "saturated": true, - "status": "ok", - "reason": "plateau" - }, - "10000": { - "stream_count": 10000, - "image_digest": "aa0b738cc42c", - "walk": [ - [ - 2, - 103985.20000000001, - 3.975, - 21.247 - ], - [ - 4, - 102119.7, - 8.903, - 35.935 - ], - [ - 8, - 107801.40000000001, - 17.455, - 58.175 - ] - ], - "pinned_pods": 2, - "throughput": 110716.0, - "p50": 2.729, - "p99": 23.455, - "pod_mem_mb": 0.0, - "saturated": true, - "status": "ok", - "reason": "plateau" - }, - "100000": { - "stream_count": 100000, - "image_digest": "aa0b738cc42c", - "walk": [ - [ - 2, - 10436.35, - 54.015, - 156.799 - ], - [ - 4, - 8514.15, - 96.639, - 829.951 - ], - [ - 8, - 6386.4, - 325.887, - 1096.703 - ] - ], - "pinned_pods": 2, - "throughput": 22756.7, - "p50": 2.083, - "p99": 229.119, - "pod_mem_mb": 0.0, - "saturated": true, - "status": "ok", - "reason": "plateau" - } - } -} \ No newline at end of file diff --git a/results/wal-multilane-shards/s4/cells.json b/results/wal-multilane-shards/s4/cells.json deleted file mode 100644 index b13439b..0000000 --- a/results/wal-multilane-shards/s4/cells.json +++ /dev/null @@ -1,144 +0,0 @@ -{ - "cells": { - "100": { - "stream_count": 100, - "image_digest": "3fde14045b8f", - "walk": [ - [ - 2, - 195818.0, - 1.395, - 15.999 - ], - [ - 4, - 254600.09999999998, - 2.915, - 22.063 - ], - [ - 8, - 222623.75, - 6.215, - 50.079 - ], - [ - 12, - 236353.5, - 9.383, - 52.799 - ] - ], - "pinned_pods": 4, - "throughput": 209048.0, - "p50": 2.957, - "p99": 28.543, - "pod_mem_mb": 0.0, - "saturated": true, - "status": "ok", - "reason": "plateau" - }, - "1000": { - "stream_count": 1000, - "image_digest": "3fde14045b8f", - "walk": [ - [ - 2, - 221033.25, - 1.608, - 11.047 - ], - [ - 4, - 226656.85, - 3.517, - 14.039 - ], - [ - 8, - 203777.15, - 7.171, - 42.911 - ] - ], - "pinned_pods": 2, - "throughput": 239381.0, - "p50": 1.576, - "p99": 8.919, - "pod_mem_mb": 0.0, - "saturated": true, - "status": "ok", - "reason": "plateau" - }, - "10000": { - "stream_count": 10000, - "image_digest": "3fde14045b8f", - "walk": [ - [ - 2, - 98871.45000000001, - 3.565, - 27.151 - ], - [ - 4, - 109948.79999999999, - 7.307, - 35.423 - ], - [ - 8, - 106981.80000000002, - 17.599, - 55.199 - ], - [ - 12, - 108550.0, - 27.999, - 69.055 - ] - ], - "pinned_pods": 4, - "throughput": 104960.0, - "p50": 8.607, - "p99": 33.407, - "pod_mem_mb": 0.0, - "saturated": true, - "status": "ok", - "reason": "plateau" - }, - "100000": { - "stream_count": 100000, - "image_digest": "3fde14045b8f", - "walk": [ - [ - 2, - 10807.3, - 44.895, - 157.951 - ], - [ - 4, - 9755.95, - 98.175, - 287.999 - ], - [ - 8, - 6471.650000000001, - 313.855, - 696.319 - ] - ], - "pinned_pods": 2, - "throughput": 13678.5, - "p50": 33.023, - "p99": 201.087, - "pod_mem_mb": 0.0, - "saturated": true, - "status": "ok", - "reason": "plateau" - } - } -} \ No newline at end of file diff --git a/results/wal-multilane-shards/s6/cells.json b/results/wal-multilane-shards/s6/cells.json deleted file mode 100644 index 8456f72..0000000 --- a/results/wal-multilane-shards/s6/cells.json +++ /dev/null @@ -1,36 +0,0 @@ -{ - "cells": { - "100": { - "stream_count": 100, - "image_digest": "e4ae264d73e0", - "walk": [ - [ - 2, - 223352.25, - 1.377, - 15.535 - ], - [ - 4, - 213347.15, - 2.799, - 35.999 - ], - [ - 8, - 225984.35, - 6.023, - 49.695 - ] - ], - "pinned_pods": 2, - "throughput": 211132.0, - "p50": 1.357, - "p99": 15.255, - "pod_mem_mb": 0.0, - "saturated": true, - "status": "ok", - "reason": "plateau" - } - } -} \ No newline at end of file diff --git a/results/wal-shard-sweep/aggregate.csv b/results/wal-shard-sweep/aggregate.csv deleted file mode 100644 index 1c88b7c..0000000 --- a/results/wal-shard-sweep/aggregate.csv +++ /dev/null @@ -1,6 +0,0 @@ -mode,stream_count,pods,throughput,p50,p99,knee_pods,knee_throughput,knee_p50,knee_p99,pod_mem_mb,pod_mem_p50_mb,saturated,status,reason -wal-s1,200000,8,66183.7,27.807,96.191,1,54072.85,4.179,21.679,1088,1055,True,ok,plateau -wal-s16,200000,8,71106.2,28.575,59.103,2,48790.9,10.223,21.247,1073,1048,True,ok,plateau -wal-s24,200000,8,64311.2,30.319,66.239,4,51795.95,18.815,46.047,1100,1063,True,ok,plateau -wal-s4,200000,8,66530.6,27.647,84.735,4,60279.2,14.743,50.495,1092,1076,True,ok,plateau -wal-s8,200000,4,67487.9,14.599,40.223,2,55098.649999999994,8.367,31.855,1041,1020,True,ok,plateau diff --git a/results/wal-shard-sweep/aggregate.json b/results/wal-shard-sweep/aggregate.json deleted file mode 100644 index db040ce..0000000 --- a/results/wal-shard-sweep/aggregate.json +++ /dev/null @@ -1,271 +0,0 @@ -[ - { - "mode": "wal-s1", - "stream_count": 200000, - "pods": 8, - "throughput": 66183.7, - "p50": 27.807, - "p99": 96.191, - "knee_pods": 1, - "knee_throughput": 54072.85, - "knee_p50": 4.179, - "knee_p99": 21.679, - "pod_mem_mb": 1088, - "pod_mem_p50_mb": 1055, - "saturated": true, - "status": "ok", - "reason": "plateau", - "walk": [ - [ - 1, - 54072.85, - 4.179, - 21.679 - ], - [ - 2, - 58946.05, - 7.435, - 38.623 - ], - [ - 4, - 62244.65000000001, - 14.063, - 65.215 - ], - [ - 8, - 72061.7, - 27.647, - 78.591 - ], - [ - 12, - 67940.1, - 41.535, - 130.687 - ], - [ - 16, - 67103.4, - 55.039, - 155.391 - ] - ] - }, - { - "mode": "wal-s16", - "stream_count": 200000, - "pods": 8, - "throughput": 71106.2, - "p50": 28.575, - "p99": 59.103, - "knee_pods": 2, - "knee_throughput": 48790.9, - "knee_p50": 10.223, - "knee_p99": 21.247, - "pod_mem_mb": 1073, - "pod_mem_p50_mb": 1048, - "saturated": true, - "status": "ok", - "reason": "plateau", - "walk": [ - [ - 1, - 34123.9, - 7.051, - 23.791 - ], - [ - 2, - 48790.9, - 10.223, - 21.247 - ], - [ - 4, - 60155.950000000004, - 16.351, - 39.263 - ], - [ - 8, - 68474.2, - 28.831, - 67.647 - ], - [ - 12, - 71914.3, - 42.111, - 80.767 - ], - [ - 16, - 72954.3, - 55.135, - 108.735 - ] - ] - }, - { - "mode": "wal-s24", - "stream_count": 200000, - "pods": 8, - "throughput": 64311.2, - "p50": 30.319, - "p99": 66.239, - "knee_pods": 4, - "knee_throughput": 51795.95, - "knee_p50": 18.815, - "knee_p99": 46.047, - "pod_mem_mb": 1100, - "pod_mem_p50_mb": 1063, - "saturated": true, - "status": "ok", - "reason": "plateau", - "walk": [ - [ - 1, - 34123.65, - 7.123, - 23.087 - ], - [ - 2, - 42837.75, - 11.415, - 29.583 - ], - [ - 4, - 51795.95, - 18.815, - 46.047 - ], - [ - 8, - 67258.7, - 29.983, - 53.983 - ], - [ - 12, - 68259.4, - 43.615, - 85.503 - ], - [ - 16, - 71206.24999999999, - 56.351, - 102.463 - ] - ] - }, - { - "mode": "wal-s4", - "stream_count": 200000, - "pods": 8, - "throughput": 66530.6, - "p50": 27.647, - "p99": 84.735, - "knee_pods": 4, - "knee_throughput": 60279.2, - "knee_p50": 14.743, - "knee_p99": 50.495, - "pod_mem_mb": 1092, - "pod_mem_p50_mb": 1076, - "saturated": true, - "status": "ok", - "reason": "plateau", - "walk": [ - [ - 1, - 48947.0, - 4.747, - 26.319 - ], - [ - 2, - 56331.65, - 7.943, - 32.047 - ], - [ - 4, - 60279.2, - 14.743, - 50.495 - ], - [ - 8, - 69483.9, - 27.871, - 73.983 - ], - [ - 12, - 70896.50000000001, - 41.279, - 95.039 - ], - [ - 16, - 75355.6, - 53.919, - 104.447 - ] - ] - }, - { - "mode": "wal-s8", - "stream_count": 200000, - "pods": 4, - "throughput": 67487.9, - "p50": 14.599, - "p99": 40.223, - "knee_pods": 2, - "knee_throughput": 55098.649999999994, - "knee_p50": 8.367, - "knee_p99": 31.855, - "pod_mem_mb": 1041, - "pod_mem_p50_mb": 1020, - "saturated": true, - "status": "ok", - "reason": "plateau", - "walk": [ - [ - 1, - 41781.65, - 5.651, - 21.247 - ], - [ - 2, - 55098.649999999994, - 8.367, - 31.855 - ], - [ - 4, - 67617.15, - 14.575, - 37.951 - ], - [ - 8, - 72084.95, - 27.791, - 59.871 - ], - [ - 12, - 73472.9, - 41.567, - 78.143 - ] - ] - } -] \ No newline at end of file diff --git a/results/wal-shard-sweep/report.md b/results/wal-shard-sweep/report.md deleted file mode 100644 index 9d87394..0000000 --- a/results/wal-shard-sweep/report.md +++ /dev/null @@ -1,41 +0,0 @@ -# wal-shard-sweep — write-throughput report - -## Throughput at saturation (ops/s) - -| streams | wal-s1 | wal-s4 | wal-s8 | wal-s16 | wal-s24 | -|---|---|---|---|---|---| -| 200000 | 66k | 67k | 67k | 71k | 64k | - -† = not saturated (ladder exhausted) — treat as a lower bound. - -## Pod memory at saturation — peak / p50 (MiB) - -| streams | wal-s1 | wal-s4 | wal-s8 | wal-s16 | wal-s24 | -|---|---|---|---|---|---| -| 200000 | 1088 / 1055 | 1092 / 1076 | 1041 / 1020 | 1073 / 1048 | 1100 / 1063 | - -_Pod working set = cgroup `memory.current − inactive_file` (anon + active page cache), sampled each second at the pinned rung. **peak** = high-water (catches bursts like an in-RAM Raft log filling); **p50** = median (what the server steadily holds resident). peak ≈ p50 ⇒ steadily resident; peak ≫ p50 ⇒ transient spikes._ - -## Latency (ms, p50 / p99) - -| streams | wal-s1 @≤80% load | wal-s1 @saturation | wal-s4 @≤80% load | wal-s4 @saturation | wal-s8 @≤80% load | wal-s8 @saturation | wal-s16 @≤80% load | wal-s16 @saturation | wal-s24 @≤80% load | wal-s24 @saturation | -|---|---|---|---|---|---|---|---|---|---|---| -| 200000 | 4.2 / 21.7 (54k @1p) | 27.8 / 96.2 | 14.7 / 50.5 (60k @4p) | 27.6 / 84.7 | 8.4 / 31.9 (55k @2p) | 14.6 / 40.2 | 10.2 / 21.2 (49k @2p) | 28.6 / 59.1 | 18.8 / 46.0 (52k @4p) | 30.3 / 66.2 | - -_@≤80% load = the largest ladder rung at ≤80% of peak throughput — the server's service latency with headroom. @saturation = the pinned plateau rung, where a closed-loop fleet measures its own queueing (≈ in-flight ÷ ceiling by Little's law), NOT the server's per-request cost. Compare against the unloaded single-request baseline (~1 ms for wal) before reading anything into large saturation values._ - -## Saturation walks (pods → ops/s, p50 ms) - -- **wal-s1 200000**: 1:54k@4.2ms → 2:59k@7.4ms → 4:62k@14.1ms → 8:72k@27.6ms → 12:68k@41.5ms → 16:67k@55.0ms (pinned 8, plateau) -- **wal-s16 200000**: 1:34k@7.1ms → 2:49k@10.2ms → 4:60k@16.4ms → 8:68k@28.8ms → 12:72k@42.1ms → 16:73k@55.1ms (pinned 8, plateau) -- **wal-s24 200000**: 1:34k@7.1ms → 2:43k@11.4ms → 4:52k@18.8ms → 8:67k@30.0ms → 12:68k@43.6ms → 16:71k@56.4ms (pinned 8, plateau) -- **wal-s4 200000**: 1:49k@4.7ms → 2:56k@7.9ms → 4:60k@14.7ms → 8:69k@27.9ms → 12:71k@41.3ms → 16:75k@53.9ms (pinned 8, plateau) -- **wal-s8 200000**: 1:42k@5.7ms → 2:55k@8.4ms → 4:68k@14.6ms → 8:72k@27.8ms → 12:73k@41.6ms (pinned 4, plateau) - -## Findings - -_TODO: written by hand on top of the generated data._ - -## Caveats - -_Single-node best-case; not 3-node Raft. Throughput is a saturation ceiling per the ladder._ diff --git a/results/wal-shard-sweep/wal-s1/cells.json b/results/wal-shard-sweep/wal-s1/cells.json deleted file mode 100644 index ab90d3a..0000000 --- a/results/wal-shard-sweep/wal-s1/cells.json +++ /dev/null @@ -1,55 +0,0 @@ -{ - "cells": { - "200000": { - "stream_count": 200000, - "image_digest": "eb1a9e1707f7", - "walk": [ - [ - 1, - 54072.85, - 4.179, - 21.679 - ], - [ - 2, - 58946.05, - 7.435, - 38.623 - ], - [ - 4, - 62244.65000000001, - 14.063, - 65.215 - ], - [ - 8, - 72061.7, - 27.647, - 78.591 - ], - [ - 12, - 67940.1, - 41.535, - 130.687 - ], - [ - 16, - 67103.4, - 55.039, - 155.391 - ] - ], - "pinned_pods": 8, - "throughput": 66183.7, - "p50": 27.807, - "p99": 96.191, - "pod_mem_mb": 1088, - "saturated": true, - "status": "ok", - "reason": "plateau", - "pod_mem_p50_mb": 1055 - } - } -} \ No newline at end of file diff --git a/results/wal-shard-sweep/wal-s16/cells.json b/results/wal-shard-sweep/wal-s16/cells.json deleted file mode 100644 index bc6fee6..0000000 --- a/results/wal-shard-sweep/wal-s16/cells.json +++ /dev/null @@ -1,55 +0,0 @@ -{ - "cells": { - "200000": { - "stream_count": 200000, - "image_digest": "ac15eab20c5a", - "walk": [ - [ - 1, - 34123.9, - 7.051, - 23.791 - ], - [ - 2, - 48790.9, - 10.223, - 21.247 - ], - [ - 4, - 60155.950000000004, - 16.351, - 39.263 - ], - [ - 8, - 68474.2, - 28.831, - 67.647 - ], - [ - 12, - 71914.3, - 42.111, - 80.767 - ], - [ - 16, - 72954.3, - 55.135, - 108.735 - ] - ], - "pinned_pods": 8, - "throughput": 71106.2, - "p50": 28.575, - "p99": 59.103, - "pod_mem_mb": 1073, - "saturated": true, - "status": "ok", - "reason": "plateau", - "pod_mem_p50_mb": 1048 - } - } -} \ No newline at end of file diff --git a/results/wal-shard-sweep/wal-s24/cells.json b/results/wal-shard-sweep/wal-s24/cells.json deleted file mode 100644 index 5670324..0000000 --- a/results/wal-shard-sweep/wal-s24/cells.json +++ /dev/null @@ -1,55 +0,0 @@ -{ - "cells": { - "200000": { - "stream_count": 200000, - "image_digest": "bd1859edf336", - "walk": [ - [ - 1, - 34123.65, - 7.123, - 23.087 - ], - [ - 2, - 42837.75, - 11.415, - 29.583 - ], - [ - 4, - 51795.95, - 18.815, - 46.047 - ], - [ - 8, - 67258.7, - 29.983, - 53.983 - ], - [ - 12, - 68259.4, - 43.615, - 85.503 - ], - [ - 16, - 71206.24999999999, - 56.351, - 102.463 - ] - ], - "pinned_pods": 8, - "throughput": 64311.2, - "p50": 30.319, - "p99": 66.239, - "pod_mem_mb": 1100, - "saturated": true, - "status": "ok", - "reason": "plateau", - "pod_mem_p50_mb": 1063 - } - } -} \ No newline at end of file diff --git a/results/wal-shard-sweep/wal-s4/cells.json b/results/wal-shard-sweep/wal-s4/cells.json deleted file mode 100644 index f7dd372..0000000 --- a/results/wal-shard-sweep/wal-s4/cells.json +++ /dev/null @@ -1,55 +0,0 @@ -{ - "cells": { - "200000": { - "stream_count": 200000, - "image_digest": "3fde14045b8f", - "walk": [ - [ - 1, - 48947.0, - 4.747, - 26.319 - ], - [ - 2, - 56331.65, - 7.943, - 32.047 - ], - [ - 4, - 60279.2, - 14.743, - 50.495 - ], - [ - 8, - 69483.9, - 27.871, - 73.983 - ], - [ - 12, - 70896.50000000001, - 41.279, - 95.039 - ], - [ - 16, - 75355.6, - 53.919, - 104.447 - ] - ], - "pinned_pods": 8, - "throughput": 66530.6, - "p50": 27.647, - "p99": 84.735, - "pod_mem_mb": 1092, - "saturated": true, - "status": "ok", - "reason": "plateau", - "pod_mem_p50_mb": 1076 - } - } -} \ No newline at end of file diff --git a/results/wal-shard-sweep/wal-s8/cells.json b/results/wal-shard-sweep/wal-s8/cells.json deleted file mode 100644 index 2e0298d..0000000 --- a/results/wal-shard-sweep/wal-s8/cells.json +++ /dev/null @@ -1,49 +0,0 @@ -{ - "cells": { - "200000": { - "stream_count": 200000, - "image_digest": "1c7d5a766789", - "walk": [ - [ - 1, - 41781.65, - 5.651, - 21.247 - ], - [ - 2, - 55098.649999999994, - 8.367, - 31.855 - ], - [ - 4, - 67617.15, - 14.575, - 37.951 - ], - [ - 8, - 72084.95, - 27.791, - 59.871 - ], - [ - 12, - 73472.9, - 41.567, - 78.143 - ] - ], - "pinned_pods": 4, - "throughput": 67487.9, - "p50": 14.599, - "p99": 40.223, - "pod_mem_mb": 1041, - "saturated": true, - "status": "ok", - "reason": "plateau", - "pod_mem_p50_mb": 1020 - } - } -} \ No newline at end of file diff --git a/results/wal-sizetrigger/aggregate.csv b/results/wal-sizetrigger/aggregate.csv deleted file mode 100644 index 614c284..0000000 --- a/results/wal-sizetrigger/aggregate.csv +++ /dev/null @@ -1,5 +0,0 @@ -mode,stream_count,pods,throughput,p50,p99,knee_pods,knee_throughput,knee_p50,knee_p99,pod_mem_mb,pod_mem_p50_mb,saturated,status,reason -ref-3s,10000,8,298060.2,,,4,298060.2,3.149,8.311,236,183,False,ok,ladder_exhausted -size-1g,10000,8,314005.9,,,4,314005.9,3.083,7.475,294,186,False,ok,ladder_exhausted -ref-3s,100000,2,275252.0,1.479,6.383,2,268921.9,1.5,6.163,663,591,True,ok,plateau -size-1g,100000,2,303152.0,1.491,6.095,2,296213.65,1.479,5.843,693,600,True,ok,plateau diff --git a/results/wal-sizetrigger/aggregate.json b/results/wal-sizetrigger/aggregate.json deleted file mode 100644 index bd56349..0000000 --- a/results/wal-sizetrigger/aggregate.json +++ /dev/null @@ -1,138 +0,0 @@ -[ - { - "mode": "ref-3s", - "stream_count": 10000, - "pods": 8, - "throughput": 298060.2, - "p50": null, - "p99": null, - "knee_pods": 4, - "knee_throughput": 298060.2, - "knee_p50": 3.149, - "knee_p99": 8.311, - "pod_mem_mb": 236, - "pod_mem_p50_mb": 183, - "saturated": false, - "status": "ok", - "reason": "ladder_exhausted", - "walk": [ - [ - 4, - 298060.2, - 3.149, - 8.311 - ], - [ - 8, - 293881.35000000003, - 6.519, - 21.375 - ] - ] - }, - { - "mode": "size-1g", - "stream_count": 10000, - "pods": 8, - "throughput": 314005.9, - "p50": null, - "p99": null, - "knee_pods": 4, - "knee_throughput": 314005.9, - "knee_p50": 3.083, - "knee_p99": 7.475, - "pod_mem_mb": 294, - "pod_mem_p50_mb": 186, - "saturated": false, - "status": "ok", - "reason": "ladder_exhausted", - "walk": [ - [ - 4, - 314005.9, - 3.083, - 7.475 - ], - [ - 8, - 309625.25, - 6.359, - 11.623 - ] - ] - }, - { - "mode": "ref-3s", - "stream_count": 100000, - "pods": 2, - "throughput": 275252.0, - "p50": 1.479, - "p99": 6.383, - "knee_pods": 2, - "knee_throughput": 268921.9, - "knee_p50": 1.5, - "knee_p99": 6.163, - "pod_mem_mb": 663, - "pod_mem_p50_mb": 591, - "saturated": true, - "status": "ok", - "reason": "plateau", - "walk": [ - [ - 2, - 268921.9, - 1.5, - 6.163 - ], - [ - 4, - 272068.4, - 3.137, - 7.847 - ], - [ - 8, - 269593.05, - 6.511, - 12.583 - ] - ] - }, - { - "mode": "size-1g", - "stream_count": 100000, - "pods": 2, - "throughput": 303152.0, - "p50": 1.491, - "p99": 6.095, - "knee_pods": 2, - "knee_throughput": 296213.65, - "knee_p50": 1.479, - "knee_p99": 5.843, - "pod_mem_mb": 693, - "pod_mem_p50_mb": 600, - "saturated": true, - "status": "ok", - "reason": "plateau", - "walk": [ - [ - 2, - 296213.65, - 1.479, - 5.843 - ], - [ - 4, - 292395.45, - 3.127, - 7.535 - ], - [ - 8, - 289558.95, - 6.395, - 11.727 - ] - ] - } -] \ No newline at end of file diff --git a/results/wal-sizetrigger/ref-3s/cells.json b/results/wal-sizetrigger/ref-3s/cells.json deleted file mode 100644 index e9b33c4..0000000 --- a/results/wal-sizetrigger/ref-3s/cells.json +++ /dev/null @@ -1,64 +0,0 @@ -{ - "cells": { - "10000": { - "stream_count": 10000, - "image_digest": "a2298d6c8011", - "walk": [ - [ - 4, - 298060.2, - 3.149, - 8.311 - ], - [ - 8, - 293881.35000000003, - 6.519, - 21.375 - ] - ], - "pinned_pods": 8, - "throughput": 293881.35000000003, - "p50": null, - "p99": null, - "pod_mem_mb": 236, - "saturated": false, - "status": "ok", - "reason": "ladder_exhausted", - "pod_mem_p50_mb": 183 - }, - "100000": { - "stream_count": 100000, - "image_digest": "a2298d6c8011", - "walk": [ - [ - 2, - 268921.9, - 1.5, - 6.163 - ], - [ - 4, - 272068.4, - 3.137, - 7.847 - ], - [ - 8, - 269593.05, - 6.511, - 12.583 - ] - ], - "pinned_pods": 2, - "throughput": 275252.0, - "p50": 1.479, - "p99": 6.383, - "pod_mem_mb": 663, - "saturated": true, - "status": "ok", - "reason": "plateau", - "pod_mem_p50_mb": 591 - } - } -} \ No newline at end of file diff --git a/results/wal-sizetrigger/report.md b/results/wal-sizetrigger/report.md deleted file mode 100644 index be733d2..0000000 --- a/results/wal-sizetrigger/report.md +++ /dev/null @@ -1,43 +0,0 @@ -# wal-sizetrigger — write-throughput report - -## Throughput at saturation (ops/s) - -| streams | ref-3s | size-1g | -|---|---|---| -| 10000 | 298k† | 314k† | -| 100000 | 275k | 303k | - -† = not saturated (ladder exhausted) — treat as a lower bound. - -## Pod memory at saturation — peak / p50 (MiB) - -| streams | ref-3s | size-1g | -|---|---|---| -| 10000 | 236 / 183 | 294 / 186 | -| 100000 | 663 / 591 | 693 / 600 | - -_Pod working set = cgroup `memory.current − inactive_file` (anon + active page cache), sampled each second at the pinned rung. **peak** = high-water (catches bursts like an in-RAM Raft log filling); **p50** = median (what the server steadily holds resident). peak ≈ p50 ⇒ steadily resident; peak ≫ p50 ⇒ transient spikes._ - -## Latency (ms, p50 / p99) - -| streams | ref-3s @≤80% load | ref-3s @saturation | size-1g @≤80% load | size-1g @saturation | -|---|---|---|---|---| -| 10000 | 3.1 / 8.3 (298k @4p) | — | 3.1 / 7.5 (314k @4p) | — | -| 100000 | 1.5 / 6.2 (269k @2p) | 1.5 / 6.4 | 1.5 / 5.8 (296k @2p) | 1.5 / 6.1 | - -_@≤80% load = the largest ladder rung at ≤80% of peak throughput — the server's service latency with headroom. @saturation = the pinned plateau rung, where a closed-loop fleet measures its own queueing (≈ in-flight ÷ ceiling by Little's law), NOT the server's per-request cost. Compare against the unloaded single-request baseline (~1 ms for wal) before reading anything into large saturation values._ - -## Saturation walks (pods → ops/s, p50 ms) - -- **ref-3s 10000**: 4:298k@3.1ms → 8:294k@6.5ms (pinned 8, ladder_exhausted) -- **size-1g 10000**: 4:314k@3.1ms → 8:310k@6.4ms (pinned 8, ladder_exhausted) -- **ref-3s 100000**: 2:269k@1.5ms → 4:272k@3.1ms → 8:270k@6.5ms (pinned 2, plateau) -- **size-1g 100000**: 2:296k@1.5ms → 4:292k@3.1ms → 8:290k@6.4ms (pinned 2, plateau) - -## Findings - -size-1g (--wal-checkpoint-wal-bytes 1GiB, 60s fallback) reaches the checkpoint-off ceiling: 309.6k @10k / 303.2k @100k vs ckpt-off 305k/306k (wal-splitlane) and ref-3s 293.9k/275.3k. The size trigger reclaims the entire 7-11% checkpoint cost while bounding crash-replay to <=1 GiB retained WAL per shard. Shipped as PR #4704 (stacked on #4697). - -## Caveats - -_Single-node best-case; not 3-node Raft. Throughput is a saturation ceiling per the ladder._ diff --git a/results/wal-sizetrigger/size-1g/cells.json b/results/wal-sizetrigger/size-1g/cells.json deleted file mode 100644 index cf73585..0000000 --- a/results/wal-sizetrigger/size-1g/cells.json +++ /dev/null @@ -1,64 +0,0 @@ -{ - "cells": { - "10000": { - "stream_count": 10000, - "image_digest": "2cfd37a9a235", - "walk": [ - [ - 4, - 314005.9, - 3.083, - 7.475 - ], - [ - 8, - 309625.25, - 6.359, - 11.623 - ] - ], - "pinned_pods": 8, - "throughput": 309625.25, - "p50": null, - "p99": null, - "pod_mem_mb": 294, - "saturated": false, - "status": "ok", - "reason": "ladder_exhausted", - "pod_mem_p50_mb": 186 - }, - "100000": { - "stream_count": 100000, - "image_digest": "2cfd37a9a235", - "walk": [ - [ - 2, - 296213.65, - 1.479, - 5.843 - ], - [ - 4, - 292395.45, - 3.127, - 7.535 - ], - [ - 8, - 289558.95, - 6.395, - 11.727 - ] - ], - "pinned_pods": 2, - "throughput": 303152.0, - "p50": 1.491, - "p99": 6.095, - "pod_mem_mb": 693, - "saturated": true, - "status": "ok", - "reason": "plateau", - "pod_mem_p50_mb": 600 - } - } -} \ No newline at end of file diff --git a/results/wal-splitlane/aggregate.csv b/results/wal-splitlane/aggregate.csv deleted file mode 100644 index 32a229a..0000000 --- a/results/wal-splitlane/aggregate.csv +++ /dev/null @@ -1,7 +0,0 @@ -mode,stream_count,pods,throughput,p50,p99,knee_pods,knee_throughput,knee_p50,knee_p99,pod_mem_mb,pod_mem_p50_mb,saturated,status,reason -ckpt-off,10000,8,309316.80000000005,,,4,309316.80000000005,3.129,7.943,291,204,False,ok,ladder_exhausted -nofsync,10000,8,269793.2,,,4,260584.6,3.135,8.831,233,190,False,ok,ladder_exhausted -ref-3s,10000,8,293223.55,,,4,293223.55,3.135,16.447,229,176,False,ok,ladder_exhausted -ckpt-off,100000,2,305989.0,1.478,6.599,2,296232.25,1.518,6.399,689,598,True,ok,plateau -nofsync,100000,2,263776.0,1.43,7.523,2,259764.05,1.447,7.371,629,584,True,ok,plateau -ref-3s,100000,2,271574.0,1.512,6.531,2,269027.80000000005,1.513,6.671,657,580,True,ok,plateau diff --git a/results/wal-splitlane/aggregate.json b/results/wal-splitlane/aggregate.json deleted file mode 100644 index 5350f74..0000000 --- a/results/wal-splitlane/aggregate.json +++ /dev/null @@ -1,206 +0,0 @@ -[ - { - "mode": "ckpt-off", - "stream_count": 10000, - "pods": 8, - "throughput": 309316.80000000005, - "p50": null, - "p99": null, - "knee_pods": 4, - "knee_throughput": 309316.80000000005, - "knee_p50": 3.129, - "knee_p99": 7.943, - "pod_mem_mb": 291, - "pod_mem_p50_mb": 204, - "saturated": false, - "status": "ok", - "reason": "ladder_exhausted", - "walk": [ - [ - 4, - 309316.80000000005, - 3.129, - 7.943 - ], - [ - 8, - 305053.85, - 6.459, - 11.687 - ] - ] - }, - { - "mode": "nofsync", - "stream_count": 10000, - "pods": 8, - "throughput": 269793.2, - "p50": null, - "p99": null, - "knee_pods": 4, - "knee_throughput": 260584.6, - "knee_p50": 3.135, - "knee_p99": 8.831, - "pod_mem_mb": 233, - "pod_mem_p50_mb": 190, - "saturated": false, - "status": "ok", - "reason": "ladder_exhausted", - "walk": [ - [ - 4, - 260584.6, - 3.135, - 8.831 - ], - [ - 8, - 269793.2, - 6.555, - 13.343 - ] - ] - }, - { - "mode": "ref-3s", - "stream_count": 10000, - "pods": 8, - "throughput": 293223.55, - "p50": null, - "p99": null, - "knee_pods": 4, - "knee_throughput": 293223.55, - "knee_p50": 3.135, - "knee_p99": 16.447, - "pod_mem_mb": 229, - "pod_mem_p50_mb": 176, - "saturated": false, - "status": "ok", - "reason": "ladder_exhausted", - "walk": [ - [ - 4, - 293223.55, - 3.135, - 16.447 - ], - [ - 8, - 286282.1, - 6.523, - 27.487 - ] - ] - }, - { - "mode": "ckpt-off", - "stream_count": 100000, - "pods": 2, - "throughput": 305989.0, - "p50": 1.478, - "p99": 6.599, - "knee_pods": 2, - "knee_throughput": 296232.25, - "knee_p50": 1.518, - "knee_p99": 6.399, - "pod_mem_mb": 689, - "pod_mem_p50_mb": 598, - "saturated": true, - "status": "ok", - "reason": "plateau", - "walk": [ - [ - 2, - 296232.25, - 1.518, - 6.399 - ], - [ - 4, - 298473.25, - 3.145, - 8.399 - ], - [ - 8, - 293619.2, - 6.515, - 12.151 - ] - ] - }, - { - "mode": "nofsync", - "stream_count": 100000, - "pods": 2, - "throughput": 263776.0, - "p50": 1.43, - "p99": 7.523, - "knee_pods": 2, - "knee_throughput": 259764.05, - "knee_p50": 1.447, - "knee_p99": 7.371, - "pod_mem_mb": 629, - "pod_mem_p50_mb": 584, - "saturated": true, - "status": "ok", - "reason": "plateau", - "walk": [ - [ - 2, - 259764.05, - 1.447, - 7.371 - ], - [ - 4, - 258613.3, - 3.151, - 8.895 - ], - [ - 8, - 250507.55000000002, - 6.679, - 14.063 - ] - ] - }, - { - "mode": "ref-3s", - "stream_count": 100000, - "pods": 2, - "throughput": 271574.0, - "p50": 1.512, - "p99": 6.531, - "knee_pods": 2, - "knee_throughput": 269027.80000000005, - "knee_p50": 1.513, - "knee_p99": 6.671, - "pod_mem_mb": 657, - "pod_mem_p50_mb": 580, - "saturated": true, - "status": "ok", - "reason": "plateau", - "walk": [ - [ - 2, - 269027.80000000005, - 1.513, - 6.671 - ], - [ - 4, - 255747.45, - 3.289, - 8.639 - ], - [ - 8, - 270907.8, - 6.727, - 13.487 - ] - ] - } -] \ No newline at end of file diff --git a/results/wal-splitlane/ckpt-off/cells.json b/results/wal-splitlane/ckpt-off/cells.json deleted file mode 100644 index 2a8eead..0000000 --- a/results/wal-splitlane/ckpt-off/cells.json +++ /dev/null @@ -1,64 +0,0 @@ -{ - "cells": { - "10000": { - "stream_count": 10000, - "image_digest": "3cf478620244", - "walk": [ - [ - 4, - 309316.80000000005, - 3.129, - 7.943 - ], - [ - 8, - 305053.85, - 6.459, - 11.687 - ] - ], - "pinned_pods": 8, - "throughput": 305053.85, - "p50": null, - "p99": null, - "pod_mem_mb": 291, - "saturated": false, - "status": "ok", - "reason": "ladder_exhausted", - "pod_mem_p50_mb": 204 - }, - "100000": { - "stream_count": 100000, - "image_digest": "3cf478620244", - "walk": [ - [ - 2, - 296232.25, - 1.518, - 6.399 - ], - [ - 4, - 298473.25, - 3.145, - 8.399 - ], - [ - 8, - 293619.2, - 6.515, - 12.151 - ] - ], - "pinned_pods": 2, - "throughput": 305989.0, - "p50": 1.478, - "p99": 6.599, - "pod_mem_mb": 689, - "saturated": true, - "status": "ok", - "reason": "plateau", - "pod_mem_p50_mb": 598 - } - } -} \ No newline at end of file diff --git a/results/wal-splitlane/nofsync/cells.json b/results/wal-splitlane/nofsync/cells.json deleted file mode 100644 index 711ea1c..0000000 --- a/results/wal-splitlane/nofsync/cells.json +++ /dev/null @@ -1,64 +0,0 @@ -{ - "cells": { - "10000": { - "stream_count": 10000, - "image_digest": "8ecce2503136", - "walk": [ - [ - 4, - 260584.6, - 3.135, - 8.831 - ], - [ - 8, - 269793.2, - 6.555, - 13.343 - ] - ], - "pinned_pods": 8, - "throughput": 269793.2, - "p50": null, - "p99": null, - "pod_mem_mb": 233, - "saturated": false, - "status": "ok", - "reason": "ladder_exhausted", - "pod_mem_p50_mb": 190 - }, - "100000": { - "stream_count": 100000, - "image_digest": "8ecce2503136", - "walk": [ - [ - 2, - 259764.05, - 1.447, - 7.371 - ], - [ - 4, - 258613.3, - 3.151, - 8.895 - ], - [ - 8, - 250507.55000000002, - 6.679, - 14.063 - ] - ], - "pinned_pods": 2, - "throughput": 263776.0, - "p50": 1.43, - "p99": 7.523, - "pod_mem_mb": 629, - "saturated": true, - "status": "ok", - "reason": "plateau", - "pod_mem_p50_mb": 584 - } - } -} \ No newline at end of file diff --git a/results/wal-splitlane/ref-3s/cells.json b/results/wal-splitlane/ref-3s/cells.json deleted file mode 100644 index 0aeda68..0000000 --- a/results/wal-splitlane/ref-3s/cells.json +++ /dev/null @@ -1,64 +0,0 @@ -{ - "cells": { - "10000": { - "stream_count": 10000, - "image_digest": "1bd30dc4c80f", - "walk": [ - [ - 4, - 293223.55, - 3.135, - 16.447 - ], - [ - 8, - 286282.1, - 6.523, - 27.487 - ] - ], - "pinned_pods": 8, - "throughput": 286282.1, - "p50": null, - "p99": null, - "pod_mem_mb": 229, - "saturated": false, - "status": "ok", - "reason": "ladder_exhausted", - "pod_mem_p50_mb": 176 - }, - "100000": { - "stream_count": 100000, - "image_digest": "1bd30dc4c80f", - "walk": [ - [ - 2, - 269027.80000000005, - 1.513, - 6.671 - ], - [ - 4, - 255747.45, - 3.289, - 8.639 - ], - [ - 8, - 270907.8, - 6.727, - 13.487 - ] - ], - "pinned_pods": 2, - "throughput": 271574.0, - "p50": 1.512, - "p99": 6.531, - "pod_mem_mb": 657, - "saturated": true, - "status": "ok", - "reason": "plateau", - "pod_mem_p50_mb": 580 - } - } -} \ No newline at end of file diff --git a/results/wal-splitlane/report.md b/results/wal-splitlane/report.md deleted file mode 100644 index 916e6f7..0000000 --- a/results/wal-splitlane/report.md +++ /dev/null @@ -1,53 +0,0 @@ -# wal-splitlane — write-throughput report - -## Throughput at saturation (ops/s) - -| streams | ref-3s | ckpt-off | nofsync | -|---|---|---|---| -| 10000 | 293k† | 309k† | 270k† | -| 100000 | 272k | 306k | 264k | - -† = not saturated (ladder exhausted) — treat as a lower bound. - -## Pod memory at saturation — peak / p50 (MiB) - -| streams | ref-3s | ckpt-off | nofsync | -|---|---|---|---| -| 10000 | 229 / 176 | 291 / 204 | 233 / 190 | -| 100000 | 657 / 580 | 689 / 598 | 629 / 584 | - -_Pod working set = cgroup `memory.current − inactive_file` (anon + active page cache), sampled each second at the pinned rung. **peak** = high-water (catches bursts like an in-RAM Raft log filling); **p50** = median (what the server steadily holds resident). peak ≈ p50 ⇒ steadily resident; peak ≫ p50 ⇒ transient spikes._ - -## Latency (ms, p50 / p99) - -| streams | ref-3s @≤80% load | ref-3s @saturation | ckpt-off @≤80% load | ckpt-off @saturation | nofsync @≤80% load | nofsync @saturation | -|---|---|---|---|---|---|---| -| 10000 | 3.1 / 16.4 (293k @4p) | — | 3.1 / 7.9 (309k @4p) | — | 3.1 / 8.8 (261k @4p) | — | -| 100000 | 1.5 / 6.7 (269k @2p) | 1.5 / 6.5 | 1.5 / 6.4 (296k @2p) | 1.5 / 6.6 | 1.4 / 7.4 (260k @2p) | 1.4 / 7.5 | - -_@≤80% load = the largest ladder rung at ≤80% of peak throughput — the server's service latency with headroom. @saturation = the pinned plateau rung, where a closed-loop fleet measures its own queueing (≈ in-flight ÷ ceiling by Little's law), NOT the server's per-request cost. Compare against the unloaded single-request baseline (~1 ms for wal) before reading anything into large saturation values._ - -## Saturation walks (pods → ops/s, p50 ms) - -- **ckpt-off 10000**: 4:309k@3.1ms → 8:305k@6.5ms (pinned 8, ladder_exhausted) -- **nofsync 10000**: 4:261k@3.1ms → 8:270k@6.6ms (pinned 8, ladder_exhausted) -- **ref-3s 10000**: 4:293k@3.1ms → 8:286k@6.5ms (pinned 8, ladder_exhausted) -- **ckpt-off 100000**: 2:296k@1.5ms → 4:298k@3.1ms → 8:294k@6.5ms (pinned 2, plateau) -- **nofsync 100000**: 2:260k@1.4ms → 4:259k@3.2ms → 8:251k@6.7ms (pinned 2, plateau) -- **ref-3s 100000**: 2:269k@1.5ms → 4:256k@3.3ms → 8:271k@6.7ms (pinned 2, plateau) - -## Findings - -**The write cardinality cliff is eliminated.** ref-3s (syncfs checkpoint @3s, full durability) holds 286k→272k ops/s from 10k→100k streams (−5%) on the split-lane layout — vs 10.4k @100k where this investigation started (26×). - -Decomposition (with wal-decomp-lane0): -- **Storage layout is the #1 lever.** Streams on the PD boot disk → 10.4k; everything on one shared NVMe lane → 46k; streams on their own lane + WAL shards on dedicated lanes → 272k. The old "~1000 fdatasync/s device ceiling" was commit-vs-checkpoint device contention. -- **Commit fdatasync on dedicated lanes is free**: ckpt-off (306k) ≥ nofsync (264k) — group-commit amortizes better under fsync backpressure than the free-running no-fsync path. -- **Checkpoint @3s costs ~7–11%** on this layout (306k → 286k/272k). The per-shard size-trigger knob (perf/wal-checkpoint-sizetrigger) can reclaim most of it by checkpointing on a retained-WAL budget instead of a timer. -- Remaining gap to memory mode (512k) is ~1.9× = WAL machinery (staging/double-write), not fsync — future work: io_uring WAL writer seam, batched mark-written. - -Production recipe: multi-device NVMe instance, streams dir on its own device, one WAL shard per remaining device, `--wal-checkpoint-syncfs on`. - -## Caveats - -_Single-node best-case; not 3-node Raft. Throughput is a saturation ceiling per the ladder._ diff --git a/results/wal-stacked-1m/aggregate.csv b/results/wal-stacked-1m/aggregate.csv deleted file mode 100644 index 08a6660..0000000 --- a/results/wal-stacked-1m/aggregate.csv +++ /dev/null @@ -1,4 +0,0 @@ -mode,stream_count,pods,throughput,p50,p99,knee_pods,knee_throughput,knee_p50,knee_p99,pod_mem_mb,pod_mem_p50_mb,saturated,status,reason -stacked,100000,8,382727.89999999997,,,4,382727.89999999997,2.537,4.539,774,668,False,ok,ladder_exhausted -stacked,500000,8,244203.8,,,8,244203.8,5.763,26.543,2925,2551,False,ok,ladder_exhausted -stacked,1000000,8,56003.35,,,8,56003.35,13.919,1186.815,4939,3852,False,ok,ladder_exhausted diff --git a/results/wal-stacked-1m/aggregate.json b/results/wal-stacked-1m/aggregate.json deleted file mode 100644 index 40ad068..0000000 --- a/results/wal-stacked-1m/aggregate.json +++ /dev/null @@ -1,83 +0,0 @@ -[ - { - "mode": "stacked", - "stream_count": 100000, - "pods": 8, - "throughput": 382727.89999999997, - "p50": null, - "p99": null, - "knee_pods": 4, - "knee_throughput": 382727.89999999997, - "knee_p50": 2.537, - "knee_p99": 4.539, - "pod_mem_mb": 774, - "pod_mem_p50_mb": 668, - "saturated": false, - "status": "ok", - "reason": "ladder_exhausted", - "walk": [ - [ - 4, - 382727.89999999997, - 2.537, - 4.539 - ], - [ - 8, - 362346.35000000003, - 5.439, - 8.751 - ] - ] - }, - { - "mode": "stacked", - "stream_count": 500000, - "pods": 8, - "throughput": 244203.8, - "p50": null, - "p99": null, - "knee_pods": 8, - "knee_throughput": 244203.8, - "knee_p50": 5.763, - "knee_p99": 26.543, - "pod_mem_mb": 2925, - "pod_mem_p50_mb": 2551, - "saturated": false, - "status": "ok", - "reason": "ladder_exhausted", - "walk": [ - [ - 8, - 244203.8, - 5.763, - 26.543 - ] - ] - }, - { - "mode": "stacked", - "stream_count": 1000000, - "pods": 8, - "throughput": 56003.35, - "p50": null, - "p99": null, - "knee_pods": 8, - "knee_throughput": 56003.35, - "knee_p50": 13.919, - "knee_p99": 1186.815, - "pod_mem_mb": 4939, - "pod_mem_p50_mb": 3852, - "saturated": false, - "status": "ok", - "reason": "ladder_exhausted", - "walk": [ - [ - 8, - 56003.35, - 13.919, - 1186.815 - ] - ] - } -] \ No newline at end of file diff --git a/results/wal-stacked-1m/report.md b/results/wal-stacked-1m/report.md deleted file mode 100644 index 141b811..0000000 --- a/results/wal-stacked-1m/report.md +++ /dev/null @@ -1,47 +0,0 @@ -# wal-stacked-1m — write-throughput report - -## Throughput at saturation (ops/s) - -| streams | stacked | -|---|---| -| 100000 | 383k† | -| 500000 | 244k† | -| 1000000 | 56k† | - -† = not saturated (ladder exhausted) — treat as a lower bound. - -## Pod memory at saturation — peak / p50 (MiB) - -| streams | stacked | -|---|---| -| 100000 | 774 / 668 | -| 500000 | 2925 / 2551 | -| 1000000 | 4939 / 3852 | - -_Pod working set = cgroup `memory.current − inactive_file` (anon + active page cache), sampled each second at the pinned rung. **peak** = high-water (catches bursts like an in-RAM Raft log filling); **p50** = median (what the server steadily holds resident). peak ≈ p50 ⇒ steadily resident; peak ≫ p50 ⇒ transient spikes._ - -## Latency (ms, p50 / p99) - -| streams | stacked @≤80% load | stacked @saturation | -|---|---|---| -| 100000 | 2.5 / 4.5 (383k @4p) | — | -| 500000 | 5.8 / 26.5 (244k @8p) | — | -| 1000000 | 13.9 / 1186.8 (56k @8p) | — | - -_@≤80% load = the largest ladder rung at ≤80% of peak throughput — the server's service latency with headroom. @saturation = the pinned plateau rung, where a closed-loop fleet measures its own queueing (≈ in-flight ÷ ceiling by Little's law), NOT the server's per-request cost. Compare against the unloaded single-request baseline (~1 ms for wal) before reading anything into large saturation values._ - -## Saturation walks (pods → ops/s, p50 ms) - -- **stacked 100000**: 4:383k@2.5ms → 8:362k@5.4ms (pinned 8, ladder_exhausted) -- **stacked 500000**: 8:244k@5.8ms (pinned 8, ladder_exhausted) -- **stacked 1000000**: 8:56k@13.9ms (pinned 8, ladder_exhausted) - -## Findings - -**Stacked ideal config validated**: 382.7k ops/s @100k streams (p4; 362.3k pinned @p8) — matches the ~360k projection from the separately-measured size-trigger (+10%) and CPU-binding (+21-24%) effects. Campaign total: 10.4k -> 383k = 37x, cardinality-flat through 100k. - -**A NEW wall near 1M streams**: 500k holds 244k (-36%), 1M collapses to 56k (-85%). This is not the old checkpoint storm (fixed, flat to 100k). Candidate mechanisms for the next investigation: 1M open fds (one per live stream ~= container nofile ceiling), ext4 directory with 1M files, stream-map/tails/meta working set, page-cache pressure from 1M dirty files. Needs a dedicated profiling pass with SRV_STATS/WAL_CKPT telemetry review at 500k/1M. - -## Caveats - -_Single-node best-case; not 3-node Raft. Throughput is a saturation ceiling per the ladder._ diff --git a/results/wal-stacked-1m/stacked/cells.json b/results/wal-stacked-1m/stacked/cells.json deleted file mode 100644 index f25df2c..0000000 --- a/results/wal-stacked-1m/stacked/cells.json +++ /dev/null @@ -1,73 +0,0 @@ -{ - "cells": { - "100000": { - "stream_count": 100000, - "image_digest": "2cfd37a9a235", - "walk": [ - [ - 4, - 382727.89999999997, - 2.537, - 4.539 - ], - [ - 8, - 362346.35000000003, - 5.439, - 8.751 - ] - ], - "pinned_pods": 8, - "throughput": 362346.35000000003, - "p50": null, - "p99": null, - "pod_mem_mb": 774, - "saturated": false, - "status": "ok", - "reason": "ladder_exhausted", - "pod_mem_p50_mb": 668 - }, - "500000": { - "stream_count": 500000, - "image_digest": "2cfd37a9a235", - "walk": [ - [ - 8, - 244203.8, - 5.763, - 26.543 - ] - ], - "pinned_pods": 8, - "throughput": 244203.8, - "p50": null, - "p99": null, - "pod_mem_mb": 2925, - "saturated": false, - "status": "ok", - "reason": "ladder_exhausted", - "pod_mem_p50_mb": 2551 - }, - "1000000": { - "stream_count": 1000000, - "image_digest": "2cfd37a9a235", - "walk": [ - [ - 8, - 56003.35, - 13.919, - 1186.815 - ] - ], - "pinned_pods": 8, - "throughput": 56003.35, - "p50": null, - "p99": null, - "pod_mem_mb": 4939, - "saturated": false, - "status": "ok", - "reason": "ladder_exhausted", - "pod_mem_p50_mb": 3852 - } - } -} \ No newline at end of file diff --git a/results/wal-streamlanes-1m/aggregate.csv b/results/wal-streamlanes-1m/aggregate.csv deleted file mode 100644 index d6ad0a5..0000000 --- a/results/wal-streamlanes-1m/aggregate.csv +++ /dev/null @@ -1,4 +0,0 @@ -mode,stream_count,pods,throughput,p50,p99,knee_pods,knee_throughput,knee_p50,knee_p99,pod_mem_mb,pod_mem_p50_mb,saturated,status,reason -lanes3x3,100000,8,374115.79999999993,,,8,374115.79999999993,5.291,8.671,1053,859,False,ok,ladder_exhausted -lanes3x3,500000,8,284627.3333333333,,,8,284627.3333333333,5.731,11.031,3160,3026,False,ok,ladder_exhausted -lanes3x3,1000000,8,211678.4333333333,,,8,211678.4333333333,6.011,29.727,5588,4968,False,ok,ladder_exhausted diff --git a/results/wal-streamlanes-1m/aggregate.json b/results/wal-streamlanes-1m/aggregate.json deleted file mode 100644 index 6344344..0000000 --- a/results/wal-streamlanes-1m/aggregate.json +++ /dev/null @@ -1,77 +0,0 @@ -[ - { - "mode": "lanes3x3", - "stream_count": 100000, - "pods": 8, - "throughput": 374115.79999999993, - "p50": null, - "p99": null, - "knee_pods": 8, - "knee_throughput": 374115.79999999993, - "knee_p50": 5.291, - "knee_p99": 8.671, - "pod_mem_mb": 1053, - "pod_mem_p50_mb": 859, - "saturated": false, - "status": "ok", - "reason": "ladder_exhausted", - "walk": [ - [ - 8, - 374115.79999999993, - 5.291, - 8.671 - ] - ] - }, - { - "mode": "lanes3x3", - "stream_count": 500000, - "pods": 8, - "throughput": 284627.3333333333, - "p50": null, - "p99": null, - "knee_pods": 8, - "knee_throughput": 284627.3333333333, - "knee_p50": 5.731, - "knee_p99": 11.031, - "pod_mem_mb": 3160, - "pod_mem_p50_mb": 3026, - "saturated": false, - "status": "ok", - "reason": "ladder_exhausted", - "walk": [ - [ - 8, - 284627.3333333333, - 5.731, - 11.031 - ] - ] - }, - { - "mode": "lanes3x3", - "stream_count": 1000000, - "pods": 8, - "throughput": 211678.4333333333, - "p50": null, - "p99": null, - "knee_pods": 8, - "knee_throughput": 211678.4333333333, - "knee_p50": 6.011, - "knee_p99": 29.727, - "pod_mem_mb": 5588, - "pod_mem_p50_mb": 4968, - "saturated": false, - "status": "ok", - "reason": "ladder_exhausted", - "walk": [ - [ - 8, - 211678.4333333333, - 6.011, - 29.727 - ] - ] - } -] \ No newline at end of file diff --git a/results/wal-streamlanes-1m/lanes3x3/cells.json b/results/wal-streamlanes-1m/lanes3x3/cells.json deleted file mode 100644 index d06b384..0000000 --- a/results/wal-streamlanes-1m/lanes3x3/cells.json +++ /dev/null @@ -1,67 +0,0 @@ -{ - "cells": { - "100000": { - "stream_count": 100000, - "image_digest": "79a7f7d5f756", - "walk": [ - [ - 8, - 374115.79999999993, - 5.291, - 8.671 - ] - ], - "pinned_pods": 8, - "throughput": 374115.79999999993, - "p50": null, - "p99": null, - "pod_mem_mb": 1053, - "saturated": false, - "status": "ok", - "reason": "ladder_exhausted", - "pod_mem_p50_mb": 859 - }, - "500000": { - "stream_count": 500000, - "image_digest": "79a7f7d5f756", - "walk": [ - [ - 8, - 284627.3333333333, - 5.731, - 11.031 - ] - ], - "pinned_pods": 8, - "throughput": 284627.3333333333, - "p50": null, - "p99": null, - "pod_mem_mb": 3160, - "saturated": false, - "status": "ok", - "reason": "ladder_exhausted", - "pod_mem_p50_mb": 3026 - }, - "1000000": { - "stream_count": 1000000, - "image_digest": "79a7f7d5f756", - "walk": [ - [ - 8, - 211678.4333333333, - 6.011, - 29.727 - ] - ], - "pinned_pods": 8, - "throughput": 211678.4333333333, - "p50": null, - "p99": null, - "pod_mem_mb": 5588, - "saturated": false, - "status": "ok", - "reason": "ladder_exhausted", - "pod_mem_p50_mb": 4968 - } - } -} \ No newline at end of file diff --git a/results/wal-streamlanes-1m/report.md b/results/wal-streamlanes-1m/report.md deleted file mode 100644 index da6641f..0000000 --- a/results/wal-streamlanes-1m/report.md +++ /dev/null @@ -1,45 +0,0 @@ -# wal-streamlanes-1m — write-throughput report - -## Throughput at saturation (ops/s) - -| streams | lanes3x3 | -|---|---| -| 100000 | 374k† | -| 500000 | 285k† | -| 1000000 | 212k† | - -† = not saturated (ladder exhausted) — treat as a lower bound. - -## Pod memory at saturation — peak / p50 (MiB) - -| streams | lanes3x3 | -|---|---| -| 100000 | 1053 / 859 | -| 500000 | 3160 / 3026 | -| 1000000 | 5588 / 4968 | - -_Pod working set = cgroup `memory.current − inactive_file` (anon + active page cache), sampled each second at the pinned rung. **peak** = high-water (catches bursts like an in-RAM Raft log filling); **p50** = median (what the server steadily holds resident). peak ≈ p50 ⇒ steadily resident; peak ≫ p50 ⇒ transient spikes._ - -## Latency (ms, p50 / p99) - -| streams | lanes3x3 @≤80% load | lanes3x3 @saturation | -|---|---|---| -| 100000 | 5.3 / 8.7 (374k @8p) | — | -| 500000 | 5.7 / 11.0 (285k @8p) | — | -| 1000000 | 6.0 / 29.7 (212k @8p) | — | - -_@≤80% load = the largest ladder rung at ≤80% of peak throughput — the server's service latency with headroom. @saturation = the pinned plateau rung, where a closed-loop fleet measures its own queueing (≈ in-flight ÷ ceiling by Little's law), NOT the server's per-request cost. Compare against the unloaded single-request baseline (~1 ms for wal) before reading anything into large saturation values._ - -## Saturation walks (pods → ops/s, p50 ms) - -- **lanes3x3 100000**: 8:374k@5.3ms (pinned 8, ladder_exhausted) -- **lanes3x3 500000**: 8:285k@5.7ms (pinned 8, ladder_exhausted) -- **lanes3x3 1000000**: 8:212k@6.0ms (pinned 8, ladder_exhausted) - -## Findings - -3 stream lanes + 3 WAL lanes (--stream-lanes 3, PR #4705) breaks the 1M writeback wall: 374.1k @100k (parity with the 5-WAL-lane stacked config), 284.6k @500k (+13% vs 1 data lane), 211.7k @1M (3.1x vs 68k). Checkpoint syncfs fell 60-74s -> 5.7-11.2s with MORE touched files per shard. Residual 100k->1M slope (-43%) = per-file writeback amplification vs total data-lane capacity; add lanes or see #4695. fd note: ~1.006M fds at 1M streams (96% of ceiling) -> #4706. - -## Caveats - -_Single-node best-case; not 3-node Raft. Throughput is a saturation ceiling per the ladder._ diff --git a/results/wal-syncfs-local/aggregate.csv b/results/wal-syncfs-local/aggregate.csv deleted file mode 100644 index 9934156..0000000 --- a/results/wal-syncfs-local/aggregate.csv +++ /dev/null @@ -1,5 +0,0 @@ -mode,stream_count,pods,throughput,p50,p99,knee_pods,knee_throughput,knee_p50,knee_p99,pod_mem_mb,pod_mem_p50_mb,saturated,status,reason -baseline,20000,4,87167.5,,,2,43451.75,5.119,18.319,133,119,False,ok,ladder_exhausted -syncfs,20000,4,75809.25,,,2,61261.25,3.361,11.567,129,107,False,ok,ladder_exhausted -baseline,50000,6,66145.25,,,2,34725.625,5.875,29.039,271,243,False,ok,ladder_exhausted -syncfs,50000,6,61155.875,,,2,48074.625,4.089,19.663,255,244,False,ok,ladder_exhausted diff --git a/results/wal-syncfs-local/aggregate.json b/results/wal-syncfs-local/aggregate.json deleted file mode 100644 index 8caff33..0000000 --- a/results/wal-syncfs-local/aggregate.json +++ /dev/null @@ -1,138 +0,0 @@ -[ - { - "mode": "baseline", - "stream_count": 20000, - "pods": 4, - "throughput": 87167.5, - "p50": null, - "p99": null, - "knee_pods": 2, - "knee_throughput": 43451.75, - "knee_p50": 5.119, - "knee_p99": 18.319, - "pod_mem_mb": 133, - "pod_mem_p50_mb": 119, - "saturated": false, - "status": "ok", - "reason": "ladder_exhausted", - "walk": [ - [ - 2, - 43451.75, - 5.119, - 18.319 - ], - [ - 4, - 87167.5, - 5.511, - 12.943 - ] - ] - }, - { - "mode": "syncfs", - "stream_count": 20000, - "pods": 4, - "throughput": 75809.25, - "p50": null, - "p99": null, - "knee_pods": 2, - "knee_throughput": 61261.25, - "knee_p50": 3.361, - "knee_p99": 11.567, - "pod_mem_mb": 129, - "pod_mem_p50_mb": 107, - "saturated": false, - "status": "ok", - "reason": "ladder_exhausted", - "walk": [ - [ - 2, - 61261.25, - 3.361, - 11.567 - ], - [ - 4, - 75809.25, - 5.071, - 20.495 - ] - ] - }, - { - "mode": "baseline", - "stream_count": 50000, - "pods": 6, - "throughput": 66145.25, - "p50": null, - "p99": null, - "knee_pods": 2, - "knee_throughput": 34725.625, - "knee_p50": 5.875, - "knee_p99": 29.039, - "pod_mem_mb": 271, - "pod_mem_p50_mb": 243, - "saturated": false, - "status": "ok", - "reason": "ladder_exhausted", - "walk": [ - [ - 2, - 34725.625, - 5.875, - 29.039 - ], - [ - 4, - 56672.125, - 8.287, - 21.071 - ], - [ - 6, - 66145.25, - 9.567, - 42.303 - ] - ] - }, - { - "mode": "syncfs", - "stream_count": 50000, - "pods": 6, - "throughput": 61155.875, - "p50": null, - "p99": null, - "knee_pods": 2, - "knee_throughput": 48074.625, - "knee_p50": 4.089, - "knee_p99": 19.663, - "pod_mem_mb": 255, - "pod_mem_p50_mb": 244, - "saturated": false, - "status": "ok", - "reason": "ladder_exhausted", - "walk": [ - [ - 2, - 48074.625, - 4.089, - 19.663 - ], - [ - 4, - 54739.875, - 6.255, - 126.335 - ], - [ - 6, - 61155.875, - 8.003, - 173.439 - ] - ] - } -] \ No newline at end of file diff --git a/results/wal-syncfs-local/baseline/cells.json b/results/wal-syncfs-local/baseline/cells.json deleted file mode 100644 index 090f24a..0000000 --- a/results/wal-syncfs-local/baseline/cells.json +++ /dev/null @@ -1,64 +0,0 @@ -{ - "cells": { - "20000": { - "stream_count": 20000, - "image_digest": "0bb2c6c89ce6", - "walk": [ - [ - 2, - 43451.75, - 5.119, - 18.319 - ], - [ - 4, - 87167.5, - 5.511, - 12.943 - ] - ], - "pinned_pods": 4, - "throughput": 87167.5, - "p50": null, - "p99": null, - "pod_mem_mb": 133, - "saturated": false, - "status": "ok", - "reason": "ladder_exhausted", - "pod_mem_p50_mb": 119 - }, - "50000": { - "stream_count": 50000, - "image_digest": "0bb2c6c89ce6", - "walk": [ - [ - 2, - 34725.625, - 5.875, - 29.039 - ], - [ - 4, - 56672.125, - 8.287, - 21.071 - ], - [ - 6, - 66145.25, - 9.567, - 42.303 - ] - ], - "pinned_pods": 6, - "throughput": 66145.25, - "p50": null, - "p99": null, - "pod_mem_mb": 271, - "saturated": false, - "status": "ok", - "reason": "ladder_exhausted", - "pod_mem_p50_mb": 243 - } - } -} \ No newline at end of file diff --git a/results/wal-syncfs-local/report.md b/results/wal-syncfs-local/report.md deleted file mode 100644 index f0022e1..0000000 --- a/results/wal-syncfs-local/report.md +++ /dev/null @@ -1,43 +0,0 @@ -# wal-syncfs-local — write-throughput report - -## Throughput at saturation (ops/s) - -| streams | baseline | syncfs | -|---|---|---| -| 20000 | 87k† | 76k† | -| 50000 | 66k† | 61k† | - -† = not saturated (ladder exhausted) — treat as a lower bound. - -## Pod memory at saturation — peak / p50 (MiB) - -| streams | baseline | syncfs | -|---|---|---| -| 20000 | 133 / 119 | 129 / 107 | -| 50000 | 271 / 243 | 255 / 244 | - -_Pod working set = cgroup `memory.current − inactive_file` (anon + active page cache), sampled each second at the pinned rung. **peak** = high-water (catches bursts like an in-RAM Raft log filling); **p50** = median (what the server steadily holds resident). peak ≈ p50 ⇒ steadily resident; peak ≫ p50 ⇒ transient spikes._ - -## Latency (ms, p50 / p99) - -| streams | baseline @≤80% load | baseline @saturation | syncfs @≤80% load | syncfs @saturation | -|---|---|---|---|---| -| 20000 | 5.1 / 18.3 (43k @2p) | — | 3.4 / 11.6 (61k @2p) | — | -| 50000 | 5.9 / 29.0 (35k @2p) | — | 4.1 / 19.7 (48k @2p) | — | - -_@≤80% load = the largest ladder rung at ≤80% of peak throughput — the server's service latency with headroom. @saturation = the pinned plateau rung, where a closed-loop fleet measures its own queueing (≈ in-flight ÷ ceiling by Little's law), NOT the server's per-request cost. Compare against the unloaded single-request baseline (~1 ms for wal) before reading anything into large saturation values._ - -## Saturation walks (pods → ops/s, p50 ms) - -- **baseline 20000**: 2:43k@5.1ms → 4:87k@5.5ms (pinned 4, ladder_exhausted) -- **syncfs 20000**: 2:61k@3.4ms → 4:76k@5.1ms (pinned 4, ladder_exhausted) -- **baseline 50000**: 2:35k@5.9ms → 4:57k@8.3ms → 6:66k@9.6ms (pinned 6, ladder_exhausted) -- **syncfs 50000**: 2:48k@4.1ms → 4:55k@6.3ms → 6:61k@8.0ms (pinned 6, ladder_exhausted) - -## Findings - -_TODO: written by hand on top of the generated data._ - -## Caveats - -_Single-node best-case; not 3-node Raft. Throughput is a saturation ceiling per the ladder._ diff --git a/results/wal-syncfs-local/syncfs/cells.json b/results/wal-syncfs-local/syncfs/cells.json deleted file mode 100644 index b0cc3b9..0000000 --- a/results/wal-syncfs-local/syncfs/cells.json +++ /dev/null @@ -1,64 +0,0 @@ -{ - "cells": { - "20000": { - "stream_count": 20000, - "image_digest": "8c09d3514768", - "walk": [ - [ - 2, - 61261.25, - 3.361, - 11.567 - ], - [ - 4, - 75809.25, - 5.071, - 20.495 - ] - ], - "pinned_pods": 4, - "throughput": 75809.25, - "p50": null, - "p99": null, - "pod_mem_mb": 129, - "saturated": false, - "status": "ok", - "reason": "ladder_exhausted", - "pod_mem_p50_mb": 107 - }, - "50000": { - "stream_count": 50000, - "image_digest": "8c09d3514768", - "walk": [ - [ - 2, - 48074.625, - 4.089, - 19.663 - ], - [ - 4, - 54739.875, - 6.255, - 126.335 - ], - [ - 6, - 61155.875, - 8.003, - 173.439 - ] - ], - "pinned_pods": 6, - "throughput": 61155.875, - "p50": null, - "p99": null, - "pod_mem_mb": 255, - "saturated": false, - "status": "ok", - "reason": "ladder_exhausted", - "pod_mem_p50_mb": 244 - } - } -} \ No newline at end of file diff --git a/results/write-accuracy-local/aggregate.csv b/results/write-accuracy-local/aggregate.csv deleted file mode 100644 index 942a18b..0000000 --- a/results/write-accuracy-local/aggregate.csv +++ /dev/null @@ -1,3 +0,0 @@ -mode,stream_count,pods,throughput,p50,p99,pod_mem_mb,pod_mem_p50_mb,saturated,status,reason -memory,2000,2,147194.5,0.732,2.601,37,25,True,ok,plateau -wal,2000,4,64688.7,,,35,26,False,ok,ladder_exhausted diff --git a/results/write-accuracy-local/aggregate.json b/results/write-accuracy-local/aggregate.json deleted file mode 100644 index 408e676..0000000 --- a/results/write-accuracy-local/aggregate.json +++ /dev/null @@ -1,48 +0,0 @@ -[ - { - "mode": "memory", - "stream_count": 2000, - "pods": 2, - "throughput": 147194.5, - "p50": 0.732, - "p99": 2.601, - "pod_mem_mb": 37, - "pod_mem_p50_mb": 25, - "saturated": true, - "status": "ok", - "reason": "plateau", - "walk": [ - [ - 2, - 147194.5 - ], - [ - 4, - 139693.9 - ] - ] - }, - { - "mode": "wal", - "stream_count": 2000, - "pods": 4, - "throughput": 64688.7, - "p50": null, - "p99": null, - "pod_mem_mb": 35, - "pod_mem_p50_mb": 26, - "saturated": false, - "status": "ok", - "reason": "ladder_exhausted", - "walk": [ - [ - 2, - 42634.5 - ], - [ - 4, - 64688.7 - ] - ] - } -] \ No newline at end of file diff --git a/results/write-accuracy-local/memory/cells.json b/results/write-accuracy-local/memory/cells.json deleted file mode 100644 index 51372b1..0000000 --- a/results/write-accuracy-local/memory/cells.json +++ /dev/null @@ -1,27 +0,0 @@ -{ - "cells": { - "2000": { - "stream_count": 2000, - "image_digest": "f60d91fcfa07", - "walk": [ - [ - 2, - 147194.5 - ], - [ - 4, - 139693.9 - ] - ], - "pinned_pods": 2, - "throughput": 147194.5, - "p50": 0.732, - "p99": 2.601, - "pod_mem_mb": 37, - "saturated": true, - "status": "ok", - "reason": "plateau", - "pod_mem_p50_mb": 25 - } - } -} \ No newline at end of file diff --git a/results/write-accuracy-local/report.md b/results/write-accuracy-local/report.md deleted file mode 100644 index f95d5b2..0000000 --- a/results/write-accuracy-local/report.md +++ /dev/null @@ -1,30 +0,0 @@ -# write-accuracy-local — write-throughput report - -## Throughput at saturation (ops/s) - -| streams | wal | memory | -|---|---|---| -| 2000 | 65k† | 147k | - -† = not saturated (ladder exhausted) — treat as a lower bound. - -## Pod memory at saturation — peak / p50 (MiB) - -| streams | wal | memory | -|---|---|---| -| 2000 | 35 / 26 | 37 / 25 | - -_Pod working set = cgroup `memory.current − inactive_file` (anon + active page cache), sampled each second at the pinned rung. **peak** = high-water (catches bursts like an in-RAM Raft log filling); **p50** = median (what the server steadily holds resident). peak ≈ p50 ⇒ steadily resident; peak ≫ p50 ⇒ transient spikes._ - -## Saturation walks (pods → ops/s) - -- **memory 2000**: 2:147k → 4:140k (pinned 2, plateau) -- **wal 2000**: 2:43k → 4:65k (pinned 4, ladder_exhausted) - -## Findings - -_TODO: written by hand on top of the generated data._ - -## Caveats - -_Single-node best-case; not 3-node Raft. Throughput is a saturation ceiling per the ladder._ diff --git a/results/write-accuracy-local/wal/cells.json b/results/write-accuracy-local/wal/cells.json deleted file mode 100644 index f13ab86..0000000 --- a/results/write-accuracy-local/wal/cells.json +++ /dev/null @@ -1,27 +0,0 @@ -{ - "cells": { - "2000": { - "stream_count": 2000, - "image_digest": "c7df7a0880e0", - "walk": [ - [ - 2, - 42634.5 - ], - [ - 4, - 64688.7 - ] - ], - "pinned_pods": 4, - "throughput": 64688.7, - "p50": null, - "p99": null, - "pod_mem_mb": 35, - "saturated": false, - "status": "ok", - "reason": "ladder_exhausted", - "pod_mem_p50_mb": 26 - } - } -} \ No newline at end of file diff --git a/results/write-cliff-local-cpu4/aggregate.csv b/results/write-cliff-local-cpu4/aggregate.csv deleted file mode 100644 index 8f681dd..0000000 --- a/results/write-cliff-local-cpu4/aggregate.csv +++ /dev/null @@ -1,3 +0,0 @@ -mode,stream_count,pods,throughput,p50,p99,knee_pods,knee_throughput,knee_p50,knee_p99,pod_mem_mb,pod_mem_p50_mb,saturated,status,reason -wal-cpu4,10000,4,44286.933333333334,,,2,30546.466666666667,3.771,12.983,78,70,False,ok,ladder_exhausted -wal-cpu4,50000,4,32143.533333333333,,,2,18012.266666666666,5.627,29.903,255,247,False,ok,ladder_exhausted diff --git a/results/write-cliff-local-cpu4/aggregate.json b/results/write-cliff-local-cpu4/aggregate.json deleted file mode 100644 index e8159b6..0000000 --- a/results/write-cliff-local-cpu4/aggregate.json +++ /dev/null @@ -1,76 +0,0 @@ -[ - { - "mode": "wal-cpu4", - "stream_count": 10000, - "pods": 4, - "throughput": 44286.933333333334, - "p50": null, - "p99": null, - "knee_pods": 2, - "knee_throughput": 30546.466666666667, - "knee_p50": 3.771, - "knee_p99": 12.983, - "pod_mem_mb": 78, - "pod_mem_p50_mb": 70, - "saturated": false, - "status": "ok", - "reason": "ladder_exhausted", - "walk": [ - [ - 1, - 13356.133333333331, - 4.163, - 16.607 - ], - [ - 2, - 30546.466666666667, - 3.771, - 12.983 - ], - [ - 4, - 44286.933333333334, - 5.047, - 18.079 - ] - ] - }, - { - "mode": "wal-cpu4", - "stream_count": 50000, - "pods": 4, - "throughput": 32143.533333333333, - "p50": null, - "p99": null, - "knee_pods": 2, - "knee_throughput": 18012.266666666666, - "knee_p50": 5.627, - "knee_p99": 29.903, - "pod_mem_mb": 255, - "pod_mem_p50_mb": 247, - "saturated": false, - "status": "ok", - "reason": "ladder_exhausted", - "walk": [ - [ - 1, - 12407.2, - 3.951, - 24.879 - ], - [ - 2, - 18012.266666666666, - 5.627, - 29.903 - ], - [ - 4, - 32143.533333333333, - 6.059, - 41.727 - ] - ] - } -] \ No newline at end of file diff --git a/results/write-cliff-local-cpu4/report.md b/results/write-cliff-local-cpu4/report.md deleted file mode 100644 index 3b354cc..0000000 --- a/results/write-cliff-local-cpu4/report.md +++ /dev/null @@ -1,41 +0,0 @@ -# write-cliff-local-cpu4 — write-throughput report - -## Throughput at saturation (ops/s) - -| streams | wal-cpu4 | -|---|---| -| 10000 | 44k† | -| 50000 | 32k† | - -† = not saturated (ladder exhausted) — treat as a lower bound. - -## Pod memory at saturation — peak / p50 (MiB) - -| streams | wal-cpu4 | -|---|---| -| 10000 | 78 / 70 | -| 50000 | 255 / 247 | - -_Pod working set = cgroup `memory.current − inactive_file` (anon + active page cache), sampled each second at the pinned rung. **peak** = high-water (catches bursts like an in-RAM Raft log filling); **p50** = median (what the server steadily holds resident). peak ≈ p50 ⇒ steadily resident; peak ≫ p50 ⇒ transient spikes._ - -## Latency (ms, p50 / p99) - -| streams | wal-cpu4 @≤80% load | wal-cpu4 @saturation | -|---|---|---| -| 10000 | 3.8 / 13.0 (31k @2p) | — | -| 50000 | 5.6 / 29.9 (18k @2p) | — | - -_@≤80% load = the largest ladder rung at ≤80% of peak throughput — the server's service latency with headroom. @saturation = the pinned plateau rung, where a closed-loop fleet measures its own queueing (≈ in-flight ÷ ceiling by Little's law), NOT the server's per-request cost. Compare against the unloaded single-request baseline (~1 ms for wal) before reading anything into large saturation values._ - -## Saturation walks (pods → ops/s, p50 ms) - -- **wal-cpu4 10000**: 1:13k@4.2ms → 2:31k@3.8ms → 4:44k@5.0ms (pinned 4, ladder_exhausted) -- **wal-cpu4 50000**: 1:12k@4.0ms → 2:18k@5.6ms → 4:32k@6.1ms (pinned 4, ladder_exhausted) - -## Findings - -_TODO: written by hand on top of the generated data._ - -## Caveats - -_Single-node best-case; not 3-node Raft. Throughput is a saturation ceiling per the ladder._ diff --git a/results/write-cliff-local-cpu4/wal-cpu4/cells.json b/results/write-cliff-local-cpu4/wal-cpu4/cells.json deleted file mode 100644 index 52ead17..0000000 --- a/results/write-cliff-local-cpu4/wal-cpu4/cells.json +++ /dev/null @@ -1,70 +0,0 @@ -{ - "cells": { - "10000": { - "stream_count": 10000, - "image_digest": "844e2da2a56c", - "walk": [ - [ - 1, - 13356.133333333331, - 4.163, - 16.607 - ], - [ - 2, - 30546.466666666667, - 3.771, - 12.983 - ], - [ - 4, - 44286.933333333334, - 5.047, - 18.079 - ] - ], - "pinned_pods": 4, - "throughput": 44286.933333333334, - "p50": null, - "p99": null, - "pod_mem_mb": 78, - "saturated": false, - "status": "ok", - "reason": "ladder_exhausted", - "pod_mem_p50_mb": 70 - }, - "50000": { - "stream_count": 50000, - "image_digest": "844e2da2a56c", - "walk": [ - [ - 1, - 12407.2, - 3.951, - 24.879 - ], - [ - 2, - 18012.266666666666, - 5.627, - 29.903 - ], - [ - 4, - 32143.533333333333, - 6.059, - 41.727 - ] - ], - "pinned_pods": 4, - "throughput": 32143.533333333333, - "p50": null, - "p99": null, - "pod_mem_mb": 255, - "saturated": false, - "status": "ok", - "reason": "ladder_exhausted", - "pod_mem_p50_mb": 247 - } - } -} \ No newline at end of file diff --git a/results/write-cliff-local/FINDINGS.md b/results/write-cliff-local/FINDINGS.md deleted file mode 100644 index feb311e..0000000 --- a/results/write-cliff-local/FINDINGS.md +++ /dev/null @@ -1,63 +0,0 @@ -# Local reproduction of the cardinality cliff (kind, 2026-07-08) - -**Purpose:** prove the remote cardinality cliff reproduces on a laptop so -server-side work can iterate locally (minutes, free) instead of on GKE -(hours, billed). It does. The authoritative how-to-iterate guide lives with the -server code: `packages/durable-streams-rust/CARDINALITY_CLIFF_REPRO.md` in the -electric repo. - -## Result: the cliff reproduces, earlier and steeper than remote - -Local kind, server pinned 2 vCPU (`--wal-shards 2 --worker-threads 2` / -`--durability memory --worker-threads 2`), pool client 64 conns/pod, ladder -1→2→4 pods. Top-rung throughput (fixed 4-pod offered load) and knee p50, -normalized to n=1000: - -| streams | wal thr | wal rel | wal knee p50 | memory thr | memory rel | memory knee p50 | -|---|---|---|---|---|---|---| -| 1k | 55.4k | 100% | 2.9 ms | 112.5k | 100% | 0.5 ms | -| 10k | 48.2k | 87% | 3.0 ms | 63.7k | 57% | 0.6 ms | -| 50k | 27.8k | 50% | 5.0 ms | 22.7k | 20% | 3.9 ms | -| 100k† | 10.4k | 19% | 6.1 ms | 17.7k | 16% | 5.0 ms | -| 200k† | 6.7k | 12% | 8.6 ms | — | — | — | - -† collected before the suite was trimmed; kept for context. **The fast loop is -[1k, 10k, 50k]** (~15 min for both configs) — the cliff is unambiguous by 50k; -bigger counts only slow iteration. - -Remote (corrected campaign, `write-wal-vs-mem-cpu4/FINDINGS.md`): wal −33%, -memory −28–39% over 100k→500k. Locally the cliff starts at ~10k instead of -~100k+ — consistent with the mechanism (per-stream working-set / page-cache / -registry physics vs a much smaller cache envelope: 2 vCPU, 8 GB Docker VM, -overlayfs). The knee p50 rising with cardinality (2.9→8.6 ms wal, 0.5→5 ms -memory at ≤80% load) is the signature that per-REQUEST cost grows — this is not -queueing (latency is quoted below the knee) and not fsync (memory mode has no -WAL and cliffs at least as hard). - -## How to iterate (summary — full guide in the server repo doc) - -```bash -# once: build server+client images from the local checkout and load into kind -BUILD_NODE=0 DS_TARGET=local bash scripts/build-images.sh -# each iteration: rebuild after a server change, wipe, re-run, compare -rm -rf results/write-cliff-local && DS_TARGET=local scripts/bench suites/write-cliff-local.json run -python3 scripts/compare-cliff.py -``` - -`scripts/compare-cliff.py` prints the normalized local curves next to the -remote reference. Judge a server change by the SHAPE (rel% at 10k/50k vs -baseline), not absolute ops/s. - -## CPU-scaling probe (`write-cliff-local-cpu4`): also reproduces - -Same client model, server at 4 vCPU / 4 shards over [10k, 50k], vs this suite's -2-vCPU / 2-shard wal cells (top rung = 4 pods × 64 conns): - -| streams | wal 2 vCPU/2 shards | wal 4 vCPU/4 shards | -|---|---|---| -| 10k | 48.2k | 43.8k | -| 50k | 27.8k | 31.8k | - -Doubling CPU+shards moves throughput −9%/+14% — flat within noise, matching the -remote pattern (47k @4 vCPU vs 43k @8 vCPU). The wal commit-path investigation -can iterate locally too. diff --git a/results/write-cliff-local/aggregate.csv b/results/write-cliff-local/aggregate.csv deleted file mode 100644 index 65c7f93..0000000 --- a/results/write-cliff-local/aggregate.csv +++ /dev/null @@ -1,10 +0,0 @@ -mode,stream_count,pods,throughput,p50,p99,knee_pods,knee_throughput,knee_p50,knee_p99,pod_mem_mb,pod_mem_p50_mb,saturated,status,reason -memory,1000,2,112530.86666666667,0.911,4.767,1,100616.4,0.518,2.443,31,19,True,ok,plateau -wal,1000,4,55448.666666666664,,,2,37571.066666666666,2.855,13.535,30,22,False,ok,ladder_exhausted -memory,10000,1,63723.46666666667,0.625,7.031,1,63723.46666666667,0.646,6.207,70,61,True,ok,plateau -wal,10000,4,48204.666666666664,,,2,36839.86666666667,3.039,11.167,79,67,False,ok,ladder_exhausted -memory,50000,1,22731.066666666666,1.574,36.991,2,14834.733333333334,3.915,67.135,233,227,True,ok,plateau -wal,50000,4,27846.13333333333,,,2,20852.13333333333,5.047,26.479,253,247,False,ok,ladder_exhausted -memory,100000,1,17723.266666666666,1.971,37.087,2,12726.4,4.967,78.527,457,346,True,ok,plateau -wal,100000,2,10376.133333333333,6.995,158.463,1,7124.866666666667,6.127,59.519,468,369,True,ok,plateau -wal,200000,2,6725.200000000001,15.703,183.807,1,5685.133333333333,8.559,62.463,922,529,True,ok,plateau diff --git a/results/write-cliff-local/aggregate.json b/results/write-cliff-local/aggregate.json deleted file mode 100644 index 1a5611d..0000000 --- a/results/write-cliff-local/aggregate.json +++ /dev/null @@ -1,317 +0,0 @@ -[ - { - "mode": "memory", - "stream_count": 1000, - "pods": 2, - "throughput": 112530.86666666667, - "p50": 0.911, - "p99": 4.767, - "knee_pods": 1, - "knee_throughput": 100616.4, - "knee_p50": 0.518, - "knee_p99": 2.443, - "pod_mem_mb": 31, - "pod_mem_p50_mb": 19, - "saturated": true, - "status": "ok", - "reason": "plateau", - "walk": [ - [ - 1, - 100616.4, - 0.518, - 2.443 - ], - [ - 2, - 112530.86666666667, - 0.931, - 5.247 - ], - [ - 4, - 108183.2, - 2.038, - 10.543 - ] - ] - }, - { - "mode": "wal", - "stream_count": 1000, - "pods": 4, - "throughput": 55448.666666666664, - "p50": null, - "p99": null, - "knee_pods": 2, - "knee_throughput": 37571.066666666666, - "knee_p50": 2.855, - "knee_p99": 13.535, - "pod_mem_mb": 30, - "pod_mem_p50_mb": 22, - "saturated": false, - "status": "ok", - "reason": "ladder_exhausted", - "walk": [ - [ - 1, - 20006.066666666666, - 2.749, - 11.199 - ], - [ - 2, - 37571.066666666666, - 2.855, - 13.535 - ], - [ - 4, - 55448.666666666664, - 3.969, - 17.567 - ] - ] - }, - { - "mode": "memory", - "stream_count": 10000, - "pods": 1, - "throughput": 63723.46666666667, - "p50": 0.625, - "p99": 7.031, - "knee_pods": 1, - "knee_throughput": 63723.46666666667, - "knee_p50": 0.646, - "knee_p99": 6.207, - "pod_mem_mb": 70, - "pod_mem_p50_mb": 61, - "saturated": true, - "status": "ok", - "reason": "plateau", - "walk": [ - [ - 1, - 63723.46666666667, - 0.646, - 6.207 - ], - [ - 2, - 65776.46666666667, - 1.191, - 30.047 - ] - ] - }, - { - "mode": "wal", - "stream_count": 10000, - "pods": 4, - "throughput": 48204.666666666664, - "p50": null, - "p99": null, - "knee_pods": 2, - "knee_throughput": 36839.86666666667, - "knee_p50": 3.039, - "knee_p99": 11.167, - "pod_mem_mb": 79, - "pod_mem_p50_mb": 67, - "saturated": false, - "status": "ok", - "reason": "ladder_exhausted", - "walk": [ - [ - 1, - 21564.6, - 2.405, - 14.255 - ], - [ - 2, - 36839.86666666667, - 3.039, - 11.167 - ], - [ - 4, - 48204.666666666664, - 4.447, - 20.895 - ] - ] - }, - { - "mode": "memory", - "stream_count": 50000, - "pods": 1, - "throughput": 22731.066666666666, - "p50": 1.574, - "p99": 36.991, - "knee_pods": 2, - "knee_throughput": 14834.733333333334, - "knee_p50": 3.915, - "knee_p99": 67.135, - "pod_mem_mb": 233, - "pod_mem_p50_mb": 227, - "saturated": true, - "status": "ok", - "reason": "plateau", - "walk": [ - [ - 1, - 22731.066666666666, - 1.386, - 35.135 - ], - [ - 2, - 14834.733333333334, - 3.915, - 67.135 - ] - ] - }, - { - "mode": "wal", - "stream_count": 50000, - "pods": 4, - "throughput": 27846.13333333333, - "p50": null, - "p99": null, - "knee_pods": 2, - "knee_throughput": 20852.13333333333, - "knee_p50": 5.047, - "knee_p99": 26.479, - "pod_mem_mb": 253, - "pod_mem_p50_mb": 247, - "saturated": false, - "status": "ok", - "reason": "ladder_exhausted", - "walk": [ - [ - 1, - 14325.733333333334, - 3.563, - 15.791 - ], - [ - 2, - 20852.13333333333, - 5.047, - 26.479 - ], - [ - 4, - 27846.13333333333, - 6.687, - 48.319 - ] - ] - }, - { - "mode": "memory", - "stream_count": 100000, - "pods": 1, - "throughput": 17723.266666666666, - "p50": 1.971, - "p99": 37.087, - "knee_pods": 2, - "knee_throughput": 12726.4, - "knee_p50": 4.967, - "knee_p99": 78.527, - "pod_mem_mb": 457, - "pod_mem_p50_mb": 346, - "saturated": true, - "status": "ok", - "reason": "plateau", - "walk": [ - [ - 1, - 17723.266666666666, - 1.613, - 36.095 - ], - [ - 2, - 12726.4, - 4.967, - 78.527 - ] - ] - }, - { - "mode": "wal", - "stream_count": 100000, - "pods": 2, - "throughput": 10376.133333333333, - "p50": 6.995, - "p99": 158.463, - "knee_pods": 1, - "knee_throughput": 7124.866666666667, - "knee_p50": 6.127, - "knee_p99": 59.519, - "pod_mem_mb": 468, - "pod_mem_p50_mb": 369, - "saturated": true, - "status": "ok", - "reason": "plateau", - "walk": [ - [ - 1, - 7124.866666666667, - 6.127, - 59.519 - ], - [ - 2, - 10376.133333333333, - 7.979, - 123.391 - ], - [ - 4, - 9534.733333333334, - 17.023, - 232.575 - ] - ] - }, - { - "mode": "wal", - "stream_count": 200000, - "pods": 2, - "throughput": 6725.200000000001, - "p50": 15.703, - "p99": 183.807, - "knee_pods": 1, - "knee_throughput": 5685.133333333333, - "knee_p50": 8.559, - "knee_p99": 62.463, - "pod_mem_mb": 922, - "pod_mem_p50_mb": 529, - "saturated": true, - "status": "ok", - "reason": "plateau", - "walk": [ - [ - 1, - 5685.133333333333, - 8.559, - 62.463 - ], - [ - 2, - 6725.200000000001, - 12.911, - 154.751 - ], - [ - 4, - 6399.133333333333, - 25.167, - 292.607 - ] - ] - } -] \ No newline at end of file diff --git a/results/write-cliff-local/memory/cells.json b/results/write-cliff-local/memory/cells.json deleted file mode 100644 index 79add9b..0000000 --- a/results/write-cliff-local/memory/cells.json +++ /dev/null @@ -1,118 +0,0 @@ -{ - "cells": { - "1000": { - "stream_count": 1000, - "image_digest": "9d445af89e81", - "walk": [ - [ - 1, - 100616.4, - 0.518, - 2.443 - ], - [ - 2, - 112530.86666666667, - 0.931, - 5.247 - ], - [ - 4, - 108183.2, - 2.038, - 10.543 - ] - ], - "pinned_pods": 2, - "throughput": 112530.86666666667, - "p50": 0.911, - "p99": 4.767, - "pod_mem_mb": 31, - "saturated": true, - "status": "ok", - "reason": "plateau", - "pod_mem_p50_mb": 19 - }, - "10000": { - "stream_count": 10000, - "image_digest": "9d445af89e81", - "walk": [ - [ - 1, - 63723.46666666667, - 0.646, - 6.207 - ], - [ - 2, - 65776.46666666667, - 1.191, - 30.047 - ] - ], - "pinned_pods": 1, - "throughput": 63723.46666666667, - "p50": 0.625, - "p99": 7.031, - "pod_mem_mb": 70, - "saturated": true, - "status": "ok", - "reason": "plateau", - "pod_mem_p50_mb": 61 - }, - "50000": { - "stream_count": 50000, - "image_digest": "9d445af89e81", - "walk": [ - [ - 1, - 22731.066666666666, - 1.386, - 35.135 - ], - [ - 2, - 14834.733333333334, - 3.915, - 67.135 - ] - ], - "pinned_pods": 1, - "throughput": 22731.066666666666, - "p50": 1.574, - "p99": 36.991, - "pod_mem_mb": 233, - "saturated": true, - "status": "ok", - "reason": "plateau", - "pod_mem_p50_mb": 227 - }, - "100000": { - "stream_count": 100000, - "image_digest": "9d445af89e81", - "walk": [ - [ - 1, - 17723.266666666666, - 1.613, - 36.095 - ], - [ - 2, - 12726.4, - 4.967, - 78.527 - ] - ], - "pinned_pods": 1, - "throughput": 17723.266666666666, - "p50": 1.971, - "p99": 37.087, - "pod_mem_mb": 457, - "saturated": true, - "status": "ok", - "reason": "plateau", - "pod_mem_p50_mb": 346 - } - } -} \ No newline at end of file diff --git a/results/write-cliff-local/report.md b/results/write-cliff-local/report.md deleted file mode 100644 index db1e9f6..0000000 --- a/results/write-cliff-local/report.md +++ /dev/null @@ -1,51 +0,0 @@ -# write-cliff-local — write-throughput report - -## Throughput at saturation (ops/s) - -| streams | wal | memory | -|---|---|---| -| 1000 | 55k† | 113k | -| 10000 | 48k† | 64k | -| 50000 | 28k† | 23k | - -† = not saturated (ladder exhausted) — treat as a lower bound. - -## Pod memory at saturation — peak / p50 (MiB) - -| streams | wal | memory | -|---|---|---| -| 1000 | 30 / 22 | 31 / 19 | -| 10000 | 79 / 67 | 70 / 61 | -| 50000 | 253 / 247 | 233 / 227 | - -_Pod working set = cgroup `memory.current − inactive_file` (anon + active page cache), sampled each second at the pinned rung. **peak** = high-water (catches bursts like an in-RAM Raft log filling); **p50** = median (what the server steadily holds resident). peak ≈ p50 ⇒ steadily resident; peak ≫ p50 ⇒ transient spikes._ - -## Latency (ms, p50 / p99) - -| streams | wal @≤80% load | wal @saturation | memory @≤80% load | memory @saturation | -|---|---|---|---|---| -| 1000 | 2.9 / 13.5 (38k @2p) | — | 0.5 / 2.4 (101k @1p) | 0.9 / 4.8 | -| 10000 | 3.0 / 11.2 (37k @2p) | — | 0.6 / 6.2 (64k @1p) | 0.6 / 7.0 | -| 50000 | 5.0 / 26.5 (21k @2p) | — | 3.9 / 67.1 (15k @2p) | 1.6 / 37.0 | - -_@≤80% load = the largest ladder rung at ≤80% of peak throughput — the server's service latency with headroom. @saturation = the pinned plateau rung, where a closed-loop fleet measures its own queueing (≈ in-flight ÷ ceiling by Little's law), NOT the server's per-request cost. Compare against the unloaded single-request baseline (~1 ms for wal) before reading anything into large saturation values._ - -## Saturation walks (pods → ops/s, p50 ms) - -- **memory 1000**: 1:101k@0.5ms → 2:113k@0.9ms → 4:108k@2.0ms (pinned 2, plateau) -- **wal 1000**: 1:20k@2.7ms → 2:38k@2.9ms → 4:55k@4.0ms (pinned 4, ladder_exhausted) -- **memory 10000**: 1:64k@0.6ms → 2:66k@1.2ms (pinned 1, plateau) -- **wal 10000**: 1:22k@2.4ms → 2:37k@3.0ms → 4:48k@4.4ms (pinned 4, ladder_exhausted) -- **memory 50000**: 1:23k@1.4ms → 2:15k@3.9ms (pinned 1, plateau) -- **wal 50000**: 1:14k@3.6ms → 2:21k@5.0ms → 4:28k@6.7ms (pinned 4, ladder_exhausted) -- **memory 100000**: 1:18k@1.6ms → 2:13k@5.0ms (pinned 1, plateau) -- **wal 100000**: 1:7k@6.1ms → 2:10k@8.0ms → 4:10k@17.0ms (pinned 2, plateau) -- **wal 200000**: 1:6k@8.6ms → 2:7k@12.9ms → 4:6k@25.2ms (pinned 2, plateau) - -## Findings - -_TODO: written by hand on top of the generated data._ - -## Caveats - -_Single-node best-case; not 3-node Raft. Throughput is a saturation ceiling per the ladder._ diff --git a/results/write-cliff-local/wal/cells.json b/results/write-cliff-local/wal/cells.json deleted file mode 100644 index 55980c6..0000000 --- a/results/write-cliff-local/wal/cells.json +++ /dev/null @@ -1,169 +0,0 @@ -{ - "cells": { - "1000": { - "stream_count": 1000, - "image_digest": "0ef8bca8f01c", - "walk": [ - [ - 1, - 20006.066666666666, - 2.749, - 11.199 - ], - [ - 2, - 37571.066666666666, - 2.855, - 13.535 - ], - [ - 4, - 55448.666666666664, - 3.969, - 17.567 - ] - ], - "pinned_pods": 4, - "throughput": 55448.666666666664, - "p50": null, - "p99": null, - "pod_mem_mb": 30, - "saturated": false, - "status": "ok", - "reason": "ladder_exhausted", - "pod_mem_p50_mb": 22 - }, - "10000": { - "stream_count": 10000, - "image_digest": "0ef8bca8f01c", - "walk": [ - [ - 1, - 21564.6, - 2.405, - 14.255 - ], - [ - 2, - 36839.86666666667, - 3.039, - 11.167 - ], - [ - 4, - 48204.666666666664, - 4.447, - 20.895 - ] - ], - "pinned_pods": 4, - "throughput": 48204.666666666664, - "p50": null, - "p99": null, - "pod_mem_mb": 79, - "saturated": false, - "status": "ok", - "reason": "ladder_exhausted", - "pod_mem_p50_mb": 67 - }, - "50000": { - "stream_count": 50000, - "image_digest": "0ef8bca8f01c", - "walk": [ - [ - 1, - 14325.733333333334, - 3.563, - 15.791 - ], - [ - 2, - 20852.13333333333, - 5.047, - 26.479 - ], - [ - 4, - 27846.13333333333, - 6.687, - 48.319 - ] - ], - "pinned_pods": 4, - "throughput": 27846.13333333333, - "p50": null, - "p99": null, - "pod_mem_mb": 253, - "saturated": false, - "status": "ok", - "reason": "ladder_exhausted", - "pod_mem_p50_mb": 247 - }, - "100000": { - "stream_count": 100000, - "image_digest": "0ef8bca8f01c", - "walk": [ - [ - 1, - 7124.866666666667, - 6.127, - 59.519 - ], - [ - 2, - 10376.133333333333, - 7.979, - 123.391 - ], - [ - 4, - 9534.733333333334, - 17.023, - 232.575 - ] - ], - "pinned_pods": 2, - "throughput": 10376.133333333333, - "p50": 6.995, - "p99": 158.463, - "pod_mem_mb": 468, - "saturated": true, - "status": "ok", - "reason": "plateau", - "pod_mem_p50_mb": 369 - }, - "200000": { - "stream_count": 200000, - "image_digest": "0ef8bca8f01c", - "walk": [ - [ - 1, - 5685.133333333333, - 8.559, - 62.463 - ], - [ - 2, - 6725.200000000001, - 12.911, - 154.751 - ], - [ - 4, - 6399.133333333333, - 25.167, - 292.607 - ] - ], - "pinned_pods": 2, - "throughput": 6725.200000000001, - "p50": 15.703, - "p99": 183.807, - "pod_mem_mb": 922, - "saturated": true, - "status": "ok", - "reason": "plateau", - "pod_mem_p50_mb": 529 - } - } -} \ No newline at end of file diff --git a/results/write-cliff-nvme/memory-gated/cells.json b/results/write-cliff-nvme/memory-gated/cells.json deleted file mode 100644 index 46295f3..0000000 --- a/results/write-cliff-nvme/memory-gated/cells.json +++ /dev/null @@ -1,80 +0,0 @@ -{ - "cells": { - "100000": { - "stream_count": 100000, - "image_digest": "159e0d8b1abd", - "walk": [ - [ - 1, - 200201.96, - 1.181, - 2.395 - ], - [ - 2, - 369682.72, - 1.294, - 2.487 - ], - [ - 4, - 364785.6, - 2.183, - 7.383 - ], - [ - 8, - 327910.68, - 2.149, - 21.807 - ] - ], - "pinned_pods": 2, - "throughput": 367876.0, - "p50": 1.308, - "p99": 2.497, - "pod_mem_mb": 689.0, - "saturated": true, - "status": "ok", - "reason": "plateau" - }, - "500000": { - "stream_count": 500000, - "image_digest": "159e0d8b1abd", - "walk": [ - [ - 1, - 186231.68, - 1.183, - 2.415 - ], - [ - 2, - 257181.8, - 1.378, - 20.927 - ], - [ - 4, - 270023.48000000004, - 2.337, - 20.127 - ], - [ - 8, - 257586.04000000004, - 2.289, - 41.151 - ] - ], - "pinned_pods": 2, - "throughput": 231658.0, - "p50": 1.363, - "p99": 16.359, - "pod_mem_mb": 2868.0, - "saturated": true, - "status": "ok", - "reason": "plateau" - } - } -} \ No newline at end of file diff --git a/results/write-cliff-nvme/memory-nogate/cells.json b/results/write-cliff-nvme/memory-nogate/cells.json deleted file mode 100644 index c39fc01..0000000 --- a/results/write-cliff-nvme/memory-nogate/cells.json +++ /dev/null @@ -1,86 +0,0 @@ -{ - "cells": { - "100000": { - "stream_count": 100000, - "image_digest": "23f7f67df1e1", - "walk": [ - [ - 1, - 195085.48, - 1.216, - 2.485 - ], - [ - 2, - 317949.8, - 1.297, - 19.135 - ], - [ - 4, - 300326.43999999994, - 2.163, - 24.335 - ], - [ - 8, - 302953.44000000006, - 2.221, - 36.287 - ] - ], - "pinned_pods": 2, - "throughput": 313013.0, - "p50": 1.308, - "p99": 19.023, - "pod_mem_mb": 691.0, - "saturated": true, - "status": "ok", - "reason": "plateau" - }, - "500000": { - "stream_count": 500000, - "image_digest": "23f7f67df1e1", - "walk": [ - [ - 1, - 158198.28, - 1.215, - 2.473 - ], - [ - 2, - 216115.44, - 1.371, - 20.671 - ], - [ - 4, - 237205.72000000003, - 2.259, - 23.183 - ], - [ - 8, - 224745.24, - 2.321, - 57.727 - ], - [ - 16, - 202831.88000000003, - 2.419, - 196.607 - ] - ], - "pinned_pods": 4, - "throughput": 236564.0, - "p50": 2.241, - "p99": 23.759, - "pod_mem_mb": 3028.0, - "saturated": true, - "status": "ok", - "reason": "plateau" - } - } -} \ No newline at end of file diff --git a/results/write-cliff-nvme/wal-fsync1/cells.json b/results/write-cliff-nvme/wal-fsync1/cells.json deleted file mode 100644 index ce31450..0000000 --- a/results/write-cliff-nvme/wal-fsync1/cells.json +++ /dev/null @@ -1,104 +0,0 @@ -{ - "cells": { - "100000": { - "stream_count": 100000, - "image_digest": "195884eee1f7", - "walk": [ - [ - 1, - 55295.4, - 4.055, - 29.135 - ], - [ - 2, - 60305.36, - 7.459, - 33.791 - ], - [ - 4, - 62912.44, - 14.559, - 44.703 - ], - [ - 8, - 66045.48, - 28.879, - 56.255 - ] - ], - "pinned_pods": 2, - "throughput": 59301.0, - "p50": 7.543, - "p99": 34.175, - "pod_mem_mb": 641.0, - "saturated": true, - "status": "ok", - "reason": "plateau" - }, - "500000": { - "stream_count": 500000, - "image_digest": "195884eee1f7", - "walk": [ - [ - 1, - 33970.6, - 5.391, - 31.247 - ], - [ - 2, - 37496.119999999995, - 10.647, - 42.783 - ], - [ - 4, - 37202.4, - 25.023, - 66.815 - ], - [ - 8, - 45211.96000000001, - 39.231, - 125.183 - ], - [ - 16, - 42289.96000000001, - 69.567, - 332.543 - ], - [ - 32, - 46918.16, - 164.607, - 366.847 - ], - [ - 48, - 45099.28, - 247.423, - 570.879 - ], - [ - 64, - 48463.23999999999, - 325.887, - 937.983 - ] - ], - "pinned_pods": 32, - "throughput": 46918.16, - "p50": 0.0, - "p99": 0.0, - "pod_mem_mb": 2871.0, - "saturated": true, - "status": "ok", - "reason": "plateau" - } - } -} \ No newline at end of file diff --git a/results/write-cliff-nvme/wal-fsync16/cells.json b/results/write-cliff-nvme/wal-fsync16/cells.json deleted file mode 100644 index d4c5fdd..0000000 --- a/results/write-cliff-nvme/wal-fsync16/cells.json +++ /dev/null @@ -1,54 +0,0 @@ -{ - "cells": { - "100000": { - "stream_count": 100000, - "image_digest": "29798c4812fb", - "walk": [ - [ - 1, - 44686.32, - 4.131, - 12.495 - ], - [ - 2, - 46483.6, - 11.095, - 23.647 - ], - [ - 4, - 54569.68, - 18.335, - 40.735 - ], - [ - 8, - 54271.64, - 35.039, - 81.471 - ], - [ - 16, - 58713.31999999999, - 64.383, - 155.775 - ], - [ - 32, - 0.0, - 0.0, - 0.0 - ] - ], - "pinned_pods": null, - "throughput": 0.0, - "p50": null, - "p99": null, - "pod_mem_mb": null, - "saturated": false, - "status": "error", - "reason": "creation_choke" - } - } -} \ No newline at end of file diff --git a/results/write-nvme-bottleneck/memory/cells.json b/results/write-nvme-bottleneck/memory/cells.json deleted file mode 100644 index c63c3d3..0000000 --- a/results/write-nvme-bottleneck/memory/cells.json +++ /dev/null @@ -1,48 +0,0 @@ -{ - "cells": { - "500000": { - "stream_count": 500000, - "image_digest": "3734ed07b9db", - "walk": [ - [ - 1, - 175274.44, - 1.191, - 2.431 - ], - [ - 2, - 234427.91999999998, - 1.404, - 18.639 - ], - [ - 4, - 267114.0, - 2.321, - 22.991 - ], - [ - 8, - 259408.03999999998, - 2.335, - 32.207 - ], - [ - 16, - 218496.95999999996, - 2.407, - 124.095 - ] - ], - "pinned_pods": 4, - "throughput": 259287.0, - "p50": 2.335, - "p99": 22.207, - "pod_mem_mb": 3247.0, - "saturated": true, - "status": "ok", - "reason": "plateau" - } - } -} \ No newline at end of file diff --git a/results/write-nvme-bottleneck/wal-s4/cells.json b/results/write-nvme-bottleneck/wal-s4/cells.json deleted file mode 100644 index 2592611..0000000 --- a/results/write-nvme-bottleneck/wal-s4/cells.json +++ /dev/null @@ -1,36 +0,0 @@ -{ - "cells": { - "500000": { - "stream_count": 500000, - "image_digest": "ce452f9cfacb", - "walk": [ - [ - 1, - 43001.4, - 5.043, - 24.207 - ], - [ - 2, - 41748.16, - 10.103, - 38.623 - ], - [ - 4, - 37944.8, - 24.495, - 67.135 - ] - ], - "pinned_pods": 1, - "throughput": 39937.7, - "p50": 4.879, - "p99": 29.407, - "pod_mem_mb": 2524.0, - "saturated": true, - "status": "ok", - "reason": "plateau" - } - } -} \ No newline at end of file diff --git a/results/write-wal-vs-mem-cpu4/FINDINGS.md b/results/write-wal-vs-mem-cpu4/FINDINGS.md deleted file mode 100644 index b642409..0000000 --- a/results/write-wal-vs-mem-cpu4/FINDINGS.md +++ /dev/null @@ -1,82 +0,0 @@ -# wal vs memory write saturation — 4 & 8 pinned vCPUs (2026-07-08, corrected) - -Covers **both** suites of the campaign: `write-wal-vs-mem-cpu4` (this dir) and -`write-wal-vs-mem-cpu8` (sibling). This is the **corrected** run: the first pass -(same day) quoted latency from the saturation plateau rung, which for a -closed-loop fleet is queueing (in-flight ÷ ceiling — Little's law), not service -time. Ladders now start at 1 pod, every rung records its own p50/p99 in the -walk, and latency is quoted from the ≤80%-of-peak **knee** rung. - -## Setup - -- **Server:** `durable-streams:dev` from `electric` branch - **`vb/ds-rust-memory-meta-sweep`** (#4675 wal coordination fixes + #4691 - memory meta-sidecar sweep). `c4d-standard-16-lssd` (physically-attached - Titanium NVMe), CPU-pinned via cgroup to **4** resp. **8** vCPUs. wal: - `--wal-shards N --worker-threads N` (N = pin); memory: - `--durability memory --worker-threads N`. -- **Client:** pool model — disjoint per-pod slices of the global key domain, - plain appends, slices pre-created before the fleet barrier, 256 - connections/pod @ `fleet_cpu=2`, batch 1, 256 B payloads, 25 s measure, - ladder 1 → 2 → 4 → … pods. Every cell `windows_aligned=true`, zero client - errors, `lazy_creates=0`. - -## Headline (plateau throughput; latency at the ≤80% knee rung) - -| server vCPU | streams | wal thr | wal p50/p99 (knee) | memory thr | memory p50/p99 (knee) | memory/wal | -|---|---|---|---|---|---|---| -| 4 | 100k | **47k** | 4.5 / 15.5 ms | **315k** | 1.2 / 2.7 ms | **6.7×** | -| 4 | 500k | **31k** | 6.7 / 30.7 ms | **226k** | 1.1 / 2.4 ms | **7.3×** | -| 8 | 100k | **43k** | 5.0 / 19.7 ms | **526k** | 1.2 / 2.5 ms | **12.2×** | -| 8 | 500k | **29k** | 6.9 / 32.9 ms | **323k** | 1.3 / 2.6 ms | **11.1×** | - -- **memory scales with cores** (315k → 526k @100k for 2× vCPU) at ~1–2 ms p50 - throughout; it pays a ~28–39% cardinality tax from 100k → 500k streams. -- **wal does not scale with cores** (47k@4 ≈ 43k@8) and its knee sits at - ~4.5–7 ms p50: the ceiling is the **commit path (fsync lanes = wal shards) on - the single NVMe partition**, reached at ~25–30% server CPU. Known lever: - shards > cores (`run-durable-tune`: s16t4 ≈ 380k @200k streams). - -## Methodology: latency is measured BELOW the knee (manually verified) - -The first pass reported wal@100k as "59k, p50 64 ms". Manual verification -(2026-07-08, `scripts/manual-wal-latency.sh` — plain curl + a single-pod -concurrency ramp on the same server shape) showed: - -| in-flight | ops/s | p50 | -|---|---|---| -| 1 (curl, not ds-bench) | — | **1.02 ms** | -| 64 | 19.8k | 2.2 ms | -| 256 | 45.5k | 5.1 ms | -| 1024 | 56.9k | 17.3 ms | -| 4096 | 61.5k | **65.7 ms** | - -One pod at 4096 in-flight reproduces the old 16-pod fleet cell (59k @ 64 ms) -almost exactly — the load generator was **accurate**, but every ladder rung sat -past the knee, so the recorded latency was queueing, not the server. Two -consequences, both fixed: - -1. **Latency** is now quoted from the largest rung at ≤80% of peak throughput - (the `## Latency` table in report.md; knee_* columns in aggregate.csv). The - plateau rung's latency is still shown, labelled as saturation queueing. -2. **"Saturation throughput" reads slightly lower than before** (wal 47k vs - 59k): the 8%-gain plateau rule with 1→2→4 rung steps now stops near the - knee instead of deep in the queue-heavy regime, where pushing 4096+ in-flight - buys ~25% more throughput at 10× the latency. The knee value is the honest - capacity number; the deep-saturation asymptote (~61k for wal@100k/4 vCPU) is - documented here for continuity. - -## Provenance - -- ds-bench branch `fix/fleet-measure-window-alignment`: pool-client rewrite - (disjoint slices, setup-before-barrier), per-rung latency in walks - (`saturation.py` / `lib-bench.sh` / `lib-saturate.sh`), knee reporting - (`report.py:knee_of`), `verify-offsets` server-truth tooling (delta 0 on kind - for both configs), fleet-apply retry + idempotent cluster adoption - (`cluster-up.sh`), scoped teardown watchdogs. -- Server image: Cloud Build 8dfd6c08 from `packages/durable-streams-rust` @ - `40b7c4677`; client image: Cloud Build a846e93c. -- Operational note: the first two rerun attempts died to (a) an unscoped - parallel watchdog sweeping the campaign's clusters mid-deploy — watchdogs are - now scoped via `CLUSTER_FILTER` — and (b) an interrupted cluster create - leaving no client pool — `cluster-up.sh` now heals that on adoption. diff --git a/results/write-wal-vs-mem-cpu4/aggregate.csv b/results/write-wal-vs-mem-cpu4/aggregate.csv deleted file mode 100644 index c7a88d1..0000000 --- a/results/write-wal-vs-mem-cpu4/aggregate.csv +++ /dev/null @@ -1,5 +0,0 @@ -mode,stream_count,pods,throughput,p50,p99,knee_pods,knee_throughput,knee_p50,knee_p99,pod_mem_mb,pod_mem_p50_mb,saturated,status,reason -memory,100000,2,315174.44,1.277,17.167,1,204417.28,1.181,2.399,631,569,True,ok,plateau -wal,100000,2,46838.119999999995,8.503,35.487,1,38245.36,4.463,35.775,613,519,True,ok,plateau -memory,500000,2,226488.52,1.347,18.431,1,164607.64,1.148,2.311,2876,2076,True,ok,plateau -wal,500000,1,30983.0,6.391,23.343,1,30983.0,6.655,26.927,2469,2162,True,ok,plateau diff --git a/results/write-wal-vs-mem-cpu4/aggregate.json b/results/write-wal-vs-mem-cpu4/aggregate.json deleted file mode 100644 index f886ada..0000000 --- a/results/write-wal-vs-mem-cpu4/aggregate.json +++ /dev/null @@ -1,144 +0,0 @@ -[ - { - "mode": "memory", - "stream_count": 100000, - "pods": 2, - "throughput": 315174.44, - "p50": 1.277, - "p99": 17.167, - "knee_pods": 1, - "knee_throughput": 204417.28, - "knee_p50": 1.181, - "knee_p99": 2.399, - "pod_mem_mb": 631, - "pod_mem_p50_mb": 569, - "saturated": true, - "status": "ok", - "reason": "plateau", - "walk": [ - [ - 1, - 204417.28, - 1.181, - 2.399 - ], - [ - 2, - 315174.44, - 1.31, - 17.759 - ], - [ - 4, - 319227.92, - 2.169, - 23.455 - ] - ] - }, - { - "mode": "wal", - "stream_count": 100000, - "pods": 2, - "throughput": 46838.119999999995, - "p50": 8.503, - "p99": 35.487, - "knee_pods": 1, - "knee_throughput": 38245.36, - "knee_p50": 4.463, - "knee_p99": 35.775, - "pod_mem_mb": 613, - "pod_mem_p50_mb": 519, - "saturated": true, - "status": "ok", - "reason": "plateau", - "walk": [ - [ - 1, - 38245.36, - 4.463, - 35.775 - ], - [ - 2, - 46838.119999999995, - 8.447, - 39.999 - ], - [ - 4, - 45983.36, - 17.823, - 55.615 - ] - ] - }, - { - "mode": "memory", - "stream_count": 500000, - "pods": 2, - "throughput": 226488.52, - "p50": 1.347, - "p99": 18.431, - "knee_pods": 1, - "knee_throughput": 164607.64, - "knee_p50": 1.148, - "knee_p99": 2.311, - "pod_mem_mb": 2876, - "pod_mem_p50_mb": 2076, - "saturated": true, - "status": "ok", - "reason": "plateau", - "walk": [ - [ - 1, - 164607.64, - 1.148, - 2.311 - ], - [ - 2, - 226488.52, - 1.352, - 13.327 - ], - [ - 4, - 206388.08000000002, - 2.229, - 31.711 - ] - ] - }, - { - "mode": "wal", - "stream_count": 500000, - "pods": 1, - "throughput": 30983.0, - "p50": 6.391, - "p99": 23.343, - "knee_pods": 1, - "knee_throughput": 30983.0, - "knee_p50": 6.655, - "knee_p99": 26.927, - "pod_mem_mb": 2469, - "pod_mem_p50_mb": 2162, - "saturated": true, - "status": "ok", - "reason": "plateau", - "walk": [ - [ - 1, - 30983.0, - 6.655, - 26.927 - ], - [ - 2, - 28356.120000000003, - 16.087, - 51.839 - ] - ] - } -] \ No newline at end of file diff --git a/results/write-wal-vs-mem-cpu4/memory/cells.json b/results/write-wal-vs-mem-cpu4/memory/cells.json deleted file mode 100644 index c1b0f75..0000000 --- a/results/write-wal-vs-mem-cpu4/memory/cells.json +++ /dev/null @@ -1,70 +0,0 @@ -{ - "cells": { - "100000": { - "stream_count": 100000, - "image_digest": "159e0d8b1abd", - "walk": [ - [ - 1, - 204417.28, - 1.181, - 2.399 - ], - [ - 2, - 315174.44, - 1.31, - 17.759 - ], - [ - 4, - 319227.92, - 2.169, - 23.455 - ] - ], - "pinned_pods": 2, - "throughput": 315174.44, - "p50": 1.277, - "p99": 17.167, - "pod_mem_mb": 631, - "saturated": true, - "status": "ok", - "reason": "plateau", - "pod_mem_p50_mb": 569 - }, - "500000": { - "stream_count": 500000, - "image_digest": "159e0d8b1abd", - "walk": [ - [ - 1, - 164607.64, - 1.148, - 2.311 - ], - [ - 2, - 226488.52, - 1.352, - 13.327 - ], - [ - 4, - 206388.08000000002, - 2.229, - 31.711 - ] - ], - "pinned_pods": 2, - "throughput": 226488.52, - "p50": 1.347, - "p99": 18.431, - "pod_mem_mb": 2876, - "saturated": true, - "status": "ok", - "reason": "plateau", - "pod_mem_p50_mb": 2076 - } - } -} \ No newline at end of file diff --git a/results/write-wal-vs-mem-cpu4/report.md b/results/write-wal-vs-mem-cpu4/report.md deleted file mode 100644 index 71a43c5..0000000 --- a/results/write-wal-vs-mem-cpu4/report.md +++ /dev/null @@ -1,43 +0,0 @@ -# write-wal-vs-mem-cpu4 — write-throughput report - -## Throughput at saturation (ops/s) - -| streams | wal | memory | -|---|---|---| -| 100000 | 47k | 315k | -| 500000 | 31k | 226k | - -† = not saturated (ladder exhausted) — treat as a lower bound. - -## Pod memory at saturation — peak / p50 (MiB) - -| streams | wal | memory | -|---|---|---| -| 100000 | 613 / 519 | 631 / 569 | -| 500000 | 2469 / 2162 | 2876 / 2076 | - -_Pod working set = cgroup `memory.current − inactive_file` (anon + active page cache), sampled each second at the pinned rung. **peak** = high-water (catches bursts like an in-RAM Raft log filling); **p50** = median (what the server steadily holds resident). peak ≈ p50 ⇒ steadily resident; peak ≫ p50 ⇒ transient spikes._ - -## Latency (ms, p50 / p99) - -| streams | wal @≤80% load | wal @saturation | memory @≤80% load | memory @saturation | -|---|---|---|---|---| -| 100000 | 4.5 / 35.8 (38k @1p) | 8.5 / 35.5 | 1.2 / 2.4 (204k @1p) | 1.3 / 17.2 | -| 500000 | 6.7 / 26.9 (31k @1p) | 6.4 / 23.3 | 1.1 / 2.3 (165k @1p) | 1.3 / 18.4 | - -_@≤80% load = the largest ladder rung at ≤80% of peak throughput — the server's service latency with headroom. @saturation = the pinned plateau rung, where a closed-loop fleet measures its own queueing (≈ in-flight ÷ ceiling by Little's law), NOT the server's per-request cost. Compare against the unloaded single-request baseline (~1 ms for wal) before reading anything into large saturation values._ - -## Saturation walks (pods → ops/s, p50 ms) - -- **memory 100000**: 1:204k@1.2ms → 2:315k@1.3ms → 4:319k@2.2ms (pinned 2, plateau) -- **wal 100000**: 1:38k@4.5ms → 2:47k@8.4ms → 4:46k@17.8ms (pinned 2, plateau) -- **memory 500000**: 1:165k@1.1ms → 2:226k@1.4ms → 4:206k@2.2ms (pinned 2, plateau) -- **wal 500000**: 1:31k@6.7ms → 2:28k@16.1ms (pinned 1, plateau) - -## Findings - -See [FINDINGS.md](FINDINGS.md) — corrected campaign write-up (memory 6.7-12x wal; latency quoted at the knee, saturation latency labelled as queueing; wal commit-path-bound, not CPU-bound). - -## Caveats - -_Single-node best-case; not 3-node Raft. Throughput is a saturation ceiling per the ladder._ diff --git a/results/write-wal-vs-mem-cpu4/wal/cells.json b/results/write-wal-vs-mem-cpu4/wal/cells.json deleted file mode 100644 index 6e37600..0000000 --- a/results/write-wal-vs-mem-cpu4/wal/cells.json +++ /dev/null @@ -1,64 +0,0 @@ -{ - "cells": { - "100000": { - "stream_count": 100000, - "image_digest": "1f64524bf39c", - "walk": [ - [ - 1, - 38245.36, - 4.463, - 35.775 - ], - [ - 2, - 46838.119999999995, - 8.447, - 39.999 - ], - [ - 4, - 45983.36, - 17.823, - 55.615 - ] - ], - "pinned_pods": 2, - "throughput": 46838.119999999995, - "p50": 8.503, - "p99": 35.487, - "pod_mem_mb": 613, - "saturated": true, - "status": "ok", - "reason": "plateau", - "pod_mem_p50_mb": 519 - }, - "500000": { - "stream_count": 500000, - "image_digest": "1f64524bf39c", - "walk": [ - [ - 1, - 30983.0, - 6.655, - 26.927 - ], - [ - 2, - 28356.120000000003, - 16.087, - 51.839 - ] - ], - "pinned_pods": 1, - "throughput": 30983.0, - "p50": 6.391, - "p99": 23.343, - "pod_mem_mb": 2469, - "saturated": true, - "status": "ok", - "reason": "plateau", - "pod_mem_p50_mb": 2162 - } - } -} \ No newline at end of file diff --git a/results/write-wal-vs-mem-cpu8/FINDINGS.md b/results/write-wal-vs-mem-cpu8/FINDINGS.md deleted file mode 100644 index d5464f8..0000000 --- a/results/write-wal-vs-mem-cpu8/FINDINGS.md +++ /dev/null @@ -1,10 +0,0 @@ -# wal vs memory write saturation — 8 pinned vCPUs (2026-07-08, corrected) - -Part of the two-suite corrected campaign. The combined write-up (headline table, -knee-latency methodology + the manual verification that motivated it, setup, -provenance) lives in -[`../write-wal-vs-mem-cpu4/FINDINGS.md`](../write-wal-vs-mem-cpu4/FINDINGS.md). - -This suite's numbers (plateau thr; knee p50): wal 43k @ 5.0 ms (100k) / 29k @ -6.9 ms (500k); memory **526k @ 1.2 ms** (100k) / **323k @ 1.3 ms** (500k) — -memory ≈ 11–12× wal at 8 vCPU, every cell `windows_aligned=true`. diff --git a/results/write-wal-vs-mem-cpu8/aggregate.csv b/results/write-wal-vs-mem-cpu8/aggregate.csv deleted file mode 100644 index b08c1c8..0000000 --- a/results/write-wal-vs-mem-cpu8/aggregate.csv +++ /dev/null @@ -1,5 +0,0 @@ -mode,stream_count,pods,throughput,p50,p99,knee_pods,knee_throughput,knee_p50,knee_p99,pod_mem_mb,pod_mem_p50_mb,saturated,status,reason -memory,100000,4,525966.44,1.575,8.087,2,388144.48,1.217,2.461,698,608,True,ok,plateau -wal,100000,1,42765.56,4.947,20.271,1,42765.56,5.023,19.711,575,490,True,ok,plateau -memory,500000,4,322757.2,1.683,12.343,1,152766.84,1.255,2.585,2629,1980,True,ok,plateau -wal,500000,1,29265.64,7.291,29.599,1,29265.64,6.875,32.927,2425,2291,True,ok,plateau diff --git a/results/write-wal-vs-mem-cpu8/aggregate.json b/results/write-wal-vs-mem-cpu8/aggregate.json deleted file mode 100644 index 2983875..0000000 --- a/results/write-wal-vs-mem-cpu8/aggregate.json +++ /dev/null @@ -1,150 +0,0 @@ -[ - { - "mode": "memory", - "stream_count": 100000, - "pods": 4, - "throughput": 525966.44, - "p50": 1.575, - "p99": 8.087, - "knee_pods": 2, - "knee_throughput": 388144.48, - "knee_p50": 1.217, - "knee_p99": 2.461, - "pod_mem_mb": 698, - "pod_mem_p50_mb": 608, - "saturated": true, - "status": "ok", - "reason": "plateau", - "walk": [ - [ - 1, - 185668.2, - 1.267, - 2.669 - ], - [ - 2, - 388144.48, - 1.217, - 2.461 - ], - [ - 4, - 525966.44, - 1.575, - 10.943 - ], - [ - 8, - 543566.3200000001, - 2.487, - 15.351 - ] - ] - }, - { - "mode": "wal", - "stream_count": 100000, - "pods": 1, - "throughput": 42765.56, - "p50": 4.947, - "p99": 20.271, - "knee_pods": 1, - "knee_throughput": 42765.56, - "knee_p50": 5.023, - "knee_p99": 19.711, - "pod_mem_mb": 575, - "pod_mem_p50_mb": 490, - "saturated": true, - "status": "ok", - "reason": "plateau", - "walk": [ - [ - 1, - 42765.56, - 5.023, - 19.711 - ], - [ - 2, - 40151.96000000001, - 9.495, - 42.655 - ] - ] - }, - { - "mode": "memory", - "stream_count": 500000, - "pods": 4, - "throughput": 322757.2, - "p50": 1.683, - "p99": 12.343, - "knee_pods": 1, - "knee_throughput": 152766.84, - "knee_p50": 1.255, - "knee_p99": 2.585, - "pod_mem_mb": 2629, - "pod_mem_p50_mb": 1980, - "saturated": true, - "status": "ok", - "reason": "plateau", - "walk": [ - [ - 1, - 152766.84, - 1.255, - 2.585 - ], - [ - 2, - 259454.36000000002, - 1.22, - 2.609 - ], - [ - 4, - 322757.2, - 1.671, - 32.863 - ], - [ - 8, - 299006.39999999997, - 2.643, - 45.759 - ] - ] - }, - { - "mode": "wal", - "stream_count": 500000, - "pods": 1, - "throughput": 29265.64, - "p50": 7.291, - "p99": 29.599, - "knee_pods": 1, - "knee_throughput": 29265.64, - "knee_p50": 6.875, - "knee_p99": 32.927, - "pod_mem_mb": 2425, - "pod_mem_p50_mb": 2291, - "saturated": true, - "status": "ok", - "reason": "plateau", - "walk": [ - [ - 1, - 29265.64, - 6.875, - 32.927 - ], - [ - 2, - 30179.879999999997, - 14.519, - 48.767 - ] - ] - } -] \ No newline at end of file diff --git a/results/write-wal-vs-mem-cpu8/memory/cells.json b/results/write-wal-vs-mem-cpu8/memory/cells.json deleted file mode 100644 index d69b75f..0000000 --- a/results/write-wal-vs-mem-cpu8/memory/cells.json +++ /dev/null @@ -1,82 +0,0 @@ -{ - "cells": { - "100000": { - "stream_count": 100000, - "image_digest": "f236da115d6e", - "walk": [ - [ - 1, - 185668.2, - 1.267, - 2.669 - ], - [ - 2, - 388144.48, - 1.217, - 2.461 - ], - [ - 4, - 525966.44, - 1.575, - 10.943 - ], - [ - 8, - 543566.3200000001, - 2.487, - 15.351 - ] - ], - "pinned_pods": 4, - "throughput": 525966.44, - "p50": 1.575, - "p99": 8.087, - "pod_mem_mb": 698, - "saturated": true, - "status": "ok", - "reason": "plateau", - "pod_mem_p50_mb": 608 - }, - "500000": { - "stream_count": 500000, - "image_digest": "f236da115d6e", - "walk": [ - [ - 1, - 152766.84, - 1.255, - 2.585 - ], - [ - 2, - 259454.36000000002, - 1.22, - 2.609 - ], - [ - 4, - 322757.2, - 1.671, - 32.863 - ], - [ - 8, - 299006.39999999997, - 2.643, - 45.759 - ] - ], - "pinned_pods": 4, - "throughput": 322757.2, - "p50": 1.683, - "p99": 12.343, - "pod_mem_mb": 2629, - "saturated": true, - "status": "ok", - "reason": "plateau", - "pod_mem_p50_mb": 1980 - } - } -} \ No newline at end of file diff --git a/results/write-wal-vs-mem-cpu8/report.md b/results/write-wal-vs-mem-cpu8/report.md deleted file mode 100644 index 8bd5828..0000000 --- a/results/write-wal-vs-mem-cpu8/report.md +++ /dev/null @@ -1,43 +0,0 @@ -# write-wal-vs-mem-cpu8 — write-throughput report - -## Throughput at saturation (ops/s) - -| streams | wal | memory | -|---|---|---| -| 100000 | 43k | 526k | -| 500000 | 29k | 323k | - -† = not saturated (ladder exhausted) — treat as a lower bound. - -## Pod memory at saturation — peak / p50 (MiB) - -| streams | wal | memory | -|---|---|---| -| 100000 | 575 / 490 | 698 / 608 | -| 500000 | 2425 / 2291 | 2629 / 1980 | - -_Pod working set = cgroup `memory.current − inactive_file` (anon + active page cache), sampled each second at the pinned rung. **peak** = high-water (catches bursts like an in-RAM Raft log filling); **p50** = median (what the server steadily holds resident). peak ≈ p50 ⇒ steadily resident; peak ≫ p50 ⇒ transient spikes._ - -## Latency (ms, p50 / p99) - -| streams | wal @≤80% load | wal @saturation | memory @≤80% load | memory @saturation | -|---|---|---|---|---| -| 100000 | 5.0 / 19.7 (43k @1p) | 4.9 / 20.3 | 1.2 / 2.5 (388k @2p) | 1.6 / 8.1 | -| 500000 | 6.9 / 32.9 (29k @1p) | 7.3 / 29.6 | 1.3 / 2.6 (153k @1p) | 1.7 / 12.3 | - -_@≤80% load = the largest ladder rung at ≤80% of peak throughput — the server's service latency with headroom. @saturation = the pinned plateau rung, where a closed-loop fleet measures its own queueing (≈ in-flight ÷ ceiling by Little's law), NOT the server's per-request cost. Compare against the unloaded single-request baseline (~1 ms for wal) before reading anything into large saturation values._ - -## Saturation walks (pods → ops/s, p50 ms) - -- **memory 100000**: 1:186k@1.3ms → 2:388k@1.2ms → 4:526k@1.6ms → 8:544k@2.5ms (pinned 4, plateau) -- **wal 100000**: 1:43k@5.0ms → 2:40k@9.5ms (pinned 1, plateau) -- **memory 500000**: 1:153k@1.3ms → 2:259k@1.2ms → 4:323k@1.7ms → 8:299k@2.6ms (pinned 4, plateau) -- **wal 500000**: 1:29k@6.9ms → 2:30k@14.5ms (pinned 1, plateau) - -## Findings - -See [FINDINGS.md](FINDINGS.md) — corrected campaign write-up (memory 6.7-12x wal; latency quoted at the knee, saturation latency labelled as queueing; wal commit-path-bound, not CPU-bound). - -## Caveats - -_Single-node best-case; not 3-node Raft. Throughput is a saturation ceiling per the ladder._ diff --git a/results/write-wal-vs-mem-cpu8/wal/cells.json b/results/write-wal-vs-mem-cpu8/wal/cells.json deleted file mode 100644 index 19a8283..0000000 --- a/results/write-wal-vs-mem-cpu8/wal/cells.json +++ /dev/null @@ -1,58 +0,0 @@ -{ - "cells": { - "100000": { - "stream_count": 100000, - "image_digest": "9d926b050142", - "walk": [ - [ - 1, - 42765.56, - 5.023, - 19.711 - ], - [ - 2, - 40151.96000000001, - 9.495, - 42.655 - ] - ], - "pinned_pods": 1, - "throughput": 42765.56, - "p50": 4.947, - "p99": 20.271, - "pod_mem_mb": 575, - "saturated": true, - "status": "ok", - "reason": "plateau", - "pod_mem_p50_mb": 490 - }, - "500000": { - "stream_count": 500000, - "image_digest": "9d926b050142", - "walk": [ - [ - 1, - 29265.64, - 6.875, - 32.927 - ], - [ - 2, - 30179.879999999997, - 14.519, - 48.767 - ] - ], - "pinned_pods": 1, - "throughput": 29265.64, - "p50": 7.291, - "p99": 29.599, - "pod_mem_mb": 2425, - "saturated": true, - "status": "ok", - "reason": "plateau", - "pod_mem_p50_mb": 2291 - } - } -} \ No newline at end of file diff --git a/suites/mixed-writes-hot.json b/suites/canonical-mixed-cal.json similarity index 58% rename from suites/mixed-writes-hot.json rename to suites/canonical-mixed-cal.json index 3d78e8d..5a1754d 100644 --- a/suites/mixed-writes-hot.json +++ b/suites/canonical-mixed-cal.json @@ -1,23 +1,20 @@ { - "suite": "mixed-writes-hot", + "suite": "canonical-mixed-cal", "workload": "mixed", "cluster": { "server_machine": "c4d-standard-16-lssd", "client_machine": "n2d-standard-32", "client_nodes": 2, "region": "europe-west4", - "cluster_name": "bench-mixed" + "cluster_name": "bench-canon-mixed" }, "mixed": { - "sweep": "readers", + "sweep": "writer_rate", "levels": [ - 16, - 64, - 256 + 0 ], "writers_per_stream": 1, - "writer_rate": 977, - "read_rate": 0, + "readers": 0, "subscribers": 0, "duration_secs": 20, "payload_bytes": 256, @@ -29,5 +26,6 @@ ], "stream_counts": [ 50 - ] -} + ], + "_doc": "CANONICAL mixed-workload calibration anchor: unthrottled writers -> single-pod mixed-shape ceiling; the interference sweeps pin writers at ~60% of this." +} \ No newline at end of file diff --git a/suites/mixed-delivery.json b/suites/canonical-mixed-delivery.json similarity index 71% rename from suites/mixed-delivery.json rename to suites/canonical-mixed-delivery.json index d66529a..7c9bf7d 100644 --- a/suites/mixed-delivery.json +++ b/suites/canonical-mixed-delivery.json @@ -1,12 +1,12 @@ { - "suite": "mixed-delivery", + "suite": "canonical-mixed-delivery", "workload": "mixed", "cluster": { "server_machine": "c4d-standard-16-lssd", "client_machine": "n2d-standard-32", "client_nodes": 2, "region": "europe-west4", - "cluster_name": "bench-mixed" + "cluster_name": "bench-canon-mixed" }, "mixed": { "sweep": "writer_rate", @@ -38,11 +38,12 @@ }, { "label": "memory", - "args": "--durability memory" + "args": "--durability memory --tier off" } ] }, "stream_counts": [ 2000 - ] -} + ], + "_doc": "CANONICAL delivery-under-write-load: spread SSE subscribers vs a write ladder; watches for the memory-mode delivery collapse found 2026-07-02." +} \ No newline at end of file diff --git a/suites/mixed-writes.json b/suites/canonical-mixed-writes.json similarity index 72% rename from suites/mixed-writes.json rename to suites/canonical-mixed-writes.json index e93406f..7536ef8 100644 --- a/suites/mixed-writes.json +++ b/suites/canonical-mixed-writes.json @@ -1,12 +1,12 @@ { - "suite": "mixed-writes", + "suite": "canonical-mixed-writes", "workload": "mixed", "cluster": { "server_machine": "c4d-standard-16-lssd", "client_machine": "n2d-standard-32", "client_nodes": 2, "region": "europe-west4", - "cluster_name": "bench-mixed" + "cluster_name": "bench-canon-mixed" }, "mixed": { "sweep": "readers", @@ -33,5 +33,6 @@ ], "stream_counts": [ 10000 - ] -} + ], + "_doc": "CANONICAL read/write interference: paced catch-up readers (1k/10k/100k) against pinned writes \u2014 the '100k readers cost writes nothing' claim." +} \ No newline at end of file diff --git a/suites/canonical-reads-live.json b/suites/canonical-reads-live.json deleted file mode 100644 index dd89bd2..0000000 --- a/suites/canonical-reads-live.json +++ /dev/null @@ -1,39 +0,0 @@ -{ - "suite": "canonical-reads-live", - "workload": "reads", - "cluster": { - "server_machine": "c4d-standard-16-lssd", - "client_machine": "n2d-standard-32", - "client_nodes": 2, - "region": "europe-west4", - "cluster_name": "bench-canon-reads", - "zone": "europe-west4-a" - }, - "reads": { - "mode": "long-poll", - "read_size_bytes": 4096, - "append_rate_per_sec": 50, - "seed_bytes": 0, - "duration_secs": 15, - "warmup_secs": 5, - "settle_secs": 2, - "connection_levels": [ - 32, - 128, - 512, - 1024, - 2048 - ], - "pods": 1, - "repeats": 1 - }, - "modes": [ - "wal", - "ursula" - ], - "stream_counts": [ - 100, - 1000 - ], - "_doc": "CANONICAL live-tail delivery benchmark (long-poll) \u2014 delivery latency vs connections." -} \ No newline at end of file diff --git a/suites/canonical-sustained.json b/suites/canonical-sustained.json deleted file mode 100644 index ae82844..0000000 --- a/suites/canonical-sustained.json +++ /dev/null @@ -1,44 +0,0 @@ -{ - "suite": "canonical-sustained", - "workload": "sustained", - "cluster": { - "server_machine": "c4d-standard-16-lssd", - "client_machine": "n2d-standard-32", - "client_nodes": 2, - "region": "europe-west4", - "cluster_name": "bench-canon", - "zone": "europe-west4-b" - }, - "sustained": { - "rate_per_stream": 10, - "duration_secs": 90, - "snapshot_secs": 5, - "pods": 1, - "fleet_cpu": 0.5, - "payload_bytes": 256, - "setup_concurrency": 64, - "repeats": 1 - }, - "modes": [ - "wal" - ], - "stream_counts": [ - 10, - 50, - 100, - 150 - ], - "server_configs": { - "wal": [ - { - "label": "wal", - "args": "--durability wal --wal-shards 4" - }, - { - "label": "memory", - "args": "--durability memory --tier off" - } - ] - }, - "_doc": "CANONICAL sustained-load stability benchmark (latency + server-memory over a long window). Standard single-fs manifest is fine here (stability, not throughput). memory arm passes --tier off (server refuses memory+tier)." -} \ No newline at end of file diff --git a/suites/catchup-durable.json b/suites/catchup-durable.json deleted file mode 100644 index 0f27589..0000000 --- a/suites/catchup-durable.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "suite": "catchup-durable", - "workload": "catchup", - "cluster": { - "server_machine": "c4d-standard-16-lssd", - "client_machine": "n2d-standard-32", - "client_nodes": 2, - "region": "europe-west4", - "cluster_name": "bench-cu-durable", - "zone": "europe-west4-c" - }, - "catchup": { "clients": 1000, "per_client_stream": true, "event_bytes": 1024, "snapshot_bytes": 51200, "pods": 1, "setup_concurrency": 64, "repeats": 1 }, - "modes": ["wal"], - "stream_counts": [200, 2000], - "server_configs": { "wal": [ { "label": "durable", "args": "--durability wal --wal-shards 4" } ] } -} diff --git a/suites/catchup-local.json b/suites/catchup-local.json deleted file mode 100644 index 514d107..0000000 --- a/suites/catchup-local.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "suite": "catchup-local", - "workload": "catchup", - "cluster": {}, - "catchup": { "clients": 20, "per_client_stream": true, "event_bytes": 1024, "snapshot_bytes": 0, "pods": 1, "setup_concurrency": 8, "repeats": 1 }, - "modes": ["wal"], - "stream_counts": [200, 1000] -} diff --git a/suites/catchup-node.json b/suites/catchup-node.json deleted file mode 100644 index a61f10c..0000000 --- a/suites/catchup-node.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "suite": "catchup-node", - "workload": "catchup", - "cluster": { - "server_machine": "c4d-standard-16-lssd", - "client_machine": "n2d-standard-32", - "client_nodes": 2, - "region": "europe-west4", - "cluster_name": "bench-cu-node", - "zone": "europe-west4-b" - }, - "catchup": { "clients": 1000, "per_client_stream": true, "event_bytes": 1024, "snapshot_bytes": 51200, "pods": 1, "setup_concurrency": 64, "repeats": 1 }, - "modes": ["node"], - "stream_counts": [200, 2000] -} diff --git a/suites/catchup-s2.json b/suites/catchup-s2.json deleted file mode 100644 index dc766cf..0000000 --- a/suites/catchup-s2.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "suite": "catchup-s2", - "workload": "catchup", - "cluster": { - "server_machine": "c4d-standard-16-lssd", - "client_machine": "n2d-standard-32", - "client_nodes": 2, - "region": "europe-west4", - "cluster_name": "bench-cu-s2", - "zone": "europe-west4-a" - }, - "catchup": { "clients": 1000, "per_client_stream": true, "event_bytes": 1024, "snapshot_bytes": 51200, "pods": 1, "setup_concurrency": 64, "repeats": 1 }, - "modes": ["s2"], - "stream_counts": [200, 2000] -} diff --git a/suites/catchup-ursula.json b/suites/catchup-ursula.json deleted file mode 100644 index 9d88e07..0000000 --- a/suites/catchup-ursula.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "suite": "catchup-ursula", - "workload": "catchup", - "cluster": { - "server_machine": "c4d-standard-16-lssd", - "client_machine": "n2d-standard-32", - "client_nodes": 2, - "region": "europe-west4", - "cluster_name": "bench-cu-ursula", - "zone": "europe-west4-b" - }, - "catchup": { "clients": 1000, "per_client_stream": true, "event_bytes": 1024, "snapshot_bytes": 51200, "pods": 1, "setup_concurrency": 64, "repeats": 1 }, - "modes": ["ursula"], - "stream_counts": [200, 2000] -} diff --git a/suites/mixed-cal-local.json b/suites/mixed-cal-local.json deleted file mode 100644 index 33af4a1..0000000 --- a/suites/mixed-cal-local.json +++ /dev/null @@ -1,36 +0,0 @@ -{ - "suite": "mixed-cal-local", - "workload": "mixed", - "cluster": {}, - "mixed": { - "sweep": "writer_rate", - "levels": [ - 0 - ], - "writers_per_stream": 1, - "readers": 0, - "subscribers": 0, - "duration_secs": 15, - "payload_bytes": 256, - "setup_concurrency": 16, - "pods": 1 - }, - "modes": [ - "wal" - ], - "stream_counts": [ - 50 - ], - "server_configs": { - "wal": [ - { - "label": "wal", - "args": "" - }, - { - "label": "memory", - "args": "--durability memory" - } - ] - } -} diff --git a/suites/mixed-cal.json b/suites/mixed-cal.json deleted file mode 100644 index a7fccea..0000000 --- a/suites/mixed-cal.json +++ /dev/null @@ -1,24 +0,0 @@ -{ - "suite": "mixed-cal", - "workload": "mixed", - "cluster": { - "server_machine": "c4d-standard-16-lssd", - "client_machine": "n2d-standard-32", - "client_nodes": 2, - "region": "europe-west4", - "cluster_name": "bench-mixed" - }, - "mixed": { - "sweep": "writer_rate", - "levels": [0], - "writers_per_stream": 1, - "readers": 0, - "subscribers": 0, - "duration_secs": 20, - "payload_bytes": 256, - "setup_concurrency": 16, - "pods": 1 - }, - "modes": ["wal"], - "stream_counts": [50] -} diff --git a/suites/mixed-delivery-local.json b/suites/mixed-delivery-local.json deleted file mode 100644 index 3d41040..0000000 --- a/suites/mixed-delivery-local.json +++ /dev/null @@ -1,40 +0,0 @@ -{ - "suite": "mixed-delivery-local", - "workload": "mixed", - "cluster": {}, - "mixed": { - "sweep": "writer_rate", - "levels": [ - 30, - 120, - 300, - 475, - 0 - ], - "writers_per_stream": 1, - "readers": 0, - "subscribers": 100, - "duration_secs": 20, - "payload_bytes": 256, - "setup_concurrency": 16, - "pods": 1 - }, - "modes": [ - "wal" - ], - "server_configs": { - "wal": [ - { - "label": "wal", - "args": "" - }, - { - "label": "memory", - "args": "--durability memory" - } - ] - }, - "stream_counts": [ - 50 - ] -} diff --git a/suites/mixed-writes-local.json b/suites/mixed-writes-local.json deleted file mode 100644 index 358f52b..0000000 --- a/suites/mixed-writes-local.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "suite": "mixed-writes-local", - "workload": "mixed", - "cluster": {}, - "mixed": { - "sweep": "readers", - "levels": [ - 0, - 4, - 16, - 64, - 128 - ], - "writers_per_stream": 1, - "writer_rate": 355, - "read_rate": 1, - "subscribers": 0, - "duration_secs": 20, - "payload_bytes": 256, - "setup_concurrency": 16, - "pods": 1 - }, - "modes": [ - "wal" - ], - "stream_counts": [ - 50 - ] -} diff --git a/suites/reads-catchup.json b/suites/reads-catchup.json deleted file mode 100644 index bdb08f0..0000000 --- a/suites/reads-catchup.json +++ /dev/null @@ -1,23 +0,0 @@ -{ - "suite": "reads-catchup", - "workload": "reads", - "cluster": { - "server_machine": "c4d-standard-16-lssd", - "client_machine": "n2d-standard-32", - "client_nodes": 2, - "region": "europe-west4" - }, - "reads": { - "mode": "catchup", - "read_size_bytes": 4096, - "seed_bytes": 16777216, - "duration_secs": 15, - "warmup_secs": 5, - "settle_secs": 2, - "connection_levels": [8, 32, 128, 512], - "pods": 1, - "repeats": 1 - }, - "modes": ["wal", "ursula"], - "stream_counts": [10, 100] -} diff --git a/suites/reads-local-ursula.json b/suites/reads-local-ursula.json deleted file mode 100644 index 5e7a3ed..0000000 --- a/suites/reads-local-ursula.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "suite": "reads-local-ursula", - "workload": "reads", - "cluster": {}, - "reads": { - "read_size_bytes": 4096, - "seed_bytes": 1048576, - "duration_secs": 10, - "warmup_secs": 2, - "settle_secs": 1, - "connection_levels": [8, 32], - "pods": 1, - "repeats": 1 - }, - "modes": ["ursula"], - "stream_counts": [10, 100] -} diff --git a/suites/reads-local.json b/suites/reads-local.json deleted file mode 100644 index 7c7cbaa..0000000 --- a/suites/reads-local.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "suite": "reads-local", - "workload": "reads", - "cluster": {}, - "reads": { - "read_size_bytes": 4096, - "seed_bytes": 1048576, - "duration_secs": 10, - "warmup_secs": 2, - "settle_secs": 1, - "connection_levels": [8, 32], - "pods": 1, - "repeats": 1 - }, - "modes": ["wal"], - "stream_counts": [10, 100] -} diff --git a/suites/reads-longpoll.json b/suites/reads-longpoll.json deleted file mode 100644 index c8f591e..0000000 --- a/suites/reads-longpoll.json +++ /dev/null @@ -1,24 +0,0 @@ -{ - "suite": "reads-longpoll", - "workload": "reads", - "cluster": { - "server_machine": "c4d-standard-16-lssd", - "client_machine": "n2d-standard-32", - "client_nodes": 2, - "region": "europe-west4" - }, - "reads": { - "mode": "long-poll", - "read_size_bytes": 4096, - "append_rate_per_sec": 50, - "seed_bytes": 0, - "duration_secs": 15, - "warmup_secs": 5, - "settle_secs": 2, - "connection_levels": [32, 128, 512, 1024, 2048], - "pods": 1, - "repeats": 1 - }, - "modes": ["wal", "ursula"], - "stream_counts": [100, 1000] -} diff --git a/suites/reads-sse-local.json b/suites/reads-sse-local.json deleted file mode 100644 index 5a85a5d..0000000 --- a/suites/reads-sse-local.json +++ /dev/null @@ -1,24 +0,0 @@ -{ - "suite": "reads-sse-local", - "workload": "reads", - "cluster": { - "server_machine": "kind", - "client_machine": "kind", - "client_nodes": 1, - "region": "local" - }, - "reads": { - "mode": "sse", - "read_size_bytes": 256, - "append_rate_per_sec": 50, - "seed_bytes": 0, - "duration_secs": 8, - "warmup_secs": 2, - "settle_secs": 1, - "connection_levels": [16, 64], - "pods": 1, - "repeats": 1 - }, - "modes": ["wal", "ursula"], - "stream_counts": [10] -} diff --git a/suites/reads-sse-remote.json b/suites/reads-sse-remote.json deleted file mode 100644 index bddc0c1..0000000 --- a/suites/reads-sse-remote.json +++ /dev/null @@ -1,24 +0,0 @@ -{ - "suite": "reads-sse-remote", - "workload": "reads", - "cluster": { - "server_machine": "c4d-standard-16-lssd", - "client_machine": "n2d-standard-32", - "client_nodes": 2, - "region": "europe-west4" - }, - "reads": { - "mode": "sse", - "read_size_bytes": 256, - "append_rate_per_sec": 50, - "seed_bytes": 0, - "duration_secs": 15, - "warmup_secs": 5, - "settle_secs": 2, - "connection_levels": [64, 256, 1024, 2048], - "pods": 1, - "repeats": 1 - }, - "modes": ["wal", "ursula"], - "stream_counts": [10, 100] -} diff --git a/suites/reads.json b/suites/reads.json deleted file mode 100644 index 4dc383e..0000000 --- a/suites/reads.json +++ /dev/null @@ -1,24 +0,0 @@ -{ - "suite": "reads", - "workload": "reads", - "cluster": { - "server_machine": "c4d-standard-16-lssd", - "client_machine": "n2d-standard-32", - "client_nodes": 2, - "region": "europe-west4", - "cluster_name": "bench-reads", - "zone": "europe-west4-b" - }, - "reads": { - "read_size_bytes": 4096, - "seed_bytes": 16777216, - "duration_secs": 60, - "warmup_secs": 10, - "settle_secs": 5, - "connection_levels": [8, 32, 128, 512], - "pods": 1, - "repeats": 1 - }, - "modes": ["wal", "ursula"], - "stream_counts": [10, 100, 1000] -} diff --git a/suites/run-durable-500k-sat.json b/suites/run-durable-500k-sat.json deleted file mode 100644 index 328b232..0000000 --- a/suites/run-durable-500k-sat.json +++ /dev/null @@ -1,28 +0,0 @@ -{ - "suite": "run-durable-500k-sat", - "cluster": { - "server_machine": "c4d-standard-32-lssd", - "client_machine": "n2d-standard-32", - "client_nodes": 6, - "region": "europe-west4" - }, - "saturation": { - "plateau_pct": 8, - "fleet_cpu": 0.5, - "repeats": 1, - "warmup_secs": 8, - "measure_secs": 12, - "setup_concurrency": 16, - "payload_bytes": 256 - }, - "modes": ["wal"], - "server_configs": { - "wal": [ - { "label": "cpu16-s16t16", "args": "--wal-shards 16 --worker-threads 16" } - ] - }, - "stream_counts": [500000], - "pod_ladder": { - "500000": [180, 250, 320] - } -} diff --git a/suites/run-durable-500k-sat2.json b/suites/run-durable-500k-sat2.json deleted file mode 100644 index c65d0a0..0000000 --- a/suites/run-durable-500k-sat2.json +++ /dev/null @@ -1,28 +0,0 @@ -{ - "suite": "run-durable-500k-sat2", - "cluster": { - "server_machine": "c4d-standard-32-lssd", - "client_machine": "n2d-standard-32", - "client_nodes": 6, - "region": "europe-west4" - }, - "saturation": { - "plateau_pct": 8, - "fleet_cpu": 0.25, - "repeats": 1, - "warmup_secs": 8, - "measure_secs": 12, - "setup_concurrency": 8, - "payload_bytes": 256 - }, - "modes": ["wal"], - "server_configs": { - "wal": [ - { "label": "cpu16-s16t16", "args": "--wal-shards 16 --worker-threads 16" } - ] - }, - "stream_counts": [500000], - "pod_ladder": { - "500000": [350, 500, 650] - } -} diff --git a/suites/run-durable-cal.json b/suites/run-durable-cal.json deleted file mode 100644 index b7f0b0f..0000000 --- a/suites/run-durable-cal.json +++ /dev/null @@ -1,30 +0,0 @@ -{ - "suite": "run-durable-cal", - "_doc": "Single-pod calibration (AGENTS.md §7 step 1): ONE fleet pod against an over-provisioned 32-core server, sweeping --connections (via CONNS_PER_POD) to find the single-pod max at fleet_cpu=2 / payload 256. 50k streams keeps perpod >= connections. The orchestrator runs this once per connection level and reads the p1 cell throughput.", - "cluster": { - "server_machine": "c4d-standard-32-lssd", - "client_machine": "n2d-standard-32", - "client_nodes": 4, - "region": "europe-west4" - }, - "saturation": { - "plateau_pct": 8, - "fleet_cpu": 2, - "connections": 256, - "repeats": 1, - "warmup_secs": 8, - "measure_secs": 15, - "setup_concurrency": 8, - "payload_bytes": 256 - }, - "modes": ["wal"], - "server_configs": { - "wal": [ - { "label": "cpu32-s32t32", "args": "--wal-shards 32 --worker-threads 32" } - ] - }, - "stream_counts": [50000], - "pod_ladder": { - "50000": [1] - } -} diff --git a/suites/run-durable-cpu16-1m-card-stats.json b/suites/run-durable-cpu16-1m-card-stats.json deleted file mode 100644 index 5d62cfc..0000000 --- a/suites/run-durable-cpu16-1m-card-stats.json +++ /dev/null @@ -1,39 +0,0 @@ -{ - "suite": "run-durable-cpu16-1m-card-stats", - "_doc": "Companion telemetry cell for run-durable-cpu16-1m-card: ONE 48-pod rung at 1M streams with --wal-stats 5, to read WAL_CONT + WAL_CKPT from server logs (checkpoint capture/fsync/tails/meta phase timing on real NVMe at 1M streams — decides whether the per-shard producer-state journal is needed next). Same image durable-streams:combined-card @ 662b0c845, same machine/config as the clean suite. Throughput/latency from this cell are secondary (stats clock reads perturb the tail).", - "cluster": { - "server_machine": "c4d-standard-16-lssd", - "client_machine": "n2d-standard-32", - "client_nodes": 6, - "region": "europe-west4" - }, - "saturation": { - "plateau_pct": -100, - "fleet_cpu": 2, - "connections": 256, - "repeats": 1, - "warmup_secs": 15, - "measure_secs": 60, - "setup_concurrency": 16, - "payload_bytes": 256 - }, - "modes": [ - "wal" - ], - "server_configs": { - "wal": [ - { - "label": "cpu16-card-stats", - "args": "--wal-shards 16 --worker-threads 16 --wal-stats 5" - } - ] - }, - "stream_counts": [ - 1000000 - ], - "pod_ladder": { - "1000000": [ - 48 - ] - } -} diff --git a/suites/run-durable-cpu16-1m-card.json b/suites/run-durable-cpu16-1m-card.json deleted file mode 100644 index 5e07792..0000000 --- a/suites/run-durable-cpu16-1m-card.json +++ /dev/null @@ -1,43 +0,0 @@ -{ - "suite": "run-durable-cpu16-1m-card", - "_doc": "Cardinality-fix validation at 16 vCPU: 1M + 500k streams on the combined stack PLUS the 2026-07-02 cardinality fixes (O(1) dirty drain, checkpoint fully off-runtime + concurrent shards, resident tails map, meta sidecar flush moved to checkpoint, single registry lookup). Image = durable-streams:combined-card @ 662b0c845. Direct baseline: run-durable-cpu16-1m-p99-combined (862k ops/s, p50 3.3/p99 32ms @ 32 pods, 77% of 16 cores, image f04baa64e). Ladder 32/48/64 pods at 1M finds the new ceiling (>=1M ops/s target); 500k rung quantifies the 500k->1M slope. clean config = NO --wal-stats (tail-accurate); stats config = one 48-pod rung with --wal-stats 1 to read WAL_CONT + WAL_CKPT (checkpoint meta/fsync phase behavior at 1M on real NVMe). SERVER_CPUS=16 on c4d-standard-16-lssd, --wal-shards 16 --worker-threads 16. Client fleet: 6x n2d-standard-32 (64 pods = 16384 conns).", - "cluster": { - "server_machine": "c4d-standard-16-lssd", - "client_machine": "n2d-standard-32", - "client_nodes": 6, - "region": "europe-west4" - }, - "saturation": { - "plateau_pct": -100, - "fleet_cpu": 2, - "connections": 256, - "repeats": 1, - "warmup_secs": 15, - "measure_secs": 60, - "setup_concurrency": 16, - "payload_bytes": 256 - }, - "modes": [ - "wal" - ], - "server_configs": { - "wal": [ - { - "label": "cpu16-card-clean", - "args": "--wal-shards 16 --worker-threads 16" - } - ] - }, - "stream_counts": [ - 500000, - 1000000 - ], - "pod_ladder": { - "500000": [ - 48 - ], - "1000000": [ - 32 - ] - } -} diff --git a/suites/run-durable-cpu16-1m-p99-combined.json b/suites/run-durable-cpu16-1m-p99-combined.json deleted file mode 100644 index 933f621..0000000 --- a/suites/run-durable-cpu16-1m-p99-combined.json +++ /dev/null @@ -1,39 +0,0 @@ -{ - "suite": "run-durable-cpu16-1m-p99-combined", - "_doc": "Write p99 at 16 vCPU (double the 8-vCPU run), SAME load: 1,000,000 streams, combined stack, single fixed rung (32 pool pods = 8192 connections). Compares directly against run-durable-cpu8-1m-p99-combined (which hit p50=2.9/p99=80ms @ 86% of 8 cores, 454k ops/s). Does doubling cores drop the tail / lift throughput at the same offered load? NO ladder. 60s measure. Image = durable-streams:combined @ f04baa64e. SERVER_CPUS=16 on c4d-standard-16-lssd, --wal-shards 16 --worker-threads 16. NO --wal-stats (would perturb the tail). Client fleet: 5x n2d-standard-32 (huge headroom for 32 pods).", - "cluster": { - "server_machine": "c4d-standard-16-lssd", - "client_machine": "n2d-standard-32", - "client_nodes": 5, - "region": "europe-west4" - }, - "saturation": { - "plateau_pct": -100, - "fleet_cpu": 2, - "connections": 256, - "repeats": 1, - "warmup_secs": 15, - "measure_secs": 60, - "setup_concurrency": 16, - "payload_bytes": 256 - }, - "modes": [ - "wal" - ], - "server_configs": { - "wal": [ - { - "label": "cpu16-combined-1m", - "args": "--wal-shards 16 --worker-threads 16" - } - ] - }, - "stream_counts": [ - 1000000 - ], - "pod_ladder": { - "1000000": [ - 32 - ] - } -} diff --git a/suites/run-durable-cpu16-clean.json b/suites/run-durable-cpu16-clean.json deleted file mode 100644 index 6686edc..0000000 --- a/suites/run-durable-cpu16-clean.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "suite": "run-durable-cpu16-clean", - "cluster": { - "server_machine": "c4d-standard-32-lssd", - "client_machine": "n2d-standard-32", - "client_nodes": 12, - "region": "europe-west4" - }, - "saturation": { - "plateau_pct": 5, - "fleet_cpu": 0.25, - "repeats": 1, - "warmup_secs": 8, - "measure_secs": 15, - "setup_concurrency": 6, - "payload_bytes": 256 - }, - "modes": ["wal"], - "server_configs": { - "wal": [ - { "label": "cpu16-s16t16", "args": "--wal-shards 16 --worker-threads 16" } - ] - }, - "stream_counts": [200000, 500000], - "pod_ladder": { - "200000": [160, 250], - "500000": [400, 625] - } -} diff --git a/suites/run-durable-cpu16.json b/suites/run-durable-cpu16.json deleted file mode 100644 index 7b9619b..0000000 --- a/suites/run-durable-cpu16.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "suite": "run-durable-cpu16", - "cluster": { - "server_machine": "c4d-standard-32-lssd", - "client_machine": "n2d-standard-32", - "client_nodes": 4, - "region": "europe-west4" - }, - "saturation": { - "plateau_pct": 8, - "fleet_cpu": 0.5, - "repeats": 1, - "warmup_secs": 8, - "measure_secs": 12, - "setup_concurrency": 16, - "payload_bytes": 256 - }, - "modes": ["wal"], - "server_configs": { - "wal": [ - { "label": "cpu16-s16t16", "args": "--wal-shards 16 --worker-threads 16" } - ] - }, - "stream_counts": [200000, 500000], - "pod_ladder": { - "200000": [80, 130], - "500000": [80, 130] - } -} diff --git a/suites/run-durable-cpu4-clean.json b/suites/run-durable-cpu4-clean.json deleted file mode 100644 index fc1704c..0000000 --- a/suites/run-durable-cpu4-clean.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "suite": "run-durable-cpu4-clean", - "cluster": { - "server_machine": "c4d-standard-32-lssd", - "client_machine": "n2d-standard-32", - "client_nodes": 12, - "region": "europe-west4" - }, - "saturation": { - "plateau_pct": 5, - "fleet_cpu": 0.25, - "repeats": 1, - "warmup_secs": 8, - "measure_secs": 15, - "setup_concurrency": 6, - "payload_bytes": 256 - }, - "modes": ["wal"], - "server_configs": { - "wal": [ - { "label": "cpu4-s4t4", "args": "--wal-shards 4 --worker-threads 4" } - ] - }, - "stream_counts": [200000, 500000], - "pod_ladder": { - "200000": [160, 250], - "500000": [400, 625] - } -} diff --git a/suites/run-durable-cpu4-pool-base.json b/suites/run-durable-cpu4-pool-base.json deleted file mode 100644 index 736e7f0..0000000 --- a/suites/run-durable-cpu4-pool-base.json +++ /dev/null @@ -1,49 +0,0 @@ -{ - "suite": "run-durable-cpu4-pool-base", - "_doc": "4-vCPU server contention test (SERVER_CPUS=4 on c4d-standard-16-lssd, --wal-shards 4 --worker-threads 4). Lean pool fleet (fleet_cpu=2, connections=256) \u2014 few big pods that still fully saturate a 4-core server, so the SERVER-SIDE metrics we care about (CPU-cap %, and the 200k->500k cardinality degradation) are measured at a fraction of the 12-node cpu4-clean cost. Compare server_cpu_pct (does it break the ~80% contention cap?) and throughput@200k vs @500k (does the cardinality penalty shrink?) across the baseline (:dev) and combined (:combined) images.", - "cluster": { - "server_machine": "c4d-standard-16-lssd", - "client_machine": "n2d-standard-32", - "client_nodes": 2, - "region": "europe-west4" - }, - "saturation": { - "plateau_pct": 8, - "fleet_cpu": 2, - "connections": 256, - "repeats": 1, - "warmup_secs": 10, - "measure_secs": 20, - "setup_concurrency": 8, - "payload_bytes": 256 - }, - "modes": [ - "wal" - ], - "server_configs": { - "wal": [ - { - "label": "cpu4-base", - "args": "--wal-shards 4 --worker-threads 4" - } - ] - }, - "stream_counts": [ - 200000, - 500000 - ], - "pod_ladder": { - "200000": [ - 12, - 16, - 20, - 24 - ], - "500000": [ - 12, - 16, - 20, - 24 - ] - } -} \ No newline at end of file diff --git a/suites/run-durable-cpu4-pool-combined.json b/suites/run-durable-cpu4-pool-combined.json deleted file mode 100644 index ea2ef6a..0000000 --- a/suites/run-durable-cpu4-pool-combined.json +++ /dev/null @@ -1,49 +0,0 @@ -{ - "suite": "run-durable-cpu4-pool-combined", - "_doc": "4-vCPU server contention test (SERVER_CPUS=4 on c4d-standard-16-lssd, --wal-shards 4 --worker-threads 4). Lean pool fleet (fleet_cpu=2, connections=256) \u2014 few big pods that still fully saturate a 4-core server, so the SERVER-SIDE metrics we care about (CPU-cap %, and the 200k->500k cardinality degradation) are measured at a fraction of the 12-node cpu4-clean cost. Compare server_cpu_pct (does it break the ~80% contention cap?) and throughput@200k vs @500k (does the cardinality penalty shrink?) across the baseline (:dev) and combined (:combined) images.", - "cluster": { - "server_machine": "c4d-standard-16-lssd", - "client_machine": "n2d-standard-32", - "client_nodes": 2, - "region": "europe-west4" - }, - "saturation": { - "plateau_pct": 8, - "fleet_cpu": 2, - "connections": 256, - "repeats": 1, - "warmup_secs": 10, - "measure_secs": 20, - "setup_concurrency": 8, - "payload_bytes": 256 - }, - "modes": [ - "wal" - ], - "server_configs": { - "wal": [ - { - "label": "cpu4-combined", - "args": "--wal-shards 4 --worker-threads 4" - } - ] - }, - "stream_counts": [ - 200000, - 500000 - ], - "pod_ladder": { - "200000": [ - 12, - 16, - 20, - 24 - ], - "500000": [ - 12, - 16, - 20, - 24 - ] - } -} \ No newline at end of file diff --git a/suites/run-durable-cpu4-sat-base.json b/suites/run-durable-cpu4-sat-base.json deleted file mode 100644 index ef71ec4..0000000 --- a/suites/run-durable-cpu4-sat-base.json +++ /dev/null @@ -1,51 +0,0 @@ -{ - "suite": "run-durable-cpu4-sat-base", - "_doc": "4-vCPU contention A/B (saturated). plateau_pct<0 forces the FULL ladder (no noisy early-stop); pick max per cell. Longer ladder to drive the 4-core server to ~400% CPU, 5 client nodes for headroom, 25s measure to cut noise. Compare (a) server CPU% at saturation: does combined break baseline's ~80% (~315/400) cap? and (b) 200k-max vs 500k-max: does combined remove the cardinality degradation? SERVER_CPUS=4, c4d-standard-16-lssd, --wal-shards 4 --worker-threads 4, pool conns=256.", - "cluster": { - "server_machine": "c4d-standard-16-lssd", - "client_machine": "n2d-standard-32", - "client_nodes": 5, - "region": "europe-west4" - }, - "saturation": { - "plateau_pct": -100, - "fleet_cpu": 2, - "connections": 256, - "repeats": 1, - "warmup_secs": 10, - "measure_secs": 25, - "setup_concurrency": 8, - "payload_bytes": 256 - }, - "modes": [ - "wal" - ], - "server_configs": { - "wal": [ - { - "label": "cpu4-base", - "args": "--wal-shards 4 --worker-threads 4" - } - ] - }, - "stream_counts": [ - 200000, - 500000 - ], - "pod_ladder": { - "200000": [ - 16, - 24, - 32, - 48, - 64 - ], - "500000": [ - 16, - 24, - 32, - 48, - 64 - ] - } -} \ No newline at end of file diff --git a/suites/run-durable-cpu4-sat-combined.json b/suites/run-durable-cpu4-sat-combined.json deleted file mode 100644 index d93d7ba..0000000 --- a/suites/run-durable-cpu4-sat-combined.json +++ /dev/null @@ -1,51 +0,0 @@ -{ - "suite": "run-durable-cpu4-sat-combined", - "_doc": "4-vCPU contention A/B (saturated). plateau_pct<0 forces the FULL ladder (no noisy early-stop); pick max per cell. Longer ladder to drive the 4-core server to ~400% CPU, 5 client nodes for headroom, 25s measure to cut noise. Compare (a) server CPU% at saturation: does combined break baseline's ~80% (~315/400) cap? and (b) 200k-max vs 500k-max: does combined remove the cardinality degradation? SERVER_CPUS=4, c4d-standard-16-lssd, --wal-shards 4 --worker-threads 4, pool conns=256.", - "cluster": { - "server_machine": "c4d-standard-16-lssd", - "client_machine": "n2d-standard-32", - "client_nodes": 5, - "region": "europe-west4" - }, - "saturation": { - "plateau_pct": -100, - "fleet_cpu": 2, - "connections": 256, - "repeats": 1, - "warmup_secs": 10, - "measure_secs": 25, - "setup_concurrency": 8, - "payload_bytes": 256 - }, - "modes": [ - "wal" - ], - "server_configs": { - "wal": [ - { - "label": "cpu4-combined", - "args": "--wal-shards 4 --worker-threads 4" - } - ] - }, - "stream_counts": [ - 200000, - 500000 - ], - "pod_ladder": { - "200000": [ - 16, - 24, - 32, - 48, - 64 - ], - "500000": [ - 16, - 24, - 32, - 48, - 64 - ] - } -} \ No newline at end of file diff --git a/suites/run-durable-cpu4-stats-base.json b/suites/run-durable-cpu4-stats-base.json deleted file mode 100644 index cb13119..0000000 --- a/suites/run-durable-cpu4-stats-base.json +++ /dev/null @@ -1,43 +0,0 @@ -{ - "suite": "run-durable-cpu4-stats-base", - "_doc": "MECHANISM PROOF (instrumented baseline). Single saturating rung (64 pods) at 200k and 500k on the fixed 4-vCPU server, --wal-stats 5 so the server emits per-interval WAL_CONT contention counters (inner_wait_load / dirty_wait_load / waiters_woken_avg). Goal: show the baseline's per-shard-lock contention RISES 200k->500k, i.e. cardinality drives contention. Image = durable-streams:baseline-stats (perf/contention-investigation, telemetry, NO fixes; reads available_parallelism under the 4-vCPU cgroup, so no --worker-threads flag). Paired with run-durable-cpu4-stats-combined.", - "cluster": { - "server_machine": "c4d-standard-16-lssd", - "client_machine": "n2d-standard-32", - "client_nodes": 5, - "region": "europe-west4" - }, - "saturation": { - "plateau_pct": -100, - "fleet_cpu": 2, - "connections": 256, - "repeats": 1, - "warmup_secs": 12, - "measure_secs": 30, - "setup_concurrency": 8, - "payload_bytes": 256 - }, - "modes": [ - "wal" - ], - "server_configs": { - "wal": [ - { - "label": "cpu4-base-stats", - "args": "--wal-shards 4 --wal-stats 5" - } - ] - }, - "stream_counts": [ - 200000, - 500000 - ], - "pod_ladder": { - "200000": [ - 64 - ], - "500000": [ - 64 - ] - } -} diff --git a/suites/run-durable-cpu4-stats-combined.json b/suites/run-durable-cpu4-stats-combined.json deleted file mode 100644 index 4fe0360..0000000 --- a/suites/run-durable-cpu4-stats-combined.json +++ /dev/null @@ -1,43 +0,0 @@ -{ - "suite": "run-durable-cpu4-stats-combined", - "_doc": "MECHANISM PROOF (combined stack). Single saturating rung (64 pods) at 200k and 500k on the fixed 4-vCPU server, --wal-stats 5 so the server emits per-interval WAL_CONT contention counters. Goal: show combined keeps inner_wait_load / dirty_wait_load ~0 and waiters_woken_avg flat across 200k->500k, i.e. cardinality no longer drives contention. Image = durable-streams:combined (perf/combined-t1a-t1c-t2a @ f04baa64e; --worker-threads 4 pins the pool to the 4-vCPU cgroup). Paired with run-durable-cpu4-stats-base.", - "cluster": { - "server_machine": "c4d-standard-16-lssd", - "client_machine": "n2d-standard-32", - "client_nodes": 5, - "region": "europe-west4" - }, - "saturation": { - "plateau_pct": -100, - "fleet_cpu": 2, - "connections": 256, - "repeats": 1, - "warmup_secs": 12, - "measure_secs": 30, - "setup_concurrency": 8, - "payload_bytes": 256 - }, - "modes": [ - "wal" - ], - "server_configs": { - "wal": [ - { - "label": "cpu4-combined-stats", - "args": "--wal-shards 4 --worker-threads 4 --wal-stats 5" - } - ] - }, - "stream_counts": [ - 200000, - 500000 - ], - "pod_ladder": { - "200000": [ - 64 - ], - "500000": [ - 64 - ] - } -} diff --git a/suites/run-durable-cpu8-1m-p99-combined.json b/suites/run-durable-cpu8-1m-p99-combined.json deleted file mode 100644 index 4da0adf..0000000 --- a/suites/run-durable-cpu8-1m-p99-combined.json +++ /dev/null @@ -1,39 +0,0 @@ -{ - "suite": "run-durable-cpu8-1m-p99-combined", - "_doc": "Write p99 at scale: 8-vCPU server, 1,000,000 streams, combined stack, single fixed rung (32 pool pods = 8192 connections, ~60-70% load on 8 vCPU -> a representative write p99 that is not sitting on the saturation cliff). NO ladder (plateau_pct<0 forces the single rung to run in full). Long 60s measure so the tail is well sampled. Image = durable-streams:combined (perf/combined-t1a-t1c-t2a @ f04baa64e). SERVER_CPUS=8, --wal-shards 8 --worker-threads 8. NO --wal-stats (hot-path timing would perturb the tail). Client fleet: 5x n2d-standard-32 (160 vCPU) for 32 pods x 2 vCPU = 64 vCPU requested -> huge client headroom so client scheduling never taints the p99.", - "cluster": { - "server_machine": "c4d-standard-16-lssd", - "client_machine": "n2d-standard-32", - "client_nodes": 5, - "region": "europe-west4" - }, - "saturation": { - "plateau_pct": -100, - "fleet_cpu": 2, - "connections": 256, - "repeats": 1, - "warmup_secs": 15, - "measure_secs": 60, - "setup_concurrency": 16, - "payload_bytes": 256 - }, - "modes": [ - "wal" - ], - "server_configs": { - "wal": [ - { - "label": "cpu8-combined-1m", - "args": "--wal-shards 8 --worker-threads 8" - } - ] - }, - "stream_counts": [ - 1000000 - ], - "pod_ladder": { - "1000000": [ - 32 - ] - } -} diff --git a/suites/run-durable-cpu8-clean.json b/suites/run-durable-cpu8-clean.json deleted file mode 100644 index e5fbeab..0000000 --- a/suites/run-durable-cpu8-clean.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "suite": "run-durable-cpu8-clean", - "cluster": { - "server_machine": "c4d-standard-32-lssd", - "client_machine": "n2d-standard-32", - "client_nodes": 12, - "region": "europe-west4" - }, - "saturation": { - "plateau_pct": 5, - "fleet_cpu": 0.25, - "repeats": 1, - "warmup_secs": 8, - "measure_secs": 15, - "setup_concurrency": 6, - "payload_bytes": 256 - }, - "modes": ["wal"], - "server_configs": { - "wal": [ - { "label": "cpu8-s8t8", "args": "--wal-shards 8 --worker-threads 8" } - ] - }, - "stream_counts": [200000, 500000], - "pod_ladder": { - "200000": [160, 250], - "500000": [400, 625] - } -} diff --git a/suites/run-durable-cpu8.json b/suites/run-durable-cpu8.json deleted file mode 100644 index bc97a4e..0000000 --- a/suites/run-durable-cpu8.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "suite": "run-durable-cpu8", - "cluster": { - "server_machine": "c4d-standard-32-lssd", - "client_machine": "n2d-standard-32", - "client_nodes": 4, - "region": "europe-west4" - }, - "saturation": { - "plateau_pct": 8, - "fleet_cpu": 0.5, - "repeats": 1, - "warmup_secs": 8, - "measure_secs": 12, - "setup_concurrency": 16, - "payload_bytes": 256 - }, - "modes": ["wal"], - "server_configs": { - "wal": [ - { "label": "cpu8-s8t8", "args": "--wal-shards 8 --worker-threads 8" } - ] - }, - "stream_counts": [200000, 500000], - "pod_ladder": { - "200000": [80, 130], - "500000": [80, 130] - } -} diff --git a/suites/run-durable-dense-local.json b/suites/run-durable-dense-local.json deleted file mode 100644 index d1b067a..0000000 --- a/suites/run-durable-dense-local.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "suite": "run-durable-dense-local", - "cluster": {}, - "saturation": { "plateau_pct": 10, "fleet_cpu": 0.1, "repeats": 1, "warmup_secs": 3, "measure_secs": 5, "setup_concurrency": 8, "payload_bytes": 256 }, - "modes": ["wal"], - "server_configs": { - "wal": [ - { "label": "wal", "args": "" } - ] - }, - "stream_counts": [5000, 20000], - "pod_ladder": { "5000": [1, 2], "20000": [2, 4] } -} diff --git a/suites/run-durable-dense.json b/suites/run-durable-dense.json deleted file mode 100644 index 6046003..0000000 --- a/suites/run-durable-dense.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "suite": "run-durable-dense", - "cluster": { - "server_machine": "c4d-standard-16-lssd", - "client_machine": "n2d-standard-32", - "client_nodes": 4, - "region": "europe-west4" - }, - "saturation": { - "plateau_pct": 8, - "fleet_cpu": 0.5, - "repeats": 1, - "warmup_secs": 8, - "measure_secs": 8, - "setup_concurrency": 16, - "payload_bytes": 256 - }, - "modes": ["wal"], - "server_configs": { - "wal": [ - { "label": "wal", "args": "" } - ] - }, - "stream_counts": [200000, 500000], - "pod_ladder": { - "200000": [15, 25, 40], - "500000": [30, 50, 80] - } -} diff --git a/suites/run-durable-pool-barrier.json b/suites/run-durable-pool-barrier.json deleted file mode 100644 index 40dfe5f..0000000 --- a/suites/run-durable-pool-barrier.json +++ /dev/null @@ -1,36 +0,0 @@ -{ - "suite": "run-durable-pool-barrier", - "_doc": "Raw write-throughput ladder: RANDOM-DOMAIN pool client (every pod's workers draw uniformly random keys over the GLOBAL stream domain; plain appends, no producers; streams created lazily on first touch — no pre-sharding, no setup gate) under the fleet start barrier. Server on c4d-standard-32-lssd: the -32 lssd shape carries more local-SSD partitions than -16 (whose single 375GB partition write-caps at ~350MB/s — the disk ceiling measured on 2026-07-07); server remains CPU-pinned to 4 vCPUs for comparability. warmup 30s absorbs the lazy-creation storm before the measured window. Ladder is intentionally tall — keep climbing until a real plateau.", - "cluster": { - "server_machine": "c4d-standard-32-lssd", - "client_machine": "n2d-standard-32", - "client_nodes": 4, - "region": "europe-west4" - }, - "saturation": { - "plateau_pct": 8, - "fleet_cpu": 2, - "connections": 256, - "repeats": 1, - "warmup_secs": 30, - "measure_secs": 20, - "setup_concurrency": 16, - "payload_bytes": 256 - }, - "modes": ["wal"], - "server_configs": { - "wal": [ - { "label": "wal", "args": "" }, - { "label": "memory", "args": "--durability memory" } - ] - }, - "stream_counts": [100, 1000, 10000, 100000, 200000, 500000], - "pod_ladder": { - "100": [4, 8, 12, 16, 24, 32], - "1000": [4, 8, 12, 16, 24, 32], - "10000": [4, 8, 12, 16, 24, 32], - "100000": [4, 8, 12, 16, 24, 32], - "200000": [4, 8, 12, 16, 24, 32], - "500000": [4, 8, 12, 16, 24, 32] - } -} diff --git a/suites/run-durable-pool-opt.json b/suites/run-durable-pool-opt.json deleted file mode 100644 index 3f0eb8f..0000000 --- a/suites/run-durable-pool-opt.json +++ /dev/null @@ -1,32 +0,0 @@ -{ - "suite": "run-durable-pool-opt", - "_doc": "Cost-optimized write saturation (AGENTS.md §7 + Fleet cost levers). Combines: (1) batch=50 records/POST — ~35x more records/s per client vCPU, so far fewer pods; (2) n2d-highcpu-32 fleet (cheaper, low-RAM — pool client is CPU-bound); (3) a TIGHTER NVMe server c4d-standard-16-lssd (SERVER_CPUS=16), since the 32-core server only used ~25 cores at saturation. connections + pod_ladder here are a STARTING POINT — re-calibrate single-pod max at batch=50/fleet_cpu=2 (it changes the knee) before trusting the ladder; the orchestrator's calibration phase does this.", - "cluster": { - "server_machine": "c4d-standard-16-lssd", - "client_machine": "n2d-highcpu-32", - "client_nodes": 2, - "region": "europe-west4" - }, - "saturation": { - "plateau_pct": 8, - "fleet_cpu": 2, - "connections": 256, - "batch": 50, - "repeats": 1, - "warmup_secs": 10, - "measure_secs": 20, - "setup_concurrency": 8, - "payload_bytes": 256 - }, - "modes": ["wal"], - "server_configs": { - "wal": [ - { "label": "cpu16-s16t16", "args": "--wal-shards 16 --worker-threads 16" } - ] - }, - "stream_counts": [200000, 500000], - "pod_ladder": { - "200000": [4, 8, 16], - "500000": [4, 8, 16] - } -} diff --git a/suites/run-durable-pool.json b/suites/run-durable-pool.json deleted file mode 100644 index 512f685..0000000 --- a/suites/run-durable-pool.json +++ /dev/null @@ -1,31 +0,0 @@ -{ - "suite": "run-durable-pool", - "_doc": "Bounded-concurrency pool client (see AGENTS.md §7). Per-pod load = 80% of the single-pod max found by the calibration phase (CONNS_PER_POD overrides saturation.connections). The pod_ladder ramps pod COUNT until the 32-core wal server saturates, at 200k and 500k streams. fleet_cpu=2 matches the calibration; client_nodes sized so the top rung (pods x fleet_cpu) fits with headroom.", - "cluster": { - "server_machine": "c4d-standard-32-lssd", - "client_machine": "n2d-standard-32", - "client_nodes": 4, - "region": "europe-west4" - }, - "saturation": { - "plateau_pct": 8, - "fleet_cpu": 2, - "connections": 1500, - "repeats": 1, - "warmup_secs": 10, - "measure_secs": 20, - "setup_concurrency": 8, - "payload_bytes": 256 - }, - "modes": ["wal"], - "server_configs": { - "wal": [ - { "label": "cpu32-s32t32", "args": "--wal-shards 32 --worker-threads 32" } - ] - }, - "stream_counts": [200000, 500000], - "pod_ladder": { - "200000": [4, 8, 16, 24, 32], - "500000": [4, 8, 16, 24, 32] - } -} diff --git a/suites/run-durable-pool2-combined.json b/suites/run-durable-pool2-combined.json deleted file mode 100644 index 7084d71..0000000 --- a/suites/run-durable-pool2-combined.json +++ /dev/null @@ -1,47 +0,0 @@ -{ - "suite": "run-durable-pool2-combined", - "_doc": "Combined Tier-1/Tier-2 stack (perf/combined-t1a-t1c-t2a) on the run-durable-pool2 saturation ladder. Isolated suite name so results don't collide with the baseline run-durable-pool2 resume store. Run with SERVER_CPUS=32 CONNS_PER_POD=256 IMG_SERVER=.../durable-streams:combined.", - "cluster": { - "server_machine": "c4d-standard-32-lssd", - "client_machine": "n2d-standard-32", - "client_nodes": 6, - "region": "europe-west4" - }, - "saturation": { - "plateau_pct": 8, - "fleet_cpu": 2, - "connections": 256, - "repeats": 1, - "warmup_secs": 10, - "measure_secs": 20, - "setup_concurrency": 8, - "payload_bytes": 256 - }, - "modes": [ - "wal" - ], - "server_configs": { - "wal": [ - { - "label": "combined-s32t32", - "args": "--wal-shards 32 --worker-threads 32" - } - ] - }, - "stream_counts": [ - 200000, - 500000 - ], - "pod_ladder": { - "200000": [ - 40, - 52, - 64 - ], - "500000": [ - 40, - 52, - 64 - ] - } -} \ No newline at end of file diff --git a/suites/run-durable-pool2.json b/suites/run-durable-pool2.json deleted file mode 100644 index 597b74e..0000000 --- a/suites/run-durable-pool2.json +++ /dev/null @@ -1,31 +0,0 @@ -{ - "suite": "run-durable-pool2", - "_doc": "Extended pod ladder for run-durable-pool: the first sweep scaled linearly to 942k ops/s @200k/32pods with the 32-core server still at ~78% CPU (ladder_exhausted, not saturated). This pushes pods to 40/52/64 (per-pod fixed at the calibrated 256 conns) on 6 client nodes to peg the server and find the true ceiling for 200k + 500k.", - "cluster": { - "server_machine": "c4d-standard-32-lssd", - "client_machine": "n2d-standard-32", - "client_nodes": 6, - "region": "europe-west4" - }, - "saturation": { - "plateau_pct": 8, - "fleet_cpu": 2, - "connections": 256, - "repeats": 1, - "warmup_secs": 10, - "measure_secs": 20, - "setup_concurrency": 8, - "payload_bytes": 256 - }, - "modes": ["wal"], - "server_configs": { - "wal": [ - { "label": "cpu32-s32t32", "args": "--wal-shards 32 --worker-threads 32" } - ] - }, - "stream_counts": [200000, 500000], - "pod_ladder": { - "200000": [40, 52, 64], - "500000": [40, 52, 64] - } -} diff --git a/suites/run-durable-tune.json b/suites/run-durable-tune.json deleted file mode 100644 index df8e25a..0000000 --- a/suites/run-durable-tune.json +++ /dev/null @@ -1,31 +0,0 @@ -{ - "suite": "run-durable-tune", - "cluster": { - "server_machine": "c4d-standard-16-lssd", - "client_machine": "n2d-standard-32", - "client_nodes": 4, - "region": "europe-west4" - }, - "saturation": { - "plateau_pct": 8, - "fleet_cpu": 0.5, - "repeats": 1, - "warmup_secs": 8, - "measure_secs": 12, - "setup_concurrency": 16, - "payload_bytes": 256 - }, - "modes": ["wal"], - "server_configs": { - "wal": [ - { "label": "s4t4", "args": "--wal-shards 4 --worker-threads 4" }, - { "label": "s8t4", "args": "--wal-shards 8 --worker-threads 4" }, - { "label": "s16t4", "args": "--wal-shards 16 --worker-threads 4" }, - { "label": "s4t8", "args": "--wal-shards 4 --worker-threads 8" }, - { "label": "s8t8", "args": "--wal-shards 8 --worker-threads 8" }, - { "label": "s16t16", "args": "--wal-shards 16 --worker-threads 16" } - ] - }, - "stream_counts": [200000], - "pod_ladder": { "200000": [50] } -} diff --git a/suites/run-durable.json b/suites/run-durable.json deleted file mode 100644 index 8888ad5..0000000 --- a/suites/run-durable.json +++ /dev/null @@ -1,77 +0,0 @@ -{ - "suite": "run-durable", - "cluster": { - "server_machine": "c4d-standard-16-lssd", - "client_machine": "n2d-standard-32", - "client_nodes": 12, - "region": "europe-west4" - }, - "saturation": { - "plateau_pct": 8, - "fleet_cpu": 0.5, - "repeats": 1, - "warmup_secs": 8, - "measure_secs": 8, - "setup_concurrency": 6, - "payload_bytes": 256 - }, - "modes": [ - "wal" - ], - "server_configs": { - "wal": [ - { - "label": "wal", - "args": "" - }, - { - "label": "wal-tailcache", - "args": "--tail-cache-bytes 65536" - }, - { - "label": "memory", - "args": "--durability memory" - } - ] - }, - "stream_counts": [ - 100, - 1000, - 10000, - 100000, - 200000, - 500000 - ], - "pod_ladder": { - "100": [ - 16, - 24, - 32 - ], - "1000": [ - 16, - 24, - 32 - ], - "10000": [ - 32, - 48, - 64 - ], - "100000": [ - 80, - 100, - 110 - ], - "200000": [ - 100, - 160, - 250 - ], - "500000": [ - 250, - 400, - 625 - ] - } -} diff --git a/suites/run-node.json b/suites/run-node.json deleted file mode 100644 index 0f349ca..0000000 --- a/suites/run-node.json +++ /dev/null @@ -1,25 +0,0 @@ -{ - "suite": "run-node", - "cluster": { - "server_machine": "c4d-standard-16-lssd", - "client_machine": "n2d-standard-32", - "client_nodes": 2, - "region": "europe-west4" - }, - "saturation": { - "plateau_pct": 8, - "fleet_cpu": 0.5, - "repeats": 1, - "warmup_secs": 8, - "measure_secs": 8, - "setup_concurrency": 16, - "payload_bytes": 256 - }, - "modes": ["node"], - "stream_counts": [100, 1000, 10000], - "pod_ladder": { - "100": [4, 8, 16], - "1000": [4, 8, 16], - "10000": [8, 16, 24] - } -} diff --git a/suites/run-s2.json b/suites/run-s2.json deleted file mode 100644 index 9056939..0000000 --- a/suites/run-s2.json +++ /dev/null @@ -1,31 +0,0 @@ -{ - "suite": "run-s2", - "cluster": { - "server_machine": "c4d-standard-16-lssd", - "client_machine": "n2d-standard-32", - "client_nodes": 1, - "region": "europe-west4" - }, - "saturation": { - "plateau_pct": 8, - "fleet_cpu": 0.5, - "repeats": 1, - "warmup_secs": 8, - "measure_secs": 8, - "setup_concurrency": 16, - "payload_bytes": 256 - }, - "modes": [ - "s2" - ], - "stream_counts": [ - 100 - ], - "pod_ladder": { - "100": [ - 2, - 4, - 8 - ] - } -} diff --git a/suites/run-ursula.json b/suites/run-ursula.json deleted file mode 100644 index 9d5fbe5..0000000 --- a/suites/run-ursula.json +++ /dev/null @@ -1,31 +0,0 @@ -{ - "suite": "run-ursula", - "cluster": { - "server_machine": "c4d-standard-16-lssd", - "client_machine": "n2d-standard-32", - "client_nodes": 2, - "region": "europe-west4" - }, - "saturation": { - "plateau_pct": 8, - "fleet_cpu": 0.5, - "repeats": 1, - "warmup_secs": 8, - "measure_secs": 8, - "setup_concurrency": 16, - "payload_bytes": 256 - }, - "modes": ["ursula"], - "server_configs": { - "ursula": [ - { "label": "ursula-memory", "args": "memory" }, - { "label": "ursula-disk", "args": "disk" } - ] - }, - "stream_counts": [100, 1000, 10000], - "pod_ladder": { - "100": [4, 8, 16], - "1000": [4, 8, 16], - "10000": [8, 16, 24] - } -} diff --git a/suites/sustained-local-base.json b/suites/sustained-local-base.json deleted file mode 100644 index 7351791..0000000 --- a/suites/sustained-local-base.json +++ /dev/null @@ -1,23 +0,0 @@ -{ - "suite": "sustained-local-base", - "workload": "sustained", - "cluster": {}, - "sustained": { - "rate_per_stream": 10, - "duration_secs": 90, - "snapshot_secs": 5, - "pods": 1, - "fleet_cpu": 0.5, - "payload_bytes": 256, - "setup_concurrency": 64, - "repeats": 1 - }, - "modes": ["wal"], - "stream_counts": [10, 100, 150], - "server_configs": { - "wal": [ - { "label": "wal", "args": "--durability wal --wal-shards 4" }, - { "label": "memory", "args": "--durability memory" } - ] - } -} diff --git a/suites/sustained-local-fix.json b/suites/sustained-local-fix.json deleted file mode 100644 index e13c419..0000000 --- a/suites/sustained-local-fix.json +++ /dev/null @@ -1,23 +0,0 @@ -{ - "suite": "sustained-local-fix", - "workload": "sustained", - "cluster": {}, - "sustained": { - "rate_per_stream": 10, - "duration_secs": 90, - "snapshot_secs": 5, - "pods": 1, - "fleet_cpu": 0.5, - "payload_bytes": 256, - "setup_concurrency": 64, - "repeats": 1 - }, - "modes": ["wal"], - "stream_counts": [10, 100, 150], - "server_configs": { - "wal": [ - { "label": "wal", "args": "--durability wal --wal-shards 4" }, - { "label": "memory", "args": "--durability memory" } - ] - } -} diff --git a/suites/sustained.json b/suites/sustained.json deleted file mode 100644 index 502342f..0000000 --- a/suites/sustained.json +++ /dev/null @@ -1,30 +0,0 @@ -{ - "suite": "sustained", - "workload": "sustained", - "cluster": { - "server_machine": "c4d-standard-16-lssd", - "client_machine": "n2d-standard-32", - "client_nodes": 2, - "region": "europe-west4", - "cluster_name": "bench-sustained", - "zone": "europe-west4-b" - }, - "sustained": { - "rate_per_stream": 10, - "duration_secs": 90, - "snapshot_secs": 5, - "pods": 1, - "fleet_cpu": 0.5, - "payload_bytes": 256, - "setup_concurrency": 64, - "repeats": 1 - }, - "modes": ["wal"], - "stream_counts": [10, 50, 100, 150], - "server_configs": { - "wal": [ - { "label": "wal", "args": "--durability wal --wal-shards 4" }, - { "label": "memory", "args": "--durability memory" } - ] - } -} diff --git a/suites/wal-1m-diag.json b/suites/wal-1m-diag.json deleted file mode 100644 index 369aab7..0000000 --- a/suites/wal-1m-diag.json +++ /dev/null @@ -1,47 +0,0 @@ -{ - "suite": "wal-1m-diag", - "_doc": "Diagnose the NEW ~1M-stream wall found by wal-stacked-1m (stacked ideal config: 383k @100k, 244k @500k, 56k @1M). Candidates: (a) one open fd per live stream vs the container nofile ceiling (~1.05M) — 1M streams sits right at it, the accept loop sheds on EMFILE; (b) ext4 streams/ directory with 1M files; (c) tails-map full rewrite per checkpoint (cumulative map = O(N_total) write); (d) post-setup settle: creating 1M streams leaves 1M dirty files + meta sidecars mid-writeback and the 12s warmup measured the storm, not steady state. This suite separates (d) from the rest via warmup_secs 90 (vs 12), and the launcher runs a telemetry capture loop (SRV_STATS / WAL_CKPT / WAL_CONT + live fd count of the server process every 15s -> results/wal-1m-diag/telemetry-.log) to identify (a)/(b)/(c) directly. Same stacked config/image as wal-stacked-1m. If 1M with a long warmup recovers to ~200k+, the wall is mostly (d); if fd count plateaus at ~1.05M with appends collapsing, it is (a); WAL_CKPT fsync_us/cadence exposes (b)/(c). Requires STATIC_CPU=1 SPLITLANE=1 GUARANTEED=1 SERVER_LOCAL_SSD_BLOCK=1 SPOT_SERVER=1 BARRIER_SETUP_TIMEOUT_SECS=3600 IMG_SERVER=...:sizetrigger.", - "cluster": { - "server_machine": "c4d-standard-64-lssd", - "server_cpus": 8, - "client_machine": "n2d-standard-32", - "client_nodes": 3, - "cluster_name": "bench-cpubind", - "zone": "europe-west4-b", - "region": "europe-west4" - }, - "saturation": { - "plateau_pct": 8, - "patience": 2, - "fleet_cpu": 2, - "connections": 256, - "repeats": 1, - "warmup_secs": 90, - "measure_secs": 30, - "setup_concurrency": 32, - "payload_bytes": 256 - }, - "modes": [ - "wal" - ], - "server_configs": { - "wal": [ - { - "label": "stacked", - "args": "--wal-shards 5 --wal-checkpoint-syncfs on --wal-checkpoint-wal-bytes 1073741824 --wal-checkpoint-interval-ms 60000 --worker-threads 8 --wal-stats 3 --server-stats 3 --data-dir /data/wal/0" - } - ] - }, - "stream_counts": [ - 500000, - 1000000 - ], - "pod_ladder": { - "500000": [ - 8 - ], - "1000000": [ - 8 - ] - } -} diff --git a/suites/wal-batched-mw-local.json b/suites/wal-batched-mw-local.json deleted file mode 100644 index 4842d19..0000000 --- a/suites/wal-batched-mw-local.json +++ /dev/null @@ -1,42 +0,0 @@ -{ - "suite": "wal-batched-mw-local", - "_doc": "A/B for the WAL-machinery optimization --wal-batched-markwritten. Baseline takes the inner shard lock twice per append (reserve + mark_written); batched pushes the completed lsn to a side buffer and lets the committer fold the batch into the watermark in one inner acquisition, cutting appender inner traffic 2->1. Hypothesis: less inner-lock contention -> faster commit cycle -> lower durwait -> higher throughput, most visible at high inflight (50k streams, 6 pods x 128 conns). Both arms syncfs on. Capture SRV_STATS (durwait/stage) from the server pod. Local kind: durwait is dominated by slow Docker fsync, so if batched helps HERE it's via reduced contention; confirm the machinery win cleanly on remote NVMe after.", - "cluster": { - "server_cpus": 4 - }, - "saturation": { - "plateau_pct": 10, - "fleet_cpu": 0.15, - "connections": 128, - "repeats": 1, - "warmup_secs": 4, - "measure_secs": 10, - "setup_concurrency": 8, - "payload_bytes": 256 - }, - "modes": [ - "wal" - ], - "server_configs": { - "wal": [ - { - "label": "baseline", - "args": "--wal-shards 4 --worker-threads 4 --wal-checkpoint-syncfs on --wal-stats 2 --server-stats 2" - }, - { - "label": "batched-mw", - "args": "--wal-shards 4 --worker-threads 4 --wal-checkpoint-syncfs on --wal-batched-markwritten on --wal-stats 2 --server-stats 2" - } - ] - }, - "stream_counts": [ - 50000 - ], - "pod_ladder": { - "50000": [ - 2, - 4, - 6 - ] - } -} diff --git a/suites/wal-checkpoint-fix-100k.json b/suites/wal-checkpoint-fix-100k.json deleted file mode 100644 index 3ce1c16..0000000 --- a/suites/wal-checkpoint-fix-100k.json +++ /dev/null @@ -1,48 +0,0 @@ -{ - "suite": "wal-checkpoint-fix-100k", - "_doc": "MINIMAL decisive rerun of the checkpoint-fix A/B (the full wal-checkpoint-fix-ab was aborted before the fix arms reached 100k). Only the missing measurement: syncfs and both at 100k streams, ladder [2,4]. Known reference points from the aborted run: baseline@100k = 10.4k ops/s at p2 degrading to 7.3k at p8 (the cliff); baseline/stagger@10k ~107-111k (fixes don't regress below the cliff). Verdict rule: if syncfs/both peak >> 10.4k the fix works. Requires MULTILANE=1 SERVER_LOCAL_SSD_BLOCK=1 SPOT_SERVER=1. Image: europe-west1-docker.pkg.dev/vaxine/ds-bench/durable-streams:dev (digest 2ebed436* = perf/wal-syncfs @ 394375168, both flags compiled in, default off).", - "cluster": { - "server_machine": "c4d-standard-64-lssd", - "server_cpus": 8, - "client_machine": "n2d-standard-32", - "client_nodes": 3, - "cluster_name": "bench-multilane", - "zone": "europe-west4-b", - "region": "europe-west4" - }, - "saturation": { - "plateau_pct": 8, - "patience": 2, - "fleet_cpu": 2, - "connections": 256, - "repeats": 1, - "warmup_secs": 12, - "measure_secs": 20, - "setup_concurrency": 16, - "payload_bytes": 256 - }, - "modes": [ - "wal" - ], - "server_configs": { - "wal": [ - { - "label": "syncfs", - "args": "--wal-shards 6 --wal-checkpoint-syncfs on --worker-threads 8 --wal-stats 3 --server-stats 3" - }, - { - "label": "both", - "args": "--wal-shards 6 --wal-checkpoint-stagger on --wal-checkpoint-syncfs on --worker-threads 8 --wal-stats 3 --server-stats 3" - } - ] - }, - "stream_counts": [ - 100000 - ], - "pod_ladder": { - "100000": [ - 2, - 4 - ] - } -} diff --git a/suites/wal-checkpoint-fix-ab.json b/suites/wal-checkpoint-fix-ab.json deleted file mode 100644 index 08bc0f8..0000000 --- a/suites/wal-checkpoint-fix-ab.json +++ /dev/null @@ -1,63 +0,0 @@ -{ - "suite": "wal-checkpoint-fix-ab", - "_doc": "A/B of the cardinality-cliff checkpoint fixes on the 6-lane NVMe cluster (bench-multilane, one physical NVMe per shard, MULTILANE=1). All configs use --wal-shards 6 (6 lanes). Compares: baseline (synchronized checkpoint wave = current), stagger (--wal-checkpoint-stagger on: round-robin per-shard checkpoints so only one device storms at a time), syncfs (--wal-checkpoint-syncfs on: one syncfs barrier per shard-device instead of O(N) per-stream fdatasync), and both. The baseline arm is also the s6@100k point the shard sweep was heading toward. Hypothesis: baseline stays ~10-11k at 100k (all lanes stall together); stagger and/or syncfs recover throughput by de-conflicting or shrinking the checkpoint storm. Requires MULTILANE=1 SERVER_LOCAL_SSD_BLOCK=1 SPOT_SERVER=1.", - "cluster": { - "server_machine": "c4d-standard-64-lssd", - "server_cpus": 8, - "client_machine": "n2d-standard-32", - "client_nodes": 3, - "cluster_name": "bench-multilane", - "zone": "europe-west4-b", - "region": "europe-west4" - }, - "saturation": { - "plateau_pct": 8, - "patience": 2, - "fleet_cpu": 2, - "connections": 256, - "repeats": 1, - "warmup_secs": 12, - "measure_secs": 20, - "setup_concurrency": 16, - "payload_bytes": 256 - }, - "modes": [ - "wal" - ], - "server_configs": { - "wal": [ - { - "label": "baseline", - "args": "--wal-shards 6 --worker-threads 8 --wal-stats 3 --server-stats 3" - }, - { - "label": "stagger", - "args": "--wal-shards 6 --wal-checkpoint-stagger on --worker-threads 8 --wal-stats 3 --server-stats 3" - }, - { - "label": "syncfs", - "args": "--wal-shards 6 --wal-checkpoint-syncfs on --worker-threads 8 --wal-stats 3 --server-stats 3" - }, - { - "label": "both", - "args": "--wal-shards 6 --wal-checkpoint-stagger on --wal-checkpoint-syncfs on --worker-threads 8 --wal-stats 3 --server-stats 3" - } - ] - }, - "stream_counts": [ - 10000, - 100000 - ], - "pod_ladder": { - "10000": [ - 2, - 4, - 8 - ], - "100000": [ - 2, - 4, - 8 - ] - } -} diff --git a/suites/wal-cpubind.json b/suites/wal-cpubind.json deleted file mode 100644 index 4721860..0000000 --- a/suites/wal-cpubind.json +++ /dev/null @@ -1,50 +0,0 @@ -{ - "suite": "wal-cpubind", - "_doc": "CPU-binding experiment (task: do exclusive pinned cores help now that wal is no longer fsync-bound at 270k+ ops/s?). Own cluster bench-cpubind created with STATIC_CPU=1 (kubelet cpuManagerPolicy=static on the server pool) and deployed with GUARANTEED=1 (gke/durable-streams-splitlane-guaranteed.yaml: every container requests==limits, server cpu integer 8 -> EXCLUSIVE pinned cores). Same split-lane layout + image (:ckptoff) + args as wal-splitlane's ref-3s, so the control is wal-splitlane ref-3s = 286k @10k / 272k @100k on shared cores. Success = bound arm meaningfully above 286k/272k (>= +5%); parity means CPU binding is not a lever at this throughput and the task closes. VERIFY at deploy: kubectl exec ... cat /sys/fs/cgroup/cpuset.cpus.effective in the server container should show a dedicated core set, not all cores. Requires STATIC_CPU=1 SPLITLANE=1 GUARANTEED=1 SERVER_LOCAL_SSD_BLOCK=1 SPOT_SERVER=1 IMG_SERVER=...:ckptoff.", - "cluster": { - "server_machine": "c4d-standard-64-lssd", - "server_cpus": 8, - "client_machine": "n2d-standard-32", - "client_nodes": 3, - "cluster_name": "bench-cpubind", - "zone": "europe-west4-b", - "region": "europe-west4" - }, - "saturation": { - "plateau_pct": 8, - "patience": 2, - "fleet_cpu": 2, - "connections": 256, - "repeats": 1, - "warmup_secs": 12, - "measure_secs": 20, - "setup_concurrency": 16, - "payload_bytes": 256 - }, - "modes": [ - "wal" - ], - "server_configs": { - "wal": [ - { - "label": "bound-3s", - "args": "--wal-shards 5 --wal-checkpoint-syncfs on --worker-threads 8 --wal-stats 3 --server-stats 3 --data-dir /data/wal/0" - } - ] - }, - "stream_counts": [ - 10000, - 100000 - ], - "pod_ladder": { - "10000": [ - 4, - 8 - ], - "100000": [ - 2, - 4, - 8 - ] - } -} diff --git a/suites/wal-decomp-lane0.json b/suites/wal-decomp-lane0.json deleted file mode 100644 index a96bc44..0000000 --- a/suites/wal-decomp-lane0.json +++ /dev/null @@ -1,61 +0,0 @@ -{ - "suite": "wal-decomp-lane0", - "_doc": "Cardinality-cliff DECOMPOSITION on true NVMe, killing a newly-found confound: stream data files live at /streams/ = the BASE /data volume, which on the raw-block multilane cluster is a plain emptyDir on the PD BOOT DISK — so every prior multilane run had the WAL commit path on NVMe but the checkpoint's O(N)-file writeback on a network PD. Here every arm appends '--data-dir /data/wal/0' (last --data-dir wins in main.rs arg parsing) so streams/ AND wal/ all sit on NVMe lane 0 (one fs; shard count is concurrency only, not lanes). Arms decompose the wal-vs-memory gap: memory (no wal, no fsync ever) | nofsync (full wal machinery, --unsafe-disable-fsync: no fsync at all) | ckpt-off (wal + commit fdatasync, checkpoint deferred 1h = off within a bench window) | ref-3s (normal syncfs @3s = PR #4697 config, re-measured on real NVMe). Verdicts: nofsync ~= memory => wal machinery is free, gap is all fsync. ckpt-off ~= nofsync => commit fdatasync is cheap (group-committed), gap is all checkpoint. ref-3s @100k >> the ~10-18k measured with streams-on-PD => the 'residual cliff' in PR #4697 was substantially the boot-disk artifact. Image :ckptoff is built from the perf/wal-lazy-checkpoint worktree (has --wal-checkpoint-interval-ms, --unsafe-disable-fsync, syncfs). Run with IMG_SERVER=europe-west1-docker.pkg.dev/vaxine/ds-bench/durable-streams:ckptoff MULTILANE=1 SERVER_LOCAL_SSD_BLOCK=1 SPOT_SERVER=1.", - "cluster": { - "server_machine": "c4d-standard-64-lssd", - "server_cpus": 8, - "client_machine": "n2d-standard-32", - "client_nodes": 3, - "cluster_name": "bench-multilane", - "zone": "europe-west4-b", - "region": "europe-west4" - }, - "saturation": { - "plateau_pct": 8, - "patience": 2, - "fleet_cpu": 2, - "connections": 256, - "repeats": 1, - "warmup_secs": 12, - "measure_secs": 20, - "setup_concurrency": 16, - "payload_bytes": 256 - }, - "modes": [ - "wal" - ], - "server_configs": { - "wal": [ - { - "label": "memory", - "args": "--durability memory --worker-threads 8 --server-stats 3 --data-dir /data/wal/0" - }, - { - "label": "nofsync", - "args": "--wal-shards 6 --unsafe-disable-fsync on --worker-threads 8 --wal-stats 3 --server-stats 3 --data-dir /data/wal/0" - }, - { - "label": "ckpt-off", - "args": "--wal-shards 6 --wal-checkpoint-syncfs on --wal-checkpoint-interval-ms 3600000 --worker-threads 8 --wal-stats 3 --server-stats 3 --data-dir /data/wal/0" - }, - { - "label": "ref-3s", - "args": "--wal-shards 6 --wal-checkpoint-syncfs on --worker-threads 8 --wal-stats 3 --server-stats 3 --data-dir /data/wal/0" - } - ] - }, - "stream_counts": [ - 10000, - 100000 - ], - "pod_ladder": { - "10000": [ - 4, - 8 - ], - "100000": [ - 2, - 4 - ] - } -} diff --git a/suites/wal-fanout-sweep.json b/suites/wal-fanout-sweep.json deleted file mode 100644 index 3fdd6a4..0000000 --- a/suites/wal-fanout-sweep.json +++ /dev/null @@ -1,59 +0,0 @@ -{ - "suite": "wal-fanout-sweep", - "_doc": "Controlled --wal-fsync-parallel (checkpoint per-stream fdatasync fan-out) ladder at the winning shard count (4), on the SAME NVMe cluster as wal-shard-sweep (cluster_name bench-walshard, idempotent reuse). The shard sweep showed throughput is device-flush-bound and shard-insensitive (~72-75k); the remaining question is whether parallelizing the checkpoint's O(N_touched) per-stream fdatasync storm (which competes with commit fsyncs for the device flush budget) helps or hurts. Prior data only had fanout 1 vs 16 (16 regressed); this sweeps 1/2/4/8 to properly characterize whether a SMALL fanout helps before device contention dominates. Same 200k streams / 8 vCPU pin / 256 conns so it's directly comparable to wal-s4.", - "cluster": { - "server_machine": "c4d-standard-16-lssd", - "server_cpus": 8, - "client_machine": "n2d-standard-32", - "client_nodes": 3, - "cluster_name": "bench-walshard", - "zone": "europe-west4-b", - "region": "europe-west4" - }, - "saturation": { - "plateau_pct": 8, - "patience": 2, - "fleet_cpu": 2, - "connections": 256, - "repeats": 1, - "warmup_secs": 12, - "measure_secs": 20, - "setup_concurrency": 16, - "payload_bytes": 256 - }, - "modes": [ - "wal" - ], - "server_configs": { - "wal": [ - { - "label": "s4-f1", - "args": "--wal-shards 4 --wal-fsync-parallel 1 --worker-threads 8 --wal-stats 3 --server-stats 3" - }, - { - "label": "s4-f2", - "args": "--wal-shards 4 --wal-fsync-parallel 2 --worker-threads 8 --wal-stats 3 --server-stats 3" - }, - { - "label": "s4-f4", - "args": "--wal-shards 4 --wal-fsync-parallel 4 --worker-threads 8 --wal-stats 3 --server-stats 3" - }, - { - "label": "s4-f8", - "args": "--wal-shards 4 --wal-fsync-parallel 8 --worker-threads 8 --wal-stats 3 --server-stats 3" - } - ] - }, - "stream_counts": [ - 200000 - ], - "pod_ladder": { - "200000": [ - 2, - 4, - 8, - 12, - 16 - ] - } -} diff --git a/suites/wal-fsync-diag-local.json b/suites/wal-fsync-diag-local.json deleted file mode 100644 index 788fad8..0000000 --- a/suites/wal-fsync-diag-local.json +++ /dev/null @@ -1,51 +0,0 @@ -{ - "suite": "wal-fsync-diag-local", - "_doc": "Decompose the WAL-path cardinality cliff: is it the fsync BARRIERS or the WAL write machinery? Three arms at the same cardinality: (1) wal-normal = syncfs+fsync on (the cliff), (2) wal-nofsync = --unsafe-disable-fsync on (keeps the full WAL write path — staging, register_dirty, group-commit, per-stream data-file writes — but no-ops every device barrier), (3) memory = no WAL at all. If wal-nofsync jumps to ~memory throughput and flattens, the cliff is the fsync barriers. If wal-nofsync stays cliffy like wal-normal, it's the WAL write machinery (register_dirty / staging / per-stream writes), not the barrier. SRV_STATS (--server-stats 2) now splits append time into applock_us / wrwire_us (per-stream data-file write) / durwait_us (fsync wait) — capture it from the server pod during the 50k cells to see exactly where the time goes. Local kind, relative signal.", - "cluster": { - "server_cpus": 4 - }, - "saturation": { - "plateau_pct": 10, - "fleet_cpu": 0.15, - "connections": 128, - "repeats": 1, - "warmup_secs": 4, - "measure_secs": 8, - "setup_concurrency": 8, - "payload_bytes": 256 - }, - "modes": [ - "wal" - ], - "server_configs": { - "wal": [ - { - "label": "wal-normal", - "args": "--wal-shards 4 --worker-threads 4 --wal-checkpoint-syncfs on --server-stats 2" - }, - { - "label": "wal-nofsync", - "args": "--wal-shards 4 --worker-threads 4 --wal-checkpoint-syncfs on --unsafe-disable-fsync on --server-stats 2" - }, - { - "label": "memory", - "args": "--durability memory --worker-threads 4 --server-stats 2" - } - ] - }, - "stream_counts": [ - 20000, - 50000 - ], - "pod_ladder": { - "20000": [ - 2, - 4 - ], - "50000": [ - 2, - 4, - 6 - ] - } -} diff --git a/suites/wal-lazy-checkpoint-local.json b/suites/wal-lazy-checkpoint-local.json deleted file mode 100644 index a20a3f8..0000000 --- a/suites/wal-lazy-checkpoint-local.json +++ /dev/null @@ -1,55 +0,0 @@ -{ - "suite": "wal-lazy-checkpoint-local", - "_doc": "Local-kind value-finder for the lazy-checkpoint approach (cardinality cliff). The checkpoint's per-stream fsync/syncfs is not a correctness barrier (ack = WAL fdatasync; recovery replays the retained WAL) — it only bounds WAL size for recycling. So the checkpoint interval is a pure amortization knob: a LARGE interval fires the O(shards) syncfs barrier far less often and coalesces many appends to the same stream into one metadata/journal update, trading a larger retained WAL + longer crash-replay for cheaper steady-state writes. This sweeps the interval (--wal-checkpoint-interval-ms) with syncfs on, against two reference arms (baseline = per-stream fdatasync @3s; syncfs @3s = PR #4697). Local kind is a relative value-finder only (Docker VM fsync != NVMe); if larger intervals lift throughput here, re-tune the actual values on the remote NVMe cluster. Watch WAL_CKPT fsync_us + cadence via --wal-stats 2.", - "cluster": { - "server_cpus": 4 - }, - "saturation": { - "plateau_pct": 10, - "fleet_cpu": 0.15, - "connections": 128, - "repeats": 1, - "warmup_secs": 4, - "measure_secs": 8, - "setup_concurrency": 8, - "payload_bytes": 256 - }, - "modes": [ - "wal" - ], - "server_configs": { - "wal": [ - { - "label": "ref-baseline-3s", - "args": "--wal-shards 4 --worker-threads 4 --wal-stats 2" - }, - { - "label": "ref-syncfs-3s", - "args": "--wal-shards 4 --worker-threads 4 --wal-checkpoint-syncfs on --wal-stats 2" - }, - { - "label": "lazy-syncfs-30s", - "args": "--wal-shards 4 --worker-threads 4 --wal-checkpoint-syncfs on --wal-checkpoint-interval-ms 30000 --wal-stats 2" - }, - { - "label": "lazy-syncfs-60s", - "args": "--wal-shards 4 --worker-threads 4 --wal-checkpoint-syncfs on --wal-checkpoint-interval-ms 60000 --wal-stats 2" - } - ] - }, - "stream_counts": [ - 20000, - 50000 - ], - "pod_ladder": { - "20000": [ - 2, - 4 - ], - "50000": [ - 2, - 4, - 6 - ] - } -} diff --git a/suites/wal-lazy-checkpoint-multilane.json b/suites/wal-lazy-checkpoint-multilane.json deleted file mode 100644 index 6e51d67..0000000 --- a/suites/wal-lazy-checkpoint-multilane.json +++ /dev/null @@ -1,63 +0,0 @@ -{ - "suite": "wal-lazy-checkpoint-multilane", - "_doc": "Remote NVMe tuning of the lazy-checkpoint interval (cardinality cliff). Runs on the 6-lane c4d-standard-64-lssd multilane cluster (one physical NVMe per WAL shard, MULTILANE=1 SERVER_LOCAL_SSD_BLOCK=1 SPOT_SERVER=1). All arms use --wal-shards 6 + --wal-checkpoint-syncfs on; the ONLY variable is --wal-checkpoint-interval-ms. ref-syncfs-3s is the reference (PR #4697 cadence on the SAME binary); 30s/60s/120s sweep the amortization knob. Local kind showed monotonic gains 3s->30s->60s at 50k with 60s still climbing, so this pushes to 120s to find the knee. Success = higher throughput at 100k as the interval grows; the winning value + the WAL_CKPT fsync cadence/cost (--wal-stats 3) sets the production default. Watch retained-WAL growth (larger interval = larger WAL + longer crash-replay) as the ceiling.", - "cluster": { - "server_machine": "c4d-standard-64-lssd", - "server_cpus": 8, - "client_machine": "n2d-standard-32", - "client_nodes": 3, - "cluster_name": "bench-multilane", - "zone": "europe-west4-b", - "region": "europe-west4" - }, - "saturation": { - "plateau_pct": 8, - "patience": 2, - "fleet_cpu": 2, - "connections": 256, - "repeats": 1, - "warmup_secs": 12, - "measure_secs": 20, - "setup_concurrency": 16, - "payload_bytes": 256 - }, - "modes": [ - "wal" - ], - "server_configs": { - "wal": [ - { - "label": "ref-syncfs-3s", - "args": "--wal-shards 6 --wal-checkpoint-syncfs on --worker-threads 8 --wal-stats 3 --server-stats 3" - }, - { - "label": "lazy-30s", - "args": "--wal-shards 6 --wal-checkpoint-syncfs on --wal-checkpoint-interval-ms 30000 --worker-threads 8 --wal-stats 3 --server-stats 3" - }, - { - "label": "lazy-60s", - "args": "--wal-shards 6 --wal-checkpoint-syncfs on --wal-checkpoint-interval-ms 60000 --worker-threads 8 --wal-stats 3 --server-stats 3" - }, - { - "label": "lazy-120s", - "args": "--wal-shards 6 --wal-checkpoint-syncfs on --wal-checkpoint-interval-ms 120000 --worker-threads 8 --wal-stats 3 --server-stats 3" - } - ] - }, - "stream_counts": [ - 10000, - 100000 - ], - "pod_ladder": { - "10000": [ - 2, - 4, - 8 - ], - "100000": [ - 2, - 4, - 8 - ] - } -} diff --git a/suites/wal-machinery-baseline-local.json b/suites/wal-machinery-baseline-local.json deleted file mode 100644 index e6c5810..0000000 --- a/suites/wal-machinery-baseline-local.json +++ /dev/null @@ -1,38 +0,0 @@ -{ - "suite": "wal-machinery-baseline-local", - "_doc": "Baseline for WAL-machinery optimization #1 (lock-free LSN/offset reservation). Runs wal with --wal-stats 2 so WAL_STATS exposes inner_lock_wait — the per-shard write-serialization signal (every stream on a shard funnels through the inner mutex twice per append: reserve + mark_written). If inner_lock_wait is a large fraction of the append time at 50k, the shard lock is the machinery tax and #1 is the right fix. Also captures SRV_STATS (svc/applock/wrwire/durwait). Local kind, relative. 50k only to be fast.", - "cluster": { - "server_cpus": 4 - }, - "saturation": { - "plateau_pct": 10, - "fleet_cpu": 0.15, - "connections": 128, - "repeats": 1, - "warmup_secs": 4, - "measure_secs": 10, - "setup_concurrency": 8, - "payload_bytes": 256 - }, - "modes": [ - "wal" - ], - "server_configs": { - "wal": [ - { - "label": "wal-baseline", - "args": "--wal-shards 4 --worker-threads 4 --wal-checkpoint-syncfs on --wal-stats 2 --server-stats 2" - } - ] - }, - "stream_counts": [ - 50000 - ], - "pod_ladder": { - "50000": [ - 2, - 4, - 6 - ] - } -} diff --git a/suites/wal-mem-diag-local.json b/suites/wal-mem-diag-local.json deleted file mode 100644 index bd93cbb..0000000 --- a/suites/wal-mem-diag-local.json +++ /dev/null @@ -1,47 +0,0 @@ -{ - "suite": "wal-mem-diag-local", - "_doc": "Diagnostic: does the cardinality cliff exist in MEMORY durability mode too? Memory mode (--durability memory) skips the WAL + all per-append fsync, but STILL writes each append into the per-stream data file (page cache) for reads. So if memory mode ALSO cliffs at high cardinality, the cause is the per-stream data files / N-inode writeback, not the WAL/fsync path — which points at the log-structured store. If memory mode is FLAT while wal cliffs, the cause is WAL durability. Local kind, relative signal. wal arm uses syncfs (best current wal). Capture SRV_STATS (svc_us/applock_us/durwait_us) from the server pod during the 50k cells for the time breakdown.", - "cluster": { - "server_cpus": 4 - }, - "saturation": { - "plateau_pct": 10, - "fleet_cpu": 0.15, - "connections": 128, - "repeats": 1, - "warmup_secs": 4, - "measure_secs": 8, - "setup_concurrency": 8, - "payload_bytes": 256 - }, - "modes": [ - "wal" - ], - "server_configs": { - "wal": [ - { - "label": "wal-syncfs", - "args": "--wal-shards 4 --worker-threads 4 --wal-checkpoint-syncfs on --wal-stats 2 --server-stats 2" - }, - { - "label": "memory", - "args": "--durability memory --worker-threads 4 --server-stats 2" - } - ] - }, - "stream_counts": [ - 20000, - 50000 - ], - "pod_ladder": { - "20000": [ - 2, - 4 - ], - "50000": [ - 2, - 4, - 6 - ] - } -} diff --git a/suites/wal-multilane-shards.json b/suites/wal-multilane-shards.json deleted file mode 100644 index 8b1e23f..0000000 --- a/suites/wal-multilane-shards.json +++ /dev/null @@ -1,79 +0,0 @@ -{ - "suite": "wal-multilane-shards", - "_doc": "MULTI-LANE re-test of WAL shard scaling. On the single-device c4d-standard-16-lssd (and the RAID0-striped -lssd ephemeral-storage layout) --wal-shards did NOT scale write throughput: every shard's fdatasync serialised behind ONE filesystem = ONE fsync barrier (a single-lane artifact, not a real ceiling). This suite instead runs on c4d-standard-64-lssd, whose 6 physically-attached Titanium NVMe devices are attached as RAW BLOCK and mkfs+mounted ONE device per WAL shard dir, so each shard's fdatasync hits an independent device queue (an independent fsync lane). It sweeps shards 1/2/4/6 (capped at the 6-device count — s7+ would spill onto the base volume and defeat the test) with the server pinned to 8 vCPU headroom (prior telemetry: wal is fsync-bound + CPU-idle) and worker-threads fixed at 8. Each config emits SRV_STATS (cpu_cores/inflight/durwait) + WAL_CKPT so fsync-lane saturation is visible directly. If throughput now rises ~linearly s1->s6, the earlier flatness was the single-lane disk, not the WAL design. REQUIRES these env vars when running (they are NOT encoded in this JSON): MULTILANE=1 (apply gke/durable-streams-multilane.yaml = one device per shard dir), SERVER_LOCAL_SSD_BLOCK=1 (create the server pool with raw-block --local-nvme-ssd-block instead of RAID0 ephemeral-storage), SPOT_SERVER=1 (cheap; a preemption just re-runs the affected cells). See MULTILANE_SETUP.md for the provisioning command + the VERIFY-AT-PROVISION-TIME checklist.", - "cluster": { - "server_machine": "c4d-standard-64-lssd", - "server_cpus": 8, - "client_machine": "n2d-standard-32", - "client_nodes": 3, - "cluster_name": "bench-multilane", - "zone": "europe-west4-b", - "region": "europe-west4" - }, - "saturation": { - "plateau_pct": 8, - "patience": 2, - "fleet_cpu": 2, - "connections": 256, - "repeats": 1, - "warmup_secs": 12, - "measure_secs": 20, - "setup_concurrency": 16, - "payload_bytes": 256 - }, - "modes": [ - "wal" - ], - "server_configs": { - "wal": [ - { - "label": "s1", - "args": "--wal-shards 1 --worker-threads 8 --wal-stats 3 --server-stats 3" - }, - { - "label": "s2", - "args": "--wal-shards 2 --worker-threads 8 --wal-stats 3 --server-stats 3" - }, - { - "label": "s4", - "args": "--wal-shards 4 --worker-threads 8 --wal-stats 3 --server-stats 3" - }, - { - "label": "s6", - "args": "--wal-shards 6 --worker-threads 8 --wal-stats 3 --server-stats 3" - } - ] - }, - "stream_counts": [ - 100, - 1000, - 10000, - 100000 - ], - "pod_ladder": { - "100": [ - 2, - 4, - 8, - 12 - ], - "1000": [ - 2, - 4, - 8, - 12 - ], - "10000": [ - 2, - 4, - 8, - 12 - ], - "100000": [ - 2, - 4, - 8, - 12 - ] - } -} diff --git a/suites/wal-shard-sweep.json b/suites/wal-shard-sweep.json deleted file mode 100644 index cb15a40..0000000 --- a/suites/wal-shard-sweep.json +++ /dev/null @@ -1,64 +0,0 @@ -{ - "suite": "wal-shard-sweep", - "_doc": "Controlled --wal-shards ladder to find the optimal wal config on real NVMe (c4d-standard-16-lssd, Titanium local SSD). Isolates shard count as the ONLY variable: server pinned to 8 vCPU (headroom so CPU is never the limit — prior telemetry showed wal at ~1.3 busy cores, fsync-bound + CPU-idle) and worker-threads fixed at 8. Sweeps shards 1/4/8/16/24 at 200k streams (matches the documented s16 reference point). Each config emits SRV_STATS (cpu_cores/inflight/durwait) + WAL_CKPT so we can see fsync-lane saturation directly. The pod ladder over-provisions offered load so even the highest-ceiling shard config saturates; the barrier-aligned walker auto-stops each config at its plateau (plateau_pct 8, patience 2). Resumable per cell (cells.json), so an interrupted run resumes without recompute.", - "cluster": { - "server_machine": "c4d-standard-16-lssd", - "server_cpus": 8, - "client_machine": "n2d-standard-32", - "client_nodes": 3, - "cluster_name": "bench-walshard", - "zone": "europe-west4-b", - "region": "europe-west4" - }, - "saturation": { - "plateau_pct": 8, - "patience": 2, - "fleet_cpu": 2, - "connections": 256, - "repeats": 1, - "warmup_secs": 12, - "measure_secs": 20, - "setup_concurrency": 16, - "payload_bytes": 256 - }, - "modes": [ - "wal" - ], - "server_configs": { - "wal": [ - { - "label": "wal-s1", - "args": "--wal-shards 1 --worker-threads 8 --wal-stats 3 --server-stats 3" - }, - { - "label": "wal-s4", - "args": "--wal-shards 4 --worker-threads 8 --wal-stats 3 --server-stats 3" - }, - { - "label": "wal-s8", - "args": "--wal-shards 8 --worker-threads 8 --wal-stats 3 --server-stats 3" - }, - { - "label": "wal-s16", - "args": "--wal-shards 16 --worker-threads 8 --wal-stats 3 --server-stats 3" - }, - { - "label": "wal-s24", - "args": "--wal-shards 24 --worker-threads 8 --wal-stats 3 --server-stats 3" - } - ] - }, - "stream_counts": [ - 200000 - ], - "pod_ladder": { - "200000": [ - 1, - 2, - 4, - 8, - 12, - 16 - ] - } -} diff --git a/suites/wal-sizetrigger.json b/suites/wal-sizetrigger.json deleted file mode 100644 index e5f1f82..0000000 --- a/suites/wal-sizetrigger.json +++ /dev/null @@ -1,54 +0,0 @@ -{ - "suite": "wal-sizetrigger", - "_doc": "Validate the per-shard checkpoint size-trigger (branch perf/wal-checkpoint-sizetrigger, image :sizetrigger) on the split-lane layout. wal-splitlane measured checkpoint @3s costing 7-11% vs checkpoint-off (ref-3s 286k/272k vs ckpt-off 305k/306k). Hypothesis: --wal-checkpoint-wal-bytes 1GiB (+60s interval fallback) amortizes the checkpoint into a rare event and recovers most of that gap, with bounded replay (<=1GiB retained WAL per shard). Arms share the SAME image; ref-3s is the control at defaults (3s timer). Success = size-1g within ~2% of the 306k ckpt-off ceiling at both stream counts. Requires SPLITLANE=1 SERVER_LOCAL_SSD_BLOCK=1 SPOT_SERVER=1 IMG_SERVER=...:sizetrigger.", - "cluster": { - "server_machine": "c4d-standard-64-lssd", - "server_cpus": 8, - "client_machine": "n2d-standard-32", - "client_nodes": 3, - "cluster_name": "bench-multilane", - "zone": "europe-west4-b", - "region": "europe-west4" - }, - "saturation": { - "plateau_pct": 8, - "patience": 2, - "fleet_cpu": 2, - "connections": 256, - "repeats": 1, - "warmup_secs": 12, - "measure_secs": 20, - "setup_concurrency": 16, - "payload_bytes": 256 - }, - "modes": [ - "wal" - ], - "server_configs": { - "wal": [ - { - "label": "ref-3s", - "args": "--wal-shards 5 --wal-checkpoint-syncfs on --worker-threads 8 --wal-stats 3 --server-stats 3 --data-dir /data/wal/0" - }, - { - "label": "size-1g", - "args": "--wal-shards 5 --wal-checkpoint-syncfs on --wal-checkpoint-wal-bytes 1073741824 --wal-checkpoint-interval-ms 60000 --worker-threads 8 --wal-stats 3 --server-stats 3 --data-dir /data/wal/0" - } - ] - }, - "stream_counts": [ - 10000, - 100000 - ], - "pod_ladder": { - "10000": [ - 4, - 8 - ], - "100000": [ - 2, - 4, - 8 - ] - } -} diff --git a/suites/wal-splitlane.json b/suites/wal-splitlane.json deleted file mode 100644 index 3e027b2..0000000 --- a/suites/wal-splitlane.json +++ /dev/null @@ -1,58 +0,0 @@ -{ - "suite": "wal-splitlane", - "_doc": "SPLIT-LANE validation of the storage-layout hypothesis from wal-decomp-lane0: with WAL shards on dedicated NVMe lanes AND stream files on their own separate NVMe lane, wal+syncfs-checkpoint should reach ~250k ops/s FLAT across 10k->100k streams (ref-3s hit 270k @10k when WAL had dedicated lanes, ~= the 260k nofsync machinery ceiling; the old 8-18k @100k collapse was streams sitting on the PD boot disk). Layout via gke/durable-streams-splitlane.yaml (SPLITLANE=1): device 0 = /data/wal/0 (data root: streams/, the checkpoint syncfs domain), devices 1-5 = /data/wal/0/wal/<0..4> (5 WAL shard lanes). All arms: --data-dir /data/wal/0 --wal-shards 5. Arms: ref-3s (syncfs @3s = production candidate), ckpt-off (checkpoint deferred 1h — isolates commit-fsync cost on this layout), nofsync (machinery ceiling control). Success = ref-3s ~= ckpt-off ~= nofsync at BOTH stream counts, proving checkpoint + commit fsync are both ~free on the right layout. Requires SPLITLANE=1 SERVER_LOCAL_SSD_BLOCK=1 SPOT_SERVER=1 IMG_SERVER=...:ckptoff (image has --wal-checkpoint-interval-ms + --unsafe-disable-fsync).", - "cluster": { - "server_machine": "c4d-standard-64-lssd", - "server_cpus": 8, - "client_machine": "n2d-standard-32", - "client_nodes": 3, - "cluster_name": "bench-multilane", - "zone": "europe-west4-b", - "region": "europe-west4" - }, - "saturation": { - "plateau_pct": 8, - "patience": 2, - "fleet_cpu": 2, - "connections": 256, - "repeats": 1, - "warmup_secs": 12, - "measure_secs": 20, - "setup_concurrency": 16, - "payload_bytes": 256 - }, - "modes": [ - "wal" - ], - "server_configs": { - "wal": [ - { - "label": "ref-3s", - "args": "--wal-shards 5 --wal-checkpoint-syncfs on --worker-threads 8 --wal-stats 3 --server-stats 3 --data-dir /data/wal/0" - }, - { - "label": "ckpt-off", - "args": "--wal-shards 5 --wal-checkpoint-syncfs on --wal-checkpoint-interval-ms 3600000 --worker-threads 8 --wal-stats 3 --server-stats 3 --data-dir /data/wal/0" - }, - { - "label": "nofsync", - "args": "--wal-shards 5 --unsafe-disable-fsync on --worker-threads 8 --wal-stats 3 --server-stats 3 --data-dir /data/wal/0" - } - ] - }, - "stream_counts": [ - 10000, - 100000 - ], - "pod_ladder": { - "10000": [ - 4, - 8 - ], - "100000": [ - 2, - 4, - 8 - ] - } -} diff --git a/suites/wal-stacked-1m.json b/suites/wal-stacked-1m.json deleted file mode 100644 index e187826..0000000 --- a/suites/wal-stacked-1m.json +++ /dev/null @@ -1,52 +0,0 @@ -{ - "suite": "wal-stacked-1m", - "_doc": "Closes the cardinality-cliff campaign: (a) measure the STACKED ideal config (split-lane + syncfs + 1GiB size-trigger checkpoint [PR #4704] + exclusive pinned cores [STATIC_CPU=1 + GUARANTEED=1]) which was previously measured only separately (size-1g 303k @100k on shared cores; bound-3s 328k @100k with 3s timer; stacked projection ~360k); (b) extreme-cardinality check at 500k and 1M streams — the mechanisms that caused the old cliff are gone, this verifies no NEW mechanism appears (candidates: 1 fd per live stream vs the container nofile limit ~1M, DashMap/meta working set, ext4 dir with 1M files, tails-map rewrite cost). An EMFILE/collapse at 1M is itself a finding, not a broken run. Single arm. 500k/1M use a single p8 rung (each rung recreates all streams after the server reset — setup dominates; BARRIER_SETUP_TIMEOUT_SECS=3600 host-side and DS_BENCH_BARRIER_TIMEOUT_SECS=3600 pod-side cover the long creation phase, setup_concurrency 32). Requires STATIC_CPU=1 SPLITLANE=1 GUARANTEED=1 SERVER_LOCAL_SSD_BLOCK=1 SPOT_SERVER=1 IMG_SERVER=...:sizetrigger BARRIER_SETUP_TIMEOUT_SECS=3600.", - "cluster": { - "server_machine": "c4d-standard-64-lssd", - "server_cpus": 8, - "client_machine": "n2d-standard-32", - "client_nodes": 3, - "cluster_name": "bench-cpubind", - "zone": "europe-west4-b", - "region": "europe-west4" - }, - "saturation": { - "plateau_pct": 8, - "patience": 2, - "fleet_cpu": 2, - "connections": 256, - "repeats": 1, - "warmup_secs": 12, - "measure_secs": 20, - "setup_concurrency": 32, - "payload_bytes": 256 - }, - "modes": [ - "wal" - ], - "server_configs": { - "wal": [ - { - "label": "stacked", - "args": "--wal-shards 5 --wal-checkpoint-syncfs on --wal-checkpoint-wal-bytes 1073741824 --wal-checkpoint-interval-ms 60000 --worker-threads 8 --wal-stats 3 --server-stats 3 --data-dir /data/wal/0" - } - ] - }, - "stream_counts": [ - 100000, - 500000, - 1000000 - ], - "pod_ladder": { - "100000": [ - 4, - 8 - ], - "500000": [ - 8 - ], - "1000000": [ - 8 - ] - } -} diff --git a/suites/wal-streamlanes-1m.json b/suites/wal-streamlanes-1m.json deleted file mode 100644 index e62f9ae..0000000 --- a/suites/wal-streamlanes-1m.json +++ /dev/null @@ -1,51 +0,0 @@ -{ - "suite": "wal-streamlanes-1m", - "_doc": "Validate the --stream-lanes fix for the ~1M-stream writeback wall (wal-1m-diag: one shared data device -> checkpoint syncfs of ~200k dirty files = 60-74s per shard, append staging starved, 68k @1M). Topology 3x3 via SERVER_MANIFEST=gke/durable-streams-splitlane3x3-guaranteed.yaml: dev0 = data root + stream lane 0, dev1-2 = stream lanes 1-2, dev3-5 = WAL shards 0-2; args --stream-lanes 3 --wal-shards 3. Controls (from wal-1m-diag, 5 WAL lanes + 1 data lane, same image family/QoS): 500k = 252k, 1M = 68k. Success = 1M meaningfully above 68k (expect 150k+ if writeback capacity was the binding constraint); 500k should hold >= 250k (3 WAL lanes must not regress the commit path - WAL fsync was measured far from saturation). 100k anchors against the stacked 383k. Requires STATIC_CPU=1 SPLITLANE=1 GUARANTEED=1 SERVER_MANIFEST=gke/durable-streams-splitlane3x3-guaranteed.yaml SERVER_LOCAL_SSD_BLOCK=1 SPOT_SERVER=1 BARRIER_SETUP_TIMEOUT_SECS=3600 IMG_SERVER=...:streamlanes.", - "cluster": { - "server_machine": "c4d-standard-64-lssd", - "server_cpus": 8, - "client_machine": "n2d-standard-32", - "client_nodes": 3, - "cluster_name": "bench-cpubind", - "zone": "europe-west4-b", - "region": "europe-west4" - }, - "saturation": { - "plateau_pct": 8, - "patience": 2, - "fleet_cpu": 2, - "connections": 256, - "repeats": 1, - "warmup_secs": 90, - "measure_secs": 30, - "setup_concurrency": 32, - "payload_bytes": 256 - }, - "modes": [ - "wal" - ], - "server_configs": { - "wal": [ - { - "label": "lanes3x3", - "args": "--wal-shards 3 --stream-lanes 3 --wal-checkpoint-syncfs on --wal-checkpoint-wal-bytes 1073741824 --wal-checkpoint-interval-ms 60000 --worker-threads 8 --wal-stats 3 --server-stats 3 --data-dir /data/wal/0" - } - ] - }, - "stream_counts": [ - 100000, - 500000, - 1000000 - ], - "pod_ladder": { - "100000": [ - 8 - ], - "500000": [ - 8 - ], - "1000000": [ - 8 - ] - } -} diff --git a/suites/wal-syncfs-local.json b/suites/wal-syncfs-local.json deleted file mode 100644 index 4691a28..0000000 --- a/suites/wal-syncfs-local.json +++ /dev/null @@ -1,47 +0,0 @@ -{ - "suite": "wal-syncfs-local", - "_doc": "Local-kind A/B for cardinality-cliff #1: checkpoint durability via one syncfs() barrier (--wal-checkpoint-syncfs on) vs the per-stream fdatasync loop (baseline). Same cluster, two labeled wal configs, high-enough cardinality (20k/50k) that the checkpoint's O(N_touched) fdatasync phase is a real cost. Compare throughput + WAL_CKPT fsync_us (--wal-stats 2) between labels. This is the local gate: if syncfs is faster (or clearly lower checkpoint fsync cost) here, promote to a remote NVMe run.", - "cluster": { - "server_cpus": 4 - }, - "saturation": { - "plateau_pct": 10, - "fleet_cpu": 0.15, - "connections": 128, - "repeats": 1, - "warmup_secs": 4, - "measure_secs": 8, - "setup_concurrency": 8, - "payload_bytes": 256 - }, - "modes": [ - "wal" - ], - "server_configs": { - "wal": [ - { - "label": "baseline", - "args": "--wal-shards 4 --worker-threads 4 --wal-stats 2" - }, - { - "label": "syncfs", - "args": "--wal-shards 4 --worker-threads 4 --wal-checkpoint-syncfs on --wal-stats 2" - } - ] - }, - "stream_counts": [ - 20000, - 50000 - ], - "pod_ladder": { - "20000": [ - 2, - 4 - ], - "50000": [ - 2, - 4, - 6 - ] - } -} diff --git a/suites/wal-syncfs-nvme.json b/suites/wal-syncfs-nvme.json deleted file mode 100644 index c3c6531..0000000 --- a/suites/wal-syncfs-nvme.json +++ /dev/null @@ -1,57 +0,0 @@ -{ - "suite": "wal-syncfs-nvme", - "_doc": "Remote NVMe A/B for cardinality-cliff #1: checkpoint durability via one syncfs() barrier (--wal-checkpoint-syncfs on) vs the per-stream fdatasync loop (baseline), on Titanium NVMe (c4d-standard-16-lssd, 8 vCPU). This is the environment where the optimization's premise holds: per-file fdatasync is an expensive ~1ms hardware barrier, so the checkpoint's O(N_touched) storm (measured ~1.4s/shard at 200k) steals device budget from commits. syncfs collapses that to ONE barrier. Tests at 200k AND 500k streams (the storm is worst at 500k). Same shards(4)/cpu(8)/conns(256) as the wal-shard-sweep baseline so numbers are comparable. If syncfs raises throughput here, merge the flag.", - "cluster": { - "server_machine": "c4d-standard-16-lssd", - "server_cpus": 8, - "client_machine": "n2d-standard-32", - "client_nodes": 3, - "cluster_name": "bench-syncfs", - "zone": "europe-west4-b", - "region": "europe-west4" - }, - "saturation": { - "plateau_pct": 8, - "patience": 2, - "fleet_cpu": 2, - "connections": 256, - "repeats": 1, - "warmup_secs": 12, - "measure_secs": 20, - "setup_concurrency": 16, - "payload_bytes": 256 - }, - "modes": [ - "wal" - ], - "server_configs": { - "wal": [ - { - "label": "baseline", - "args": "--wal-shards 4 --worker-threads 8 --wal-stats 3 --server-stats 3" - }, - { - "label": "syncfs", - "args": "--wal-shards 4 --worker-threads 8 --wal-checkpoint-syncfs on --wal-stats 3 --server-stats 3" - } - ] - }, - "stream_counts": [ - 200000, - 500000 - ], - "pod_ladder": { - "200000": [ - 2, - 4, - 8, - 12 - ], - "500000": [ - 2, - 4, - 8, - 12 - ] - } -} diff --git a/suites/write-accuracy-local.json b/suites/write-accuracy-local.json deleted file mode 100644 index 597a814..0000000 --- a/suites/write-accuracy-local.json +++ /dev/null @@ -1,24 +0,0 @@ -{ - "suite": "write-accuracy-local", - "_doc": "Local kind validation of the pool write client + fleet coordination: disjoint per-pod slices of a global key domain, plain appends, setup-before-barrier. Small ladder exercises the walker, barrier, window alignment and hdr-merge end to end; after the run, `ds-bench verify-offsets` compares server-side stream offsets against the fleet's client-observed totals (see scripts/verify-write-accuracy.sh).", - "cluster": {}, - "saturation": { - "plateau_pct": 10, - "fleet_cpu": 0.25, - "repeats": 1, - "connections": 64, - "warmup_secs": 5, - "measure_secs": 10, - "setup_concurrency": 32, - "payload_bytes": 256 - }, - "modes": ["wal"], - "server_configs": { - "wal": [ - { "label": "wal", "args": "" }, - { "label": "memory", "args": "--durability memory" } - ] - }, - "stream_counts": [2000], - "pod_ladder": { "2000": [2, 4] } -} diff --git a/suites/write-cliff-local-cpu4.json b/suites/write-cliff-local-cpu4.json deleted file mode 100644 index a603f99..0000000 --- a/suites/write-cliff-local-cpu4.json +++ /dev/null @@ -1,44 +0,0 @@ -{ - "suite": "write-cliff-local-cpu4", - "_doc": "Local (kind) probe of the second suspicious remote pattern: wal throughput NOT scaling with the CPU pin (47k@4vCPU vs 43k@8vCPU remotely \u2014 fsync-lane/commit-path bound at ~25-30% CPU). Same client model as write-cliff-local but the server gets 4 vCPUs and 4 wal shards; compare against write-cliff-local's 2-vCPU wal cells at the same stream counts. If wal stays flat 2->4 vCPU locally too, the commit-path investigation can iterate on a laptop.", - "cluster": { - "server_cpus": 4 - }, - "saturation": { - "plateau_pct": 10, - "fleet_cpu": 0.25, - "connections": 64, - "repeats": 1, - "warmup_secs": 5, - "measure_secs": 15, - "setup_concurrency": 64, - "payload_bytes": 256 - }, - "modes": [ - "wal" - ], - "server_configs": { - "wal": [ - { - "label": "wal-cpu4", - "args": "--wal-shards 4 --worker-threads 4" - } - ] - }, - "stream_counts": [ - 10000, - 50000 - ], - "pod_ladder": { - "10000": [ - 1, - 2, - 4 - ], - "50000": [ - 1, - 2, - 4 - ] - } -} \ No newline at end of file diff --git a/suites/write-cliff-local.json b/suites/write-cliff-local.json deleted file mode 100644 index 3efc131..0000000 --- a/suites/write-cliff-local.json +++ /dev/null @@ -1,55 +0,0 @@ -{ - "suite": "write-cliff-local", - "_doc": "Local (kind) reproduction of the remote cardinality cliff: does write throughput fall with stream count on a laptop the way it does on GKE (2026-07-08 corrected campaign: wal -33%, memory -28..39% from 100k to 500k streams)? Purpose: a fast LOCAL iteration loop for server-side cardinality work (registry/page-cache/fd physics \u2014 WRITE_BOTTLENECKS_1M.md bottleneck #2). Server pinned to 2 vCPUs (kind default); pool client 64 conns/pod, ladder from 1 pod so knee latency lands in the walk. FAST LOOP: the cliff is fully visible by 50k streams locally (~15 min for both configs) \u2014 do NOT add 100k/200k counts back, they only slow iteration. Compare the SHAPE (normalized throughput vs streams), not absolute numbers \u2014 local disk/fsync differs from the c4d NVMe.", - "cluster": { - "server_cpus": 2 - }, - "saturation": { - "plateau_pct": 10, - "patience": 2, - "fleet_cpu": 0.25, - "connections": 64, - "repeats": 1, - "warmup_secs": 5, - "measure_secs": 15, - "setup_concurrency": 64, - "payload_bytes": 256 - }, - "modes": [ - "wal" - ], - "server_configs": { - "wal": [ - { - "label": "wal", - "args": "--wal-shards 2 --worker-threads 2" - }, - { - "label": "memory", - "args": "--durability memory --worker-threads 2" - } - ] - }, - "stream_counts": [ - 1000, - 10000, - 50000 - ], - "pod_ladder": { - "1000": [ - 1, - 2, - 4 - ], - "10000": [ - 1, - 2, - 4 - ], - "50000": [ - 1, - 2, - 4 - ] - } -} \ No newline at end of file diff --git a/suites/write-cliff-nvme.json b/suites/write-cliff-nvme.json deleted file mode 100644 index 0b58b0c..0000000 --- a/suites/write-cliff-nvme.json +++ /dev/null @@ -1,55 +0,0 @@ -{ - "suite": "write-cliff-nvme", - "_doc": "Remote real-NVMe validation of the cardinality-cliff work on c4d-standard-16-lssd (physically-attached Titanium NVMe local SSD), server pinned to 4 vCPU / 4 shards (production write shape, AGENTS.md §3). Four configs, same image, same-binary A/Bs: (1) memory-gated vs memory-nogate validates the #1 memory sidecar fix on real NVMe; (2) wal-fsync1 vs wal-fsync16 tests whether parallelizing the checkpoint fdatasync scales on real NVMe (it regressed on 2-vCPU/virtiofs locally). Pool client: disjoint per-pod slices, plain appends, pre-created before the fleet barrier. connections=256 @ fleet_cpu=2 is the ~80% single-pod calibration (AGENTS.md §7).", - "cluster": { - "server_machine": "c4d-standard-16-lssd", - "server_cpus": 4, - "client_machine": "n2d-standard-32", - "client_nodes": 5, - "cluster_name": "bench-cliff-nvme", - "zone": "europe-west4-a", - "region": "europe-west4" - }, - "saturation": { - "plateau_pct": 8, - "patience": 2, - "fleet_cpu": 2, - "connections": 256, - "repeats": 1, - "warmup_secs": 15, - "measure_secs": 25, - "setup_concurrency": 16, - "payload_bytes": 256 - }, - "modes": [ - "wal" - ], - "server_configs": { - "wal": [ - { - "label": "memory-gated", - "args": "--durability memory --worker-threads 4" - }, - { - "label": "memory-nogate", - "args": "--durability memory --worker-threads 4 --mem-meta-gate off" - }, - { - "label": "wal-fsync1", - "args": "--wal-shards 4 --worker-threads 4 --wal-fsync-parallel 1 --wal-stats 3" - }, - { - "label": "wal-fsync16", - "args": "--wal-shards 4 --worker-threads 4 --wal-fsync-parallel 16 --wal-stats 3" - } - ] - }, - "stream_counts": [ - 100000, - 500000 - ], - "pod_ladder": { - "100000": [1, 2, 4, 8, 16, 32, 48, 64], - "500000": [1, 2, 4, 8, 16, 32, 48, 64] - } -} diff --git a/suites/write-nvme-bottleneck.json b/suites/write-nvme-bottleneck.json deleted file mode 100644 index 6b6dd18..0000000 --- a/suites/write-nvme-bottleneck.json +++ /dev/null @@ -1,60 +0,0 @@ -{ - "suite": "write-nvme-bottleneck", - "_doc": "Real-NVMe bottleneck confirmation + wal shard-scaling on c4d-standard-16-lssd (Titanium NVMe), 4 vCPU pin. Uses --server-stats (SRV_STATS: cpu_cores/inflight/durwait) to confirm on NVMe what the local Docker probe showed: memory is CPU-bound, wal is fsync-bound (CPU-idle). Sweeps wal shards (4/8/16) to test whether wal throughput scales with parallel fsync lanes (the documented lever) rather than CPU. 500k streams (heaviest checkpoint/fsync pressure); shorter ladder since wal plateaus early on fsync and memory saturates cores by ~a few pods.", - "cluster": { - "server_machine": "c4d-standard-16-lssd", - "server_cpus": 4, - "client_machine": "n2d-standard-32", - "client_nodes": 3, - "cluster_name": "bench-nvme-bneck", - "zone": "europe-west4-b", - "region": "europe-west4" - }, - "saturation": { - "plateau_pct": 8, - "patience": 2, - "fleet_cpu": 2, - "connections": 256, - "repeats": 1, - "warmup_secs": 15, - "measure_secs": 25, - "setup_concurrency": 16, - "payload_bytes": 256 - }, - "modes": [ - "wal" - ], - "server_configs": { - "wal": [ - { - "label": "memory", - "args": "--durability memory --worker-threads 4 --server-stats 3" - }, - { - "label": "wal-s4", - "args": "--wal-shards 4 --worker-threads 4 --wal-stats 3 --server-stats 3" - }, - { - "label": "wal-s8", - "args": "--wal-shards 8 --worker-threads 4 --wal-stats 3 --server-stats 3" - }, - { - "label": "wal-s16", - "args": "--wal-shards 16 --worker-threads 4 --wal-stats 3 --server-stats 3" - } - ] - }, - "stream_counts": [ - 500000 - ], - "pod_ladder": { - "500000": [ - 1, - 2, - 4, - 8, - 16, - 24 - ] - } -} \ No newline at end of file diff --git a/suites/write-throughput-local.json b/suites/write-throughput-local.json deleted file mode 100644 index 4212775..0000000 --- a/suites/write-throughput-local.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "suite": "write-throughput-local", - "cluster": {}, - "saturation": { "plateau_pct": 10, "fleet_cpu": 0.1, "repeats": 1, "warmup_secs": 3, "measure_secs": 5, "setup_concurrency": 8, "payload_bytes": 256 }, - "modes": ["wal"], - "stream_counts": [10, 100], - "pod_ladder": { "10": [1, 2], "100": [2, 4] } -} diff --git a/suites/write-wal-vs-mem-cpu4.json b/suites/write-wal-vs-mem-cpu4.json deleted file mode 100644 index dc6f409..0000000 --- a/suites/write-wal-vs-mem-cpu4.json +++ /dev/null @@ -1,67 +0,0 @@ -{ - "suite": "write-wal-vs-mem-cpu4", - "_doc": "Raw append saturation, wal vs memory, server pinned to 4 vCPUs (SERVER_CPUS via cluster.server_cpus) on c4d-standard-16-lssd \u2014 physically-attached Titanium NVMe local SSD, the canonical write-benchmark server shape (see AGENTS.md \u00a73; note its single 375GB partition write-caps at ~350MB/s, measured 2026-07-07 \u2014 wal may hit that wall before CPU, memory mode does not touch it). Pool client: pods own DISJOINT slices of the global key domain (even coverage, no cross-pod appender-lock conflicts), PLAIN appends (no producer sessions), slices pre-created before the fleet start barrier so the measure window is appends-only. connections=256 @ fleet_cpu=2 is the documented ~80% single-pod calibration at batch=1/256B (AGENTS.md \u00a77) \u2014 pods stay in their linear region so the sweep measures the server, never the client. Ladder climbs until a real plateau; memory (vb/ds-rust-memory-meta-sweep build) is expected to clear wal, so its rungs extend higher. Ladder starts at 1 pod so the walk samples the PRE-saturation region: per-rung p50/p99 land in the walk and the report quotes latency from the <=80%-of-peak knee rung; the pinned plateau rung's latency is labelled as saturation queueing (Little's law), not service time.", - "cluster": { - "server_machine": "c4d-standard-16-lssd", - "server_cpus": 4, - "client_machine": "n2d-standard-32", - "client_nodes": 5, - "cluster_name": "bench-cpu4", - "zone": "europe-west4-a", - "region": "europe-west4" - }, - "saturation": { - "plateau_pct": 8, - "patience": 2, - "fleet_cpu": 2, - "connections": 256, - "repeats": 2, - "warmup_secs": 15, - "measure_secs": 25, - "setup_concurrency": 16, - "payload_bytes": 256 - }, - "modes": [ - "wal" - ], - "server_configs": { - "wal": [ - { - "label": "wal", - "args": "--wal-shards 4 --worker-threads 4" - }, - { - "label": "memory", - "args": "--durability memory --worker-threads 4" - } - ] - }, - "stream_counts": [ - 100000, - 500000 - ], - "pod_ladder": { - "100000": [ - 1, - 2, - 4, - 8, - 16, - 24, - 32, - 48, - 64 - ], - "500000": [ - 1, - 2, - 4, - 8, - 16, - 24, - 32, - 48, - 64 - ] - } -} \ No newline at end of file diff --git a/suites/write-wal-vs-mem-cpu8.json b/suites/write-wal-vs-mem-cpu8.json deleted file mode 100644 index 23686ff..0000000 --- a/suites/write-wal-vs-mem-cpu8.json +++ /dev/null @@ -1,69 +0,0 @@ -{ - "suite": "write-wal-vs-mem-cpu8", - "_doc": "Raw append saturation, wal vs memory, server pinned to 8 vCPUs (SERVER_CPUS via cluster.server_cpus) on c4d-standard-16-lssd \u2014 physically-attached Titanium NVMe local SSD, the canonical write-benchmark server shape (see AGENTS.md \u00a73; its single 375GB partition write-caps at ~350MB/s, measured 2026-07-07 \u2014 wal may hit that wall before CPU, memory mode does not touch it). Same client model as write-wal-vs-mem-cpu4: disjoint per-pod slices of the global domain, plain appends, setup-before-barrier, connections=256 @ fleet_cpu=2 (the documented ~80% single-pod calibration at batch=1/256B). Own cluster/zone so it runs in parallel with the cpu4 suite. Ladder starts at 1 pod so the walk samples the PRE-saturation region: per-rung p50/p99 land in the walk and the report quotes latency from the <=80%-of-peak knee rung; the pinned plateau rung's latency is labelled as saturation queueing (Little's law), not service time.", - "cluster": { - "server_machine": "c4d-standard-16-lssd", - "server_cpus": 8, - "client_machine": "n2d-standard-32", - "client_nodes": 7, - "cluster_name": "bench-cpu8", - "zone": "europe-west4-b", - "region": "europe-west4" - }, - "saturation": { - "plateau_pct": 8, - "patience": 2, - "fleet_cpu": 2, - "connections": 256, - "repeats": 2, - "warmup_secs": 15, - "measure_secs": 25, - "setup_concurrency": 16, - "payload_bytes": 256 - }, - "modes": [ - "wal" - ], - "server_configs": { - "wal": [ - { - "label": "wal", - "args": "--wal-shards 8 --worker-threads 8" - }, - { - "label": "memory", - "args": "--durability memory --worker-threads 8" - } - ] - }, - "stream_counts": [ - 100000, - 500000 - ], - "pod_ladder": { - "100000": [ - 1, - 2, - 4, - 8, - 16, - 24, - 32, - 48, - 64, - 96 - ], - "500000": [ - 1, - 2, - 4, - 8, - 16, - 24, - 32, - 48, - 64, - 96 - ] - } -} \ No newline at end of file From 2b0ada5c830ffd46c3b33a53ba97fb8ca3ee4347 Mon Sep 17 00:00:00 2001 From: Valter Balegas Date: Tue, 14 Jul 2026 00:02:42 +0100 Subject: [PATCH 7/8] bench: canonical reads suites use stock per-mode clusters A shared cluster_name override made the wal and ursula modes (which the harness runs in PARALLEL, one cluster per mode by design) race to create the same cluster; the loser had no kube context. Reverting to the derived per-mode names/zones. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01Y3x7bcT9vLGiT4tXZeQpnk --- suites/canonical-reads-catchup.json | 4 +--- suites/canonical-reads-sse.json | 4 +--- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/suites/canonical-reads-catchup.json b/suites/canonical-reads-catchup.json index b07b94a..cb2c442 100644 --- a/suites/canonical-reads-catchup.json +++ b/suites/canonical-reads-catchup.json @@ -5,9 +5,7 @@ "server_machine": "c4d-standard-16-lssd", "client_machine": "n2d-standard-32", "client_nodes": 2, - "region": "europe-west4", - "cluster_name": "bench-canon-reads", - "zone": "europe-west4-a" + "region": "europe-west4" }, "reads": { "mode": "catchup", diff --git a/suites/canonical-reads-sse.json b/suites/canonical-reads-sse.json index 10a9c42..7ef886d 100644 --- a/suites/canonical-reads-sse.json +++ b/suites/canonical-reads-sse.json @@ -5,9 +5,7 @@ "server_machine": "c4d-standard-16-lssd", "client_machine": "n2d-standard-32", "client_nodes": 2, - "region": "europe-west4", - "cluster_name": "bench-canon-reads", - "zone": "europe-west4-a" + "region": "europe-west4" }, "reads": { "mode": "sse", From 2e9783bf9cabbdbef2bb41219857f8f61838dee9 Mon Sep 17 00:00:00 2001 From: Valter Balegas Date: Tue, 14 Jul 2026 00:54:23 +0100 Subject: [PATCH 8/8] =?UTF-8?q?bench:=20canonical=20campaign=202026-07-14?= =?UTF-8?q?=20=E2=80=94=20full=20results,=20REPORT.md=20+=20PROVENANCE.md?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Write: wal-ideal 417k/382k @10k/100k, memory 680k/632k (gates passed, no cliff); ursula v0.2.0 baselines. Reads: wal cardinality-flat (2.3-2.7 GiB/s catchup; SSE p99 2-3ms @2048 conns). Mixed: 100k readers cost writes nothing; 2026-07-02 memory delivery collapse is gone (126.7k del/s @ 127k writes). Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01Y3x7bcT9vLGiT4tXZeQpnk --- results/PROVENANCE.md | 7 + results/REPORT.md | 76 +++++ results/canonical-mixed-cal/aggregate.csv | 2 + results/canonical-mixed-cal/aggregate.json | 28 ++ results/canonical-mixed-cal/report.md | 13 + results/canonical-mixed-cal/wal/cells.json | 40 +++ .../canonical-mixed-delivery/aggregate.csv | 11 + .../canonical-mixed-delivery/aggregate.json | 262 ++++++++++++++++++ .../memory/cells.json | 156 +++++++++++ results/canonical-mixed-delivery/report.md | 27 ++ .../canonical-mixed-delivery/wal/cells.json | 156 +++++++++++ results/canonical-mixed-writes/aggregate.csv | 5 + results/canonical-mixed-writes/aggregate.json | 106 +++++++ results/canonical-mixed-writes/report.md | 16 ++ results/canonical-mixed-writes/wal/cells.json | 127 +++++++++ results/canonical-reads-catchup/aggregate.csv | 17 ++ .../canonical-reads-catchup/aggregate.json | 226 +++++++++++++++ results/canonical-reads-catchup/report.md | 29 ++ .../canonical-reads-catchup/ursula/cells.json | 106 +++++++ .../canonical-reads-catchup/wal/cells.json | 106 +++++++ results/canonical-reads-sse/aggregate.csv | 17 ++ results/canonical-reads-sse/aggregate.json | 226 +++++++++++++++ results/canonical-reads-sse/report.md | 29 ++ results/canonical-reads-sse/ursula/cells.json | 106 +++++++ results/canonical-reads-sse/wal/cells.json | 106 +++++++ results/canonical-write-ursula/aggregate.csv | 7 + results/canonical-write-ursula/aggregate.json | 188 +++++++++++++ results/canonical-write-ursula/report.md | 48 ++++ .../ursula-disk/cells.json | 85 ++++++ .../ursula-memory/cells.json | 85 ++++++ results/canonical-write/aggregate.csv | 5 + results/canonical-write/aggregate.json | 126 +++++++++ results/canonical-write/memory/cells.json | 58 ++++ results/canonical-write/report.md | 43 +++ results/canonical-write/wal-ideal/cells.json | 58 ++++ 35 files changed, 2703 insertions(+) create mode 100644 results/PROVENANCE.md create mode 100644 results/REPORT.md create mode 100644 results/canonical-mixed-cal/aggregate.csv create mode 100644 results/canonical-mixed-cal/aggregate.json create mode 100644 results/canonical-mixed-cal/report.md create mode 100644 results/canonical-mixed-cal/wal/cells.json create mode 100644 results/canonical-mixed-delivery/aggregate.csv create mode 100644 results/canonical-mixed-delivery/aggregate.json create mode 100644 results/canonical-mixed-delivery/memory/cells.json create mode 100644 results/canonical-mixed-delivery/report.md create mode 100644 results/canonical-mixed-delivery/wal/cells.json create mode 100644 results/canonical-mixed-writes/aggregate.csv create mode 100644 results/canonical-mixed-writes/aggregate.json create mode 100644 results/canonical-mixed-writes/report.md create mode 100644 results/canonical-mixed-writes/wal/cells.json create mode 100644 results/canonical-reads-catchup/aggregate.csv create mode 100644 results/canonical-reads-catchup/aggregate.json create mode 100644 results/canonical-reads-catchup/report.md create mode 100644 results/canonical-reads-catchup/ursula/cells.json create mode 100644 results/canonical-reads-catchup/wal/cells.json create mode 100644 results/canonical-reads-sse/aggregate.csv create mode 100644 results/canonical-reads-sse/aggregate.json create mode 100644 results/canonical-reads-sse/report.md create mode 100644 results/canonical-reads-sse/ursula/cells.json create mode 100644 results/canonical-reads-sse/wal/cells.json create mode 100644 results/canonical-write-ursula/aggregate.csv create mode 100644 results/canonical-write-ursula/aggregate.json create mode 100644 results/canonical-write-ursula/report.md create mode 100644 results/canonical-write-ursula/ursula-disk/cells.json create mode 100644 results/canonical-write-ursula/ursula-memory/cells.json create mode 100644 results/canonical-write/aggregate.csv create mode 100644 results/canonical-write/aggregate.json create mode 100644 results/canonical-write/memory/cells.json create mode 100644 results/canonical-write/report.md create mode 100644 results/canonical-write/wal-ideal/cells.json diff --git a/results/PROVENANCE.md b/results/PROVENANCE.md new file mode 100644 index 0000000..ec5e9d8 --- /dev/null +++ b/results/PROVENANCE.md @@ -0,0 +1,7 @@ +# Provenance — canonical campaign 2026-07-14 + +- **Server build:** electric-sql/electric#4697 head `d5589289d` (perf/wal-checkpoint-syncfs), image `durable-streams:dev` digest `0e8bfe920065`. +- **Ursula:** upstream `ghcr.io/tonbo-io/ursula:v0.2.0`. +- **Hardware:** canonical-write on `c4d-standard-64-lssd` raw-block (STATIC_CPU=1, GUARANTEED=1, splitlane3x3 manifest); ursula/reads/mixed on `c4d-standard-16-lssd` stock manifests. Clients `n2d-standard-32` Spot. europe-west4. +- **Harness:** ds-bench bench/canonical-suites @ 2b0ada5. +- All suites self-tore-down; zero clusters at campaign end (verified). diff --git a/results/REPORT.md b/results/REPORT.md new file mode 100644 index 0000000..1c06e15 --- /dev/null +++ b/results/REPORT.md @@ -0,0 +1,76 @@ +# ds-bench canonical campaign — 2026-07-14 (post-cardinality-cliff build) + +First run of the canonical suite set, on the electric#4697 merged-state build +(syncfs checkpoints, per-shard triggers, stream lanes; see WAL_TUNING.md). +Provenance in `PROVENANCE.md`; per-suite grids in each subdirectory. + +## 1. Write saturation (`canonical-write`, `canonical-write-ursula`) + +Peak append/s at saturation (256 B payloads): + +| streams | wal-ideal | memory | ursula-mem | ursula-disk | +|---|---|---|---|---| +| 100 | — | — | 52.0k | 4.5k | +| 1k | — | — | 54.7k | 7.4k | +| 10k | 417.0k | 680.4k | 49.0k | 8.3k | +| 100k | **382.3k** | **631.5k** | — | — | + +- All four durable-streams cells are true plateaus (pinned rungs). **No + cardinality cliff: −8% (wal) / −7% (memory) from 10k→100k.** Regression gates + (wal@100k > 250k, memory@100k > 400k) passed with wide margin. +- wal-ideal = the WAL_TUNING.md configuration (3 stream lanes + 3 WAL lanes, + 1 GiB checkpoint budget, pinned cores). memory benefits from the same pinned + cores (previous best 512k on shared cores). +- Physics sanity: memory > wal everywhere, as it must be (cf. the retracted + 2026-07-02 numbers). + +## 3. Read scalability (`canonical-reads-catchup`, `canonical-reads-sse`) + +**Catch-up** (MiB/s @ p99 ms): wal is cardinality-flat — 2.27 GiB/s @62ms +(8 conns) to ~2.7 GiB/s (512 conns) at BOTH 10 and 100 streams. ursula matches +at 10 streams (2.9 GiB/s peak, worse p99: 12.1s vs 5.1s at 512) and returns +errors at every level at 100 streams (recorded as a gap — matches its historic +client-OOM ceiling there). + +**SSE tail** (per-connection paced): wal flat at both cardinalities up to 2048 +conns, p99 2–3 ms. ursula matches at 10 streams but degrades at 100 streams +(p99 39–62 ms) — same shape as 2026-07-02. + +## 4. Mixed read/write interference (`canonical-mixed-*`) + +- **Anchor:** 81.7k ops/s single-pod mixed-shape ceiling (was 81.4k — stable). +- **Paced readers vs pinned writes (10k streams, 50k ops/s pinned):** writes hold + 49.9–50.0k at 0 / 1k / 10k / 100k readers while serving up to 4,986 replays/s + at 303 MiB/s. **The premise holds: 100k concurrent catch-up readers cost the + write path nothing.** +- **Delivery under write load (2000 SSE subscribers):** + +| writes/s | wal del/s (p99 ms) | memory del/s (p99 ms) | +|---|---|---| +| 4k | 3.3k (138) | 3.3k (151) | +| 16k | 15.9k (54) | 13.3k (5) | +| 40k | 33.3k (86) | 33.2k (2) | +| 66k | 54.9k (107) | 65.7k (8) | +| max | 63.8k @ 85k writes (106) | **126.7k @ 127k writes (45)** | + + **The 2026-07-02 memory-mode delivery collapse is GONE** on this build: + memory delivery tracks writes 1:1 all the way to 127k/s. wal delivery keeps + pace to ~40k writes/s and caps at ~64k del/s at full write saturation. + +## Known gaps & artifacts + +- ursula catch-up @100 streams: ERR at all connection levels (historic + client-OOM ceiling) — gap, not a zero. +- SSE fan-out (single-stream subscriber ladder, `run-sse.sh`) not run this + campaign. +- mixed-writes write p99 (1.4–3.8 s tails) is the single-pod 10k-writer client + shape, present at zero readers too — not reader interference. +- The mixed chain was restarted once between `mixed-cal` and `mixed-writes` + (orchestration, not measurement); every published cell is a clean run. + +## Pre-publication checklist + +Layout verified (splitlane3x3 for canonical-write) ✓ · aligned windows ✓ · +plateau-labeled cells ✓ · digests in PROVENANCE.md ✓ · physics sanity (memory > +wal; fsync/s plausible) ✓ · cardinality shape reported ✓ · client-bound cells +marked as gaps ✓ diff --git a/results/canonical-mixed-cal/aggregate.csv b/results/canonical-mixed-cal/aggregate.csv new file mode 100644 index 0000000..c7b4e6e --- /dev/null +++ b/results/canonical-mixed-cal/aggregate.csv @@ -0,0 +1,2 @@ +mode,stream_count,sweep,level,readers,subscribers,writer_rate,read_rate,write_ops_per_sec,write_p50,write_p99,read_ops_per_sec,read_mib_per_sec,read_p50,read_p99,events_per_sec,delivery_p50,delivery_p99,write_bp,write_err,read_bp,read_err,status,reason +wal,50,writer_rate,0,0,0,0,0,81670.65,0.474,0.961,0.0,0.0,,,0.0,,,0,0,0,0,ok,complete diff --git a/results/canonical-mixed-cal/aggregate.json b/results/canonical-mixed-cal/aggregate.json new file mode 100644 index 0000000..2bacbd1 --- /dev/null +++ b/results/canonical-mixed-cal/aggregate.json @@ -0,0 +1,28 @@ +[ + { + "mode": "wal", + "stream_count": 50, + "sweep": "writer_rate", + "level": 0, + "readers": 0, + "subscribers": 0, + "writer_rate": 0, + "read_rate": 0, + "write_ops_per_sec": 81670.65, + "write_p50": 0.474, + "write_p99": 0.961, + "read_ops_per_sec": 0.0, + "read_mib_per_sec": 0.0, + "read_p50": null, + "read_p99": null, + "events_per_sec": 0.0, + "delivery_p50": null, + "delivery_p99": null, + "write_bp": 0, + "write_err": 0, + "read_bp": 0, + "read_err": 0, + "status": "ok", + "reason": "complete" + } +] \ No newline at end of file diff --git a/results/canonical-mixed-cal/report.md b/results/canonical-mixed-cal/report.md new file mode 100644 index 0000000..e8dbade --- /dev/null +++ b/results/canonical-mixed-cal/report.md @@ -0,0 +1,13 @@ +# canonical-mixed-cal — mixed read/write interference report + +Sweep axis: **writer_rate**. Latency cells are p50/p99 ms. ‡ = backpressure (503/429) observed in that class. + +## wal — 50 streams + +| rate/writer | write ops/s | write ms | read ops/s | read MiB/s | read ms | deliv rec/s | deliv ms | status | +|---|---|---|---|---|---|---|---|---| +| max | 81671 | 0.5/1.0 | 0 | 0.0 | — | 0 | — | ok | + +## Findings + +_TODO: written by hand on top of the generated data._ diff --git a/results/canonical-mixed-cal/wal/cells.json b/results/canonical-mixed-cal/wal/cells.json new file mode 100644 index 0000000..1f5cf11 --- /dev/null +++ b/results/canonical-mixed-cal/wal/cells.json @@ -0,0 +1,40 @@ +{ + "cells": { + "50": { + "stream_count": 50, + "image_digest": "92f4a6387253", + "complete": true, + "levels": { + "0": { + "write_ops_per_sec": 81670.65, + "write_p50": 0.474, + "write_p99": 0.961, + "read_ops_per_sec": 0.0, + "read_mib_per_sec": 0.0, + "read_p50": null, + "read_p99": null, + "events_per_sec": 0.0, + "delivery_p50": null, + "delivery_p99": null, + "write_ok": 1633413, + "write_bp": 0, + "write_err": 0, + "read_ok": 0, + "read_bp": 0, + "read_err": 0, + "events_received": 0, + "control_events_received": 0, + "elapsed_secs": 20.088478845, + "drive_secs": 20.0, + "level": 0, + "readers": 0, + "subscribers": 0, + "writer_rate": 0, + "read_rate": 0, + "status": "ok", + "reason": "complete" + } + } + } + } +} \ No newline at end of file diff --git a/results/canonical-mixed-delivery/aggregate.csv b/results/canonical-mixed-delivery/aggregate.csv new file mode 100644 index 0000000..3f9ee30 --- /dev/null +++ b/results/canonical-mixed-delivery/aggregate.csv @@ -0,0 +1,11 @@ +mode,stream_count,sweep,level,readers,subscribers,writer_rate,read_rate,write_ops_per_sec,write_p50,write_p99,read_ops_per_sec,read_mib_per_sec,read_p50,read_p99,events_per_sec,delivery_p50,delivery_p99,write_bp,write_err,read_bp,read_err,status,reason +memory,2000,writer_rate,0,0,2000,0,0,127139.0,14.527,41.279,0.0,0.0,,,126729.93333333333,12.015,45.023,0,0,0,0,ok,complete +memory,2000,writer_rate,2,0,2000,2,0,4066.6666666666665,1.682,129.407,0.0,0.0,,,3341.3333333333335,0.924,150.655,0,0,0,0,ok,complete +memory,2000,writer_rate,8,0,2000,8,0,16047.333333333334,1.654,3.761,0.0,0.0,,,13305.3,0.954,5.063,0,0,0,0,ok,complete +memory,2000,writer_rate,20,0,2000,20,0,39984.833333333336,1.822,3.033,0.0,0.0,,,33226.433333333334,1.09,2.333,0,0,0,0,ok,complete +memory,2000,writer_rate,33,0,2000,33,0,65920.43333333333,2.101,93.759,0.0,0.0,,,65691.26666666666,1.214,8.367,0,0,0,0,ok,complete +wal,2000,writer_rate,0,0,2000,0,0,85168.3,21.935,95.871,0.0,0.0,,,63848.26666666667,35.807,105.599,0,0,0,0,ok,complete +wal,2000,writer_rate,2,0,2000,2,0,4066.6666666666665,1.67,121.151,0.0,0.0,,,3336.4666666666667,1.011,137.471,0,0,0,0,ok,complete +wal,2000,writer_rate,8,0,2000,8,0,16048.966666666667,1.693,58.143,0.0,0.0,,,15944.433333333332,1.063,54.175,0,0,0,0,ok,complete +wal,2000,writer_rate,20,0,2000,20,0,39977.566666666666,2.251,135.807,0.0,0.0,,,33261.933333333334,1.437,86.079,0,0,0,0,ok,complete +wal,2000,writer_rate,33,0,2000,33,0,65915.2,10.119,262.399,0.0,0.0,,,54853.333333333336,10.063,107.071,0,0,0,0,ok,complete diff --git a/results/canonical-mixed-delivery/aggregate.json b/results/canonical-mixed-delivery/aggregate.json new file mode 100644 index 0000000..bc6166c --- /dev/null +++ b/results/canonical-mixed-delivery/aggregate.json @@ -0,0 +1,262 @@ +[ + { + "mode": "memory", + "stream_count": 2000, + "sweep": "writer_rate", + "level": 0, + "readers": 0, + "subscribers": 2000, + "writer_rate": 0, + "read_rate": 0, + "write_ops_per_sec": 127139.0, + "write_p50": 14.527, + "write_p99": 41.279, + "read_ops_per_sec": 0.0, + "read_mib_per_sec": 0.0, + "read_p50": null, + "read_p99": null, + "events_per_sec": 126729.93333333333, + "delivery_p50": 12.015, + "delivery_p99": 45.023, + "write_bp": 0, + "write_err": 0, + "read_bp": 0, + "read_err": 0, + "status": "ok", + "reason": "complete" + }, + { + "mode": "memory", + "stream_count": 2000, + "sweep": "writer_rate", + "level": 2, + "readers": 0, + "subscribers": 2000, + "writer_rate": 2, + "read_rate": 0, + "write_ops_per_sec": 4066.6666666666665, + "write_p50": 1.682, + "write_p99": 129.407, + "read_ops_per_sec": 0.0, + "read_mib_per_sec": 0.0, + "read_p50": null, + "read_p99": null, + "events_per_sec": 3341.3333333333335, + "delivery_p50": 0.924, + "delivery_p99": 150.655, + "write_bp": 0, + "write_err": 0, + "read_bp": 0, + "read_err": 0, + "status": "ok", + "reason": "complete" + }, + { + "mode": "memory", + "stream_count": 2000, + "sweep": "writer_rate", + "level": 8, + "readers": 0, + "subscribers": 2000, + "writer_rate": 8, + "read_rate": 0, + "write_ops_per_sec": 16047.333333333334, + "write_p50": 1.654, + "write_p99": 3.761, + "read_ops_per_sec": 0.0, + "read_mib_per_sec": 0.0, + "read_p50": null, + "read_p99": null, + "events_per_sec": 13305.3, + "delivery_p50": 0.954, + "delivery_p99": 5.063, + "write_bp": 0, + "write_err": 0, + "read_bp": 0, + "read_err": 0, + "status": "ok", + "reason": "complete" + }, + { + "mode": "memory", + "stream_count": 2000, + "sweep": "writer_rate", + "level": 20, + "readers": 0, + "subscribers": 2000, + "writer_rate": 20, + "read_rate": 0, + "write_ops_per_sec": 39984.833333333336, + "write_p50": 1.822, + "write_p99": 3.033, + "read_ops_per_sec": 0.0, + "read_mib_per_sec": 0.0, + "read_p50": null, + "read_p99": null, + "events_per_sec": 33226.433333333334, + "delivery_p50": 1.09, + "delivery_p99": 2.333, + "write_bp": 0, + "write_err": 0, + "read_bp": 0, + "read_err": 0, + "status": "ok", + "reason": "complete" + }, + { + "mode": "memory", + "stream_count": 2000, + "sweep": "writer_rate", + "level": 33, + "readers": 0, + "subscribers": 2000, + "writer_rate": 33, + "read_rate": 0, + "write_ops_per_sec": 65920.43333333333, + "write_p50": 2.101, + "write_p99": 93.759, + "read_ops_per_sec": 0.0, + "read_mib_per_sec": 0.0, + "read_p50": null, + "read_p99": null, + "events_per_sec": 65691.26666666666, + "delivery_p50": 1.214, + "delivery_p99": 8.367, + "write_bp": 0, + "write_err": 0, + "read_bp": 0, + "read_err": 0, + "status": "ok", + "reason": "complete" + }, + { + "mode": "wal", + "stream_count": 2000, + "sweep": "writer_rate", + "level": 0, + "readers": 0, + "subscribers": 2000, + "writer_rate": 0, + "read_rate": 0, + "write_ops_per_sec": 85168.3, + "write_p50": 21.935, + "write_p99": 95.871, + "read_ops_per_sec": 0.0, + "read_mib_per_sec": 0.0, + "read_p50": null, + "read_p99": null, + "events_per_sec": 63848.26666666667, + "delivery_p50": 35.807, + "delivery_p99": 105.599, + "write_bp": 0, + "write_err": 0, + "read_bp": 0, + "read_err": 0, + "status": "ok", + "reason": "complete" + }, + { + "mode": "wal", + "stream_count": 2000, + "sweep": "writer_rate", + "level": 2, + "readers": 0, + "subscribers": 2000, + "writer_rate": 2, + "read_rate": 0, + "write_ops_per_sec": 4066.6666666666665, + "write_p50": 1.67, + "write_p99": 121.151, + "read_ops_per_sec": 0.0, + "read_mib_per_sec": 0.0, + "read_p50": null, + "read_p99": null, + "events_per_sec": 3336.4666666666667, + "delivery_p50": 1.011, + "delivery_p99": 137.471, + "write_bp": 0, + "write_err": 0, + "read_bp": 0, + "read_err": 0, + "status": "ok", + "reason": "complete" + }, + { + "mode": "wal", + "stream_count": 2000, + "sweep": "writer_rate", + "level": 8, + "readers": 0, + "subscribers": 2000, + "writer_rate": 8, + "read_rate": 0, + "write_ops_per_sec": 16048.966666666667, + "write_p50": 1.693, + "write_p99": 58.143, + "read_ops_per_sec": 0.0, + "read_mib_per_sec": 0.0, + "read_p50": null, + "read_p99": null, + "events_per_sec": 15944.433333333332, + "delivery_p50": 1.063, + "delivery_p99": 54.175, + "write_bp": 0, + "write_err": 0, + "read_bp": 0, + "read_err": 0, + "status": "ok", + "reason": "complete" + }, + { + "mode": "wal", + "stream_count": 2000, + "sweep": "writer_rate", + "level": 20, + "readers": 0, + "subscribers": 2000, + "writer_rate": 20, + "read_rate": 0, + "write_ops_per_sec": 39977.566666666666, + "write_p50": 2.251, + "write_p99": 135.807, + "read_ops_per_sec": 0.0, + "read_mib_per_sec": 0.0, + "read_p50": null, + "read_p99": null, + "events_per_sec": 33261.933333333334, + "delivery_p50": 1.437, + "delivery_p99": 86.079, + "write_bp": 0, + "write_err": 0, + "read_bp": 0, + "read_err": 0, + "status": "ok", + "reason": "complete" + }, + { + "mode": "wal", + "stream_count": 2000, + "sweep": "writer_rate", + "level": 33, + "readers": 0, + "subscribers": 2000, + "writer_rate": 33, + "read_rate": 0, + "write_ops_per_sec": 65915.2, + "write_p50": 10.119, + "write_p99": 262.399, + "read_ops_per_sec": 0.0, + "read_mib_per_sec": 0.0, + "read_p50": null, + "read_p99": null, + "events_per_sec": 54853.333333333336, + "delivery_p50": 10.063, + "delivery_p99": 107.071, + "write_bp": 0, + "write_err": 0, + "read_bp": 0, + "read_err": 0, + "status": "ok", + "reason": "complete" + } +] \ No newline at end of file diff --git a/results/canonical-mixed-delivery/memory/cells.json b/results/canonical-mixed-delivery/memory/cells.json new file mode 100644 index 0000000..0f33627 --- /dev/null +++ b/results/canonical-mixed-delivery/memory/cells.json @@ -0,0 +1,156 @@ +{ + "cells": { + "2000": { + "stream_count": 2000, + "image_digest": "ea3e9784de6d", + "complete": true, + "levels": { + "2": { + "write_ops_per_sec": 4066.6666666666665, + "write_p50": 1.682, + "write_p99": 129.407, + "read_ops_per_sec": 0.0, + "read_mib_per_sec": 0.0, + "read_p50": null, + "read_p99": null, + "events_per_sec": 3341.3333333333335, + "delivery_p50": 0.924, + "delivery_p99": 150.655, + "write_ok": 122000, + "write_bp": 0, + "write_err": 0, + "read_ok": 0, + "read_bp": 0, + "read_err": 0, + "events_received": 100240, + "control_events_received": 102240, + "elapsed_secs": 35.314730835, + "drive_secs": 30.0, + "level": 2, + "readers": 0, + "subscribers": 2000, + "writer_rate": 2, + "read_rate": 0, + "status": "ok", + "reason": "complete" + }, + "8": { + "write_ops_per_sec": 16047.333333333334, + "write_p50": 1.654, + "write_p99": 3.761, + "read_ops_per_sec": 0.0, + "read_mib_per_sec": 0.0, + "read_p50": null, + "read_p99": null, + "events_per_sec": 13305.3, + "delivery_p50": 0.954, + "delivery_p99": 5.063, + "write_ok": 481420, + "write_bp": 0, + "write_err": 0, + "read_ok": 0, + "read_bp": 0, + "read_err": 0, + "events_received": 399159, + "control_events_received": 401159, + "elapsed_secs": 35.198380154, + "drive_secs": 30.0, + "level": 8, + "readers": 0, + "subscribers": 2000, + "writer_rate": 8, + "read_rate": 0, + "status": "ok", + "reason": "complete" + }, + "20": { + "write_ops_per_sec": 39984.833333333336, + "write_p50": 1.822, + "write_p99": 3.033, + "read_ops_per_sec": 0.0, + "read_mib_per_sec": 0.0, + "read_p50": null, + "read_p99": null, + "events_per_sec": 33226.433333333334, + "delivery_p50": 1.09, + "delivery_p99": 2.333, + "write_ok": 1199545, + "write_bp": 0, + "write_err": 0, + "read_ok": 0, + "read_bp": 0, + "read_err": 0, + "events_received": 996793, + "control_events_received": 998793, + "elapsed_secs": 35.333831543, + "drive_secs": 30.0, + "level": 20, + "readers": 0, + "subscribers": 2000, + "writer_rate": 20, + "read_rate": 0, + "status": "ok", + "reason": "complete" + }, + "33": { + "write_ops_per_sec": 65920.43333333333, + "write_p50": 2.101, + "write_p99": 93.759, + "read_ops_per_sec": 0.0, + "read_mib_per_sec": 0.0, + "read_p50": null, + "read_p99": null, + "events_per_sec": 65691.26666666666, + "delivery_p50": 1.214, + "delivery_p99": 8.367, + "write_ok": 1977613, + "write_bp": 0, + "write_err": 0, + "read_ok": 0, + "read_bp": 0, + "read_err": 0, + "events_received": 1970738, + "control_events_received": 1972738, + "elapsed_secs": 30.25392982, + "drive_secs": 30.0, + "level": 33, + "readers": 0, + "subscribers": 2000, + "writer_rate": 33, + "read_rate": 0, + "status": "ok", + "reason": "complete" + }, + "0": { + "write_ops_per_sec": 127139.0, + "write_p50": 14.527, + "write_p99": 41.279, + "read_ops_per_sec": 0.0, + "read_mib_per_sec": 0.0, + "read_p50": null, + "read_p99": null, + "events_per_sec": 126729.93333333333, + "delivery_p50": 12.015, + "delivery_p99": 45.023, + "write_ok": 3814170, + "write_bp": 0, + "write_err": 0, + "read_ok": 0, + "read_bp": 0, + "read_err": 0, + "events_received": 3801898, + "control_events_received": 3803898, + "elapsed_secs": 30.436254117, + "drive_secs": 30.0, + "level": 0, + "readers": 0, + "subscribers": 2000, + "writer_rate": 0, + "read_rate": 0, + "status": "ok", + "reason": "complete" + } + } + } + } +} \ No newline at end of file diff --git a/results/canonical-mixed-delivery/report.md b/results/canonical-mixed-delivery/report.md new file mode 100644 index 0000000..471e9f5 --- /dev/null +++ b/results/canonical-mixed-delivery/report.md @@ -0,0 +1,27 @@ +# canonical-mixed-delivery — mixed read/write interference report + +Sweep axis: **writer_rate**. Latency cells are p50/p99 ms. ‡ = backpressure (503/429) observed in that class. + +## wal — 2000 streams + +| rate/writer | write ops/s | write ms | read ops/s | read MiB/s | read ms | deliv rec/s | deliv ms | status | +|---|---|---|---|---|---|---|---|---| +| 2 | 4067 | 1.7/121.2 | 0 | 0.0 | — | 3336 | 1.0/137.5 | ok | +| 8 | 16049 | 1.7/58.1 | 0 | 0.0 | — | 15944 | 1.1/54.2 | ok | +| 20 | 39978 | 2.3/135.8 | 0 | 0.0 | — | 33262 | 1.4/86.1 | ok | +| 33 | 65915 | 10.1/262.4 | 0 | 0.0 | — | 54853 | 10.1/107.1 | ok | +| max | 85168 | 21.9/95.9 | 0 | 0.0 | — | 63848 | 35.8/105.6 | ok | + +## memory — 2000 streams + +| rate/writer | write ops/s | write ms | read ops/s | read MiB/s | read ms | deliv rec/s | deliv ms | status | +|---|---|---|---|---|---|---|---|---| +| 2 | 4067 | 1.7/129.4 | 0 | 0.0 | — | 3341 | 0.9/150.7 | ok | +| 8 | 16047 | 1.7/3.8 | 0 | 0.0 | — | 13305 | 1.0/5.1 | ok | +| 20 | 39985 | 1.8/3.0 | 0 | 0.0 | — | 33226 | 1.1/2.3 | ok | +| 33 | 65920 | 2.1/93.8 | 0 | 0.0 | — | 65691 | 1.2/8.4 | ok | +| max | 127139 | 14.5/41.3 | 0 | 0.0 | — | 126730 | 12.0/45.0 | ok | + +## Findings + +_TODO: written by hand on top of the generated data._ diff --git a/results/canonical-mixed-delivery/wal/cells.json b/results/canonical-mixed-delivery/wal/cells.json new file mode 100644 index 0000000..aeec4a7 --- /dev/null +++ b/results/canonical-mixed-delivery/wal/cells.json @@ -0,0 +1,156 @@ +{ + "cells": { + "2000": { + "stream_count": 2000, + "image_digest": "92f4a6387253", + "complete": true, + "levels": { + "2": { + "write_ops_per_sec": 4066.6666666666665, + "write_p50": 1.67, + "write_p99": 121.151, + "read_ops_per_sec": 0.0, + "read_mib_per_sec": 0.0, + "read_p50": null, + "read_p99": null, + "events_per_sec": 3336.4666666666667, + "delivery_p50": 1.011, + "delivery_p99": 137.471, + "write_ok": 122000, + "write_bp": 0, + "write_err": 0, + "read_ok": 0, + "read_bp": 0, + "read_err": 0, + "events_received": 100094, + "control_events_received": 102094, + "elapsed_secs": 35.276377325, + "drive_secs": 30.0, + "level": 2, + "readers": 0, + "subscribers": 2000, + "writer_rate": 2, + "read_rate": 0, + "status": "ok", + "reason": "complete" + }, + "8": { + "write_ops_per_sec": 16048.966666666667, + "write_p50": 1.693, + "write_p99": 58.143, + "read_ops_per_sec": 0.0, + "read_mib_per_sec": 0.0, + "read_p50": null, + "read_p99": null, + "events_per_sec": 15944.433333333332, + "delivery_p50": 1.063, + "delivery_p99": 54.175, + "write_ok": 481469, + "write_bp": 0, + "write_err": 0, + "read_ok": 0, + "read_bp": 0, + "read_err": 0, + "events_received": 478333, + "control_events_received": 480333, + "elapsed_secs": 30.420944699, + "drive_secs": 30.0, + "level": 8, + "readers": 0, + "subscribers": 2000, + "writer_rate": 8, + "read_rate": 0, + "status": "ok", + "reason": "complete" + }, + "20": { + "write_ops_per_sec": 39977.566666666666, + "write_p50": 2.251, + "write_p99": 135.807, + "read_ops_per_sec": 0.0, + "read_mib_per_sec": 0.0, + "read_p50": null, + "read_p99": null, + "events_per_sec": 33261.933333333334, + "delivery_p50": 1.437, + "delivery_p99": 86.079, + "write_ok": 1199327, + "write_bp": 0, + "write_err": 0, + "read_ok": 0, + "read_bp": 0, + "read_err": 0, + "events_received": 997858, + "control_events_received": 999858, + "elapsed_secs": 35.153495591, + "drive_secs": 30.0, + "level": 20, + "readers": 0, + "subscribers": 2000, + "writer_rate": 20, + "read_rate": 0, + "status": "ok", + "reason": "complete" + }, + "33": { + "write_ops_per_sec": 65915.2, + "write_p50": 10.119, + "write_p99": 262.399, + "read_ops_per_sec": 0.0, + "read_mib_per_sec": 0.0, + "read_p50": null, + "read_p99": null, + "events_per_sec": 54853.333333333336, + "delivery_p50": 10.063, + "delivery_p99": 107.071, + "write_ok": 1977456, + "write_bp": 0, + "write_err": 0, + "read_ok": 0, + "read_bp": 0, + "read_err": 0, + "events_received": 1645600, + "control_events_received": 1647600, + "elapsed_secs": 35.307107715, + "drive_secs": 30.0, + "level": 33, + "readers": 0, + "subscribers": 2000, + "writer_rate": 33, + "read_rate": 0, + "status": "ok", + "reason": "complete" + }, + "0": { + "write_ops_per_sec": 85168.3, + "write_p50": 21.935, + "write_p99": 95.871, + "read_ops_per_sec": 0.0, + "read_mib_per_sec": 0.0, + "read_p50": null, + "read_p99": null, + "events_per_sec": 63848.26666666667, + "delivery_p50": 35.807, + "delivery_p99": 105.599, + "write_ok": 2555049, + "write_bp": 0, + "write_err": 0, + "read_ok": 0, + "read_bp": 0, + "read_err": 0, + "events_received": 1915448, + "control_events_received": 1917448, + "elapsed_secs": 35.193152446, + "drive_secs": 30.0, + "level": 0, + "readers": 0, + "subscribers": 2000, + "writer_rate": 0, + "read_rate": 0, + "status": "ok", + "reason": "complete" + } + } + } + } +} \ No newline at end of file diff --git a/results/canonical-mixed-writes/aggregate.csv b/results/canonical-mixed-writes/aggregate.csv new file mode 100644 index 0000000..8c83cd7 --- /dev/null +++ b/results/canonical-mixed-writes/aggregate.csv @@ -0,0 +1,5 @@ +mode,stream_count,sweep,level,readers,subscribers,writer_rate,read_rate,write_ops_per_sec,write_p50,write_p99,read_ops_per_sec,read_mib_per_sec,read_p50,read_p99,events_per_sec,delivery_p50,delivery_p99,write_bp,write_err,read_bp,read_err,status,reason +wal,10000,readers,0,0,0,5,0,49910.63333333333,10.839,1701.887,0.0,0.0,,,0.0,,,0,0,0,0,ok,complete +wal,10000,readers,1000,1000,0,5,0,50025.26666666667,7.955,1419.263,49.95,3.051070149739583,2.303,399.359,0.0,,,0,0,0,0,ok,complete +wal,10000,readers,10000,10000,0,5,0,49977.98333333333,9.031,1516.543,499.1166666666667,30.42186279296875,2.115,413.695,0.0,,,0,0,0,0,ok,complete +wal,10000,readers,100000,100000,0,5,0,50011.86666666667,22.543,3770.367,4986.233333333334,302.88669840494794,2.095,1404.927,0.0,,,0,0,0,0,ok,complete diff --git a/results/canonical-mixed-writes/aggregate.json b/results/canonical-mixed-writes/aggregate.json new file mode 100644 index 0000000..79ab967 --- /dev/null +++ b/results/canonical-mixed-writes/aggregate.json @@ -0,0 +1,106 @@ +[ + { + "mode": "wal", + "stream_count": 10000, + "sweep": "readers", + "level": 0, + "readers": 0, + "subscribers": 0, + "writer_rate": 5, + "read_rate": 0, + "write_ops_per_sec": 49910.63333333333, + "write_p50": 10.839, + "write_p99": 1701.887, + "read_ops_per_sec": 0.0, + "read_mib_per_sec": 0.0, + "read_p50": null, + "read_p99": null, + "events_per_sec": 0.0, + "delivery_p50": null, + "delivery_p99": null, + "write_bp": 0, + "write_err": 0, + "read_bp": 0, + "read_err": 0, + "status": "ok", + "reason": "complete" + }, + { + "mode": "wal", + "stream_count": 10000, + "sweep": "readers", + "level": 1000, + "readers": 1000, + "subscribers": 0, + "writer_rate": 5, + "read_rate": 0, + "write_ops_per_sec": 50025.26666666667, + "write_p50": 7.955, + "write_p99": 1419.263, + "read_ops_per_sec": 49.95, + "read_mib_per_sec": 3.051070149739583, + "read_p50": 2.303, + "read_p99": 399.359, + "events_per_sec": 0.0, + "delivery_p50": null, + "delivery_p99": null, + "write_bp": 0, + "write_err": 0, + "read_bp": 0, + "read_err": 0, + "status": "ok", + "reason": "complete" + }, + { + "mode": "wal", + "stream_count": 10000, + "sweep": "readers", + "level": 10000, + "readers": 10000, + "subscribers": 0, + "writer_rate": 5, + "read_rate": 0, + "write_ops_per_sec": 49977.98333333333, + "write_p50": 9.031, + "write_p99": 1516.543, + "read_ops_per_sec": 499.1166666666667, + "read_mib_per_sec": 30.42186279296875, + "read_p50": 2.115, + "read_p99": 413.695, + "events_per_sec": 0.0, + "delivery_p50": null, + "delivery_p99": null, + "write_bp": 0, + "write_err": 0, + "read_bp": 0, + "read_err": 0, + "status": "ok", + "reason": "complete" + }, + { + "mode": "wal", + "stream_count": 10000, + "sweep": "readers", + "level": 100000, + "readers": 100000, + "subscribers": 0, + "writer_rate": 5, + "read_rate": 0, + "write_ops_per_sec": 50011.86666666667, + "write_p50": 22.543, + "write_p99": 3770.367, + "read_ops_per_sec": 4986.233333333334, + "read_mib_per_sec": 302.88669840494794, + "read_p50": 2.095, + "read_p99": 1404.927, + "events_per_sec": 0.0, + "delivery_p50": null, + "delivery_p99": null, + "write_bp": 0, + "write_err": 0, + "read_bp": 0, + "read_err": 0, + "status": "ok", + "reason": "complete" + } +] \ No newline at end of file diff --git a/results/canonical-mixed-writes/report.md b/results/canonical-mixed-writes/report.md new file mode 100644 index 0000000..4cc8fd4 --- /dev/null +++ b/results/canonical-mixed-writes/report.md @@ -0,0 +1,16 @@ +# canonical-mixed-writes — mixed read/write interference report + +Sweep axis: **readers**. Latency cells are p50/p99 ms. ‡ = backpressure (503/429) observed in that class. + +## wal — 10000 streams + +| readers | write ops/s | write ms | read ops/s | read MiB/s | read ms | deliv rec/s | deliv ms | status | +|---|---|---|---|---|---|---|---|---| +| 0 | 49911 | 10.8/1701.9 | 0 | 0.0 | — | 0 | — | ok | +| 1000 | 50025 | 8.0/1419.3 | 50 | 3.1 | 2.3/399.4 | 0 | — | ok | +| 10000 | 49978 | 9.0/1516.5 | 499 | 30.4 | 2.1/413.7 | 0 | — | ok | +| 100000 | 50012 | 22.5/3770.4 | 4986 | 302.9 | 2.1/1404.9 | 0 | — | ok | + +## Findings + +_TODO: written by hand on top of the generated data._ diff --git a/results/canonical-mixed-writes/wal/cells.json b/results/canonical-mixed-writes/wal/cells.json new file mode 100644 index 0000000..846e748 --- /dev/null +++ b/results/canonical-mixed-writes/wal/cells.json @@ -0,0 +1,127 @@ +{ + "cells": { + "10000": { + "stream_count": 10000, + "image_digest": "92f4a6387253", + "complete": true, + "levels": { + "0": { + "write_ops_per_sec": 49910.63333333333, + "write_p50": 10.839, + "write_p99": 1701.887, + "read_ops_per_sec": 0.0, + "read_mib_per_sec": 0.0, + "read_p50": null, + "read_p99": null, + "events_per_sec": 0.0, + "delivery_p50": null, + "delivery_p99": null, + "write_ok": 2994638, + "write_bp": 0, + "write_err": 0, + "read_ok": 0, + "read_bp": 0, + "read_err": 0, + "events_received": 0, + "control_events_received": 0, + "elapsed_secs": 60.575083058, + "drive_secs": 60.0, + "level": 0, + "readers": 0, + "subscribers": 0, + "writer_rate": 5, + "read_rate": 0, + "status": "ok", + "reason": "complete" + }, + "1000": { + "write_ops_per_sec": 50025.26666666667, + "write_p50": 7.955, + "write_p99": 1419.263, + "read_ops_per_sec": 49.95, + "read_mib_per_sec": 3.051070149739583, + "read_p50": 2.303, + "read_p99": 399.359, + "events_per_sec": 0.0, + "delivery_p50": null, + "delivery_p99": null, + "write_ok": 3001516, + "write_bp": 0, + "write_err": 0, + "read_ok": 2997, + "read_bp": 0, + "read_err": 0, + "events_received": 0, + "control_events_received": 0, + "elapsed_secs": 90.017702397, + "drive_secs": 60.0, + "level": 1000, + "readers": 1000, + "subscribers": 0, + "writer_rate": 5, + "read_rate": 0, + "status": "ok", + "reason": "complete" + }, + "10000": { + "write_ops_per_sec": 49977.98333333333, + "write_p50": 9.031, + "write_p99": 1516.543, + "read_ops_per_sec": 499.1166666666667, + "read_mib_per_sec": 30.42186279296875, + "read_p50": 2.115, + "read_p99": 413.695, + "events_per_sec": 0.0, + "delivery_p50": null, + "delivery_p99": null, + "write_ok": 2998679, + "write_bp": 0, + "write_err": 0, + "read_ok": 29947, + "read_bp": 0, + "read_err": 0, + "events_received": 0, + "control_events_received": 0, + "elapsed_secs": 90.02590709, + "drive_secs": 60.0, + "level": 10000, + "readers": 10000, + "subscribers": 0, + "writer_rate": 5, + "read_rate": 0, + "status": "ok", + "reason": "complete" + }, + "100000": { + "write_ops_per_sec": 50011.86666666667, + "write_p50": 22.543, + "write_p99": 3770.367, + "read_ops_per_sec": 4986.233333333334, + "read_mib_per_sec": 302.88669840494794, + "read_p50": 2.095, + "read_p99": 1404.927, + "events_per_sec": 0.0, + "delivery_p50": null, + "delivery_p99": null, + "write_ok": 3000712, + "write_bp": 0, + "write_err": 0, + "read_ok": 299174, + "read_bp": 0, + "read_err": 0, + "events_received": 0, + "control_events_received": 0, + "elapsed_secs": 90.176862163, + "drive_secs": 60.0, + "level": 100000, + "readers": 100000, + "subscribers": 0, + "writer_rate": 5, + "read_rate": 0, + "status": "ok", + "reason": "complete" + } + } + } + } +} \ No newline at end of file diff --git a/results/canonical-reads-catchup/aggregate.csv b/results/canonical-reads-catchup/aggregate.csv new file mode 100644 index 0000000..f534556 --- /dev/null +++ b/results/canonical-reads-catchup/aggregate.csv @@ -0,0 +1,17 @@ +mode,stream_count,connections,ops_per_sec,bytes_per_sec,mib_per_sec,p50,p99,backpressure,other_err,status,is_peak +ursula,10,8,147.06666666666666,2467369233.0666666,2353.1,54.143,66.303,0,0,ok,False +ursula,10,32,148.4,2489738854.4,2374.4,198.271,485.119,0,0,ok,False +ursula,10,128,154.33333333333334,2589283669.3333335,2469.3,807.423,1763.327,0,0,ok,False +ursula,10,512,181.4,3043386982.4,2902.4,3123.199,12099.583,0,0,ok,True +ursula,100,8,0.0,0.0,0.0,,,0,0,error,True +ursula,100,32,0.0,0.0,0.0,,,0,0,error,False +ursula,100,128,0.0,0.0,0.0,,,0,0,error,False +ursula,100,512,0.0,0.0,0.0,,,0,0,error,False +wal,10,8,142.06666666666666,2383483153.0666666,2273.1,56.447,62.239,0,0,ok,False +wal,10,32,149.53333333333333,2508753032.5333333,2392.5,217.727,248.831,0,0,ok,False +wal,10,128,148.0,2483027968.0,2368.0,904.703,1477.631,0,0,ok,False +wal,10,512,170.46666666666667,2859956087.4666667,2727.5,3753.983,5070.847,0,0,ok,True +wal,100,8,142.26666666666668,2386838596.266667,2276.3,56.319,62.303,0,0,ok,False +wal,100,32,149.13333333333333,2502042146.133333,2386.1,218.239,248.959,0,0,ok,False +wal,100,128,147.13333333333333,2468487714.133333,2354.1,941.567,1376.255,0,0,ok,False +wal,100,512,171.2,2872259379.2,2739.2,3721.215,4644.863,0,0,ok,True diff --git a/results/canonical-reads-catchup/aggregate.json b/results/canonical-reads-catchup/aggregate.json new file mode 100644 index 0000000..b7c3e56 --- /dev/null +++ b/results/canonical-reads-catchup/aggregate.json @@ -0,0 +1,226 @@ +[ + { + "mode": "ursula", + "stream_count": 10, + "connections": 8, + "ops_per_sec": 147.06666666666666, + "bytes_per_sec": 2467369233.0666666, + "mib_per_sec": 2353.1, + "p50": 54.143, + "p99": 66.303, + "backpressure": 0, + "other_err": 0, + "status": "ok", + "is_peak": false + }, + { + "mode": "ursula", + "stream_count": 10, + "connections": 32, + "ops_per_sec": 148.4, + "bytes_per_sec": 2489738854.4, + "mib_per_sec": 2374.4, + "p50": 198.271, + "p99": 485.119, + "backpressure": 0, + "other_err": 0, + "status": "ok", + "is_peak": false + }, + { + "mode": "ursula", + "stream_count": 10, + "connections": 128, + "ops_per_sec": 154.33333333333334, + "bytes_per_sec": 2589283669.3333335, + "mib_per_sec": 2469.3, + "p50": 807.423, + "p99": 1763.327, + "backpressure": 0, + "other_err": 0, + "status": "ok", + "is_peak": false + }, + { + "mode": "ursula", + "stream_count": 10, + "connections": 512, + "ops_per_sec": 181.4, + "bytes_per_sec": 3043386982.4, + "mib_per_sec": 2902.4, + "p50": 3123.199, + "p99": 12099.583, + "backpressure": 0, + "other_err": 0, + "status": "ok", + "is_peak": true + }, + { + "mode": "ursula", + "stream_count": 100, + "connections": 8, + "ops_per_sec": 0.0, + "bytes_per_sec": 0.0, + "mib_per_sec": 0.0, + "p50": null, + "p99": null, + "backpressure": 0, + "other_err": 0, + "status": "error", + "is_peak": true + }, + { + "mode": "ursula", + "stream_count": 100, + "connections": 32, + "ops_per_sec": 0.0, + "bytes_per_sec": 0.0, + "mib_per_sec": 0.0, + "p50": null, + "p99": null, + "backpressure": 0, + "other_err": 0, + "status": "error", + "is_peak": false + }, + { + "mode": "ursula", + "stream_count": 100, + "connections": 128, + "ops_per_sec": 0.0, + "bytes_per_sec": 0.0, + "mib_per_sec": 0.0, + "p50": null, + "p99": null, + "backpressure": 0, + "other_err": 0, + "status": "error", + "is_peak": false + }, + { + "mode": "ursula", + "stream_count": 100, + "connections": 512, + "ops_per_sec": 0.0, + "bytes_per_sec": 0.0, + "mib_per_sec": 0.0, + "p50": null, + "p99": null, + "backpressure": 0, + "other_err": 0, + "status": "error", + "is_peak": false + }, + { + "mode": "wal", + "stream_count": 10, + "connections": 8, + "ops_per_sec": 142.06666666666666, + "bytes_per_sec": 2383483153.0666666, + "mib_per_sec": 2273.1, + "p50": 56.447, + "p99": 62.239, + "backpressure": 0, + "other_err": 0, + "status": "ok", + "is_peak": false + }, + { + "mode": "wal", + "stream_count": 10, + "connections": 32, + "ops_per_sec": 149.53333333333333, + "bytes_per_sec": 2508753032.5333333, + "mib_per_sec": 2392.5, + "p50": 217.727, + "p99": 248.831, + "backpressure": 0, + "other_err": 0, + "status": "ok", + "is_peak": false + }, + { + "mode": "wal", + "stream_count": 10, + "connections": 128, + "ops_per_sec": 148.0, + "bytes_per_sec": 2483027968.0, + "mib_per_sec": 2368.0, + "p50": 904.703, + "p99": 1477.631, + "backpressure": 0, + "other_err": 0, + "status": "ok", + "is_peak": false + }, + { + "mode": "wal", + "stream_count": 10, + "connections": 512, + "ops_per_sec": 170.46666666666667, + "bytes_per_sec": 2859956087.4666667, + "mib_per_sec": 2727.5, + "p50": 3753.983, + "p99": 5070.847, + "backpressure": 0, + "other_err": 0, + "status": "ok", + "is_peak": true + }, + { + "mode": "wal", + "stream_count": 100, + "connections": 8, + "ops_per_sec": 142.26666666666668, + "bytes_per_sec": 2386838596.266667, + "mib_per_sec": 2276.3, + "p50": 56.319, + "p99": 62.303, + "backpressure": 0, + "other_err": 0, + "status": "ok", + "is_peak": false + }, + { + "mode": "wal", + "stream_count": 100, + "connections": 32, + "ops_per_sec": 149.13333333333333, + "bytes_per_sec": 2502042146.133333, + "mib_per_sec": 2386.1, + "p50": 218.239, + "p99": 248.959, + "backpressure": 0, + "other_err": 0, + "status": "ok", + "is_peak": false + }, + { + "mode": "wal", + "stream_count": 100, + "connections": 128, + "ops_per_sec": 147.13333333333333, + "bytes_per_sec": 2468487714.133333, + "mib_per_sec": 2354.1, + "p50": 941.567, + "p99": 1376.255, + "backpressure": 0, + "other_err": 0, + "status": "ok", + "is_peak": false + }, + { + "mode": "wal", + "stream_count": 100, + "connections": 512, + "ops_per_sec": 171.2, + "bytes_per_sec": 2872259379.2, + "mib_per_sec": 2739.2, + "p50": 3721.215, + "p99": 4644.863, + "backpressure": 0, + "other_err": 0, + "status": "ok", + "is_peak": true + } +] \ No newline at end of file diff --git a/results/canonical-reads-catchup/report.md b/results/canonical-reads-catchup/report.md new file mode 100644 index 0000000..67bb641 --- /dev/null +++ b/results/canonical-reads-catchup/report.md @@ -0,0 +1,29 @@ +# canonical-reads-catchup — read-scalability report + +Each cell: aggregate read throughput (MiB/s) @ p99 latency (ms). ‡ = backpressure (503/429) observed at this load. + +## wal — throughput @ p99 over stream_count × connections + +| streams | 8 | 32 | 128 | 512 | +|---|---|---|---|---| +| 10 | 2273MiB/s@62ms | 2393MiB/s@249ms | 2368MiB/s@1478ms | 2727MiB/s@5071ms | +| 100 | 2276MiB/s@62ms | 2386MiB/s@249ms | 2354MiB/s@1376ms | 2739MiB/s@4645ms | + +Peak read throughput per cardinality: +- streams=10: 2727 MiB/s at 512 connections +- streams=100: 2739 MiB/s at 512 connections + +## ursula — throughput @ p99 over stream_count × connections + +| streams | 8 | 32 | 128 | 512 | +|---|---|---|---|---| +| 10 | 2353MiB/s@66ms | 2374MiB/s@485ms | 2469MiB/s@1763ms | 2902MiB/s@12100ms | +| 100 | ERR(0) | ERR(0) | ERR(0) | ERR(0) | + +Peak read throughput per cardinality: +- streams=10: 2902 MiB/s at 512 connections +- streams=100: 0 MiB/s at 8 connections + +## Findings + +_TODO: written by hand on top of the generated data._ diff --git a/results/canonical-reads-catchup/ursula/cells.json b/results/canonical-reads-catchup/ursula/cells.json new file mode 100644 index 0000000..59a75b5 --- /dev/null +++ b/results/canonical-reads-catchup/ursula/cells.json @@ -0,0 +1,106 @@ +{ + "cells": { + "10": { + "stream_count": 10, + "image_digest": "a56ce0252087", + "complete": true, + "connections": { + "8": { + "connections": 8, + "ops_per_sec": 147.06666666666666, + "bytes_per_sec": 2467369233.0666666, + "p50": 54.143, + "p99": 66.303, + "backpressure": 0, + "other_err": 0, + "status": "ok", + "reason": "complete" + }, + "32": { + "connections": 32, + "ops_per_sec": 148.4, + "bytes_per_sec": 2489738854.4, + "p50": 198.271, + "p99": 485.119, + "backpressure": 0, + "other_err": 0, + "status": "ok", + "reason": "complete" + }, + "128": { + "connections": 128, + "ops_per_sec": 154.33333333333334, + "bytes_per_sec": 2589283669.3333335, + "p50": 807.423, + "p99": 1763.327, + "backpressure": 0, + "other_err": 0, + "status": "ok", + "reason": "complete" + }, + "512": { + "connections": 512, + "ops_per_sec": 181.4, + "bytes_per_sec": 3043386982.4, + "p50": 3123.199, + "p99": 12099.583, + "backpressure": 0, + "other_err": 0, + "status": "ok", + "reason": "complete" + } + } + }, + "100": { + "stream_count": 100, + "image_digest": "a56ce0252087", + "complete": true, + "connections": { + "8": { + "connections": 8, + "ops_per_sec": 0.0, + "bytes_per_sec": 0.0, + "p50": null, + "p99": null, + "backpressure": 0, + "other_err": 0, + "status": "error", + "reason": "no_reads" + }, + "32": { + "connections": 32, + "ops_per_sec": 0.0, + "bytes_per_sec": 0.0, + "p50": null, + "p99": null, + "backpressure": 0, + "other_err": 0, + "status": "error", + "reason": "no_reads" + }, + "128": { + "connections": 128, + "ops_per_sec": 0.0, + "bytes_per_sec": 0.0, + "p50": null, + "p99": null, + "backpressure": 0, + "other_err": 0, + "status": "error", + "reason": "no_reads" + }, + "512": { + "connections": 512, + "ops_per_sec": 0.0, + "bytes_per_sec": 0.0, + "p50": null, + "p99": null, + "backpressure": 0, + "other_err": 0, + "status": "error", + "reason": "no_reads" + } + } + } + } +} \ No newline at end of file diff --git a/results/canonical-reads-catchup/wal/cells.json b/results/canonical-reads-catchup/wal/cells.json new file mode 100644 index 0000000..bc228c7 --- /dev/null +++ b/results/canonical-reads-catchup/wal/cells.json @@ -0,0 +1,106 @@ +{ + "cells": { + "10": { + "stream_count": 10, + "image_digest": "92f4a6387253", + "complete": true, + "connections": { + "8": { + "connections": 8, + "ops_per_sec": 142.06666666666666, + "bytes_per_sec": 2383483153.0666666, + "p50": 56.447, + "p99": 62.239, + "backpressure": 0, + "other_err": 0, + "status": "ok", + "reason": "complete" + }, + "32": { + "connections": 32, + "ops_per_sec": 149.53333333333333, + "bytes_per_sec": 2508753032.5333333, + "p50": 217.727, + "p99": 248.831, + "backpressure": 0, + "other_err": 0, + "status": "ok", + "reason": "complete" + }, + "128": { + "connections": 128, + "ops_per_sec": 148.0, + "bytes_per_sec": 2483027968.0, + "p50": 904.703, + "p99": 1477.631, + "backpressure": 0, + "other_err": 0, + "status": "ok", + "reason": "complete" + }, + "512": { + "connections": 512, + "ops_per_sec": 170.46666666666667, + "bytes_per_sec": 2859956087.4666667, + "p50": 3753.983, + "p99": 5070.847, + "backpressure": 0, + "other_err": 0, + "status": "ok", + "reason": "complete" + } + } + }, + "100": { + "stream_count": 100, + "image_digest": "92f4a6387253", + "complete": true, + "connections": { + "8": { + "connections": 8, + "ops_per_sec": 142.26666666666668, + "bytes_per_sec": 2386838596.266667, + "p50": 56.319, + "p99": 62.303, + "backpressure": 0, + "other_err": 0, + "status": "ok", + "reason": "complete" + }, + "32": { + "connections": 32, + "ops_per_sec": 149.13333333333333, + "bytes_per_sec": 2502042146.133333, + "p50": 218.239, + "p99": 248.959, + "backpressure": 0, + "other_err": 0, + "status": "ok", + "reason": "complete" + }, + "128": { + "connections": 128, + "ops_per_sec": 147.13333333333333, + "bytes_per_sec": 2468487714.133333, + "p50": 941.567, + "p99": 1376.255, + "backpressure": 0, + "other_err": 0, + "status": "ok", + "reason": "complete" + }, + "512": { + "connections": 512, + "ops_per_sec": 171.2, + "bytes_per_sec": 2872259379.2, + "p50": 3721.215, + "p99": 4644.863, + "backpressure": 0, + "other_err": 0, + "status": "ok", + "reason": "complete" + } + } + } + } +} \ No newline at end of file diff --git a/results/canonical-reads-sse/aggregate.csv b/results/canonical-reads-sse/aggregate.csv new file mode 100644 index 0000000..07a19d8 --- /dev/null +++ b/results/canonical-reads-sse/aggregate.csv @@ -0,0 +1,17 @@ +mode,stream_count,connections,ops_per_sec,bytes_per_sec,mib_per_sec,p50,p99,backpressure,other_err,status,is_peak +ursula,10,64,3204.266666666667,820292.2666666667,0.8,0.929,1.466,0,0,ok,False +ursula,10,256,12817.066666666668,3281169.066666667,3.1,1.384,1.888,0,0,ok,False +ursula,10,1024,51267.066666666666,13124369.066666666,12.5,1.56,2.347,0,0,ok,False +ursula,10,2048,102536.53333333334,26249352.533333335,25.0,1.989,3.347,0,0,ok,True +ursula,100,64,3151.8,806860.8,0.8,1.141,38.655,0,0,ok,False +ursula,100,256,12140.6,3107993.6,3.0,1.356,43.999,0,0,ok,False +ursula,100,1024,45777.13333333333,11718946.133333333,11.2,1.518,56.063,0,0,ok,False +ursula,100,2048,82798.4,21196390.4,20.2,1.661,62.399,0,0,ok,True +wal,10,64,3204.266666666667,820292.2666666667,0.8,0.475,0.705,0,0,ok,False +wal,10,256,12817.066666666668,3281169.066666667,3.1,0.676,1.222,0,0,ok,False +wal,10,1024,51268.26666666667,13124676.266666668,12.5,1.071,2.157,0,0,ok,False +wal,10,2048,102544.66666666669,26251434.666666668,25.0,1.145,1.985,0,0,ok,True +wal,100,64,3204.266666666667,820292.2666666667,0.8,0.352,0.584,0,0,ok,False +wal,100,256,12817.066666666668,3281169.066666667,3.1,0.958,1.67,0,0,ok,False +wal,100,1024,51267.73333333333,13124539.733333332,12.5,1.074,3.131,0,0,ok,False +wal,100,2048,102536.53333333334,26249352.533333335,25.0,0.734,1.958,0,0,ok,True diff --git a/results/canonical-reads-sse/aggregate.json b/results/canonical-reads-sse/aggregate.json new file mode 100644 index 0000000..a1f5a43 --- /dev/null +++ b/results/canonical-reads-sse/aggregate.json @@ -0,0 +1,226 @@ +[ + { + "mode": "ursula", + "stream_count": 10, + "connections": 64, + "ops_per_sec": 3204.266666666667, + "bytes_per_sec": 820292.2666666667, + "mib_per_sec": 0.8, + "p50": 0.929, + "p99": 1.466, + "backpressure": 0, + "other_err": 0, + "status": "ok", + "is_peak": false + }, + { + "mode": "ursula", + "stream_count": 10, + "connections": 256, + "ops_per_sec": 12817.066666666668, + "bytes_per_sec": 3281169.066666667, + "mib_per_sec": 3.1, + "p50": 1.384, + "p99": 1.888, + "backpressure": 0, + "other_err": 0, + "status": "ok", + "is_peak": false + }, + { + "mode": "ursula", + "stream_count": 10, + "connections": 1024, + "ops_per_sec": 51267.066666666666, + "bytes_per_sec": 13124369.066666666, + "mib_per_sec": 12.5, + "p50": 1.56, + "p99": 2.347, + "backpressure": 0, + "other_err": 0, + "status": "ok", + "is_peak": false + }, + { + "mode": "ursula", + "stream_count": 10, + "connections": 2048, + "ops_per_sec": 102536.53333333334, + "bytes_per_sec": 26249352.533333335, + "mib_per_sec": 25.0, + "p50": 1.989, + "p99": 3.347, + "backpressure": 0, + "other_err": 0, + "status": "ok", + "is_peak": true + }, + { + "mode": "ursula", + "stream_count": 100, + "connections": 64, + "ops_per_sec": 3151.8, + "bytes_per_sec": 806860.8, + "mib_per_sec": 0.8, + "p50": 1.141, + "p99": 38.655, + "backpressure": 0, + "other_err": 0, + "status": "ok", + "is_peak": false + }, + { + "mode": "ursula", + "stream_count": 100, + "connections": 256, + "ops_per_sec": 12140.6, + "bytes_per_sec": 3107993.6, + "mib_per_sec": 3.0, + "p50": 1.356, + "p99": 43.999, + "backpressure": 0, + "other_err": 0, + "status": "ok", + "is_peak": false + }, + { + "mode": "ursula", + "stream_count": 100, + "connections": 1024, + "ops_per_sec": 45777.13333333333, + "bytes_per_sec": 11718946.133333333, + "mib_per_sec": 11.2, + "p50": 1.518, + "p99": 56.063, + "backpressure": 0, + "other_err": 0, + "status": "ok", + "is_peak": false + }, + { + "mode": "ursula", + "stream_count": 100, + "connections": 2048, + "ops_per_sec": 82798.4, + "bytes_per_sec": 21196390.4, + "mib_per_sec": 20.2, + "p50": 1.661, + "p99": 62.399, + "backpressure": 0, + "other_err": 0, + "status": "ok", + "is_peak": true + }, + { + "mode": "wal", + "stream_count": 10, + "connections": 64, + "ops_per_sec": 3204.266666666667, + "bytes_per_sec": 820292.2666666667, + "mib_per_sec": 0.8, + "p50": 0.475, + "p99": 0.705, + "backpressure": 0, + "other_err": 0, + "status": "ok", + "is_peak": false + }, + { + "mode": "wal", + "stream_count": 10, + "connections": 256, + "ops_per_sec": 12817.066666666668, + "bytes_per_sec": 3281169.066666667, + "mib_per_sec": 3.1, + "p50": 0.676, + "p99": 1.222, + "backpressure": 0, + "other_err": 0, + "status": "ok", + "is_peak": false + }, + { + "mode": "wal", + "stream_count": 10, + "connections": 1024, + "ops_per_sec": 51268.26666666667, + "bytes_per_sec": 13124676.266666668, + "mib_per_sec": 12.5, + "p50": 1.071, + "p99": 2.157, + "backpressure": 0, + "other_err": 0, + "status": "ok", + "is_peak": false + }, + { + "mode": "wal", + "stream_count": 10, + "connections": 2048, + "ops_per_sec": 102544.66666666669, + "bytes_per_sec": 26251434.666666668, + "mib_per_sec": 25.0, + "p50": 1.145, + "p99": 1.985, + "backpressure": 0, + "other_err": 0, + "status": "ok", + "is_peak": true + }, + { + "mode": "wal", + "stream_count": 100, + "connections": 64, + "ops_per_sec": 3204.266666666667, + "bytes_per_sec": 820292.2666666667, + "mib_per_sec": 0.8, + "p50": 0.352, + "p99": 0.584, + "backpressure": 0, + "other_err": 0, + "status": "ok", + "is_peak": false + }, + { + "mode": "wal", + "stream_count": 100, + "connections": 256, + "ops_per_sec": 12817.066666666668, + "bytes_per_sec": 3281169.066666667, + "mib_per_sec": 3.1, + "p50": 0.958, + "p99": 1.67, + "backpressure": 0, + "other_err": 0, + "status": "ok", + "is_peak": false + }, + { + "mode": "wal", + "stream_count": 100, + "connections": 1024, + "ops_per_sec": 51267.73333333333, + "bytes_per_sec": 13124539.733333332, + "mib_per_sec": 12.5, + "p50": 1.074, + "p99": 3.131, + "backpressure": 0, + "other_err": 0, + "status": "ok", + "is_peak": false + }, + { + "mode": "wal", + "stream_count": 100, + "connections": 2048, + "ops_per_sec": 102536.53333333334, + "bytes_per_sec": 26249352.533333335, + "mib_per_sec": 25.0, + "p50": 0.734, + "p99": 1.958, + "backpressure": 0, + "other_err": 0, + "status": "ok", + "is_peak": true + } +] \ No newline at end of file diff --git a/results/canonical-reads-sse/report.md b/results/canonical-reads-sse/report.md new file mode 100644 index 0000000..39a1254 --- /dev/null +++ b/results/canonical-reads-sse/report.md @@ -0,0 +1,29 @@ +# canonical-reads-sse — read-scalability report + +Each cell: aggregate read throughput (MiB/s) @ p99 latency (ms). ‡ = backpressure (503/429) observed at this load. + +## wal — throughput @ p99 over stream_count × connections + +| streams | 64 | 256 | 1024 | 2048 | +|---|---|---|---|---| +| 10 | 1MiB/s@1ms | 3MiB/s@1ms | 13MiB/s@2ms | 25MiB/s@2ms | +| 100 | 1MiB/s@1ms | 3MiB/s@2ms | 13MiB/s@3ms | 25MiB/s@2ms | + +Peak read throughput per cardinality: +- streams=10: 25 MiB/s at 2048 connections +- streams=100: 25 MiB/s at 2048 connections + +## ursula — throughput @ p99 over stream_count × connections + +| streams | 64 | 256 | 1024 | 2048 | +|---|---|---|---|---| +| 10 | 1MiB/s@1ms | 3MiB/s@2ms | 13MiB/s@2ms | 25MiB/s@3ms | +| 100 | 1MiB/s@39ms | 3MiB/s@44ms | 11MiB/s@56ms | 20MiB/s@62ms | + +Peak read throughput per cardinality: +- streams=10: 25 MiB/s at 2048 connections +- streams=100: 20 MiB/s at 2048 connections + +## Findings + +_TODO: written by hand on top of the generated data._ diff --git a/results/canonical-reads-sse/ursula/cells.json b/results/canonical-reads-sse/ursula/cells.json new file mode 100644 index 0000000..9961f5b --- /dev/null +++ b/results/canonical-reads-sse/ursula/cells.json @@ -0,0 +1,106 @@ +{ + "cells": { + "10": { + "stream_count": 10, + "image_digest": "a56ce0252087", + "complete": true, + "connections": { + "64": { + "connections": 64, + "ops_per_sec": 3204.266666666667, + "bytes_per_sec": 820292.2666666667, + "p50": 0.929, + "p99": 1.466, + "backpressure": 0, + "other_err": 0, + "status": "ok", + "reason": "complete" + }, + "256": { + "connections": 256, + "ops_per_sec": 12817.066666666668, + "bytes_per_sec": 3281169.066666667, + "p50": 1.384, + "p99": 1.888, + "backpressure": 0, + "other_err": 0, + "status": "ok", + "reason": "complete" + }, + "1024": { + "connections": 1024, + "ops_per_sec": 51267.066666666666, + "bytes_per_sec": 13124369.066666666, + "p50": 1.56, + "p99": 2.347, + "backpressure": 0, + "other_err": 0, + "status": "ok", + "reason": "complete" + }, + "2048": { + "connections": 2048, + "ops_per_sec": 102536.53333333334, + "bytes_per_sec": 26249352.533333335, + "p50": 1.989, + "p99": 3.347, + "backpressure": 0, + "other_err": 0, + "status": "ok", + "reason": "complete" + } + } + }, + "100": { + "stream_count": 100, + "image_digest": "a56ce0252087", + "complete": true, + "connections": { + "64": { + "connections": 64, + "ops_per_sec": 3151.8, + "bytes_per_sec": 806860.8, + "p50": 1.141, + "p99": 38.655, + "backpressure": 0, + "other_err": 0, + "status": "ok", + "reason": "complete" + }, + "256": { + "connections": 256, + "ops_per_sec": 12140.6, + "bytes_per_sec": 3107993.6, + "p50": 1.356, + "p99": 43.999, + "backpressure": 0, + "other_err": 0, + "status": "ok", + "reason": "complete" + }, + "1024": { + "connections": 1024, + "ops_per_sec": 45777.13333333333, + "bytes_per_sec": 11718946.133333333, + "p50": 1.518, + "p99": 56.063, + "backpressure": 0, + "other_err": 0, + "status": "ok", + "reason": "complete" + }, + "2048": { + "connections": 2048, + "ops_per_sec": 82798.4, + "bytes_per_sec": 21196390.4, + "p50": 1.661, + "p99": 62.399, + "backpressure": 0, + "other_err": 0, + "status": "ok", + "reason": "complete" + } + } + } + } +} \ No newline at end of file diff --git a/results/canonical-reads-sse/wal/cells.json b/results/canonical-reads-sse/wal/cells.json new file mode 100644 index 0000000..e3cf781 --- /dev/null +++ b/results/canonical-reads-sse/wal/cells.json @@ -0,0 +1,106 @@ +{ + "cells": { + "10": { + "stream_count": 10, + "image_digest": "92f4a6387253", + "complete": true, + "connections": { + "64": { + "connections": 64, + "ops_per_sec": 3204.266666666667, + "bytes_per_sec": 820292.2666666667, + "p50": 0.475, + "p99": 0.705, + "backpressure": 0, + "other_err": 0, + "status": "ok", + "reason": "complete" + }, + "256": { + "connections": 256, + "ops_per_sec": 12817.066666666668, + "bytes_per_sec": 3281169.066666667, + "p50": 0.676, + "p99": 1.222, + "backpressure": 0, + "other_err": 0, + "status": "ok", + "reason": "complete" + }, + "1024": { + "connections": 1024, + "ops_per_sec": 51268.26666666667, + "bytes_per_sec": 13124676.266666668, + "p50": 1.071, + "p99": 2.157, + "backpressure": 0, + "other_err": 0, + "status": "ok", + "reason": "complete" + }, + "2048": { + "connections": 2048, + "ops_per_sec": 102544.66666666669, + "bytes_per_sec": 26251434.666666668, + "p50": 1.145, + "p99": 1.985, + "backpressure": 0, + "other_err": 0, + "status": "ok", + "reason": "complete" + } + } + }, + "100": { + "stream_count": 100, + "image_digest": "92f4a6387253", + "complete": true, + "connections": { + "64": { + "connections": 64, + "ops_per_sec": 3204.266666666667, + "bytes_per_sec": 820292.2666666667, + "p50": 0.352, + "p99": 0.584, + "backpressure": 0, + "other_err": 0, + "status": "ok", + "reason": "complete" + }, + "256": { + "connections": 256, + "ops_per_sec": 12817.066666666668, + "bytes_per_sec": 3281169.066666667, + "p50": 0.958, + "p99": 1.67, + "backpressure": 0, + "other_err": 0, + "status": "ok", + "reason": "complete" + }, + "1024": { + "connections": 1024, + "ops_per_sec": 51267.73333333333, + "bytes_per_sec": 13124539.733333332, + "p50": 1.074, + "p99": 3.131, + "backpressure": 0, + "other_err": 0, + "status": "ok", + "reason": "complete" + }, + "2048": { + "connections": 2048, + "ops_per_sec": 102536.53333333334, + "bytes_per_sec": 26249352.533333335, + "p50": 0.734, + "p99": 1.958, + "backpressure": 0, + "other_err": 0, + "status": "ok", + "reason": "complete" + } + } + } + } +} \ No newline at end of file diff --git a/results/canonical-write-ursula/aggregate.csv b/results/canonical-write-ursula/aggregate.csv new file mode 100644 index 0000000..90f98bc --- /dev/null +++ b/results/canonical-write-ursula/aggregate.csv @@ -0,0 +1,7 @@ +mode,stream_count,pods,throughput,p50,p99,knee_pods,knee_throughput,knee_p50,knee_p99,pod_mem_mb,pod_mem_p50_mb,saturated,status,reason +ursula-disk,100,4,4275.12,6.627,84.607,4,4299.5,6.607,84.735,1311,1133,True,ok,plateau +ursula-memory,100,4,49109.1,1.135,34.335,4,48446.875,1.142,34.943,3172,2094,True,ok,plateau +ursula-disk,1000,4,7333.75,112.063,297.471,4,7395.5,111.359,298.495,1744,1508,True,ok,plateau +ursula-memory,1000,4,54660.2,10.975,63.199,4,54418.5,10.975,62.943,2320,1746,True,ok,plateau +ursula-disk,10000,8,8207.0,1404.927,1901.567,8,8261.0,1402.879,1905.663,2610,2334,True,ok,plateau +ursula-memory,10000,8,48672.5,203.391,304.639,8,49027.5,203.007,296.447,3430,2936,True,ok,plateau diff --git a/results/canonical-write-ursula/aggregate.json b/results/canonical-write-ursula/aggregate.json new file mode 100644 index 0000000..96985c6 --- /dev/null +++ b/results/canonical-write-ursula/aggregate.json @@ -0,0 +1,188 @@ +[ + { + "mode": "ursula-disk", + "stream_count": 100, + "pods": 4, + "throughput": 4275.12, + "p50": 6.627, + "p99": 84.607, + "knee_pods": 4, + "knee_throughput": 4299.5, + "knee_p50": 6.607, + "knee_p99": 84.735, + "pod_mem_mb": 1311, + "pod_mem_p50_mb": 1133, + "saturated": true, + "status": "ok", + "reason": "plateau", + "walk": [ + [ + 4, + 4299.5, + 6.607, + 84.735 + ], + [ + 8, + 4541.5, + 6.519, + 84.415 + ] + ] + }, + { + "mode": "ursula-memory", + "stream_count": 100, + "pods": 4, + "throughput": 49109.1, + "p50": 1.135, + "p99": 34.335, + "knee_pods": 4, + "knee_throughput": 48446.875, + "knee_p50": 1.142, + "knee_p99": 34.943, + "pod_mem_mb": 3172, + "pod_mem_p50_mb": 2094, + "saturated": true, + "status": "ok", + "reason": "plateau", + "walk": [ + [ + 4, + 48446.875, + 1.142, + 34.943 + ], + [ + 8, + 51968.125, + 1.16, + 34.431 + ] + ] + }, + { + "mode": "ursula-disk", + "stream_count": 1000, + "pods": 4, + "throughput": 7333.75, + "p50": 112.063, + "p99": 297.471, + "knee_pods": 4, + "knee_throughput": 7395.5, + "knee_p50": 111.359, + "knee_p99": 298.495, + "pod_mem_mb": 1744, + "pod_mem_p50_mb": 1508, + "saturated": true, + "status": "ok", + "reason": "plateau", + "walk": [ + [ + 4, + 7395.5, + 111.359, + 298.495 + ], + [ + 8, + 7296.5, + 111.295, + 303.103 + ] + ] + }, + { + "mode": "ursula-memory", + "stream_count": 1000, + "pods": 4, + "throughput": 54660.2, + "p50": 10.975, + "p99": 63.199, + "knee_pods": 4, + "knee_throughput": 54418.5, + "knee_p50": 10.975, + "knee_p99": 62.943, + "pod_mem_mb": 2320, + "pod_mem_p50_mb": 1746, + "saturated": true, + "status": "ok", + "reason": "plateau", + "walk": [ + [ + 4, + 54418.5, + 10.975, + 62.943 + ], + [ + 8, + 54198.0, + 11.087, + 64.415 + ] + ] + }, + { + "mode": "ursula-disk", + "stream_count": 10000, + "pods": 8, + "throughput": 8207.0, + "p50": 1404.927, + "p99": 1901.567, + "knee_pods": 8, + "knee_throughput": 8261.0, + "knee_p50": 1402.879, + "knee_p99": 1905.663, + "pod_mem_mb": 2610, + "pod_mem_p50_mb": 2334, + "saturated": true, + "status": "ok", + "reason": "plateau", + "walk": [ + [ + 8, + 8261.0, + 1402.879, + 1905.663 + ], + [ + 16, + 8240.75, + 1398.783, + 2019.327 + ] + ] + }, + { + "mode": "ursula-memory", + "stream_count": 10000, + "pods": 8, + "throughput": 48672.5, + "p50": 203.391, + "p99": 304.639, + "knee_pods": 8, + "knee_throughput": 49027.5, + "knee_p50": 203.007, + "knee_p99": 296.447, + "pod_mem_mb": 3430, + "pod_mem_p50_mb": 2936, + "saturated": true, + "status": "ok", + "reason": "plateau", + "walk": [ + [ + 8, + 49027.5, + 203.007, + 296.447 + ], + [ + 16, + 48271.0, + 203.519, + 342.527 + ] + ] + } +] \ No newline at end of file diff --git a/results/canonical-write-ursula/report.md b/results/canonical-write-ursula/report.md new file mode 100644 index 0000000..0cc9e0f --- /dev/null +++ b/results/canonical-write-ursula/report.md @@ -0,0 +1,48 @@ +# canonical-write-ursula — write-throughput report + +## Throughput at saturation (ops/s) + +| streams | ursula-memory | ursula-disk | +|---|---|---| +| 100 | 49k | 4k | +| 1000 | 55k | 7k | +| 10000 | 49k | 8k | + +† = not saturated (ladder exhausted) — treat as a lower bound. + +## Pod memory at saturation — peak / p50 (MiB) + +| streams | ursula-memory | ursula-disk | +|---|---|---| +| 100 | 3172 / 2094 | 1311 / 1133 | +| 1000 | 2320 / 1746 | 1744 / 1508 | +| 10000 | 3430 / 2936 | 2610 / 2334 | + +_Pod working set = cgroup `memory.current − inactive_file` (anon + active page cache), sampled each second at the pinned rung. **peak** = high-water (catches bursts like an in-RAM Raft log filling); **p50** = median (what the server steadily holds resident). peak ≈ p50 ⇒ steadily resident; peak ≫ p50 ⇒ transient spikes._ + +## Latency (ms, p50 / p99) + +| streams | ursula-memory @≤80% load | ursula-memory @saturation | ursula-disk @≤80% load | ursula-disk @saturation | +|---|---|---|---|---| +| 100 | 1.1 / 34.9 (48k @4p) | 1.1 / 34.3 | 6.6 / 84.7 (4k @4p) | 6.6 / 84.6 | +| 1000 | 11.0 / 62.9 (54k @4p) | 11.0 / 63.2 | 111.4 / 298.5 (7k @4p) | 112.1 / 297.5 | +| 10000 | 203.0 / 296.4 (49k @8p) | 203.4 / 304.6 | 1402.9 / 1905.7 (8k @8p) | 1404.9 / 1901.6 | + +_@≤80% load = the largest ladder rung at ≤80% of peak throughput — the server's service latency with headroom. @saturation = the pinned plateau rung, where a closed-loop fleet measures its own queueing (≈ in-flight ÷ ceiling by Little's law), NOT the server's per-request cost. Compare against the unloaded single-request baseline (~1 ms for wal) before reading anything into large saturation values._ + +## Saturation walks (pods → ops/s, p50 ms) + +- **ursula-disk 100**: 4:4k@6.6ms → 8:5k@6.5ms (pinned 4, plateau) +- **ursula-memory 100**: 4:48k@1.1ms → 8:52k@1.2ms (pinned 4, plateau) +- **ursula-disk 1000**: 4:7k@111.4ms → 8:7k@111.3ms (pinned 4, plateau) +- **ursula-memory 1000**: 4:54k@11.0ms → 8:54k@11.1ms (pinned 4, plateau) +- **ursula-disk 10000**: 8:8k@1402.9ms → 16:8k@1398.8ms (pinned 8, plateau) +- **ursula-memory 10000**: 8:49k@203.0ms → 16:48k@203.5ms (pinned 8, plateau) + +## Findings + +_TODO: written by hand on top of the generated data._ + +## Caveats + +_Single-node best-case; not 3-node Raft. Throughput is a saturation ceiling per the ladder._ diff --git a/results/canonical-write-ursula/ursula-disk/cells.json b/results/canonical-write-ursula/ursula-disk/cells.json new file mode 100644 index 0000000..0c46c6c --- /dev/null +++ b/results/canonical-write-ursula/ursula-disk/cells.json @@ -0,0 +1,85 @@ +{ + "cells": { + "100": { + "stream_count": 100, + "image_digest": "bed0b89cab02", + "walk": [ + [ + 4, + 4299.5, + 6.607, + 84.735 + ], + [ + 8, + 4541.5, + 6.519, + 84.415 + ] + ], + "pinned_pods": 4, + "throughput": 4275.12, + "p50": 6.627, + "p99": 84.607, + "pod_mem_mb": 1311, + "saturated": true, + "status": "ok", + "reason": "plateau", + "pod_mem_p50_mb": 1133 + }, + "1000": { + "stream_count": 1000, + "image_digest": "bed0b89cab02", + "walk": [ + [ + 4, + 7395.5, + 111.359, + 298.495 + ], + [ + 8, + 7296.5, + 111.295, + 303.103 + ] + ], + "pinned_pods": 4, + "throughput": 7333.75, + "p50": 112.063, + "p99": 297.471, + "pod_mem_mb": 1744, + "saturated": true, + "status": "ok", + "reason": "plateau", + "pod_mem_p50_mb": 1508 + }, + "10000": { + "stream_count": 10000, + "image_digest": "bed0b89cab02", + "walk": [ + [ + 8, + 8261.0, + 1402.879, + 1905.663 + ], + [ + 16, + 8240.75, + 1398.783, + 2019.327 + ] + ], + "pinned_pods": 8, + "throughput": 8207.0, + "p50": 1404.927, + "p99": 1901.567, + "pod_mem_mb": 2610, + "saturated": true, + "status": "ok", + "reason": "plateau", + "pod_mem_p50_mb": 2334 + } + } +} \ No newline at end of file diff --git a/results/canonical-write-ursula/ursula-memory/cells.json b/results/canonical-write-ursula/ursula-memory/cells.json new file mode 100644 index 0000000..e1ecb38 --- /dev/null +++ b/results/canonical-write-ursula/ursula-memory/cells.json @@ -0,0 +1,85 @@ +{ + "cells": { + "100": { + "stream_count": 100, + "image_digest": "f71bb4708fe1", + "walk": [ + [ + 4, + 48446.875, + 1.142, + 34.943 + ], + [ + 8, + 51968.125, + 1.16, + 34.431 + ] + ], + "pinned_pods": 4, + "throughput": 49109.1, + "p50": 1.135, + "p99": 34.335, + "pod_mem_mb": 3172, + "saturated": true, + "status": "ok", + "reason": "plateau", + "pod_mem_p50_mb": 2094 + }, + "1000": { + "stream_count": 1000, + "image_digest": "f71bb4708fe1", + "walk": [ + [ + 4, + 54418.5, + 10.975, + 62.943 + ], + [ + 8, + 54198.0, + 11.087, + 64.415 + ] + ], + "pinned_pods": 4, + "throughput": 54660.2, + "p50": 10.975, + "p99": 63.199, + "pod_mem_mb": 2320, + "saturated": true, + "status": "ok", + "reason": "plateau", + "pod_mem_p50_mb": 1746 + }, + "10000": { + "stream_count": 10000, + "image_digest": "f71bb4708fe1", + "walk": [ + [ + 8, + 49027.5, + 203.007, + 296.447 + ], + [ + 16, + 48271.0, + 203.519, + 342.527 + ] + ], + "pinned_pods": 8, + "throughput": 48672.5, + "p50": 203.391, + "p99": 304.639, + "pod_mem_mb": 3430, + "saturated": true, + "status": "ok", + "reason": "plateau", + "pod_mem_p50_mb": 2936 + } + } +} \ No newline at end of file diff --git a/results/canonical-write/aggregate.csv b/results/canonical-write/aggregate.csv new file mode 100644 index 0000000..0cc48fe --- /dev/null +++ b/results/canonical-write/aggregate.csv @@ -0,0 +1,5 @@ +mode,stream_count,pods,throughput,p50,p99,knee_pods,knee_throughput,knee_p50,knee_p99,pod_mem_mb,pod_mem_p50_mb,saturated,status,reason +memory,10000,8,680429.25,,,4,655795.45,1.418,3.621,298,206,False,ok,ladder_exhausted +wal-ideal,10000,8,416986.55000000005,,,4,416986.55000000005,2.369,4.235,323,221,False,ok,ladder_exhausted +memory,100000,8,631519.5,,,4,631519.5,1.433,3.643,714,630,False,ok,ladder_exhausted +wal-ideal,100000,8,382296.4,,,4,382296.4,2.501,4.563,777,658,False,ok,ladder_exhausted diff --git a/results/canonical-write/aggregate.json b/results/canonical-write/aggregate.json new file mode 100644 index 0000000..f1139f1 --- /dev/null +++ b/results/canonical-write/aggregate.json @@ -0,0 +1,126 @@ +[ + { + "mode": "memory", + "stream_count": 10000, + "pods": 8, + "throughput": 680429.25, + "p50": null, + "p99": null, + "knee_pods": 4, + "knee_throughput": 655795.45, + "knee_p50": 1.418, + "knee_p99": 3.621, + "pod_mem_mb": 298, + "pod_mem_p50_mb": 206, + "saturated": false, + "status": "ok", + "reason": "ladder_exhausted", + "walk": [ + [ + 4, + 655795.45, + 1.418, + 3.621 + ], + [ + 8, + 680429.25, + 2.753, + 6.475 + ] + ] + }, + { + "mode": "wal-ideal", + "stream_count": 10000, + "pods": 8, + "throughput": 416986.55000000005, + "p50": null, + "p99": null, + "knee_pods": 4, + "knee_throughput": 416986.55000000005, + "knee_p50": 2.369, + "knee_p99": 4.235, + "pod_mem_mb": 323, + "pod_mem_p50_mb": 221, + "saturated": false, + "status": "ok", + "reason": "ladder_exhausted", + "walk": [ + [ + 4, + 416986.55000000005, + 2.369, + 4.235 + ], + [ + 8, + 399101.1500000001, + 4.999, + 8.231 + ] + ] + }, + { + "mode": "memory", + "stream_count": 100000, + "pods": 8, + "throughput": 631519.5, + "p50": null, + "p99": null, + "knee_pods": 4, + "knee_throughput": 631519.5, + "knee_p50": 1.433, + "knee_p99": 3.643, + "pod_mem_mb": 714, + "pod_mem_p50_mb": 630, + "saturated": false, + "status": "ok", + "reason": "ladder_exhausted", + "walk": [ + [ + 4, + 631519.5, + 1.433, + 3.643 + ], + [ + 8, + 621656.3, + 2.865, + 7.031 + ] + ] + }, + { + "mode": "wal-ideal", + "stream_count": 100000, + "pods": 8, + "throughput": 382296.4, + "p50": null, + "p99": null, + "knee_pods": 4, + "knee_throughput": 382296.4, + "knee_p50": 2.501, + "knee_p99": 4.563, + "pod_mem_mb": 777, + "pod_mem_p50_mb": 658, + "saturated": false, + "status": "ok", + "reason": "ladder_exhausted", + "walk": [ + [ + 4, + 382296.4, + 2.501, + 4.563 + ], + [ + 8, + 364593.49999999994, + 5.307, + 8.647 + ] + ] + } +] \ No newline at end of file diff --git a/results/canonical-write/memory/cells.json b/results/canonical-write/memory/cells.json new file mode 100644 index 0000000..84f128f --- /dev/null +++ b/results/canonical-write/memory/cells.json @@ -0,0 +1,58 @@ +{ + "cells": { + "10000": { + "stream_count": 10000, + "image_digest": "53835b8ca25c", + "walk": [ + [ + 4, + 655795.45, + 1.418, + 3.621 + ], + [ + 8, + 680429.25, + 2.753, + 6.475 + ] + ], + "pinned_pods": 8, + "throughput": 680429.25, + "p50": null, + "p99": null, + "pod_mem_mb": 298, + "saturated": false, + "status": "ok", + "reason": "ladder_exhausted", + "pod_mem_p50_mb": 206 + }, + "100000": { + "stream_count": 100000, + "image_digest": "53835b8ca25c", + "walk": [ + [ + 4, + 631519.5, + 1.433, + 3.643 + ], + [ + 8, + 621656.3, + 2.865, + 7.031 + ] + ], + "pinned_pods": 8, + "throughput": 621656.3, + "p50": null, + "p99": null, + "pod_mem_mb": 714, + "saturated": false, + "status": "ok", + "reason": "ladder_exhausted", + "pod_mem_p50_mb": 630 + } + } +} \ No newline at end of file diff --git a/results/canonical-write/report.md b/results/canonical-write/report.md new file mode 100644 index 0000000..ea1f3f8 --- /dev/null +++ b/results/canonical-write/report.md @@ -0,0 +1,43 @@ +# canonical-write — write-throughput report + +## Throughput at saturation (ops/s) + +| streams | wal-ideal | memory | +|---|---|---| +| 10000 | 417k† | 680k† | +| 100000 | 382k† | 632k† | + +† = not saturated (ladder exhausted) — treat as a lower bound. + +## Pod memory at saturation — peak / p50 (MiB) + +| streams | wal-ideal | memory | +|---|---|---| +| 10000 | 323 / 221 | 298 / 206 | +| 100000 | 777 / 658 | 714 / 630 | + +_Pod working set = cgroup `memory.current − inactive_file` (anon + active page cache), sampled each second at the pinned rung. **peak** = high-water (catches bursts like an in-RAM Raft log filling); **p50** = median (what the server steadily holds resident). peak ≈ p50 ⇒ steadily resident; peak ≫ p50 ⇒ transient spikes._ + +## Latency (ms, p50 / p99) + +| streams | wal-ideal @≤80% load | wal-ideal @saturation | memory @≤80% load | memory @saturation | +|---|---|---|---|---| +| 10000 | 2.4 / 4.2 (417k @4p) | — | 1.4 / 3.6 (656k @4p) | — | +| 100000 | 2.5 / 4.6 (382k @4p) | — | 1.4 / 3.6 (632k @4p) | — | + +_@≤80% load = the largest ladder rung at ≤80% of peak throughput — the server's service latency with headroom. @saturation = the pinned plateau rung, where a closed-loop fleet measures its own queueing (≈ in-flight ÷ ceiling by Little's law), NOT the server's per-request cost. Compare against the unloaded single-request baseline (~1 ms for wal) before reading anything into large saturation values._ + +## Saturation walks (pods → ops/s, p50 ms) + +- **memory 10000**: 4:656k@1.4ms → 8:680k@2.8ms (pinned 8, ladder_exhausted) +- **wal-ideal 10000**: 4:417k@2.4ms → 8:399k@5.0ms (pinned 8, ladder_exhausted) +- **memory 100000**: 4:632k@1.4ms → 8:622k@2.9ms (pinned 8, ladder_exhausted) +- **wal-ideal 100000**: 4:382k@2.5ms → 8:365k@5.3ms (pinned 8, ladder_exhausted) + +## Findings + +_TODO: written by hand on top of the generated data._ + +## Caveats + +_Single-node best-case; not 3-node Raft. Throughput is a saturation ceiling per the ladder._ diff --git a/results/canonical-write/wal-ideal/cells.json b/results/canonical-write/wal-ideal/cells.json new file mode 100644 index 0000000..b0ccc66 --- /dev/null +++ b/results/canonical-write/wal-ideal/cells.json @@ -0,0 +1,58 @@ +{ + "cells": { + "10000": { + "stream_count": 10000, + "image_digest": "09395f644017", + "walk": [ + [ + 4, + 416986.55000000005, + 2.369, + 4.235 + ], + [ + 8, + 399101.1500000001, + 4.999, + 8.231 + ] + ], + "pinned_pods": 8, + "throughput": 399101.1500000001, + "p50": null, + "p99": null, + "pod_mem_mb": 323, + "saturated": false, + "status": "ok", + "reason": "ladder_exhausted", + "pod_mem_p50_mb": 221 + }, + "100000": { + "stream_count": 100000, + "image_digest": "09395f644017", + "walk": [ + [ + 4, + 382296.4, + 2.501, + 4.563 + ], + [ + 8, + 364593.49999999994, + 5.307, + 8.647 + ] + ], + "pinned_pods": 8, + "throughput": 364593.49999999994, + "p50": null, + "p99": null, + "pod_mem_mb": 777, + "saturated": false, + "status": "ok", + "reason": "ladder_exhausted", + "pod_mem_p50_mb": 658 + } + } +} \ No newline at end of file