From 156eb7789c4be5a5c0432e9ef201c87c9cc6fc7e Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 15 Aug 2026 00:09:37 +0000 Subject: [PATCH 1/5] perf(bench): add a repeatable benchmark suite MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds `bench/`: a performance suite covering the paths behind the CPU and memory complaints, plus a regression gate CI can run. Suites: store (per backend, and how write cost scales with workspace size), render (shiki/markdown-it/diff SSR and output size), api (route latency and response bytes), events (SSE fan-out, long-poll wakeups), process (startup, idle RSS, and which imports the memory belongs to), and viewer (browser CPU, layout work, heap, DOM size via CDP). The design problem is deciding when a number moving is news, so each metric carries a kind: bytes/count are deterministic and gated near-exactly, memory and time get generous ratios plus an absolute floor. Timings are normalized by a per-run machine index so a baseline recorded on one machine still means something on another. CI runs with `--gate deterministic`, failing only on machine-independent metrics — timings are still measured and printed, but a flaky perf gate is one people learn to ignore. Fixtures are pure functions of a seed, so a difference in the numbers is a difference in the code. Write benchmarks use fixed iteration counts because each write changes the thing being measured. Tooling only; no runtime behavior changes. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01MGMuudU8anVrxB7sjY9hpt --- .changeset/perf-benchmark-suite.md | 5 + .github/workflows/ci.yml | 22 + .gitignore | 5 + AGENTS.md | 15 + bench/README.md | 177 +++ bench/baseline.json | 1827 ++++++++++++++++++++++++++++ bench/compare.ts | 179 +++ bench/fixtures.ts | 312 +++++ bench/harness.ts | 310 +++++ bench/report.ts | 138 +++ bench/run.ts | 289 +++++ bench/suites/api.bench.ts | 244 ++++ bench/suites/events.bench.ts | 205 ++++ bench/suites/process.bench.ts | 271 +++++ bench/suites/render.bench.ts | 206 ++++ bench/suites/store.bench.ts | 267 ++++ bench/suites/viewer.bench.ts | 358 ++++++ package.json | 4 + test/bench.test.ts | 179 +++ tsconfig.json | 2 +- 20 files changed, 5014 insertions(+), 1 deletion(-) create mode 100644 .changeset/perf-benchmark-suite.md create mode 100644 bench/README.md create mode 100644 bench/baseline.json create mode 100644 bench/compare.ts create mode 100644 bench/fixtures.ts create mode 100644 bench/harness.ts create mode 100644 bench/report.ts create mode 100644 bench/run.ts create mode 100644 bench/suites/api.bench.ts create mode 100644 bench/suites/events.bench.ts create mode 100644 bench/suites/process.bench.ts create mode 100644 bench/suites/render.bench.ts create mode 100644 bench/suites/store.bench.ts create mode 100644 bench/suites/viewer.bench.ts create mode 100644 test/bench.test.ts diff --git a/.changeset/perf-benchmark-suite.md b/.changeset/perf-benchmark-suite.md new file mode 100644 index 00000000..0c6893a8 --- /dev/null +++ b/.changeset/perf-benchmark-suite.md @@ -0,0 +1,5 @@ +--- +--- + +Add a repeatable performance benchmark suite (`bench/`). Tooling only — no +runtime behavior changes. diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 65742e72..3b410190 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -56,6 +56,28 @@ jobs: sideshow demo | grep -q "Seeded 2 demo sessions" curl -sf localhost:8228/guide > /dev/null + # Performance regression gate. Runs the in-process benchmark suites and compares + # against the committed baseline (bench/baseline.json). + # + # `--gate deterministic` means only byte counts and operation counts can fail + # the job. Those are pure functions of our code, so they mean the same thing on + # a shared runner as on a laptop. Timings and memory are still measured, still + # compared, and still printed — a regression shows up in the log — but they + # can't fail the build, because a flaky perf gate is one people learn to re-run + # and then stop reading. See bench/README.md. + bench: + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@v7 + - uses: actions/setup-node@v6 + with: + node-version: 22 + cache: npm + - run: npm install -g npm@11.17.0 + - run: npm ci --ignore-scripts + - run: npm run bench:check -- --gate deterministic + e2e: runs-on: ubuntu-latest steps: diff --git a/.gitignore b/.gitignore index baa81d72..a725782a 100644 --- a/.gitignore +++ b/.gitignore @@ -11,3 +11,8 @@ test-results/ playwright-report/ coverage/ .video-work/ + +# Local-only benchmark baseline for `npm run bench:all`. The browser and +# process suites depend on the machine and the installed Chromium, so this one +# is recorded per-machine rather than committed (bench/baseline.json is). +bench/baseline-all.json diff --git a/AGENTS.md b/AGENTS.md index c18e0951..f16f361d 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -189,12 +189,27 @@ npm run format:check # oxfmt npm run security:audit npm run test:e2e # Playwright, chromium + webkit (separate CI job); # builds the viewer first via e2e/globalSetup.ts +npm run bench # performance suite (separate CI job); bench:check gates it ``` The first seven must pass before committing; e2e should pass before merge for viewer/rendering changes. CI also gates PRs on changeset status and smoke-tests the packed CLI. Pre-commit formats staged files (`npm run prepare` after a fresh clone). +Performance (`bench/`, see `bench/README.md`): + +- `npm run bench` runs the in-process suites (store, render, api, events); + `npm run bench:all` adds the ones that spawn processes or a browser. +- `npm run bench:check` compares against the committed `bench/baseline.json` and + exits non-zero on regression. CI runs it with `--gate deterministic`, so only + byte/count metrics can fail the job — those are machine-independent, while + timings on a shared runner are not (they're still measured and printed). +- Changed a hot path deliberately? Re-record with `npm run bench:baseline` and + say so in the PR — the baseline is committed, so the trade is visible in the diff. +- Prefer adding a deterministic metric (bytes, counts) over a timing where one + exists: it gates reliably, and payload size is itself a CPU/memory cost for the + viewer. + Testing notes: - Coverage is deliberately split by runtime. c8 uses `--all` for every shipped diff --git a/bench/README.md b/bench/README.md new file mode 100644 index 00000000..c1ea24a1 --- /dev/null +++ b/bench/README.md @@ -0,0 +1,177 @@ +# Benchmarks + +A repeatable performance suite for sideshow: what costs CPU, what costs memory, +and whether a change made either worse. + +```sh +npm run bench # the fast in-process suites (~1 min) +npm run bench:all # everything, including child processes + browser +npm run bench:check # run and fail on regression vs the committed baseline +npm run bench:baseline # re-record the baseline +``` + +## What it measures + +| Suite | Covers | Default | +| --------- | -------------------------------------------------------------------------- | ------- | +| `store` | Read/write paths per backend; how write cost scales with workspace size | yes | +| `render` | Server-side surface rendering (shiki, markdown-it, diff SSR) + output size | yes | +| `api` | HTTP routes end to end: latency and response bytes | yes | +| `events` | SSE fan-out, event-bus dispatch, comment long-poll wakeups | yes | +| `process` | Process startup, idle RSS, and which imports the memory belongs to | `--all` | +| `viewer` | Browser-side CPU, layout work, heap, and DOM size (Chromium via CDP) | `--all` | + +`process` and `viewer` are excluded from the default run because they spawn real +processes and a browser. Run them directly when you need them: + +```sh +node --expose-gc bench/run.ts process viewer +``` + +The `viewer` suite needs a built viewer (`npm run build:viewer`). If your +Chromium doesn't match the pinned Playwright revision, point at one: + +```sh +SIDESHOW_BENCH_CHROMIUM=/path/to/chromium npm run bench:all +``` + +Both skip with an explanation rather than failing the run when their +prerequisites are missing. + +## Reading the output + +``` + metric value rate p95 detail + markdown/small 3.81 ms 262/s 5.63 ms 1760 B source + markdown/small document 11.0 KB +``` + +`value` is the comparable number — the **median** per-op time, or the raw +byte/count/memory figure. Medians, not means, so one GC pause doesn't move a +number people compare across commits. + +## Metric kinds, and how much to trust them + +The suite's real job isn't measuring — it's deciding when a number moving is +news. Get that wrong strictly and CI is permanently red for reasons nobody can +fix; get it wrong loosely and the suite green-lights the regression it exists to +catch. So each metric carries a kind, and thresholds are set per kind: + +| Kind | Stability | Gate | +| -------- | ----------------------------------- | -------------------- | +| `bytes` | Deterministic — same on any machine | 1.02× (+256 B floor) | +| `count` | Deterministic | 1.02× | +| `memory` | Semi-stable; GC timing moves it | 1.25× (+1 MB floor) | +| `time` | Noisy; shared runners vary a lot | 1.4× (+0.5 ms floor) | + +A metric only fails when it exceeds **both** the ratio and the absolute floor, +so a 40% regression on a 0.1 ms operation doesn't fail the build. Individual +metrics can override the ratio (`tolerance`) — prefer that over loosening the +global default for everyone. + +Deterministic metrics are the ones worth trusting most. `bytes` in particular +catches the class of regression that timing can't see: shipping more data to the +browser costs the user CPU and memory even when the server got no slower. + +## Cross-machine comparison + +Every run measures a fixed synthetic workload and records the result as a +machine index. `--check` scales the baseline's **timings** by the ratio of the +two indices, so a baseline recorded on a laptop still means something on a +slower CI runner. + +This is a coarse correction. It does not account for a different Node major, a +different CPU architecture's cache behavior, or a noisy neighbor on a shared +runner. Treat a cross-machine timing regression as a reason to investigate, not +as proof — and if a machine is going to police the baseline, record the baseline +on that machine. `bytes` and `count` are never scaled; they don't need it. + +## Methodology notes + +- **Iteration counts auto-scale** to a time budget rather than being pinned, so + fast and slow machines both collect enough samples. +- **Writes use fixed iteration counts.** A write grows the store, and on the JSON + store the next write then costs more — an auto-scaled loop would run a + different workload on every machine. +- **Memory needs `--expose-gc`** (the npm scripts pass it). Without it the runner + warns, and the numbers include uncollected garbage. +- **Fixtures are deterministic**: every input is a pure function of a seed, so a + difference in the numbers is a difference in the code. +- **Cache misses are forced with a unique cache key**, not by rotating over the + three real themes — after three iterations those are all cached and the rest of + the run would silently measure hits. + +## What CI gates on + +CI runs `npm run bench:check -- --gate deterministic`. In that mode only `bytes` +and `count` metrics can fail the job. Timings and memory are still measured, +compared, and printed — a regression is visible in the log and called out in +yellow — but they can't turn the build red. + +That's deliberate. A byte count is a pure function of our code and means the same +thing on a shared runner as on a laptop. A timing on a shared runner does not. A +perf gate that fails randomly is one people learn to re-run and then stop +reading, which is worse protection than no gate at all. + +Locally, `npm run bench:check` gates everything (`--gate all`, the default), +which is the right setting on a machine that isn't shared. + +## Baselines + +Two baselines, one per scope, because the default suites and `--all` measure +different metric sets: + +| Command | Baseline | Committed? | +| ------------------- | ------------------------- | -------------------------------- | +| `npm run bench` | `bench/baseline.json` | yes — this is what CI gates | +| `npm run bench:all` | `bench/baseline-all.json` | no — gitignored, record your own | + +`baseline-all.json` stays local because the browser and process suites depend on +the machine and the installed Chromium; a committed copy would mostly measure +whoever recorded it. Record one with `npm run bench:all -- --baseline`. + +Recording from a subset (`bench store --baseline`, or with `--filter`) is +refused: it would silently drop every metric it didn't run, leaving a baseline +that had quietly stopped policing most of the suite. Use `--save ` for a +scratch run instead. + +## Regressions + +`npm run bench:check` prints a comparison table and exits non-zero on regression: + +``` +store + metric baseline current change + json-file/createPost 7.88 ms 12.4 ms +57.4% REGRESSED +``` + +If the change is **intended** (you knowingly traded speed for something), +re-record with `npm run bench:baseline` and say so in the PR. The baseline is +committed, so the diff makes the trade visible to reviewers instead of hiding it. + +Deleting a benchmark reports as `missing` rather than failing — removing one is a +normal deliberate act, and failing on it would make every intentional removal +look like a regression. + +## Adding a benchmark + +Add to an existing suite in `bench/suites/`, or create one and register it in +`bench/run.ts`: + +```ts +export const mySuite: Suite = { + name: "mine", + description: "…", + async run(ctx) { + await ctx.time("thing I care about", () => doTheThing(), { note: "at N items" }); + ctx.add(bytes("mine", "payload size", Buffer.byteLength(payload))); + }, +}; +``` + +Two rules worth following: + +1. **Prefer a deterministic metric where one exists.** A byte count or an + operation count gates far more reliably than a timing. +2. **Benchmark the shape users actually hit.** A microbenchmark of a function + nobody calls in a loop measures nothing anyone will feel. diff --git a/bench/baseline.json b/bench/baseline.json new file mode 100644 index 00000000..5e320d32 --- /dev/null +++ b/bench/baseline.json @@ -0,0 +1,1827 @@ +{ + "format": 1, + "recordedAt": "2026-08-15T00:04:42.691Z", + "git": { + "commit": "9b66556", + "branch": "claude/sideshow-perf-benchmarks-9zhrho" + }, + "machine": { + "platform": "linux", + "arch": "x64", + "cpus": 4, + "cpuModel": "Intel(R) Xeon(R) Processor @ 2.80GHz", + "nodeVersion": "v22.22.2", + "totalMemory": 16856068096, + "index": 8891.705493781223 + }, + "results": [ + { + "suite": "store", + "name": "sqlite-memory/getPost", + "kind": "time", + "unit": "ms/op", + "value": 0.034613785714285375, + "stats": { + "iterations": 10010, + "samples": 286, + "min": 0.031170228571428586, + "median": 0.034613785714285375, + "p95": 0.049504478571429894, + "max": 0.19214122857142685, + "opsPerSec": 28890.223342062593 + }, + "note": "SqlStore(:memory:), 144 posts / 96 comments" + }, + { + "suite": "store", + "name": "sqlite-memory/listPosts(session)", + "kind": "time", + "unit": "ms/op", + "value": 0.39970083333332695, + "stats": { + "iterations": 960, + "samples": 320, + "min": 0.3678070000000086, + "median": 0.39970083333332695, + "p95": 0.507585016666667, + "max": 0.7785416666667212, + "opsPerSec": 2501.871191161763 + }, + "note": "SqlStore(:memory:), 12 posts in session" + }, + { + "suite": "store", + "name": "sqlite-memory/listPosts(all)", + "kind": "time", + "unit": "ms/op", + "value": 4.731358499999942, + "stats": { + "iterations": 76, + "samples": 76, + "min": 4.4434040000001005, + "median": 4.731358499999942, + "p95": 7.196898749999832, + "max": 11.945956999999908, + "opsPerSec": 211.35578713809412 + }, + "note": "SqlStore(:memory:), 144 posts / 96 comments" + }, + { + "suite": "store", + "name": "sqlite-memory/listRecentPosts(20)", + "kind": "time", + "unit": "ms/op", + "value": 0.6540833333333467, + "stats": { + "iterations": 573, + "samples": 191, + "min": 0.5992056666667244, + "median": 0.6540833333333467, + "p95": 0.9849848333333284, + "max": 1.449440333333314, + "opsPerSec": 1528.857179258473 + }, + "note": "SqlStore(:memory:), 144 posts / 96 comments" + }, + { + "suite": "store", + "name": "sqlite-memory/listComments(session)", + "kind": "time", + "unit": "ms/op", + "value": 0.04182148571428245, + "stats": { + "iterations": 8050, + "samples": 230, + "min": 0.038746999999992635, + "median": 0.04182148571428245, + "p95": 0.076123408571425, + "max": 0.3384484000000027, + "opsPerSec": 23911.154348552715 + }, + "note": "SqlStore(:memory:), 144 posts / 96 comments" + }, + { + "suite": "store", + "name": "sqlite-memory/listSessions", + "kind": "time", + "unit": "ms/op", + "value": 0.04424722222222499, + "stats": { + "iterations": 8028, + "samples": 223, + "min": 0.04083891666665648, + "median": 0.04424722222222499, + "p95": 0.05344691666666292, + "max": 0.2576869444444456, + "opsPerSec": 22600.288781466348 + }, + "note": "SqlStore(:memory:), 12 sessions" + }, + { + "suite": "store", + "name": "sqlite-memory/countPostsBySession", + "kind": "time", + "unit": "ms/op", + "value": 0.03504255102040811, + "stats": { + "iterations": 10192, + "samples": 208, + "min": 0.032274857142863544, + "median": 0.03504255102040811, + "p95": 0.043985715306119284, + "max": 0.2250860816326464, + "opsPerSec": 28536.735222775853 + }, + "note": "SqlStore(:memory:), 144 posts / 96 comments" + }, + { + "suite": "store", + "name": "sqlite-memory/isAssetReferenced(miss)", + "kind": "time", + "unit": "ms/op", + "value": 0.0001111151315789763, + "stats": { + "iterations": 3376528, + "samples": 383, + "min": 0.00008978221415615624, + "median": 0.0001111151315789763, + "p95": 0.00016115510435577786, + "max": 0.00020321109346649132, + "opsPerSec": 8999674.353886168 + }, + "note": "SqlStore(:memory:), 144 posts / 96 comments" + }, + { + "suite": "store", + "name": "sqlite-memory/createPost", + "kind": "time", + "unit": "ms/op", + "value": 0.11053850000007515, + "stats": { + "iterations": 40, + "samples": 40, + "min": 0.07198599999992439, + "median": 0.11053850000007515, + "p95": 0.18154740000027214, + "max": 0.463502000000517, + "opsPerSec": 9046.621765261156 + }, + "note": "SqlStore(:memory:), 144 posts / 96 comments" + }, + { + "suite": "store", + "name": "sqlite-memory/updatePost", + "kind": "time", + "unit": "ms/op", + "value": 0.36556849999988117, + "stats": { + "iterations": 40, + "samples": 40, + "min": 0.20033799999964685, + "median": 0.36556849999988117, + "p95": 0.49073974999982956, + "max": 1.696125999999822, + "opsPerSec": 2735.465446285238 + }, + "note": "SqlStore(:memory:), history at cap" + }, + { + "suite": "store", + "name": "sqlite-memory/createComment", + "kind": "time", + "unit": "ms/op", + "value": 0.1557794999998805, + "stats": { + "iterations": 40, + "samples": 40, + "min": 0.09776400000009744, + "median": 0.1557794999998805, + "p95": 0.23081220000026403, + "max": 0.24728800000048068, + "opsPerSec": 6419.329886158109 + }, + "note": "SqlStore(:memory:), 144 posts / 96 comments" + }, + { + "suite": "store", + "name": "sqlite-file/getPost", + "kind": "time", + "unit": "ms/op", + "value": 0.03840621951220361, + "stats": { + "iterations": 9225, + "samples": 225, + "min": 0.03548236585365663, + "median": 0.03840621951220361, + "p95": 0.06737120000000607, + "max": 0.14738526829267623, + "opsPerSec": 26037.449473053424 + }, + "note": "SqlStore(file), 144 posts / 96 comments" + }, + { + "suite": "store", + "name": "sqlite-file/listPosts(session)", + "kind": "time", + "unit": "ms/op", + "value": 0.42086349999999584, + "stats": { + "iterations": 916, + "samples": 229, + "min": 0.39616175000014664, + "median": 0.42086349999999584, + "p95": 0.6201491500000884, + "max": 0.729702750000115, + "opsPerSec": 2376.067299730221 + }, + "note": "SqlStore(file), 12 posts in session" + }, + { + "suite": "store", + "name": "sqlite-file/listPosts(all)", + "kind": "time", + "unit": "ms/op", + "value": 5.0059510000000955, + "stats": { + "iterations": 73, + "samples": 73, + "min": 4.67334000000028, + "median": 5.0059510000000955, + "p95": 7.0836891999999345, + "max": 7.563390000000254, + "opsPerSec": 199.7622429784033 + }, + "note": "SqlStore(file), 144 posts / 96 comments" + }, + { + "suite": "store", + "name": "sqlite-file/listRecentPosts(20)", + "kind": "time", + "unit": "ms/op", + "value": 0.7438904999999068, + "stats": { + "iterations": 510, + "samples": 255, + "min": 0.6827775000001566, + "median": 0.7438904999999068, + "p95": 1.142371400000138, + "max": 1.3079160000002048, + "opsPerSec": 1344.283869736373 + }, + "note": "SqlStore(file), 144 posts / 96 comments" + }, + { + "suite": "store", + "name": "sqlite-file/listComments(session)", + "kind": "time", + "unit": "ms/op", + "value": 0.04919448611110511, + "stats": { + "iterations": 6984, + "samples": 194, + "min": 0.042023472222227715, + "median": 0.04919448611110511, + "p95": 0.07824290694444218, + "max": 0.4209769444444444, + "opsPerSec": 20327.4813714186 + }, + "note": "SqlStore(file), 144 posts / 96 comments" + }, + { + "suite": "store", + "name": "sqlite-file/listSessions", + "kind": "time", + "unit": "ms/op", + "value": 0.04679093939395181, + "stats": { + "iterations": 7425, + "samples": 225, + "min": 0.04307184848485948, + "median": 0.04679093939395181, + "p95": 0.0594284787878981, + "max": 0.3468611818181781, + "opsPerSec": 21371.658978260646 + }, + "note": "SqlStore(file), 12 sessions" + }, + { + "suite": "store", + "name": "sqlite-file/countPostsBySession", + "kind": "time", + "unit": "ms/op", + "value": 0.038225222222222834, + "stats": { + "iterations": 9540, + "samples": 212, + "min": 0.034087999999994484, + "median": 0.038225222222222834, + "p95": 0.04912179999999957, + "max": 0.2531072444444362, + "opsPerSec": 26160.73738398398 + }, + "note": "SqlStore(file), 144 posts / 96 comments" + }, + { + "suite": "store", + "name": "sqlite-file/isAssetReferenced(miss)", + "kind": "time", + "unit": "ms/op", + "value": 0.00011079669664476878, + "stats": { + "iterations": 3411380, + "samples": 590, + "min": 0.00009790591490834072, + "median": 0.00011079669664476878, + "p95": 0.00016421719128334054, + "max": 0.00034833137322733436, + "opsPerSec": 9025539.84263767 + }, + "note": "SqlStore(file), 144 posts / 96 comments" + }, + { + "suite": "store", + "name": "sqlite-file/createPost", + "kind": "time", + "unit": "ms/op", + "value": 0.12265800000022864, + "stats": { + "iterations": 40, + "samples": 40, + "min": 0.10285499999918102, + "median": 0.12265800000022864, + "p95": 0.19955445000050517, + "max": 0.3980699999992794, + "opsPerSec": 8152.749922533679 + }, + "note": "SqlStore(file), 144 posts / 96 comments" + }, + { + "suite": "store", + "name": "sqlite-file/updatePost", + "kind": "time", + "unit": "ms/op", + "value": 0.4303924999994706, + "stats": { + "iterations": 40, + "samples": 40, + "min": 0.21944100000109756, + "median": 0.4303924999994706, + "p95": 0.4923694999999497, + "max": 1.8614739999993617, + "opsPerSec": 2323.460562164141 + }, + "note": "SqlStore(file), history at cap" + }, + { + "suite": "store", + "name": "sqlite-file/createComment", + "kind": "time", + "unit": "ms/op", + "value": 0.11581099999966682, + "stats": { + "iterations": 40, + "samples": 40, + "min": 0.09817599999951199, + "median": 0.11581099999966682, + "p95": 0.22707395000079483, + "max": 10.421438999999737, + "opsPerSec": 8634.75835631224 + }, + "note": "SqlStore(file), 144 posts / 96 comments" + }, + { + "suite": "store", + "name": "sqlite-file/cold open + first read", + "kind": "time", + "unit": "ms/op", + "value": 0.7581345000007786, + "stats": { + "iterations": 350, + "samples": 175, + "min": 0.6985580000000482, + "median": 0.7581345000007786, + "p95": 0.9937225499999838, + "max": 4.206881000000067, + "opsPerSec": 1319.0271647035888 + }, + "note": "SqlStore(file), 144 posts / 96 comments" + }, + { + "suite": "store", + "name": "json-file/getPost", + "kind": "time", + "unit": "ms/op", + "value": 0.016411252747255297, + "stats": { + "iterations": 23296, + "samples": 256, + "min": 0.015309912087904306, + "median": 0.016411252747255297, + "p95": 0.02314657692307434, + "max": 0.027903538461532162, + "opsPerSec": 60933.80044782049 + }, + "note": "JsonFileStore, 144 posts / 96 comments" + }, + { + "suite": "store", + "name": "json-file/listPosts(session)", + "kind": "time", + "unit": "ms/op", + "value": 0.5045552500000667, + "stats": { + "iterations": 764, + "samples": 191, + "min": 0.47796924999965995, + "median": 0.5045552500000667, + "p95": 0.667314124999848, + "max": 0.7793567499998062, + "opsPerSec": 1981.9435037091928 + }, + "note": "JsonFileStore, 12 posts in session" + }, + { + "suite": "store", + "name": "json-file/listPosts(all)", + "kind": "time", + "unit": "ms/op", + "value": 6.3928975000007995, + "stats": { + "iterations": 58, + "samples": 58, + "min": 5.803273999999874, + "median": 6.3928975000007995, + "p95": 8.850587550001272, + "max": 10.551391000000876, + "opsPerSec": 156.42359352701573 + }, + "note": "JsonFileStore, 144 posts / 96 comments" + }, + { + "suite": "store", + "name": "json-file/listRecentPosts(20)", + "kind": "time", + "unit": "ms/op", + "value": 0.9378512500002216, + "stats": { + "iterations": 416, + "samples": 208, + "min": 0.8688295000001744, + "median": 0.9378512500002216, + "p95": 1.236448900000323, + "max": 2.0988930000003165, + "opsPerSec": 1066.2671718993429 + }, + "note": "JsonFileStore, 144 posts / 96 comments" + }, + { + "suite": "store", + "name": "json-file/listComments(session)", + "kind": "time", + "unit": "ms/op", + "value": 0.023168696969702127, + "stats": { + "iterations": 16500, + "samples": 250, + "min": 0.022022590909066574, + "median": 0.023168696969702127, + "p95": 0.027103380303020504, + "max": 0.09133016666667325, + "opsPerSec": 43161.684979854814 + }, + "note": "JsonFileStore, 144 posts / 96 comments" + }, + { + "suite": "store", + "name": "json-file/listSessions", + "kind": "time", + "unit": "ms/op", + "value": 0.03197912244898789, + "stats": { + "iterations": 12103, + "samples": 247, + "min": 0.030096551020376085, + "median": 0.03197912244898789, + "p95": 0.04366843469388001, + "max": 0.05813248979590141, + "opsPerSec": 31270.401543856286 + }, + "note": "JsonFileStore, 12 sessions" + }, + { + "suite": "store", + "name": "json-file/countPostsBySession", + "kind": "time", + "unit": "ms/op", + "value": 0.004537276422761499, + "stats": { + "iterations": 82902, + "samples": 337, + "min": 0.004340597560975419, + "median": 0.004537276422761499, + "p95": 0.006713969918694683, + "max": 0.008803455284556193, + "opsPerSec": 220396.5345781986 + }, + "note": "JsonFileStore, 144 posts / 96 comments" + }, + { + "suite": "store", + "name": "json-file/isAssetReferenced(miss)", + "kind": "time", + "unit": "ms/op", + "value": 0.000268329577464639, + "stats": { + "iterations": 1437750, + "samples": 225, + "min": 0.00022767793427209453, + "median": 0.000268329577464639, + "p95": 0.0003215271674490283, + "max": 0.0003963466353679009, + "opsPerSec": 3726760.238094818 + }, + "note": "JsonFileStore, 144 posts / 96 comments" + }, + { + "suite": "store", + "name": "json-file/createPost", + "kind": "time", + "unit": "ms/op", + "value": 8.302538000000823, + "stats": { + "iterations": 40, + "samples": 40, + "min": 7.216768000000229, + "median": 8.302538000000823, + "p95": 9.50251315000032, + "max": 11.17880900000091, + "opsPerSec": 120.44509763157976 + }, + "note": "JsonFileStore, 144 posts / 96 comments" + }, + { + "suite": "store", + "name": "json-file/updatePost", + "kind": "time", + "unit": "ms/op", + "value": 9.825980499998877, + "stats": { + "iterations": 40, + "samples": 40, + "min": 7.951912000000448, + "median": 9.825980499998877, + "p95": 13.580894749999425, + "max": 14.73130300000048, + "opsPerSec": 101.77101409880818 + }, + "note": "JsonFileStore, history at cap" + }, + { + "suite": "store", + "name": "json-file/createComment", + "kind": "time", + "unit": "ms/op", + "value": 8.732371999999486, + "stats": { + "iterations": 40, + "samples": 40, + "min": 7.88097899999957, + "median": 8.732371999999486, + "p95": 10.820051250000688, + "max": 11.296034999999392, + "opsPerSec": 114.51642234206912 + }, + "note": "JsonFileStore, 144 posts / 96 comments" + }, + { + "suite": "store", + "name": "json-file/cold open + first read", + "kind": "time", + "unit": "ms/op", + "value": 9.031462999999349, + "stats": { + "iterations": 27, + "samples": 27, + "min": 8.409542999999758, + "median": 9.031462999999349, + "p95": 14.093383899999935, + "max": 51.87620100000095, + "opsPerSec": 110.72403219722786 + }, + "note": "JsonFileStore, 144 posts / 96 comments" + }, + { + "suite": "store", + "name": "sqlite-memory/createPost @ 50 posts", + "kind": "time", + "unit": "ms/op", + "value": 0.1009650000005422, + "stats": { + "iterations": 20, + "samples": 20, + "min": 0.07481099999858998, + "median": 0.1009650000005422, + "p95": 0.2506658000013886, + "max": 0.3402089999999589, + "opsPerSec": 9904.422324514731 + }, + "note": "SqlStore(:memory:), workspace already holds 50 posts" + }, + { + "suite": "store", + "name": "sqlite-memory/createPost @ 200 posts", + "kind": "time", + "unit": "ms/op", + "value": 0.07565600000089034, + "stats": { + "iterations": 20, + "samples": 20, + "min": 0.06927100000029895, + "median": 0.07565600000089034, + "p95": 0.13423439999860423, + "max": 0.1772579999997106, + "opsPerSec": 13217.722321933907 + }, + "note": "SqlStore(:memory:), workspace already holds 200 posts" + }, + { + "suite": "store", + "name": "sqlite-memory/createPost @ 500 posts", + "kind": "time", + "unit": "ms/op", + "value": 0.0792570000003252, + "stats": { + "iterations": 20, + "samples": 20, + "min": 0.06852999999864551, + "median": 0.0792570000003252, + "p95": 0.1328072500002236, + "max": 0.14616899999964517, + "opsPerSec": 12617.182078502805 + }, + "note": "SqlStore(:memory:), workspace already holds 500 posts" + }, + { + "suite": "store", + "name": "sqlite-file/createPost @ 50 posts", + "kind": "time", + "unit": "ms/op", + "value": 0.13259949999974197, + "stats": { + "iterations": 20, + "samples": 20, + "min": 0.10356400000091526, + "median": 0.13259949999974197, + "p95": 0.20030765000010436, + "max": 0.2615189999996801, + "opsPerSec": 7541.506566781519 + }, + "note": "SqlStore(file), workspace already holds 50 posts" + }, + { + "suite": "store", + "name": "sqlite-file/createPost @ 200 posts", + "kind": "time", + "unit": "ms/op", + "value": 0.12705300000106945, + "stats": { + "iterations": 20, + "samples": 20, + "min": 0.09517700000105833, + "median": 0.12705300000106945, + "p95": 0.1865948000010576, + "max": 0.20496400000047288, + "opsPerSec": 7870.731112146763 + }, + "note": "SqlStore(file), workspace already holds 200 posts" + }, + { + "suite": "store", + "name": "sqlite-file/createPost @ 500 posts", + "kind": "time", + "unit": "ms/op", + "value": 0.13933849999921222, + "stats": { + "iterations": 20, + "samples": 20, + "min": 0.0998520000030112, + "median": 0.13933849999921222, + "p95": 0.22354524999864225, + "max": 0.2342849999986356, + "opsPerSec": 7176.7673687147035 + }, + "note": "SqlStore(file), workspace already holds 500 posts" + }, + { + "suite": "store", + "name": "json-file/createPost @ 50 posts", + "kind": "time", + "unit": "ms/op", + "value": 1.2645945000003849, + "stats": { + "iterations": 20, + "samples": 20, + "min": 1.086012000003393, + "median": 1.2645945000003849, + "p95": 1.3843087999994168, + "max": 1.3880669999998645, + "opsPerSec": 790.7673171120827 + }, + "note": "JsonFileStore, workspace already holds 50 posts" + }, + { + "suite": "store", + "name": "json-file/createPost @ 200 posts", + "kind": "time", + "unit": "ms/op", + "value": 2.104668000001766, + "stats": { + "iterations": 20, + "samples": 20, + "min": 1.9422969999977795, + "median": 2.104668000001766, + "p95": 2.642122600000768, + "max": 4.287818999997398, + "opsPerSec": 475.1343204719989 + }, + "note": "JsonFileStore, workspace already holds 200 posts" + }, + { + "suite": "store", + "name": "json-file/createPost @ 500 posts", + "kind": "time", + "unit": "ms/op", + "value": 4.370087499999499, + "stats": { + "iterations": 20, + "samples": 20, + "min": 4.057629999999335, + "median": 4.370087499999499, + "p95": 5.125313050000479, + "max": 7.349017000000458, + "opsPerSec": 228.82837014135634 + }, + "note": "JsonFileStore, workspace already holds 500 posts" + }, + { + "suite": "store", + "name": "sqlite-memory/heap for loaded workspace", + "kind": "memory", + "unit": "bytes", + "value": 256, + "note": "SqlStore(:memory:), 12×25 posts" + }, + { + "suite": "store", + "name": "sqlite-memory/heap for listPosts(all) result", + "kind": "memory", + "unit": "bytes", + "value": 2201440, + "note": "SqlStore(:memory:), 12×25 posts, 300 materialized" + }, + { + "suite": "store", + "name": "sqlite-file/heap for loaded workspace", + "kind": "memory", + "unit": "bytes", + "value": 4928, + "note": "SqlStore(file), 12×25 posts" + }, + { + "suite": "store", + "name": "sqlite-file/heap for listPosts(all) result", + "kind": "memory", + "unit": "bytes", + "value": 2199616, + "note": "SqlStore(file), 12×25 posts, 300 materialized" + }, + { + "suite": "store", + "name": "json-file/heap for loaded workspace", + "kind": "memory", + "unit": "bytes", + "value": 2607040, + "note": "JsonFileStore, 12×25 posts" + }, + { + "suite": "store", + "name": "json-file/heap for listPosts(all) result", + "kind": "memory", + "unit": "bytes", + "value": 2651672, + "note": "JsonFileStore, 12×25 posts, 300 materialized" + }, + { + "suite": "render", + "name": "shiki cold init (first code render)", + "kind": "time", + "unit": "ms/op", + "value": 277.8204659999974, + "note": "one-time per process; loads every registry theme", + "tolerance": 2 + }, + { + "suite": "render", + "name": "shiki resident heap after warmup", + "kind": "memory", + "unit": "bytes", + "value": 2953112, + "note": "themes + grammars, per process" + }, + { + "suite": "render", + "name": "markdown/small", + "kind": "time", + "unit": "ms/op", + "value": 3.7866030000004685, + "stats": { + "iterations": 99, + "samples": 99, + "min": 3.290779000002658, + "median": 3.7866030000004685, + "p95": 5.625986199999169, + "max": 8.543934000001173, + "opsPerSec": 264.0889472701195 + }, + "note": "1760 B source" + }, + { + "suite": "render", + "name": "markdown/small document", + "kind": "bytes", + "unit": "bytes", + "value": 11256 + }, + { + "suite": "render", + "name": "markdown/large", + "kind": "time", + "unit": "ms/op", + "value": 51.90582100000029, + "stats": { + "iterations": 15, + "samples": 15, + "min": 48.21957900000052, + "median": 51.90582100000029, + "p95": 55.21429749999952, + "max": 55.25917800000025, + "opsPerSec": 19.265661937993322 + }, + "note": "34621 B source" + }, + { + "suite": "render", + "name": "markdown/large document", + "kind": "bytes", + "unit": "bytes", + "value": 88120 + }, + { + "suite": "render", + "name": "code/small", + "kind": "time", + "unit": "ms/op", + "value": 39.18193099999917, + "stats": { + "iterations": 15, + "samples": 15, + "min": 37.27079700000104, + "median": 39.18193099999917, + "p95": 43.45370629999997, + "max": 44.5009560000035, + "opsPerSec": 25.52196827665337 + }, + "note": "48 lines" + }, + { + "suite": "render", + "name": "code/small document", + "kind": "bytes", + "unit": "bytes", + "value": 34826 + }, + { + "suite": "render", + "name": "code/large", + "kind": "time", + "unit": "ms/op", + "value": 1138.8138330000002, + "stats": { + "iterations": 3, + "samples": 3, + "min": 1096.683001999998, + "median": 1138.8138330000002, + "p95": 1237.3471101000032, + "max": 1248.2952520000035, + "opsPerSec": 0.8781066501147777 + }, + "note": "1402 lines" + }, + { + "suite": "render", + "name": "code/large document", + "kind": "bytes", + "unit": "bytes", + "value": 769132 + }, + { + "suite": "render", + "name": "terminal/small", + "kind": "time", + "unit": "ms/op", + "value": 0.1271825357141227, + "stats": { + "iterations": 2716, + "samples": 194, + "min": 0.11191635714255556, + "median": 0.1271825357141227, + "p95": 0.19956953571423322, + "max": 1.773545928571236, + "opsPerSec": 7862.7147539169355 + }, + "note": "30 lines with SGR codes" + }, + { + "suite": "render", + "name": "terminal/small document", + "kind": "bytes", + "unit": "bytes", + "value": 11164 + }, + { + "suite": "render", + "name": "terminal/large", + "kind": "time", + "unit": "ms/op", + "value": 6.56851800000004, + "stats": { + "iterations": 57, + "samples": 57, + "min": 5.999559000003501, + "median": 6.56851800000004, + "p95": 8.427512000002022, + "max": 9.987591000004613, + "opsPerSec": 152.24134271992463 + }, + "note": "1500 lines with SGR codes" + }, + { + "suite": "render", + "name": "terminal/large document", + "kind": "bytes", + "unit": "bytes", + "value": 232521 + }, + { + "suite": "render", + "name": "diff/small", + "kind": "time", + "unit": "ms/op", + "value": 5.528104999997595, + "stats": { + "iterations": 52, + "samples": 52, + "min": 4.861931999999797, + "median": 5.528104999997595, + "p95": 7.447519500004999, + "max": 9.025180000004184, + "opsPerSec": 180.8938144265413 + }, + "note": "368 B patch" + }, + { + "suite": "render", + "name": "diff/small document", + "kind": "bytes", + "unit": "bytes", + "value": 61778 + }, + { + "suite": "render", + "name": "diff/large", + "kind": "time", + "unit": "ms/op", + "value": 251.4284370000023, + "stats": { + "iterations": 7, + "samples": 7, + "min": 235.10585300000093, + "median": 251.4284370000023, + "p95": 271.61501959999805, + "max": 274.98200599999836, + "opsPerSec": 3.9772748537588485 + }, + "note": "13086 B patch" + }, + { + "suite": "render", + "name": "diff/large document", + "kind": "bytes", + "unit": "bytes", + "value": 925798 + }, + { + "suite": "render", + "name": "html/small page wrap", + "kind": "time", + "unit": "ms/op", + "value": 0.0062783214285792345, + "stats": { + "iterations": 59640, + "samples": 284, + "min": 0.005764742857164016, + "median": 0.0062783214285792345, + "p95": 0.009416457142876986, + "max": 0.01127477142855198, + "opsPerSec": 159278.24202309072 + }, + "note": "784 B source" + }, + { + "suite": "render", + "name": "html/small document", + "kind": "bytes", + "unit": "bytes", + "value": 11064 + }, + { + "suite": "render", + "name": "html/large page wrap", + "kind": "time", + "unit": "ms/op", + "value": 0.006226331818172845, + "stats": { + "iterations": 60940, + "samples": 277, + "min": 0.005593345454524916, + "median": 0.006226331818172845, + "p95": 0.008822907272730265, + "max": 0.01375391818182834, + "opsPerSec": 160608.20868577738 + }, + "note": "37049 B source" + }, + { + "suite": "render", + "name": "html/large document", + "kind": "bytes", + "unit": "bytes", + "value": 47329 + }, + { + "suite": "render", + "name": "mermaid/small page wrap", + "kind": "time", + "unit": "ms/op", + "value": 0.009985105691068783, + "stats": { + "iterations": 37515, + "samples": 305, + "min": 0.009171333333334935, + "median": 0.009985105691068783, + "p95": 0.01504726666665865, + "max": 0.02245682113821906, + "opsPerSec": 100149.16526065957 + }, + "note": "250 B source" + }, + { + "suite": "render", + "name": "mermaid/small document", + "kind": "bytes", + "unit": "bytes", + "value": 9147 + }, + { + "suite": "render", + "name": "mermaid/large page wrap", + "kind": "time", + "unit": "ms/op", + "value": 0.02121194444437909, + "stats": { + "iterations": 18216, + "samples": 253, + "min": 0.01983090277776177, + "median": 0.02121194444437909, + "p95": 0.029159594444495774, + "max": 0.038419055555500056, + "opsPerSec": 47143.25000341909 + }, + "note": "3819 B source" + }, + { + "suite": "render", + "name": "mermaid/large document", + "kind": "bytes", + "unit": "bytes", + "value": 13086 + }, + { + "suite": "render", + "name": "theme switch re-render (10 mixed surfaces)", + "kind": "time", + "unit": "ms/op", + "value": 99.8628200000021, + "stats": { + "iterations": 7, + "samples": 7, + "min": 98.11342799999693, + "median": 99.8628200000021, + "p95": 103.0595550000049, + "max": 103.2796770000059, + "opsPerSec": 10.013736844202667 + }, + "note": "burst cost when the workspace theme changes" + }, + { + "suite": "api", + "name": "typical: GET /api/sessions", + "kind": "time", + "unit": "ms/op", + "value": 0.2943039999991015, + "stats": { + "iterations": 1190, + "samples": 595, + "min": 0.2344069999999192, + "median": 0.2943039999991015, + "p95": 0.43002094999901597, + "max": 5.1711114999998244, + "opsPerSec": 3397.8471240725676 + }, + "note": "144 posts / 96 comments" + }, + { + "suite": "api", + "name": "typical: GET /api/sessions bytes", + "kind": "bytes", + "unit": "bytes", + "value": 2751, + "note": "144 posts / 96 comments" + }, + { + "suite": "api", + "name": "typical: GET /api/posts/recent?limit=20", + "kind": "time", + "unit": "ms/op", + "value": 1.8851049999975658, + "stats": { + "iterations": 194, + "samples": 194, + "min": 1.6733889999959501, + "median": 1.8851049999975658, + "p95": 2.68266625000615, + "max": 6.822669000001042, + "opsPerSec": 530.4744298069822 + }, + "note": "144 posts / 96 comments" + }, + { + "suite": "api", + "name": "typical: GET /api/posts/recent?limit=20 bytes", + "kind": "bytes", + "unit": "bytes", + "value": 132989, + "note": "144 posts / 96 comments" + }, + { + "suite": "api", + "name": "typical: GET /api/sessions/:id/posts", + "kind": "time", + "unit": "ms/op", + "value": 1.0049739999994927, + "stats": { + "iterations": 362, + "samples": 362, + "min": 0.8916099999987637, + "median": 1.0049739999994927, + "p95": 1.5954183500023646, + "max": 4.1199889999988955, + "opsPerSec": 995.0506182254514 + }, + "note": "144 posts / 96 comments" + }, + { + "suite": "api", + "name": "typical: GET /api/sessions/:id/posts bytes", + "kind": "bytes", + "unit": "bytes", + "value": 72743, + "note": "144 posts / 96 comments" + }, + { + "suite": "api", + "name": "typical: GET /api/sessions/:id/posts?hydrate=1", + "kind": "time", + "unit": "ms/op", + "value": 0.7676956666667442, + "stats": { + "iterations": 489, + "samples": 163, + "min": 0.6834593333333032, + "median": 0.7676956666667442, + "p95": 1.3966703333334107, + "max": 1.6033759999991162, + "opsPerSec": 1302.5995110040121 + }, + "note": "144 posts / 96 comments" + }, + { + "suite": "api", + "name": "typical: GET /api/sessions/:id/posts?hydrate=1 bytes", + "kind": "bytes", + "unit": "bytes", + "value": 7071, + "note": "144 posts / 96 comments" + }, + { + "suite": "api", + "name": "typical: GET /api/comments?session", + "kind": "time", + "unit": "ms/op", + "value": 0.15433637499972974, + "stats": { + "iterations": 2408, + "samples": 301, + "min": 0.11185837499942863, + "median": 0.15433637499972974, + "p95": 0.21399612499953946, + "max": 0.7090522500002407, + "opsPerSec": 6479.353943629628 + }, + "note": "144 posts / 96 comments" + }, + { + "suite": "api", + "name": "typical: GET /api/comments?session bytes", + "kind": "bytes", + "unit": "bytes", + "value": 1994, + "note": "144 posts / 96 comments" + }, + { + "suite": "api", + "name": "heavy: GET /api/sessions", + "kind": "time", + "unit": "ms/op", + "value": 0.352337599999737, + "stats": { + "iterations": 1065, + "samples": 213, + "min": 0.3065810000000056, + "median": 0.352337599999737, + "p95": 0.4533312399999704, + "max": 1.429665199998999, + "opsPerSec": 2838.187011550134 + }, + "note": "900 posts / 600 comments" + }, + { + "suite": "api", + "name": "heavy: GET /api/sessions bytes", + "kind": "bytes", + "unit": "bytes", + "value": 6910, + "note": "900 posts / 600 comments" + }, + { + "suite": "api", + "name": "heavy: GET /api/posts/recent?limit=20", + "kind": "time", + "unit": "ms/op", + "value": 2.636497999996209, + "stats": { + "iterations": 144, + "samples": 144, + "min": 2.503145000002405, + "median": 2.636497999996209, + "p95": 4.405809349999253, + "max": 4.6635649999952875, + "opsPerSec": 379.2910140654148 + }, + "note": "900 posts / 600 comments" + }, + { + "suite": "api", + "name": "heavy: GET /api/posts/recent?limit=20 bytes", + "kind": "bytes", + "unit": "bytes", + "value": 196269, + "note": "900 posts / 600 comments" + }, + { + "suite": "api", + "name": "heavy: GET /api/sessions/:id/posts", + "kind": "time", + "unit": "ms/op", + "value": 3.487602999997762, + "stats": { + "iterations": 110, + "samples": 110, + "min": 3.282379000003857, + "median": 3.487602999997762, + "p95": 5.109663499998351, + "max": 5.550014000000374, + "opsPerSec": 286.7298829599131 + }, + "note": "900 posts / 600 comments" + }, + { + "suite": "api", + "name": "heavy: GET /api/sessions/:id/posts bytes", + "kind": "bytes", + "unit": "bytes", + "value": 270041, + "note": "900 posts / 600 comments" + }, + { + "suite": "api", + "name": "heavy: GET /api/sessions/:id/posts?hydrate=1", + "kind": "time", + "unit": "ms/op", + "value": 2.5542610000047716, + "stats": { + "iterations": 143, + "samples": 143, + "min": 2.3615519999948447, + "median": 2.5542610000047716, + "p95": 4.419611799997074, + "max": 6.281531999993604, + "opsPerSec": 391.5026694602204 + }, + "note": "900 posts / 600 comments" + }, + { + "suite": "api", + "name": "heavy: GET /api/sessions/:id/posts?hydrate=1 bytes", + "kind": "bytes", + "unit": "bytes", + "value": 23661, + "note": "900 posts / 600 comments" + }, + { + "suite": "api", + "name": "heavy: GET /api/comments?session", + "kind": "time", + "unit": "ms/op", + "value": 0.18444327777777086, + "stats": { + "iterations": 1944, + "samples": 216, + "min": 0.15777333333406002, + "median": 0.18444327777777086, + "p95": 0.2629653888889152, + "max": 0.8409713333328708, + "opsPerSec": 5421.721041006788 + }, + "note": "900 posts / 600 comments" + }, + { + "suite": "api", + "name": "heavy: GET /api/comments?session bytes", + "kind": "bytes", + "unit": "bytes", + "value": 4952, + "note": "900 posts / 600 comments" + }, + { + "suite": "api", + "name": "POST /api/posts (publish)", + "kind": "time", + "unit": "ms/op", + "value": 0.49710112500088144, + "stats": { + "iterations": 712, + "samples": 178, + "min": 0.42816449999918405, + "median": 0.49710112500088144, + "p95": 0.6046600000001491, + "max": 4.053106500001377, + "opsPerSec": 2011.6631198495616 + }, + "note": "144 posts" + }, + { + "suite": "api", + "name": "PUT /api/posts/:id (revise)", + "kind": "time", + "unit": "ms/op", + "value": 0.7600546666665953, + "stats": { + "iterations": 480, + "samples": 160, + "min": 0.6913466666665045, + "median": 0.7600546666665953, + "p95": 1.2115256166653123, + "max": 2.3452593333325544, + "opsPerSec": 1315.6948359856053 + }, + "note": "144 posts" + }, + { + "suite": "api", + "name": "POST /api/comments", + "kind": "time", + "unit": "ms/op", + "value": 0.641979750000246, + "stats": { + "iterations": 560, + "samples": 280, + "min": 0.5738304999977117, + "median": 0.641979750000246, + "p95": 0.8144162000004146, + "max": 2.629928500002279, + "opsPerSec": 1557.681531231502 + }, + "note": "144 posts" + }, + { + "suite": "api", + "name": "GET /s/:id html (cache hit)", + "kind": "time", + "unit": "ms/op", + "value": 0.19807200000026828, + "stats": { + "iterations": 1757, + "samples": 251, + "min": 0.16414628571406606, + "median": 0.19807200000026828, + "p95": 0.29082928571473793, + "max": 1.3039982857143124, + "opsPerSec": 5048.669170799737 + }, + "note": "render-cache hit" + }, + { + "suite": "api", + "name": "GET /s/:id html bytes", + "kind": "bytes", + "unit": "bytes", + "value": 11966 + }, + { + "suite": "api", + "name": "GET /s/:id markdown (cache hit)", + "kind": "time", + "unit": "ms/op", + "value": 0.19946578571450246, + "stats": { + "iterations": 1792, + "samples": 256, + "min": 0.17621114285741765, + "median": 0.19946578571450246, + "p95": 0.2401052857141102, + "max": 1.121802000000441, + "opsPerSec": 5013.39112579092 + }, + "note": "render-cache hit" + }, + { + "suite": "api", + "name": "GET /s/:id markdown bytes", + "kind": "bytes", + "unit": "bytes", + "value": 11545 + }, + { + "suite": "api", + "name": "GET /s/:id code (cache hit)", + "kind": "time", + "unit": "ms/op", + "value": 0.36957859999965875, + "stats": { + "iterations": 995, + "samples": 199, + "min": 0.33679620000038996, + "median": 0.36957859999965875, + "p95": 0.5435153200002968, + "max": 1.3997988000002806, + "opsPerSec": 2705.7843717166616 + }, + "note": "render-cache hit" + }, + { + "suite": "api", + "name": "GET /s/:id code bytes", + "kind": "bytes", + "unit": "bytes", + "value": 35097 + }, + { + "suite": "api", + "name": "GET /s/:id diff (cache hit)", + "kind": "time", + "unit": "ms/op", + "value": 0.51357700000032, + "stats": { + "iterations": 723, + "samples": 241, + "min": 0.4786696666681867, + "median": 0.51357700000032, + "p95": 0.6817536666664333, + "max": 1.756462333331001, + "opsPerSec": 1947.127694580125 + }, + "note": "render-cache hit" + }, + { + "suite": "api", + "name": "GET /s/:id diff bytes", + "kind": "bytes", + "unit": "bytes", + "value": 61819 + }, + { + "suite": "api", + "name": "GET /s/:id terminal (cache hit)", + "kind": "time", + "unit": "ms/op", + "value": 0.19818199999956831, + "stats": { + "iterations": 1764, + "samples": 252, + "min": 0.17413271428605576, + "median": 0.19818199999956831, + "p95": 0.33540783571427574, + "max": 1.164485714286067, + "opsPerSec": 5045.866930408302 + }, + "note": "render-cache hit" + }, + { + "suite": "api", + "name": "GET /s/:id terminal bytes", + "kind": "bytes", + "unit": "bytes", + "value": 11410 + }, + { + "suite": "api", + "name": "GET /s/:id html (cache miss)", + "kind": "time", + "unit": "ms/op", + "value": 0.2058208571428882, + "stats": { + "iterations": 1155, + "samples": 165, + "min": 0.1875468571420892, + "median": 0.2058208571428882, + "p95": 0.8027861142857311, + "max": 1.6305274285717002, + "opsPerSec": 4858.594089450148 + }, + "note": "forced re-render" + }, + { + "suite": "api", + "name": "GET /s/:id markdown (cache miss)", + "kind": "time", + "unit": "ms/op", + "value": 3.4799459999994724, + "stats": { + "iterations": 81, + "samples": 81, + "min": 3.323792000002868, + "median": 3.4799459999994724, + "p95": 5.63822199999413, + "max": 5.967258000004222, + "opsPerSec": 287.36078088572395 + }, + "note": "forced re-render" + }, + { + "suite": "api", + "name": "GET /s/:id code (cache miss)", + "kind": "time", + "unit": "ms/op", + "value": 38.454201500000636, + "stats": { + "iterations": 8, + "samples": 8, + "min": 36.23347900000226, + "median": 38.454201500000636, + "p95": 39.59185875000003, + "max": 39.854693000001134, + "opsPerSec": 26.004960732313826 + }, + "note": "forced re-render" + }, + { + "suite": "api", + "name": "GET /s/:id diff (cache miss)", + "kind": "time", + "unit": "ms/op", + "value": 5.479161999999633, + "stats": { + "iterations": 52, + "samples": 52, + "min": 5.148033999998006, + "median": 5.479161999999633, + "p95": 7.8641177000015885, + "max": 9.085332999995444, + "opsPerSec": 182.50966114892515 + }, + "note": "forced re-render" + }, + { + "suite": "api", + "name": "GET /s/:id terminal (cache miss)", + "kind": "time", + "unit": "ms/op", + "value": 0.4143813333333431, + "stats": { + "iterations": 609, + "samples": 203, + "min": 0.3752163333338103, + "median": 0.4143813333333431, + "p95": 1.2993985999998712, + "max": 2.125185999999909, + "opsPerSec": 2413.2361174569714 + }, + "note": "forced re-render" + }, + { + "suite": "api", + "name": "GET / (viewer document)", + "kind": "time", + "unit": "ms/op", + "value": 0.02684650999995938, + "stats": { + "iterations": 13700, + "samples": 274, + "min": 0.02316336000003503, + "median": 0.02684650999995938, + "p95": 0.04629047799993712, + "max": 0.06190402000007453, + "opsPerSec": 37248.78950751934 + }, + "note": "in-memory single-file viewer" + }, + { + "suite": "api", + "name": "render cache heap (64 mixed surfaces)", + "kind": "memory", + "unit": "bytes", + "value": 2779704, + "note": "cache holds up to 512 entries" + }, + { + "suite": "api", + "name": "render cache entries per surface view", + "kind": "count", + "unit": "count", + "value": 1, + "note": "one entry per (post, surface, version, theme, mode)" + }, + { + "suite": "events", + "name": "bus.broadcast → 1 subscribers", + "kind": "time", + "unit": "ms/op", + "value": 0.00011540424631020816, + "stats": { + "iterations": 3273336, + "samples": 198, + "min": 0.00009843787805490364, + "median": 0.00011540424631020816, + "p95": 0.00014854479494300386, + "max": 0.0001899944350350349, + "opsPerSec": 8665192.416854287 + }, + "note": "1 listeners" + }, + { + "suite": "events", + "name": "bus.broadcast → 10 subscribers", + "kind": "time", + "unit": "ms/op", + "value": 0.00016940602503061325, + "stats": { + "iterations": 2239192, + "samples": 248, + "min": 0.0001450443016939769, + "median": 0.00016940602503061325, + "p95": 0.00022877351866189166, + "max": 0.00026348931221635807, + "opsPerSec": 5902977.7708277535 + }, + "note": "10 listeners" + }, + { + "suite": "events", + "name": "bus.broadcast → 100 subscribers", + "kind": "time", + "unit": "ms/op", + "value": 0.000663621452422953, + "stats": { + "iterations": 594208, + "samples": 248, + "min": 0.0006222683639403682, + "median": 0.000663621452422953, + "p95": 0.0007566636894812296, + "max": 0.0009908580968278605, + "opsPerSec": 1506883.1731537504 + }, + "note": "100 listeners" + }, + { + "suite": "events", + "name": "publish → SSE delivery to 1 tabs", + "kind": "time", + "unit": "ms/op", + "value": 0.575905333332533, + "stats": { + "iterations": 624, + "samples": 208, + "min": 0.4647973333315652, + "median": 0.575905333332533, + "p95": 0.7826120333333764, + "max": 4.330366666666426, + "opsPerSec": 1736.3964910923837 + }, + "note": "1 open streams" + }, + { + "suite": "events", + "name": "frames per publish (1 tabs)", + "kind": "count", + "unit": "count", + "value": 1, + "note": "one frame per open stream" + }, + { + "suite": "events", + "name": "publish → SSE delivery to 5 tabs", + "kind": "time", + "unit": "ms/op", + "value": 0.5499659999998887, + "stats": { + "iterations": 654, + "samples": 218, + "min": 0.48065666666661855, + "median": 0.5499659999998887, + "p95": 0.6935038666681063, + "max": 3.389102333332024, + "opsPerSec": 1818.2942218249898 + }, + "note": "5 open streams" + }, + { + "suite": "events", + "name": "frames per publish (5 tabs)", + "kind": "count", + "unit": "count", + "value": 5, + "note": "one frame per open stream" + }, + { + "suite": "events", + "name": "publish → SSE delivery to 20 tabs", + "kind": "time", + "unit": "ms/op", + "value": 0.6798331666674737, + "stats": { + "iterations": 504, + "samples": 168, + "min": 0.6147533333326768, + "median": 0.6798331666674737, + "p95": 1.5987446833343733, + "max": 2.4767419999989215, + "opsPerSec": 1470.9491225648442 + }, + "note": "20 open streams" + }, + { + "suite": "events", + "name": "frames per publish (20 tabs)", + "kind": "count", + "unit": "count", + "value": 20, + "note": "one frame per open stream" + }, + { + "suite": "events", + "name": "heap per open SSE connection", + "kind": "memory", + "unit": "bytes", + "value": 3226, + "note": "measured across 20 concurrent streams" + }, + { + "suite": "events", + "name": "comment long-poll wakeup latency", + "kind": "time", + "unit": "ms/op", + "value": 0.6885714999989432, + "stats": { + "iterations": 526, + "samples": 263, + "min": 0.5753264999984822, + "median": 0.6885714999989432, + "p95": 0.8695627499986585, + "max": 4.8331154999978025, + "opsPerSec": 1452.282007026917 + }, + "note": "post → parked agent wakes" + } + ] +} diff --git a/bench/compare.ts b/bench/compare.ts new file mode 100644 index 00000000..1c4fb783 --- /dev/null +++ b/bench/compare.ts @@ -0,0 +1,179 @@ +// Baseline comparison and the regression gate. +// +// The hard problem a benchmark suite has to solve is not measuring — it's +// deciding when a number moving is NEWS. Get that wrong in the strict direction +// and CI is permanently red for reasons nobody can fix; get it wrong in the +// loose direction and the suite silently green-lights the regression it exists +// to catch. So thresholds are set per metric CLASS, and timings additionally get +// an absolute floor and a machine-speed correction: +// +// bytes/count DETERMINISTIC — identical input produces an identical number on +// every machine. Gated at 2%: effectively exact, with just enough +// slack for a version string or timestamp changing width. +// memory SEMI-STABLE — GC timing and allocator behavior move it a few +// percent run to run. Gated at 1.25× with a 1 MB floor. +// time NOISY — shared CI runners vary by more than this gate. Gated at +// 1.4× with a 0.5 ms floor, after normalizing by the machine +// index (see below). +// +// Machine normalization: each run measures a fixed synthetic workload and records +// the resulting ops/sec. Comparing across machines scales the baseline's timings +// by the ratio of the two indices. This corrects for "the CI runner is 2× slower +// than the laptop that recorded the baseline" — it does NOT correct for a +// different Node major, a different CPU architecture's cache behavior, or a +// noisy-neighbor runner. Treat a cross-machine timing comparison as a signal to +// investigate, and re-record the baseline on the machine that will police it. + +import type { BenchResult, MetricKind } from "./harness.ts"; + +export interface BenchRun { + /** Schema version of this file's shape, so an old baseline fails loudly. */ + format: 1; + recordedAt: string; + git?: { commit?: string; branch?: string }; + machine: { + platform: string; + arch: string; + cpus: number; + cpuModel: string; + nodeVersion: string; + totalMemory: number; + /** Calibration workload ops/sec — higher is faster. See harness.ts. */ + index: number; + }; + results: BenchResult[]; +} + +export interface Threshold { + /** Fail when current / baseline exceeds this. */ + ratio: number; + /** …but only when the absolute change also exceeds this, in the metric's unit. */ + floor: number; +} + +export const DEFAULT_THRESHOLDS: Record = { + bytes: { ratio: 1.02, floor: 256 }, + count: { ratio: 1.02, floor: 0 }, + memory: { ratio: 1.25, floor: 1024 * 1024 }, + time: { ratio: 1.4, floor: 0.5 }, +}; + +export type Verdict = "regressed" | "improved" | "unchanged" | "new" | "missing"; + +export interface Comparison { + key: string; + suite: string; + name: string; + kind: MetricKind; + unit: string; + baseline: number | null; + current: number | null; + /** Baseline scaled for machine speed (timings only); equals `baseline` otherwise. */ + expected: number | null; + ratio: number | null; + verdict: Verdict; + note?: string; +} + +export const resultKey = (r: Pick) => `${r.suite}/${r.name}`; + +/** + * Scale factor applied to baseline timings so a baseline recorded on a faster or + * slower machine still compares sensibly. Returns 1 when either index is missing + * or the ratio is implausible (a wildly different index usually means the + * calibration itself was disturbed, and silently trusting it would be worse than + * ignoring it). + */ +export function machineScale(baseline: BenchRun, current: BenchRun): number { + const b = baseline.machine?.index ?? 0; + const c = current.machine?.index ?? 0; + if (!b || !c) return 1; + const scale = b / c; + if (!Number.isFinite(scale) || scale <= 0.05 || scale >= 20) return 1; + return scale; +} + +export function compareRuns( + baseline: BenchRun, + current: BenchRun, + thresholds: Record = DEFAULT_THRESHOLDS, +): { comparisons: Comparison[]; regressions: Comparison[]; scale: number } { + const scale = machineScale(baseline, current); + const baseByKey = new Map(baseline.results.map((r) => [resultKey(r), r])); + const curByKey = new Map(current.results.map((r) => [resultKey(r), r])); + const comparisons: Comparison[] = []; + + for (const cur of current.results) { + const key = resultKey(cur); + const base = baseByKey.get(key); + if (!base) { + comparisons.push({ + key, + suite: cur.suite, + name: cur.name, + kind: cur.kind, + unit: cur.unit, + baseline: null, + current: cur.value, + expected: null, + ratio: null, + verdict: "new", + note: cur.note, + }); + continue; + } + // Only timings are machine-speed dependent; bytes and counts are not, and + // scaling memory by CPU speed would be nonsense. + // + // Direction matters: `scale` is baselineIndex/currentIndex, and the index is + // ops/sec, so scale > 1 means this machine is SLOWER. The same code should + // then take proportionally longer, which means multiplying the baseline. + const expected = cur.kind === "time" ? base.value * scale : base.value; + const threshold = thresholds[cur.kind]; + const ratio = expected > 0 ? cur.value / expected : cur.value > 0 ? Infinity : 1; + const delta = cur.value - expected; + const limit = cur.tolerance ?? threshold.ratio; + const regressed = ratio > limit && Math.abs(delta) > threshold.floor; + const improved = ratio < 1 / limit && Math.abs(delta) > threshold.floor; + comparisons.push({ + key, + suite: cur.suite, + name: cur.name, + kind: cur.kind, + unit: cur.unit, + baseline: base.value, + current: cur.value, + expected, + ratio, + verdict: regressed ? "regressed" : improved ? "improved" : "unchanged", + note: cur.note, + }); + } + + // A metric that vanished is reported, not failed: deleting a benchmark is a + // normal thing to do deliberately, and failing on it would make every + // intentional removal look like a regression. + for (const base of baseline.results) { + const key = resultKey(base); + if (curByKey.has(key)) continue; + comparisons.push({ + key, + suite: base.suite, + name: base.name, + kind: base.kind, + unit: base.unit, + baseline: base.value, + current: null, + expected: base.value, + ratio: null, + verdict: "missing", + note: base.note, + }); + } + + return { + comparisons, + regressions: comparisons.filter((c) => c.verdict === "regressed"), + scale, + }; +} diff --git a/bench/fixtures.ts b/bench/fixtures.ts new file mode 100644 index 00000000..7283fe0e --- /dev/null +++ b/bench/fixtures.ts @@ -0,0 +1,312 @@ +// Deterministic benchmark content. Every generator is a pure function of a seed +// and a size, so two runs on two machines measure the SAME bytes — a diff in the +// numbers is a diff in the code, never a diff in the input. +// +// Sizes are named after the shapes agents actually publish, not after round +// numbers: `small` is a typical card, `large` is the kind of payload a user +// notices ("why is my fan on?"). Both are measured, because a regression that +// only shows up at scale is exactly the one that reaches users. + +import type { Store } from "../server/types.ts"; + +export type Size = "small" | "large"; + +// --------------------------------------------------------------------------- +// Seeded PRNG — mulberry32, same generator the storage stress test uses, so a +// surprising number can be reproduced from its seed alone. +// --------------------------------------------------------------------------- + +export function rng(seed: number): () => number { + let s = seed; + return () => { + s |= 0; + s = (s + 0x6d2b79f5) | 0; + let t = Math.imul(s ^ (s >>> 15), s | 1); + t ^= t + Math.imul(t ^ (t >>> 7), t | 61); + return ((t ^ (t >>> 14)) >>> 0) / 4294967296; + }; +} + +const WORDS = [ + "session", + "surface", + "render", + "viewer", + "agent", + "comment", + "publish", + "sandbox", + "iframe", + "theme", + "store", + "sqlite", + "cursor", + "stream", + "payload", + "cache", +]; + +function words(r: () => number, n: number): string { + const out: string[] = []; + for (let i = 0; i < n; i++) out.push(WORDS[Math.floor(r() * WORDS.length)]); + return out.join(" "); +} + +// --------------------------------------------------------------------------- +// Surface content +// --------------------------------------------------------------------------- + +export function markdownSource(size: Size, seed = 1): string { + const r = rng(seed); + const sections = size === "small" ? 3 : 60; + const out: string[] = ["# Benchmark document", ""]; + for (let i = 0; i < sections; i++) { + out.push(`## Section ${i}: ${words(r, 4)}`, ""); + out.push(words(r, 45), ""); + out.push(`- ${words(r, 6)}`, `- ${words(r, 6)}`, `- ${words(r, 6)}`, ""); + // Fenced code forces shiki to load a grammar and highlight — the expensive + // half of markdown rendering, and the half agents actually use. + out.push( + "```ts", + `const ${WORDS[i % WORDS.length]}${i} = compute(${i}, "${words(r, 2)}");`, + "```", + "", + ); + if (i % 5 === 0) out.push(`> ${words(r, 12)}`, ""); + } + return out.join("\n"); +} + +export function codeSource(size: Size, seed = 2): string { + const r = rng(seed); + const lines = size === "small" ? 40 : 1200; + const out: string[] = ["import { createApp } from './app.ts';", ""]; + for (let i = 0; i < lines; i++) { + const kind = i % 6; + if (kind === 0) out.push(`export function handler${i}(input: { id: string; n: number }) {`); + else if (kind === 1) + out.push(` const ${WORDS[i % WORDS.length]} = input.n * ${i}; // ${words(r, 3)}`); + else if (kind === 2) + out.push(` if (!${WORDS[i % WORDS.length]}) throw new Error("${words(r, 3)}");`); + else if (kind === 3) + out.push(` const parts = [${i}, ${i + 1}].map((v) => ({ v, id: input.id }));`); + else if (kind === 4) out.push(` return { ok: true, parts, label: "${words(r, 2)}" };`); + else out.push("}", ""); + } + return out.join("\n"); +} + +export function terminalSource(size: Size, seed = 3): string { + const r = rng(seed); + const lines = size === "small" ? 30 : 1500; + const out: string[] = []; + for (let i = 0; i < lines; i++) { + // Mixed SGR codes: ansi_up's state machine is the thing under test, so the + // fixture has to actually exercise color transitions, not plain text. + const color = 31 + (i % 7); + out.push( + `\u001b[${color}m[${String(i).padStart(4, "0")}]\u001b[0m \u001b[1m${words(r, 3)}\u001b[0m ${words(r, 6)}`, + ); + } + return out.join("\n"); +} + +export function diffPatch(size: Size, seed = 4): string { + const r = rng(seed); + const files = size === "small" ? 1 : 12; + const hunksPerFile = size === "small" ? 1 : 4; + const out: string[] = []; + for (let f = 0; f < files; f++) { + const name = `server/module${f}.ts`; + out.push( + `diff --git a/${name} b/${name}`, + "index 1111111..2222222 100644", + `--- a/${name}`, + `+++ b/${name}`, + ); + for (let h = 0; h < hunksPerFile; h++) { + const start = 1 + h * 40; + out.push(`@@ -${start},9 +${start},10 @@`); + out.push(` const before${h} = ${h};`); + out.push(`- const removed = "${words(r, 3)}";`); + out.push(`+ const added = "${words(r, 3)}";`); + out.push(`+ const extra = compute(${h});`); + out.push(` function keep${h}() {`); + out.push(` return ${h} + ${f};`); + out.push(" }"); + out.push(`- legacy(${h});`); + out.push(`+ modern(${h});`); + out.push(` const after${h} = ${h};`); + } + } + return out.join("\n"); +} + +export function htmlSource(size: Size, seed = 5): string { + const r = rng(seed); + const rows = size === "small" ? 8 : 400; + const cells: string[] = []; + for (let i = 0; i < rows; i++) { + cells.push(`${i}${words(r, 4)}${words(r, 2)}`); + } + return `${cells.join("")}
`; +} + +export function mermaidSource(size: Size, seed = 6): string { + const r = rng(seed); + const nodes = size === "small" ? 6 : 80; + const out = ["flowchart TD"]; + for (let i = 1; i < nodes; i++) + out.push(` n${i - 1}["${words(r, 2)}"] --> n${i}["${words(r, 2)}"]`); + return out.join("\n"); +} + +export function jsonData(size: Size, seed = 7): unknown { + const r = rng(seed); + const rows = size === "small" ? 10 : 800; + return { + generated: "fixture", + rows: Array.from({ length: rows }, (_, i) => ({ + id: i, + label: words(r, 3), + value: Math.floor(r() * 10000) / 100, + tags: [words(r, 1), words(r, 1)], + })), + }; +} + +export function surfaceOfKind(kind: string, size: Size, seed = 1): Record { + switch (kind) { + case "markdown": + return { kind, markdown: markdownSource(size, seed) }; + case "code": + return { kind, code: codeSource(size, seed), language: "typescript", title: "module.ts" }; + case "terminal": + return { kind, text: terminalSource(size, seed), title: "build" }; + case "diff": + return { kind, patch: diffPatch(size, seed) }; + case "mermaid": + return { kind, mermaid: mermaidSource(size, seed) }; + case "json": + return { kind, data: jsonData(size, seed) }; + default: + return { kind: "html", html: htmlSource(size, seed) }; + } +} + +/** The kind mix a realistic workspace holds, weighted toward what agents publish most. */ +export const KIND_MIX = ["html", "markdown", "code", "diff", "terminal", "json"] as const; + +// --------------------------------------------------------------------------- +// Workspaces +// --------------------------------------------------------------------------- + +export interface WorkspaceShape { + sessions: number; + postsPerSession: number; + /** Revisions applied to each post — drives history growth (capped at HISTORY_LIMIT). */ + updatesPerPost: number; + commentsPerSession: number; + /** Surfaces per post. */ + surfacesPerPost: number; + size: Size; +} + +/** A typical single-user workspace after a few days of agent work. */ +export const TYPICAL: WorkspaceShape = { + sessions: 12, + postsPerSession: 12, + updatesPerPost: 2, + commentsPerSession: 8, + surfacesPerPost: 2, + size: "small", +}; + +/** A heavy workspace — the shape behind "sideshow is eating my laptop". */ +export const HEAVY: WorkspaceShape = { + sessions: 30, + postsPerSession: 30, + updatesPerPost: 4, + commentsPerSession: 20, + surfacesPerPost: 3, + size: "small", +}; + +export interface BuiltWorkspace { + sessionIds: string[]; + postIds: string[]; + /** The session with the most posts — the one a viewer load is benchmarked against. */ + busiestSessionId: string; + totalPosts: number; + totalComments: number; +} + +/** + * Populate a store with a deterministic workspace. Content is generated once per + * (kind, index) pair and reused, so building a big workspace measures the STORE, + * not the fixture generator. + */ +export async function buildWorkspace( + store: Store, + shape: WorkspaceShape, + seed = 42, +): Promise { + const r = rng(seed); + const cache = new Map>(); + const surfaceFor = (kind: string, variant: number) => { + const key = `${kind}:${variant}`; + let made = cache.get(key); + if (!made) { + made = surfaceOfKind(kind, shape.size, seed + variant); + cache.set(key, made); + } + return made; + }; + + const sessionIds: string[] = []; + const postIds: string[] = []; + let totalComments = 0; + + for (let s = 0; s < shape.sessions; s++) { + const session = await store.createSession({ + agent: ["pi", "claude", "amp"][s % 3], + title: `Session ${s}: ${words(r, 3)}`, + cwd: `/work/project-${s}`, + }); + sessionIds.push(session.id); + + for (let p = 0; p < shape.postsPerSession; p++) { + const surfaces = Array.from({ length: shape.surfacesPerPost }, (_, i) => + surfaceFor(KIND_MIX[(p + i) % KIND_MIX.length], (p + i) % 4), + ); + const post = await store.createPost({ + sessionId: session.id, + title: `Post ${p}: ${words(r, 3)}`, + surfaces: surfaces as never, + }); + if (!post) continue; + postIds.push(post.id); + for (let u = 0; u < shape.updatesPerPost; u++) { + await store.updatePost(post.id, { title: `Post ${p} rev ${u + 1}` }); + } + } + + for (let c = 0; c < shape.commentsPerSession; c++) { + await store.createComment({ + sessionId: session.id, + postId: c % 2 === 0 ? postIds[postIds.length - 1] : undefined, + author: c % 3 === 0 ? "user" : "agent", + text: words(r, 12), + }); + totalComments++; + } + } + + return { + sessionIds, + postIds, + busiestSessionId: sessionIds[0], + totalPosts: postIds.length, + totalComments, + }; +} diff --git a/bench/harness.ts b/bench/harness.ts new file mode 100644 index 00000000..faa1644b --- /dev/null +++ b/bench/harness.ts @@ -0,0 +1,310 @@ +// Benchmark measurement primitives. Node built-ins only (matching the repo's +// zero-build, type-stripping ethos) so `node bench/run.ts` just runs. +// +// Three ideas hold the suite together: +// +// 1. A metric is more than a number — it carries a `kind` that says how much +// to trust it. `bytes`/`count` are DETERMINISTIC (same input, same output, +// any machine), so a regression check can gate them almost exactly. +// `time`/`memory` are machine- and noise-dependent, so they get generous +// tolerances and an absolute floor. Mixing the two classes under one +// threshold is how benchmark suites end up permanently red or useless. +// 2. Iteration counts auto-scale to a time budget rather than being pinned, so +// a fast laptop and a slow CI runner both collect enough samples. +// 3. We report the MEDIAN, not the mean. One GC pause shouldn't move a number +// that people are asked to compare across commits. + +export type MetricKind = "time" | "bytes" | "count" | "memory"; + +export interface Stats { + iterations: number; + samples: number; + min: number; + median: number; + p95: number; + max: number; + opsPerSec: number; +} + +export interface BenchResult { + suite: string; + name: string; + kind: MetricKind; + unit: string; + /** The comparable value: median per-op time, or the raw byte/count/memory number. */ + value: number; + /** Present for `time` metrics. */ + stats?: Stats; + /** Free-text detail shown in the table (input size, backend, etc.). */ + note?: string; + /** + * Per-metric override of the regression ratio (see DEFAULT_TOLERANCE). Set it + * on a metric that is known-noisy or known-tight, rather than loosening the + * global default for everyone. + */ + tolerance?: number; +} + +export interface TimeOptions { + /** Iterations discarded before measuring. Default: auto (~50ms of work). */ + warmup?: number; + /** Minimum measured samples. Default 15. */ + minSamples?: number; + /** Keep sampling until this much wall time has been measured. Default 400ms. */ + minMs?: number; + /** Hard stop, even if minSamples/minMs are unmet. Default 3000ms. */ + maxMs?: number; + /** + * Run EXACTLY this many operations, one per sample, with no auto-scaling and + * no warmup unless asked for. + * + * Use it for benchmarks whose operation changes the thing being measured — a + * write that grows the store, a publish that lengthens a session. Auto-scaling + * would run a different number of those on every machine, so the workload + * itself (not just the clock) would differ run to run and the number would not + * be comparable. A fixed count makes the accumulated side effects identical + * everywhere, at the cost of fewer samples on fast machines. + */ + iterations?: number; + /** Shown in the results table next to the name. */ + note?: string; + tolerance?: number; +} + +const now = () => performance.now(); + +function quantile(sorted: number[], q: number): number { + if (sorted.length === 0) return 0; + const pos = (sorted.length - 1) * q; + const lo = Math.floor(pos); + const hi = Math.ceil(pos); + if (lo === hi) return sorted[lo]; + return sorted[lo] + (sorted[hi] - sorted[lo]) * (pos - lo); +} + +function summarize(perOpSamples: number[], iterations: number): Stats { + const sorted = [...perOpSamples].sort((a, b) => a - b); + const median = quantile(sorted, 0.5); + return { + iterations, + samples: sorted.length, + min: sorted[0] ?? 0, + median, + p95: quantile(sorted, 0.95), + max: sorted[sorted.length - 1] ?? 0, + opsPerSec: median > 0 ? 1000 / median : 0, + }; +} + +/** + * A batch size that makes one sample last long enough to out-resolve the clock. + * Sub-millisecond samples are mostly timer quantization, so cheap operations get + * measured in batches and the total divided back down. + */ +async function calibrateBatch(fn: () => unknown | Promise): Promise { + let batch = 1; + for (let attempt = 0; attempt < 20; attempt++) { + const start = now(); + for (let i = 0; i < batch; i++) await fn(); + const elapsed = now() - start; + if (elapsed >= 1) return batch; + batch = Math.max(batch * 2, Math.ceil(batch * (1.5 / Math.max(elapsed, 0.001)))); + if (batch > 1_000_000) return batch; + } + return batch; +} + +/** Time one operation, auto-scaling iterations to a budget. Reports the median. */ +export async function time( + suite: string, + name: string, + fn: () => unknown | Promise, + opts: TimeOptions = {}, +): Promise { + const { minSamples = 15, minMs = 400, maxMs = 3000, iterations: fixed, note, tolerance } = opts; + + // Warm up: let JIT tiering and lazy initialization settle before measuring. + // A fixed-iteration bench defaults to no warmup — its warmup ops would be + // side effects that shift the starting state the count is meant to pin. + const warmupIters = opts.warmup ?? (fixed !== undefined ? 0 : undefined); + if (warmupIters !== undefined) { + for (let i = 0; i < warmupIters; i++) await fn(); + } else { + const warmStart = now(); + let warmed = 0; + while (now() - warmStart < 50 && warmed < 10_000) { + await fn(); + warmed++; + } + } + + if (fixed !== undefined) { + const samples: number[] = []; + for (let i = 0; i < fixed; i++) { + const t0 = now(); + await fn(); + samples.push(now() - t0); + } + const stats = summarize(samples, fixed); + return { + suite, + name, + kind: "time", + unit: "ms/op", + value: stats.median, + stats, + note, + tolerance, + }; + } + + const batch = await calibrateBatch(fn); + const perOp: number[] = []; + const start = now(); + let iterations = 0; + while (perOp.length < minSamples || now() - start < minMs) { + const sampleStart = now(); + for (let i = 0; i < batch; i++) await fn(); + perOp.push((now() - sampleStart) / batch); + iterations += batch; + if (now() - start > maxMs) break; + } + + const stats = summarize(perOp, iterations); + return { suite, name, kind: "time", unit: "ms/op", value: stats.median, stats, note, tolerance }; +} + +/** A deterministic size metric — same input, same number, on any machine. */ +export function bytes(suite: string, name: string, value: number, note?: string): BenchResult { + return { suite, name, kind: "bytes", unit: "bytes", value, note }; +} + +/** A deterministic count metric (queries issued, events delivered, DOM nodes…). */ +export function count( + suite: string, + name: string, + value: number, + note?: string, + tolerance?: number, +): BenchResult { + return { suite, name, kind: "count", unit: "count", value, note, tolerance }; +} + +export function memory( + suite: string, + name: string, + value: number, + note?: string, + tolerance?: number, +): BenchResult { + return { suite, name, kind: "memory", unit: "bytes", value, note, tolerance }; +} + +/** + * Best-effort major GC. Requires `--expose-gc` (the npm scripts pass it); without + * it, memory numbers include whatever garbage happened to survive, so the runner + * warns once rather than silently reporting inflated deltas. + */ +export function collectGarbage(): boolean { + const gc = (globalThis as { gc?: (opts?: { execution?: string }) => void }).gc; + if (!gc) return false; + // Three passes: the first frees most of it, later ones catch objects kept alive + // only by finalizers/weak refs cleared in the previous pass. + for (let i = 0; i < 3; i++) gc(); + return true; +} + +export const gcAvailable = () => typeof (globalThis as { gc?: unknown }).gc === "function"; + +/** + * Heap retained by whatever `build` returns. The returned value is held live + * across the second sample (and only released after), so this measures RETAINED + * memory rather than allocation churn. + */ +export async function retainedHeap( + build: () => T | Promise, +): Promise<{ retained: number; value: T }> { + collectGarbage(); + const before = process.memoryUsage().heapUsed; + const value = await build(); + collectGarbage(); + const after = process.memoryUsage().heapUsed; + // Keep `value` reachable past the measurement so the optimizer can't drop it. + void (value as unknown); + return { retained: Math.max(0, after - before), value }; +} + +/** + * A fixed synthetic workload used to estimate how fast the current machine is, + * so a baseline recorded on one machine can be compared on another. It mixes + * integer math, string building, and small-object allocation to avoid rewarding + * any single microarchitectural strength. + * + * This is a coarse correction, not a promise of portability — see bench/README.md. + */ +export function calibrationWorkload(): number { + let acc = 0; + const parts: string[] = []; + for (let i = 0; i < 20_000; i++) { + acc = (acc + Math.imul(i ^ (acc >>> 3), 0x9e3779b1)) | 0; + if ((i & 0x3ff) === 0) parts.push(`${acc.toString(36)}:${i}`); + } + const joined = parts.join(","); + let hash = 0; + for (let i = 0; i < joined.length; i++) hash = (hash * 31 + joined.charCodeAt(i)) | 0; + const objs = []; + for (let i = 0; i < 2000; i++) objs.push({ i, k: `k${i & 63}`, v: hash ^ i }); + return objs.reduce((sum, o) => sum + (o.v & 0xff), 0) + hash; +} + +/** Machine speed index in workload-runs per second. Higher is faster. */ +export async function measureMachineIndex(): Promise { + const result = await time("_calibration", "machine index", () => calibrationWorkload(), { + minSamples: 9, + minMs: 300, + maxMs: 1500, + }); + return result.stats ? result.stats.opsPerSec : 0; +} + +// --------------------------------------------------------------------------- +// Suite plumbing +// --------------------------------------------------------------------------- + +export interface SuiteContext { + /** Record a finished result. */ + add: (result: BenchResult) => void; + /** Time an operation and record it under this suite. */ + time: (name: string, fn: () => unknown | Promise, opts?: TimeOptions) => Promise; + /** True when the caller asked for a fuller (slower) run. */ + full: boolean; + /** Only run benches whose "suite/name" matches; always true when unset. */ + matches: (name: string) => boolean; +} + +export interface Suite { + name: string; + description: string; + /** Suites needing a browser or child processes are excluded from the default run. */ + optional?: boolean; + run: (ctx: SuiteContext) => Promise; +} + +export function makeContext( + suite: string, + sink: BenchResult[], + opts: { full: boolean; filter?: RegExp }, +): SuiteContext { + const matches = (name: string) => !opts.filter || opts.filter.test(`${suite}/${name}`); + return { + full: opts.full, + matches, + add: (result) => { + if (matches(result.name)) sink.push(result); + }, + time: async (name, fn, timeOpts) => { + if (!matches(name)) return; + sink.push(await time(suite, name, fn, timeOpts)); + }, + }; +} diff --git a/bench/report.ts b/bench/report.ts new file mode 100644 index 00000000..eacc4994 --- /dev/null +++ b/bench/report.ts @@ -0,0 +1,138 @@ +// Terminal and markdown formatting for benchmark output. + +import type { BenchResult } from "./harness.ts"; +import type { BenchRun, Comparison } from "./compare.ts"; + +const isTty = () => Boolean(process.stdout.isTTY) && process.env.NO_COLOR === undefined; +const paint = (code: string, s: string) => (isTty() ? `\u001b[${code}m${s}\u001b[0m` : s); +export const dim = (s: string) => paint("2", s); +export const bold = (s: string) => paint("1", s); +export const red = (s: string) => paint("31", s); +export const green = (s: string) => paint("32", s); +export const yellow = (s: string) => paint("33", s); + +export function formatValue(value: number, unit: string): string { + if (unit === "bytes") { + if (value >= 1024 * 1024) return `${(value / (1024 * 1024)).toFixed(2)} MB`; + if (value >= 1024) return `${(value / 1024).toFixed(1)} KB`; + return `${Math.round(value)} B`; + } + if (unit === "ms/op") { + if (value >= 1000) return `${(value / 1000).toFixed(2)} s`; + if (value >= 1) return `${value.toFixed(2)} ms`; + if (value >= 0.001) return `${(value * 1000).toFixed(1)} µs`; + return `${(value * 1_000_000).toFixed(0)} ns`; + } + return String(Math.round(value)); +} + +const perSec = (msPerOp: number) => + msPerOp > 0 ? `${Math.round(1000 / msPerOp).toLocaleString("en-US")}/s` : "—"; + +/** Width ignoring ANSI escapes, so colored cells still line up. */ +// oxlint-disable-next-line no-control-regex +const visibleWidth = (s: string) => s.replace(/\u001b\[[0-9;]*m/g, "").length; + +function table(rows: string[][], align: ("left" | "right")[]): string { + const widths = rows[0].map((_, col) => Math.max(...rows.map((r) => visibleWidth(r[col] ?? "")))); + return rows + .map((row) => + row + .map((cell, col) => { + const pad = " ".repeat(Math.max(0, widths[col] - visibleWidth(cell))); + return align[col] === "right" ? pad + cell : cell + pad; + }) + .join(" ") + .trimEnd(), + ) + .join("\n"); +} + +export function renderResults(results: BenchResult[]): string { + const out: string[] = []; + const suites = [...new Set(results.map((r) => r.suite))]; + for (const suite of suites) { + const rows = results.filter((r) => r.suite === suite); + out.push("", bold(suite)); + const body: string[][] = [[" metric", "value", "rate", "p95", "detail"]]; + for (const r of rows) { + body.push([ + ` ${r.name}`, + formatValue(r.value, r.unit), + r.kind === "time" ? perSec(r.value) : "", + r.stats && r.kind === "time" ? formatValue(r.stats.p95, r.unit) : "", + dim(r.note ?? ""), + ]); + } + out.push(table(body, ["left", "right", "right", "right", "left"])); + } + return out.join("\n"); +} + +export function renderComparison(comparisons: Comparison[], scale: number): string { + const out: string[] = []; + if (Math.abs(scale - 1) > 0.05) { + out.push( + dim( + `Baseline timings scaled by ${scale.toFixed(2)}× for machine speed ` + + `(this machine is ${scale > 1 ? "slower" : "faster"} than the one that recorded the baseline).`, + ), + ); + } + const suites = [...new Set(comparisons.map((c) => c.suite))]; + for (const suite of suites) { + const rows = comparisons.filter((c) => c.suite === suite); + out.push("", bold(suite)); + const body: string[][] = [[" metric", "baseline", "current", "change", ""]]; + for (const c of rows) { + const change = + c.ratio === null ? "—" : `${c.ratio >= 1 ? "+" : ""}${((c.ratio - 1) * 100).toFixed(1)}%`; + const marker = + c.verdict === "regressed" + ? red("REGRESSED") + : c.verdict === "improved" + ? green("improved") + : c.verdict === "new" + ? yellow("new") + : c.verdict === "missing" + ? yellow("missing") + : ""; + body.push([ + ` ${c.name}`, + c.baseline === null ? "—" : formatValue(c.baseline, c.unit), + c.current === null ? "—" : formatValue(c.current, c.unit), + c.verdict === "regressed" ? red(change) : c.verdict === "improved" ? green(change) : change, + marker, + ]); + } + out.push(table(body, ["left", "right", "right", "right", "left"])); + } + return out.join("\n"); +} + +export function renderMachine(run: BenchRun): string { + const m = run.machine; + return dim( + `${m.platform}/${m.arch} · ${m.cpus}× ${m.cpuModel} · node ${m.nodeVersion} · ` + + `index ${Math.round(m.index).toLocaleString("en-US")}/s`, + ); +} + +/** A compact markdown table, for pasting into a PR or an issue. */ +export function renderMarkdown(run: BenchRun): string { + const lines = [ + `# sideshow benchmarks`, + "", + `Recorded ${run.recordedAt} on ${run.machine.platform}/${run.machine.arch}, ` + + `node ${run.machine.nodeVersion}, machine index ${Math.round(run.machine.index)}/s.`, + "", + ]; + for (const suite of new Set(run.results.map((r) => r.suite))) { + lines.push(`## ${suite}`, "", "| metric | value | detail |", "| --- | ---: | --- |"); + for (const r of run.results.filter((x) => x.suite === suite)) { + lines.push(`| ${r.name} | ${formatValue(r.value, r.unit)} | ${r.note ?? ""} |`); + } + lines.push(""); + } + return lines.join("\n"); +} diff --git a/bench/run.ts b/bench/run.ts new file mode 100644 index 00000000..724cd034 --- /dev/null +++ b/bench/run.ts @@ -0,0 +1,289 @@ +// Benchmark runner. +// +// node --expose-gc bench/run.ts # default suites, print a table +// node --expose-gc bench/run.ts store render # only these suites +// node --expose-gc bench/run.ts --all # include optional (slow) suites +// node --expose-gc bench/run.ts --filter 'diff' # only matching metrics +// node --expose-gc bench/run.ts --save out.json # write results +// node --expose-gc bench/run.ts --baseline # record bench/baseline.json +// node --expose-gc bench/run.ts --check # compare, exit 1 on regression +// node --expose-gc bench/run.ts --check --gate deterministic +// # …but only fail on bytes/counts +// node --expose-gc bench/run.ts --markdown # markdown table on stdout +// +// See bench/README.md for what the numbers mean and how the gate decides. + +import { execFileSync } from "node:child_process"; +import { readFileSync, writeFileSync } from "node:fs"; +import { relative } from "node:path"; +import { cpus, arch, platform, totalmem } from "node:os"; +import { fileURLToPath } from "node:url"; +import { compareRuns, type BenchRun } from "./compare.ts"; +import { + gcAvailable, + makeContext, + measureMachineIndex, + type BenchResult, + type MetricKind, + type Suite, +} from "./harness.ts"; +import { + bold, + dim, + red, + renderComparison, + renderMachine, + renderMarkdown, + renderResults, + yellow, +} from "./report.ts"; +import { apiSuite } from "./suites/api.bench.ts"; +import { eventsSuite } from "./suites/events.bench.ts"; +import { processSuite } from "./suites/process.bench.ts"; +import { renderSuite } from "./suites/render.bench.ts"; +import { storeSuite } from "./suites/store.bench.ts"; +import { viewerSuite } from "./suites/viewer.bench.ts"; + +const SUITES: Suite[] = [storeSuite, renderSuite, apiSuite, eventsSuite, processSuite, viewerSuite]; + +/** + * Baselines are per-scope. The default suites and `--all` measure different sets + * of metrics, so sharing one file would make every default run report the + * browser and process metrics as "missing" and every `--all` run report them as + * "new" — noise that trains people to ignore the diff. + */ +const baselinePath = (all: boolean) => + fileURLToPath(new URL(all ? "./baseline-all.json" : "./baseline.json", import.meta.url)); + +interface Options { + suites: string[]; + all: boolean; + filter?: RegExp; + save?: string; + baseline: boolean; + check: boolean; + /** Explicit --check path; defaults to the scope's baseline. */ + checkPath?: string; + markdown: boolean; + full: boolean; + gate: GateMode; +} + +/** + * Which metric kinds may FAIL the build. Everything is always compared and + * printed; this only decides what turns into a non-zero exit. + * + * `deterministic` exists for shared CI runners. A byte count means the same + * thing on any machine, so gating it there is sound. A timing on a noisy + * runner is not — gate those and the suite becomes a flaky test that people + * learn to re-run, which is worse than not gating at all. Timings still print, + * so a real slowdown is visible in the log even when it can't fail the job. + */ +type GateMode = "all" | "deterministic"; +const GATED_KINDS: Record = { + all: ["bytes", "count", "memory", "time"], + deterministic: ["bytes", "count"], +}; + +function parseArgs(argv: string[]): Options { + const opts: Options = { + suites: [], + all: false, + baseline: false, + check: false, + markdown: false, + full: false, + gate: "all", + }; + for (let i = 0; i < argv.length; i++) { + const arg = argv[i]; + if (arg === "--all") opts.all = true; + else if (arg === "--full") opts.full = true; + else if (arg === "--markdown") opts.markdown = true; + else if (arg === "--baseline") opts.baseline = true; + else if (arg === "--filter") opts.filter = new RegExp(argv[++i], "i"); + else if (arg === "--save") opts.save = argv[++i]; + else if (arg === "--gate") { + const mode = argv[++i]; + if (mode !== "all" && mode !== "deterministic") { + console.error(`--gate must be "all" or "deterministic", got: ${mode}`); + process.exit(2); + } + opts.gate = mode; + } else if (arg === "--check") { + opts.check = true; + // `--check path.json` is optional; a bare --check uses the committed baseline. + if (argv[i + 1] && !argv[i + 1].startsWith("--")) opts.checkPath = argv[++i]; + } else if (arg === "--help" || arg === "-h") { + console.log( + readFileSync(fileURLToPath(new URL("./run.ts", import.meta.url)), "utf8") + .split("\n") + .filter((l) => l.startsWith("//")) + .map((l) => l.slice(3)) + .join("\n"), + ); + process.exit(0); + } else if (arg.startsWith("--")) { + console.error(`unknown flag: ${arg}`); + process.exit(2); + } else opts.suites.push(arg); + } + return opts; +} + +function gitInfo(): { commit?: string; branch?: string } { + const run = (args: string[]) => { + try { + return execFileSync("git", args, { + encoding: "utf8", + stdio: ["ignore", "pipe", "ignore"], + }).trim(); + } catch { + return undefined; + } + }; + return { + commit: run(["rev-parse", "--short", "HEAD"]), + branch: run(["rev-parse", "--abbrev-ref", "HEAD"]), + }; +} + +async function main() { + const opts = parseArgs(process.argv.slice(2)); + + if (!gcAvailable()) { + console.error( + yellow( + "warning: running without --expose-gc; memory numbers include uncollected garbage " + + "and are not comparable to a baseline. Use `npm run bench`.", + ), + ); + } + + // A baseline recorded from a subset would silently drop every metric it didn't + // run, and the next full --check would report them all as "new" — a baseline + // that quietly stopped policing most of the suite. Refuse rather than record it. + if (opts.baseline && (opts.suites.length > 0 || opts.filter)) { + console.error( + "refusing to record a partial baseline: drop the suite names and --filter, " + + "or use --save to keep a scratch run.", + ); + process.exit(2); + } + + const selected = SUITES.filter((s) => { + if (opts.suites.length > 0) return opts.suites.includes(s.name); + return opts.all || !s.optional; + }); + if (selected.length === 0) { + console.error(`no suites matched. available: ${SUITES.map((s) => s.name).join(", ")}`); + process.exit(2); + } + + const results: BenchResult[] = []; + const started = Date.now(); + + // Calibrate before the suites, while the process is quiet — an index measured + // after a heavy suite has already thrashed the caches would understate the + // machine and inflate every scaled comparison. + const index = await measureMachineIndex(); + + for (const suite of selected) { + process.stderr.write(dim(`running ${suite.name}…\n`)); + const ctx = makeContext(suite.name, results, { full: opts.full, filter: opts.filter }); + await suite.run(ctx); + } + + const cpuList = cpus(); + const run: BenchRun = { + format: 1, + recordedAt: new Date().toISOString(), + git: gitInfo(), + machine: { + platform: platform(), + arch: arch(), + cpus: cpuList.length, + cpuModel: cpuList[0]?.model?.replace(/\s+/g, " ").trim() ?? "unknown", + nodeVersion: process.version, + totalMemory: totalmem(), + index, + }, + results, + }; + + if (opts.markdown) { + console.log(renderMarkdown(run)); + } else { + console.log(renderMachine(run)); + console.log(renderResults(results)); + console.log( + dim(`\n${results.length} metrics in ${((Date.now() - started) / 1000).toFixed(1)}s`), + ); + } + + if (opts.save) { + writeFileSync(opts.save, `${JSON.stringify(run, null, 2)}\n`); + console.log(dim(`saved ${opts.save}`)); + } + + if (opts.baseline) { + const path = baselinePath(opts.all); + writeFileSync(path, `${JSON.stringify(run, null, 2)}\n`); + console.log(`\n${bold("baseline recorded")} → ${relative(process.cwd(), path)}`); + return; + } + + if (opts.check) { + const checkPath = opts.checkPath ?? baselinePath(opts.all); + let baseline: BenchRun; + try { + baseline = JSON.parse(readFileSync(checkPath, "utf8")) as BenchRun; + } catch (err) { + console.error(red(`\ncannot read baseline at ${checkPath}: ${(err as Error).message}`)); + console.error("record one with `npm run bench:baseline`."); + process.exit(2); + } + if (baseline.format !== 1) { + console.error(red(`\nbaseline format ${baseline.format} is not supported; re-record it.`)); + process.exit(2); + } + const { comparisons, regressions, scale } = compareRuns(baseline, run); + console.log( + `\n${bold("vs baseline")} ${dim(`(${baseline.recordedAt}, ${baseline.git?.commit ?? "unknown"})`)}`, + ); + console.log(renderComparison(comparisons, scale)); + + const gated = GATED_KINDS[opts.gate]; + const failing = regressions.filter((r) => gated.includes(r.kind)); + const advisory = regressions.filter((r) => !gated.includes(r.kind)); + + // Advisory regressions are printed as prominently as failing ones. The point + // of --gate is to avoid FLAKY FAILURES, not to hide slowdowns: a timing + // regression on a shared runner still deserves a human's attention, it just + // shouldn't be the thing that blocks a merge. + if (advisory.length > 0) { + console.error(yellow(`\n${advisory.length} regression(s) not gated in "${opts.gate}" mode:`)); + for (const r of advisory) { + console.error(yellow(` ${r.key}: +${((r.ratio! - 1) * 100).toFixed(1)}% (${r.kind})`)); + } + } + + if (failing.length > 0) { + console.error(red(`\n${failing.length} regression${failing.length === 1 ? "" : "s"}:`)); + for (const r of failing) { + console.error( + red(` ${r.key}: ${((r.ratio! - 1) * 100).toFixed(1)}% slower/larger than baseline`), + ); + } + console.error( + dim( + "\nIf the change is intended, re-record with `npm run bench:baseline` and say so in the PR.", + ), + ); + process.exit(1); + } + console.log(`\n${bold(advisory.length > 0 ? "no gated regressions" : "no regressions")}`); + } +} + +await main(); diff --git a/bench/suites/api.bench.ts b/bench/suites/api.bench.ts new file mode 100644 index 00000000..1c5ae7a3 --- /dev/null +++ b/bench/suites/api.bench.ts @@ -0,0 +1,244 @@ +// End-to-end HTTP benchmarks against the real Hono app (in-process via +// `app.request`, so no socket noise between the measurement and the handler). +// +// These are the numbers a user actually feels: what an agent's publish costs, +// what a viewer's stream load costs, and what a surface iframe costs to serve. +// +// Response SIZE is recorded alongside latency and matters just as much. Every +// byte here is parsed and retained by the browser tab, and the two most recent +// viewer perf fixes were both about shipping less — so a size regression is a +// real regression even when the server-side timing is unchanged. + +import { createApp } from "../../server/app.ts"; +import { SqlStore } from "../../server/sqlStore.ts"; +import { createSqliteStorage } from "../../server/sqliteStorage.ts"; +import type { Store } from "../../server/types.ts"; +import { buildWorkspace, HEAVY, surfaceOfKind, TYPICAL } from "../fixtures.ts"; +import { bytes, count, memory, retainedHeap, type Suite, type SuiteContext } from "../harness.ts"; + +const VIEWER_HTML = "viewer"; + +function makeApp(store: Store) { + return createApp({ + store, + viewerHtml: VIEWER_HTML, + guideMarkdown: "# guide", + setupText: "# setup", + agentHowtoText: "# agent how-to", + // Empty version disables the npm-registry update check, keeping the bench + // off the network (and off a variable that has nothing to do with our code). + version: "", + }); +} + +type App = ReturnType; + +const jsonPost = (body: unknown) => ({ + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify(body), +}); + +/** Drain a response and return its byte length — what the client pays to receive. */ +async function responseBytes(app: App, path: string): Promise { + const res = await app.request(path); + return Buffer.byteLength(await res.text(), "utf8"); +} + +/** Issue a request and fully drain the body, so the bench includes serialization. */ +const hit = async (app: App, path: string) => { + await (await app.request(path)).text(); +}; + +async function benchReads(ctx: SuiteContext, app: App, sessionId: string, scale: string) { + const reads: [string, string][] = [ + ["GET /api/sessions", "/api/sessions"], + ["GET /api/posts/recent?limit=20", "/api/posts/recent?limit=20"], + ["GET /api/sessions/:id/posts", `/api/sessions/${sessionId}/posts`], + // The hydrate flavor is what a viewer tab loads on open — one response + // carrying the session's whole stream. + ["GET /api/sessions/:id/posts?hydrate=1", `/api/sessions/${sessionId}/posts?hydrate=1`], + ["GET /api/comments?session", `/api/comments?session=${sessionId}`], + ]; + for (const [name, path] of reads) { + await ctx.time(name, () => hit(app, path), { note: scale }); + ctx.add(bytes("api", `${name} bytes`, await responseBytes(app, path), scale)); + } +} + +export const apiSuite: Suite = { + name: "api", + description: "HTTP request paths through the real app: latency and response size", + async run(ctx) { + // --- reads at two workspace sizes -------------------------------------- + // The same routes at typical and heavy scale. A route whose cost grows with + // total workspace size (rather than with what it returns) shows up as a gap + // between these two. + for (const [label, shape] of [ + ["typical", TYPICAL], + ["heavy", HEAVY], + ] as const) { + const store = new SqlStore(createSqliteStorage()); + const built = await buildWorkspace(store, shape); + const app = makeApp(store); + await benchReads( + { + ...ctx, + add: (r) => ctx.add({ ...r, name: `${label}: ${r.name}` }), + time: (name, fn, opts) => ctx.time(`${label}: ${name}`, fn, opts), + }, + app, + built.busiestSessionId, + `${built.totalPosts} posts / ${built.totalComments} comments`, + ); + } + + // --- writes ------------------------------------------------------------ + { + const store = new SqlStore(createSqliteStorage()); + const built = await buildWorkspace(store, TYPICAL); + const app = makeApp(store); + const scale = `${built.totalPosts} posts`; + + const publishBody = { + session: built.busiestSessionId, + title: "bench", + parts: [surfaceOfKind("markdown", "small")], + }; + await ctx.time( + "POST /api/posts (publish)", + async () => { + await (await app.request("/api/posts", jsonPost(publishBody))).text(); + }, + { note: scale }, + ); + + const target = built.postIds[0]; + let rev = 0; + await ctx.time( + "PUT /api/posts/:id (revise)", + async () => { + const res = await app.request(`/api/posts/${target}`, { + method: "PUT", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ title: `rev ${rev++}` }), + }); + await res.text(); + }, + { note: scale }, + ); + + await ctx.time( + "POST /api/comments", + async () => { + const res = await app.request( + "/api/comments", + // Comments attach to a post, not a session (see createComment). + jsonPost({ surface: target, text: "bench comment" }), + ); + await res.text(); + }, + { note: scale }, + ); + } + + // --- surface documents (/s/:id) ---------------------------------------- + // The route each sandboxed iframe loads. Cold is a real render; warm is a + // render-cache hit. A viewer showing N surfaces issues N of these, so the + // warm number is multiplied by everything on screen. + { + const store = new SqlStore(createSqliteStorage()); + const session = await store.createSession({ agent: "bench", title: "surfaces" }); + const perKind: Record = {}; + for (const kind of ["html", "markdown", "code", "diff", "terminal"]) { + const post = await store.createPost({ + sessionId: session.id, + title: kind, + surfaces: [surfaceOfKind(kind, "small")] as never, + }); + if (post) perKind[kind] = post.id; + } + const app = makeApp(store); + + for (const [kind, id] of Object.entries(perKind)) { + const path = `/s/${id}?part=0`; + // Warm: every request after the first hits the memoized document. + await ctx.time(`GET /s/:id ${kind} (cache hit)`, () => hit(app, path), { + note: "render-cache hit", + }); + ctx.add(bytes("api", `GET /s/:id ${kind} bytes`, await responseBytes(app, path))); + } + + // Cold: force a miss on EVERY iteration. `?theme=` is part of the cache key + // but an unknown id resolves to the default theme (themeById falls back), so + // a counter in the theme slot gives a fresh key with byte-identical render + // work. Rotating over the three real themes would not do it — after three + // iterations they are all cached and the rest of the run measures hits. + // This is what a first view, a theme switch, or a cache eviction costs. + for (const [kind, id] of Object.entries(perKind)) { + let n = 0; + await ctx.time( + `GET /s/:id ${kind} (cache miss)`, + () => hit(app, `/s/${id}?part=0&theme=bench-${n++}`), + { note: "forced re-render", minSamples: 7, minMs: 300 }, + ); + } + } + + // --- viewer document ---------------------------------------------------- + { + const store = new SqlStore(createSqliteStorage()); + const app = makeApp(store); + await ctx.time("GET / (viewer document)", () => hit(app, "/"), { + note: "in-memory single-file viewer", + }); + } + + // --- render cache footprint --------------------------------------------- + // MAX_RENDER_CACHE bounds the ENTRY COUNT, not the bytes, so the ceiling is + // "512 × whatever a document happens to weigh". This measures what a + // realistically-filled cache actually holds — the number that decides whether + // that bound is generous or dangerous. + if (ctx.matches("render cache heap (64 mixed surfaces)")) { + const store = new SqlStore(createSqliteStorage()); + const session = await store.createSession({ agent: "bench", title: "cache" }); + const ids: string[] = []; + for (let i = 0; i < 64; i++) { + const kind = ["markdown", "code", "diff", "terminal", "html"][i % 5]; + const post = await store.createPost({ + sessionId: session.id, + title: `p${i}`, + surfaces: [surfaceOfKind(kind, "small", i)] as never, + }); + if (post) ids.push(post.id); + } + const app = makeApp(store); + // Warm the shared highlighter first so its one-time heap isn't billed here. + await hit(app, `/s/${ids[0]}?part=0`); + const { retained } = await retainedHeap(async () => { + const fresh = makeApp(store); + for (const id of ids) await hit(fresh, `/s/${id}?part=0`); + return fresh; + }); + ctx.add( + memory( + "api", + "render cache heap (64 mixed surfaces)", + retained, + "cache holds up to 512 entries", + ), + ); + ctx.add( + count( + "api", + "render cache entries per surface view", + 1, + "one entry per (post, surface, version, theme, mode)", + ), + ); + } + }, +}; + +/** Exported for the memory suite, which boots an app of its own. */ +export { makeApp as createBenchApp }; diff --git a/bench/suites/events.bench.ts b/bench/suites/events.bench.ts new file mode 100644 index 00000000..a69fc211 --- /dev/null +++ b/bench/suites/events.bench.ts @@ -0,0 +1,205 @@ +// Live-feed benchmarks: the SSE fan-out and the comment long-poll. +// +// This is the path most likely to burn CPU while nobody is looking. A publish +// broadcasts to every open viewer tab; each tab holds a connection for the life +// of the session and gets a keepalive ping every 15s. An agent mid-task can +// publish in a tight loop, so per-event cost is multiplied by both the event +// rate and the tab count. +// +// The per-connection heap number matters for the same reason: connections are +// long-lived, so whatever a connection retains is retained for hours. + +import { createApp } from "../../server/app.ts"; +import { EventBus } from "../../server/events.ts"; +import { SqlStore } from "../../server/sqlStore.ts"; +import { createSqliteStorage } from "../../server/sqliteStorage.ts"; +import { buildWorkspace, surfaceOfKind, TYPICAL } from "../fixtures.ts"; +import { count, memory, retainedHeap, type Suite, time } from "../harness.ts"; + +function makeApp(store: SqlStore) { + return createApp({ + store, + viewerHtml: "viewer", + guideMarkdown: "# guide", + setupText: "# setup", + agentHowtoText: "# agent how-to", + version: "", + }); +} + +/** + * Open an SSE connection and collect events until `expected` data frames have + * arrived. Returns a stop() that aborts the request, so a bench can't leak a + * held connection into the next one (the app caps concurrent holds, and a leak + * would show up as a mysterious 503 several benches later). + */ +function openSse( + app: ReturnType, + onEvent: () => void, +): { stop: () => void; ready: Promise } { + const controller = new AbortController(); + let markReady!: () => void; + const ready = new Promise((resolve) => { + markReady = resolve; + }); + + void (async () => { + try { + const res = await app.request("/api/events", { signal: controller.signal }); + const body = res.body; + if (!body) return; + const reader = body.getReader(); + const decoder = new TextDecoder(); + let buffer = ""; + for (;;) { + const { done, value } = await reader.read(); + if (done) return; + buffer += decoder.decode(value, { stream: true }); + let idx: number; + while ((idx = buffer.indexOf("\n\n")) !== -1) { + const frame = buffer.slice(0, idx); + buffer = buffer.slice(idx + 2); + if (frame.includes("event: hello")) markReady(); + else if (frame.includes("data:") && !frame.includes("event: ping")) onEvent(); + } + } + } catch { + // Aborting the request is the normal way these end. + } + })(); + + return { stop: () => controller.abort(), ready }; +} + +export const eventsSuite: Suite = { + name: "events", + description: "SSE fan-out, event bus dispatch, and comment long-poll wakeups", + async run(ctx) { + // --- raw bus dispatch --------------------------------------------------- + // The floor: what a broadcast costs before any serialization or I/O. If this + // number moves, the cost is structural (listener bookkeeping), not transport. + for (const subscribers of [1, 10, 100]) { + const bus = new EventBus(); + let seen = 0; + for (let i = 0; i < subscribers; i++) bus.subscribe(() => seen++); + await ctx.time( + `bus.broadcast → ${subscribers} subscribers`, + () => bus.broadcast({ type: "post-updated", id: "p", sessionId: "s", version: 2 }), + { note: `${subscribers} listeners` }, + ); + void seen; + } + + // --- SSE end to end ----------------------------------------------------- + // A publish, from the write landing to the frame arriving on N open streams. + // This includes JSON serialization and the stream writes — the part that + // actually scales with open tabs. + for (const tabs of [1, 5, 20]) { + if (!ctx.matches(`publish → SSE delivery to ${tabs} tabs`)) continue; + const store = new SqlStore(createSqliteStorage()); + const built = await buildWorkspace(store, { ...TYPICAL, sessions: 1, postsPerSession: 2 }); + const app = makeApp(store); + + let delivered = 0; + const conns = Array.from({ length: tabs }, () => openSse(app, () => delivered++)); + await Promise.all(conns.map((c) => c.ready)); + + const surface = surfaceOfKind("markdown", "small"); + const publishOnce = async () => { + const target = delivered + tabs; + const res = await app.request("/api/posts", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + session: built.busiestSessionId, + title: "bench", + parts: [surface], + }), + }); + await res.text(); + // Wait until every open stream has actually seen it — otherwise this + // measures the write and leaves the fan-out to happen off-clock. + while (delivered < target) await new Promise((r) => setImmediate(r)); + }; + + ctx.add( + await time("events", `publish → SSE delivery to ${tabs} tabs`, publishOnce, { + note: `${tabs} open streams`, + minSamples: 10, + minMs: 400, + }), + ); + for (const c of conns) c.stop(); + // Let the aborts settle so the next iteration starts from zero holds. + await new Promise((r) => setTimeout(r, 20)); + ctx.add( + count("events", `frames per publish (${tabs} tabs)`, tabs, "one frame per open stream"), + ); + } + + // --- per-connection heap ------------------------------------------------- + // Long-lived by nature: a tab left open all day holds one of these. + if (ctx.matches("heap per open SSE connection")) { + const store = new SqlStore(createSqliteStorage()); + await buildWorkspace(store, { ...TYPICAL, sessions: 1, postsPerSession: 2 }); + const app = makeApp(store); + const N = 20; + const { retained, value } = await retainedHeap(async () => { + const conns = Array.from({ length: N }, () => openSse(app, () => {})); + await Promise.all(conns.map((c) => c.ready)); + return conns; + }); + ctx.add( + memory( + "events", + "heap per open SSE connection", + Math.round(retained / N), + `measured across ${N} concurrent streams`, + ), + ); + for (const c of value) c.stop(); + await new Promise((r) => setTimeout(r, 20)); + } + + // --- comment long-poll --------------------------------------------------- + // The agent-side half of the feedback loop: an agent parks on + // /api/comments?wait and must be woken promptly when the user comments. + // Latency here is felt directly as "the agent didn't notice my comment". + if (ctx.matches("comment long-poll wakeup latency")) { + const store = new SqlStore(createSqliteStorage()); + const session = await store.createSession({ agent: "bench", title: "poll" }); + // Comments attach to a post, so the poll needs one to point at. + const post = await store.createPost({ + sessionId: session.id, + title: "poll target", + surfaces: [surfaceOfKind("markdown", "small")] as never, + }); + const app = makeApp(store); + let n = 0; + ctx.add( + await time( + "events", + "comment long-poll wakeup latency", + async () => { + const waiting = (async () => { + const res = await app.request( + `/api/comments?session=${session.id}&author=user&wait=5`, + ); + return res.text(); + })(); + // Yield so the wait is genuinely parked before the comment lands. + await new Promise((r) => setImmediate(r)); + const posted = await app.request("/api/comments", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ surface: post!.id, text: `c${n++}`, author: "user" }), + }); + await posted.text(); + await waiting; + }, + { note: "post → parked agent wakes", minSamples: 10, minMs: 400 }, + ), + ); + } + }, +}; diff --git a/bench/suites/process.bench.ts b/bench/suites/process.bench.ts new file mode 100644 index 00000000..5cf36eea --- /dev/null +++ b/bench/suites/process.bench.ts @@ -0,0 +1,271 @@ +// Whole-process benchmarks: what sideshow costs before it does any work. +// +// These spawn real child processes, so they're slower than the in-process suites +// and run with a small iteration count. They're worth the wall time because +// they're the numbers a user meets first: how long `sideshow` takes to respond, +// and how much memory the server holds while idle. +// +// RSS (not heap) is the number reported here — it's what a user sees in Activity +// Monitor, which is where the "sideshow uses too much memory" complaint starts. + +import { spawn } from "node:child_process"; +import { mkdtempSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { fileURLToPath } from "node:url"; +import { join } from "node:path"; +import { memory, type Suite, time } from "../harness.ts"; + +const repoRoot = fileURLToPath(new URL("../..", import.meta.url)); + +function tmpDataDir(): string { + return mkdtempSync(join(tmpdir(), "sideshow-bench-proc-")); +} + +/** Run a command to completion and return its wall time. */ +function runToCompletion(args: string[], env: Record = {}): Promise { + const started = performance.now(); + return new Promise((resolve, reject) => { + const proc = spawn(process.execPath, args, { + cwd: repoRoot, + env: { ...process.env, ...env }, + stdio: "ignore", + }); + proc.on("error", reject); + proc.on("exit", () => resolve(performance.now() - started)); + }); +} + +interface RunningServer { + url: string; + pid: number; + bootMs: number; + stop: () => void; +} + +/** Boot the real server and resolve once it reports a listening port. */ +function bootServer(env: Record = {}): Promise { + const dir = tmpDataDir(); + const started = performance.now(); + const proc = spawn(process.execPath, ["server/index.ts"], { + cwd: repoRoot, + env: { + ...process.env, + PORT: "0", + SIDESHOW_DB: join(dir, "bench.db"), + SIDESHOW_DATA: join(dir, "bench.json"), + // Empty version disables the update check — otherwise boot time includes a + // network round trip that has nothing to do with our code. + SIDESHOW_VERSION: "", + SIDESHOW_TOKEN: "", + ...env, + }, + stdio: ["ignore", "pipe", "ignore"], + }); + return new Promise((resolve, reject) => { + let out = ""; + const timer = setTimeout(() => { + proc.kill(); + reject(new Error(`server did not boot in time; output: ${out}`)); + }, 30_000); + proc.stdout?.on("data", (chunk: Buffer) => { + out += chunk.toString(); + const match = out.match(/listening on (http:\/\/localhost:\d+)/); + if (match) { + clearTimeout(timer); + resolve({ + url: match[1], + pid: proc.pid ?? 0, + bootMs: performance.now() - started, + stop: () => proc.kill(), + }); + } + }); + proc.on("exit", (code) => { + clearTimeout(timer); + reject(new Error(`server exited early with code ${code}: ${out}`)); + }); + }); +} + +/** Resident set size of another process, in bytes. Linux/macOS `ps`. */ +function rssOf(pid: number): Promise { + return new Promise((resolve) => { + const proc = spawn("ps", ["-o", "rss=", "-p", String(pid)], { + stdio: ["ignore", "pipe", "ignore"], + }); + let out = ""; + proc.stdout.on("data", (c: Buffer) => (out += c.toString())); + proc.on("error", () => resolve(0)); + // `ps` reports kilobytes. + proc.on("exit", () => resolve((Number(out.trim()) || 0) * 1024)); + }); +} + +const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms)); + +/** + * RSS and wall time for importing one module into an otherwise-empty process. + * + * The idle-server number says the server holds a lot of memory; it doesn't say + * whose. This does. Each module is loaded in its own child process and the child + * reports its own RSS, so the cost is attributed rather than guessed — and the + * `none` row gives the Node baseline to subtract. + */ +function importCost(specifier: string | null): Promise<{ rss: number; ms: number }> { + const script = specifier + ? `const t=performance.now();await import(${JSON.stringify(specifier)});` + + `console.log(JSON.stringify({rss:process.memoryUsage().rss,ms:performance.now()-t}));` + : `console.log(JSON.stringify({rss:process.memoryUsage().rss,ms:0}));`; + return new Promise((resolve, reject) => { + const proc = spawn(process.execPath, ["--input-type=module", "--eval", script], { + cwd: repoRoot, + stdio: ["ignore", "pipe", "ignore"], + }); + let out = ""; + proc.stdout.on("data", (c: Buffer) => (out += c.toString())); + proc.on("error", reject); + proc.on("exit", (code) => { + try { + resolve(JSON.parse(out.trim()) as { rss: number; ms: number }); + } catch { + reject(new Error(`import probe for ${specifier ?? "none"} failed (exit ${code}): ${out}`)); + } + }); + }); +} + +/** Modules that dominate the server's module graph, loaded cheapest-first. */ +const IMPORT_TARGETS: [name: string, specifier: string | null][] = [ + ["node baseline (no imports)", null], + ["hono", "hono"], + ["markdown-it", "markdown-it"], + ["shiki", "shiki"], + ["@pierre/diffs", "@pierre/diffs"], + ["server/richRender.ts", "./server/richRender.ts"], + ["server/app.ts", "./server/app.ts"], +]; + +export const processSuite: Suite = { + name: "process", + description: "Process startup time and idle/loaded resident memory (spawns real processes)", + // Spawning servers is slow and needs `ps`; keep it out of the default run. + optional: true, + async run(ctx) { + // --- CLI startup --------------------------------------------------------- + // Every CLI invocation pays module load. Agents call the CLI per publish, so + // this is multiplied by however chatty the agent is. + if (ctx.matches("CLI: sideshow help")) { + ctx.add( + await time( + "process", + "CLI: sideshow help", + () => runToCompletion(["bin/sideshow.js", "help"]), + { + warmup: 1, + minSamples: 5, + minMs: 200, + maxMs: 8000, + note: "cold node process per invocation", + }, + ), + ); + } + + // --- server boot + idle footprint --------------------------------------- + if (ctx.matches("server boot to listening")) { + const boots: number[] = []; + for (let i = 0; i < 3; i++) { + const server = await bootServer(); + boots.push(server.bootMs); + server.stop(); + await sleep(50); + } + boots.sort((a, b) => a - b); + ctx.add({ + suite: "process", + name: "server boot to listening", + kind: "time", + unit: "ms/op", + value: boots[Math.floor(boots.length / 2)], + note: "empty workspace, update check disabled", + stats: { + iterations: boots.length, + samples: boots.length, + min: boots[0], + median: boots[Math.floor(boots.length / 2)], + p95: boots[boots.length - 1], + max: boots[boots.length - 1], + opsPerSec: 0, + }, + tolerance: 1.8, + }); + } + + if (ctx.matches("server RSS idle")) { + const server = await bootServer(); + // Let boot allocations settle before sampling. + await sleep(750); + ctx.add( + memory("process", "server RSS idle", await rssOf(server.pid), "just booted, no requests"), + ); + + // The first rich surface loads shiki's themes and grammars into the server + // process; that step is invisible in an idle reading but permanent + // afterwards. Measuring both makes the jump attributable. + const publish = await fetch(`${server.url}/api/posts`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + title: "bench", + agent: "bench", + parts: [{ kind: "code", code: "const x = 1;\n", language: "typescript" }], + }), + }); + const post = (await publish.json()) as { id: string }; + await (await fetch(`${server.url}/s/${post.id}?part=0`)).text(); + await sleep(750); + ctx.add( + memory( + "process", + "server RSS after first rich render", + await rssOf(server.pid), + "shiki themes + grammars now resident", + ), + ); + server.stop(); + } + + // --- where the idle memory goes ----------------------------------------- + // Attribution for the idle RSS above. The server's module graph is loaded + // eagerly — importing server/app.ts pulls in richRender.ts, which pulls in + // shiki and @pierre/diffs — so a server that never renders a rich surface + // still pays for the ones it might. + for (const [name, specifier] of IMPORT_TARGETS) { + const metric = `import RSS: ${name}`; + if (!ctx.matches(metric)) continue; + // Median of three: module loading hits the filesystem, and a cold cache on + // the first probe would otherwise be read as a difference between modules. + const runs = [ + await importCost(specifier), + await importCost(specifier), + await importCost(specifier), + ]; + const rss = runs.map((r) => r.rss).sort((a, b) => a - b)[1]; + const ms = runs.map((r) => r.ms).sort((a, b) => a - b)[1]; + ctx.add( + memory("process", metric, rss, specifier ? `import "${specifier}"` : "empty process"), + ); + if (specifier) { + ctx.add({ + suite: "process", + name: `import time: ${name}`, + kind: "time", + unit: "ms/op", + value: ms, + note: "cold module graph load", + tolerance: 1.8, + }); + } + } + }, +}; diff --git a/bench/suites/render.bench.ts b/bench/suites/render.bench.ts new file mode 100644 index 00000000..077edcce --- /dev/null +++ b/bench/suites/render.bench.ts @@ -0,0 +1,206 @@ +// Server-side rendering benchmarks — the biggest single CPU consumer in the +// server process. +// +// Every sandboxed surface (markdown/code/diff/terminal/html/mermaid) is rendered +// to a string on the server and served from /s/:id. markdown and code run shiki; +// diff runs @pierre/diffs SSR on top of shiki. That work happens per +// (post, surface, version, theme, mode) and is cached, so two numbers matter and +// are measured separately: +// +// - COLD INIT: creating the shared highlighter loads every registry theme. +// It's paid once per process, but it's large enough to be felt at startup +// and is measured on its own rather than being smeared across the first +// render's average. +// - STEADY RENDER: what a cache miss costs once the highlighter exists. +// +// Output size is recorded too. A rendered document is held in the render cache +// (up to MAX_RENDER_CACHE entries) and shipped to the browser, so its size is +// both a memory and a bandwidth number — and unlike timings, it's deterministic +// enough to gate hard. + +import { renderCode, renderDiff, renderMarkdown, renderTerminal } from "../../server/richRender.ts"; +import { + renderHtmlPage, + renderMermaidPage, + renderSandboxedPart, +} from "../../server/surfacePage.ts"; +import { + codeSource, + diffPatch, + htmlSource, + markdownSource, + mermaidSource, + type Size, + terminalSource, +} from "../fixtures.ts"; +import { bytes, memory, retainedHeap, type Suite, time } from "../harness.ts"; + +const THEME = { theme: "github", mode: "dark" as const }; +const ORIGIN = "http://localhost:8228"; + +/** Byte length of a rendered document as it goes over the wire. */ +const utf8 = (s: string) => Buffer.byteLength(s, "utf8"); + +// Wrapping is what /s/:id actually serves, so the recorded sizes are the real +// document sizes, not just the inner body. +const wrap = (rendered: { body: string; css: string }) => + renderSandboxedPart({ ...rendered, origin: ORIGIN, theme: THEME.theme, mode: THEME.mode }); + +export const renderSuite: Suite = { + name: "render", + description: "Server-side surface rendering (shiki, markdown-it, diff SSR) and output size", + async run(ctx) { + // --- cold highlighter init --------------------------------------------- + // Measured in a fresh module instance so the singleton highlighter is + // genuinely cold. This is startup cost paid on the first rich surface a + // process ever renders. + if (ctx.matches("shiki cold init (first code render)")) { + const started = performance.now(); + const fresh = await import(`../../server/richRender.ts?cold=${Date.now()}`); + await (fresh as typeof import("../../server/richRender.ts")).renderCode( + { kind: "code", code: "const x = 1;", language: "typescript" }, + THEME, + ); + const elapsed = performance.now() - started; + ctx.add({ + suite: "render", + name: "shiki cold init (first code render)", + kind: "time", + unit: "ms/op", + value: elapsed, + note: "one-time per process; loads every registry theme", + // A single unrepeatable sample, so it is noisier than the sampled + // benchmarks — gate it loosely. + tolerance: 2, + }); + + // Heap held by the loaded highlighter: themes + grammars stay resident for + // the life of the process. + const { retained } = await retainedHeap(async () => { + const mod = await import(`../../server/richRender.ts?heap=${Date.now()}`); + const m = mod as typeof import("../../server/richRender.ts"); + // Touch every renderer that pulls in a grammar, so the number reflects a + // warmed-up server rather than a bare highlighter. + await m.renderCode({ kind: "code", code: "const x = 1;", language: "typescript" }, THEME); + await m.renderMarkdown({ kind: "markdown", markdown: markdownSource("small") }, THEME); + return m; + }); + ctx.add( + memory( + "render", + "shiki resident heap after warmup", + retained, + "themes + grammars, per process", + ), + ); + } + + // --- steady-state renders ---------------------------------------------- + const sizes: Size[] = ["small", "large"]; + + for (const size of sizes) { + const md = markdownSource(size); + const rendered = await renderMarkdown({ kind: "markdown", markdown: md }, THEME); + await ctx.time( + `markdown/${size}`, + () => renderMarkdown({ kind: "markdown", markdown: md }, THEME), + { note: `${utf8(md)} B source` }, + ); + ctx.add(bytes("render", `markdown/${size} document`, utf8(wrap(rendered)))); + } + + for (const size of sizes) { + const code = codeSource(size); + const surface = { kind: "code" as const, code, language: "typescript", title: "module.ts" }; + const rendered = await renderCode(surface, THEME); + await ctx.time(`code/${size}`, () => renderCode(surface, THEME), { + note: `${code.split("\n").length} lines`, + }); + ctx.add(bytes("render", `code/${size} document`, utf8(wrap(rendered)))); + } + + for (const size of sizes) { + const text = terminalSource(size); + const surface = { kind: "terminal" as const, text, title: "build" }; + const rendered = renderTerminal(surface); + await ctx.time(`terminal/${size}`, () => renderTerminal(surface), { + note: `${text.split("\n").length} lines with SGR codes`, + }); + ctx.add(bytes("render", `terminal/${size} document`, utf8(wrap(rendered)))); + } + + for (const size of sizes) { + const patch = diffPatch(size); + const surface = { kind: "diff" as const, patch }; + const rendered = await renderDiff(surface, THEME); + await ctx.time(`diff/${size}`, () => renderDiff(surface, THEME), { + note: `${utf8(patch)} B patch`, + // Diff SSR is the slowest renderer; a short budget here keeps the whole + // suite interactive without dropping below a usable sample count. + minSamples: 7, + minMs: 300, + }); + ctx.add(bytes("render", `diff/${size} document`, utf8(wrap(rendered)))); + } + + // --- string-wrapping paths --------------------------------------------- + // html and mermaid never touch shiki: the server only wraps the agent's + // source in a sandboxed document. They should be orders of magnitude cheaper + // than the shiki kinds — this pins that they stay that way. + for (const size of sizes) { + const html = htmlSource(size); + const build = () => + renderHtmlPage({ + title: "bench", + html, + origin: ORIGIN, + theme: THEME.theme, + mode: THEME.mode, + }); + await ctx.time(`html/${size} page wrap`, build, { note: `${utf8(html)} B source` }); + ctx.add(bytes("render", `html/${size} document`, utf8(build()))); + } + + for (const size of sizes) { + const mermaid = mermaidSource(size); + const build = () => + renderMermaidPage({ mermaid, origin: ORIGIN, theme: THEME.theme, mode: THEME.mode }); + await ctx.time(`mermaid/${size} page wrap`, build, { note: `${utf8(mermaid)} B source` }); + ctx.add(bytes("render", `mermaid/${size} document`, utf8(build()))); + } + + // --- theme switching ----------------------------------------------------- + // Switching the workspace theme invalidates every cached document at once, so + // the whole visible stream re-renders. This measures that burst for a + // representative card mix. + if (ctx.matches("theme switch re-render (10 mixed surfaces)")) { + const mix = [ + () => renderMarkdown({ kind: "markdown", markdown: markdownSource("small") }, THEME), + () => + renderCode({ kind: "code", code: codeSource("small"), language: "typescript" }, THEME), + () => renderDiff({ kind: "diff", patch: diffPatch("small") }, THEME), + () => Promise.resolve(renderTerminal({ kind: "terminal", text: terminalSource("small") })), + () => + Promise.resolve( + renderHtmlPage({ + title: "bench", + html: htmlSource("small"), + origin: ORIGIN, + theme: THEME.theme, + mode: THEME.mode, + }), + ), + ]; + ctx.add( + await time( + "render", + "theme switch re-render (10 mixed surfaces)", + async () => { + for (let i = 0; i < 10; i++) await mix[i % mix.length](); + }, + { note: "burst cost when the workspace theme changes", minSamples: 7, minMs: 300 }, + ), + ); + } + }, +}; diff --git a/bench/suites/store.bench.ts b/bench/suites/store.bench.ts new file mode 100644 index 00000000..ab7bbec0 --- /dev/null +++ b/bench/suites/store.bench.ts @@ -0,0 +1,267 @@ +// Store benchmarks: the read/write paths every request sits on top of. +// +// Both backends are measured because they have genuinely different cost curves, +// and the app can run either (`SIDESHOW_STORE=json`). Where a JSON-store number +// is dramatically worse, that IS the finding — the numbers exist to make that +// visible rather than to be quietly excused. +// +// Two methodology notes worth knowing before reading the numbers: +// +// - Steady state, not first insert. Every backend is benchmarked against a +// pre-built workspace, so the numbers describe a store people have actually +// been using, not an empty table. +// - Writes use a FIXED iteration count. A write grows the store, and on the +// JSON store the next write then costs more (it rewrites the whole file), so +// an auto-scaled loop would run a different workload on every machine. A +// fixed count keeps the accumulated growth identical everywhere. +// +// The scaling probe at the end exists because that JSON write cost is the single +// steepest curve in the codebase: it's O(workspace) per write, so it is invisible +// on a small workspace and pathological on a large one. Measuring create cost at +// three sizes shows the slope directly, instead of asking anyone to infer it from +// one number. + +import { mkdtempSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { SqlStore } from "../../server/sqlStore.ts"; +import { createSqliteStorage } from "../../server/sqliteStorage.ts"; +import { JsonFileStore } from "../../server/storage.ts"; +import type { Store } from "../../server/types.ts"; +import { buildWorkspace, surfaceOfKind, TYPICAL, type WorkspaceShape } from "../fixtures.ts"; +import { memory, retainedHeap, type Suite, type SuiteContext, time } from "../harness.ts"; + +const tmpPath = (name: string) => join(mkdtempSync(join(tmpdir(), "sideshow-bench-")), name); + +/** Ops per write benchmark. Enough samples for a stable median, few enough that + * the JSON store's quadratic growth doesn't dominate the suite's wall time. */ +const WRITE_ITERATIONS = 40; + +/** + * Shape used for the memory comparison. Deliberately smaller than the API + * suite's HEAVY: it has to be affordable on the JSON store (every write rewrites + * the file, so building 900 posts × 5 revisions there means gigabytes of I/O and + * minutes of wall time). Same shape for every backend, so the heap numbers are + * comparable to each other. + */ +const MEMORY_SHAPE: WorkspaceShape = { + sessions: 12, + postsPerSession: 25, + updatesPerPost: 1, + commentsPerSession: 15, + surfacesPerPost: 2, + size: "small", +}; + +type BackendId = "sqlite-memory" | "sqlite-file" | "json-file"; + +interface Backend { + id: BackendId; + label: string; + create: () => Store; + /** Reopen the same underlying storage — measures cold-load cost. */ + reopen?: (store: Store) => Store; +} + +const BACKENDS: Backend[] = [ + { + id: "sqlite-memory", + label: "SqlStore(:memory:)", + create: () => new SqlStore(createSqliteStorage()), + }, + { + id: "sqlite-file", + label: "SqlStore(file)", + create: () => { + const path = tmpPath("bench.db"); + const store = new SqlStore(createSqliteStorage(path)) as SqlStore & { __path: string }; + store.__path = path; + return store; + }, + reopen: (store) => + new SqlStore(createSqliteStorage((store as SqlStore & { __path: string }).__path)), + }, + { + id: "json-file", + label: "JsonFileStore", + create: () => { + const path = tmpPath("bench.json"); + const store = new JsonFileStore(path) as JsonFileStore & { __path: string }; + store.__path = path; + return store; + }, + reopen: (store) => new JsonFileStore((store as JsonFileStore & { __path: string }).__path), + }, +]; + +async function benchBackend(ctx: SuiteContext, backend: Backend) { + const store = backend.create(); + const built = await buildWorkspace(store, TYPICAL); + const label = backend.label; + const scale = `${label}, ${built.totalPosts} posts / ${built.totalComments} comments`; + + // --- reads ------------------------------------------------------------- + await ctx.time(`${backend.id}/getPost`, () => store.getPost(built.postIds[0]), { note: scale }); + + await ctx.time( + `${backend.id}/listPosts(session)`, + () => store.listPosts(built.busiestSessionId), + { + note: `${label}, ${TYPICAL.postsPerSession} posts in session`, + }, + ); + + // The whole-workspace read. Both stores hydrate every post's surfaces AND its + // history here, so this is the one most likely to dominate a large workspace. + await ctx.time(`${backend.id}/listPosts(all)`, () => store.listPosts(), { note: scale }); + + await ctx.time(`${backend.id}/listRecentPosts(20)`, () => store.listRecentPosts(20), { + note: scale, + }); + + await ctx.time( + `${backend.id}/listComments(session)`, + () => store.listComments({ sessionId: built.busiestSessionId }), + { note: scale }, + ); + + await ctx.time(`${backend.id}/listSessions`, () => store.listSessions(), { + note: `${label}, ${TYPICAL.sessions} sessions`, + }); + + if (store.countPostsBySession) { + const countPosts = store.countPostsBySession.bind(store); + await ctx.time(`${backend.id}/countPostsBySession`, () => countPosts(), { note: scale }); + } + + // isAssetReferenced scans every post's surfaces + history looking for the id. + // It runs on the asset-serving path, so its cost is proportional to workspace + // size on every image load — worth watching explicitly. + await ctx.time( + `${backend.id}/isAssetReferenced(miss)`, + () => store.isAssetReferenced("asset-that-does-not-exist"), + { note: scale }, + ); + + // --- writes ------------------------------------------------------------ + const surface = surfaceOfKind("markdown", "small"); + await ctx.time( + `${backend.id}/createPost`, + () => + store.createPost({ + sessionId: built.busiestSessionId, + title: "bench post", + surfaces: [surface] as never, + }), + { note: scale, iterations: WRITE_ITERATIONS }, + ); + + const target = built.postIds[Math.floor(built.postIds.length / 2)]; + let rev = 0; + await ctx.time( + `${backend.id}/updatePost`, + () => store.updatePost(target, { title: `rev ${rev++}` }), + { note: `${label}, history at cap`, iterations: WRITE_ITERATIONS }, + ); + + await ctx.time( + `${backend.id}/createComment`, + () => + store.createComment({ + sessionId: built.busiestSessionId, + author: "user", + text: "bench comment", + }), + { note: scale, iterations: WRITE_ITERATIONS }, + ); + + // --- cold open --------------------------------------------------------- + // What a server restart pays before serving its first request. The JSON store + // parses the entire workspace file here; SQLite opens a handle and runs its + // migration probes. + if (backend.reopen) { + const reopen = backend.reopen; + await ctx.time( + `${backend.id}/cold open + first read`, + async () => { + const fresh = reopen(store); + await fresh.listRecentPosts(20); + }, + { note: scale, minSamples: 7, minMs: 300 }, + ); + } +} + +/** + * How create cost grows with workspace size. Reported per backend at three + * sizes: a flat line means the write cost is independent of what's already + * stored, a rising one means every existing post is being paid for again. + */ +async function benchWriteScaling(ctx: SuiteContext, backend: Backend) { + for (const existing of [50, 200, 500]) { + const name = `${backend.id}/createPost @ ${existing} posts`; + if (!ctx.matches(name)) continue; + const store = backend.create(); + const session = await store.createSession({ agent: "bench", title: "scaling" }); + const surface = surfaceOfKind("markdown", "small"); + for (let i = 0; i < existing; i++) { + await store.createPost({ + sessionId: session.id, + title: `seed ${i}`, + surfaces: [surface] as never, + }); + } + ctx.add( + await time( + "store", + name, + () => + store.createPost({ + sessionId: session.id, + title: "probe", + surfaces: [surface] as never, + }), + { note: `${backend.label}, workspace already holds ${existing} posts`, iterations: 20 }, + ), + ); + } +} + +export const storeSuite: Suite = { + name: "store", + description: "Store read/write paths per backend, plus how write cost scales with workspace size", + async run(ctx) { + for (const backend of BACKENDS) await benchBackend(ctx, backend); + for (const backend of BACKENDS) await benchWriteScaling(ctx, backend); + + // --- memory ------------------------------------------------------------ + // How much heap a loaded workspace costs. This is the direct answer to "why + // does the sideshow server hold so much memory?" — the JSON store keeps the + // entire workspace resident by design; SQLite should not. + for (const backend of BACKENDS) { + const loadedName = `${backend.id}/heap for loaded workspace`; + const listName = `${backend.id}/heap for listPosts(all) result`; + if (!ctx.matches(loadedName) && !ctx.matches(listName)) continue; + + const shapeNote = `${backend.label}, ${MEMORY_SHAPE.sessions}×${MEMORY_SHAPE.postsPerSession} posts`; + const { retained, value: store } = await retainedHeap(async () => { + const fresh = backend.create(); + await buildWorkspace(fresh, MEMORY_SHAPE); + return fresh; + }); + ctx.add(memory("store", loadedName, retained, shapeNote)); + + // A whole-workspace read materializes every post as JS objects. This is the + // transient cost of any route that fans out over the workspace. + const listed = await retainedHeap(() => store.listPosts()); + ctx.add( + memory( + "store", + listName, + listed.retained, + `${shapeNote}, ${listed.value.length} materialized`, + ), + ); + } + }, +}; diff --git a/bench/suites/viewer.bench.ts b/bench/suites/viewer.bench.ts new file mode 100644 index 00000000..6c7f3b1c --- /dev/null +++ b/bench/suites/viewer.bench.ts @@ -0,0 +1,358 @@ +// Viewer benchmarks — the browser half of the CPU/memory story. +// +// The server can be fast and the product can still feel heavy: the viewer holds +// one sandboxed iframe per rendered surface, keeps an SSE connection open for the +// life of the tab, and re-renders on every live update. That's where a user's fan +// actually spins up, and none of the Node-side suites can see it. +// +// Measured through Chrome DevTools Protocol rather than wall-clock timers, +// because CDP exposes the numbers that are both meaningful and comparatively +// stable across runs: +// +// ScriptDuration / TaskDuration CPU seconds actually spent — the complaint, +// quantified. +// LayoutCount / RecalcStyleCount Deterministic-ish work counters. These catch +// a render-thrash regression that a timing +// number would bury in noise. +// JSHeapUsedSize / Nodes What the tab retains while it sits open. +// +// Chromium only: WebKit has no equivalent metrics channel. Correctness on WebKit +// is covered by the e2e suite; this is about cost, and the cost profile we can +// measure is the one worth tracking. + +import { spawn } from "node:child_process"; +import { existsSync, mkdtempSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; +// Type-only: erased at runtime, so the default `npm run bench` still never loads +// Playwright (the value import below is dynamic and lives inside run()). +import type { CDPSession } from "@playwright/test"; +import { KIND_MIX, surfaceOfKind } from "../fixtures.ts"; +import { bytes, count, memory, type Suite, type SuiteContext } from "../harness.ts"; + +const repoRoot = fileURLToPath(new URL("../..", import.meta.url)); +const viewerBundle = join(repoRoot, "viewer", "dist", "index.html"); + +/** Posts seeded into the benchmarked session. Enough to make per-card cost visible. */ +const POSTS = 30; +/** Live updates pushed while the tab is open, to measure steady-state churn. */ +const LIVE_UPDATES = 20; + +interface Server { + url: string; + stop: () => void; +} + +function bootServer(): Promise { + const dir = mkdtempSync(join(tmpdir(), "sideshow-bench-viewer-")); + const proc = spawn(process.execPath, ["server/index.ts"], { + cwd: repoRoot, + env: { + ...process.env, + PORT: "0", + SIDESHOW_DB: join(dir, "bench.db"), + SIDESHOW_DATA: join(dir, "bench.json"), + SIDESHOW_VERSION: "", + SIDESHOW_TOKEN: "", + }, + stdio: ["ignore", "pipe", "ignore"], + }); + return new Promise((resolve, reject) => { + let out = ""; + const timer = setTimeout(() => { + proc.kill(); + reject(new Error("server did not boot in time")); + }, 30_000); + proc.stdout?.on("data", (chunk: Buffer) => { + out += chunk.toString(); + const m = out.match(/listening on (http:\/\/localhost:\d+)/); + if (m) { + clearTimeout(timer); + resolve({ url: m[1], stop: () => proc.kill() }); + } + }); + proc.on("exit", (code) => { + clearTimeout(timer); + reject(new Error(`server exited early (${code})`)); + }); + }); +} + +async function publish(url: string, sessionId: string | undefined, index: number) { + const res = await fetch(`${url}/api/posts`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + ...(sessionId ? { session: sessionId } : { agent: "bench", sessionTitle: "Viewer bench" }), + title: `Post ${index}`, + parts: [surfaceOfKind(KIND_MIX[index % KIND_MIX.length], "small", index % 4)], + }), + }); + if (!res.ok) throw new Error(`publish failed: ${res.status}`); + return (await res.json()) as { id: string; sessionId: string }; +} + +type Metrics = Record; + +const readMetrics = async (cdp: CDPSession): Promise => { + const { metrics } = await cdp.send("Performance.getMetrics"); + return Object.fromEntries(metrics.map((m) => [m.name, m.value])); +}; + +/** + * Card-count predicate as a source string rather than a closure. The node + * typecheck program has no DOM lib (correctly — this file runs in Node), so a + * closure referencing `document` would not compile even though it only ever + * executes in the browser. + */ +const cardsPresent = (selector: string, n: number) => + `document.querySelectorAll(${JSON.stringify(selector)}).length >= ${n}`; + +const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms)); + +export const viewerSuite: Suite = { + name: "viewer", + description: "Browser-side cost of the viewer: CPU, layout work, heap, and DOM size", + // Needs a built viewer and a Chromium launch; kept out of the default run. + optional: true, + async run(ctx: SuiteContext) { + if (!existsSync(viewerBundle)) { + console.error( + `viewer bundle missing at ${viewerBundle} — run \`npm run build:viewer\` first; skipping viewer suite.`, + ); + return; + } + + // Imported lazily so the default `npm run bench` never loads Playwright. + const { chromium } = await import("@playwright/test"); + + // Some environments ship a Chromium that doesn't match the pinned Playwright + // revision. SIDESHOW_BENCH_CHROMIUM points at one explicitly rather than + // forcing a download; without it we use whatever Playwright resolves. + const executablePath = process.env.SIDESHOW_BENCH_CHROMIUM || undefined; + const launch = () => chromium.launch({ executablePath }); + + // Probe the launch before booting a server, so a missing browser is a clean + // skip rather than a crash that takes down a whole `--all` run. + try { + await (await launch()).close(); + } catch (err) { + console.error( + `skipping viewer suite: cannot launch Chromium (${(err as Error).message.split("\n")[0]}).\n` + + `Set SIDESHOW_BENCH_CHROMIUM to a Chromium binary, or run \`npx playwright install chromium\`.`, + ); + return; + } + + const server = await bootServer(); + let browser: Awaited> | null = null; + try { + const first = await publish(server.url, undefined, 0); + const sessionId = first.sessionId; + for (let i = 1; i < POSTS; i++) await publish(server.url, sessionId, i); + + browser = await launch(); + const context = await browser.newContext({ viewport: { width: 1280, height: 900 } }); + const page = await context.newPage(); + const cdp = await context.newCDPSession(page); + await cdp.send("Performance.enable"); + + // Byte accounting through Playwright's response event, which follows every + // frame — including the sandboxed surface iframes. Neither of the obvious + // alternatives covers them: the top document's PerformanceResourceTiming + // reports transferSize 0 for opaque-origin frames, and a page-scoped CDP + // session never sees the frames that site isolation puts in their own + // process. Those frames are the bytes most worth counting here. + const responseSizes: Promise[] = []; + page.on("response", (response) => { + // /api/events is the SSE stream: it stays open for the life of the tab, so + // its sizes() never settles and awaiting it would hang the whole bench. + // Its bytes are a keepalive ping every 15s — nothing this metric is about. + if (new URL(response.url()).pathname === "/api/events") return; + responseSizes.push( + Promise.race([ + response + .request() + .sizes() + .then((s) => s.responseBodySize + s.responseHeadersSize), + // Belt and braces: any other request that never finishes contributes + // zero instead of stalling the run. + new Promise((resolve) => setTimeout(() => resolve(0), 5000)), + ]).catch(() => 0), + ); + }); + + // --- initial load ------------------------------------------------------- + const cardSelector = ".card:not(#whatsNew)"; + const startedAt = Date.now(); + await page.goto(`${server.url}/session/${sessionId}`, { waitUntil: "load" }); + await page.waitForFunction(cardsPresent(cardSelector, POSTS), undefined, { + timeout: 30_000, + }); + const loadMs = Date.now() - startedAt; + + // Let the surface iframes finish loading and settle before sampling, so the + // numbers describe a rendered stream rather than one mid-flight. + await page.waitForLoadState("networkidle").catch(() => {}); + await sleep(1500); + + const afterLoad = await readMetrics(cdp); + ctx.add({ + suite: "viewer", + name: `load ${POSTS}-post session to all cards`, + kind: "time", + unit: "ms/op", + value: loadMs, + note: "navigation → every card in the DOM", + // A single unrepeatable navigation: noisier than a sampled benchmark. + tolerance: 1.8, + }); + ctx.add({ + suite: "viewer", + name: "script CPU for initial load", + kind: "time", + unit: "ms/op", + value: (afterLoad.ScriptDuration ?? 0) * 1000, + note: `${POSTS} posts, CDP ScriptDuration`, + tolerance: 1.6, + }); + ctx.add({ + suite: "viewer", + name: "total task CPU for initial load", + kind: "time", + unit: "ms/op", + value: (afterLoad.TaskDuration ?? 0) * 1000, + note: "CDP TaskDuration (script + layout + paint)", + tolerance: 1.6, + }); + ctx.add( + memory( + "viewer", + "JS heap after load", + afterLoad.JSHeapUsedSize ?? 0, + `${POSTS} posts rendered`, + ), + ); + ctx.add(count("viewer", "DOM nodes after load", afterLoad.Nodes ?? 0, `${POSTS} posts`, 1.1)); + ctx.add( + count( + "viewer", + "layout count for initial load", + afterLoad.LayoutCount ?? 0, + undefined, + 1.25, + ), + ); + ctx.add( + count( + "viewer", + "style recalcs for initial load", + afterLoad.RecalcStyleCount ?? 0, + undefined, + 1.25, + ), + ); + + const iframes = await page.locator(`${cardSelector} iframe`).count(); + ctx.add( + count( + "viewer", + "surface iframes", + iframes, + "one opaque-origin frame per sandboxed surface", + ), + ); + + const transferred = (await Promise.all(responseSizes)).reduce((a, b) => a + b, 0); + ctx.add( + bytes( + "viewer", + "bytes transferred for load", + transferred, + "viewer bundle + API + every surface iframe", + ), + ); + + // --- live churn ---------------------------------------------------------- + // The steady state of a tab left open next to a working agent: posts stream + // in over SSE and the stream re-renders. The two recent viewer perf fixes + // (coalescing refreshes, compacting updates) both live on this path, so it + // gets its own measurement rather than being folded into load cost. + const beforeChurn = await readMetrics(cdp); + for (let i = 0; i < LIVE_UPDATES; i++) await publish(server.url, sessionId, POSTS + i); + await page.waitForFunction(cardsPresent(cardSelector, POSTS + LIVE_UPDATES), undefined, { + timeout: 30_000, + }); + await sleep(1500); + const afterChurn = await readMetrics(cdp); + + const churnScript = + ((afterChurn.ScriptDuration ?? 0) - (beforeChurn.ScriptDuration ?? 0)) * 1000; + const churnTask = ((afterChurn.TaskDuration ?? 0) - (beforeChurn.TaskDuration ?? 0)) * 1000; + ctx.add({ + suite: "viewer", + name: "script CPU per live post", + kind: "time", + unit: "ms/op", + value: churnScript / LIVE_UPDATES, + note: `${LIVE_UPDATES} posts streamed into an open tab`, + tolerance: 1.6, + }); + ctx.add({ + suite: "viewer", + name: "total task CPU per live post", + kind: "time", + unit: "ms/op", + value: churnTask / LIVE_UPDATES, + note: "CDP TaskDuration delta / posts", + tolerance: 1.6, + }); + ctx.add( + count( + "viewer", + "layouts per live post", + Math.round( + ((afterChurn.LayoutCount ?? 0) - (beforeChurn.LayoutCount ?? 0)) / LIVE_UPDATES, + ), + "re-layout churn per streamed post", + 1.3, + ), + ); + ctx.add( + memory( + "viewer", + "JS heap growth per live post", + Math.max( + 0, + ((afterChurn.JSHeapUsedSize ?? 0) - (beforeChurn.JSHeapUsedSize ?? 0)) / LIVE_UPDATES, + ), + "retention drift while a tab stays open", + // Heap sampling without a forced GC in the page is inherently jumpy. + 2, + ), + ); + + // --- idle ----------------------------------------------------------------- + // A tab nobody is looking at should cost ~nothing: the SSE keepalive is one + // frame every 15s. Anything meaningful here is a polling or timer leak. + const beforeIdle = await readMetrics(cdp); + await sleep(5000); + const afterIdle = await readMetrics(cdp); + ctx.add({ + suite: "viewer", + name: "idle task CPU per second", + kind: "time", + unit: "ms/op", + value: (((afterIdle.TaskDuration ?? 0) - (beforeIdle.TaskDuration ?? 0)) * 1000) / 5, + note: "open tab, no activity — should be near zero", + tolerance: 2.5, + }); + + await context.close(); + } finally { + await browser?.close(); + server.stop(); + } + }, +}; diff --git a/package.json b/package.json index ee3d1e61..6cda9a80 100644 --- a/package.json +++ b/package.json @@ -78,6 +78,10 @@ "coverage:node": "c8 node --test 'test/**/*.test.ts'", "coverage:viewer": "vitest run --coverage --config vitest.viewer.config.ts", "test:e2e": "playwright test", + "bench": "node --expose-gc bench/run.ts", + "bench:all": "node --expose-gc bench/run.ts --all", + "bench:check": "node --expose-gc bench/run.ts --check", + "bench:baseline": "node --expose-gc bench/run.ts --baseline", "format": "oxfmt --write .", "format:check": "oxfmt --check .", "lint": "oxlint . --deny-warnings", diff --git a/test/bench.test.ts b/test/bench.test.ts new file mode 100644 index 00000000..af614d85 --- /dev/null +++ b/test/bench.test.ts @@ -0,0 +1,179 @@ +// The benchmark suite's gate decides whether CI goes red, so the deciding logic +// gets tested like any other code. These cover the parts where a subtle mistake +// would be invisible in practice: a threshold that never fires, a floor that +// suppresses a real regression, or machine scaling applied to a metric that +// isn't machine-dependent. + +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { + compareRuns, + DEFAULT_THRESHOLDS, + machineScale, + resultKey, + type BenchRun, +} from "../bench/compare.ts"; +import { bytes, count, memory, time } from "../bench/harness.ts"; +import { buildWorkspace, markdownSource, rng, TYPICAL } from "../bench/fixtures.ts"; +import { SqlStore } from "../server/sqlStore.ts"; +import { createSqliteStorage } from "../server/sqliteStorage.ts"; +import type { BenchResult } from "../bench/harness.ts"; + +const run = (results: BenchResult[], index = 1000): BenchRun => ({ + format: 1, + recordedAt: "2026-01-01T00:00:00.000Z", + machine: { + platform: "linux", + arch: "x64", + cpus: 8, + cpuModel: "test", + nodeVersion: "v22.0.0", + totalMemory: 1, + index, + }, + results, +}); + +const timeResult = (name: string, value: number, tolerance?: number): BenchResult => ({ + suite: "s", + name, + kind: "time", + unit: "ms/op", + value, + tolerance, +}); + +test("a deterministic metric fails on a small change; a timing does not", () => { + // 10% is nothing for a timing and everything for a byte count. Both directions + // of that asymmetry matter: gate bytes loosely and payload bloat sails through; + // gate timings tightly and CI is red on runner noise alone. + const base = run([bytes("s", "payload", 10_000), timeResult("op", 10)]); + const cur = run([bytes("s", "payload", 11_000), timeResult("op", 11)]); + const { regressions } = compareRuns(base, cur); + assert.deepEqual( + regressions.map((r) => r.name), + ["payload"], + ); +}); + +test("the absolute floor suppresses a large ratio on a tiny value", () => { + // 10µs → 30µs is 3×, but nobody can feel 20µs and CI runners produce swings + // this size for free. The floor is what keeps microbenchmarks from becoming + // the loudest thing in the report. + const base = run([timeResult("tiny", 0.01)]); + const cur = run([timeResult("tiny", 0.03)]); + assert.equal(compareRuns(base, cur).regressions.length, 0); + + // The same 3× on a value above the floor does fail. + const bigBase = run([timeResult("real", 10)]); + const bigCur = run([timeResult("real", 30)]); + assert.equal(compareRuns(bigBase, bigCur).regressions.length, 1); +}); + +test("a per-metric tolerance overrides the default", () => { + // 2.5× would fail the default 1.4×, but this metric declares itself noisy. + // The tolerance is read from the CURRENT result, so loosening a gate is a + // visible source change rather than a quiet baseline edit. + const base = run([timeResult("noisy", 10)]); + const cur = run([timeResult("noisy", 25, 3)]); + assert.equal(compareRuns(base, cur).regressions.length, 0); + + // Without the declaration, the same change fails. + assert.equal(compareRuns(base, run([timeResult("noisy", 25)])).regressions.length, 1); +}); + +test("machine scaling adjusts timings but never bytes, counts, or memory", () => { + const base = run( + [bytes("s", "b", 1000), count("s", "c", 10), memory("s", "m", 10_000_000), timeResult("t", 10)], + 1000, + ); + // Half the index = half the speed: the same code should take twice as long. + const cur = run( + [bytes("s", "b", 1000), count("s", "c", 10), memory("s", "m", 10_000_000), timeResult("t", 20)], + 500, + ); + const { comparisons, regressions, scale } = compareRuns(base, cur); + assert.equal(scale, 2); + assert.equal(regressions.length, 0, "a 2x slower machine is not a 2x regression"); + + const byName = new Map(comparisons.map((c) => [c.name, c])); + assert.equal(byName.get("t")!.expected, 20, "timing baseline is scaled"); + assert.equal(byName.get("b")!.expected, 1000, "bytes are not scaled"); + assert.equal(byName.get("c")!.expected, 10, "counts are not scaled"); + assert.equal(byName.get("m")!.expected, 10_000_000, "memory is not scaled"); +}); + +test("an implausible or missing machine index is ignored rather than trusted", () => { + // A wildly different index means the calibration itself was disturbed. Scaling + // by it would quietly excuse (or invent) an arbitrary regression, so it is + // discarded in favour of an unscaled comparison. + assert.equal(machineScale(run([], 1000), run([], 1)), 1); + assert.equal(machineScale(run([], 1000), run([], 0)), 1); + assert.equal(machineScale(run([], 0), run([], 1000)), 1); + assert.equal(machineScale(run([], 1000), run([], 2000)), 0.5); +}); + +test("new and removed metrics are reported, and neither fails the gate", () => { + const base = run([timeResult("gone", 10)]); + const cur = run([timeResult("fresh", 10)]); + const { comparisons, regressions } = compareRuns(base, cur); + assert.equal(regressions.length, 0); + const verdicts = Object.fromEntries(comparisons.map((c) => [c.name, c.verdict])); + assert.deepEqual(verdicts, { fresh: "new", gone: "missing" }); +}); + +test("an improvement is labeled, not just ignored", () => { + const base = run([bytes("s", "payload", 10_000)]); + const cur = run([bytes("s", "payload", 5_000)]); + const { comparisons, regressions } = compareRuns(base, cur); + assert.equal(regressions.length, 0); + assert.equal(comparisons[0].verdict, "improved"); +}); + +test("thresholds are ordered by how trustworthy each metric kind is", () => { + // Pins the intent rather than the exact constants: deterministic metrics must + // always be gated more tightly than noisy ones, whatever the numbers become. + assert.ok(DEFAULT_THRESHOLDS.bytes.ratio < DEFAULT_THRESHOLDS.memory.ratio); + assert.ok(DEFAULT_THRESHOLDS.memory.ratio < DEFAULT_THRESHOLDS.time.ratio); +}); + +test("resultKey namespaces by suite so two suites can share a metric name", () => { + assert.equal(resultKey({ suite: "store", name: "getPost" }), "store/getPost"); + assert.notEqual( + resultKey({ suite: "api", name: "getPost" }), + resultKey({ suite: "store", name: "getPost" }), + ); +}); + +test("time() reports a median and honours a fixed iteration count", async () => { + let calls = 0; + const result = await time("s", "counted", () => calls++, { iterations: 12 }); + assert.equal(calls, 12, "no warmup ops added to a fixed-iteration bench"); + assert.equal(result.stats?.samples, 12); + assert.equal(result.kind, "time"); + assert.ok(result.value >= 0); +}); + +test("fixtures are deterministic — the whole point of comparing across machines", async () => { + assert.equal(markdownSource("small", 7), markdownSource("small", 7)); + assert.notEqual(markdownSource("small", 7), markdownSource("small", 8)); + assert.notEqual(markdownSource("small"), markdownSource("large")); + + const a = rng(3); + const b = rng(3); + assert.deepEqual([a(), a(), a()], [b(), b(), b()]); +}); + +test("buildWorkspace produces the shape it promises", async () => { + const shape = { ...TYPICAL, sessions: 2, postsPerSession: 3, commentsPerSession: 4 }; + const store = new SqlStore(createSqliteStorage()); + const built = await buildWorkspace(store, shape); + assert.equal(built.sessionIds.length, 2); + assert.equal(built.totalPosts, 6); + assert.equal(built.totalComments, 8); + assert.equal((await store.listPosts()).length, 6); + // Revisions actually applied, so history-dependent benchmarks aren't measuring + // a workspace of untouched posts. + const post = await store.getPost(built.postIds[0]); + assert.equal(post?.version, shape.updatesPerPost + 1); +}); diff --git a/tsconfig.json b/tsconfig.json index 75629171..e7528082 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -14,5 +14,5 @@ "skipLibCheck": true, "forceConsistentCasingInFileNames": true }, - "include": ["server", "mcp", "test"] + "include": ["server", "mcp", "test", "bench"] } From b26e1549ed2d107b948ddc6b93fab775e911e80e Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 15 Aug 2026 03:30:05 +0000 Subject: [PATCH 2/5] perf(server): load rich renderers and parsers on first use MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The benchmark suite's process attribution showed a server holding ~132 MB before serving a request, with ~72 MB of it attributable to importing server/app.ts — mostly shiki, @pierre/diffs, markdown-it and @mermaid-js/parser, loaded at boot whether or not the workspace ever used them. Two dynamic imports, both at the point of use: - app.ts loads richRender.ts when it renders a markdown/code/diff/terminal surface. It sits after the html and mermaid branches, so an html-only workspace never loads any of it. - postSurfaces.ts loads @pierre/diffs and @mermaid-js/parser when validating a published diff or mermaid surface. Both halves were needed: leaving either static keeps the whole graph resident and neither saves anything. Idle RSS 132 MB -> 102 MB, boot 486 ms -> 275 ms. The mermaid parser and the diff parsers now load outside the try/catch that reports parse failures, so a module-load error can't be reported to the user as invalid mermaid (a 400 where it should be a 500). A dynamic import is the kind of change that works on Node and fails only once deployed, so test/workerIntegration now renders all four rich surface kinds on real workerd and asserts on markup only the real renderers emit — the existing html render deliberately never reaches that path. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01MGMuudU8anVrxB7sjY9hpt --- .changeset/lazy-rich-render-imports.md | 10 +++++ bench/suites/process.bench.ts | 20 +++++++++- server/app.ts | 16 +++++++- server/postSurfaces.ts | 30 ++++++++++++-- test/workerIntegration.integration.ts | 55 +++++++++++++++++++++++++- 5 files changed, 124 insertions(+), 7 deletions(-) create mode 100644 .changeset/lazy-rich-render-imports.md diff --git a/.changeset/lazy-rich-render-imports.md b/.changeset/lazy-rich-render-imports.md new file mode 100644 index 00000000..11c95899 --- /dev/null +++ b/.changeset/lazy-rich-render-imports.md @@ -0,0 +1,10 @@ +--- +"sideshow": patch +--- + +Load the rich-surface renderers and publish-time parsers on first use instead of +at boot. shiki, `@pierre/diffs`, markdown-it and `@mermaid-js/parser` are now +imported when a markdown/code/diff/terminal surface is rendered or a diff/mermaid +surface is published, rather than by every server at startup. Idle memory drops +from ~132 MB to ~102 MB and boot time from ~486 ms to ~275 ms; a workspace that +only ever uses html surfaces never loads them at all. No behavior change. diff --git a/bench/suites/process.bench.ts b/bench/suites/process.bench.ts index 5cf36eea..83951c2a 100644 --- a/bench/suites/process.bench.ts +++ b/bench/suites/process.bench.ts @@ -134,14 +134,32 @@ function importCost(specifier: string | null): Promise<{ rss: number; ms: number }); } -/** Modules that dominate the server's module graph, loaded cheapest-first. */ +/** + * Modules that dominate the server's module graph, loaded cheapest-first. + * + * Reading these: subtract the `node baseline` row to get a module's own cost, and + * note that any row importing a LOCAL `.ts` file also carries Node's type-stripping + * overhead — the transpiler loads on the first `.ts` import and costs ~25 MB by + * itself, which is why `server/types.ts` is in the list as a floor to subtract. + * That overhead is a dev-run artifact: the published package ships compiled `.js` + * in `dist/`, so an installed CLI never pays it. The `server RSS idle` number + * above does include it, because it runs `server/index.ts` from source. + * + * The npm-package rows have no such caveat and are directly comparable. + */ const IMPORT_TARGETS: [name: string, specifier: string | null][] = [ ["node baseline (no imports)", null], + ["server/types.ts (type-strip floor)", "./server/types.ts"], ["hono", "hono"], ["markdown-it", "markdown-it"], ["shiki", "shiki"], + ["@mermaid-js/parser", "@mermaid-js/parser"], ["@pierre/diffs", "@pierre/diffs"], + // The two modules deliberately kept OFF the boot path (see the dynamic imports + // in app.ts and postSurfaces.ts). If either becomes a static import again, + // server/app.ts jumps to roughly the richRender row and this suite shows it. ["server/richRender.ts", "./server/richRender.ts"], + ["server/postSurfaces.ts", "./server/postSurfaces.ts"], ["server/app.ts", "./server/app.ts"], ]; diff --git a/server/app.ts b/server/app.ts index 92c9eedb..66c89b10 100644 --- a/server/app.ts +++ b/server/app.ts @@ -23,7 +23,6 @@ import { renderMermaidPage, renderSandboxedPart, } from "./surfacePage.ts"; -import { renderCode, renderDiff, renderMarkdown, renderTerminal } from "./richRender.ts"; import { DEFAULT_THEME_ID, themeById, themeOptions } from "./themes.ts"; import { type Asset, @@ -1571,6 +1570,21 @@ export function createApp({ if (surface.kind === "mermaid") { return renderMermaidPage({ mermaid: surface.mermaid, origin, theme, mode }); } + // Load the rich renderers on first use, not at module load. richRender.ts + // pulls in shiki, @pierre/diffs, markdown-it and ansi_up — measured at ~48 MB + // of RSS and ~240 ms of import time (`npm run bench:all`, process suite), which + // every server paid at boot whether or not it ever rendered a rich surface. + // Deferring it past the html and mermaid branches above means an html-only + // workspace never loads any of it. + // + // The runtime's module cache makes every later call cheap, so there's no memo + // here to keep in sync. On the Worker the module is already inside the + // deployed bundle — the import defers evaluating it, not fetching it — so this + // needs no network at runtime. test/workerIntegration covers that on real + // workerd, because a dynamic import resolving differently there is exactly the + // way this optimization could break in production and nowhere else. + const { renderCode, renderDiff, renderMarkdown, renderTerminal } = + await import("./richRender.ts"); const rendered = surface.kind === "markdown" ? await renderMarkdown(surface as MarkdownSurface, { theme: themeId, mode }) diff --git a/server/postSurfaces.ts b/server/postSurfaces.ts index 3a051a39..6bd3d85c 100644 --- a/server/postSurfaces.ts +++ b/server/postSurfaces.ts @@ -1,6 +1,4 @@ import { z } from "zod"; -import { processFile, parsePatchFiles } from "@pierre/diffs"; -import { parse as parseMermaid } from "@mermaid-js/parser"; import { isKnownKit, KIT_IDS } from "./kits.ts"; import { isSurfaceKind, type Surface, type SurfaceKind } from "./types.ts"; @@ -308,7 +306,16 @@ export async function validateSurfaces( // class, state, er, gantt, …) are not yet in that package, so we skip // validation for them — the viewer's existing graceful fallback handles any // render failure. No `node:` imports, no DOM usage on the parse path. -function diffPatchHasContent(patch: string): boolean { +// +// Both parsers are imported lazily, at their point of use in validateSemantics +// below — see the note there. + +// Takes the parsers rather than importing them, so the caller can load the module +// OUTSIDE its try/catch — see the mermaid branch below for why that matters. +function diffPatchHasContent( + patch: string, + { processFile, parsePatchFiles }: typeof import("@pierre/diffs"), +): boolean { let files = 0; for (const parsed of parsePatchFiles(patch)) files += parsed.files.length; if (files > 0) return true; @@ -328,10 +335,21 @@ function mermaidDiagramType(src: string): string | null { return null; } +// The two parsers load at their point of use, not at module load. Together they +// are ~51 MB of RSS and ~210 ms of import time (`npm run bench:all`, process +// suite), and they are only reachable when someone publishes a diff or a mermaid +// surface — a static import made every server pay that at boot to validate content +// it may never receive. The runtime's module cache makes later publishes cheap, +// and @pierre/diffs is shared with richRender.ts, so a workspace that renders +// diffs loads it once for both. +// +// This is the other half of app.ts's lazy richRender import: leaving either half +// static keeps the whole dependency graph resident and neither one saves anything. async function validateSemantics(surface: Surface): Promise { if (surface.kind === "diff" && surface.patch) { + const diffParsers = await import("@pierre/diffs"); try { - if (!diffPatchHasContent(surface.patch)) + if (!diffPatchHasContent(surface.patch, diffParsers)) return [ 'diff surface "patch" did not parse to any file — expected a unified/git patch with --- /+++ headers and @@ hunks', ]; @@ -345,6 +363,10 @@ async function validateSemantics(surface: Surface): Promise { const diagramType = mermaidDiagramType(surface.mermaid); if (!diagramType) return ['mermaid surface has no diagram type (first line should be e.g. "flowchart TD")']; + // Outside the try: the catch below turns a throw into "your mermaid is + // invalid", which would be a wrong answer (and a 400 instead of a 500) if what + // actually failed was loading the parser. + const { parse: parseMermaid } = await import("@mermaid-js/parser"); try { await parseMermaid(diagramType as never, surface.mermaid); } catch (e) { diff --git a/test/workerIntegration.integration.ts b/test/workerIntegration.integration.ts index 5f4e67cb..3b16e6a9 100644 --- a/test/workerIntegration.integration.ts +++ b/test/workerIntegration.integration.ts @@ -172,6 +172,57 @@ test( assert.equal(rendered.headers.get("x-content-type-options"), "nosniff"); assert.match(rendered.headers.get("cache-control") ?? "", /immutable/); + // Rich surfaces are the ones that pull in richRender.ts, and app.ts imports it + // DYNAMICALLY so a server that never renders one doesn't pay ~48 MB of RSS for + // shiki/@pierre/diffs at boot. A dynamic import is the kind of thing that works + // on Node and fails only once deployed, so it gets exercised on real workerd + // here — the html render above deliberately never reaches that code path. + // + // Each assertion looks for markup only the real renderer emits (shiki's span + // classes, ansi_up's inline colors, the diff web component), so a renderer that + // loaded but silently produced a fallback still fails. + const richPost = await expectJson( + await worker.fetch( + "/api/posts", + json({ + session: post.sessionId, + title: "Rich surfaces", + surfaces: [ + { kind: "markdown", markdown: "# Heading\n\n```ts\nconst x: number = 1;\n```" }, + { kind: "code", code: "export const y = 2;", language: "typescript" }, + { kind: "terminal", text: "\u001b[31mred\u001b[0m plain" }, + { + kind: "diff", + patch: [ + "diff --git a/a.ts b/a.ts", + "--- a/a.ts", + "+++ b/a.ts", + "@@ -1,2 +1,2 @@", + " const keep = 1;", + "-const before = 2;", + "+const after = 3;", + ].join("\n"), + }, + ], + }), + ), + 201, + ); + const richExpectations: Array<[kind: string, pattern: RegExp]> = [ + ["markdown", /

Heading<\/h1>/], + ["code", /class="shiki/], + ["terminal", /rgb\(/], + ["diff", /diffs-container/], + ]; + for (const [index, [kind, pattern]] of richExpectations.entries()) { + const page = await worker.fetch(`/p/${richPost.id}?surface=${index}&theme=github&mode=dark`, { + headers: AUTH, + }); + const body = await page.text(); + assert.equal(page.status, 200, `${kind} surface failed to render: ${body.slice(0, 400)}`); + assert.match(body, pattern, `${kind} surface rendered without the real renderer's markup`); + } + assert.equal((await worker.fetch(`/p/${post.id}.png?card=1`, { method: "HEAD" })).status, 401); const screenshot = await worker.fetch(`/p/${post.id}.png?card=1`, { method: "HEAD", @@ -344,9 +395,11 @@ test( await worker.fetch("/api/sessions", { headers: AUTH }), 200, ); + // Two posts: the html one this test drives throughout, plus the rich-surface + // post published above to exercise the lazily-imported renderers. assert.deepEqual( sessions.map(({ id, postCount }) => ({ id, postCount })), - [{ id: post.sessionId, postCount: 1 }], + [{ id: post.sessionId, postCount: 2 }], ); }, ); From f4ce9bbfdf3b5525789fc80e542386abd04b885e Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 15 Aug 2026 04:30:02 +0000 Subject: [PATCH 3/5] fix(bench): validate ids from HTTP responses before using them in URLs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CodeQL flagged one high-severity alert on the benchmark suite: a post id parsed out of a publish response flowed straight into the URL of the next request (request forgery). The viewer suite did the same with a session id. The bench only ever talks to a server it spawned itself, so this was not a live vulnerability — but the shape genuinely was unverified, and the unchecked version also fails confusingly: when a publish returns an error body instead of a post, the next line 404s rather than reporting what actually went wrong. Both ids are now checked against the url-safe base64 shape newId produces, and throw with the offending value when they don't match. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01MGMuudU8anVrxB7sjY9hpt --- bench/suites/process.bench.ts | 20 +++++++++++++++++++- bench/suites/viewer.bench.ts | 7 +++++++ 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/bench/suites/process.bench.ts b/bench/suites/process.bench.ts index 83951c2a..e8e3bcfd 100644 --- a/bench/suites/process.bench.ts +++ b/bench/suites/process.bench.ts @@ -103,6 +103,24 @@ function rssOf(pid: number): Promise { const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms)); +/** + * Check that an id parsed out of an HTTP response looks like one before it goes + * into another request's URL (see newId in server/types.ts: url-safe base64). + * + * The bench only ever talks to a server it spawned itself, so this isn't + * defending against a hostile peer — it's refusing to build a URL out of a value + * we haven't looked at. It also fails loudly and immediately if a publish returns + * an error body instead of a post, which otherwise shows up as a confusing 404 on + * the next line. CodeQL flags the unchecked version as request forgery, and it's + * right that the shape was unverified. + */ +function postId(value: unknown): string { + if (typeof value !== "string" || !/^[A-Za-z0-9_-]{1,64}$/.test(value)) { + throw new Error(`publish did not return a usable post id: ${JSON.stringify(value)}`); + } + return value; +} + /** * RSS and wall time for importing one module into an otherwise-empty process. * @@ -240,7 +258,7 @@ export const processSuite: Suite = { }), }); const post = (await publish.json()) as { id: string }; - await (await fetch(`${server.url}/s/${post.id}?part=0`)).text(); + await (await fetch(`${server.url}/s/${postId(post.id)}?part=0`)).text(); await sleep(750); ctx.add( memory( diff --git a/bench/suites/viewer.bench.ts b/bench/suites/viewer.bench.ts index 6c7f3b1c..c9d2f732 100644 --- a/bench/suites/viewer.bench.ts +++ b/bench/suites/viewer.bench.ts @@ -149,7 +149,14 @@ export const viewerSuite: Suite = { let browser: Awaited> | null = null; try { const first = await publish(server.url, undefined, 0); + // Validated before it goes into a navigation URL below — same reasoning as + // postId in process.bench.ts: don't build a URL out of a response value + // whose shape we never checked, and fail loudly if a publish returned an + // error body instead of a post. const sessionId = first.sessionId; + if (!/^[A-Za-z0-9_-]{1,64}$/.test(sessionId)) { + throw new Error(`publish did not return a usable session id: ${JSON.stringify(sessionId)}`); + } for (let i = 1; i < POSTS; i++) await publish(server.url, sessionId, i); browser = await launch(); From 01c8ceb9373c5d65eb384d7a89046e26834eaf01 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 15 Aug 2026 04:35:41 +0000 Subject: [PATCH 4/5] fix(bench): make --filter literal substrings instead of a regex MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CodeQL's high-severity alert on this PR was regex injection at bench/run.ts:104 — a command-line argument going straight into the RegExp constructor. --filter now takes comma-separated literal substrings, OR'd, case-insensitive. That removes the taint flow, and it is the better CLI anyway: metric names are full of regex metacharacters ("GET /s/:id code (cache hit)"), so the obvious move — pasting a name off the results table — used to match nothing. Now it works, and no hand-written pattern can backtrack a benchmark run into a hang. Lowercasing happens inside the matcher rather than in the arg parser, so a caller constructing a context directly still gets case-insensitive matching; a test covers that seam along with the pasted-name and OR cases. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01MGMuudU8anVrxB7sjY9hpt --- bench/README.md | 10 ++++++++++ bench/harness.ts | 17 +++++++++++++++-- bench/run.ts | 25 +++++++++++++++++++++---- test/bench.test.ts | 26 +++++++++++++++++++++++++- 4 files changed, 71 insertions(+), 7 deletions(-) diff --git a/bench/README.md b/bench/README.md index c1ea24a1..f1d5b932 100644 --- a/bench/README.md +++ b/bench/README.md @@ -10,6 +10,16 @@ npm run bench:check # run and fail on regression vs the committed bas npm run bench:baseline # re-record the baseline ``` +Narrow a run to the metrics you care about with `--filter`: comma-separated +literal substrings, OR'd, case-insensitive. It is deliberately not a regex — +metric names are full of `/`, `:` and parentheses, so pasting one straight off +the results table just works. + +```sh +npm run bench -- --filter diff,code # anything mentioning diff or code +npm run bench -- --filter 'GET /s/:id code' # one metric, pasted from the table +``` + ## What it measures | Suite | Covers | Default | diff --git a/bench/harness.ts b/bench/harness.ts index faa1644b..f5999694 100644 --- a/bench/harness.ts +++ b/bench/harness.ts @@ -293,9 +293,22 @@ export interface Suite { export function makeContext( suite: string, sink: BenchResult[], - opts: { full: boolean; filter?: RegExp }, + // `filter` holds lowercased literal substrings; a metric runs if its + // "suite/name" contains any of them. Literals rather than a pattern so a name + // copied straight off the results table works — those are full of `/`, `:` and + // parentheses — and so a command-line string never reaches the RegExp + // constructor. + opts: { full: boolean; filter?: string[] }, ): SuiteContext { - const matches = (name: string) => !opts.filter || opts.filter.test(`${suite}/${name}`); + // Lowercased here rather than trusting the caller to have done it: + // case-insensitivity is a property of matching, and splitting it across the + // arg parser and this function is how one of the two ends up forgetting. + const terms = opts.filter?.length ? opts.filter.map((term) => term.toLowerCase()) : null; + const matches = (name: string) => { + if (!terms) return true; + const key = `${suite}/${name}`.toLowerCase(); + return terms.some((term) => key.includes(term)); + }; return { full: opts.full, matches, diff --git a/bench/run.ts b/bench/run.ts index 724cd034..cb71bc20 100644 --- a/bench/run.ts +++ b/bench/run.ts @@ -3,7 +3,7 @@ // node --expose-gc bench/run.ts # default suites, print a table // node --expose-gc bench/run.ts store render # only these suites // node --expose-gc bench/run.ts --all # include optional (slow) suites -// node --expose-gc bench/run.ts --filter 'diff' # only matching metrics +// node --expose-gc bench/run.ts --filter diff,code # only metrics containing these // node --expose-gc bench/run.ts --save out.json # write results // node --expose-gc bench/run.ts --baseline # record bench/baseline.json // node --expose-gc bench/run.ts --check # compare, exit 1 on regression @@ -58,7 +58,8 @@ const baselinePath = (all: boolean) => interface Options { suites: string[]; all: boolean; - filter?: RegExp; + /** Literal, lowercased substrings; a metric matches if it contains any of them. */ + filter?: string[]; save?: string; baseline: boolean; check: boolean; @@ -85,6 +86,14 @@ const GATED_KINDS: Record = { deterministic: ["bytes", "count"], }; +/** Split a --filter value into lowercased literal terms; empty terms dropped. */ +function parseFilter(raw: string | undefined): string[] { + return (raw ?? "") + .split(",") + .map((term) => term.trim().toLowerCase()) + .filter(Boolean); +} + function parseArgs(argv: string[]): Options { const opts: Options = { suites: [], @@ -101,7 +110,15 @@ function parseArgs(argv: string[]): Options { else if (arg === "--full") opts.full = true; else if (arg === "--markdown") opts.markdown = true; else if (arg === "--baseline") opts.baseline = true; - else if (arg === "--filter") opts.filter = new RegExp(argv[++i], "i"); + // Comma-separated literal substrings, OR'd, case-insensitive — deliberately + // not a regex. Metric names are full of regex metacharacters + // ("GET /s/:id code (cache hit)"), so a regex filter made the obvious thing + // — pasting a metric name straight off the results table — silently match + // nothing. It also kept a command-line argument out of the RegExp + // constructor, which CodeQL flags as regex injection: a hand-written pattern + // can backtrack catastrophically, and turning your own benchmark run into a + // hang is a bad way to find that out. + else if (arg === "--filter") opts.filter = parseFilter(argv[++i]); else if (arg === "--save") opts.save = argv[++i]; else if (arg === "--gate") { const mode = argv[++i]; @@ -163,7 +180,7 @@ async function main() { // A baseline recorded from a subset would silently drop every metric it didn't // run, and the next full --check would report them all as "new" — a baseline // that quietly stopped policing most of the suite. Refuse rather than record it. - if (opts.baseline && (opts.suites.length > 0 || opts.filter)) { + if (opts.baseline && (opts.suites.length > 0 || opts.filter?.length)) { console.error( "refusing to record a partial baseline: drop the suite names and --filter, " + "or use --save to keep a scratch run.", diff --git a/test/bench.test.ts b/test/bench.test.ts index af614d85..83b2701d 100644 --- a/test/bench.test.ts +++ b/test/bench.test.ts @@ -13,7 +13,7 @@ import { resultKey, type BenchRun, } from "../bench/compare.ts"; -import { bytes, count, memory, time } from "../bench/harness.ts"; +import { bytes, count, makeContext, memory, time } from "../bench/harness.ts"; import { buildWorkspace, markdownSource, rng, TYPICAL } from "../bench/fixtures.ts"; import { SqlStore } from "../server/sqlStore.ts"; import { createSqliteStorage } from "../server/sqliteStorage.ts"; @@ -145,6 +145,30 @@ test("resultKey namespaces by suite so two suites can share a metric name", () = ); }); +test("--filter matches literal substrings, including regex metacharacters", async () => { + // The filter is literal on purpose: metric names carry `/`, `:` and parens, so + // the obvious move — pasting a name off the results table — has to work, and a + // command-line string must never reach the RegExp constructor. + const run = (filter: string[] | undefined, names: string[]) => { + const sink: BenchResult[] = []; + const ctx = makeContext("api", sink, { full: false, filter }); + for (const name of names) ctx.add(bytes("api", name, 1)); + return sink.map((r) => r.name); + }; + const names = ["GET /s/:id code (cache hit)", "GET /s/:id diff (cache hit)", "POST /api/posts"]; + + assert.deepEqual(run(["get /s/:id code (cache hit)"], names), [names[0]], "pasted name matches"); + assert.deepEqual(run(["code", "diff"], names), [names[0], names[1]], "comma terms are OR'd"); + assert.deepEqual(run(["CODE"], names), [names[0]], "matching is case-insensitive"); + assert.deepEqual(run(undefined, names), names, "no filter runs everything"); + assert.deepEqual(run([], names), names, "an empty filter runs everything"); + // A regex-looking term is treated as text, so it matches nothing rather than + // quietly behaving as alternation. + assert.deepEqual(run(["code|diff"], names), []); + // The suite name is part of the searched key, so a suite can be selected by name. + assert.deepEqual(run(["api/"], names), names); +}); + test("time() reports a median and honours a fixed iteration count", async () => { let calls = 0; const result = await time("s", "counted", () => calls++, { iterations: 12 }); From 5f2379994a842c49759a576697565951443c5ed8 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 15 Aug 2026 04:54:14 +0000 Subject: [PATCH 5/5] fix(bench): request surface documents the way the viewer does MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The /s/:id benches asked for `?part=0` with no `&theme=&mode=`. The viewer always sends both (Card.tsx builds every surface iframe src that way), and the server renders differently when the mode is pinned — so the suite was measuring a URL shape no client ever requests. That is the failure mode a benchmark can't afford: profiling a shiki change against these numbers showed no improvement at all, while the same change through the real viewer URL was 47% faster and 30% smaller. A bench that misses the win is worse than no bench, because it argues against the fix. Baseline re-recorded for the corrected URLs. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01MGMuudU8anVrxB7sjY9hpt --- bench/baseline.json | 1336 ++++++++++++++++++------------------- bench/suites/api.bench.ts | 9 +- 2 files changed, 675 insertions(+), 670 deletions(-) diff --git a/bench/baseline.json b/bench/baseline.json index 5e320d32..bc4526ed 100644 --- a/bench/baseline.json +++ b/bench/baseline.json @@ -1,8 +1,8 @@ { "format": 1, - "recordedAt": "2026-08-15T00:04:42.691Z", + "recordedAt": "2026-08-15T04:53:46.800Z", "git": { - "commit": "9b66556", + "commit": "01c8ceb", "branch": "claude/sideshow-perf-benchmarks-9zhrho" }, "machine": { @@ -12,7 +12,7 @@ "cpuModel": "Intel(R) Xeon(R) Processor @ 2.80GHz", "nodeVersion": "v22.22.2", "totalMemory": 16856068096, - "index": 8891.705493781223 + "index": 9320.700677015857 }, "results": [ { @@ -20,15 +20,15 @@ "name": "sqlite-memory/getPost", "kind": "time", "unit": "ms/op", - "value": 0.034613785714285375, + "value": 0.03278715909090899, "stats": { - "iterations": 10010, - "samples": 286, - "min": 0.031170228571428586, - "median": 0.034613785714285375, - "p95": 0.049504478571429894, - "max": 0.19214122857142685, - "opsPerSec": 28890.223342062593 + "iterations": 10868, + "samples": 247, + "min": 0.030398727272726814, + "median": 0.03278715909090899, + "p95": 0.08673261136363483, + "max": 0.12173104545454355, + "opsPerSec": 30499.745257809587 }, "note": "SqlStore(:memory:), 144 posts / 96 comments" }, @@ -37,15 +37,15 @@ "name": "sqlite-memory/listPosts(session)", "kind": "time", "unit": "ms/op", - "value": 0.39970083333332695, + "value": 0.3908609999998589, "stats": { - "iterations": 960, - "samples": 320, - "min": 0.3678070000000086, - "median": 0.39970083333332695, - "p95": 0.507585016666667, - "max": 0.7785416666667212, - "opsPerSec": 2501.871191161763 + "iterations": 975, + "samples": 975, + "min": 0.3407520000000659, + "median": 0.3908609999998589, + "p95": 0.4696559999999181, + "max": 1.3134840000000167, + "opsPerSec": 2558.454284260545 }, "note": "SqlStore(:memory:), 12 posts in session" }, @@ -54,15 +54,15 @@ "name": "sqlite-memory/listPosts(all)", "kind": "time", "unit": "ms/op", - "value": 4.731358499999942, + "value": 4.655306999999993, "stats": { - "iterations": 76, - "samples": 76, - "min": 4.4434040000001005, - "median": 4.731358499999942, - "p95": 7.196898749999832, - "max": 11.945956999999908, - "opsPerSec": 211.35578713809412 + "iterations": 79, + "samples": 79, + "min": 4.449718000000075, + "median": 4.655306999999993, + "p95": 6.4813622000001025, + "max": 6.884097000000111, + "opsPerSec": 214.80860445938396 }, "note": "SqlStore(:memory:), 144 posts / 96 comments" }, @@ -71,15 +71,15 @@ "name": "sqlite-memory/listRecentPosts(20)", "kind": "time", "unit": "ms/op", - "value": 0.6540833333333467, + "value": 0.6447630000000117, "stats": { - "iterations": 573, - "samples": 191, - "min": 0.5992056666667244, - "median": 0.6540833333333467, - "p95": 0.9849848333333284, - "max": 1.449440333333314, - "opsPerSec": 1528.857179258473 + "iterations": 600, + "samples": 200, + "min": 0.6046503333333627, + "median": 0.6447630000000117, + "p95": 0.9251579499999706, + "max": 1.0390786666666827, + "opsPerSec": 1550.9574836024738 }, "note": "SqlStore(:memory:), 144 posts / 96 comments" }, @@ -88,15 +88,15 @@ "name": "sqlite-memory/listComments(session)", "kind": "time", "unit": "ms/op", - "value": 0.04182148571428245, + "value": 0.04174636904761948, "stats": { - "iterations": 8050, - "samples": 230, - "min": 0.038746999999992635, - "median": 0.04182148571428245, - "p95": 0.076123408571425, - "max": 0.3384484000000027, - "opsPerSec": 23911.154348552715 + "iterations": 8568, + "samples": 204, + "min": 0.03759035714285476, + "median": 0.04174636904761948, + "p95": 0.04873627976189925, + "max": 0.2881252857142889, + "opsPerSec": 23954.17907745017 }, "note": "SqlStore(:memory:), 144 posts / 96 comments" }, @@ -105,15 +105,15 @@ "name": "sqlite-memory/listSessions", "kind": "time", "unit": "ms/op", - "value": 0.04424722222222499, + "value": 0.04481858974358124, "stats": { - "iterations": 8028, - "samples": 223, - "min": 0.04083891666665648, - "median": 0.04424722222222499, - "p95": 0.05344691666666292, - "max": 0.2576869444444456, - "opsPerSec": 22600.288781466348 + "iterations": 7917, + "samples": 203, + "min": 0.04101684615383979, + "median": 0.04481858974358124, + "p95": 0.05008507692307783, + "max": 0.269639666666675, + "opsPerSec": 22312.17014460426 }, "note": "SqlStore(:memory:), 12 sessions" }, @@ -122,15 +122,15 @@ "name": "sqlite-memory/countPostsBySession", "kind": "time", "unit": "ms/op", - "value": 0.03504255102040811, + "value": 0.03410506382978712, "stats": { - "iterations": 10192, - "samples": 208, - "min": 0.032274857142863544, - "median": 0.03504255102040811, - "p95": 0.043985715306119284, - "max": 0.2250860816326464, - "opsPerSec": 28536.735222775853 + "iterations": 10528, + "samples": 224, + "min": 0.03190599999999826, + "median": 0.03410506382978712, + "p95": 0.03711657765957751, + "max": 0.2558304893617052, + "opsPerSec": 29321.159021746418 }, "note": "SqlStore(:memory:), 144 posts / 96 comments" }, @@ -139,15 +139,15 @@ "name": "sqlite-memory/isAssetReferenced(miss)", "kind": "time", "unit": "ms/op", - "value": 0.0001111151315789763, + "value": 0.00010750313755906627, "stats": { - "iterations": 3376528, - "samples": 383, - "min": 0.00008978221415615624, - "median": 0.0001111151315789763, - "p95": 0.00016115510435577786, - "max": 0.00020321109346649132, - "opsPerSec": 8999674.353886168 + "iterations": 3616665, + "samples": 255, + "min": 0.0000958854262144772, + "median": 0.00010750313755906627, + "p95": 0.0001270693083268462, + "max": 0.00015420009870974422, + "opsPerSec": 9302054.08610109 }, "note": "SqlStore(:memory:), 144 posts / 96 comments" }, @@ -156,15 +156,15 @@ "name": "sqlite-memory/createPost", "kind": "time", "unit": "ms/op", - "value": 0.11053850000007515, + "value": 0.08670949999986988, "stats": { "iterations": 40, "samples": 40, - "min": 0.07198599999992439, - "median": 0.11053850000007515, - "p95": 0.18154740000027214, - "max": 0.463502000000517, - "opsPerSec": 9046.621765261156 + "min": 0.07478299999911542, + "median": 0.08670949999986988, + "p95": 0.19185415000051753, + "max": 0.5166230000004362, + "opsPerSec": 11532.761692796068 }, "note": "SqlStore(:memory:), 144 posts / 96 comments" }, @@ -173,15 +173,15 @@ "name": "sqlite-memory/updatePost", "kind": "time", "unit": "ms/op", - "value": 0.36556849999988117, + "value": 0.33067999999957465, "stats": { "iterations": 40, "samples": 40, - "min": 0.20033799999964685, - "median": 0.36556849999988117, - "p95": 0.49073974999982956, - "max": 1.696125999999822, - "opsPerSec": 2735.465446285238 + "min": 0.18460700000014185, + "median": 0.33067999999957465, + "p95": 0.44441425000004525, + "max": 0.48703000000023167, + "opsPerSec": 3024.071610019615 }, "note": "SqlStore(:memory:), history at cap" }, @@ -190,15 +190,15 @@ "name": "sqlite-memory/createComment", "kind": "time", "unit": "ms/op", - "value": 0.1557794999998805, + "value": 0.07491149999987101, "stats": { "iterations": 40, "samples": 40, - "min": 0.09776400000009744, - "median": 0.1557794999998805, - "p95": 0.23081220000026403, - "max": 0.24728800000048068, - "opsPerSec": 6419.329886158109 + "min": 0.06531399999948917, + "median": 0.07491149999987101, + "p95": 0.14941494999943639, + "max": 0.24567800000022544, + "opsPerSec": 13349.08525395596 }, "note": "SqlStore(:memory:), 144 posts / 96 comments" }, @@ -207,15 +207,15 @@ "name": "sqlite-file/getPost", "kind": "time", "unit": "ms/op", - "value": 0.03840621951220361, + "value": 0.03802074999998695, "stats": { - "iterations": 9225, - "samples": 225, - "min": 0.03548236585365663, - "median": 0.03840621951220361, - "p95": 0.06737120000000607, - "max": 0.14738526829267623, - "opsPerSec": 26037.449473053424 + "iterations": 9372, + "samples": 213, + "min": 0.0353577727272626, + "median": 0.03802074999998695, + "p95": 0.06907646363635331, + "max": 0.14508761363636455, + "opsPerSec": 26301.427509987134 }, "note": "SqlStore(file), 144 posts / 96 comments" }, @@ -224,15 +224,15 @@ "name": "sqlite-file/listPosts(session)", "kind": "time", "unit": "ms/op", - "value": 0.42086349999999584, + "value": 0.4324545000000626, "stats": { - "iterations": 916, - "samples": 229, - "min": 0.39616175000014664, - "median": 0.42086349999999584, - "p95": 0.6201491500000884, - "max": 0.729702750000115, - "opsPerSec": 2376.067299730221 + "iterations": 896, + "samples": 224, + "min": 0.40242125000008855, + "median": 0.4324545000000626, + "p95": 0.6585197499999139, + "max": 0.7219672500000343, + "opsPerSec": 2312.381996255919 }, "note": "SqlStore(file), 12 posts in session" }, @@ -241,15 +241,15 @@ "name": "sqlite-file/listPosts(all)", "kind": "time", "unit": "ms/op", - "value": 5.0059510000000955, + "value": 4.90140700000029, "stats": { - "iterations": 73, - "samples": 73, - "min": 4.67334000000028, - "median": 5.0059510000000955, - "p95": 7.0836891999999345, - "max": 7.563390000000254, - "opsPerSec": 199.7622429784033 + "iterations": 75, + "samples": 75, + "min": 4.6387439999998605, + "median": 4.90140700000029, + "p95": 6.892220500000075, + "max": 7.188204999999471, + "opsPerSec": 204.02304889186732 }, "note": "SqlStore(file), 144 posts / 96 comments" }, @@ -258,15 +258,15 @@ "name": "sqlite-file/listRecentPosts(20)", "kind": "time", "unit": "ms/op", - "value": 0.7438904999999068, + "value": 0.7079435000000558, "stats": { - "iterations": 510, - "samples": 255, - "min": 0.6827775000001566, - "median": 0.7438904999999068, - "p95": 1.142371400000138, - "max": 1.3079160000002048, - "opsPerSec": 1344.283869736373 + "iterations": 538, + "samples": 538, + "min": 0.6464200000000346, + "median": 0.7079435000000558, + "p95": 0.844244449999587, + "max": 1.869235000000117, + "opsPerSec": 1412.5421025829337 }, "note": "SqlStore(file), 144 posts / 96 comments" }, @@ -275,15 +275,15 @@ "name": "sqlite-file/listComments(session)", "kind": "time", "unit": "ms/op", - "value": 0.04919448611110511, + "value": 0.0424536153846195, "stats": { - "iterations": 6984, - "samples": 194, - "min": 0.042023472222227715, - "median": 0.04919448611110511, - "p95": 0.07824290694444218, - "max": 0.4209769444444444, - "opsPerSec": 20327.4813714186 + "iterations": 8216, + "samples": 316, + "min": 0.04083019230770683, + "median": 0.0424536153846195, + "p95": 0.04920380769231787, + "max": 0.493603499999993, + "opsPerSec": 23555.119886498276 }, "note": "SqlStore(file), 144 posts / 96 comments" }, @@ -292,15 +292,15 @@ "name": "sqlite-file/listSessions", "kind": "time", "unit": "ms/op", - "value": 0.04679093939395181, + "value": 0.04617821875001482, "stats": { - "iterations": 7425, - "samples": 225, - "min": 0.04307184848485948, - "median": 0.04679093939395181, - "p95": 0.0594284787878981, - "max": 0.3468611818181781, - "opsPerSec": 21371.658978260646 + "iterations": 7584, + "samples": 237, + "min": 0.04242706249999628, + "median": 0.04617821875001482, + "p95": 0.057259575000000465, + "max": 0.3355770000000007, + "opsPerSec": 21655.231125598126 }, "note": "SqlStore(file), 12 sessions" }, @@ -309,15 +309,15 @@ "name": "sqlite-file/countPostsBySession", "kind": "time", "unit": "ms/op", - "value": 0.038225222222222834, + "value": 0.036346909090918256, "stats": { - "iterations": 9540, - "samples": 212, - "min": 0.034087999999994484, - "median": 0.038225222222222834, - "p95": 0.04912179999999957, - "max": 0.2531072444444362, - "opsPerSec": 26160.73738398398 + "iterations": 9856, + "samples": 224, + "min": 0.03419393181818784, + "median": 0.036346909090918256, + "p95": 0.045977863636362165, + "max": 0.26813281818181167, + "opsPerSec": 27512.655821671036 }, "note": "SqlStore(file), 144 posts / 96 comments" }, @@ -326,15 +326,15 @@ "name": "sqlite-file/isAssetReferenced(miss)", "kind": "time", "unit": "ms/op", - "value": 0.00011079669664476878, + "value": 0.00011465017502911164, "stats": { - "iterations": 3411380, - "samples": 590, - "min": 0.00009790591490834072, - "median": 0.00011079669664476878, - "p95": 0.00016421719128334054, - "max": 0.00034833137322733436, - "opsPerSec": 9025539.84263767 + "iterations": 3012355, + "samples": 703, + "min": 0.00009109638273043256, + "median": 0.00011465017502911164, + "p95": 0.00018708651108526863, + "max": 0.00849851341890306, + "opsPerSec": 8722184.678270947 }, "note": "SqlStore(file), 144 posts / 96 comments" }, @@ -343,15 +343,15 @@ "name": "sqlite-file/createPost", "kind": "time", "unit": "ms/op", - "value": 0.12265800000022864, + "value": 0.13465599999972255, "stats": { "iterations": 40, "samples": 40, - "min": 0.10285499999918102, - "median": 0.12265800000022864, - "p95": 0.19955445000050517, - "max": 0.3980699999992794, - "opsPerSec": 8152.749922533679 + "min": 0.10810700000001816, + "median": 0.13465599999972255, + "p95": 0.21038515000027447, + "max": 0.4812920000003942, + "opsPerSec": 7426.330798494389 }, "note": "SqlStore(file), 144 posts / 96 comments" }, @@ -360,15 +360,15 @@ "name": "sqlite-file/updatePost", "kind": "time", "unit": "ms/op", - "value": 0.4303924999994706, + "value": 0.449845500000265, "stats": { "iterations": 40, "samples": 40, - "min": 0.21944100000109756, - "median": 0.4303924999994706, - "p95": 0.4923694999999497, - "max": 1.8614739999993617, - "opsPerSec": 2323.460562164141 + "min": 0.23486899999988964, + "median": 0.449845500000265, + "p95": 0.4962515000001531, + "max": 0.5815519999996468, + "opsPerSec": 2222.98544722446 }, "note": "SqlStore(file), history at cap" }, @@ -377,15 +377,15 @@ "name": "sqlite-file/createComment", "kind": "time", "unit": "ms/op", - "value": 0.11581099999966682, + "value": 0.11506549999921845, "stats": { "iterations": 40, "samples": 40, - "min": 0.09817599999951199, - "median": 0.11581099999966682, - "p95": 0.22707395000079483, - "max": 10.421438999999737, - "opsPerSec": 8634.75835631224 + "min": 0.09638799999993353, + "median": 0.11506549999921845, + "p95": 0.2083520000005591, + "max": 10.513995999999679, + "opsPerSec": 8690.702252254518 }, "note": "SqlStore(file), 144 posts / 96 comments" }, @@ -394,15 +394,15 @@ "name": "sqlite-file/cold open + first read", "kind": "time", "unit": "ms/op", - "value": 0.7581345000007786, + "value": 0.7695420000000013, "stats": { - "iterations": 350, - "samples": 175, - "min": 0.6985580000000482, - "median": 0.7581345000007786, - "p95": 0.9937225499999838, - "max": 4.206881000000067, - "opsPerSec": 1319.0271647035888 + "iterations": 344, + "samples": 172, + "min": 0.702826499999901, + "median": 0.7695420000000013, + "p95": 0.9991592000001217, + "max": 4.964671999999155, + "opsPerSec": 1299.474232725437 }, "note": "SqlStore(file), 144 posts / 96 comments" }, @@ -411,15 +411,15 @@ "name": "json-file/getPost", "kind": "time", "unit": "ms/op", - "value": 0.016411252747255297, + "value": 0.01614038947369217, "stats": { - "iterations": 23296, - "samples": 256, - "min": 0.015309912087904306, - "median": 0.016411252747255297, - "p95": 0.02314657692307434, - "max": 0.027903538461532162, - "opsPerSec": 60933.80044782049 + "iterations": 23655, + "samples": 249, + "min": 0.014985463157886682, + "median": 0.01614038947369217, + "p95": 0.023176564210535662, + "max": 0.02944848421052723, + "opsPerSec": 61956.373582554355 }, "note": "JsonFileStore, 144 posts / 96 comments" }, @@ -428,15 +428,15 @@ "name": "json-file/listPosts(session)", "kind": "time", "unit": "ms/op", - "value": 0.5045552500000667, + "value": 0.5204158333335727, "stats": { - "iterations": 764, - "samples": 191, - "min": 0.47796924999965995, - "median": 0.5045552500000667, - "p95": 0.667314124999848, - "max": 0.7793567499998062, - "opsPerSec": 1981.9435037091928 + "iterations": 684, + "samples": 228, + "min": 0.45871666666668415, + "median": 0.5204158333335727, + "p95": 0.8394498166666987, + "max": 1.1837623333334097, + "opsPerSec": 1921.5403067089749 }, "note": "JsonFileStore, 12 posts in session" }, @@ -445,15 +445,15 @@ "name": "json-file/listPosts(all)", "kind": "time", "unit": "ms/op", - "value": 6.3928975000007995, + "value": 5.993367999999464, "stats": { - "iterations": 58, - "samples": 58, - "min": 5.803273999999874, - "median": 6.3928975000007995, - "p95": 8.850587550001272, - "max": 10.551391000000876, - "opsPerSec": 156.42359352701573 + "iterations": 63, + "samples": 63, + "min": 5.74029100000007, + "median": 5.993367999999464, + "p95": 8.43700740000022, + "max": 9.939164000001256, + "opsPerSec": 166.8510927411915 }, "note": "JsonFileStore, 144 posts / 96 comments" }, @@ -462,15 +462,15 @@ "name": "json-file/listRecentPosts(20)", "kind": "time", "unit": "ms/op", - "value": 0.9378512500002216, + "value": 0.9203260000003866, "stats": { - "iterations": 416, - "samples": 208, - "min": 0.8688295000001744, - "median": 0.9378512500002216, - "p95": 1.236448900000323, - "max": 2.0988930000003165, - "opsPerSec": 1066.2671718993429 + "iterations": 426, + "samples": 213, + "min": 0.8571394999999029, + "median": 0.9203260000003866, + "p95": 1.0785030000000912, + "max": 1.396543500000007, + "opsPerSec": 1086.57149749065 }, "note": "JsonFileStore, 144 posts / 96 comments" }, @@ -479,15 +479,15 @@ "name": "json-file/listComments(session)", "kind": "time", "unit": "ms/op", - "value": 0.023168696969702127, + "value": 0.02265336764704069, "stats": { - "iterations": 16500, - "samples": 250, - "min": 0.022022590909066574, - "median": 0.023168696969702127, - "p95": 0.027103380303020504, - "max": 0.09133016666667325, - "opsPerSec": 43161.684979854814 + "iterations": 16932, + "samples": 249, + "min": 0.02136257352942587, + "median": 0.02265336764704069, + "p95": 0.03194909117648207, + "max": 0.04019189705882929, + "opsPerSec": 44143.54702492256 }, "note": "JsonFileStore, 144 posts / 96 comments" }, @@ -496,15 +496,15 @@ "name": "json-file/listSessions", "kind": "time", "unit": "ms/op", - "value": 0.03197912244898789, + "value": 0.03186185416666376, "stats": { - "iterations": 12103, - "samples": 247, - "min": 0.030096551020376085, - "median": 0.03197912244898789, - "p95": 0.04366843469388001, - "max": 0.05813248979590141, - "opsPerSec": 31270.401543856286 + "iterations": 12288, + "samples": 256, + "min": 0.029656645833332124, + "median": 0.03186185416666376, + "p95": 0.03480674479165676, + "max": 0.050398249999981694, + "opsPerSec": 31385.492971287098 }, "note": "JsonFileStore, 12 sessions" }, @@ -513,15 +513,15 @@ "name": "json-file/countPostsBySession", "kind": "time", "unit": "ms/op", - "value": 0.004537276422761499, + "value": 0.004613224137925687, "stats": { - "iterations": 82902, - "samples": 337, - "min": 0.004340597560975419, - "median": 0.004537276422761499, - "p95": 0.006713969918694683, - "max": 0.008803455284556193, - "opsPerSec": 220396.5345781986 + "iterations": 82650, + "samples": 285, + "min": 0.0042993931034477, + "median": 0.004613224137925687, + "p95": 0.006524214482756336, + "max": 0.007585437931033837, + "opsPerSec": 216768.13657912682 }, "note": "JsonFileStore, 144 posts / 96 comments" }, @@ -530,15 +530,15 @@ "name": "json-file/isAssetReferenced(miss)", "kind": "time", "unit": "ms/op", - "value": 0.000268329577464639, + "value": 0.0002651433129147562, "stats": { - "iterations": 1437750, - "samples": 225, - "min": 0.00022767793427209453, - "median": 0.000268329577464639, - "p95": 0.0003215271674490283, - "max": 0.0003963466353679009, - "opsPerSec": 3726760.238094818 + "iterations": 1449660, + "samples": 185, + "min": 0.00023923545176102707, + "median": 0.0002651433129147562, + "p95": 0.0003186351710055528, + "max": 0.0003561924451249872, + "opsPerSec": 3771545.240975023 }, "note": "JsonFileStore, 144 posts / 96 comments" }, @@ -547,15 +547,15 @@ "name": "json-file/createPost", "kind": "time", "unit": "ms/op", - "value": 8.302538000000823, + "value": 8.644445500000074, "stats": { "iterations": 40, "samples": 40, - "min": 7.216768000000229, - "median": 8.302538000000823, - "p95": 9.50251315000032, - "max": 11.17880900000091, - "opsPerSec": 120.44509763157976 + "min": 7.5775830000002316, + "median": 8.644445500000074, + "p95": 11.319358400001146, + "max": 11.532438000000184, + "opsPerSec": 115.68121980756214 }, "note": "JsonFileStore, 144 posts / 96 comments" }, @@ -564,15 +564,15 @@ "name": "json-file/updatePost", "kind": "time", "unit": "ms/op", - "value": 9.825980499998877, + "value": 8.603773499999988, "stats": { "iterations": 40, "samples": 40, - "min": 7.951912000000448, - "median": 9.825980499998877, - "p95": 13.580894749999425, - "max": 14.73130300000048, - "opsPerSec": 101.77101409880818 + "min": 7.837500000001455, + "median": 8.603773499999988, + "p95": 11.26097569999947, + "max": 11.872676000000865, + "opsPerSec": 116.2280713224263 }, "note": "JsonFileStore, history at cap" }, @@ -581,15 +581,15 @@ "name": "json-file/createComment", "kind": "time", "unit": "ms/op", - "value": 8.732371999999486, + "value": 9.250788000000284, "stats": { "iterations": 40, "samples": 40, - "min": 7.88097899999957, - "median": 8.732371999999486, - "p95": 10.820051250000688, - "max": 11.296034999999392, - "opsPerSec": 114.51642234206912 + "min": 7.866990999998961, + "median": 9.250788000000284, + "p95": 11.98593900000078, + "max": 12.454119999998511, + "opsPerSec": 108.09889925052539 }, "note": "JsonFileStore, 144 posts / 96 comments" }, @@ -598,15 +598,15 @@ "name": "json-file/cold open + first read", "kind": "time", "unit": "ms/op", - "value": 9.031462999999349, + "value": 9.233320999999705, "stats": { "iterations": 27, "samples": 27, - "min": 8.409542999999758, - "median": 9.031462999999349, - "p95": 14.093383899999935, - "max": 51.87620100000095, - "opsPerSec": 110.72403219722786 + "min": 8.54478299999937, + "median": 9.233320999999705, + "p95": 14.955438400000821, + "max": 41.679371000000174, + "opsPerSec": 108.30339376265938 }, "note": "JsonFileStore, 144 posts / 96 comments" }, @@ -615,15 +615,15 @@ "name": "sqlite-memory/createPost @ 50 posts", "kind": "time", "unit": "ms/op", - "value": 0.1009650000005422, + "value": 0.08792300000004616, "stats": { "iterations": 20, "samples": 20, - "min": 0.07481099999858998, - "median": 0.1009650000005422, - "p95": 0.2506658000013886, - "max": 0.3402089999999589, - "opsPerSec": 9904.422324514731 + "min": 0.07265000000006694, + "median": 0.08792300000004616, + "p95": 0.16825185000034254, + "max": 0.17759700000169687, + "opsPerSec": 11373.588253352082 }, "note": "SqlStore(:memory:), workspace already holds 50 posts" }, @@ -632,15 +632,15 @@ "name": "sqlite-memory/createPost @ 200 posts", "kind": "time", "unit": "ms/op", - "value": 0.07565600000089034, + "value": 0.07489050000003772, "stats": { "iterations": 20, "samples": 20, - "min": 0.06927100000029895, - "median": 0.07565600000089034, - "p95": 0.13423439999860423, - "max": 0.1772579999997106, - "opsPerSec": 13217.722321933907 + "min": 0.0694229999990057, + "median": 0.07489050000003772, + "p95": 0.13356389999962634, + "max": 0.18818699999974342, + "opsPerSec": 13352.828462882426 }, "note": "SqlStore(:memory:), workspace already holds 200 posts" }, @@ -649,15 +649,15 @@ "name": "sqlite-memory/createPost @ 500 posts", "kind": "time", "unit": "ms/op", - "value": 0.0792570000003252, + "value": 0.08162049999918963, "stats": { "iterations": 20, "samples": 20, - "min": 0.06852999999864551, - "median": 0.0792570000003252, - "p95": 0.1328072500002236, - "max": 0.14616899999964517, - "opsPerSec": 12617.182078502805 + "min": 0.06854399999974703, + "median": 0.08162049999918963, + "p95": 0.14707125000031743, + "max": 0.2049120000010589, + "opsPerSec": 12251.823990418197 }, "note": "SqlStore(:memory:), workspace already holds 500 posts" }, @@ -666,15 +666,15 @@ "name": "sqlite-file/createPost @ 50 posts", "kind": "time", "unit": "ms/op", - "value": 0.13259949999974197, + "value": 0.1258095000002868, "stats": { "iterations": 20, "samples": 20, - "min": 0.10356400000091526, - "median": 0.13259949999974197, - "p95": 0.20030765000010436, - "max": 0.2615189999996801, - "opsPerSec": 7541.506566781519 + "min": 0.10846899999887682, + "median": 0.1258095000002868, + "p95": 0.23989224999968428, + "max": 0.24704100000053586, + "opsPerSec": 7948.525349816353 }, "note": "SqlStore(file), workspace already holds 50 posts" }, @@ -683,15 +683,15 @@ "name": "sqlite-file/createPost @ 200 posts", "kind": "time", "unit": "ms/op", - "value": 0.12705300000106945, + "value": 0.11100449999958073, "stats": { "iterations": 20, "samples": 20, - "min": 0.09517700000105833, - "median": 0.12705300000106945, - "p95": 0.1865948000010576, - "max": 0.20496400000047288, - "opsPerSec": 7870.731112146763 + "min": 0.09524600000077044, + "median": 0.11100449999958073, + "p95": 0.1720955999998296, + "max": 0.1865090000010241, + "opsPerSec": 9008.643793754101 }, "note": "SqlStore(file), workspace already holds 200 posts" }, @@ -700,15 +700,15 @@ "name": "sqlite-file/createPost @ 500 posts", "kind": "time", "unit": "ms/op", - "value": 0.13933849999921222, + "value": 0.12231599999995524, "stats": { "iterations": 20, "samples": 20, - "min": 0.0998520000030112, - "median": 0.13933849999921222, - "p95": 0.22354524999864225, - "max": 0.2342849999986356, - "opsPerSec": 7176.7673687147035 + "min": 0.09742000000005646, + "median": 0.12231599999995524, + "p95": 0.1605111000001671, + "max": 0.21042599999964295, + "opsPerSec": 8175.545308875094 }, "note": "SqlStore(file), workspace already holds 500 posts" }, @@ -717,15 +717,15 @@ "name": "json-file/createPost @ 50 posts", "kind": "time", "unit": "ms/op", - "value": 1.2645945000003849, + "value": 1.0160115000007863, "stats": { "iterations": 20, "samples": 20, - "min": 1.086012000003393, - "median": 1.2645945000003849, - "p95": 1.3843087999994168, - "max": 1.3880669999998645, - "opsPerSec": 790.7673171120827 + "min": 0.8750789999994595, + "median": 1.0160115000007863, + "p95": 1.783846449998427, + "max": 13.314308999999412, + "opsPerSec": 984.2408279819924 }, "note": "JsonFileStore, workspace already holds 50 posts" }, @@ -734,15 +734,15 @@ "name": "json-file/createPost @ 200 posts", "kind": "time", "unit": "ms/op", - "value": 2.104668000001766, + "value": 2.3249864999997953, "stats": { "iterations": 20, "samples": 20, - "min": 1.9422969999977795, - "median": 2.104668000001766, - "p95": 2.642122600000768, - "max": 4.287818999997398, - "opsPerSec": 475.1343204719989 + "min": 2.161910999999236, + "median": 2.3249864999997953, + "p95": 3.875720550000189, + "max": 5.881560999998328, + "opsPerSec": 430.1100242948026 }, "note": "JsonFileStore, workspace already holds 200 posts" }, @@ -751,15 +751,15 @@ "name": "json-file/createPost @ 500 posts", "kind": "time", "unit": "ms/op", - "value": 4.370087499999499, + "value": 4.418359999997847, "stats": { "iterations": 20, "samples": 20, - "min": 4.057629999999335, - "median": 4.370087499999499, - "p95": 5.125313050000479, - "max": 7.349017000000458, - "opsPerSec": 228.82837014135634 + "min": 3.9412300000003597, + "median": 4.418359999997847, + "p95": 4.927181900000871, + "max": 7.18734399999812, + "opsPerSec": 226.3283209155631 }, "note": "JsonFileStore, workspace already holds 500 posts" }, @@ -768,7 +768,7 @@ "name": "sqlite-memory/heap for loaded workspace", "kind": "memory", "unit": "bytes", - "value": 256, + "value": 120, "note": "SqlStore(:memory:), 12×25 posts" }, { @@ -776,7 +776,7 @@ "name": "sqlite-memory/heap for listPosts(all) result", "kind": "memory", "unit": "bytes", - "value": 2201440, + "value": 2201584, "note": "SqlStore(:memory:), 12×25 posts, 300 materialized" }, { @@ -800,7 +800,7 @@ "name": "json-file/heap for loaded workspace", "kind": "memory", "unit": "bytes", - "value": 2607040, + "value": 2606704, "note": "JsonFileStore, 12×25 posts" }, { @@ -816,7 +816,7 @@ "name": "shiki cold init (first code render)", "kind": "time", "unit": "ms/op", - "value": 277.8204659999974, + "value": 275.9442850000014, "note": "one-time per process; loads every registry theme", "tolerance": 2 }, @@ -825,7 +825,7 @@ "name": "shiki resident heap after warmup", "kind": "memory", "unit": "bytes", - "value": 2953112, + "value": 2951592, "note": "themes + grammars, per process" }, { @@ -833,15 +833,15 @@ "name": "markdown/small", "kind": "time", "unit": "ms/op", - "value": 3.7866030000004685, + "value": 3.6510084999990795, "stats": { - "iterations": 99, - "samples": 99, - "min": 3.290779000002658, - "median": 3.7866030000004685, - "p95": 5.625986199999169, - "max": 8.543934000001173, - "opsPerSec": 264.0889472701195 + "iterations": 104, + "samples": 104, + "min": 3.301951999997982, + "median": 3.6510084999990795, + "p95": 5.205377799997949, + "max": 6.938107000001764, + "opsPerSec": 273.8969246443146 }, "note": "1760 B source" }, @@ -857,15 +857,15 @@ "name": "markdown/large", "kind": "time", "unit": "ms/op", - "value": 51.90582100000029, + "value": 48.80531699999847, "stats": { "iterations": 15, "samples": 15, - "min": 48.21957900000052, - "median": 51.90582100000029, - "p95": 55.21429749999952, - "max": 55.25917800000025, - "opsPerSec": 19.265661937993322 + "min": 47.954821999999695, + "median": 48.80531699999847, + "p95": 52.717446000000564, + "max": 55.06776600000012, + "opsPerSec": 20.48957083917786 }, "note": "34621 B source" }, @@ -881,15 +881,15 @@ "name": "code/small", "kind": "time", "unit": "ms/op", - "value": 39.18193099999917, + "value": 37.00863000000027, "stats": { "iterations": 15, "samples": 15, - "min": 37.27079700000104, - "median": 39.18193099999917, - "p95": 43.45370629999997, - "max": 44.5009560000035, - "opsPerSec": 25.52196827665337 + "min": 35.47996499999863, + "median": 37.00863000000027, + "p95": 42.16878729999989, + "max": 45.74122400000124, + "opsPerSec": 27.020724625580378 }, "note": "48 lines" }, @@ -905,15 +905,15 @@ "name": "code/large", "kind": "time", "unit": "ms/op", - "value": 1138.8138330000002, + "value": 1084.8303860000015, "stats": { "iterations": 3, "samples": 3, - "min": 1096.683001999998, - "median": 1138.8138330000002, - "p95": 1237.3471101000032, - "max": 1248.2952520000035, - "opsPerSec": 0.8781066501147777 + "min": 1056.9614409999995, + "median": 1084.8303860000015, + "p95": 1086.7145035999995, + "max": 1086.9238499999992, + "opsPerSec": 0.9218030882110622 }, "note": "1402 lines" }, @@ -929,15 +929,15 @@ "name": "terminal/small", "kind": "time", "unit": "ms/op", - "value": 0.1271825357141227, + "value": 0.11563766666646795, "stats": { - "iterations": 2716, - "samples": 194, - "min": 0.11191635714255556, - "median": 0.1271825357141227, - "p95": 0.19956953571423322, - "max": 1.773545928571236, - "opsPerSec": 7862.7147539169355 + "iterations": 3300, + "samples": 275, + "min": 0.10670541666634865, + "median": 0.11563766666646795, + "p95": 0.16150641666699814, + "max": 0.2037614999996246, + "opsPerSec": 8647.701296881798 }, "note": "30 lines with SGR codes" }, @@ -953,15 +953,15 @@ "name": "terminal/large", "kind": "time", "unit": "ms/op", - "value": 6.56851800000004, + "value": 6.180776000004698, "stats": { - "iterations": 57, - "samples": 57, - "min": 5.999559000003501, - "median": 6.56851800000004, - "p95": 8.427512000002022, - "max": 9.987591000004613, - "opsPerSec": 152.24134271992463 + "iterations": 60, + "samples": 60, + "min": 5.725174999999581, + "median": 6.180776000004698, + "p95": 9.395330349995492, + "max": 11.911801999995077, + "opsPerSec": 161.79198210697814 }, "note": "1500 lines with SGR codes" }, @@ -977,15 +977,15 @@ "name": "diff/small", "kind": "time", "unit": "ms/op", - "value": 5.528104999997595, + "value": 4.939244999994116, "stats": { - "iterations": 52, - "samples": 52, - "min": 4.861931999999797, - "median": 5.528104999997595, - "p95": 7.447519500004999, - "max": 9.025180000004184, - "opsPerSec": 180.8938144265413 + "iterations": 57, + "samples": 57, + "min": 4.735113000002457, + "median": 4.939244999994116, + "p95": 7.273367600000347, + "max": 8.019012000004295, + "opsPerSec": 202.4600925852415 }, "note": "368 B patch" }, @@ -1001,15 +1001,15 @@ "name": "diff/large", "kind": "time", "unit": "ms/op", - "value": 251.4284370000023, + "value": 225.57247300000017, "stats": { "iterations": 7, "samples": 7, - "min": 235.10585300000093, - "median": 251.4284370000023, - "p95": 271.61501959999805, - "max": 274.98200599999836, - "opsPerSec": 3.9772748537588485 + "min": 221.74022500000137, + "median": 225.57247300000017, + "p95": 232.30616450000306, + "max": 234.6491060000044, + "opsPerSec": 4.433165034281462 }, "note": "13086 B patch" }, @@ -1025,15 +1025,15 @@ "name": "html/small page wrap", "kind": "time", "unit": "ms/op", - "value": 0.0062783214285792345, + "value": 0.005994022026431638, "stats": { - "iterations": 59640, + "iterations": 64468, "samples": 284, - "min": 0.005764742857164016, - "median": 0.0062783214285792345, - "p95": 0.009416457142876986, - "max": 0.01127477142855198, - "opsPerSec": 159278.24202309072 + "min": 0.005577299559459661, + "median": 0.005994022026431638, + "p95": 0.007725456828195836, + "max": 0.008723092511014722, + "opsPerSec": 166832.88709823447 }, "note": "784 B source" }, @@ -1049,15 +1049,15 @@ "name": "html/large page wrap", "kind": "time", "unit": "ms/op", - "value": 0.006226331818172845, + "value": 0.005956285024145596, "stats": { - "iterations": 60940, - "samples": 277, - "min": 0.005593345454524916, - "median": 0.006226331818172845, - "p95": 0.008822907272730265, - "max": 0.01375391818182834, - "opsPerSec": 160608.20868577738 + "iterations": 64377, + "samples": 311, + "min": 0.005609386473418054, + "median": 0.005956285024145596, + "p95": 0.00790398792271049, + "max": 0.009521328502406078, + "opsPerSec": 167889.8837020389 }, "note": "37049 B source" }, @@ -1073,15 +1073,15 @@ "name": "mermaid/small page wrap", "kind": "time", "unit": "ms/op", - "value": 0.009985105691068783, + "value": 0.00943555405405818, "stats": { - "iterations": 37515, - "samples": 305, - "min": 0.009171333333334935, - "median": 0.009985105691068783, - "p95": 0.01504726666665865, - "max": 0.02245682113821906, - "opsPerSec": 100149.16526065957 + "iterations": 40848, + "samples": 276, + "min": 0.009005405405430891, + "median": 0.00943555405405818, + "p95": 0.012476626689206297, + "max": 0.01822427027026778, + "opsPerSec": 105982.11766588369 }, "note": "250 B source" }, @@ -1097,15 +1097,15 @@ "name": "mermaid/large page wrap", "kind": "time", "unit": "ms/op", - "value": 0.02121194444437909, + "value": 0.020245792207859197, "stats": { - "iterations": 18216, - "samples": 253, - "min": 0.01983090277776177, - "median": 0.02121194444437909, - "p95": 0.029159594444495774, - "max": 0.038419055555500056, - "opsPerSec": 47143.25000341909 + "iterations": 19173, + "samples": 249, + "min": 0.019308870129873854, + "median": 0.020245792207859197, + "p95": 0.02774856883114772, + "max": 0.03160487012990581, + "opsPerSec": 49392.979525484356 }, "note": "3819 B source" }, @@ -1121,15 +1121,15 @@ "name": "theme switch re-render (10 mixed surfaces)", "kind": "time", "unit": "ms/op", - "value": 99.8628200000021, + "value": 93.0356180000017, "stats": { "iterations": 7, "samples": 7, - "min": 98.11342799999693, - "median": 99.8628200000021, - "p95": 103.0595550000049, - "max": 103.2796770000059, - "opsPerSec": 10.013736844202667 + "min": 90.29730600000039, + "median": 93.0356180000017, + "p95": 94.13107479999962, + "max": 94.32709299999988, + "opsPerSec": 10.748571584701912 }, "note": "burst cost when the workspace theme changes" }, @@ -1138,15 +1138,15 @@ "name": "typical: GET /api/sessions", "kind": "time", "unit": "ms/op", - "value": 0.2943039999991015, + "value": 0.2604277499995078, "stats": { - "iterations": 1190, - "samples": 595, - "min": 0.2344069999999192, - "median": 0.2943039999991015, - "p95": 0.43002094999901597, - "max": 5.1711114999998244, - "opsPerSec": 3397.8471240725676 + "iterations": 1428, + "samples": 714, + "min": 0.21896250000281725, + "median": 0.2604277499995078, + "p95": 0.32141255000060487, + "max": 2.86197650000031, + "opsPerSec": 3839.836576562559 }, "note": "144 posts / 96 comments" }, @@ -1163,15 +1163,15 @@ "name": "typical: GET /api/posts/recent?limit=20", "kind": "time", "unit": "ms/op", - "value": 1.8851049999975658, + "value": 1.7669479999967734, "stats": { - "iterations": 194, - "samples": 194, - "min": 1.6733889999959501, - "median": 1.8851049999975658, - "p95": 2.68266625000615, - "max": 6.822669000001042, - "opsPerSec": 530.4744298069822 + "iterations": 213, + "samples": 213, + "min": 1.6660439999977825, + "median": 1.7669479999967734, + "p95": 1.9562640000018285, + "max": 5.916741000000911, + "opsPerSec": 565.947611362545 }, "note": "144 posts / 96 comments" }, @@ -1188,15 +1188,15 @@ "name": "typical: GET /api/sessions/:id/posts", "kind": "time", "unit": "ms/op", - "value": 1.0049739999994927, + "value": 0.9479025000000547, "stats": { - "iterations": 362, - "samples": 362, - "min": 0.8916099999987637, - "median": 1.0049739999994927, - "p95": 1.5954183500023646, - "max": 4.1199889999988955, - "opsPerSec": 995.0506182254514 + "iterations": 404, + "samples": 202, + "min": 0.8911774999978661, + "median": 0.9479025000000547, + "p95": 1.5920816749983675, + "max": 1.7638015000011364, + "opsPerSec": 1054.9608213924346 }, "note": "144 posts / 96 comments" }, @@ -1213,15 +1213,15 @@ "name": "typical: GET /api/sessions/:id/posts?hydrate=1", "kind": "time", "unit": "ms/op", - "value": 0.7676956666667442, + "value": 0.7019473333324034, "stats": { - "iterations": 489, - "samples": 163, - "min": 0.6834593333333032, - "median": 0.7676956666667442, - "p95": 1.3966703333334107, - "max": 1.6033759999991162, - "opsPerSec": 1302.5995110040121 + "iterations": 543, + "samples": 181, + "min": 0.667041666667501, + "median": 0.7019473333324034, + "p95": 1.192501666667037, + "max": 1.3312329999995807, + "opsPerSec": 1424.608303948717 }, "note": "144 posts / 96 comments" }, @@ -1238,15 +1238,15 @@ "name": "typical: GET /api/comments?session", "kind": "time", "unit": "ms/op", - "value": 0.15433637499972974, + "value": 0.1307765624997046, "stats": { - "iterations": 2408, - "samples": 301, - "min": 0.11185837499942863, - "median": 0.15433637499972974, - "p95": 0.21399612499953946, - "max": 0.7090522500002407, - "opsPerSec": 6479.353943629628 + "iterations": 2752, + "samples": 344, + "min": 0.1052349999999933, + "median": 0.1307765624997046, + "p95": 0.17934354374992834, + "max": 0.6950582499994198, + "opsPerSec": 7646.630106233744 }, "note": "144 posts / 96 comments" }, @@ -1263,15 +1263,15 @@ "name": "heavy: GET /api/sessions", "kind": "time", "unit": "ms/op", - "value": 0.352337599999737, + "value": 0.3410856000002241, "stats": { - "iterations": 1065, - "samples": 213, - "min": 0.3065810000000056, - "median": 0.352337599999737, - "p95": 0.4533312399999704, - "max": 1.429665199998999, - "opsPerSec": 2838.187011550134 + "iterations": 1095, + "samples": 219, + "min": 0.3052270000000135, + "median": 0.3410856000002241, + "p95": 0.404193620000442, + "max": 1.5259619999997085, + "opsPerSec": 2931.8153566123665 }, "note": "900 posts / 600 comments" }, @@ -1288,15 +1288,15 @@ "name": "heavy: GET /api/posts/recent?limit=20", "kind": "time", "unit": "ms/op", - "value": 2.636497999996209, + "value": 2.652928999999858, "stats": { "iterations": 144, "samples": 144, - "min": 2.503145000002405, - "median": 2.636497999996209, - "p95": 4.405809349999253, - "max": 4.6635649999952875, - "opsPerSec": 379.2910140654148 + "min": 2.4809669999958714, + "median": 2.652928999999858, + "p95": 4.534050999994361, + "max": 5.033357000000251, + "opsPerSec": 376.9418631256447 }, "note": "900 posts / 600 comments" }, @@ -1313,15 +1313,15 @@ "name": "heavy: GET /api/sessions/:id/posts", "kind": "time", "unit": "ms/op", - "value": 3.487602999997762, + "value": 3.5445119999931194, "stats": { - "iterations": 110, - "samples": 110, - "min": 3.282379000003857, - "median": 3.487602999997762, - "p95": 5.109663499998351, - "max": 5.550014000000374, - "opsPerSec": 286.7298829599131 + "iterations": 107, + "samples": 107, + "min": 3.376322000003711, + "median": 3.5445119999931194, + "p95": 5.488593799999944, + "max": 6.242750999997952, + "opsPerSec": 282.1262842393935 }, "note": "900 posts / 600 comments" }, @@ -1338,15 +1338,15 @@ "name": "heavy: GET /api/sessions/:id/posts?hydrate=1", "kind": "time", "unit": "ms/op", - "value": 2.5542610000047716, + "value": 2.5840709999974933, "stats": { - "iterations": 143, - "samples": 143, - "min": 2.3615519999948447, - "median": 2.5542610000047716, - "p95": 4.419611799997074, - "max": 6.281531999993604, - "opsPerSec": 391.5026694602204 + "iterations": 141, + "samples": 141, + "min": 2.4076979999954347, + "median": 2.5840709999974933, + "p95": 4.489228999998886, + "max": 4.8457909999997355, + "opsPerSec": 386.9862708884431 }, "note": "900 posts / 600 comments" }, @@ -1363,15 +1363,15 @@ "name": "heavy: GET /api/comments?session", "kind": "time", "unit": "ms/op", - "value": 0.18444327777777086, + "value": 0.16878837500007648, "stats": { - "iterations": 1944, - "samples": 216, - "min": 0.15777333333406002, - "median": 0.18444327777777086, - "p95": 0.2629653888889152, - "max": 0.8409713333328708, - "opsPerSec": 5421.721041006788 + "iterations": 2136, + "samples": 267, + "min": 0.15077012500023557, + "median": 0.16878837500007648, + "p95": 0.20876519999983428, + "max": 0.9712487500000861, + "opsPerSec": 5924.578632856362 }, "note": "900 posts / 600 comments" }, @@ -1388,15 +1388,15 @@ "name": "POST /api/posts (publish)", "kind": "time", "unit": "ms/op", - "value": 0.49710112500088144, + "value": 0.4546188333333703, "stats": { - "iterations": 712, - "samples": 178, - "min": 0.42816449999918405, - "median": 0.49710112500088144, - "p95": 0.6046600000001491, - "max": 4.053106500001377, - "opsPerSec": 2011.6631198495616 + "iterations": 750, + "samples": 250, + "min": 0.4157623333327744, + "median": 0.4546188333333703, + "p95": 0.5489607333335393, + "max": 5.074804999998984, + "opsPerSec": 2199.6449039908202 }, "note": "144 posts" }, @@ -1405,15 +1405,15 @@ "name": "PUT /api/posts/:id (revise)", "kind": "time", "unit": "ms/op", - "value": 0.7600546666665953, + "value": 0.7054755000002236, "stats": { - "iterations": 480, - "samples": 160, - "min": 0.6913466666665045, - "median": 0.7600546666665953, - "p95": 1.2115256166653123, - "max": 2.3452593333325544, - "opsPerSec": 1315.6948359856053 + "iterations": 528, + "samples": 176, + "min": 0.6634960000010324, + "median": 0.7054755000002236, + "p95": 0.8444310833347117, + "max": 1.9035213333312033, + "opsPerSec": 1417.4836688158314 }, "note": "144 posts" }, @@ -1422,15 +1422,15 @@ "name": "POST /api/comments", "kind": "time", "unit": "ms/op", - "value": 0.641979750000246, + "value": 0.6060996666662201, "stats": { - "iterations": 560, - "samples": 280, - "min": 0.5738304999977117, - "median": 0.641979750000246, - "p95": 0.8144162000004146, - "max": 2.629928500002279, - "opsPerSec": 1557.681531231502 + "iterations": 603, + "samples": 201, + "min": 0.5570519999988998, + "median": 0.6060996666662201, + "p95": 1.5059876666685643, + "max": 1.9170019999995322, + "opsPerSec": 1649.8936643545483 }, "note": "144 posts" }, @@ -1439,15 +1439,15 @@ "name": "GET /s/:id html (cache hit)", "kind": "time", "unit": "ms/op", - "value": 0.19807200000026828, + "value": 0.1616573999999673, "stats": { - "iterations": 1757, - "samples": 251, - "min": 0.16414628571406606, - "median": 0.19807200000026828, - "p95": 0.29082928571473793, - "max": 1.3039982857143124, - "opsPerSec": 5048.669170799737 + "iterations": 2210, + "samples": 221, + "min": 0.14280760000037845, + "median": 0.1616573999999673, + "p95": 0.220179000000644, + "max": 0.7946782999999413, + "opsPerSec": 6185.921584784874 }, "note": "render-cache hit" }, @@ -1456,22 +1456,22 @@ "name": "GET /s/:id html bytes", "kind": "bytes", "unit": "bytes", - "value": 11966 + "value": 11052 }, { "suite": "api", "name": "GET /s/:id markdown (cache hit)", "kind": "time", "unit": "ms/op", - "value": 0.19946578571450246, + "value": 0.16925133333360362, "stats": { - "iterations": 1792, - "samples": 256, - "min": 0.17621114285741765, - "median": 0.19946578571450246, - "p95": 0.2401052857141102, - "max": 1.121802000000441, - "opsPerSec": 5013.39112579092 + "iterations": 2187, + "samples": 243, + "min": 0.1546691111111108, + "median": 0.16925133333360362, + "p95": 0.1989401111115713, + "max": 0.6584673333337479, + "opsPerSec": 5908.372952246972 }, "note": "render-cache hit" }, @@ -1480,22 +1480,22 @@ "name": "GET /s/:id markdown bytes", "kind": "bytes", "unit": "bytes", - "value": 11545 + "value": 11246 }, { "suite": "api", "name": "GET /s/:id code (cache hit)", "kind": "time", "unit": "ms/op", - "value": 0.36957859999965875, + "value": 0.32431929999947895, "stats": { - "iterations": 995, - "samples": 199, - "min": 0.33679620000038996, - "median": 0.36957859999965875, - "p95": 0.5435153200002968, - "max": 1.3997988000002806, - "opsPerSec": 2705.7843717166616 + "iterations": 1150, + "samples": 230, + "min": 0.30191780000022844, + "median": 0.32431929999947895, + "p95": 0.43222877000014687, + "max": 0.965800800001307, + "opsPerSec": 3083.381100050495 }, "note": "render-cache hit" }, @@ -1504,22 +1504,22 @@ "name": "GET /s/:id code bytes", "kind": "bytes", "unit": "bytes", - "value": 35097 + "value": 34798 }, { "suite": "api", "name": "GET /s/:id diff (cache hit)", "kind": "time", "unit": "ms/op", - "value": 0.51357700000032, + "value": 0.4791334999990795, "stats": { - "iterations": 723, - "samples": 241, - "min": 0.4786696666681867, - "median": 0.51357700000032, - "p95": 0.6817536666664333, - "max": 1.756462333331001, - "opsPerSec": 1947.127694580125 + "iterations": 788, + "samples": 197, + "min": 0.4546470000004774, + "median": 0.4791334999990795, + "p95": 0.8184851500001968, + "max": 0.9328232499992737, + "opsPerSec": 2087.10098542874 }, "note": "render-cache hit" }, @@ -1528,22 +1528,22 @@ "name": "GET /s/:id diff bytes", "kind": "bytes", "unit": "bytes", - "value": 61819 + "value": 61579 }, { "suite": "api", "name": "GET /s/:id terminal (cache hit)", "kind": "time", "unit": "ms/op", - "value": 0.19818199999956831, + "value": 0.1749382142858979, "stats": { - "iterations": 1764, - "samples": 252, - "min": 0.17413271428605576, - "median": 0.19818199999956831, - "p95": 0.33540783571427574, - "max": 1.164485714286067, - "opsPerSec": 5045.866930408302 + "iterations": 2086, + "samples": 298, + "min": 0.1603950000000103, + "median": 0.1749382142858979, + "p95": 0.20851737142807647, + "max": 0.8390112857149299, + "opsPerSec": 5716.303919541106 }, "note": "render-cache hit" }, @@ -1552,22 +1552,22 @@ "name": "GET /s/:id terminal bytes", "kind": "bytes", "unit": "bytes", - "value": 11410 + "value": 11148 }, { "suite": "api", "name": "GET /s/:id html (cache miss)", "kind": "time", "unit": "ms/op", - "value": 0.2058208571428882, + "value": 0.18619768750022558, "stats": { - "iterations": 1155, - "samples": 165, - "min": 0.1875468571420892, - "median": 0.2058208571428882, - "p95": 0.8027861142857311, - "max": 1.6305274285717002, - "opsPerSec": 4858.594089450148 + "iterations": 1264, + "samples": 158, + "min": 0.17052699999931065, + "median": 0.18619768750022558, + "p95": 0.4421718437502496, + "max": 1.4917308750000302, + "opsPerSec": 5370.635980636379 }, "note": "forced re-render" }, @@ -1576,15 +1576,15 @@ "name": "GET /s/:id markdown (cache miss)", "kind": "time", "unit": "ms/op", - "value": 3.4799459999994724, + "value": 3.423541500000283, "stats": { - "iterations": 81, - "samples": 81, - "min": 3.323792000002868, - "median": 3.4799459999994724, - "p95": 5.63822199999413, - "max": 5.967258000004222, - "opsPerSec": 287.36078088572395 + "iterations": 84, + "samples": 84, + "min": 3.284511999998358, + "median": 3.423541500000283, + "p95": 5.247293250000075, + "max": 5.771963000006508, + "opsPerSec": 292.09518856421556 }, "note": "forced re-render" }, @@ -1593,15 +1593,15 @@ "name": "GET /s/:id code (cache miss)", "kind": "time", "unit": "ms/op", - "value": 38.454201500000636, + "value": 37.0162049999999, "stats": { - "iterations": 8, - "samples": 8, - "min": 36.23347900000226, - "median": 38.454201500000636, - "p95": 39.59185875000003, - "max": 39.854693000001134, - "opsPerSec": 26.004960732313826 + "iterations": 9, + "samples": 9, + "min": 35.73860499999864, + "median": 37.0162049999999, + "p95": 38.71794220000011, + "max": 39.269735000001674, + "opsPerSec": 27.01519510171296 }, "note": "forced re-render" }, @@ -1610,15 +1610,15 @@ "name": "GET /s/:id diff (cache miss)", "kind": "time", "unit": "ms/op", - "value": 5.479161999999633, + "value": 5.172302999999374, "stats": { - "iterations": 52, - "samples": 52, - "min": 5.148033999998006, - "median": 5.479161999999633, - "p95": 7.8641177000015885, - "max": 9.085332999995444, - "opsPerSec": 182.50966114892515 + "iterations": 55, + "samples": 55, + "min": 5.031182000006083, + "median": 5.172302999999374, + "p95": 7.829863499994098, + "max": 9.734932999999728, + "opsPerSec": 193.33747462206313 }, "note": "forced re-render" }, @@ -1627,15 +1627,15 @@ "name": "GET /s/:id terminal (cache miss)", "kind": "time", "unit": "ms/op", - "value": 0.4143813333333431, + "value": 0.4073397499996645, "stats": { - "iterations": 609, - "samples": 203, - "min": 0.3752163333338103, - "median": 0.4143813333333431, - "p95": 1.2993985999998712, - "max": 2.125185999999909, - "opsPerSec": 2413.2361174569714 + "iterations": 652, + "samples": 163, + "min": 0.3828452500001731, + "median": 0.4073397499996645, + "p95": 1.0642420500003937, + "max": 1.396208750000369, + "opsPerSec": 2454.9531441525746 }, "note": "forced re-render" }, @@ -1644,15 +1644,15 @@ "name": "GET / (viewer document)", "kind": "time", "unit": "ms/op", - "value": 0.02684650999995938, + "value": 0.02477656779654168, "stats": { - "iterations": 13700, - "samples": 274, - "min": 0.02316336000003503, - "median": 0.02684650999995938, - "p95": 0.04629047799993712, - "max": 0.06190402000007453, - "opsPerSec": 37248.78950751934 + "iterations": 14868, + "samples": 252, + "min": 0.02279196610174564, + "median": 0.02477656779654168, + "p95": 0.03969537966099855, + "max": 0.06435677966103451, + "opsPerSec": 40360.71534248502 }, "note": "in-memory single-file viewer" }, @@ -1661,7 +1661,7 @@ "name": "render cache heap (64 mixed surfaces)", "kind": "memory", "unit": "bytes", - "value": 2779704, + "value": 2795512, "note": "cache holds up to 512 entries" }, { @@ -1677,15 +1677,15 @@ "name": "bus.broadcast → 1 subscribers", "kind": "time", "unit": "ms/op", - "value": 0.00011540424631020816, + "value": 0.00011390950993383426, "stats": { - "iterations": 3273336, - "samples": 198, - "min": 0.00009843787805490364, - "median": 0.00011540424631020816, - "p95": 0.00014854479494300386, - "max": 0.0001899944350350349, - "opsPerSec": 8665192.416854287 + "iterations": 3378625, + "samples": 179, + "min": 0.00010170272847680709, + "median": 0.00011390950993383426, + "p95": 0.00013396627284776856, + "max": 0.00018834558940401596, + "opsPerSec": 8778898.272680325 }, "note": "1 listeners" }, @@ -1694,15 +1694,15 @@ "name": "bus.broadcast → 10 subscribers", "kind": "time", "unit": "ms/op", - "value": 0.00016940602503061325, + "value": 0.00016594739723329558, "stats": { - "iterations": 2239192, - "samples": 248, - "min": 0.0001450443016939769, - "median": 0.00016940602503061325, - "p95": 0.00022877351866189166, - "max": 0.00026348931221635807, - "opsPerSec": 5902977.7708277535 + "iterations": 2350998, + "samples": 234, + "min": 0.00014979566039577635, + "median": 0.00016594739723329558, + "p95": 0.00020288548322877447, + "max": 0.0002554353538373657, + "opsPerSec": 6026005.931229879 }, "note": "10 listeners" }, @@ -1711,15 +1711,15 @@ "name": "bus.broadcast → 100 subscribers", "kind": "time", "unit": "ms/op", - "value": 0.000663621452422953, + "value": 0.0006580681922217912, "stats": { - "iterations": 594208, - "samples": 248, - "min": 0.0006222683639403682, - "median": 0.000663621452422953, - "p95": 0.0007566636894812296, - "max": 0.0009908580968278605, - "opsPerSec": 1506883.1731537504 + "iterations": 596505, + "samples": 273, + "min": 0.0006251395881006566, + "median": 0.0006580681922217912, + "p95": 0.0007242100686500875, + "max": 0.0010049331807787157, + "opsPerSec": 1519599.354321879 }, "note": "100 listeners" }, @@ -1728,15 +1728,15 @@ "name": "publish → SSE delivery to 1 tabs", "kind": "time", "unit": "ms/op", - "value": 0.575905333332533, + "value": 0.5570706666670352, "stats": { - "iterations": 624, - "samples": 208, - "min": 0.4647973333315652, - "median": 0.575905333332533, - "p95": 0.7826120333333764, - "max": 4.330366666666426, - "opsPerSec": 1736.3964910923837 + "iterations": 666, + "samples": 222, + "min": 0.4525090000000394, + "median": 0.5570706666670352, + "p95": 0.6692637333344463, + "max": 4.010468666667293, + "opsPerSec": 1795.1043913028482 }, "note": "1 open streams" }, @@ -1753,15 +1753,15 @@ "name": "publish → SSE delivery to 5 tabs", "kind": "time", "unit": "ms/op", - "value": 0.5499659999998887, + "value": 0.5657738333332721, "stats": { - "iterations": 654, - "samples": 218, - "min": 0.48065666666661855, - "median": 0.5499659999998887, - "p95": 0.6935038666681063, - "max": 3.389102333332024, - "opsPerSec": 1818.2942218249898 + "iterations": 612, + "samples": 204, + "min": 0.5066080000009, + "median": 0.5657738333332721, + "p95": 0.7991219666649461, + "max": 3.4259690000011083, + "opsPerSec": 1767.490719937457 }, "note": "5 open streams" }, @@ -1778,15 +1778,15 @@ "name": "publish → SSE delivery to 20 tabs", "kind": "time", "unit": "ms/op", - "value": 0.6798331666674737, + "value": 0.6637884999994033, "stats": { - "iterations": 504, - "samples": 168, - "min": 0.6147533333326768, - "median": 0.6798331666674737, - "p95": 1.5987446833343733, - "max": 2.4767419999989215, - "opsPerSec": 1470.9491225648442 + "iterations": 546, + "samples": 182, + "min": 0.6101163333344933, + "median": 0.6637884999994033, + "p95": 0.8761031833337861, + "max": 2.3414170000008503, + "opsPerSec": 1506.5039541976082 }, "note": "20 open streams" }, @@ -1803,7 +1803,7 @@ "name": "heap per open SSE connection", "kind": "memory", "unit": "bytes", - "value": 3226, + "value": 5598, "note": "measured across 20 concurrent streams" }, { @@ -1811,15 +1811,15 @@ "name": "comment long-poll wakeup latency", "kind": "time", "unit": "ms/op", - "value": 0.6885714999989432, + "value": 0.6973897500010935, "stats": { - "iterations": 526, - "samples": 263, - "min": 0.5753264999984822, - "median": 0.6885714999989432, - "p95": 0.8695627499986585, - "max": 4.8331154999978025, - "opsPerSec": 1452.282007026917 + "iterations": 524, + "samples": 262, + "min": 0.583179499997641, + "median": 0.6973897500010935, + "p95": 0.8723623500018218, + "max": 4.8926529999989725, + "opsPerSec": 1433.918407889465 }, "note": "post → parked agent wakes" } diff --git a/bench/suites/api.bench.ts b/bench/suites/api.bench.ts index 1c5ae7a3..d52fa8a4 100644 --- a/bench/suites/api.bench.ts +++ b/bench/suites/api.bench.ts @@ -160,8 +160,13 @@ export const apiSuite: Suite = { } const app = makeApp(store); + // The viewer appends `&theme=&mode=` to every surface iframe src (Card.tsx), + // and the server renders differently when the mode is pinned — so a bench + // that omits them measures a URL shape no viewer ever sends, and would score + // a change to the pinned path as no change at all. Match the real client. + const viewerQuery = "part=0&theme=github&mode=dark"; for (const [kind, id] of Object.entries(perKind)) { - const path = `/s/${id}?part=0`; + const path = `/s/${id}?${viewerQuery}`; // Warm: every request after the first hits the memoized document. await ctx.time(`GET /s/:id ${kind} (cache hit)`, () => hit(app, path), { note: "render-cache hit", @@ -179,7 +184,7 @@ export const apiSuite: Suite = { let n = 0; await ctx.time( `GET /s/:id ${kind} (cache miss)`, - () => hit(app, `/s/${id}?part=0&theme=bench-${n++}`), + () => hit(app, `/s/${id}?part=0&mode=dark&theme=bench-${n++}`), { note: "forced re-render", minSamples: 7, minMs: 300 }, ); }