From 2f55825c9479b7a4ad3814cc93023c55b7237410 Mon Sep 17 00:00:00 2001 From: Vladimir Sheremet Date: Mon, 3 Aug 2026 16:10:15 +0200 Subject: [PATCH] feat: add long-haul, the worker-lifetime endurance app 80 jsdom test files through 2 workers so each worker serves 40 files. Every file's world holds a ~15MB module-level dataset and renders DOM tables over it, and the committed config pins vmMemoryLimit to 512MB so vm workers recycle several times per run on any machine. Short fixtures structurally understate worker-lifetime behavior: node pools rebuild the environment and re-import externalized dependencies for every file, while vm pool workers amortize both across their run and periodically pay for recycling. This is the first app where the vm pools win by a wide margin (5.9s vs 19.2s on jsdom) and the first one that enters the worker recycle path at all. World retention is deliberately out of scope: workers report lazy heap numbers, so leak regressions belong to Vitest's own reachability tests rather than wall clock. --- README.md | 15 ++- apps/long-haul/generate.mjs | 161 ++++++++++++++++++++++++++++++++ apps/long-haul/package.json | 17 ++++ apps/long-haul/vitest.config.ts | 14 +++ pnpm-lock.yaml | 21 +++++ scripts/matrix.mjs | 13 +++ 6 files changed, 240 insertions(+), 1 deletion(-) create mode 100644 apps/long-haul/generate.mjs create mode 100644 apps/long-haul/package.json create mode 100644 apps/long-haul/vitest.config.ts diff --git a/README.md b/README.md index 040d176..6bb27ce 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # vitest benchmarks -Generated reference apps for measuring Vitest performance. Each app models a real category of project — tiny utility packages, libraries, barrel-file graphs, DOM component suites, dependency-heavy services, a 1300-module monolith — and the bench runner measures the options that move run time against each of them: `pool`, `environment` (jsdom, happy-dom and headless Chromium via browser mode), `isolate`, `fsModuleCache`, `maxWorkers`, cold vs warm caches. +Generated reference apps for measuring Vitest performance. Each app models a real category of project — tiny utility packages, libraries, barrel-file graphs, DOM component suites, dependency-heavy services, a 1300-module monolith, a long-haul DOM suite that ages its workers — and the bench runner measures the options that move run time against each of them: `pool`, `environment` (jsdom, happy-dom and headless Chromium via browser mode), `isolate`, `fsModuleCache`, `maxWorkers`, cold vs warm caches. ## Usage @@ -204,6 +204,19 @@ Big-repo CI: ~1280 modules with 12-deep import chains, import cycles, path alias | threads | false | true | default | 2.44s | 2.11s | | forks | false | false | 50% | — | 3.26s | +### long-haul + +The worker-lifetime endurance fixture: 80 jsdom test files through 2 workers, every file holding a ~15MB module-level dataset and rendering tables over it. Node pools rebuild the environment and re-import the externalized dependencies for each of a worker's 40 files; vm pool workers pay once, reuse compiled scripts across contexts, and get recycled several times per run by the pinned 512MB `vmMemoryLimit` — the recycle path no other app enters. Short fixtures understate the vm pools; this is the fixture where they win by a wide margin. (World *retention* is deliberately out of scope: workers report lazy heap numbers, so leak regressions are covered by Vitest's own reachability tests, not wall clock.) + +| pool | env | isolate | workers | cold | warm | +|---|---|---|---|---:|---:| +| forks | jsdom | true | 2 | — | 19.24s | +| threads | jsdom | true | 2 | — | 17.32s | +| vmThreads | jsdom | true | 2 | — | 5.89s | +| vmForks | jsdom | true | 2 | 6.04s | 6.08s | +| forks | happy-dom | true | 2 | — | 12.13s | +| vmForks | happy-dom | true | 2 | — | 5.67s | + ### cpu-bound 30 test files that burn real CPU (hashing, sieving, matrix multiplication) on an 8-module graph. The tests themselves dominate, so only scheduling — `maxWorkers`, pool choice — changes anything. diff --git a/apps/long-haul/generate.mjs b/apps/long-haul/generate.mjs new file mode 100644 index 0000000..d06997d --- /dev/null +++ b/apps/long-haul/generate.mjs @@ -0,0 +1,161 @@ +// long-haul — the worker-lifetime endurance fixture. +// +// Every other app in this suite finishes before a worker has lived long +// enough for worker-lifetime behavior to matter. This one pushes 80 jsdom +// test files through 2 workers, so each worker serves 40 files in one +// process and everything a worker amortizes or accumulates gets 40 chances +// to show up: +// +// - amortization: node pools re-create the environment and re-import the +// externalized dependencies for every file; vm pool workers pay once and +// reuse compiled scripts across contexts. Short fixtures understate this +// advantage; here it decides the result. +// - recycling: every file's world holds a ~15MB module-level dataset, and +// the committed config pins `vmMemoryLimit` to 512MB, so vm workers are +// recycled several times per run on any machine. No other app enters the +// recycle path at all; a regression that makes recycling expensive (it +// tears an isolate down in-process on vmThreads) lands here. +// +// Wall clock cannot see world *retention* at this scale — the worker reports +// lazy heap numbers and V8 collects when it pleases — so leak regressions +// are the job of vitest's own reachability tests, not this fixture. What +// this fixture answers is: which pool should a large DOM suite use, once +// workers live long enough for the answer to change. +import { createApp } from '../../tools/generator/helpers.mjs' + +const FEATURES = 80 +const ROWS = 60_000 +const RENDERED_ROWS = 600 + +const app = createApp(import.meta.url) + +app.write('src/shared/format.ts', `export function formatScore(score: number): string { + return score >= 900 ? \`\${score} (top)\` : String(score) +} + +export function formatKey(key: string, bucket: number): string { + return \`\${key}/\${bucket.toString(16)}\` +} +`) + +for (let i = 0; i < FEATURES; i++) { + app.write(`src/feature${i}/data.ts`, `export interface Row { + id: number + key: string + score: number + active: boolean + tags: string[] + meta: { weight: number, bucket: number } +} + +// module-level state: alive for as long as this file's world is reachable +export const rows: Row[] = Array.from({ length: ${ROWS} }, (_, k) => ({ + id: k, + key: \`k\${(k * 31 + ${i}) % 9973}\`, + score: (k * 7 + ${i}) % 1000, + active: k % 3 === 0, + tags: [\`t\${k % 11}\`, \`t\${(k + ${i}) % 13}\`], + meta: { weight: (k % 97) / 97, bucket: k % 16 }, +})) +`) + + app.write(`src/feature${i}/logic.ts`, `import type { Row } from './data' +import { rows } from './data' + +export function topScores(limit: number): Row[] { + return [...rows].sort((a, b) => b.score - a.score).slice(0, limit) +} + +export function countByBucket(): Map { + const counts = new Map() + for (const row of rows) + counts.set(row.meta.bucket, (counts.get(row.meta.bucket) ?? 0) + 1) + return counts +} + +export function activeWithTag(tag: string): number { + let total = 0 + for (const row of rows) { + if (row.active && row.tags.includes(tag)) + total++ + } + return total +} +`) + + app.write(`src/feature${i}/view.ts`, `import { formatKey, formatScore } from '../shared/format' +import { rows } from './data' + +export function renderTable(target: HTMLElement): HTMLTableElement { + const table = document.createElement('table') + table.className = 'feature${i}' + const body = document.createElement('tbody') + for (const row of rows.slice(0, ${RENDERED_ROWS})) { + const tr = document.createElement('tr') + tr.className = row.active ? 'row active' : 'row' + tr.dataset.id = String(row.id) + const key = document.createElement('td') + key.textContent = formatKey(row.key, row.meta.bucket) + const score = document.createElement('td') + score.textContent = formatScore(row.score) + tr.append(key, score) + body.append(tr) + } + table.append(body) + target.append(table) + return table +} + +export function toggleRows(table: HTMLTableElement): number { + let toggled = 0 + for (const tr of table.querySelectorAll('tr.row')) { + tr.classList.toggle('selected') + toggled++ + } + return toggled +} +`) + + app.write(`tests/feature${i}.test.ts`, `import { screen } from '@testing-library/dom' +import { describe, expect, it } from 'vitest' +import { rows } from '../src/feature${i}/data' +import { activeWithTag, countByBucket, topScores } from '../src/feature${i}/logic' +import { renderTable, toggleRows } from '../src/feature${i}/view' + +describe('feature${i}', () => { + it('holds the full dataset', () => { + expect(rows).toHaveLength(${ROWS}) + expect(rows[${ROWS} - 1].id).toBe(${ROWS} - 1) + }) + + it('aggregates over every row', () => { + const counts = countByBucket() + expect([...counts.values()].reduce((a, b) => a + b, 0)).toBe(${ROWS}) + expect(topScores(5)[0].score).toBeGreaterThanOrEqual(topScores(5)[4].score) + expect(activeWithTag('t1')).toBeGreaterThan(0) + }) + + it('renders the table', () => { + const table = renderTable(document.body) + expect(table.querySelectorAll('tr.row')).toHaveLength(${RENDERED_ROWS}) + expect(screen.getAllByText(/\\(top\\)$/).length).toBeGreaterThan(0) + }) + + it('toggles every rendered row', () => { + const table = renderTable(document.body) + expect(toggleRows(table)).toBe(${RENDERED_ROWS}) + expect(table.querySelectorAll('tr.selected')).toHaveLength(${RENDERED_ROWS}) + }) +}) +`) +} + +app.write('tests/setup.ts', `import '@testing-library/jest-dom/vitest' +import { afterEach } from 'vitest' + +afterEach(() => { + document.body.innerHTML = '' +}) +`) + +app.report('long-haul', `${FEATURES} features, ${ROWS} rows each`) diff --git a/apps/long-haul/package.json b/apps/long-haul/package.json new file mode 100644 index 0000000..cea9f0a --- /dev/null +++ b/apps/long-haul/package.json @@ -0,0 +1,17 @@ +{ + "name": "@bench-app/long-haul", + "private": true, + "type": "module", + "scripts": { + "generate": "node generate.mjs", + "test": "vitest run" + }, + "devDependencies": { + "@testing-library/dom": "10.4.1", + "@testing-library/jest-dom": "6.9.1", + "happy-dom": "20.10.6", + "jsdom": "29.1.1", + "vite": "8.1.4", + "vitest": "4.1.10" + } +} diff --git a/apps/long-haul/vitest.config.ts b/apps/long-haul/vitest.config.ts new file mode 100644 index 0000000..a77cd4e --- /dev/null +++ b/apps/long-haul/vitest.config.ts @@ -0,0 +1,14 @@ +import { defineConfig } from 'vitest/config' +import { benchTest } from '../../tools/config/bench-config.js' + +export default defineConfig({ + test: { + ...benchTest({ + environment: 'jsdom', + setupFiles: ['./tests/setup.ts'], + }), + // pinned so recycling pressure is deterministic instead of scaling with + // the host's RAM — see generate.mjs + vmMemoryLimit: '512MB', + }, +}) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 8b621d8..8062cd0 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -155,6 +155,27 @@ importers: specifier: 4.1.10 version: 4.1.10(@types/node@26.1.1)(@vitest/browser-playwright@4.1.10)(@vitest/coverage-istanbul@4.1.10)(@vitest/coverage-v8@4.1.10)(happy-dom@20.10.6)(jsdom@29.1.1)(vite@8.1.4(@types/node@26.1.1)(yaml@2.9.0)) + apps/long-haul: + devDependencies: + '@testing-library/dom': + specifier: 10.4.1 + version: 10.4.1 + '@testing-library/jest-dom': + specifier: 6.9.1 + version: 6.9.1 + happy-dom: + specifier: 20.10.6 + version: 20.10.6 + jsdom: + specifier: 29.1.1 + version: 29.1.1 + vite: + specifier: 8.1.4 + version: 8.1.4(@types/node@26.1.1)(yaml@2.9.0) + vitest: + specifier: 4.1.10 + version: 4.1.10(@types/node@26.1.1)(@vitest/browser-playwright@4.1.10)(@vitest/coverage-istanbul@4.1.10)(@vitest/coverage-v8@4.1.10)(happy-dom@20.10.6)(jsdom@29.1.1)(vite@8.1.4(@types/node@26.1.1)(yaml@2.9.0)) + apps/micro-utils: devDependencies: '@vitest/coverage-istanbul': diff --git a/scripts/matrix.mjs b/scripts/matrix.mjs index 03a75c8..0c0cc7b 100644 --- a/scripts/matrix.mjs +++ b/scripts/matrix.mjs @@ -101,6 +101,19 @@ const APPS = { { pool: 'forks', env: 'node', isolate: f, fsCache: f, state: 'warm', workers: '50%' }, ], }, + 'long-haul': { + envs: ['jsdom', 'happy-dom'], + primary: 'jsdom', + workers: ['2'], + // 2 workers x 40 heavy files each: long enough worker lifetimes for + // retention, memory-limit recycling and aging to show up in wall time + dims: { pool: POOLS, env: ['jsdom'], isolate: [t], fsCache: [f], state: ['warm'], workers: ['2'] }, + extra: [ + { pool: 'vmForks', env: 'jsdom', isolate: t, fsCache: f, state: 'cold', workers: '2' }, + { pool: 'forks', env: 'happy-dom', isolate: t, fsCache: f, state: 'warm', workers: '2' }, + { pool: 'vmForks', env: 'happy-dom', isolate: t, fsCache: f, state: 'warm', workers: '2' }, + ], + }, 'cpu-bound': { envs: ['node'], primary: 'node',