diff --git a/README.md b/README.md index 040d176..6bb27ce 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # vitest benchmarks -Generated reference apps for measuring Vitest performance. Each app models a real category of project — tiny utility packages, libraries, barrel-file graphs, DOM component suites, dependency-heavy services, a 1300-module monolith — and the bench runner measures the options that move run time against each of them: `pool`, `environment` (jsdom, happy-dom and headless Chromium via browser mode), `isolate`, `fsModuleCache`, `maxWorkers`, cold vs warm caches. +Generated reference apps for measuring Vitest performance. Each app models a real category of project — tiny utility packages, libraries, barrel-file graphs, DOM component suites, dependency-heavy services, a 1300-module monolith, a long-haul DOM suite that ages its workers — and the bench runner measures the options that move run time against each of them: `pool`, `environment` (jsdom, happy-dom and headless Chromium via browser mode), `isolate`, `fsModuleCache`, `maxWorkers`, cold vs warm caches. ## Usage @@ -204,6 +204,19 @@ Big-repo CI: ~1280 modules with 12-deep import chains, import cycles, path alias | threads | false | true | default | 2.44s | 2.11s | | forks | false | false | 50% | — | 3.26s | +### long-haul + +The worker-lifetime endurance fixture: 80 jsdom test files through 2 workers, every file holding a ~15MB module-level dataset and rendering tables over it. Node pools rebuild the environment and re-import the externalized dependencies for each of a worker's 40 files; vm pool workers pay once, reuse compiled scripts across contexts, and get recycled several times per run by the pinned 512MB `vmMemoryLimit` — the recycle path no other app enters. Short fixtures understate the vm pools; this is the fixture where they win by a wide margin. (World *retention* is deliberately out of scope: workers report lazy heap numbers, so leak regressions are covered by Vitest's own reachability tests, not wall clock.) + +| pool | env | isolate | workers | cold | warm | +|---|---|---|---|---:|---:| +| forks | jsdom | true | 2 | — | 19.24s | +| threads | jsdom | true | 2 | — | 17.32s | +| vmThreads | jsdom | true | 2 | — | 5.89s | +| vmForks | jsdom | true | 2 | 6.04s | 6.08s | +| forks | happy-dom | true | 2 | — | 12.13s | +| vmForks | happy-dom | true | 2 | — | 5.67s | + ### cpu-bound 30 test files that burn real CPU (hashing, sieving, matrix multiplication) on an 8-module graph. The tests themselves dominate, so only scheduling — `maxWorkers`, pool choice — changes anything. diff --git a/apps/long-haul/generate.mjs b/apps/long-haul/generate.mjs new file mode 100644 index 0000000..d06997d --- /dev/null +++ b/apps/long-haul/generate.mjs @@ -0,0 +1,161 @@ +// long-haul — the worker-lifetime endurance fixture. +// +// Every other app in this suite finishes before a worker has lived long +// enough for worker-lifetime behavior to matter. This one pushes 80 jsdom +// test files through 2 workers, so each worker serves 40 files in one +// process and everything a worker amortizes or accumulates gets 40 chances +// to show up: +// +// - amortization: node pools re-create the environment and re-import the +// externalized dependencies for every file; vm pool workers pay once and +// reuse compiled scripts across contexts. Short fixtures understate this +// advantage; here it decides the result. +// - recycling: every file's world holds a ~15MB module-level dataset, and +// the committed config pins `vmMemoryLimit` to 512MB, so vm workers are +// recycled several times per run on any machine. No other app enters the +// recycle path at all; a regression that makes recycling expensive (it +// tears an isolate down in-process on vmThreads) lands here. +// +// Wall clock cannot see world *retention* at this scale — the worker reports +// lazy heap numbers and V8 collects when it pleases — so leak regressions +// are the job of vitest's own reachability tests, not this fixture. What +// this fixture answers is: which pool should a large DOM suite use, once +// workers live long enough for the answer to change. +import { createApp } from '../../tools/generator/helpers.mjs' + +const FEATURES = 80 +const ROWS = 60_000 +const RENDERED_ROWS = 600 + +const app = createApp(import.meta.url) + +app.write('src/shared/format.ts', `export function formatScore(score: number): string { + return score >= 900 ? \`\${score} (top)\` : String(score) +} + +export function formatKey(key: string, bucket: number): string { + return \`\${key}/\${bucket.toString(16)}\` +} +`) + +for (let i = 0; i < FEATURES; i++) { + app.write(`src/feature${i}/data.ts`, `export interface Row { + id: number + key: string + score: number + active: boolean + tags: string[] + meta: { weight: number, bucket: number } +} + +// module-level state: alive for as long as this file's world is reachable +export const rows: Row[] = Array.from({ length: ${ROWS} }, (_, k) => ({ + id: k, + key: \`k\${(k * 31 + ${i}) % 9973}\`, + score: (k * 7 + ${i}) % 1000, + active: k % 3 === 0, + tags: [\`t\${k % 11}\`, \`t\${(k + ${i}) % 13}\`], + meta: { weight: (k % 97) / 97, bucket: k % 16 }, +})) +`) + + app.write(`src/feature${i}/logic.ts`, `import type { Row } from './data' +import { rows } from './data' + +export function topScores(limit: number): Row[] { + return [...rows].sort((a, b) => b.score - a.score).slice(0, limit) +} + +export function countByBucket(): Map { + const counts = new Map() + for (const row of rows) + counts.set(row.meta.bucket, (counts.get(row.meta.bucket) ?? 0) + 1) + return counts +} + +export function activeWithTag(tag: string): number { + let total = 0 + for (const row of rows) { + if (row.active && row.tags.includes(tag)) + total++ + } + return total +} +`) + + app.write(`src/feature${i}/view.ts`, `import { formatKey, formatScore } from '../shared/format' +import { rows } from './data' + +export function renderTable(target: HTMLElement): HTMLTableElement { + const table = document.createElement('table') + table.className = 'feature${i}' + const body = document.createElement('tbody') + for (const row of rows.slice(0, ${RENDERED_ROWS})) { + const tr = document.createElement('tr') + tr.className = row.active ? 'row active' : 'row' + tr.dataset.id = String(row.id) + const key = document.createElement('td') + key.textContent = formatKey(row.key, row.meta.bucket) + const score = document.createElement('td') + score.textContent = formatScore(row.score) + tr.append(key, score) + body.append(tr) + } + table.append(body) + target.append(table) + return table +} + +export function toggleRows(table: HTMLTableElement): number { + let toggled = 0 + for (const tr of table.querySelectorAll('tr.row')) { + tr.classList.toggle('selected') + toggled++ + } + return toggled +} +`) + + app.write(`tests/feature${i}.test.ts`, `import { screen } from '@testing-library/dom' +import { describe, expect, it } from 'vitest' +import { rows } from '../src/feature${i}/data' +import { activeWithTag, countByBucket, topScores } from '../src/feature${i}/logic' +import { renderTable, toggleRows } from '../src/feature${i}/view' + +describe('feature${i}', () => { + it('holds the full dataset', () => { + expect(rows).toHaveLength(${ROWS}) + expect(rows[${ROWS} - 1].id).toBe(${ROWS} - 1) + }) + + it('aggregates over every row', () => { + const counts = countByBucket() + expect([...counts.values()].reduce((a, b) => a + b, 0)).toBe(${ROWS}) + expect(topScores(5)[0].score).toBeGreaterThanOrEqual(topScores(5)[4].score) + expect(activeWithTag('t1')).toBeGreaterThan(0) + }) + + it('renders the table', () => { + const table = renderTable(document.body) + expect(table.querySelectorAll('tr.row')).toHaveLength(${RENDERED_ROWS}) + expect(screen.getAllByText(/\\(top\\)$/).length).toBeGreaterThan(0) + }) + + it('toggles every rendered row', () => { + const table = renderTable(document.body) + expect(toggleRows(table)).toBe(${RENDERED_ROWS}) + expect(table.querySelectorAll('tr.selected')).toHaveLength(${RENDERED_ROWS}) + }) +}) +`) +} + +app.write('tests/setup.ts', `import '@testing-library/jest-dom/vitest' +import { afterEach } from 'vitest' + +afterEach(() => { + document.body.innerHTML = '' +}) +`) + +app.report('long-haul', `${FEATURES} features, ${ROWS} rows each`) diff --git a/apps/long-haul/package.json b/apps/long-haul/package.json new file mode 100644 index 0000000..cea9f0a --- /dev/null +++ b/apps/long-haul/package.json @@ -0,0 +1,17 @@ +{ + "name": "@bench-app/long-haul", + "private": true, + "type": "module", + "scripts": { + "generate": "node generate.mjs", + "test": "vitest run" + }, + "devDependencies": { + "@testing-library/dom": "10.4.1", + "@testing-library/jest-dom": "6.9.1", + "happy-dom": "20.10.6", + "jsdom": "29.1.1", + "vite": "8.1.4", + "vitest": "4.1.10" + } +} diff --git a/apps/long-haul/vitest.config.ts b/apps/long-haul/vitest.config.ts new file mode 100644 index 0000000..a77cd4e --- /dev/null +++ b/apps/long-haul/vitest.config.ts @@ -0,0 +1,14 @@ +import { defineConfig } from 'vitest/config' +import { benchTest } from '../../tools/config/bench-config.js' + +export default defineConfig({ + test: { + ...benchTest({ + environment: 'jsdom', + setupFiles: ['./tests/setup.ts'], + }), + // pinned so recycling pressure is deterministic instead of scaling with + // the host's RAM — see generate.mjs + vmMemoryLimit: '512MB', + }, +}) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 8b621d8..8062cd0 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -155,6 +155,27 @@ importers: specifier: 4.1.10 version: 4.1.10(@types/node@26.1.1)(@vitest/browser-playwright@4.1.10)(@vitest/coverage-istanbul@4.1.10)(@vitest/coverage-v8@4.1.10)(happy-dom@20.10.6)(jsdom@29.1.1)(vite@8.1.4(@types/node@26.1.1)(yaml@2.9.0)) + apps/long-haul: + devDependencies: + '@testing-library/dom': + specifier: 10.4.1 + version: 10.4.1 + '@testing-library/jest-dom': + specifier: 6.9.1 + version: 6.9.1 + happy-dom: + specifier: 20.10.6 + version: 20.10.6 + jsdom: + specifier: 29.1.1 + version: 29.1.1 + vite: + specifier: 8.1.4 + version: 8.1.4(@types/node@26.1.1)(yaml@2.9.0) + vitest: + specifier: 4.1.10 + version: 4.1.10(@types/node@26.1.1)(@vitest/browser-playwright@4.1.10)(@vitest/coverage-istanbul@4.1.10)(@vitest/coverage-v8@4.1.10)(happy-dom@20.10.6)(jsdom@29.1.1)(vite@8.1.4(@types/node@26.1.1)(yaml@2.9.0)) + apps/micro-utils: devDependencies: '@vitest/coverage-istanbul': diff --git a/scripts/matrix.mjs b/scripts/matrix.mjs index 03a75c8..0c0cc7b 100644 --- a/scripts/matrix.mjs +++ b/scripts/matrix.mjs @@ -101,6 +101,19 @@ const APPS = { { pool: 'forks', env: 'node', isolate: f, fsCache: f, state: 'warm', workers: '50%' }, ], }, + 'long-haul': { + envs: ['jsdom', 'happy-dom'], + primary: 'jsdom', + workers: ['2'], + // 2 workers x 40 heavy files each: long enough worker lifetimes for + // retention, memory-limit recycling and aging to show up in wall time + dims: { pool: POOLS, env: ['jsdom'], isolate: [t], fsCache: [f], state: ['warm'], workers: ['2'] }, + extra: [ + { pool: 'vmForks', env: 'jsdom', isolate: t, fsCache: f, state: 'cold', workers: '2' }, + { pool: 'forks', env: 'happy-dom', isolate: t, fsCache: f, state: 'warm', workers: '2' }, + { pool: 'vmForks', env: 'happy-dom', isolate: t, fsCache: f, state: 'warm', workers: '2' }, + ], + }, 'cpu-bound': { envs: ['node'], primary: 'node',