From 3f8ddeb5821e3695b36a5414449c22836841ade0 Mon Sep 17 00:00:00 2001 From: Hiroshi Ogawa <4232207+hi-ogawa@users.noreply.github.com> Date: Tue, 27 Oct 2026 12:32:57 +0900 Subject: [PATCH 1/8] feat: add setup-mocks, a non-isolated suite with mocking setup files A 1500-module shared tree, 800 test files, and 6 setup files that each vi.mock a service. A reused worker re-runs the setup files before every file, so per-file and per-run-request costs show up here and nowhere else. Co-Authored-By: Claude Opus 5.5 (1M context) --- README.md | 8 ++- apps/setup-mocks/generate.mjs | 85 +++++++++++++++++++++++++++++++ apps/setup-mocks/package.json | 15 ++++++ apps/setup-mocks/vitest.config.ts | 9 ++++ pnpm-lock.yaml | 15 ++++++ scripts/matrix.mjs | 14 +++++ 6 files changed, 145 insertions(+), 1 deletion(-) create mode 100644 apps/setup-mocks/generate.mjs create mode 100644 apps/setup-mocks/package.json create mode 100644 apps/setup-mocks/vitest.config.ts diff --git a/README.md b/README.md index 8599480..f10fe6c 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ > The apps, generators, and bench runner in this repository were generated with [Claude Fable 5](https://www.anthropic.com/news/claude-fable-5-mythos-5) and reviewed by the Vitest team. The results below were measured by hand on the hardware stated in each section. -Generated reference apps for measuring Vitest performance. Each app stands for one kind of project: a tiny utility package, a library, barrel-file graphs, DOM component suites, a dependency-heavy service, a 1300-module monolith, and a long-running DOM suite. The bench runner measures the options that change run time on each of them: `pool`, `environment` (jsdom, happy-dom, and headless Chromium via browser mode), `isolate`, `fsModuleCache`, `maxWorkers`, and cold vs warm caches. +Generated reference apps for measuring Vitest performance. Each app stands for one kind of project: a tiny utility package, a library, barrel-file graphs, DOM component suites, a dependency-heavy service, a 1300-module monolith, a long-running DOM suite, and a large non-isolated suite with mocking setup files. The bench runner measures the options that change run time on each of them: `pool`, `environment` (jsdom, happy-dom, and headless Chromium via browser mode), `isolate`, `fsModuleCache`, `maxWorkers`, and cold vs warm caches. ## Usage @@ -257,6 +257,12 @@ A long-running DOM suite: 80 jsdom test files through 2 workers, each file holdi | forks | happy-dom | — | — | — | 11.38s | 10.89s | −4% | | **vmForks** | **happy-dom** | — | — | — | 5.43s | 4.06s | −25% | +### setup-mocks + +A large suite meant to run with `isolate: false`: a shared tree of 1500 modules that every test file evaluates through its root, 800 test files that each also import one module of their own, and 6 setup files that each `vi.mock` a service the tests assert on. A reused worker re-runs the setup files before each of its files, so whatever Vitest does per file or per run request (resolving setup files and mock paths, asking the main process for warm modules) is paid hundreds of times per worker, and each round trip to the main process waits behind the other workers' calls. The 25% and 50% `maxWorkers` rows show how that cost scales with the number of workers. `isolate: true` takes about a minute here and only runs in the full matrix. Modelled on [vjeux/vitest-warm-rpc-repro](https://github.com/vjeux/vitest-warm-rpc-repro). + +Not measured for 4.1.10 vs 5.0.0 yet. + ### cpu-bound 30 test files that burn real CPU (hashing, sieving, matrix multiplication) on an 8-module graph. The tests dominate, so only scheduling (`maxWorkers`, pool choice) changes anything. diff --git a/apps/setup-mocks/generate.mjs b/apps/setup-mocks/generate.mjs new file mode 100644 index 0000000..4010965 --- /dev/null +++ b/apps/setup-mocks/generate.mjs @@ -0,0 +1,85 @@ +// setup-mocks — a large non-isolated suite whose setup files mock modules. +// +// The shape of big real-world suites that run with `isolate: false`: a wide +// shared module graph, many test files, and a handful of setup files that +// register `vi.mock` calls for every test file. A reused worker re-runs the +// setup files before each of its files, so whatever the runtime does per +// file or per run request (resolving setup files and their mock paths, +// asking the main process for warm module snapshots, ...) is paid hundreds +// of times per worker, and every round trip to the main process queues +// behind the other workers' calls. No other app has mocking setup files, so +// none of them can see this cost. +// +// - core: a binary tree of CORE modules; every test file evaluates all of +// it through the root, like a monorepo's shared packages +// - features: one module per test file that no other test file imports, +// so every file also loads something new to its worker +// - services: SETUPS modules, each mocked by one setup file; tests assert +// the mocked values so the mocks are load-bearing +// +// Modelled on https://github.com/vjeux/vitest-warm-rpc-repro, scaled down +// so the app stays affordable in the default matrix. +import { createApp, tsModule } from '../../tools/generator/helpers.mjs' + +const CORE = 1500 +const SETUPS = 6 +const TESTS = 800 + +const app = createApp(import.meta.url) + +for (let i = 0; i < CORE; i++) { + const children = [2 * i + 1, 2 * i + 2].filter(c => c < CORE) + const imports = children.map(c => `import { m${c}Walk } from './m${c}'`).join('\n') + const walk = ` +export function m${i}Walk(): number { + return 1${children.map(c => ` + m${c}Walk()`).join('')} +}` + app.write(`src/core/m${i}.ts`, `${imports}${imports ? '\n' : ''}${tsModule(`m${i}`, i, walk)}`) +} + +for (let s = 0; s < SETUPS; s++) { + app.write(`src/services/s${s}.ts`, `export function s${s}Fetch(): string { + return 'real-s${s}' +} +`) + app.write(`tests/setup/setup${s}.ts`, `import { vi } from 'vitest' + +vi.mock('../../src/services/s${s}', () => ({ + s${s}Fetch: () => 'mocked-s${s}', +})) +`) +} + +for (let t = 0; t < TESTS; t++) { + // each feature leans on a different deep node of the shared tree + const node = CORE - 1 - (t % (CORE >> 1)) + app.write(`src/features/f${t}.ts`, `import { m${node}Compute, m${node}Default, m${node}Format } from '../core/m${node}' + +export function f${t}Run(): string { + return m${node}Format(m${node}Compute({ ...m${node}Default, id: ${t} })) +} +`) + + const s = t % SETUPS + app.write(`tests/t${t}.test.ts`, `import { describe, expect, it } from 'vitest' +import { m0Walk } from '../src/core/m0' +import { f${t}Run } from '../src/features/f${t}' +import { s${s}Fetch } from '../src/services/s${s}' + +describe('t${t}', () => { + it('evaluates the shared tree', () => { + expect(m0Walk()).toBe(${CORE}) + }) + + it('runs its own feature', () => { + expect(f${t}Run()).toMatch(/^ok:/) + }) + + it('sees the mock from the setup file', () => { + expect(s${s}Fetch()).toBe('mocked-s${s}') + }) +}) +`) +} + +app.report('setup-mocks', `${CORE} core modules, ${TESTS} test files, ${SETUPS} mocking setup files`) diff --git a/apps/setup-mocks/package.json b/apps/setup-mocks/package.json new file mode 100644 index 0000000..b4a3ad9 --- /dev/null +++ b/apps/setup-mocks/package.json @@ -0,0 +1,15 @@ +{ + "name": "@bench-app/setup-mocks", + "private": true, + "type": "module", + "scripts": { + "generate": "node generate.mjs", + "test": "vitest run" + }, + "devDependencies": { + "@vitest/coverage-istanbul": "4.1.10", + "@vitest/coverage-v8": "4.1.10", + "vite": "8.1.4", + "vitest": "4.1.10" + } +} diff --git a/apps/setup-mocks/vitest.config.ts b/apps/setup-mocks/vitest.config.ts new file mode 100644 index 0000000..31e7538 --- /dev/null +++ b/apps/setup-mocks/vitest.config.ts @@ -0,0 +1,9 @@ +import { defineConfig } from 'vitest/config' +import { benchTest } from '../../tools/config/bench-config.js' + +export default defineConfig({ + test: benchTest({ + environment: 'node', + setupFiles: Array.from({ length: 6 }, (_, s) => `./tests/setup/setup${s}.ts`), + }), +}) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 8062cd0..543338f 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -298,6 +298,21 @@ importers: specifier: 4.1.10 version: 4.1.10(@types/node@26.1.1)(@vitest/browser-playwright@4.1.10)(@vitest/coverage-istanbul@4.1.10)(@vitest/coverage-v8@4.1.10)(happy-dom@20.10.6)(jsdom@29.1.1)(vite@8.1.4(@types/node@26.1.1)(yaml@2.9.0)) + apps/setup-mocks: + devDependencies: + '@vitest/coverage-istanbul': + specifier: 4.1.10 + version: 4.1.10(vitest@4.1.10) + '@vitest/coverage-v8': + specifier: 4.1.10 + version: 4.1.10(@vitest/browser@4.1.10)(vitest@4.1.10) + vite: + specifier: 8.1.4 + version: 8.1.4(@types/node@26.1.1)(yaml@2.9.0) + vitest: + specifier: 4.1.10 + version: 4.1.10(@types/node@26.1.1)(@vitest/browser-playwright@4.1.10)(@vitest/coverage-istanbul@4.1.10)(@vitest/coverage-v8@4.1.10)(happy-dom@20.10.6)(jsdom@29.1.1)(vite@8.1.4(@types/node@26.1.1)(yaml@2.9.0)) + apps/vue-spa: dependencies: vue: diff --git a/scripts/matrix.mjs b/scripts/matrix.mjs index e55c85b..59a6ea9 100644 --- a/scripts/matrix.mjs +++ b/scripts/matrix.mjs @@ -128,6 +128,20 @@ const APPS = { { pool: 'vmForks', env: 'happy-dom', isolate: t, fsCache: f, state: 'warm', workers: '2' }, ], }, + 'setup-mocks': { + best: { pool: 'threads', env: 'node', isolate: f, fsCache: t, state: 'warm' }, + envs: ['node'], + primary: 'node', + // isolate:true re-evaluates the 1500-module tree for every one of the + // 800 files (~1 minute per rep), so it only runs in the full matrix; + // the worker-count rows show how per-file round trips scale with the + // number of workers queueing on the main thread + dims: { pool: ['forks', 'threads'], env: ['node'], isolate: [f], fsCache: [f, t], state: ['cold', 'warm'] }, + extra: [ + { pool: 'forks', env: 'node', isolate: f, fsCache: f, state: 'warm', workers: '25%' }, + { pool: 'forks', env: 'node', isolate: f, fsCache: f, state: 'warm', workers: '50%' }, + ], + }, 'cpu-bound': { best: { pool: 'forks', env: 'node', isolate: f, fsCache: f, state: 'warm', workers: '100%' }, envs: ['node'], From 12f2d6942b9aed9d523ee833dd2ca2c628eb9d4a Mon Sep 17 00:00:00 2001 From: Hiroshi Ogawa <4232207+hi-ogawa@users.noreply.github.com> Date: Tue, 27 Oct 2026 15:40:12 +0900 Subject: [PATCH 2/8] feat(setup-mocks): add a maxWorkers 1 row Co-Authored-By: Claude Opus 5.5 (1M context) --- README.md | 2 +- scripts/matrix.mjs | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index f10fe6c..2761ca9 100644 --- a/README.md +++ b/README.md @@ -259,7 +259,7 @@ A long-running DOM suite: 80 jsdom test files through 2 workers, each file holdi ### setup-mocks -A large suite meant to run with `isolate: false`: a shared tree of 1500 modules that every test file evaluates through its root, 800 test files that each also import one module of their own, and 6 setup files that each `vi.mock` a service the tests assert on. A reused worker re-runs the setup files before each of its files, so whatever Vitest does per file or per run request (resolving setup files and mock paths, asking the main process for warm modules) is paid hundreds of times per worker, and each round trip to the main process waits behind the other workers' calls. The 25% and 50% `maxWorkers` rows show how that cost scales with the number of workers. `isolate: true` takes about a minute here and only runs in the full matrix. Modelled on [vjeux/vitest-warm-rpc-repro](https://github.com/vjeux/vitest-warm-rpc-repro). +A large suite meant to run with `isolate: false`: a shared tree of 1500 modules that every test file evaluates through its root, 800 test files that each also import one module of their own, and 6 setup files that each `vi.mock` a service the tests assert on. A reused worker re-runs the setup files before each of its files, so whatever Vitest does per file or per run request (resolving setup files and mock paths, asking the main process for warm modules) is paid hundreds of times per worker, and each round trip to the main process waits behind the other workers' calls. The 25% and 50% `maxWorkers` rows show how that cost scales with the number of workers; with `maxWorkers: 1` all files go to one worker as a single run request, so only the per-file cost remains, with no other worker waiting on the main process. `isolate: true` takes about a minute here and only runs in the full matrix. Modelled on [vjeux/vitest-warm-rpc-repro](https://github.com/vjeux/vitest-warm-rpc-repro). Not measured for 4.1.10 vs 5.0.0 yet. diff --git a/scripts/matrix.mjs b/scripts/matrix.mjs index 59a6ea9..1facafb 100644 --- a/scripts/matrix.mjs +++ b/scripts/matrix.mjs @@ -135,9 +135,11 @@ const APPS = { // isolate:true re-evaluates the 1500-module tree for every one of the // 800 files (~1 minute per rep), so it only runs in the full matrix; // the worker-count rows show how per-file round trips scale with the - // number of workers queueing on the main thread + // number of workers queueing on the main thread; with one worker all + // files share one run request and nothing else queues dims: { pool: ['forks', 'threads'], env: ['node'], isolate: [f], fsCache: [f, t], state: ['cold', 'warm'] }, extra: [ + { pool: 'forks', env: 'node', isolate: f, fsCache: f, state: 'warm', workers: '1' }, { pool: 'forks', env: 'node', isolate: f, fsCache: f, state: 'warm', workers: '25%' }, { pool: 'forks', env: 'node', isolate: f, fsCache: f, state: 'warm', workers: '50%' }, ], From 295a2db90e1f0337a4f2fa387dcbc2029d187893 Mon Sep 17 00:00:00 2001 From: Hiroshi Ogawa <4232207+hi-ogawa@users.noreply.github.com> Date: Tue, 27 Oct 2026 15:50:45 +0900 Subject: [PATCH 3/8] docs(setup-mocks): cite the suite the app is modelled on Co-Authored-By: Claude Opus 5.5 (1M context) --- apps/setup-mocks/generate.mjs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/setup-mocks/generate.mjs b/apps/setup-mocks/generate.mjs index 4010965..e63b329 100644 --- a/apps/setup-mocks/generate.mjs +++ b/apps/setup-mocks/generate.mjs @@ -1,8 +1,8 @@ // setup-mocks — a large non-isolated suite whose setup files mock modules. // -// The shape of big real-world suites that run with `isolate: false`: a wide -// shared module graph, many test files, and a handful of setup files that -// register `vi.mock` calls for every test file. A reused worker re-runs the +// The shape of the 6,600-file suite reported in vitest-dev/vitest#11752: +// `isolate: false`, a wide shared module graph, many test files, and a +// handful of setup files that register `vi.mock` calls for every test file. A reused worker re-runs the // setup files before each of its files, so whatever the runtime does per // file or per run request (resolving setup files and their mock paths, // asking the main process for warm module snapshots, ...) is paid hundreds From f22752e1fd83960136d7b6609ddff37777667604 Mon Sep 17 00:00:00 2001 From: Hiroshi Ogawa <4232207+hi-ogawa@users.noreply.github.com> Date: Tue, 27 Oct 2026 15:51:05 +0900 Subject: [PATCH 4/8] docs(setup-mocks): reflow the header comment Co-Authored-By: Claude Opus 5.5 (1M context) --- apps/setup-mocks/generate.mjs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/apps/setup-mocks/generate.mjs b/apps/setup-mocks/generate.mjs index e63b329..a68ab89 100644 --- a/apps/setup-mocks/generate.mjs +++ b/apps/setup-mocks/generate.mjs @@ -2,12 +2,12 @@ // // The shape of the 6,600-file suite reported in vitest-dev/vitest#11752: // `isolate: false`, a wide shared module graph, many test files, and a -// handful of setup files that register `vi.mock` calls for every test file. A reused worker re-runs the -// setup files before each of its files, so whatever the runtime does per -// file or per run request (resolving setup files and their mock paths, -// asking the main process for warm module snapshots, ...) is paid hundreds -// of times per worker, and every round trip to the main process queues -// behind the other workers' calls. No other app has mocking setup files, so +// handful of setup files that register `vi.mock` calls for every test file. +// A reused worker re-runs the setup files before each of its files, so +// whatever the runtime does per file or per run request (resolving setup +// files and their mock paths, asking the main process for warm module +// snapshots, ...) is paid hundreds of times per worker, and every round trip +// to the main process queues behind the other workers' calls. No other app has mocking setup files, so // none of them can see this cost. // // - core: a binary tree of CORE modules; every test file evaluates all of From f0427f19cb162ddd6cd021f60a374e2c6f7b84f0 Mon Sep 17 00:00:00 2001 From: Hiroshi Ogawa <4232207+hi-ogawa@users.noreply.github.com> Date: Tue, 27 Oct 2026 15:51:15 +0900 Subject: [PATCH 5/8] docs(setup-mocks): rewrap the header comment Co-Authored-By: Claude Opus 5.5 (1M context) --- apps/setup-mocks/generate.mjs | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/apps/setup-mocks/generate.mjs b/apps/setup-mocks/generate.mjs index a68ab89..6c85505 100644 --- a/apps/setup-mocks/generate.mjs +++ b/apps/setup-mocks/generate.mjs @@ -2,13 +2,13 @@ // // The shape of the 6,600-file suite reported in vitest-dev/vitest#11752: // `isolate: false`, a wide shared module graph, many test files, and a -// handful of setup files that register `vi.mock` calls for every test file. -// A reused worker re-runs the setup files before each of its files, so -// whatever the runtime does per file or per run request (resolving setup -// files and their mock paths, asking the main process for warm module -// snapshots, ...) is paid hundreds of times per worker, and every round trip -// to the main process queues behind the other workers' calls. No other app has mocking setup files, so -// none of them can see this cost. +// handful of setup files that register `vi.mock` calls for every test file. A +// reused worker re-runs the setup files before each of its files, so whatever +// the runtime does per file or per run request (resolving setup files and +// their mock paths, asking the main process for warm module snapshots, ...) +// is paid hundreds of times per worker, and every round trip to the main +// process queues behind the other workers' calls. No other app has mocking +// setup files, so none of them can see this cost. // // - core: a binary tree of CORE modules; every test file evaluates all of // it through the root, like a monorepo's shared packages From 36cddb1cd2d799cfc3b619ffaaeb158e8fcb13a9 Mon Sep 17 00:00:00 2001 From: Hiroshi Ogawa <4232207+hi-ogawa@users.noreply.github.com> Date: Tue, 27 Oct 2026 18:02:26 +0900 Subject: [PATCH 6/8] feat: add late-graphs, large graphs after each worker's first file Co-Authored-By: Claude Opus 5.5 (1M context) --- README.md | 8 ++++- apps/late-graphs/generate.mjs | 56 +++++++++++++++++++++++++++++++ apps/late-graphs/package.json | 15 +++++++++ apps/late-graphs/vitest.config.ts | 20 +++++++++++ pnpm-lock.yaml | 15 +++++++++ scripts/matrix.mjs | 8 +++++ 6 files changed, 121 insertions(+), 1 deletion(-) create mode 100644 apps/late-graphs/generate.mjs create mode 100644 apps/late-graphs/package.json create mode 100644 apps/late-graphs/vitest.config.ts diff --git a/README.md b/README.md index 2761ca9..a3d71fa 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ > The apps, generators, and bench runner in this repository were generated with [Claude Fable 5](https://www.anthropic.com/news/claude-fable-5-mythos-5) and reviewed by the Vitest team. The results below were measured by hand on the hardware stated in each section. -Generated reference apps for measuring Vitest performance. Each app stands for one kind of project: a tiny utility package, a library, barrel-file graphs, DOM component suites, a dependency-heavy service, a 1300-module monolith, a long-running DOM suite, and a large non-isolated suite with mocking setup files. The bench runner measures the options that change run time on each of them: `pool`, `environment` (jsdom, happy-dom, and headless Chromium via browser mode), `isolate`, `fsModuleCache`, `maxWorkers`, and cold vs warm caches. +Generated reference apps for measuring Vitest performance. Each app stands for one kind of project: a tiny utility package, a library, barrel-file graphs, DOM component suites, a dependency-heavy service, a 1300-module monolith, a long-running DOM suite, a large non-isolated suite with mocking setup files, and a non-isolated suite whose large graphs arrive after each worker's first file. The bench runner measures the options that change run time on each of them: `pool`, `environment` (jsdom, happy-dom, and headless Chromium via browser mode), `isolate`, `fsModuleCache`, `maxWorkers`, and cold vs warm caches. ## Usage @@ -263,6 +263,12 @@ A large suite meant to run with `isolate: false`: a shared tree of 1500 modules Not measured for 4.1.10 vs 5.0.0 yet. +### late-graphs + +A non-isolated suite where every worker's first test file is trivial and the files after it bring large graphs of their own: 4 trivial files that sort first (the app sorts files by name and fixes `maxWorkers` to 4), then 24 test files that each import their own chain of 250 modules. Anything a worker fetches or decides once, from its first run request, says nothing about the modules its later files need, so a per-worker shortcut shows up here as one round trip per module. A setup file with no work in it lets a watch rerun be triggered by touching it. + +Not measured for 4.1.10 vs 5.0.0 yet. + ### cpu-bound 30 test files that burn real CPU (hashing, sieving, matrix multiplication) on an 8-module graph. The tests dominate, so only scheduling (`maxWorkers`, pool choice) changes anything. diff --git a/apps/late-graphs/generate.mjs b/apps/late-graphs/generate.mjs new file mode 100644 index 0000000..9dc7a2e --- /dev/null +++ b/apps/late-graphs/generate.mjs @@ -0,0 +1,56 @@ +// late-graphs — reused workers whose large graphs arrive after their first file. +// +// A non-isolated suite where every worker's first test file is trivial and +// the files after it each import a large graph no other file shares. Whatever +// a worker learns from its first run request (a warm-module snapshot, for +// example) says nothing about the modules its later files need, so anything +// that is fetched or decided once per worker instead of once per file shows +// up here as one round trip per module. +// +// - trivial: TRIVIAL files that sort first (the config sorts files by +// name and fixes maxWorkers to TRIVIAL), one per worker +// - heavy: HEAVY files, each importing its own chain of CHAIN modules; +// a chain is discovered one import at a time, like a deep dependency +// - a setup file with no work in it, so touching it reruns every file in +// watch mode +import { createApp, tsModule } from '../../tools/generator/helpers.mjs' + +const TRIVIAL = 4 +const HEAVY = 24 +const CHAIN = 250 + +const app = createApp(import.meta.url) + +app.write('tests/setup.ts', `export {} +`) + +for (let t = 0; t < TRIVIAL; t++) { + app.write(`tests/a-trivial-${t}.test.ts`, `import { expect, it } from 'vitest' + +it('is trivial', () => { + expect(${t}).toBe(${t}) +}) +`) +} + +for (let h = 0; h < HEAVY; h++) { + for (let i = 0; i < CHAIN; i++) { + const next = i + 1 < CHAIN + ? `import { h${h}m${i + 1}Walk } from './m${i + 1}'\n` + : '' + const walk = ` +export function h${h}m${i}Walk(): number { + return 1${i + 1 < CHAIN ? ` + h${h}m${i + 1}Walk()` : ''} +}` + app.write(`src/h${h}/m${i}.ts`, `${next}${tsModule(`h${h}m${i}`, i, walk)}`) + } + app.write(`tests/b-heavy-${String(h).padStart(2, '0')}.test.ts`, `import { expect, it } from 'vitest' +import { h${h}m0Walk } from '../src/h${h}/m0' + +it('walks its own chain', () => { + expect(h${h}m0Walk()).toBe(${CHAIN}) +}) +`) +} + +app.report('late-graphs', `${TRIVIAL} trivial files first, ${HEAVY} heavy files with ${CHAIN}-module chains of their own`) diff --git a/apps/late-graphs/package.json b/apps/late-graphs/package.json new file mode 100644 index 0000000..82332cc --- /dev/null +++ b/apps/late-graphs/package.json @@ -0,0 +1,15 @@ +{ + "name": "@bench-app/late-graphs", + "private": true, + "type": "module", + "scripts": { + "generate": "node generate.mjs", + "test": "vitest run" + }, + "devDependencies": { + "@vitest/coverage-istanbul": "4.1.10", + "@vitest/coverage-v8": "4.1.10", + "vite": "8.1.4", + "vitest": "4.1.10" + } +} diff --git a/apps/late-graphs/vitest.config.ts b/apps/late-graphs/vitest.config.ts new file mode 100644 index 0000000..01fcc5c --- /dev/null +++ b/apps/late-graphs/vitest.config.ts @@ -0,0 +1,20 @@ +import type { TestSpecification } from 'vitest/node' +import { BaseSequencer } from 'vitest/node' +import { defineConfig } from 'vitest/config' +import { benchTest } from '../../tools/config/bench-config.js' + +// files run in name order, so every worker starts with a trivial file +class NameOrderSequencer extends BaseSequencer { + async sort(files: TestSpecification[]) { + return [...files].sort((a, b) => a.moduleId.localeCompare(b.moduleId)) + } +} + +export default defineConfig({ + test: benchTest({ + environment: 'node', + maxWorkers: 4, + setupFiles: ['./tests/setup.ts'], + sequence: { sequencer: NameOrderSequencer }, + }), +}) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 543338f..b59af2a 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -155,6 +155,21 @@ importers: specifier: 4.1.10 version: 4.1.10(@types/node@26.1.1)(@vitest/browser-playwright@4.1.10)(@vitest/coverage-istanbul@4.1.10)(@vitest/coverage-v8@4.1.10)(happy-dom@20.10.6)(jsdom@29.1.1)(vite@8.1.4(@types/node@26.1.1)(yaml@2.9.0)) + apps/late-graphs: + devDependencies: + '@vitest/coverage-istanbul': + specifier: 4.1.10 + version: 4.1.10(vitest@4.1.10) + '@vitest/coverage-v8': + specifier: 4.1.10 + version: 4.1.10(@vitest/browser@4.1.10)(vitest@4.1.10) + vite: + specifier: 8.1.4 + version: 8.1.4(@types/node@26.1.1)(yaml@2.9.0) + vitest: + specifier: 4.1.10 + version: 4.1.10(@types/node@26.1.1)(@vitest/browser-playwright@4.1.10)(@vitest/coverage-istanbul@4.1.10)(@vitest/coverage-v8@4.1.10)(happy-dom@20.10.6)(jsdom@29.1.1)(vite@8.1.4(@types/node@26.1.1)(yaml@2.9.0)) + apps/long-haul: devDependencies: '@testing-library/dom': diff --git a/scripts/matrix.mjs b/scripts/matrix.mjs index 1facafb..802b1df 100644 --- a/scripts/matrix.mjs +++ b/scripts/matrix.mjs @@ -144,6 +144,14 @@ const APPS = { { pool: 'forks', env: 'node', isolate: f, fsCache: f, state: 'warm', workers: '50%' }, ], }, + 'late-graphs': { + best: { pool: 'forks', env: 'node', isolate: f, fsCache: t, state: 'warm' }, + envs: ['node'], + primary: 'node', + // the shape only exists when workers are reused across files, so only + // isolate:false; the app fixes maxWorkers itself + dims: { pool: ['forks', 'threads'], env: ['node'], isolate: [f], fsCache: [f, t], state: ['cold', 'warm'] }, + }, 'cpu-bound': { best: { pool: 'forks', env: 'node', isolate: f, fsCache: f, state: 'warm', workers: '100%' }, envs: ['node'], From b41860d4c1cf67d541bb78a1a253ac735bed7f97 Mon Sep 17 00:00:00 2001 From: Hiroshi Ogawa <4232207+hi-ogawa@users.noreply.github.com> Date: Tue, 27 Oct 2026 18:32:07 +0900 Subject: [PATCH 7/8] feat(late-graphs): share chains across workers instead of per file Co-Authored-By: Claude Opus 5.5 (1M context) --- README.md | 4 +-- apps/late-graphs/generate.mjs | 49 ++++++++++++++++--------------- apps/late-graphs/vitest.config.ts | 1 - 3 files changed, 28 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index a3d71fa..3fd6d6b 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ > The apps, generators, and bench runner in this repository were generated with [Claude Fable 5](https://www.anthropic.com/news/claude-fable-5-mythos-5) and reviewed by the Vitest team. The results below were measured by hand on the hardware stated in each section. -Generated reference apps for measuring Vitest performance. Each app stands for one kind of project: a tiny utility package, a library, barrel-file graphs, DOM component suites, a dependency-heavy service, a 1300-module monolith, a long-running DOM suite, a large non-isolated suite with mocking setup files, and a non-isolated suite whose large graphs arrive after each worker's first file. The bench runner measures the options that change run time on each of them: `pool`, `environment` (jsdom, happy-dom, and headless Chromium via browser mode), `isolate`, `fsModuleCache`, `maxWorkers`, and cold vs warm caches. +Generated reference apps for measuring Vitest performance. Each app stands for one kind of project: a tiny utility package, a library, barrel-file graphs, DOM component suites, a dependency-heavy service, a 1300-module monolith, a long-running DOM suite, a large non-isolated suite with mocking setup files, and a non-isolated suite whose later files reuse graphs other workers loaded. The bench runner measures the options that change run time on each of them: `pool`, `environment` (jsdom, happy-dom, and headless Chromium via browser mode), `isolate`, `fsModuleCache`, `maxWorkers`, and cold vs warm caches. ## Usage @@ -265,7 +265,7 @@ Not measured for 4.1.10 vs 5.0.0 yet. ### late-graphs -A non-isolated suite where every worker's first test file is trivial and the files after it bring large graphs of their own: 4 trivial files that sort first (the app sorts files by name and fixes `maxWorkers` to 4), then 24 test files that each import their own chain of 250 modules. Anything a worker fetches or decides once, from its first run request, says nothing about the modules its later files need, so a per-worker shortcut shows up here as one round trip per module. A setup file with no work in it lets a watch rerun be triggered by touching it. +A non-isolated suite where every worker's first test file is trivial and later files reuse large graphs that files on other workers loaded earlier: 4 trivial files that sort first (the app sorts files by name and fixes `maxWorkers` to 4), then 24 test files sharing 12 chains of 250 modules, so the second dozen reuses the chains the first dozen loaded. Anything a worker fetches or decides once, from its first run request, says nothing about the modules its later files need, so a per-worker shortcut shows up here as one round trip per module. Not measured for 4.1.10 vs 5.0.0 yet. diff --git a/apps/late-graphs/generate.mjs b/apps/late-graphs/generate.mjs index 9dc7a2e..3c4ba4b 100644 --- a/apps/late-graphs/generate.mjs +++ b/apps/late-graphs/generate.mjs @@ -1,29 +1,28 @@ -// late-graphs — reused workers whose large graphs arrive after their first file. +// late-graphs — reused workers whose later files need modules other workers +// already loaded. // -// A non-isolated suite where every worker's first test file is trivial and -// the files after it each import a large graph no other file shares. Whatever -// a worker learns from its first run request (a warm-module snapshot, for -// example) says nothing about the modules its later files need, so anything -// that is fetched or decided once per worker instead of once per file shows -// up here as one round trip per module. +// A non-isolated suite where every worker's first test file is trivial, and +// the files after it import large graphs that a file on some other worker +// loaded earlier. Whatever a worker learns from its first run request (a +// warm-module snapshot, for example) says nothing about the modules its later +// files need, so anything decided once per worker instead of once per file +// shows up here as one round trip per module. // // - trivial: TRIVIAL files that sort first (the config sorts files by // name and fixes maxWorkers to TRIVIAL), one per worker -// - heavy: HEAVY files, each importing its own chain of CHAIN modules; -// a chain is discovered one import at a time, like a deep dependency -// - a setup file with no work in it, so touching it reruns every file in -// watch mode +// - chains: CHAINS chains of CHAIN modules; a chain is discovered one +// import at a time, like a deep dependency +// - heavy: HEAVY files, file h importing chain h % CHAINS, so the second +// half reuses the chains the first half loaded import { createApp, tsModule } from '../../tools/generator/helpers.mjs' const TRIVIAL = 4 +const CHAINS = 12 const HEAVY = 24 const CHAIN = 250 const app = createApp(import.meta.url) -app.write('tests/setup.ts', `export {} -`) - for (let t = 0; t < TRIVIAL; t++) { app.write(`tests/a-trivial-${t}.test.ts`, `import { expect, it } from 'vitest' @@ -33,24 +32,28 @@ it('is trivial', () => { `) } -for (let h = 0; h < HEAVY; h++) { +for (let c = 0; c < CHAINS; c++) { for (let i = 0; i < CHAIN; i++) { const next = i + 1 < CHAIN - ? `import { h${h}m${i + 1}Walk } from './m${i + 1}'\n` + ? `import { c${c}m${i + 1}Walk } from './m${i + 1}'\n` : '' const walk = ` -export function h${h}m${i}Walk(): number { - return 1${i + 1 < CHAIN ? ` + h${h}m${i + 1}Walk()` : ''} +export function c${c}m${i}Walk(): number { + return 1${i + 1 < CHAIN ? ` + c${c}m${i + 1}Walk()` : ''} }` - app.write(`src/h${h}/m${i}.ts`, `${next}${tsModule(`h${h}m${i}`, i, walk)}`) + app.write(`src/c${c}/m${i}.ts`, `${next}${tsModule(`c${c}m${i}`, i, walk)}`) } +} + +for (let h = 0; h < HEAVY; h++) { + const c = h % CHAINS app.write(`tests/b-heavy-${String(h).padStart(2, '0')}.test.ts`, `import { expect, it } from 'vitest' -import { h${h}m0Walk } from '../src/h${h}/m0' +import { c${c}m0Walk } from '../src/c${c}/m0' -it('walks its own chain', () => { - expect(h${h}m0Walk()).toBe(${CHAIN}) +it('walks chain ${c}', () => { + expect(c${c}m0Walk()).toBe(${CHAIN}) }) `) } -app.report('late-graphs', `${TRIVIAL} trivial files first, ${HEAVY} heavy files with ${CHAIN}-module chains of their own`) +app.report('late-graphs', `${TRIVIAL} trivial files first, ${HEAVY} heavy files sharing ${CHAINS} chains of ${CHAIN} modules`) diff --git a/apps/late-graphs/vitest.config.ts b/apps/late-graphs/vitest.config.ts index 01fcc5c..f0b0113 100644 --- a/apps/late-graphs/vitest.config.ts +++ b/apps/late-graphs/vitest.config.ts @@ -14,7 +14,6 @@ export default defineConfig({ test: benchTest({ environment: 'node', maxWorkers: 4, - setupFiles: ['./tests/setup.ts'], sequence: { sequencer: NameOrderSequencer }, }), }) From 9cbb493f320723fb657eeb3c2b464576cf500a72 Mon Sep 17 00:00:00 2001 From: Hiroshi Ogawa <4232207+hi-ogawa@users.noreply.github.com> Date: Wed, 28 Oct 2026 09:28:42 +0900 Subject: [PATCH 8/8] feat: add enterprise-monolith-xl, 800 node test files over the same graph Co-Authored-By: Claude Opus 5.5 (1M context) --- README.md | 8 ++++++- apps/enterprise-monolith-xl/generate.mjs | 13 ++++++++++ apps/enterprise-monolith-xl/package.json | 20 ++++++++++++++++ apps/enterprise-monolith-xl/vitest.config.ts | 16 +++++++++++++ apps/enterprise-monolith/generate.mjs | 15 ++++++++---- pnpm-lock.yaml | 25 ++++++++++++++++++++ scripts/matrix.mjs | 8 +++++++ 7 files changed, 100 insertions(+), 5 deletions(-) create mode 100644 apps/enterprise-monolith-xl/generate.mjs create mode 100644 apps/enterprise-monolith-xl/package.json create mode 100644 apps/enterprise-monolith-xl/vitest.config.ts diff --git a/README.md b/README.md index 3fd6d6b..6d79a1b 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ > The apps, generators, and bench runner in this repository were generated with [Claude Fable 5](https://www.anthropic.com/news/claude-fable-5-mythos-5) and reviewed by the Vitest team. The results below were measured by hand on the hardware stated in each section. -Generated reference apps for measuring Vitest performance. Each app stands for one kind of project: a tiny utility package, a library, barrel-file graphs, DOM component suites, a dependency-heavy service, a 1300-module monolith, a long-running DOM suite, a large non-isolated suite with mocking setup files, and a non-isolated suite whose later files reuse graphs other workers loaded. The bench runner measures the options that change run time on each of them: `pool`, `environment` (jsdom, happy-dom, and headless Chromium via browser mode), `isolate`, `fsModuleCache`, `maxWorkers`, and cold vs warm caches. +Generated reference apps for measuring Vitest performance. Each app stands for one kind of project: a tiny utility package, a library, barrel-file graphs, DOM component suites, a dependency-heavy service, a 1300-module monolith (and a variant with 800 test files), a long-running DOM suite, a large non-isolated suite with mocking setup files, and a non-isolated suite whose later files reuse graphs other workers loaded. The bench runner measures the options that change run time on each of them: `pool`, `environment` (jsdom, happy-dom, and headless Chromium via browser mode), `isolate`, `fsModuleCache`, `maxWorkers`, and cold vs warm caches. ## Usage @@ -244,6 +244,12 @@ A large monorepo: about 1280 modules with 12-deep import chains, import cycles, | **threads** | **false** | **true** | **default** | 2.48s | 2.45s | ~0 | 1.90s | 1.98s | +5% | | forks | false | false | 50% | — | — | — | 3.36s | 3.17s | −6% | +### enterprise-monolith-xl + +enterprise-monolith's graph with 800 test files instead of 150, from the same generator, all in the node environment. That is about 70 files per worker on 12 cores instead of about 14, so under `isolate: false` whatever a reused worker pays per file over a large shared graph is paid five times as often. The original's jsdom files are left out: a worker isn't reused across environments, so mixing them in replaces workers every few files. The default matrix runs only `isolate: false`. + +Not measured for 4.1.10 vs 5.0.0 yet. + ### long-haul A long-running DOM suite: 80 jsdom test files through 2 workers, each file holding a 15MB module-level dataset and rendering tables over it. Node pools rebuild the environment and import the external dependencies again for each of a worker's 40 files. vm pool workers pay once, reuse compiled scripts across contexts, and get recycled by the pinned 512MB `vmMemoryLimit` several times per run. This is the app where the vm pools win by a wide margin. Memory retention across files is out of scope here. diff --git a/apps/enterprise-monolith-xl/generate.mjs b/apps/enterprise-monolith-xl/generate.mjs new file mode 100644 index 0000000..48a8ae7 --- /dev/null +++ b/apps/enterprise-monolith-xl/generate.mjs @@ -0,0 +1,13 @@ +// enterprise-monolith-xl — enterprise-monolith's graph with 800 test files +// instead of 150, all in the node environment. +// +// The same ~1280-module graph, so the difference is how many files each +// worker runs over it: about 70 per worker on 12 cores. The original's jsdom +// files are left out because a worker can't be reused across environments, +// which would replace workers every few files. +// Under isolate:false, anything a reused worker pays per file over a large +// shared graph (such as a warm-module snapshot of the file's graph) grows with +// that count, which the smaller app keeps too low to see. +import { generate } from '../enterprise-monolith/generate.mjs' + +generate(import.meta.url, { name: 'enterprise-monolith-xl', tests: 800, jsdom: false }) diff --git a/apps/enterprise-monolith-xl/package.json b/apps/enterprise-monolith-xl/package.json new file mode 100644 index 0000000..64c55c6 --- /dev/null +++ b/apps/enterprise-monolith-xl/package.json @@ -0,0 +1,20 @@ +{ + "name": "@bench-app/enterprise-monolith-xl", + "private": true, + "type": "module", + "scripts": { + "generate": "node generate.mjs", + "test": "vitest run" + }, + "dependencies": { + "dayjs": "1.11.21", + "zod": "4.4.3" + }, + "devDependencies": { + "@vitest/coverage-istanbul": "4.1.10", + "@vitest/coverage-v8": "4.1.10", + "jsdom": "29.1.1", + "vite": "8.1.4", + "vitest": "4.1.10" + } +} diff --git a/apps/enterprise-monolith-xl/vitest.config.ts b/apps/enterprise-monolith-xl/vitest.config.ts new file mode 100644 index 0000000..f1c690a --- /dev/null +++ b/apps/enterprise-monolith-xl/vitest.config.ts @@ -0,0 +1,16 @@ +import { fileURLToPath } from 'node:url' +import { defineConfig } from 'vitest/config' +import { benchTest } from '../../tools/config/bench-config.js' + +export default defineConfig({ + resolve: { + alias: { + '@util': fileURLToPath(new URL('./src/util', import.meta.url)), + '@core': fileURLToPath(new URL('./src/core', import.meta.url)), + '@svc': fileURLToPath(new URL('./src/svc', import.meta.url)), + }, + }, + test: benchTest({ + environment: 'node', + }), +}) diff --git a/apps/enterprise-monolith/generate.mjs b/apps/enterprise-monolith/generate.mjs index 487fa8f..4c5d7e1 100644 --- a/apps/enterprise-monolith/generate.mjs +++ b/apps/enterprise-monolith/generate.mjs @@ -14,6 +14,7 @@ // Dimensions stressed: module graph scale (transform + per-file RPC fetch // waterfall), fs module cache cold/warm, isolate cost at scale, alias // resolution, mixed environments. +import { pathToFileURL } from 'node:url' import { createApp, tsModule } from '../../tools/generator/helpers.mjs' const UTIL = 400 @@ -22,10 +23,12 @@ const SVC = 250 const FEAT = 250 const CHAIN = 60 const GROUPS = 10 -const TESTS = 150 const CYCLES = [20, 70, 120, 170, 220] -const app = createApp(import.meta.url) +// the -xl variant reuses this graph with more test files; the body stays +// unindented so the template literals keep their output byte-for-byte +export function generate(importMetaUrl, { name, tests: TESTS, jsdom = true }) { +const app = createApp(importMetaUrl) for (let i = 0; i < UTIL; i++) app.write(`src/util/u${i}.ts`, tsModule(`u${i}`, i)) @@ -137,7 +140,7 @@ ${tsModule(`d${i}`, i)}`) for (let t = 0; t < TESTS; t++) { const feats = [(t * 3) % FEAT, (t * 3 + 11) % FEAT, (t * 3 + 23) % FEAT] const viaBarrel = t % 3 === 0 - const dom = t % 10 === 7 + const dom = jsdom && t % 10 === 7 const imports = feats.map((f, idx) => viaBarrel && idx === 0 ? `import { f${f}Render } from '../src/feat/group-${Math.floor(f / perGroup)}'` : `import { f${f}Render } from '../src/feat/f${f}'`) @@ -184,4 +187,8 @@ ${extraTests.join('\n')} `) } -app.report('enterprise-monolith', `${UTIL + CORE + SVC + FEAT + CHAIN + GROUPS} src modules + ${GROUPS} json, ${TESTS} test files (15 jsdom via pragma)`) +app.report(name, `${UTIL + CORE + SVC + FEAT + CHAIN + GROUPS} src modules + ${GROUPS} json, ${TESTS} test files (${jsdom ? Math.ceil((TESTS - 7) / 10) : 0} jsdom via pragma)`) +} + +if (import.meta.url === pathToFileURL(process.argv[1]).href) + generate(import.meta.url, { name: 'enterprise-monolith', tests: 150 }) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index b59af2a..4b08464 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -155,6 +155,31 @@ importers: specifier: 4.1.10 version: 4.1.10(@types/node@26.1.1)(@vitest/browser-playwright@4.1.10)(@vitest/coverage-istanbul@4.1.10)(@vitest/coverage-v8@4.1.10)(happy-dom@20.10.6)(jsdom@29.1.1)(vite@8.1.4(@types/node@26.1.1)(yaml@2.9.0)) + apps/enterprise-monolith-xl: + dependencies: + dayjs: + specifier: 1.11.21 + version: 1.11.21 + zod: + specifier: 4.4.3 + version: 4.4.3 + devDependencies: + '@vitest/coverage-istanbul': + specifier: 4.1.10 + version: 4.1.10(vitest@4.1.10) + '@vitest/coverage-v8': + specifier: 4.1.10 + version: 4.1.10(@vitest/browser@4.1.10)(vitest@4.1.10) + jsdom: + specifier: 29.1.1 + version: 29.1.1 + vite: + specifier: 8.1.4 + version: 8.1.4(@types/node@26.1.1)(yaml@2.9.0) + vitest: + specifier: 4.1.10 + version: 4.1.10(@types/node@26.1.1)(@vitest/browser-playwright@4.1.10)(@vitest/coverage-istanbul@4.1.10)(@vitest/coverage-v8@4.1.10)(happy-dom@20.10.6)(jsdom@29.1.1)(vite@8.1.4(@types/node@26.1.1)(yaml@2.9.0)) + apps/late-graphs: devDependencies: '@vitest/coverage-istanbul': diff --git a/scripts/matrix.mjs b/scripts/matrix.mjs index 802b1df..7f4c371 100644 --- a/scripts/matrix.mjs +++ b/scripts/matrix.mjs @@ -114,6 +114,14 @@ const APPS = { { pool: 'forks', env: 'node', isolate: f, fsCache: f, state: 'warm', workers: '50%' }, ], }, + 'enterprise-monolith-xl': { + best: { pool: 'threads', env: 'node', isolate: f, fsCache: t, state: 'warm' }, + envs: ['node'], + primary: 'node', + // the same graph as enterprise-monolith with ~5x the files; isolate:true + // only gets slower in proportion, so it's left to the full matrix + dims: { pool: ['forks', 'threads'], env: ['node'], isolate: [f], fsCache: [f, t], state: ['warm'] }, + }, 'long-haul': { best: { pool: 'vmForks', env: 'happy-dom', isolate: t, fsCache: f, state: 'warm', workers: '2' }, envs: ['jsdom', 'happy-dom'],