From c8e6930922a10a21a3fbbd5070f9e9f41b29caf2 Mon Sep 17 00:00:00 2001 From: avivkeller Date: Mon, 9 Nov 2026 00:35:28 -0500 Subject: [PATCH] chore(benchmarks): codspeed --- .github/workflows/codspeed.yml | 55 +++++ .github/workflows/generate.yml | 5 +- CONTRIBUTING.md | 12 + bench/generate.bench.mjs | 66 ++++++ docs/contributing/comparators.md | 12 +- package.json | 4 + pnpm-lock.yaml | 266 +++++++++++++++++++++++ scripts/__tests__/comparators.test.mjs | 80 +------ scripts/comparators/file-size.mjs | 6 - scripts/comparators/files.mjs | 6 +- scripts/comparators/object-assertion.mjs | 6 - scripts/comparators/performance.mjs | 108 --------- scripts/constants.mjs | 1 - 13 files changed, 411 insertions(+), 216 deletions(-) create mode 100644 .github/workflows/codspeed.yml create mode 100644 bench/generate.bench.mjs delete mode 100644 scripts/comparators/performance.mjs diff --git a/.github/workflows/codspeed.yml b/.github/workflows/codspeed.yml new file mode 100644 index 000000000..f32a68b16 --- /dev/null +++ b/.github/workflows/codspeed.yml @@ -0,0 +1,55 @@ +name: CodSpeed + +on: + push: + branches: [main] + pull_request: + branches: [main] + # Lets CodSpeed replay a run when it needs a baseline. + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + benchmarks: + name: Benchmarks + runs-on: ubuntu-latest + permissions: + contents: read + id-token: write + env: + NODE_SHA: 768315cf7165e9ff9b636844b776488e9d1703fc + steps: + - name: Harden Runner + uses: step-security/harden-runner@dc2769abcc0fcfbadfc1df54081c1608579ef6b2 # v2.22.2 + with: + egress-policy: audit + + - name: Setup environment + uses: nodejs/web-team/actions/setup-environment@4c0b18db11a8357664683d70ef1f27af08ecc766 + with: + use-version-file: true + pnpm: true + + - name: Checkout Node.js sources + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + repository: nodejs/node + ref: ${{ env.NODE_SHA }} + sparse-checkout: | + doc + lib + . + path: node + + - name: Run benchmarks + uses: CodSpeedHQ/action@af75f77f17a054769abefaa42fe8dc2975469b5f # v5.6.3 + with: + mode: simulation,memory + run: node --run bench diff --git a/.github/workflows/generate.yml b/.github/workflows/generate.yml index 2d2e288bd..0f96c16a4 100644 --- a/.github/workflows/generate.yml +++ b/.github/workflows/generate.yml @@ -145,10 +145,7 @@ jobs: - name: Generate ${{ matrix.target }} run: | - /usr/bin/time \ - --output out/benchmark.json \ - --format '{"elapsedSeconds": %e, "userCpuSeconds": %U, "systemCpuSeconds": %S, "maxRssKiB": %M}' \ - node packages/cli/bin/cli.mjs generate \ + node packages/cli/bin/cli.mjs generate \ -t ${{ matrix.target }} \ -i "${{ matrix.input }}" \ -o out \ diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index afcd1b2b0..c99616cd0 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -20,6 +20,7 @@ Thank you for your interest in contributing to the `doc-kit` project! We welcome - [Best Practices](#best-practices) - [Example Test File](#example-test-file) - [Running Tests](#running-tests) +- [Benchmarks](#benchmarks) - [Code Quality](#code-quality) - [Linting and Formatting](#linting-and-formatting) - [Pre-commit Hooks](#pre-commit-hooks) @@ -334,6 +335,17 @@ node --run test:coverage node --test packages/core/src/utils/__tests__/parser.test.mjs ``` +## Benchmarks + +Generation speed and memory are measured with [CodSpeed](https://codspeed.io) +(`bench/generate.bench.mjs`), on one thread. The workflow checks `nodejs/node` +out at `NODE_SHA` into `./node`, and the suite reads that directory. It runs on +every pull request and reports CPU and memory there. + +```bash +node --run bench +``` + ## Code Quality ### Linting and Formatting diff --git a/bench/generate.bench.mjs b/bench/generate.bench.mjs new file mode 100644 index 000000000..f7972136a --- /dev/null +++ b/bench/generate.bench.mjs @@ -0,0 +1,66 @@ +import { mkdir, rm } from 'node:fs/promises'; +import path from 'node:path'; + +import { withCodSpeed } from '@codspeed/tinybench-plugin'; +import createGenerator from '@doc-kit/core'; +import logger from '@doc-kit/core/logger/index.mjs'; +import { setConfig } from '@doc-kit/core/utils/configuration/index.mjs'; +import { Bench } from 'tinybench'; + +logger.setLogLevel('warn'); + +const node = path.join(import.meta.dirname, '../node'); +const at = (...parts) => path.join(node, ...parts); + +const shared = { + threads: 1, + index: at('doc/api/index.md'), + typeMap: at('doc/type-map.json'), + changelog: at('CHANGELOG.md'), +}; + +const docs = at('doc/api/*.md'); + +const targets = [ + ['json', docs], + ['json-simple', docs], + ['legacy-json', docs], + ['legacy-html', docs], + ['html', docs], + ['llms-txt', docs], + ['orama-db', docs], + ['man-page', at('doc/api/cli.md')], + ['addon-verify', at('doc/api/addons.md')], + ['api-links', at('lib/*.js')], +]; + +const bench = withCodSpeed( + new Bench({ warmup: false, iterations: 1, time: 0, throws: true }) +); + +for (const [name, input] of targets) { + const output = path.join(import.meta.dirname, '../out', name); + let runGenerators; + let config; + + bench.add(name, () => runGenerators(config), { + beforeAll: async () => { + await rm(output, { recursive: true, force: true }); + await mkdir(output, { recursive: true }); + ({ runGenerators } = createGenerator()); + config = await setConfig({ + ...shared, + target: [name], + input: [input], + output, + }); + }, + afterAll: () => rm(output, { recursive: true, force: true }), + }); +} + +await bench.run(); + +if (process.env.CODSPEED_ENV === undefined) { + console.table(bench.table()); +} diff --git a/docs/contributing/comparators.md b/docs/contributing/comparators.md index 2743a7e94..180408bce 100644 --- a/docs/contributing/comparators.md +++ b/docs/contributing/comparators.md @@ -7,7 +7,7 @@ This guide explains how to create build comparison scripts for `@doc-kit/core`. Comparators are scripts that: 1. **Compare** generated documentation between two builds (base vs. head) -2. **Identify differences** in content, structure, file size, or performance +2. **Identify differences** in content, structure, or file size 3. **Report results** in a format suitable for CI/CD systems 4. **Help catch regressions** before merging changes @@ -15,7 +15,6 @@ Comparators are scripts that: - **Verify backward compatibility** - Ensure new code produces same output - **Track file size changes** - Monitor bundle size growth -- **Catch performance regressions** - Compare elapsed time, CPU time, and peak memory - **Validate transformations** - Check that refactors don't alter output - **Debug generation issues** - Understand what changed between versions @@ -28,10 +27,9 @@ sharing the `BASE`, `HEAD`, and `TITLE` constants from `scripts/constants.mjs`: scripts/ ├── constants.mjs # Shared constants (BASE, HEAD, TITLE) └── comparators/ - ├── file-size.mjs # Compare file sizes and performance between builds + ├── file-size.mjs # Compare file sizes between builds ├── files.mjs # Shared output-file listing helpers ├── object-assertion.mjs # Deep equality assertion for JSON objects - ├── performance.mjs # Compare benchmark measurements └── your-comparator.mjs # Your new comparator ``` @@ -43,12 +41,6 @@ Comparators can be reused across multiple generators. You specify which comparat - `object-assertion.mjs` can compare JSON output from `legacy-json`, `json-simple`, etc. - `my-comparator.mjs` would be a custom comparator for specific needs -The generation workflow also stores timing, CPU, and peak resident memory in -`benchmark.json`. The built-in comparators include these measurements in their -Markdown report and exclude the metadata file from output comparisons. If a -base artifact predates performance measurement, the output comparison still -runs and the performance section is omitted. - ## Creating a Comparator ### Step 1: Create the Comparator File diff --git a/package.json b/package.json index 9f4249ca8..7505d2967 100644 --- a/package.json +++ b/package.json @@ -17,6 +17,7 @@ "test:ci": "c8 --reporter=lcov node --test --experimental-test-module-mocks \"packages/*/src/**/*.test.mjs\" \"scripts/**/*.test.mjs\" --test-reporter=@reporters/github --test-reporter-destination=stdout --test-reporter=junit --test-reporter-destination=junit.xml --test-reporter=spec --test-reporter-destination=stdout", "test:update-snapshots": "node --test --experimental-test-module-mocks --test-update-snapshots \"packages/*/src/**/*.test.mjs\" \"scripts/**/*.test.mjs\"", "test:watch": "node --test --experimental-test-module-mocks --watch \"packages/*/src/**/*.test.mjs\" \"scripts/**/*.test.mjs\"", + "bench": "node bench/generate.bench.mjs", "test:e2e": "playwright test", "prepare": "husky || exit 0", "run": "node packages/cli/bin/cli.mjs", @@ -36,6 +37,8 @@ "devDependencies": { "@changesets/changelog-github": "^1.0.1", "@changesets/cli": "^3.0.3", + "@codspeed/tinybench-plugin": "6.1.0", + "@doc-kit/core": "workspace:*", "@node-core/doc-kit": "workspace:*", "@playwright/test": "~1.64.1", "@reporters/github": "~2.1.4", @@ -50,6 +53,7 @@ "lint-staged": "17.6.0", "oxfmt": "0.74.0", "oxlint": "1.89.0", + "tinybench": "6.2.3", "wrangler": "4.151.1" } } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index e54d9efae..3053d1c2a 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -14,6 +14,12 @@ importers: '@changesets/cli': specifier: ^3.0.3 version: 3.0.3 + '@codspeed/tinybench-plugin': + specifier: 6.1.0 + version: 6.1.0(debug@4.4.3(supports-color@10.2.2))(supports-color@10.2.2)(tinybench@6.2.3) + '@doc-kit/core': + specifier: workspace:* + version: link:packages/core '@node-core/doc-kit': specifier: workspace:* version: link:packages/node @@ -56,6 +62,9 @@ importers: oxlint: specifier: 1.89.0 version: 1.89.0 + tinybench: + specifier: 6.2.3 + version: 6.2.3 wrangler: specifier: 4.151.1 version: 4.151.1(@types/node@26.6.6) @@ -467,6 +476,14 @@ packages: cpu: [x64] os: [win32] + '@codspeed/core@6.1.0': + resolution: {integrity: sha512-lxFkexeuf6iOWUZmgCeuGSVgk0fKBbtnCVd3w3UWi3627/4t55FhA046LhFLzedCQUi6183s99DnM3/6TRjzHQ==} + + '@codspeed/tinybench-plugin@6.1.0': + resolution: {integrity: sha512-R34+lpwSyCll2CpHSFaVDJq7uHWexlUpSq+FheiaI9qzKb5BNN3z/WNGwFPMcEXO5xpUfjg7qepO83aGuwcSyg==} + peerDependencies: + tinybench: '>=4.0.1' + '@cspotcode/source-map-support@0.8.1': resolution: {integrity: sha512-IchNf6dN4tHoMFIn/7OE8LWZ19Y6q/67Bmf6vnGREv8RSbBVb9LPJxEcnwrcwX6ixSvaiGoomAUvu4YSxXrVgw==} engines: {node: '>=12'} @@ -1984,6 +2001,10 @@ packages: engines: {node: '>=0.4.0'} hasBin: true + agent-base@6.0.2: + resolution: {integrity: sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==} + engines: {node: '>= 6.0.0'} + ajv@6.15.0: resolution: {integrity: sha512-fgFx7Hfoq60ytK2c7DhnF8jIvzYgOMxfugjLOSMHjLIPgenqa7S7oaagATUq99mV6IYvN2tRmC0wnTYX6iPbMw==} @@ -2013,6 +2034,12 @@ packages: resolution: {integrity: sha512-LElXdjswlqjWrPpJFg1Fx4wpkOCxj1TDHlSV4PlaRxHGWko024xICaa97ZkMfs6DRKlCguiAI+rbXv5GWwXIkg==} hasBin: true + asynckit@0.4.0: + resolution: {integrity: sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==} + + axios@1.21.0: + resolution: {integrity: sha512-YCzXMB53Q9PZBsh7yN7wtapit4WepwcHS2fP8QhGsBxSz5YtgYobV7GGlWlAJi+RxfU77GQTQi2DSLUma42EYA==} + bail@2.0.2: resolution: {integrity: sha512-0xO6mYd7JB2YesxDKplafRpsiOzPt9V02ddPCLbY1xYGPOX24NTyN50qnUxgCPcSoYMhKpAuBTjQoRZCAkUDRw==} @@ -2044,6 +2071,10 @@ packages: resolution: {integrity: sha512-tixWYgm5ZoOD+3g6UTea91eow5z6AAHaho3g0V9CNSNb45gM8SmflpAc+GRd1InC4AqN/07Unrgp56Y94N9hJQ==} engines: {node: '>=20.19.0'} + call-bind-apply-helpers@1.0.2: + resolution: {integrity: sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==} + engines: {node: '>= 0.4'} + ccount@2.0.1: resolution: {integrity: sha512-eyrF0jiFpY+3drT6383f1qhkbGsLSifNAjA61IUjZjmLCWjItY6LB9ft9YhoDgwfmclB2zhu51Lc7+95b8NRAg==} @@ -2066,6 +2097,10 @@ packages: resolution: {integrity: sha512-k7ndgKhwoQveBL+/1tqGJYNz097I7WOvwbmmU2AR5+magtbjPWQTS1C5vzGkBC8Ym8UWRzfKUzUUqFLypY4Q+w==} engines: {node: '>=20'} + combined-stream@1.0.8: + resolution: {integrity: sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==} + engines: {node: '>= 0.8'} + comma-separated-tokens@2.0.3: resolution: {integrity: sha512-Fu4hJdvzeylCfQPp9SGWidpzrMs7tTrlu6Vb8XGaRGck8QSNZJJp538Wrb60Lax4fPwR64ViY468OIUTbRlGZg==} @@ -2131,6 +2166,10 @@ packages: deep-is@0.1.4: resolution: {integrity: sha512-oIPzksmTg4/MriiaYGO+okXDT7ztn/w3Eptv/+gSIdMdKsJo0u4CfYNFJPy+4SKMuCqGw2wxnA+URMg3t8a/bQ==} + delayed-stream@1.0.0: + resolution: {integrity: sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==} + engines: {node: '>=0.4.0'} + dequal@2.0.3: resolution: {integrity: sha512-0je+qPKHEMohvfRTCEo3CrPG6cAzAYgmzKyxRiYSSDkS6eGJdyVJm7WaYA5ECaAD9wLB2T4EEeymA5aFVcYXCA==} engines: {node: '>=6'} @@ -2145,6 +2184,10 @@ packages: devlop@1.1.0: resolution: {integrity: sha512-RWmIqhcFf1lRYBvNmr7qTNuyCt/7/ns2jbpp1+PalgE/rDQcBT0fioSMUpJ93irlUhC5hrg4cYqe6U+0ImW0rA==} + dunder-proto@1.0.1: + resolution: {integrity: sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==} + engines: {node: '>= 0.4'} + emoji-regex@10.6.0: resolution: {integrity: sha512-toUI84YS5YmxW219erniWD0CIVOo46xGKColeNQRgOzDorgBi1v4D71/OFzgD9GO2UGKIv1C3Sp8DAn0+j5w7A==} @@ -2163,6 +2206,22 @@ packages: error-stack-parser-es@1.0.5: resolution: {integrity: sha512-5qucVt2XcuGMcEGgWI7i+yZpmpByQ8J1lHhcL7PwqCwu9FPP3VUXzT4ltHe5i2z9dePwEHcDVOAfSnHsOlCXRA==} + es-define-property@1.0.1: + resolution: {integrity: sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==} + engines: {node: '>= 0.4'} + + es-errors@1.3.0: + resolution: {integrity: sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==} + engines: {node: '>= 0.4'} + + es-object-atoms@1.1.2: + resolution: {integrity: sha512-HWcBoN6NileqtSydK2FqHbS/LoDd2pqrnQHLyJzBj4kOp/ky2MWMN694xOfkK8/SnUsW2DH7EfyVlydKCsm1Zw==} + engines: {node: '>= 0.4'} + + es-set-tostringtag@2.1.0: + resolution: {integrity: sha512-j6vWzfrGVfyXxge+O0x5sh6cvxAog0a/4Rdd2K36zCMV5eJ+/+tOAngRO8cODMNWbVRdVlmGZQL2YS3yR8bIUA==} + engines: {node: '>= 0.4'} + esast-util-from-estree@2.0.0: resolution: {integrity: sha512-4CyanoAudUSBAn5K13H4JhsMH6L9ZP7XbLVe/dKybkxMO7eDyLsT8UHl9TRNrU2Gr9nz+FovfSIjuXWJ81uVwQ==} @@ -2317,15 +2376,31 @@ packages: flatted@3.4.4: resolution: {integrity: sha512-5+ybhBZANEJxaH3X5evAFatUxLfEHSr7n6kYJ+1Qd0mUqr4eu9gIf6GDbWHf8RJijHrjjO8G+la14SlL2SeS1Q==} + follow-redirects@1.16.2: + resolution: {integrity: sha512-/kVP2U8z596Ia0gwFrwyIeo74ta5aVmSeU1FfAUMAmcEP5EP7zydmag3SLiN64NaRKef3lxsq7i/g7ltJQDGow==} + engines: {node: '>=4.0'} + peerDependencies: + debug: '*' + peerDependenciesMeta: + debug: + optional: true + foreground-child@3.3.1: resolution: {integrity: sha512-gIXjKqtFuWEgzFRJA9WCQeSJLZDjgJUOMCMzxtvFq/37KojM1BFGufqsCy0r4qSQmYLsZYMeyRqzIWOMup03sw==} engines: {node: '>=14'} + form-data@4.0.6: + resolution: {integrity: sha512-vKatAh4SlVfgbv+YtmhiRjhEMJsYpsG1Y2rMQtR+SVSbytsSD1YGzDIcrAJmdFec88u/+VoGmxnl+80gL1tRCQ==} + engines: {node: '>= 6'} + fsevents@2.3.3: resolution: {integrity: sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==} engines: {node: ^8.16.0 || ^10.6.0 || >=11.0.0} os: [darwin] + function-bind@1.1.2: + resolution: {integrity: sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==} + get-caller-file@2.0.5: resolution: {integrity: sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==} engines: {node: 6.* || 8.* || >= 10.*} @@ -2334,10 +2409,18 @@ packages: resolution: {integrity: sha512-QRbvDIbx6YklUe6RxeTeleMR0yv3cYH6PsPZHcnVn7xv7zO1BHN8r0XETu8n6Ye3Q+ahtSarc3WgtNWmehIBfA==} engines: {node: '>=18'} + get-intrinsic@1.3.0: + resolution: {integrity: sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==} + engines: {node: '>= 0.4'} + get-nonce@1.0.1: resolution: {integrity: sha512-FJhYRoDaiatfEkUK8HKlicmu/3SGFD51q3itKDGoSTysQJBnfOcxU5GxnhE1E6soB76MbT0MBtnKJuXyAx+96Q==} engines: {node: '>=6'} + get-proto@1.0.1: + resolution: {integrity: sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==} + engines: {node: '>= 0.4'} + github-slugger@2.0.0: resolution: {integrity: sha512-IaOQ9puYtjrkq7Y0Ygl9KDZnrf/aiUJYUpVf89y8kyaxbRG7Y1SrX/jaumrv81vc61+kiMempujsM3Yw7w5qcw==} @@ -2349,10 +2432,26 @@ packages: resolution: {integrity: sha512-Wjlyrolmm8uDpm/ogGyXZXb1Z+Ca2B8NbJwqBVg0axK9GbBeoS7yGV6vjXnYdGm6X53iehEuxxbyiKp8QmN4Vw==} engines: {node: 18 || 20 || >=22} + gopd@1.2.0: + resolution: {integrity: sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==} + engines: {node: '>= 0.4'} + has-flag@4.0.0: resolution: {integrity: sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==} engines: {node: '>=8'} + has-symbols@1.1.0: + resolution: {integrity: sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==} + engines: {node: '>= 0.4'} + + has-tostringtag@1.0.2: + resolution: {integrity: sha512-NqADB8VjPFLM2V0VvHUewwwsw0ZWBaIdgo+ieHtK3hasLz4qeCRjYcqfB6AQrBggRKppKF8L52/VqdVsO47Dlw==} + engines: {node: '>= 0.4'} + + hasown@2.0.4: + resolution: {integrity: sha512-T2UbfbBEF32wiepXIsMlTW9+dDYC6wMh/t/vYA4tuOMKqWz/n3vr1NFSxQiyP+zk2mXsoMA/i/7qV6LKut1t1A==} + engines: {node: '>= 0.4'} + hast-util-from-parse5@8.0.3: resolution: {integrity: sha512-3kxEVkEKt0zvcZ3hCRYI8rqrgwtlIOFMWkbclACvjlDw8Li9S2hk/d51OI0nr/gIpdMHNepwgOKqZ/sy0Clpyg==} @@ -2395,6 +2494,10 @@ packages: html-void-elements@3.0.0: resolution: {integrity: sha512-bEqo66MRXsUGxWHV5IP0PUiAWwoEjba4VCzg0LjFJBpchPaTfyfCKTG6bc5F8ucKec3q5y6qOdGyYTSBEvhCrg==} + https-proxy-agent@5.0.1: + resolution: {integrity: sha512-dFcAjpTQFgoLMzC2VwU+C/CbS7uRL0lWmxDITmqm7C+7F0Odmj6s9l6alZc6AELXhrnggM2CeWSXHGOdX2YtwA==} + engines: {node: '>= 6'} + human-id@4.2.1: resolution: {integrity: sha512-zPGsiS+dWoTZtZ4AtpA9Y+BdSFSNWvnouNlWNoUFyAM6xHOHmdCvqO3k8AIbdamCOv4gUFUVNPf6rJFfc4UiJw==} hasBin: true @@ -2620,6 +2723,10 @@ packages: markdown-table@3.0.4: resolution: {integrity: sha512-wiYz4+JrLyb/DqW2hkFJxP7Vd7JuTDm77fvbM8VfEQdmSMqcImWeeRbHwZjBjIFki/VaMK2BhFi7oUUZeM5bqw==} + math-intrinsics@1.1.0: + resolution: {integrity: sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==} + engines: {node: '>= 0.4'} + mdast-util-find-and-replace@3.0.2: resolution: {integrity: sha512-Tmd1Vg/m3Xz43afeNxDIhWRtFZgM2VLyaf4vSTYwudTyeuTneoL3qtWMA5jeLyz/O1vDJmmV4QuScFCA2tBPwg==} @@ -2779,6 +2886,14 @@ packages: micromark@4.0.2: resolution: {integrity: sha512-zpe98Q6kvavpCr1NPVSCMebCKfD7CA2NqZ+rykeNhONIJBpc1tFKt9hucLGwha3jNTNI8lHpctWJWoimVF4PfA==} + mime-db@1.52.0: + resolution: {integrity: sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==} + engines: {node: '>= 0.6'} + + mime-types@2.1.35: + resolution: {integrity: sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==} + engines: {node: '>= 0.6'} + miniflare@5.20261012.1-beta: resolution: {integrity: sha512-8Canhy6g9+vAhqMc0o80YphqYIwpiA3aAp8aX+uDXU7J/qbUeoZzcII9c2V1P4uT/eHD4SB93qet1o1bRY3BuQ==} engines: {node: '>=22.0.0'} @@ -2805,6 +2920,10 @@ packages: natural-compare@1.4.0: resolution: {integrity: sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw==} + node-gyp-build@4.8.4: + resolution: {integrity: sha512-LA4ZjwlnUblHVgq0oBF3Jl/6h/Nvs5fzBLwdEF4nuxnFdsfajde4WfxtJr3CaiH+F6ewcIB/q4jQ4UzPyid+CQ==} + hasBin: true + nth-check@2.1.1: resolution: {integrity: sha512-lqjrjmaOoAnWfMmBPL+XNnynZh2+swxiX3WUE0s4yEHI6m+AwrK2UZOimIRl3X/4QctVqS8AiZjFqyOGrMXb/w==} @@ -2946,6 +3065,10 @@ packages: property-information@7.2.0: resolution: {integrity: sha512-IAtzIB6sUiWaJYrX9smp3V46pBGbBeLFRGdh25kg1334VcBlD8HzhPeNIWQH9zhGmo2itIe25EHt9dQP7G5hmg==} + proxy-from-env@2.1.0: + resolution: {integrity: sha512-cJ+oHTW1VAEa8cJslgmUZrc+sjRKgAKl3Zyse6+PV38hZe/V6Z14TbCuXcan9F9ghlz4QrFr2c92TNF82UkYHA==} + engines: {node: '>=10'} + punycode.js@2.3.1: resolution: {integrity: sha512-uxFIHU0YlHYhDQtV4R9J6a52SLx28BCjT+4ieh7IGbgwVJWO+km431c4yRlREUAsAmt/uMjQUyQHNEPf0M39CA==} engines: {node: '>=6'} @@ -3120,6 +3243,10 @@ packages: spdx-license-ids@3.0.23: resolution: {integrity: sha512-CWLcCCH7VLu13TgOH+r8p1O/Znwhqv/dbb6lqWy67G+pT1kHmeD/+V36AVb/vq8QMIQwVShJ6Ssl5FPh0fuSdw==} + stack-trace@1.0.0-pre2: + resolution: {integrity: sha512-2ztBJRek8IVofG9DBJqdy2N5kulaacX30Nz7xmkYF6ale9WBVmIy6mFBchvGX7Vx/MyjBhx+Rcxqrj+dbOnQ6A==} + engines: {node: '>=16'} + stack-utils@2.0.6: resolution: {integrity: sha512-XlkWvfIm6RmsWtNJx+uqtKLS8eqFbxUg0ZzLXqY0caEy9l7hruX8IpiDnjsLavoBgqCCR71TqWO8MaXYheJ3RQ==} engines: {node: '>=10'} @@ -3168,6 +3295,10 @@ packages: resolution: {integrity: sha512-nt6AMGKW1p/70DF/hGBdJB57B8Tspmbp5gfJ8ilhLnt7kkr2ye7hzD6NVG8GGErk2HWF34igrL2CXmNIkzKqKw==} engines: {node: '>=18'} + tinybench@6.2.3: + resolution: {integrity: sha512-Al01C2JIBnO2CXCAPbpAl6sUiOZHeThGtHYNTa51JRw9PyWOCdr+9Ong05R4qg/Sk3+H645E8ov1h2c58ityNw==} + engines: {node: '>=20.0.0'} + tinyexec@1.3.1: resolution: {integrity: sha512-GCvB3aoys96IuDFBMcTB46JOR6mdMtAToqwiW8JlWhsoh1mhHi/xn9ss/Dg7N555GiJyEt2qzoG/NHCwM6h1EA==} engines: {node: '>=18'} @@ -3626,6 +3757,24 @@ snapshots: '@cloudflare/workerd-windows-64@1.20261012.1': optional: true + '@codspeed/core@6.1.0(debug@4.4.3(supports-color@10.2.2))(supports-color@10.2.2)': + dependencies: + axios: 1.21.0(debug@4.4.3(supports-color@10.2.2))(supports-color@10.2.2) + form-data: 4.0.6 + node-gyp-build: 4.8.4 + stack-trace: 1.0.0-pre2 + transitivePeerDependencies: + - debug + - supports-color + + '@codspeed/tinybench-plugin@6.1.0(debug@4.4.3(supports-color@10.2.2))(supports-color@10.2.2)(tinybench@6.2.3)': + dependencies: + '@codspeed/core': 6.1.0(debug@4.4.3(supports-color@10.2.2))(supports-color@10.2.2) + tinybench: 6.2.3 + transitivePeerDependencies: + - debug + - supports-color + '@cspotcode/source-map-support@0.8.1': dependencies: '@jridgewell/trace-mapping': 0.3.9 @@ -4901,6 +5050,12 @@ snapshots: acorn@8.19.0: {} + agent-base@6.0.2(supports-color@10.2.2): + dependencies: + debug: 4.4.3(supports-color@10.2.2) + transitivePeerDependencies: + - supports-color + ajv@6.15.0: dependencies: fast-deep-equal: 3.1.3 @@ -4929,6 +5084,18 @@ snapshots: astring@1.9.0: {} + asynckit@0.4.0: {} + + axios@1.21.0(debug@4.4.3(supports-color@10.2.2))(supports-color@10.2.2): + dependencies: + follow-redirects: 1.16.2(debug@4.4.3(supports-color@10.2.2)) + form-data: 4.0.6 + https-proxy-agent: 5.0.1(supports-color@10.2.2) + proxy-from-env: 2.1.0 + transitivePeerDependencies: + - debug + - supports-color + bail@2.0.2: {} balanced-match@4.0.4: {} @@ -4957,6 +5124,11 @@ snapshots: cac@7.0.0: {} + call-bind-apply-helpers@1.0.2: + dependencies: + es-errors: 1.3.0 + function-bind: 1.1.2 + ccount@2.0.1: {} character-entities-html4@2.1.0: {} @@ -4975,6 +5147,10 @@ snapshots: strip-ansi: 7.2.0 wrap-ansi: 9.0.2 + combined-stream@1.0.8: + dependencies: + delayed-stream: 1.0.0 + comma-separated-tokens@2.0.3: {} commander@15.0.0: {} @@ -5020,6 +5196,8 @@ snapshots: deep-is@0.1.4: {} + delayed-stream@1.0.0: {} + dequal@2.0.3: {} detect-libc@2.1.2: {} @@ -5030,6 +5208,12 @@ snapshots: dependencies: dequal: 2.0.3 + dunder-proto@1.0.1: + dependencies: + call-bind-apply-helpers: 1.0.2 + es-errors: 1.3.0 + gopd: 1.2.0 + emoji-regex@10.6.0: {} entities@4.5.0: {} @@ -5040,6 +5224,21 @@ snapshots: error-stack-parser-es@1.0.5: {} + es-define-property@1.0.1: {} + + es-errors@1.3.0: {} + + es-object-atoms@1.1.2: + dependencies: + es-errors: 1.3.0 + + es-set-tostringtag@2.1.0: + dependencies: + es-errors: 1.3.0 + get-intrinsic: 1.3.0 + has-tostringtag: 1.0.2 + hasown: 2.0.4 + esast-util-from-estree@2.0.0: dependencies: '@types/estree-jsx': 1.0.5 @@ -5260,20 +5459,52 @@ snapshots: flatted@3.4.4: {} + follow-redirects@1.16.2(debug@4.4.3(supports-color@10.2.2)): + optionalDependencies: + debug: 4.4.3(supports-color@10.2.2) + foreground-child@3.3.1: dependencies: cross-spawn: 7.0.6 signal-exit: 4.1.0 + form-data@4.0.6: + dependencies: + asynckit: 0.4.0 + combined-stream: 1.0.8 + es-set-tostringtag: 2.1.0 + hasown: 2.0.4 + mime-types: 2.1.35 + fsevents@2.3.3: optional: true + function-bind@1.1.2: {} + get-caller-file@2.0.5: {} get-east-asian-width@1.6.0: {} + get-intrinsic@1.3.0: + dependencies: + call-bind-apply-helpers: 1.0.2 + es-define-property: 1.0.1 + es-errors: 1.3.0 + es-object-atoms: 1.1.2 + function-bind: 1.1.2 + get-proto: 1.0.1 + gopd: 1.2.0 + has-symbols: 1.1.0 + hasown: 2.0.4 + math-intrinsics: 1.1.0 + get-nonce@1.0.1: {} + get-proto@1.0.1: + dependencies: + dunder-proto: 1.0.1 + es-object-atoms: 1.1.2 + github-slugger@2.0.0: {} glob-parent@6.0.2: @@ -5286,8 +5517,20 @@ snapshots: minipass: 7.1.3 path-scurry: 2.0.2 + gopd@1.2.0: {} + has-flag@4.0.0: {} + has-symbols@1.1.0: {} + + has-tostringtag@1.0.2: + dependencies: + has-symbols: 1.1.0 + + hasown@2.0.4: + dependencies: + function-bind: 1.1.2 + hast-util-from-parse5@8.0.3: dependencies: '@types/hast': 3.0.5 @@ -5408,6 +5651,13 @@ snapshots: html-void-elements@3.0.0: {} + https-proxy-agent@5.0.1(supports-color@10.2.2): + dependencies: + agent-base: 6.0.2(supports-color@10.2.2) + debug: 4.4.3(supports-color@10.2.2) + transitivePeerDependencies: + - supports-color + human-id@4.2.1: {} husky@9.1.7: {} @@ -5595,6 +5845,8 @@ snapshots: markdown-table@3.0.4: {} + math-intrinsics@1.1.0: {} + mdast-util-find-and-replace@3.0.2: dependencies: '@types/mdast': 4.0.4 @@ -6026,6 +6278,12 @@ snapshots: transitivePeerDependencies: - supports-color + mime-db@1.52.0: {} + + mime-types@2.1.35: + dependencies: + mime-db: 1.52.0 + miniflare@5.20261012.1-beta(@types/node@26.6.6): dependencies: '@cspotcode/source-map-support': 0.8.1 @@ -6053,6 +6311,8 @@ snapshots: natural-compare@1.4.0: {} + node-gyp-build@4.8.4: {} + nth-check@2.1.1: dependencies: boolbase: 1.0.0 @@ -6201,6 +6461,8 @@ snapshots: property-information@7.2.0: {} + proxy-from-env@2.1.0: {} + punycode.js@2.3.1: {} punycode@2.3.1: {} @@ -6462,6 +6724,8 @@ snapshots: spdx-license-ids@3.0.23: {} + stack-trace@1.0.0-pre2: {} + stack-utils@2.0.6: dependencies: escape-string-regexp: 2.0.0 @@ -6512,6 +6776,8 @@ snapshots: throttleit@2.1.0: {} + tinybench@6.2.3: {} + tinyexec@1.3.1: {} tinyglobby@0.2.17: diff --git a/scripts/__tests__/comparators.test.mjs b/scripts/__tests__/comparators.test.mjs index e398ae7da..cd0547aed 100644 --- a/scripts/__tests__/comparators.test.mjs +++ b/scripts/__tests__/comparators.test.mjs @@ -7,15 +7,6 @@ import { test } from 'node:test'; import { fileURLToPath } from 'node:url'; import { promisify } from 'node:util'; -import { comparePerformance } from '../comparators/performance.mjs'; - -const benchmark = { - elapsedSeconds: 2, - userCpuSeconds: 1, - systemCpuSeconds: 0.5, - maxRssKiB: 1024, -}; - const executeFile = promisify(execFile); const comparatorDirectory = fileURLToPath( new URL('../comparators/', import.meta.url) @@ -35,13 +26,6 @@ const createDirectories = async t => { return { base, head }; }; -const writeBenchmark = (directory, value = benchmark) => - writeFile( - path.join(directory, 'benchmark.json'), - JSON.stringify(value), - 'utf8' - ); - const runComparator = async (name, base, head) => { const { stdout } = await executeFile( process.execPath, @@ -54,63 +38,12 @@ const runComparator = async (name, base, head) => { return stdout; }; -test('comparePerformance summarizes benchmark differences', async t => { - const { base, head } = await createDirectories(t); - - await Promise.all([ - writeBenchmark(base), - writeBenchmark(head, { - elapsedSeconds: 3, - userCpuSeconds: 0.75, - systemCpuSeconds: 0.5, - maxRssKiB: 1536, - }), - ]); - - const result = await comparePerformance(base, head); - - assert.match( - result, - /\*\*Generation time:\*\* 50\.0% slower \(2\.00 s → 3\.00 s\)/ - ); - assert.match( - result, - /\*\*Peak memory:\*\* 50\.0% higher \(1\.00 MB → 1\.50 MB\)/ - ); - assert.match(result, /single CI run/); - assert.doesNotMatch(result, /CPU time/); -}); - -test('comparePerformance omits results when an artifact has no benchmark', async t => { - const { base, head } = await createDirectories(t); - - await writeBenchmark(head); - - assert.equal(await comparePerformance(base, head), ''); -}); - -test('comparePerformance rejects invalid benchmark values', async t => { - const { base, head } = await createDirectories(t); - - await Promise.all([ - writeBenchmark(base), - writeBenchmark(head, { ...benchmark, maxRssKiB: undefined }), - ]); - - await assert.rejects( - comparePerformance(base, head), - /Invalid peak memory benchmark value/ - ); -}); - -test('file-size comparator combines output and performance results', async t => { +test('file-size comparator reports output changes', async t => { const { base, head } = await createDirectories(t); await Promise.all([ writeFile(path.join(base, 'result.txt'), 'base', 'utf8'), writeFile(path.join(head, 'result.txt'), 'a larger result', 'utf8'), - writeBenchmark(base), - writeBenchmark(head, { ...benchmark, elapsedSeconds: 3 }), ]); const result = await runComparator('file-size', base, head); @@ -123,26 +56,19 @@ test('file-size comparator combines output and performance results', async t => /\| File \| Main \| PR \| Change \|\n\| --- \| ---: \| ---: \| ---: \|/ ); assert.match(result, /`result\.txt`/); - assert.match(result, /Performance estimate/); - assert.doesNotMatch(result, /benchmark\.json/); }); -test('object comparator treats benchmark data as metadata', async t => { +test('object comparator is silent when output matches', async t => { const { base, head } = await createDirectories(t); await Promise.all([ writeFile(path.join(base, 'result.json'), '{"value":true}', 'utf8'), writeFile(path.join(head, 'result.json'), '{"value":true}', 'utf8'), - writeBenchmark(base), - writeBenchmark(head, { ...benchmark, maxRssKiB: 2048 }), ]); const result = await runComparator('object-assertion', base, head); - assert.equal(result.match(/## `test` Generator/g)?.length, 1); - assert.doesNotMatch(result, /\*\*Output:/); - assert.match(result, /Performance estimate/); - assert.doesNotMatch(result, /benchmark\.json/); + assert.equal(result, ''); }); test('comparators report added and removed output files', async t => { diff --git a/scripts/comparators/file-size.mjs b/scripts/comparators/file-size.mjs index 81e2e8653..b167cb318 100644 --- a/scripts/comparators/file-size.mjs +++ b/scripts/comparators/file-size.mjs @@ -3,7 +3,6 @@ import path from 'node:path'; import { BASE, HEAD, TITLE } from '../constants.mjs'; import { listOutputFiles } from './files.mjs'; -import { comparePerformance } from './performance.mjs'; const UNITS = ['B', 'KB', 'MB', 'GB']; @@ -84,11 +83,6 @@ if (changed.length) { ); } -const performance = await comparePerformance(); -if (performance) { - sections.push(performance); -} - if (sections.length) { console.log(`${TITLE}\n\n${sections.join('\n\n')}\n`); } diff --git a/scripts/comparators/files.mjs b/scripts/comparators/files.mjs index 7e7d1035c..cb67f4246 100644 --- a/scripts/comparators/files.mjs +++ b/scripts/comparators/files.mjs @@ -1,15 +1,13 @@ import { glob } from 'node:fs/promises'; import path from 'node:path'; -import { BENCHMARK_FILE, COMPARISON_FILE } from '../constants.mjs'; - -const METADATA_FILES = new Set([BENCHMARK_FILE, COMPARISON_FILE]); +import { COMPARISON_FILE } from '../constants.mjs'; export const listOutputFiles = async directory => { const entries = glob('**/*', { cwd: directory, withFileTypes: true, - exclude: entry => METADATA_FILES.has(entry.name), + exclude: entry => entry.name === COMPARISON_FILE, }); return (await Array.fromAsync(entries)) diff --git a/scripts/comparators/object-assertion.mjs b/scripts/comparators/object-assertion.mjs index 33408523d..70b77e18f 100644 --- a/scripts/comparators/object-assertion.mjs +++ b/scripts/comparators/object-assertion.mjs @@ -4,7 +4,6 @@ import { join } from 'node:path'; import { BASE, HEAD, TITLE } from '../constants.mjs'; import { listOutputFiles } from './files.mjs'; -import { comparePerformance } from './performance.mjs'; const [baseFiles, headFiles] = await Promise.all( [BASE, HEAD].map(directory => listOutputFiles(directory)) @@ -51,11 +50,6 @@ if (filteredResults.length) { ); } -const performance = await comparePerformance(); -if (performance) { - sections.push(performance); -} - if (sections.length) { console.log(`${TITLE}\n\n${sections.join('\n\n')}\n`); } diff --git a/scripts/comparators/performance.mjs b/scripts/comparators/performance.mjs deleted file mode 100644 index f3c594ee8..000000000 --- a/scripts/comparators/performance.mjs +++ /dev/null @@ -1,108 +0,0 @@ -import { readFile } from 'node:fs/promises'; -import path from 'node:path'; - -import { BASE, BENCHMARK_FILE, HEAD } from '../constants.mjs'; - -const UNITS = ['B', 'KB', 'MB', 'GB']; - -const formatBytes = bytes => { - if (!bytes) { - return '0 B'; - } - - const unit = Math.min( - Math.floor(Math.log(Math.abs(bytes)) / Math.log(1024)), - UNITS.length - 1 - ); - return `${(bytes / Math.pow(1024, unit)).toFixed(2)} ${UNITS[unit]}`; -}; - -const formatSeconds = seconds => - Math.abs(seconds) < 1 - ? `${(seconds * 1_000).toFixed(2)} ms` - : `${seconds.toFixed(2)} s`; - -const formatPercent = (base, diff) => { - if (base === 0) { - return 'n/a'; - } - - return `${Math.abs((diff / base) * 100).toFixed(1)}%`; -}; - -const formatChange = (base, head, { increase, decrease }) => { - const diff = head - base; - - if (diff === 0) { - return 'unchanged'; - } - - return `${formatPercent(base, diff)} ${diff > 0 ? increase : decrease}`; -}; - -const readBenchmark = async directory => { - try { - return JSON.parse( - await readFile(path.join(directory, BENCHMARK_FILE), 'utf8') - ); - } catch (error) { - if (error.code === 'ENOENT') { - return null; - } - - throw error; - } -}; - -const METRICS = [ - { - label: 'Generation time', - value: benchmark => benchmark.elapsedSeconds, - format: formatSeconds, - change: { increase: 'slower', decrease: 'faster' }, - }, - { - label: 'Peak memory', - value: benchmark => benchmark.maxRssKiB * 1024, - format: formatBytes, - change: { increase: 'higher', decrease: 'lower' }, - }, -]; - -/** - * Builds a Markdown comparison of generation performance. - * - * Missing benchmark files are expected while artifacts created before - * performance measurement are still being used as the comparison base. - * - * @param {string} baseDirectory - Base artifact directory - * @param {string} headDirectory - Head artifact directory - * @returns {Promise} Markdown table, or an empty string - */ -export const comparePerformance = async ( - baseDirectory = BASE, - headDirectory = HEAD -) => { - const [base, head] = await Promise.all( - [baseDirectory, headDirectory].map(readBenchmark) - ); - - if (!base || !head) { - return ''; - } - - const rows = METRICS.map(({ label, value, format, change }) => { - const baseValue = value(base); - const headValue = value(head); - - if (!Number.isFinite(baseValue) || !Number.isFinite(headValue)) { - throw new TypeError(`Invalid ${label.toLowerCase()} benchmark value`); - } - - return `- **${label}:** ${formatChange(baseValue, headValue, change)} (${format(baseValue)} → ${format(headValue)})`; - }); - - return ['**Performance estimate** (single CI run)', ...rows].join( - '\n' - ); -}; diff --git a/scripts/constants.mjs b/scripts/constants.mjs index 2c885d6db..2476dd4ab 100644 --- a/scripts/constants.mjs +++ b/scripts/constants.mjs @@ -10,7 +10,6 @@ export const HEAD = export const TITLE = process.env.TITLE || `## \`${process.env.GENERATOR ?? '...'}\` Generator`; -export const BENCHMARK_FILE = 'benchmark.json'; export const COMPARISON_FILE = 'comparison.txt'; // MDN Constants