From 8cf1e1a74394e610f25f7e28b8397aab226f9891 Mon Sep 17 00:00:00 2001 From: owjs3901 Date: Sat, 3 Oct 2026 22:40:29 +0900 Subject: [PATCH 1/9] ci: fail the benchmark job on a Devup UI regression and keep a WASM size budget Refs #683 Ultraworked with [Sisyphus](https://github.com/code-yeongyu/oh-my-openagent) Co-authored-by: Sisyphus --- .github/workflows/publish.yml | 34 ++- .gitignore | 1 + README.md | 48 ++-- README_ko.md | 48 ++-- benchmark-baseline.json | 77 +++++ benchmark-check.js | 63 +++++ benchmark-gate.js | 1 + benchmark-manifest.json | 28 ++ benchmark-results.json | 110 ++++++-- benchmark.js | 149 ++++------ bindings/devup-ui-wasm/size-budget.json | 10 + libs/extractor/Cargo.toml | 4 + .../benches/semantic_pass_benchmark.rs | 111 ++++++++ .../src/__tests__/benchmark-gate.test.ts | 265 ++++++++++++++++++ render-benchmark-readme.js | 72 +++-- verify-wasm-size.js | 71 +++++ 16 files changed, 880 insertions(+), 212 deletions(-) create mode 100644 benchmark-baseline.json create mode 100644 benchmark-check.js create mode 100644 benchmark-gate.js create mode 100644 benchmark-manifest.json create mode 100644 bindings/devup-ui-wasm/size-budget.json create mode 100644 libs/extractor/benches/semantic_pass_benchmark.rs create mode 100644 packages/plugin-utils/src/__tests__/benchmark-gate.test.ts create mode 100644 verify-wasm-size.js diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 424d3fde3..3a69ca26e 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -73,16 +73,12 @@ jobs: node-version: ${{ env.NODE_VERSION }} - run: bun install - run: bun run build + # Every row is the median of the same number of cold samples taken in + # rotating order (benchmark-manifest.json). The run goes to an artifact and + # the Devup UI rows are compared with the checked baseline at the end of the + # job, so the profile and the Rust measurement still run when it fails. - name: Benchmark run: bun run benchmark - - name: Upload the measured run - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 - if: ${{ !cancelled() }} - with: - name: benchmark-run - path: benchmark-run.json - if-no-files-found: ignore - retention-days: 30 - name: Profile Devup UI Turbopack env: DEVUP_UI_PROFILE: "1" @@ -91,6 +87,18 @@ jobs: benchmark/next-devup-ui-single-turbo/df \ benchmark/next-devup-ui-single-turbo/tsconfig.tsbuildinfo bun run --filter next-devup-ui-single-turbo-benchmark build + - name: Upload the measured run + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + if: ${{ !cancelled() }} + with: + name: benchmark-run + path: benchmark-run.json + if-no-files-found: ignore + retention-days: 30 + - name: Measure the SemanticBuilder passes of the extractor + run: cargo bench -p extractor --bench semantic_pass_benchmark -- --quick + - name: Compare the Devup UI rows with the baseline + run: bun benchmark-check.js # The landing app is the only real-scale acceptance target in this repo, and # `publish` runs it against the deployed vinext/@devup-ui/vite-plugin build. @@ -297,6 +305,16 @@ jobs: - run: bun run build - name: Verify built package entry points run: bun run verify:dist + - name: Check the WASM size budget + run: bun verify-wasm-size.js + - name: Upload the WASM size + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + if: ${{ !cancelled() }} + with: + name: wasm-size + path: wasm-size.json + if-no-files-found: ignore + retention-days: 30 - name: Verify the README benchmark tables run: bun render-benchmark-readme.js --check - name: Verify publish tarballs diff --git a/.gitignore b/.gitignore index 7e5ea71be..0eb1f2217 100644 --- a/.gitignore +++ b/.gitignore @@ -25,3 +25,4 @@ playwright-report .omo /.changepacks/publish.tgz benchmark-run.json +wasm-size.json diff --git a/README.md b/README.md index 1596e7478..8057e46c4 100644 --- a/README.md +++ b/README.md @@ -80,33 +80,33 @@ npm install @devup-ui/webpack-plugin ## Comparison Benchmarks -[Latest CI benchmark](https://github.com/dev-five-git/devup-ui/actions/runs/36702867475) (commit `8d6c2f62`, Devup UI 1.0.44) on `ubuntu-24.04` with Next.js 16.3.3. All Next.js builds use the native TypeScript 7 CLI for type checking. The English and Korean tables are generated from the same checked result file, `benchmark-results.json`. +[Latest CI benchmark](https://github.com/dev-five-git/devup-ui/actions/runs/36702867475) (commit `8d6c2f62`, Devup UI 1.0.44) on `ubuntu-24.04` with Next.js 16.3.3. Every row is the median of 1 cold builds that ran in rotating order, shown with the range of the samples. All Next.js builds use the native TypeScript 7 CLI for type checking. The English and Korean tables are generated from the same checked result file, `benchmark-results.json`. Build Size is every byte of the build output directory (`.next`, or `dist` for vinext); CSS Size is the emitted `.css` files alone, measured separately. -Webpack values are one cold build: - -| Library | Version | Build Time | Build Size | CSS Size | -| --------------------------- | ------- | ---------- | ----------------- | ------------ | -| tailwindcss | 4.3.3 | 16.67s | 67,332,642 bytes | 5,852 bytes | -| styleX | 0.19.0 | 38.08s | 96,291,287 bytes | 425 bytes | -| vanilla-extract | 1.21.2 | 16.09s | 68,565,036 bytes | 294 bytes | -| kuma-ui | 1.6.4 | 17.91s | 75,627,783 bytes | 340 bytes | -| panda-css | 1.12.0 | 18.16s | 71,849,418 bytes | 15,615 bytes | -| chakra-ui | 3.37.0 | 27.22s | 207,426,274 bytes | 0 bytes | -| mui | 9.4.0 | 18.78s | 101,473,260 bytes | 0 bytes | -| **devup-ui (per-file CSS)** | 1.0.44 | **14.27s** | 67,424,897 bytes | 562 bytes | -| **devup-ui (single CSS)** | 1.0.44 | **14.17s** | 67,426,959 bytes | 790 bytes | - -Turbopack values are medians of six cold builds in alternating order: - -| Library | Version | Median Build Time | Build Size | CSS Size | -| -------------------------------------- | ------- | ----------------- | ---------------- | ----------- | -| tailwindcss | 4.3.3 | 6.98s | 38,422,141 bytes | 6,197 bytes | -| **devup-ui (direct APIs, single CSS)** | 1.0.44 | **6.93s** | 36,518,952 bytes | 327 bytes | -| **devup-ui (static `.css.ts`)** | 1.0.44 | **6.92s** | 36,578,506 bytes | 268 bytes | - -The Turbopack ranges overlap, so the direct-API median is 0.05s (0.7%) ahead of Tailwind on this fixture and the two are effectively at parity. The six cold samples were Tailwind `7.10s, 6.99s, 6.98s, 6.90s, 6.91s, 6.97s`, direct Devup UI `6.98s, 6.83s, 6.95s, 6.91s, 6.90s, 6.94s`, and static `.css.ts` `6.93s, 6.95s, 6.85s, 6.97s, 6.87s, 6.91s`. The fixtures have comparable app shapes, not pixel-identical styling: Tailwind styles the leading paragraph and button more heavily, while Devup UI exercises typed component/style props. Treat these as build-pipeline results rather than a per-rule microbenchmark. Every `.css.ts` module runs on the full Boa evaluator, which adds about 20 ms for the first module and under 1 ms for each further one. +Webpack: + +| Library | Version | Median Build Time | Range | Build Size | CSS Size | +| --------------------------- | ------- | ----------------- | --------------- | ----------------- | ------------ | +| tailwindcss | 4.3.3 | 16.67s | 16.67s - 16.67s | 67,332,642 bytes | 5,852 bytes | +| styleX | 0.19.0 | 38.08s | 38.08s - 38.08s | 96,291,287 bytes | 425 bytes | +| vanilla-extract | 1.21.2 | 16.09s | 16.09s - 16.09s | 68,565,036 bytes | 294 bytes | +| kuma-ui | 1.6.4 | 17.91s | 17.91s - 17.91s | 75,627,783 bytes | 340 bytes | +| panda-css | 1.12.0 | 18.16s | 18.16s - 18.16s | 71,849,418 bytes | 15,615 bytes | +| chakra-ui | 3.37.0 | 27.22s | 27.22s - 27.22s | 207,426,274 bytes | 0 bytes | +| mui | 9.4.0 | 18.78s | 18.78s - 18.78s | 101,473,260 bytes | 0 bytes | +| **devup-ui (per-file CSS)** | 1.0.44 | **14.27s** | 14.27s - 14.27s | 67,424,897 bytes | 562 bytes | +| **devup-ui (single CSS)** | 1.0.44 | **14.17s** | 14.17s - 14.17s | 67,426,959 bytes | 790 bytes | + +Turbopack: + +| Library | Version | Median Build Time | Range | Build Size | CSS Size | +| -------------------------------------- | ------- | ----------------- | ------------- | ---------------- | ----------- | +| tailwindcss | 4.3.3 | 7.04s | 7.04s - 7.04s | 38,422,141 bytes | 6,197 bytes | +| **devup-ui (direct APIs, single CSS)** | 1.0.44 | **6.83s** | 6.83s - 6.83s | 36,518,952 bytes | 327 bytes | +| **devup-ui (static `.css.ts`)** | 1.0.44 | **6.99s** | 6.99s - 6.99s | 36,578,506 bytes | 268 bytes | + +The Turbopack ranges do not overlap, so the direct-API median is 0.21s (3.0%) ahead of Tailwind on this fixture. The 1 cold samples were Tailwind `7.04s`, direct Devup UI `6.83s`, and static `.css.ts` `6.99s`. The fixtures have comparable app shapes, not pixel-identical styling: Tailwind styles the leading paragraph and button more heavily, while Devup UI exercises typed component/style props. Treat these as build-pipeline results rather than a per-rule microbenchmark. Every `.css.ts` module runs on the full Boa evaluator, which adds about 20 ms for the first module and under 1 ms for each further one. CI compares the Devup UI rows with a checked baseline (see `benchmark-gate.js` for the rule) and fails on a regression beyond the measured noise. ## How it works diff --git a/README_ko.md b/README_ko.md index 2c3554b1b..1a6524f9f 100644 --- a/README_ko.md +++ b/README_ko.md @@ -80,33 +80,33 @@ npm install @devup-ui/webpack-plugin ## 비교 벤치마크 -[최신 CI 벤치마크](https://github.com/dev-five-git/devup-ui/actions/runs/36702867475) (커밋 `8d6c2f62`, Devup UI 1.0.44)는 `ubuntu-24.04`에서 Next.js 16.3.3로 실행했습니다. 모든 Next.js 빌드는 네이티브 TypeScript 7 CLI로 타입 검사를 수행합니다. 영어와 한국어 표는 같은 결과 파일 `benchmark-results.json`에서 생성합니다. +[최신 CI 벤치마크](https://github.com/dev-five-git/devup-ui/actions/runs/36702867475) (커밋 `8d6c2f62`, Devup UI 1.0.44)는 `ubuntu-24.04`에서 Next.js 16.3.3로 실행했습니다. 모든 행은 순서를 돌려 가며 측정한 cold build 1회의 중앙값이며 샘플의 범위를 함께 보여줍니다. 모든 Next.js 빌드는 네이티브 TypeScript 7 CLI로 타입 검사를 수행합니다. 영어와 한국어 표는 같은 결과 파일 `benchmark-results.json`에서 생성합니다. 빌드 사이즈는 빌드 출력 디렉터리(`.next`, vinext는 `dist`)의 모든 바이트이고, CSS 사이즈는 출력된 `.css` 파일만 따로 잰 값입니다. -Webpack 값은 cold build 1회 결과입니다. - -| 라이브러리 | 버전 | 빌드 시간 | 빌드 사이즈 | CSS 사이즈 | -| --------------------------- | ------ | ---------- | ----------------- | ------------ | -| tailwindcss | 4.3.3 | 16.67s | 67,332,642 bytes | 5,852 bytes | -| styleX | 0.19.0 | 38.08s | 96,291,287 bytes | 425 bytes | -| vanilla-extract | 1.21.2 | 16.09s | 68,565,036 bytes | 294 bytes | -| kuma-ui | 1.6.4 | 17.91s | 75,627,783 bytes | 340 bytes | -| panda-css | 1.12.0 | 18.16s | 71,849,418 bytes | 15,615 bytes | -| chakra-ui | 3.37.0 | 27.22s | 207,426,274 bytes | 0 bytes | -| mui | 9.4.0 | 18.78s | 101,473,260 bytes | 0 bytes | -| **devup-ui (per-file CSS)** | 1.0.44 | **14.27s** | 67,424,897 bytes | 562 bytes | -| **devup-ui (single CSS)** | 1.0.44 | **14.17s** | 67,426,959 bytes | 790 bytes | - -Turbopack 값은 실행 순서를 번갈아 측정한 cold build 6회의 중앙값입니다. - -| 라이브러리 | 버전 | 빌드 시간 중앙값 | 빌드 사이즈 | CSS 사이즈 | -| -------------------------------------- | ------ | --------- | ---------------- | ----------- | -| tailwindcss | 4.3.3 | 6.98s | 38,422,141 bytes | 6,197 bytes | -| **devup-ui (direct APIs, single CSS)** | 1.0.44 | **6.93s** | 36,518,952 bytes | 327 bytes | -| **devup-ui (static `.css.ts`)** | 1.0.44 | **6.92s** | 36,578,506 bytes | 268 bytes | - -Turbopack 측정 범위는 서로 겹치므로 direct API 중앙값은 이 fixture에서 Tailwind보다 0.05s(0.7%) 빠르며 사실상 동률입니다. cold 샘플 6개는 Tailwind `7.10s, 6.99s, 6.98s, 6.90s, 6.91s, 6.97s`, direct Devup UI `6.98s, 6.83s, 6.95s, 6.91s, 6.90s, 6.94s`, static `.css.ts` `6.93s, 6.95s, 6.85s, 6.97s, 6.87s, 6.91s`입니다. fixture는 앱 구조만 비슷하고 스타일이 픽셀 단위로 같지는 않습니다. Tailwind는 첫 문단과 버튼에 더 많은 스타일을 주고, Devup UI는 타입이 있는 컴포넌트/스타일 props를 사용합니다. 규칙 단위 마이크로벤치마크가 아니라 빌드 파이프라인 결과로 보세요. 모든 `.css.ts`는 full Boa evaluator로 처리하며, 첫 모듈에 약 20ms, 이후 모듈마다 1ms 미만이 더 듭니다. +Webpack: + +| 라이브러리 | 버전 | 빌드 시간 중앙값 | 범위 | 빌드 사이즈 | CSS 사이즈 | +| --------------------------- | ------ | ---------- | --------------- | ----------------- | ------------ | +| tailwindcss | 4.3.3 | 16.67s | 16.67s - 16.67s | 67,332,642 bytes | 5,852 bytes | +| styleX | 0.19.0 | 38.08s | 38.08s - 38.08s | 96,291,287 bytes | 425 bytes | +| vanilla-extract | 1.21.2 | 16.09s | 16.09s - 16.09s | 68,565,036 bytes | 294 bytes | +| kuma-ui | 1.6.4 | 17.91s | 17.91s - 17.91s | 75,627,783 bytes | 340 bytes | +| panda-css | 1.12.0 | 18.16s | 18.16s - 18.16s | 71,849,418 bytes | 15,615 bytes | +| chakra-ui | 3.37.0 | 27.22s | 27.22s - 27.22s | 207,426,274 bytes | 0 bytes | +| mui | 9.4.0 | 18.78s | 18.78s - 18.78s | 101,473,260 bytes | 0 bytes | +| **devup-ui (per-file CSS)** | 1.0.44 | **14.27s** | 14.27s - 14.27s | 67,424,897 bytes | 562 bytes | +| **devup-ui (single CSS)** | 1.0.44 | **14.17s** | 14.17s - 14.17s | 67,426,959 bytes | 790 bytes | + +Turbopack: + +| 라이브러리 | 버전 | 빌드 시간 중앙값 | 범위 | 빌드 사이즈 | CSS 사이즈 | +| -------------------------------------- | ------ | --------- | ------------- | ---------------- | ----------- | +| tailwindcss | 4.3.3 | 7.04s | 7.04s - 7.04s | 38,422,141 bytes | 6,197 bytes | +| **devup-ui (direct APIs, single CSS)** | 1.0.44 | **6.83s** | 6.83s - 6.83s | 36,518,952 bytes | 327 bytes | +| **devup-ui (static `.css.ts`)** | 1.0.44 | **6.99s** | 6.99s - 6.99s | 36,578,506 bytes | 268 bytes | + +Turbopack 측정 범위는 겹치지 않으므로 direct API 중앙값은 이 fixture에서 Tailwind보다 0.21s(3.0%) 빠르며. cold 샘플 1개는 Tailwind `7.04s`, direct Devup UI `6.83s`, static `.css.ts` `6.99s`입니다. fixture는 앱 구조만 비슷하고 스타일이 픽셀 단위로 같지는 않습니다. Tailwind는 첫 문단과 버튼에 더 많은 스타일을 주고, Devup UI는 타입이 있는 컴포넌트/스타일 props를 사용합니다. 규칙 단위 마이크로벤치마크가 아니라 빌드 파이프라인 결과로 보세요. 모든 `.css.ts`는 full Boa evaluator로 처리하며, 첫 모듈에 약 20ms, 이후 모듈마다 1ms 미만이 더 듭니다. CI는 Devup UI 행을 체크인된 기준값과 비교해 측정된 노이즈를 넘는 회귀에서 실패합니다(규칙은 `benchmark-gate.js`). ## 작동 원리 diff --git a/benchmark-baseline.json b/benchmark-baseline.json new file mode 100644 index 000000000..bfed7c7f7 --- /dev/null +++ b/benchmark-baseline.json @@ -0,0 +1,77 @@ +{ + "samples": 3, + "rows": { + "stylex-turbo-devup-ui": { + "control": "tailwind-turbo", + "ratio": 1.3471, + "calibration": [1.3509, 1.3471, 1.2939, 1.3096, 1.4427], + "timeTolerance": 0.221, + "bytes": 31244290, + "cssBytes": 321 + }, + "devup-ui": { + "control": "tailwind", + "ratio": 0.856, + "calibration": [0.856, 0.8538, 0.9013, 0.8735, 0.8515], + "timeTolerance": 0.1163, + "bytes": 67427563, + "cssBytes": 562 + }, + "devup-ui-single": { + "control": "tailwind", + "ratio": 0.8704, + "calibration": [0.85, 0.8727, 0.9024, 0.8704, 0.859], + "timeTolerance": 0.1204, + "bytes": 67426959, + "cssBytes": 790 + }, + "devup-ui-single-turbo": { + "control": "tailwind-turbo", + "ratio": 0.9794, + "calibration": [0.9702, 0.9794, 0.9701, 1.003, 0.9981], + "timeTolerance": 0.1, + "bytes": 36535115, + "cssBytes": 327 + }, + "devup-ui-turbo": { + "control": "tailwind-turbo", + "ratio": 0.9901, + "calibration": [0.9901, 1, 0.9753, 1.0089, 0.9885], + "timeTolerance": 0.1, + "bytes": 36539386, + "cssBytes": 358 + }, + "vanilla-extract-devup-ui": { + "control": "tailwind-turbo", + "ratio": 0.9882, + "calibration": [0.9929, 0.9882, 0.9428, 0.9867, 1.0439], + "timeTolerance": 0.2046, + "bytes": 36570432, + "cssBytes": 268 + }, + "tailwind-turbo-devup-ui": { + "control": "tailwind-turbo", + "ratio": 0.9986, + "calibration": [0.9986, 1.0059, 0.9532, 0.997, 1.0115], + "timeTolerance": 0.1167, + "bytes": 36572725, + "cssBytes": 530 + }, + "vinext-devup-ui": { + "control": "tailwind", + "ratio": 0.1995, + "calibration": [0.1962, 0.2066, 0.2104, 0.1995, 0.1868], + "timeTolerance": 0.2367, + "bytes": 1198750, + "cssBytes": 29746 + }, + "devup-ui-collapse": { + "control": "tailwind-turbo", + "ratio": 1.0118, + "calibration": [1.0384, 1.0118, 0.9519, 1.0015, 1.021], + "timeTolerance": 0.1709, + "bytes": 31844629, + "cssBytes": 1286 + } + } +} diff --git a/benchmark-check.js b/benchmark-check.js new file mode 100644 index 000000000..6dfa3b61b --- /dev/null +++ b/benchmark-check.js @@ -0,0 +1,63 @@ +// Compares benchmark-run.json (written by `bun benchmark.js`) with the checked +// baseline, and fails on a regression. The rule is in benchmark-gate.js. +// +// bun benchmark-check.js compare, exit 1 on a regression +// bun benchmark-check.js --write-baseline add this run to the calibration and rewrite benchmark-baseline.json +// bun benchmark-check.js --write-results rewrite benchmark-results.json (the README tables) from this run +// +// A change that makes a build slower or larger on purpose runs the two writes +// on a CI run's `benchmark-run` artifact and commits the files in the same pull +// request. +import { readFileSync, writeFileSync } from 'node:fs' + +import { evaluate, nextBaseline, report } from './benchmark-gate.js' + +const read = (file) => JSON.parse(readFileSync(file, 'utf8')) +const write = (file, value) => + writeFileSync(file, JSON.stringify(value, null, 2) + '\n') + +const manifest = read('benchmark-manifest.json') +const run = read('benchmark-run.json') + +if (process.argv.includes('--write-baseline')) { + let previous + try { + previous = read('benchmark-baseline.json') + } catch { + previous = undefined + } + write('benchmark-baseline.json', nextBaseline(run, manifest, previous)) + console.info('benchmark-baseline.json rewritten') +} else if (process.argv.includes('--write-results')) { + const results = read('benchmark-results.json') + const runId = run.run.id ?? results.run.id + results.run = { + ...results.run, + id: runId, + url: `https://github.com/dev-five-git/devup-ui/actions/runs/${runId}`, + commit: (run.run.commit ?? results.run.commit).slice(0, 8), + } + results.samples = run.samples + results.rows = results.rows.map((row) => ({ + ...row, + samples: run.rows[row.target].samples, + median: run.rows[row.target].median, + min: run.rows[row.target].min, + max: run.rows[row.target].max, + bytes: run.rows[row.target].bytes, + cssBytes: run.rows[row.target].cssBytes, + })) + write('benchmark-results.json', results) + console.info('benchmark-results.json rewritten') +} else { + const baseline = read('benchmark-baseline.json') + const evaluation = evaluate(run, baseline) + console.info(report(run, baseline, evaluation)) + if (evaluation.failed) { + console.error( + '\nbenchmark-check: a Devup UI row regressed beyond the measured noise (rule: benchmark-gate.js). If the change is intended, add this run to the baseline with `bun benchmark-check.js --write-baseline` in the same pull request.', + ) + process.exit(1) + } + console.info('\nbenchmark-check: no regression beyond the measured noise') +} diff --git a/benchmark-gate.js b/benchmark-gate.js new file mode 100644 index 000000000..2de619703 --- /dev/null +++ b/benchmark-gate.js @@ -0,0 +1 @@ +// The regression rule behind `benchmark-check.js`, kept free of file and process // access so it can be tested. // // Only the Devup UI rows of benchmark-manifest.json (`gate: true`) can fail; // every other library's row is informational. // // Build time is compared as a ratio to a control row measured in the same run // (Tailwind on the same bundler), because the shared CI runner drifts by about // 25% between runs while the ratio drifts by a few percent. A row fails when // ratio > baseline ratio * (1 + timeTolerance) // where the tolerance is at least 10% and at least twice the spread the ratio // showed over the calibration runs stored with the baseline, so ordinary noise // stays well inside it and a real slowdown does not. // // Output sizes are deterministic to about 0.05% between runs. A row fails when // its bytes grow beyond baseline * 1.01, or its CSS bytes beyond the baseline // plus the larger of 5% and 32 bytes. A change that grows a size on purpose // updates benchmark-baseline.json in the same pull request. export const MIN_TIME_TOLERANCE = 0.1 export const SPREAD_FACTOR = 2 export const BYTES_TOLERANCE = 0.01 export const CSS_TOLERANCE = 0.05 export const CSS_MIN_SLACK = 32 export const KEPT_CALIBRATION_RUNS = 10 export function median(values) { const sorted = values.toSorted((a, b) => a - b) const middle = Math.floor(sorted.length / 2) return sorted.length % 2 ? sorted[middle] : (sorted[middle - 1] + sorted[middle]) / 2 } /** The targets in the order of one round: the list, started ound steps of its length / rounds further on */ export function roundOrder(names, round, rounds) { const offset = Math.floor((names.length * round) / rounds) return [...names.slice(offset), ...names.slice(0, offset)] } /** (max - min) / median */ export function relativeSpread(values) { return (Math.max(...values) - Math.min(...values)) / median(values) } /** The time of a row as a ratio to the control of its group in the same run */ export function timeRatio(run, target, control) { return run.rows[target].median / run.rows[control].median } export function timeTolerance(calibration) { return Math.max( MIN_TIME_TOLERANCE, SPREAD_FACTOR * relativeSpread(calibration), ) } /** Checks the run against the baseline: one entry per gated row, and whether any failed */ export function evaluate(run, baseline) { const results = [] if (run.samples !== baseline.samples) results.push({ target: '(policy)', failures: [ `the run took ${run.samples} samples per row, the baseline was made with ${baseline.samples}`, ], }) for (const [target, expected] of Object.entries(baseline.rows)) { const row = run.rows[target] const failures = [] if (!row || !run.rows[expected.control]) { failures.push('missing from the run') } else { const ratio = timeRatio(run, target, expected.control) const limit = expected.ratio * (1 + expected.timeTolerance) if (ratio > limit) failures.push( `build time is ${ratio.toFixed(3)}x ${expected.control}, over the limit ${limit.toFixed(3)}x (baseline ${expected.ratio.toFixed(3)}x + ${(expected.timeTolerance * 100).toFixed(0)}%)`, ) const bytesLimit = Math.floor(expected.bytes * (1 + BYTES_TOLERANCE)) if (row.bytes > bytesLimit) failures.push( `output is ${row.bytes.toLocaleString()} bytes, over the limit ${bytesLimit.toLocaleString()} (baseline ${expected.bytes.toLocaleString()} + 1%)`, ) const cssLimit = expected.cssBytes + Math.max(Math.floor(expected.cssBytes * CSS_TOLERANCE), CSS_MIN_SLACK) if (row.cssBytes > cssLimit) failures.push( `CSS is ${row.cssBytes.toLocaleString()} bytes, over the limit ${cssLimit.toLocaleString()} (baseline ${expected.cssBytes.toLocaleString()})`, ) } results.push({ target, failures }) } return { results, failed: results.some((entry) => entry.failures.length > 0) } } /** The baseline for this run: the stored calibration runs plus this one, and what they say */ export function nextBaseline(run, manifest, previous) { const gated = manifest.targets.filter((target) => target.gate) const rows = {} for (const { name, group } of gated) { const control = manifest.controls[group] const calibration = [ ...(previous?.rows[name]?.calibration ?? []), timeRatio(run, name, control), ].slice(-KEPT_CALIBRATION_RUNS) rows[name] = { control, ratio: median(calibration), calibration, timeTolerance: timeTolerance(calibration), bytes: run.rows[name].bytes, cssBytes: run.rows[name].cssBytes, } } return { samples: run.samples, rows } } /** One line per gated row for the job log */ export function report(run, baseline, evaluation) { return evaluation.results .map(({ target, failures }) => { const expected = baseline.rows[target] const row = run.rows[target] const status = failures.length ? 'FAIL' : 'ok ' if (!expected || !row) return `${status} ${target}: ${failures.join('; ')}` const ratio = timeRatio(run, target, expected.control) return `${status} ${target}: ${row.median.toFixed(2)}s (${ratio.toFixed(3)}x ${expected.control}, baseline ${expected.ratio.toFixed(3)}x ±${(expected.timeTolerance * 100).toFixed(0)}%), ${row.bytes.toLocaleString()} bytes, css ${row.cssBytes.toLocaleString()} bytes${failures.length ? '\n ' + failures.join('\n ') : ''}` }) .join('\n') } \ No newline at end of file diff --git a/benchmark-manifest.json b/benchmark-manifest.json new file mode 100644 index 000000000..945f84e53 --- /dev/null +++ b/benchmark-manifest.json @@ -0,0 +1,28 @@ +{ + "samples": 3, + "targets": [ + { "name": "tailwind", "group": "webpack" }, + { "name": "stylex", "group": "webpack" }, + { "name": "stylex-turbo", "group": "turbopack" }, + { "name": "stylex-turbo-devup-ui", "group": "turbopack", "gate": true }, + { "name": "vanilla-extract", "group": "webpack" }, + { "name": "kuma-ui", "group": "webpack" }, + { "name": "panda-css", "group": "webpack" }, + { "name": "chakra-ui", "group": "webpack" }, + { "name": "mui", "group": "webpack" }, + { "name": "devup-ui", "group": "webpack", "gate": true }, + { "name": "devup-ui-single", "group": "webpack", "gate": true }, + { "name": "tailwind-turbo", "group": "turbopack" }, + { "name": "devup-ui-single-turbo", "group": "turbopack", "gate": true }, + { "name": "devup-ui-turbo", "group": "turbopack", "gate": true }, + { "name": "vanilla-extract-devup-ui", "group": "turbopack", "gate": true }, + { "name": "tailwind-turbo-devup-ui", "group": "turbopack", "gate": true }, + { "name": "vinext-devup-ui", "group": "vite", "gate": true }, + { "name": "devup-ui-collapse", "group": "turbopack", "gate": true } + ], + "controls": { + "webpack": "tailwind", + "turbopack": "tailwind-turbo", + "vite": "tailwind" + } +} diff --git a/benchmark-results.json b/benchmark-results.json index 5ef26dd7b..0db0976af 100644 --- a/benchmark-results.json +++ b/benchmark-results.json @@ -7,99 +7,155 @@ "next": "16.3.3", "devupUi": "1.0.44" }, - "webpack": [ + "samples": 1, + "rows": [ { + "table": "webpack", + "target": "tailwind", "label": "tailwindcss", "version": "4.3.3", - "seconds": 16.67, + "samples": [16.67], + "median": 16.67, + "min": 16.67, + "max": 16.67, "bytes": 67332642, "cssBytes": 5852 }, { + "table": "webpack", + "target": "stylex", "label": "styleX", "version": "0.19.0", - "seconds": 38.08, + "samples": [38.08], + "median": 38.08, + "min": 38.08, + "max": 38.08, "bytes": 96291287, "cssBytes": 425 }, { + "table": "webpack", + "target": "vanilla-extract", "label": "vanilla-extract", "version": "1.21.2", - "seconds": 16.09, + "samples": [16.09], + "median": 16.09, + "min": 16.09, + "max": 16.09, "bytes": 68565036, "cssBytes": 294 }, { + "table": "webpack", + "target": "kuma-ui", "label": "kuma-ui", "version": "1.6.4", - "seconds": 17.91, + "samples": [17.91], + "median": 17.91, + "min": 17.91, + "max": 17.91, "bytes": 75627783, "cssBytes": 340 }, { + "table": "webpack", + "target": "panda-css", "label": "panda-css", "version": "1.12.0", - "seconds": 18.16, + "samples": [18.16], + "median": 18.16, + "min": 18.16, + "max": 18.16, "bytes": 71849418, "cssBytes": 15615 }, { + "table": "webpack", + "target": "chakra-ui", "label": "chakra-ui", "version": "3.37.0", - "seconds": 27.22, + "samples": [27.22], + "median": 27.22, + "min": 27.22, + "max": 27.22, "bytes": 207426274, "cssBytes": 0 }, { + "table": "webpack", + "target": "mui", "label": "mui", "version": "9.4.0", - "seconds": 18.78, + "samples": [18.78], + "median": 18.78, + "min": 18.78, + "max": 18.78, "bytes": 101473260, "cssBytes": 0 }, { + "table": "webpack", + "target": "devup-ui", "label": "devup-ui (per-file CSS)", "version": "1.0.44", - "seconds": 14.27, + "devup": true, + "samples": [14.27], + "median": 14.27, + "min": 14.27, + "max": 14.27, "bytes": 67424897, - "cssBytes": 562, - "devup": true + "cssBytes": 562 }, { + "table": "webpack", + "target": "devup-ui-single", "label": "devup-ui (single CSS)", "version": "1.0.44", - "seconds": 14.17, + "devup": true, + "samples": [14.17], + "median": 14.17, + "min": 14.17, + "max": 14.17, "bytes": 67426959, - "cssBytes": 790, - "devup": true - } - ], - "turbopack": [ + "cssBytes": 790 + }, { + "table": "turbopack", + "target": "tailwind-turbo", "label": "tailwindcss", "version": "4.3.3", - "samples": [7.1, 6.99, 6.98, 6.9, 6.91, 6.97], - "median": 6.98, + "samples": [7.04], + "median": 7.04, + "min": 7.04, + "max": 7.04, "bytes": 38422141, "cssBytes": 6197 }, { + "table": "turbopack", + "target": "devup-ui-single-turbo", "label": "devup-ui (direct APIs, single CSS)", "version": "1.0.44", - "samples": [6.98, 6.83, 6.95, 6.91, 6.9, 6.94], - "median": 6.93, + "devup": true, + "samples": [6.83], + "median": 6.83, + "min": 6.83, + "max": 6.83, "bytes": 36518952, - "cssBytes": 327, - "devup": true + "cssBytes": 327 }, { + "table": "turbopack", + "target": "vanilla-extract-devup-ui", "label": "devup-ui (static `.css.ts`)", "version": "1.0.44", - "samples": [6.93, 6.95, 6.85, 6.97, 6.87, 6.91], - "median": 6.92, + "devup": true, + "samples": [6.99], + "median": 6.99, + "min": 6.99, + "max": 6.99, "bytes": 36578506, - "cssBytes": 268, - "devup": true + "cssBytes": 268 } ] } diff --git a/benchmark.js b/benchmark.js index 3c6c4b2cb..6c9005dbc 100644 --- a/benchmark.js +++ b/benchmark.js @@ -1,6 +1,18 @@ +// Cold-build benchmark of every fixture in benchmark-manifest.json. +// +// Sampling policy: every target gets the same number of cold samples +// (`samples` in the manifest). Samples run in rounds, and each round starts the +// list at a different place, so no target always runs first (when the runner is +// coolest) or last. A row is reported as the median of its samples with its +// spread (min and max); the median is what gets compared and published. +// +// The machine-readable run goes to benchmark-run.json: `benchmark-check.js` +// compares it with the checked baseline, and the README tables are written from +// the checked results (benchmark-results.json, render-benchmark-readme.js). import { existsSync, readdirSync, + readFileSync, rmSync, statSync, writeFileSync, @@ -9,6 +21,10 @@ import { join } from 'node:path' import { execSync } from 'child_process' +import { median, roundOrder } from './benchmark-gate.js' + +const manifest = JSON.parse(readFileSync('benchmark-manifest.json', 'utf8')) + function clearBuildFile(dir) { const base = join('./benchmark', dir) for (const output of ['.next', 'dist', 'df', 'tsconfig.tsbuildinfo']) { @@ -25,10 +41,9 @@ function checkDirSize(path, filter) { for (const entry of entries) { const entryPath = join(directory, entry) if (statSync(entryPath).isDirectory()) { - calculateSize(entryPath) // 재귀적으로 하위 폴더 크기 계산 + calculateSize(entryPath) } else if (!filter || filter(entryPath)) { - const stats = statSync(entryPath) - totalSize += stats.size // 파일 크기 합산 + totalSize += statSync(entryPath).size } } } @@ -70,94 +85,54 @@ function benchmark(target) { // exists; fall back to `.next` for pure Next.js apps (which never emit dist). const distDir = join(benchmarkDir, 'dist') const outputDir = existsSync(distDir) ? distDir : join(benchmarkDir, '.next') - const duration = performance.getEntriesByName(run)[0].duration / 1000 - const bytes = checkDirSize(outputDir) - const cssBytes = checkCssSize(outputDir) return { - duration, - bytes, - cssBytes, - result: `${target} ${duration.toFixed(2)}s ${bytes.toLocaleString()} bytes (css ${cssBytes.toLocaleString()} bytes)`, + seconds: performance.getEntriesByName(run)[0].duration / 1000, + bytes: checkDirSize(outputDir), + cssBytes: checkCssSize(outputDir), } } -let result = [] -const turboSamples = new Map([ - ['tailwind-turbo', []], - ['devup-ui-single-turbo', []], - ['vanilla-extract-devup-ui', []], -]) - -const measured = [] - -function record(target) { - const sample = benchmark(target) - result.push(sample.result) - measured.push({ - target, - seconds: sample.duration, - bytes: sample.bytes, - cssBytes: sample.cssBytes, - }) -} +const names = manifest.targets.map((target) => target.name) +const samples = new Map( + names.map((name) => [name, { seconds: [], bytes: [], cssBytes: [] }]), +) -record('tailwind') -record('stylex') -record('stylex-turbo') -record('stylex-turbo-devup-ui') -record('vanilla-extract') -record('kuma-ui') -record('panda-css') -record('chakra-ui') -record('mui') -record('devup-ui') -record('devup-ui-single') -record('tailwind-turbo') -record('devup-ui-single-turbo') -record('devup-ui-turbo') -record('vanilla-extract-devup-ui') -record('tailwind-turbo-devup-ui') -record('vinext-devup-ui') -// Multi-component app exercising single-importer collapse (atom dedup). -record('devup-ui-collapse') - -// A single fixed-order result on a shared CI runner is too noisy for direct -// comparisons. Run six cold samples in a repeated Latin-square order so each -// target occupies each position twice, then report their medians. The vanilla -// target contains a static `.css.ts` module; the single target uses Devup APIs. -const turboTargets = [ - 'tailwind-turbo', - 'devup-ui-single-turbo', - 'vanilla-extract-devup-ui', -] -for (let sample = 0; sample < 6; sample++) { - const offset = sample % turboTargets.length - const order = [ - ...turboTargets.slice(offset), - ...turboTargets.slice(0, offset), - ] - for (const target of order) { - turboSamples.get(target).push(benchmark(target).duration) +for (let round = 0; round < manifest.samples; round++) { + for (const name of roundOrder(names, round, manifest.samples)) { + const sample = benchmark(name) + const row = samples.get(name) + row.seconds.push(sample.seconds) + row.bytes.push(sample.bytes) + row.cssBytes.push(sample.cssBytes) } } -function median(samples) { - const sorted = samples.toSorted((a, b) => a - b) - const middle = sorted.length / 2 - return (sorted[middle - 1] + sorted[middle]) / 2 -} - -for (const target of turboTargets) { - const samples = turboSamples.get(target) - result.push( - `${target} median ${median(samples).toFixed(2)}s (${samples.length} cold samples: ${samples.map((sample) => sample.toFixed(2) + 's').join(', ')})`, - ) -} +const rows = Object.fromEntries( + names.map((name) => { + const { seconds, bytes, cssBytes } = samples.get(name) + return [ + name, + { + samples: seconds, + median: median(seconds), + min: Math.min(...seconds), + max: Math.max(...seconds), + bytes: median(bytes), + cssBytes: median(cssBytes), + }, + ] + }), +) -console.info(result.join('\n')) +console.info( + names + .map((name) => { + const row = rows[name] + return `${name} median ${row.median.toFixed(2)}s (${row.samples.length} cold samples: ${row.samples.map((sample) => sample.toFixed(2) + 's').join(', ')}) ${row.bytes.toLocaleString()} bytes (css ${row.cssBytes.toLocaleString()} bytes)` + }) + .join('\n'), +) -// The machine-readable run, the CI artifact the README tables are written from -// (see benchmark-results.json and render-benchmark-readme.js) writeFileSync( 'benchmark-run.json', JSON.stringify( @@ -166,16 +141,8 @@ writeFileSync( id: process.env.GITHUB_RUN_ID ?? null, commit: process.env.GITHUB_SHA ?? null, }, - measured, - turbopack: Object.fromEntries( - turboTargets.map((target) => [ - target, - { - samples: turboSamples.get(target), - median: median(turboSamples.get(target)), - }, - ]), - ), + samples: manifest.samples, + rows, }, null, 2, diff --git a/bindings/devup-ui-wasm/size-budget.json b/bindings/devup-ui-wasm/size-budget.json new file mode 100644 index 000000000..f7488542c --- /dev/null +++ b/bindings/devup-ui-wasm/size-budget.json @@ -0,0 +1,10 @@ +{ + "wasm": { + "raw": 6059008, + "gzip": 2170880 + }, + "glue": { + "raw": 47104, + "gzip": 8192 + } +} diff --git a/libs/extractor/Cargo.toml b/libs/extractor/Cargo.toml index 33565e1c7..9a9ae95e4 100644 --- a/libs/extractor/Cargo.toml +++ b/libs/extractor/Cargo.toml @@ -44,3 +44,7 @@ harness = false [[bench]] name = "alloc_benchmark" harness = false + +[[bench]] +name = "semantic_pass_benchmark" +harness = false diff --git a/libs/extractor/benches/semantic_pass_benchmark.rs b/libs/extractor/benches/semantic_pass_benchmark.rs new file mode 100644 index 000000000..41c7f8964 --- /dev/null +++ b/libs/extractor/benches/semantic_pass_benchmark.rs @@ -0,0 +1,111 @@ +#![allow(clippy::expect_used, clippy::unwrap_used)] + +//! What the `SemanticBuilder` passes cost the extractor. +//! +//! A file that imports `css` or `keyframes` pays one scope-analysis pass before +//! extraction (to read the style results it binds); a file with an error pays a +//! second one to report reads of what the build compiled away. Each group below +//! pairs two inputs that differ only in that pass, and `semantic_floor` is the +//! pass on its own, so the difference between the pair can be read against it. +//! Run: `cargo bench -p extractor --bench semantic_pass_benchmark`. + +use criterion::{Criterion, criterion_group, criterion_main}; +use std::collections::HashMap; +use std::hint::black_box; + +use css::class_map::reset_class_map; +use css::debug::set_debug; +use css::file_map::reset_file_map; +use css::set_prefix; +use extractor::{ExtractOption, extract}; +use oxc_allocator::Allocator; +use oxc_parser::Parser; +use oxc_semantic::SemanticBuilder; +use oxc_span::SourceType; + +fn make_option() -> ExtractOption { + ExtractOption { + package: "@devup-ui/react".to_string(), + css_dir: "@devup-ui/react".to_string(), + single_css: true, + import_main_css: false, + import_aliases: HashMap::new(), + } +} + +fn reset_state() { + reset_class_map(); + reset_file_map(); + set_debug(false); + set_prefix(None); +} + +const BODY: &str = r#" +export const a = + + + + + +"#; + +/// The same file with and without the import that turns the pass on +fn without_pass() -> String { + format!("import {{Box, Flex, Text}} from '@devup-ui/react'{BODY}") +} + +fn with_pass() -> String { + format!("import {{Box, Flex, Text, css}} from '@devup-ui/react'{BODY}") +} + +/// A file that reads what the build compiled away, so the error pass runs, and +/// the file that does not read it +fn with_error_pass() -> String { + format!( + "import {{Box, Flex, Text, css}} from '@devup-ui/react'\nexport const runtime = css{BODY}" + ) +} + +fn bench_extract(c: &mut Criterion, group: &str, name: &str, input: &str) { + c.benchmark_group(group).bench_function(name, |b| { + b.iter(|| { + reset_state(); + let _ = extract(black_box("test.tsx"), black_box(input), make_option()); + }); + }); +} + +/// The floor: parsing alone, and parsing plus the scope analysis the passes run +fn bench_floor(c: &mut Criterion, input: &str) { + let mut floor = c.benchmark_group("semantic_floor"); + floor.bench_function("parse", |b| { + b.iter(|| { + let allocator = Allocator::default(); + let parsed = Parser::new(&allocator, black_box(input), SourceType::tsx()).parse(); + black_box(parsed.program.body.len()) + }); + }); + floor.bench_function("parse_and_semantic", |b| { + b.iter(|| { + let allocator = Allocator::default(); + let parsed = Parser::new(&allocator, black_box(input), SourceType::tsx()).parse(); + let semantic = SemanticBuilder::new().build(&parsed.program).semantic; + black_box(semantic.scoping().symbols_len()) + }); + }); + floor.finish(); +} + +fn criterion_benchmark(c: &mut Criterion) { + let (off, on, error) = (without_pass(), with_pass(), with_error_pass()); + + bench_floor(c, &on); + + bench_extract(c, "style_values_pass", "off", &off); + bench_extract(c, "style_values_pass", "on", &on); + bench_extract(c, "report_compiled_reads_pass", "off", &on); + bench_extract(c, "report_compiled_reads_pass", "on", &error); +} + +criterion_group!(benches, criterion_benchmark); +criterion_main!(benches); diff --git a/packages/plugin-utils/src/__tests__/benchmark-gate.test.ts b/packages/plugin-utils/src/__tests__/benchmark-gate.test.ts new file mode 100644 index 000000000..83fa669ce --- /dev/null +++ b/packages/plugin-utils/src/__tests__/benchmark-gate.test.ts @@ -0,0 +1,265 @@ +import { describe, expect, it } from 'bun:test' + +import { + BYTES_TOLERANCE, + CSS_MIN_SLACK, + evaluate, + KEPT_CALIBRATION_RUNS, + median, + MIN_TIME_TOLERANCE, + nextBaseline, + relativeSpread, + report, + roundOrder, + timeRatio, + timeTolerance, +} from '../../../../benchmark-gate.js' + +interface Row { + samples: number[] + median: number + min: number + max: number + bytes: number + cssBytes: number +} +interface Run { + samples: number + rows: Record +} + +const manifest = { + samples: 3, + targets: [ + { name: 'tailwind', group: 'webpack' }, + { name: 'tailwind-turbo', group: 'turbopack' }, + { name: 'devup-ui', group: 'webpack', gate: true }, + { name: 'devup-ui-turbo', group: 'turbopack', gate: true }, + { name: 'mui', group: 'webpack' }, + ], + controls: { webpack: 'tailwind', turbopack: 'tailwind-turbo' }, +} + +function row(seconds: number, bytes: number, cssBytes: number): Row { + return { + samples: [seconds, seconds, seconds], + median: seconds, + min: seconds, + max: seconds, + bytes, + cssBytes, + } +} + +/** A run of the normal speed: Tailwind takes `control`, Devup UI 0.86 of it */ +function normalRun(control = 16): Run { + return { + samples: 3, + rows: { + tailwind: row(control, 67_000_000, 5_000), + 'tailwind-turbo': row(control / 2, 38_000_000, 6_000), + 'devup-ui': row(control * 0.86, 67_400_000, 562), + 'devup-ui-turbo': row(control / 2, 36_500_000, 327), + mui: row(control * 1.2, 101_000_000, 0), + }, + } +} + +const calibration = (ratios: Record) => + Object.fromEntries( + Object.entries(ratios).map(([name, values]) => [name, values]), + ) + +function baselineOf(run: Run) { + const first = nextBaseline(run, manifest, undefined) + // The ratios of five earlier runs, a few percent apart + const history = calibration({ + 'devup-ui': [0.85, 0.87, 0.9, 0.86, 0.86], + 'devup-ui-turbo': [0.97, 1.0, 0.98, 1.0, 1.0], + }) + for (const [name, values] of Object.entries(history)) { + first.rows[name].calibration = values + first.rows[name].ratio = median(values) + first.rows[name].timeTolerance = timeTolerance(values) + } + return first +} + +describe('sampling helpers', () => { + it('takes the median and the spread of samples', () => { + expect(median([3, 1, 2])).toBe(2) + expect(median([4, 1, 3, 2])).toBe(2.5) + expect(relativeSpread([9, 10, 11])).toBe(0.2) + }) + + it('starts each round at another place of the list, so every target takes other positions', () => { + const names = ['a', 'b', 'c', 'd', 'e', 'f'] + expect(roundOrder(names, 0, 3)).toEqual(['a', 'b', 'c', 'd', 'e', 'f']) + expect(roundOrder(names, 1, 3)).toEqual(['c', 'd', 'e', 'f', 'a', 'b']) + expect(roundOrder(names, 2, 3)).toEqual(['e', 'f', 'a', 'b', 'c', 'd']) + const firsts = [0, 1, 2].map((round) => roundOrder(names, round, 3)[0]) + expect(new Set(firsts).size).toBe(3) + }) + + it('allows at least the minimum tolerance and twice the observed spread', () => { + expect(timeTolerance([1, 1.01, 1])).toBe(MIN_TIME_TOLERANCE) + expect(timeTolerance([1, 1.2, 1])).toBeCloseTo(0.4, 5) + }) + + it('reads a row as a ratio to its control of the same run', () => { + expect(timeRatio(normalRun(), 'devup-ui', 'tailwind')).toBeCloseTo(0.86, 5) + }) +}) + +describe('benchmark gate', () => { + const baseline = baselineOf(normalRun()) + + it('passes a normal run', () => { + const evaluation = evaluate(normalRun(), baseline) + + expect(evaluation.failed).toBe(false) + expect(evaluation.results.map((entry) => entry.target)).toEqual([ + 'devup-ui', + 'devup-ui-turbo', + ]) + }) + + it('passes a run on a slower or faster machine, as every row moves with its control', () => { + expect(evaluate(normalRun(24), baseline).failed).toBe(false) + expect(evaluate(normalRun(11), baseline).failed).toBe(false) + }) + + it('passes ordinary noise between runs', () => { + const run = normalRun() + run.rows['devup-ui'] = row(16 * 0.9, 67_400_000 + 30_000, 562) + run.rows['devup-ui-turbo'] = row(8.2, 36_500_000, 327) + + expect(evaluate(run, baseline).failed).toBe(false) + }) + + it('fails a build that got slower than its control by more than the noise', () => { + const run = normalRun() + // An artificial delay of 3 s on a 13.8 s build + run.rows['devup-ui'] = row(16 * 0.86 + 3, 67_400_000, 562) + + const evaluation = evaluate(run, baseline) + + expect(evaluation.failed).toBe(true) + expect(evaluation.results[0].failures[0]).toMatch( + /build time is 1\.0\d\dx tailwind, over the limit/, + ) + expect(evaluation.results[1].failures).toEqual([]) + }) + + it('does not fail a slower control row, or any row that is not gated', () => { + const run = normalRun() + run.rows.mui = row(60, 500_000_000, 99_999) + + expect(evaluate(run, baseline).failed).toBe(false) + }) + + it('fails an output that grew beyond one percent', () => { + const run = normalRun() + run.rows['devup-ui-turbo'] = row( + 8, + Math.ceil(36_500_000 * (1 + BYTES_TOLERANCE)) + 1, + 327, + ) + + const evaluation = evaluate(run, baseline) + + expect(evaluation.failed).toBe(true) + expect(evaluation.results[1].failures[0]).toMatch( + /output is .* over the limit/, + ) + }) + + it('fails CSS that grew beyond five percent and the slack', () => { + const run = normalRun() + run.rows['devup-ui'] = row(16 * 0.86, 67_400_000, 562 + 28 + CSS_MIN_SLACK) + expect(evaluate(run, baseline).failed).toBe(true) + + run.rows['devup-ui'] = row(16 * 0.86, 67_400_000, 562 + CSS_MIN_SLACK) + expect(evaluate(run, baseline).failed).toBe(false) + }) + + it('fails a gated row missing from the run, and a run with another sampling policy', () => { + const run = normalRun() + delete run.rows['devup-ui'] + expect(evaluate(run, baseline).results[0].failures).toEqual([ + 'missing from the run', + ]) + + const lone = { ...normalRun(), samples: 1 } + const evaluation = evaluate(lone, baseline) + expect(evaluation.failed).toBe(true) + expect(evaluation.results[0].target).toBe('(policy)') + }) + + it('fails a row whose control is missing from the run', () => { + const run = normalRun() + delete run.rows.tailwind + + expect(evaluate(run, baseline).failed).toBe(true) + }) +}) + +describe('benchmark baseline', () => { + it('records the gated rows with their control, ratio, sizes and calibration', () => { + const baseline = nextBaseline(normalRun(), manifest, undefined) + + expect(Object.keys(baseline.rows)).toEqual(['devup-ui', 'devup-ui-turbo']) + expect(baseline.samples).toBe(3) + expect(baseline.rows['devup-ui']).toMatchObject({ + control: 'tailwind', + bytes: 67_400_000, + cssBytes: 562, + timeTolerance: MIN_TIME_TOLERANCE, + }) + expect(baseline.rows['devup-ui'].ratio).toBeCloseTo(0.86, 5) + expect(baseline.rows['devup-ui-turbo'].control).toBe('tailwind-turbo') + }) + + it('adds a run to the calibration, keeps the latest ones and widens the tolerance by the spread', () => { + let baseline = nextBaseline(normalRun(), manifest, undefined) + for (let index = 0; index < KEPT_CALIBRATION_RUNS + 2; index++) + baseline = nextBaseline(normalRun(16 + (index % 2)), manifest, baseline) + expect(baseline.rows['devup-ui'].calibration).toHaveLength( + KEPT_CALIBRATION_RUNS, + ) + + const noisy = normalRun() + noisy.rows['devup-ui'] = row(16 * 0.86 * 1.4, 67_400_000, 562) + const widened = nextBaseline(noisy, manifest, baseline) + expect(widened.rows['devup-ui'].timeTolerance).toBeGreaterThan( + MIN_TIME_TOLERANCE, + ) + }) +}) + +describe('benchmark report', () => { + it('prints a line per gated row and what failed', () => { + const baseline = baselineOf(normalRun()) + const run = normalRun() + run.rows['devup-ui'] = row(16 * 0.86 + 3, 67_400_000, 562) + const text = report(run, baseline, evaluate(run, baseline)) + + expect(text).toMatch(/^FAIL devup-ui: 16\.76s/) + expect(text).toContain('over the limit') + expect(text).toMatch(/ok {3}devup-ui-turbo: 8\.00s/) + }) + + it('prints a failure without a row to describe', () => { + const baseline = baselineOf(normalRun()) + const run = normalRun() + delete run.rows['devup-ui'] + const text = report(run, baseline, evaluate(run, baseline)) + + expect(text).toContain('FAIL devup-ui: missing from the run') + + const lone = { ...normalRun(), samples: 1 } + expect(report(lone, baseline, evaluate(lone, baseline))).toContain( + 'FAIL (policy): the run took 1 samples per row', + ) + }) +}) diff --git a/render-benchmark-readme.js b/render-benchmark-readme.js index 37418e76d..b45a0a288 100644 --- a/render-benchmark-readme.js +++ b/render-benchmark-readme.js @@ -1,5 +1,7 @@ // Writes the benchmark section of README.md and README_ko.md from one checked // result file, so both languages publish the same run and the same numbers. +// Every row is the median of the same number of cold samples, shown with its +// range. `benchmark-check.js --write-results` refreshes the file from a run. // // bun render-benchmark-readme.js rewrite the sections // bun render-benchmark-readme.js --check fail when a README differs from them @@ -9,80 +11,73 @@ const START = '' const END = '' const results = JSON.parse(readFileSync('benchmark-results.json', 'utf8')) +const rowsOf = (table) => results.rows.filter((row) => row.table === table) const bytes = (value) => `${value.toLocaleString('en-US')} bytes` const seconds = (value) => `${value.toFixed(2)}s` +const range = (row) => `${seconds(row.min)} - ${seconds(row.max)}` const list = (samples) => samples.map(seconds).join(', ') const TEXT = { en: { file: 'README.md', - intro: ({ run }) => - `[Latest CI benchmark](${run.url}) (commit \`${run.commit}\`, Devup UI ${run.devupUi}) on \`${run.runner}\` with Next.js ${run.next}. All Next.js builds use the native TypeScript 7 CLI for type checking. The English and Korean tables are generated from the same checked result file, \`benchmark-results.json\`.`, + intro: ({ run, samples }) => + `[Latest CI benchmark](${run.url}) (commit \`${run.commit}\`, Devup UI ${run.devupUi}) on \`${run.runner}\` with Next.js ${run.next}. Every row is the median of ${samples} cold builds that ran in rotating order, shown with the range of the samples. All Next.js builds use the native TypeScript 7 CLI for type checking. The English and Korean tables are generated from the same checked result file, \`benchmark-results.json\`.`, sizes: 'Build Size is every byte of the build output directory (`.next`, or `dist` for vinext); CSS Size is the emitted `.css` files alone, measured separately.', - webpack: 'Webpack values are one cold build:', - turbopack: - 'Turbopack values are medians of six cold builds in alternating order:', + webpack: 'Webpack:', + turbopack: 'Turbopack:', columns: { library: 'Library', version: 'Version', - time: 'Build Time', - median: 'Median Build Time', + time: 'Median Build Time', + range: 'Range', size: 'Build Size', css: 'CSS Size', }, - note: ({ turbopack }) => { - const [tailwind, direct, stylesheet] = turbopack - const range = (row) => [ - Math.min(...row.samples), - Math.max(...row.samples), - ] + note: ({ samples }, [tailwind, direct, stylesheet]) => { + const spread = (row) => [row.min, row.max] const overlap = - range(tailwind)[0] <= range(direct)[1] && - range(direct)[0] <= range(tailwind)[1] + spread(tailwind)[0] <= spread(direct)[1] && + spread(direct)[0] <= spread(tailwind)[1] const gap = tailwind.median - direct.median - return `${overlap ? 'The Turbopack ranges overlap' : 'The Turbopack ranges do not overlap'}, so the direct-API median is ${Math.abs(gap).toFixed(2)}s (${Math.abs((gap / tailwind.median) * 100).toFixed(1)}%) ${gap >= 0 ? 'ahead of' : 'behind'} Tailwind on this fixture${overlap ? ' and the two are effectively at parity' : ''}. The six cold samples were Tailwind \`${list(tailwind.samples)}\`, direct Devup UI \`${list(direct.samples)}\`, and static \`.css.ts\` \`${list(stylesheet.samples)}\`. The fixtures have comparable app shapes, not pixel-identical styling: Tailwind styles the leading paragraph and button more heavily, while Devup UI exercises typed component/style props. Treat these as build-pipeline results rather than a per-rule microbenchmark. Every \`.css.ts\` module runs on the full Boa evaluator, which adds about 20 ms for the first module and under 1 ms for each further one.` + return `${overlap ? 'The Turbopack ranges overlap' : 'The Turbopack ranges do not overlap'}, so the direct-API median is ${Math.abs(gap).toFixed(2)}s (${Math.abs((gap / tailwind.median) * 100).toFixed(1)}%) ${gap >= 0 ? 'ahead of' : 'behind'} Tailwind on this fixture${overlap ? ' and the two are effectively at parity' : ''}. The ${samples} cold samples were Tailwind \`${list(tailwind.samples)}\`, direct Devup UI \`${list(direct.samples)}\`, and static \`.css.ts\` \`${list(stylesheet.samples)}\`. The fixtures have comparable app shapes, not pixel-identical styling: Tailwind styles the leading paragraph and button more heavily, while Devup UI exercises typed component/style props. Treat these as build-pipeline results rather than a per-rule microbenchmark. Every \`.css.ts\` module runs on the full Boa evaluator, which adds about 20 ms for the first module and under 1 ms for each further one. CI compares the Devup UI rows with a checked baseline (see \`benchmark-gate.js\` for the rule) and fails on a regression beyond the measured noise.` }, }, ko: { file: 'README_ko.md', - intro: ({ run }) => - `[최신 CI 벤치마크](${run.url}) (커밋 \`${run.commit}\`, Devup UI ${run.devupUi})는 \`${run.runner}\`에서 Next.js ${run.next}로 실행했습니다. 모든 Next.js 빌드는 네이티브 TypeScript 7 CLI로 타입 검사를 수행합니다. 영어와 한국어 표는 같은 결과 파일 \`benchmark-results.json\`에서 생성합니다.`, + intro: ({ run, samples }) => + `[최신 CI 벤치마크](${run.url}) (커밋 \`${run.commit}\`, Devup UI ${run.devupUi})는 \`${run.runner}\`에서 Next.js ${run.next}로 실행했습니다. 모든 행은 순서를 돌려 가며 측정한 cold build ${samples}회의 중앙값이며 샘플의 범위를 함께 보여줍니다. 모든 Next.js 빌드는 네이티브 TypeScript 7 CLI로 타입 검사를 수행합니다. 영어와 한국어 표는 같은 결과 파일 \`benchmark-results.json\`에서 생성합니다.`, sizes: '빌드 사이즈는 빌드 출력 디렉터리(`.next`, vinext는 `dist`)의 모든 바이트이고, CSS 사이즈는 출력된 `.css` 파일만 따로 잰 값입니다.', - webpack: 'Webpack 값은 cold build 1회 결과입니다.', - turbopack: - 'Turbopack 값은 실행 순서를 번갈아 측정한 cold build 6회의 중앙값입니다.', + webpack: 'Webpack:', + turbopack: 'Turbopack:', columns: { library: '라이브러리', version: '버전', - time: '빌드 시간', - median: '빌드 시간 중앙값', + time: '빌드 시간 중앙값', + range: '범위', size: '빌드 사이즈', css: 'CSS 사이즈', }, - note: ({ turbopack }) => { - const [tailwind, direct, stylesheet] = turbopack - const range = (row) => [ - Math.min(...row.samples), - Math.max(...row.samples), - ] + note: ({ samples }, [tailwind, direct, stylesheet]) => { + const spread = (row) => [row.min, row.max] const overlap = - range(tailwind)[0] <= range(direct)[1] && - range(direct)[0] <= range(tailwind)[1] + spread(tailwind)[0] <= spread(direct)[1] && + spread(direct)[0] <= spread(tailwind)[1] const gap = tailwind.median - direct.median - return `Turbopack 측정 범위는 ${overlap ? '서로 겹치므로' : '겹치지 않으므로'} direct API 중앙값은 이 fixture에서 Tailwind보다 ${Math.abs(gap).toFixed(2)}s(${Math.abs((gap / tailwind.median) * 100).toFixed(1)}%) ${gap >= 0 ? '빠르며' : '느리며'}${overlap ? ' 사실상 동률입니다' : ''}. cold 샘플 6개는 Tailwind \`${list(tailwind.samples)}\`, direct Devup UI \`${list(direct.samples)}\`, static \`.css.ts\` \`${list(stylesheet.samples)}\`입니다. fixture는 앱 구조만 비슷하고 스타일이 픽셀 단위로 같지는 않습니다. Tailwind는 첫 문단과 버튼에 더 많은 스타일을 주고, Devup UI는 타입이 있는 컴포넌트/스타일 props를 사용합니다. 규칙 단위 마이크로벤치마크가 아니라 빌드 파이프라인 결과로 보세요. 모든 \`.css.ts\`는 full Boa evaluator로 처리하며, 첫 모듈에 약 20ms, 이후 모듈마다 1ms 미만이 더 듭니다.` + return `Turbopack 측정 범위는 ${overlap ? '서로 겹치므로' : '겹치지 않으므로'} direct API 중앙값은 이 fixture에서 Tailwind보다 ${Math.abs(gap).toFixed(2)}s(${Math.abs((gap / tailwind.median) * 100).toFixed(1)}%) ${gap >= 0 ? '빠르며' : '느리며'}${overlap ? ' 사실상 동률입니다' : ''}. cold 샘플 ${samples}개는 Tailwind \`${list(tailwind.samples)}\`, direct Devup UI \`${list(direct.samples)}\`, static \`.css.ts\` \`${list(stylesheet.samples)}\`입니다. fixture는 앱 구조만 비슷하고 스타일이 픽셀 단위로 같지는 않습니다. Tailwind는 첫 문단과 버튼에 더 많은 스타일을 주고, Devup UI는 타입이 있는 컴포넌트/스타일 props를 사용합니다. 규칙 단위 마이크로벤치마크가 아니라 빌드 파이프라인 결과로 보세요. 모든 \`.css.ts\`는 full Boa evaluator로 처리하며, 첫 모듈에 약 20ms, 이후 모듈마다 1ms 미만이 더 듭니다. CI는 Devup UI 행을 체크인된 기준값과 비교해 측정된 노이즈를 넘는 회귀에서 실패합니다(규칙은 \`benchmark-gate.js\`).` }, }, } -function table(text, rows, timeColumn, timeOf) { +function table(text, rows) { const columns = text.columns const header = [ columns.library, columns.version, - timeColumn, + columns.time, + columns.range, columns.size, columns.css, ] @@ -91,7 +86,8 @@ function table(text, rows, timeColumn, timeOf) { return [ bold(row.label), row.version, - bold(seconds(timeOf(row))), + bold(seconds(row.median)), + range(row), bytes(row.bytes), bytes(row.cssBytes), ] @@ -117,13 +113,13 @@ function section(text) { '', text.webpack, '', - table(text, results.webpack, text.columns.time, (row) => row.seconds), + table(text, rowsOf('webpack')), '', text.turbopack, '', - table(text, results.turbopack, text.columns.median, (row) => row.median), + table(text, rowsOf('turbopack')), '', - text.note(results), + text.note(results, rowsOf('turbopack')), END, ].join('\n') } diff --git a/verify-wasm-size.js b/verify-wasm-size.js new file mode 100644 index 000000000..ec06a4e67 --- /dev/null +++ b/verify-wasm-size.js @@ -0,0 +1,71 @@ +// Tracks the size of the published WASM package and fails when it grows beyond +// the budget checked into bindings/devup-ui-wasm/size-budget.json. +// +// bun verify-wasm-size.js measure pkg/, write wasm-size.json, fail above the budget +// bun verify-wasm-size.js --write set the budget to the measurement plus 1%, rounded up to 1 KiB +// bun verify-wasm-size.js --write --from F the same from a measurement file F (a CI run's `wasm-size` artifact) +// +// The budget is a ceiling, not a target: a pull request that grows the binary +// beyond it raises the budget in the same pull request, where a reviewer sees +// the size change next to the code that caused it. +import { readFileSync, writeFileSync } from 'node:fs' +import { join } from 'node:path' +import { gzipSync } from 'node:zlib' + +const PACKAGE = 'bindings/devup-ui-wasm' +const BUDGET = join(PACKAGE, 'size-budget.json') +const HEADROOM = 1.01 + +const read = (file) => JSON.parse(readFileSync(file, 'utf8')) + +function measure() { + const sizes = {} + for (const [name, file] of [ + ['wasm', 'index_bg.wasm'], + ['glue', 'index.js'], + ]) { + const content = readFileSync(join(PACKAGE, 'pkg', file)) + sizes[name] = { + raw: content.length, + gzip: gzipSync(content, { level: 9 }).length, + } + } + return sizes +} + +const roundUp = (value) => Math.ceil((value * HEADROOM) / 1024) * 1024 + +const from = process.argv.indexOf('--from') +const measured = from === -1 ? measure() : read(process.argv[from + 1]) + +if (process.argv.includes('--write')) { + const budget = {} + for (const [name, size] of Object.entries(measured)) + budget[name] = { raw: roundUp(size.raw), gzip: roundUp(size.gzip) } + writeFileSync(BUDGET, JSON.stringify(budget, null, 2) + '\n') + console.info(`${BUDGET} rewritten`) +} else { + writeFileSync('wasm-size.json', JSON.stringify(measured, null, 2) + '\n') + const budget = read(BUDGET) + const failures = [] + for (const [name, size] of Object.entries(measured)) { + for (const kind of ['raw', 'gzip']) { + const limit = budget[name][kind] + const share = ((size[kind] / limit) * 100).toFixed(1) + console.info( + `${name} ${kind}: ${size[kind].toLocaleString()} bytes of the ${limit.toLocaleString()} budget (${share}%)`, + ) + if (size[kind] > limit) + failures.push( + `${name} ${kind} is ${size[kind].toLocaleString()} bytes, ${(size[kind] - limit).toLocaleString()} over the budget of ${limit.toLocaleString()}`, + ) + } + } + if (failures.length > 0) { + console.error( + `verify-wasm-size: the package grew beyond its budget:\n - ${failures.join('\n - ')}\nIf the growth is intended, raise the budget with \`bun verify-wasm-size.js --write\` in the same pull request.`, + ) + process.exit(1) + } + console.info('verify-wasm-size: within the budget') +} From 56a37f1d0f24ad9c562c2796cf7b5b937552fd92 Mon Sep 17 00:00:00 2001 From: owjs3901 Date: Sat, 3 Oct 2026 23:02:01 +0900 Subject: [PATCH 2/9] ci: calibrate the benchmark baseline, results and WASM budget from the first CI run of the new policy Refs #683 Ultraworked with [Sisyphus](https://github.com/code-yeongyu/oh-my-openagent) Co-authored-by: Sisyphus --- README.md | 28 +++--- README_ko.md | 28 +++--- benchmark-baseline.json | 80 +++++++++------ benchmark-results.json | 128 ++++++++++++------------ bindings/devup-ui-wasm/size-budget.json | 8 +- 5 files changed, 143 insertions(+), 129 deletions(-) diff --git a/README.md b/README.md index 8057e46c4..bc9602e35 100644 --- a/README.md +++ b/README.md @@ -80,7 +80,7 @@ npm install @devup-ui/webpack-plugin ## Comparison Benchmarks -[Latest CI benchmark](https://github.com/dev-five-git/devup-ui/actions/runs/36702867475) (commit `8d6c2f62`, Devup UI 1.0.44) on `ubuntu-24.04` with Next.js 16.3.3. Every row is the median of 1 cold builds that ran in rotating order, shown with the range of the samples. All Next.js builds use the native TypeScript 7 CLI for type checking. The English and Korean tables are generated from the same checked result file, `benchmark-results.json`. +[Latest CI benchmark](https://github.com/dev-five-git/devup-ui/actions/runs/37127187436) (commit `49b33534`, Devup UI 1.0.44) on `ubuntu-24.04` with Next.js 16.3.3. Every row is the median of 3 cold builds that ran in rotating order, shown with the range of the samples. All Next.js builds use the native TypeScript 7 CLI for type checking. The English and Korean tables are generated from the same checked result file, `benchmark-results.json`. Build Size is every byte of the build output directory (`.next`, or `dist` for vinext); CSS Size is the emitted `.css` files alone, measured separately. @@ -88,25 +88,25 @@ Webpack: | Library | Version | Median Build Time | Range | Build Size | CSS Size | | --------------------------- | ------- | ----------------- | --------------- | ----------------- | ------------ | -| tailwindcss | 4.3.3 | 16.67s | 16.67s - 16.67s | 67,332,642 bytes | 5,852 bytes | -| styleX | 0.19.0 | 38.08s | 38.08s - 38.08s | 96,291,287 bytes | 425 bytes | -| vanilla-extract | 1.21.2 | 16.09s | 16.09s - 16.09s | 68,565,036 bytes | 294 bytes | -| kuma-ui | 1.6.4 | 17.91s | 17.91s - 17.91s | 75,627,783 bytes | 340 bytes | -| panda-css | 1.12.0 | 18.16s | 18.16s - 18.16s | 71,849,418 bytes | 15,615 bytes | -| chakra-ui | 3.37.0 | 27.22s | 27.22s - 27.22s | 207,426,274 bytes | 0 bytes | -| mui | 9.4.0 | 18.78s | 18.78s - 18.78s | 101,473,260 bytes | 0 bytes | -| **devup-ui (per-file CSS)** | 1.0.44 | **14.27s** | 14.27s - 14.27s | 67,424,897 bytes | 562 bytes | -| **devup-ui (single CSS)** | 1.0.44 | **14.17s** | 14.17s - 14.17s | 67,426,959 bytes | 790 bytes | +| tailwindcss | 4.3.3 | 12.57s | 12.53s - 13.07s | 67,337,100 bytes | 5,852 bytes | +| styleX | 0.19.0 | 28.43s | 28.39s - 28.90s | 96,290,977 bytes | 425 bytes | +| vanilla-extract | 1.21.2 | 12.18s | 12.10s - 12.49s | 68,577,913 bytes | 294 bytes | +| kuma-ui | 1.6.4 | 13.50s | 13.23s - 14.56s | 75,627,639 bytes | 340 bytes | +| panda-css | 1.12.0 | 13.61s | 13.44s - 14.35s | 71,844,396 bytes | 15,615 bytes | +| chakra-ui | 3.37.0 | 20.19s | 20.16s - 20.75s | 207,420,203 bytes | 0 bytes | +| mui | 9.4.0 | 14.18s | 14.03s - 14.44s | 101,478,873 bytes | 0 bytes | +| **devup-ui (per-file CSS)** | 1.0.44 | **10.97s** | 10.83s - 11.25s | 67,438,054 bytes | 562 bytes | +| **devup-ui (single CSS)** | 1.0.44 | **10.98s** | 10.90s - 11.03s | 67,415,063 bytes | 790 bytes | Turbopack: | Library | Version | Median Build Time | Range | Build Size | CSS Size | | -------------------------------------- | ------- | ----------------- | ------------- | ---------------- | ----------- | -| tailwindcss | 4.3.3 | 7.04s | 7.04s - 7.04s | 38,422,141 bytes | 6,197 bytes | -| **devup-ui (direct APIs, single CSS)** | 1.0.44 | **6.83s** | 6.83s - 6.83s | 36,518,952 bytes | 327 bytes | -| **devup-ui (static `.css.ts`)** | 1.0.44 | **6.99s** | 6.99s - 6.99s | 36,578,506 bytes | 268 bytes | +| tailwindcss | 4.3.3 | 5.83s | 5.55s - 5.90s | 38,422,653 bytes | 6,197 bytes | +| **devup-ui (direct APIs, single CSS)** | 1.0.44 | **5.66s** | 5.58s - 5.68s | 36,545,972 bytes | 327 bytes | +| **devup-ui (static `.css.ts`)** | 1.0.44 | **5.52s** | 5.51s - 5.65s | 36,571,452 bytes | 268 bytes | -The Turbopack ranges do not overlap, so the direct-API median is 0.21s (3.0%) ahead of Tailwind on this fixture. The 1 cold samples were Tailwind `7.04s`, direct Devup UI `6.83s`, and static `.css.ts` `6.99s`. The fixtures have comparable app shapes, not pixel-identical styling: Tailwind styles the leading paragraph and button more heavily, while Devup UI exercises typed component/style props. Treat these as build-pipeline results rather than a per-rule microbenchmark. Every `.css.ts` module runs on the full Boa evaluator, which adds about 20 ms for the first module and under 1 ms for each further one. CI compares the Devup UI rows with a checked baseline (see `benchmark-gate.js` for the rule) and fails on a regression beyond the measured noise. +The Turbopack ranges overlap, so the direct-API median is 0.17s (2.9%) ahead of Tailwind on this fixture and the two are effectively at parity. The 3 cold samples were Tailwind `5.90s, 5.55s, 5.83s`, direct Devup UI `5.66s, 5.58s, 5.68s`, and static `.css.ts` `5.65s, 5.52s, 5.51s`. The fixtures have comparable app shapes, not pixel-identical styling: Tailwind styles the leading paragraph and button more heavily, while Devup UI exercises typed component/style props. Treat these as build-pipeline results rather than a per-rule microbenchmark. Every `.css.ts` module runs on the full Boa evaluator, which adds about 20 ms for the first module and under 1 ms for each further one. CI compares the Devup UI rows with a checked baseline (see `benchmark-gate.js` for the rule) and fails on a regression beyond the measured noise. ## How it works diff --git a/README_ko.md b/README_ko.md index 1a6524f9f..81ffa9e99 100644 --- a/README_ko.md +++ b/README_ko.md @@ -80,7 +80,7 @@ npm install @devup-ui/webpack-plugin ## 비교 벤치마크 -[최신 CI 벤치마크](https://github.com/dev-five-git/devup-ui/actions/runs/36702867475) (커밋 `8d6c2f62`, Devup UI 1.0.44)는 `ubuntu-24.04`에서 Next.js 16.3.3로 실행했습니다. 모든 행은 순서를 돌려 가며 측정한 cold build 1회의 중앙값이며 샘플의 범위를 함께 보여줍니다. 모든 Next.js 빌드는 네이티브 TypeScript 7 CLI로 타입 검사를 수행합니다. 영어와 한국어 표는 같은 결과 파일 `benchmark-results.json`에서 생성합니다. +[최신 CI 벤치마크](https://github.com/dev-five-git/devup-ui/actions/runs/37127187436) (커밋 `49b33534`, Devup UI 1.0.44)는 `ubuntu-24.04`에서 Next.js 16.3.3로 실행했습니다. 모든 행은 순서를 돌려 가며 측정한 cold build 3회의 중앙값이며 샘플의 범위를 함께 보여줍니다. 모든 Next.js 빌드는 네이티브 TypeScript 7 CLI로 타입 검사를 수행합니다. 영어와 한국어 표는 같은 결과 파일 `benchmark-results.json`에서 생성합니다. 빌드 사이즈는 빌드 출력 디렉터리(`.next`, vinext는 `dist`)의 모든 바이트이고, CSS 사이즈는 출력된 `.css` 파일만 따로 잰 값입니다. @@ -88,25 +88,25 @@ Webpack: | 라이브러리 | 버전 | 빌드 시간 중앙값 | 범위 | 빌드 사이즈 | CSS 사이즈 | | --------------------------- | ------ | ---------- | --------------- | ----------------- | ------------ | -| tailwindcss | 4.3.3 | 16.67s | 16.67s - 16.67s | 67,332,642 bytes | 5,852 bytes | -| styleX | 0.19.0 | 38.08s | 38.08s - 38.08s | 96,291,287 bytes | 425 bytes | -| vanilla-extract | 1.21.2 | 16.09s | 16.09s - 16.09s | 68,565,036 bytes | 294 bytes | -| kuma-ui | 1.6.4 | 17.91s | 17.91s - 17.91s | 75,627,783 bytes | 340 bytes | -| panda-css | 1.12.0 | 18.16s | 18.16s - 18.16s | 71,849,418 bytes | 15,615 bytes | -| chakra-ui | 3.37.0 | 27.22s | 27.22s - 27.22s | 207,426,274 bytes | 0 bytes | -| mui | 9.4.0 | 18.78s | 18.78s - 18.78s | 101,473,260 bytes | 0 bytes | -| **devup-ui (per-file CSS)** | 1.0.44 | **14.27s** | 14.27s - 14.27s | 67,424,897 bytes | 562 bytes | -| **devup-ui (single CSS)** | 1.0.44 | **14.17s** | 14.17s - 14.17s | 67,426,959 bytes | 790 bytes | +| tailwindcss | 4.3.3 | 12.57s | 12.53s - 13.07s | 67,337,100 bytes | 5,852 bytes | +| styleX | 0.19.0 | 28.43s | 28.39s - 28.90s | 96,290,977 bytes | 425 bytes | +| vanilla-extract | 1.21.2 | 12.18s | 12.10s - 12.49s | 68,577,913 bytes | 294 bytes | +| kuma-ui | 1.6.4 | 13.50s | 13.23s - 14.56s | 75,627,639 bytes | 340 bytes | +| panda-css | 1.12.0 | 13.61s | 13.44s - 14.35s | 71,844,396 bytes | 15,615 bytes | +| chakra-ui | 3.37.0 | 20.19s | 20.16s - 20.75s | 207,420,203 bytes | 0 bytes | +| mui | 9.4.0 | 14.18s | 14.03s - 14.44s | 101,478,873 bytes | 0 bytes | +| **devup-ui (per-file CSS)** | 1.0.44 | **10.97s** | 10.83s - 11.25s | 67,438,054 bytes | 562 bytes | +| **devup-ui (single CSS)** | 1.0.44 | **10.98s** | 10.90s - 11.03s | 67,415,063 bytes | 790 bytes | Turbopack: | 라이브러리 | 버전 | 빌드 시간 중앙값 | 범위 | 빌드 사이즈 | CSS 사이즈 | | -------------------------------------- | ------ | --------- | ------------- | ---------------- | ----------- | -| tailwindcss | 4.3.3 | 7.04s | 7.04s - 7.04s | 38,422,141 bytes | 6,197 bytes | -| **devup-ui (direct APIs, single CSS)** | 1.0.44 | **6.83s** | 6.83s - 6.83s | 36,518,952 bytes | 327 bytes | -| **devup-ui (static `.css.ts`)** | 1.0.44 | **6.99s** | 6.99s - 6.99s | 36,578,506 bytes | 268 bytes | +| tailwindcss | 4.3.3 | 5.83s | 5.55s - 5.90s | 38,422,653 bytes | 6,197 bytes | +| **devup-ui (direct APIs, single CSS)** | 1.0.44 | **5.66s** | 5.58s - 5.68s | 36,545,972 bytes | 327 bytes | +| **devup-ui (static `.css.ts`)** | 1.0.44 | **5.52s** | 5.51s - 5.65s | 36,571,452 bytes | 268 bytes | -Turbopack 측정 범위는 겹치지 않으므로 direct API 중앙값은 이 fixture에서 Tailwind보다 0.21s(3.0%) 빠르며. cold 샘플 1개는 Tailwind `7.04s`, direct Devup UI `6.83s`, static `.css.ts` `6.99s`입니다. fixture는 앱 구조만 비슷하고 스타일이 픽셀 단위로 같지는 않습니다. Tailwind는 첫 문단과 버튼에 더 많은 스타일을 주고, Devup UI는 타입이 있는 컴포넌트/스타일 props를 사용합니다. 규칙 단위 마이크로벤치마크가 아니라 빌드 파이프라인 결과로 보세요. 모든 `.css.ts`는 full Boa evaluator로 처리하며, 첫 모듈에 약 20ms, 이후 모듈마다 1ms 미만이 더 듭니다. CI는 Devup UI 행을 체크인된 기준값과 비교해 측정된 노이즈를 넘는 회귀에서 실패합니다(규칙은 `benchmark-gate.js`). +Turbopack 측정 범위는 서로 겹치므로 direct API 중앙값은 이 fixture에서 Tailwind보다 0.17s(2.9%) 빠르며 사실상 동률입니다. cold 샘플 3개는 Tailwind `5.90s, 5.55s, 5.83s`, direct Devup UI `5.66s, 5.58s, 5.68s`, static `.css.ts` `5.65s, 5.52s, 5.51s`입니다. fixture는 앱 구조만 비슷하고 스타일이 픽셀 단위로 같지는 않습니다. Tailwind는 첫 문단과 버튼에 더 많은 스타일을 주고, Devup UI는 타입이 있는 컴포넌트/스타일 props를 사용합니다. 규칙 단위 마이크로벤치마크가 아니라 빌드 파이프라인 결과로 보세요. 모든 `.css.ts`는 full Boa evaluator로 처리하며, 첫 모듈에 약 20ms, 이후 모듈마다 1ms 미만이 더 듭니다. CI는 Devup UI 행을 체크인된 기준값과 비교해 측정된 노이즈를 넘는 회귀에서 실패합니다(규칙은 `benchmark-gate.js`). ## 작동 원리 diff --git a/benchmark-baseline.json b/benchmark-baseline.json index bfed7c7f7..c28493adf 100644 --- a/benchmark-baseline.json +++ b/benchmark-baseline.json @@ -3,74 +3,88 @@ "rows": { "stylex-turbo-devup-ui": { "control": "tailwind-turbo", - "ratio": 1.3471, - "calibration": [1.3509, 1.3471, 1.2939, 1.3096, 1.4427], - "timeTolerance": 0.221, - "bytes": 31244290, + "ratio": 1.349, + "calibration": [ + 1.3509, 1.3471, 1.2939, 1.3096, 1.4427, 1.417800722121156 + ], + "timeTolerance": 0.22060785767234994, + "bytes": 31227953, "cssBytes": 321 }, "devup-ui": { "control": "tailwind", - "ratio": 0.856, - "calibration": [0.856, 0.8538, 0.9013, 0.8735, 0.8515], - "timeTolerance": 0.1163, - "bytes": 67427563, + "ratio": 0.8645789338279827, + "calibration": [ + 0.856, 0.8538, 0.9013, 0.8735, 0.8515, 0.8731578676559654 + ], + "timeTolerance": 0.1152005862079175, + "bytes": 67438054, "cssBytes": 562 }, "devup-ui-single": { "control": "tailwind", - "ratio": 0.8704, - "calibration": [0.85, 0.8727, 0.9024, 0.8704, 0.859], - "timeTolerance": 0.1204, - "bytes": 67426959, + "ratio": 0.87155, + "calibration": [0.85, 0.8727, 0.9024, 0.8704, 0.859, 0.874014974821355], + "timeTolerance": 0.12024553955596351, + "bytes": 67415063, "cssBytes": 790 }, "devup-ui-single-turbo": { "control": "tailwind-turbo", - "ratio": 0.9794, - "calibration": [0.9702, 0.9794, 0.9701, 1.003, 0.9981], + "ratio": 0.9753609130605841, + "calibration": [ + 0.9702, 0.9794, 0.9701, 1.003, 0.9981, 0.9713218261211682 + ], "timeTolerance": 0.1, - "bytes": 36535115, + "bytes": 36545972, "cssBytes": 327 }, "devup-ui-turbo": { "control": "tailwind-turbo", - "ratio": 0.9901, - "calibration": [0.9901, 1, 0.9753, 1.0089, 0.9885], + "ratio": 0.9893000000000001, + "calibration": [0.9901, 1, 0.9753, 1.0089, 0.9885, 0.9761607019540564], "timeTolerance": 0.1, - "bytes": 36539386, + "bytes": 36539932, "cssBytes": 358 }, "vanilla-extract-devup-ui": { "control": "tailwind-turbo", - "ratio": 0.9882, - "calibration": [0.9929, 0.9882, 0.9428, 0.9867, 1.0439], - "timeTolerance": 0.2046, - "bytes": 36570432, + "ratio": 0.9874499999999999, + "calibration": [ + 0.9929, 0.9882, 0.9428, 0.9867, 1.0439, 0.9471234772686924 + ], + "timeTolerance": 0.20476986176515283, + "bytes": 36571452, "cssBytes": 268 }, "tailwind-turbo-devup-ui": { "control": "tailwind-turbo", - "ratio": 0.9986, - "calibration": [0.9986, 1.0059, 0.9532, 0.997, 1.0115], - "timeTolerance": 0.1167, - "bytes": 36572725, + "ratio": 0.9978, + "calibration": [ + 0.9986, 1.0059, 0.9532, 0.997, 1.0115, 0.9732950875989826 + ], + "timeTolerance": 0.11685708558829429, + "bytes": 36558611, "cssBytes": 530 }, "vinext-devup-ui": { "control": "tailwind", - "ratio": 0.1995, - "calibration": [0.1962, 0.2066, 0.2104, 0.1995, 0.1868], - "timeTolerance": 0.2367, + "ratio": 0.19785000000000003, + "calibration": [ + 0.1962, 0.2066, 0.2104, 0.1995, 0.1868, 0.1894402619634694 + ], + "timeTolerance": 0.23856456911801877, "bytes": 1198750, "cssBytes": 29746 }, "devup-ui-collapse": { "control": "tailwind-turbo", - "ratio": 1.0118, - "calibration": [1.0384, 1.0118, 0.9519, 1.0015, 1.021], - "timeTolerance": 0.1709, - "bytes": 31844629, + "ratio": 1.00665, + "calibration": [ + 1.0384, 1.0118, 0.9519, 1.0015, 1.021, 0.9659282187960732 + ], + "timeTolerance": 0.17185714995281381, + "bytes": 31851810, "cssBytes": 1286 } } diff --git a/benchmark-results.json b/benchmark-results.json index 0db0976af..d614f59b7 100644 --- a/benchmark-results.json +++ b/benchmark-results.json @@ -1,24 +1,24 @@ { "run": { - "id": "36702867475", - "url": "https://github.com/dev-five-git/devup-ui/actions/runs/36702867475", - "commit": "8d6c2f62", + "id": "37127187436", + "url": "https://github.com/dev-five-git/devup-ui/actions/runs/37127187436", + "commit": "49b33534", "runner": "ubuntu-24.04", "next": "16.3.3", "devupUi": "1.0.44" }, - "samples": 1, + "samples": 3, "rows": [ { "table": "webpack", "target": "tailwind", "label": "tailwindcss", "version": "4.3.3", - "samples": [16.67], - "median": 16.67, - "min": 16.67, - "max": 16.67, - "bytes": 67332642, + "samples": [13.073913441999998, 12.53008934299997, 12.567363142999994], + "median": 12.567363142999994, + "min": 12.53008934299997, + "max": 13.073913441999998, + "bytes": 67337100, "cssBytes": 5852 }, { @@ -26,11 +26,11 @@ "target": "stylex", "label": "styleX", "version": "0.19.0", - "samples": [38.08], - "median": 38.08, - "min": 38.08, - "max": 38.08, - "bytes": 96291287, + "samples": [28.899089018999998, 28.39018241399998, 28.426503754000006], + "median": 28.426503754000006, + "min": 28.39018241399998, + "max": 28.899089018999998, + "bytes": 96290977, "cssBytes": 425 }, { @@ -38,11 +38,11 @@ "target": "vanilla-extract", "label": "vanilla-extract", "version": "1.21.2", - "samples": [16.09], - "median": 16.09, - "min": 16.09, - "max": 16.09, - "bytes": 68565036, + "samples": [12.486133507000005, 12.17654423, 12.095946322000003], + "median": 12.17654423, + "min": 12.095946322000003, + "max": 12.486133507000005, + "bytes": 68577913, "cssBytes": 294 }, { @@ -50,11 +50,11 @@ "target": "kuma-ui", "label": "kuma-ui", "version": "1.6.4", - "samples": [17.91], - "median": 17.91, - "min": 17.91, - "max": 17.91, - "bytes": 75627783, + "samples": [14.556600542999993, 13.22699236699997, 13.500704490000032], + "median": 13.500704490000032, + "min": 13.22699236699997, + "max": 14.556600542999993, + "bytes": 75627639, "cssBytes": 340 }, { @@ -62,11 +62,11 @@ "target": "panda-css", "label": "panda-css", "version": "1.12.0", - "samples": [18.16], - "median": 18.16, - "min": 18.16, - "max": 18.16, - "bytes": 71849418, + "samples": [14.353009642999998, 13.608209632000014, 13.440839359000034], + "median": 13.608209632000014, + "min": 13.440839359000034, + "max": 14.353009642999998, + "bytes": 71844396, "cssBytes": 15615 }, { @@ -74,11 +74,11 @@ "target": "chakra-ui", "label": "chakra-ui", "version": "3.37.0", - "samples": [27.22], - "median": 27.22, - "min": 27.22, - "max": 27.22, - "bytes": 207426274, + "samples": [20.753550944000004, 20.160875633999996, 20.191317410000018], + "median": 20.191317410000018, + "min": 20.160875633999996, + "max": 20.753550944000004, + "bytes": 207420203, "cssBytes": 0 }, { @@ -86,11 +86,11 @@ "target": "mui", "label": "mui", "version": "9.4.0", - "samples": [18.78], - "median": 18.78, - "min": 18.78, - "max": 18.78, - "bytes": 101473260, + "samples": [14.440934410000002, 14.029546139999992, 14.179219555000076], + "median": 14.179219555000076, + "min": 14.029546139999992, + "max": 14.440934410000002, + "bytes": 101478873, "cssBytes": 0 }, { @@ -99,11 +99,11 @@ "label": "devup-ui (per-file CSS)", "version": "1.0.44", "devup": true, - "samples": [14.27], - "median": 14.27, - "min": 14.27, - "max": 14.27, - "bytes": 67424897, + "samples": [11.254953350000026, 10.825207491000008, 10.973292004000045], + "median": 10.973292004000045, + "min": 10.825207491000008, + "max": 11.254953350000026, + "bytes": 67438054, "cssBytes": 562 }, { @@ -112,11 +112,11 @@ "label": "devup-ui (single CSS)", "version": "1.0.44", "devup": true, - "samples": [14.17], - "median": 14.17, - "min": 14.17, - "max": 14.17, - "bytes": 67426959, + "samples": [11.032363211999996, 10.896242967, 10.984063580999965], + "median": 10.984063580999965, + "min": 10.896242967, + "max": 11.032363211999996, + "bytes": 67415063, "cssBytes": 790 }, { @@ -124,11 +124,11 @@ "target": "tailwind-turbo", "label": "tailwindcss", "version": "4.3.3", - "samples": [7.04], - "median": 7.04, - "min": 7.04, - "max": 7.04, - "bytes": 38422141, + "samples": [5.900064186999982, 5.547877848000004, 5.831188684000051], + "median": 5.831188684000051, + "min": 5.547877848000004, + "max": 5.900064186999982, + "bytes": 38422653, "cssBytes": 6197 }, { @@ -137,11 +137,11 @@ "label": "devup-ui (direct APIs, single CSS)", "version": "1.0.44", "devup": true, - "samples": [6.83], - "median": 6.83, - "min": 6.83, - "max": 6.83, - "bytes": 36518952, + "samples": [5.663960841000022, 5.579608844000031, 5.6779602329999905], + "median": 5.663960841000022, + "min": 5.579608844000031, + "max": 5.6779602329999905, + "bytes": 36545972, "cssBytes": 327 }, { @@ -150,11 +150,11 @@ "label": "devup-ui (static `.css.ts`)", "version": "1.0.44", "devup": true, - "samples": [6.99], - "median": 6.99, - "min": 6.99, - "max": 6.99, - "bytes": 36578506, + "samples": [5.647029271000006, 5.522855702999979, 5.507302220999962], + "median": 5.522855702999979, + "min": 5.507302220999962, + "max": 5.647029271000006, + "bytes": 36571452, "cssBytes": 268 } ] diff --git a/bindings/devup-ui-wasm/size-budget.json b/bindings/devup-ui-wasm/size-budget.json index f7488542c..a0a9703c6 100644 --- a/bindings/devup-ui-wasm/size-budget.json +++ b/bindings/devup-ui-wasm/size-budget.json @@ -1,10 +1,10 @@ { "wasm": { - "raw": 6059008, - "gzip": 2170880 + "raw": 5479424, + "gzip": 1970176 }, "glue": { - "raw": 47104, - "gzip": 8192 + "raw": 41984, + "gzip": 7168 } } From 06e5beaaa948838b3c47bd2f16450bb48b8fbeeb Mon Sep 17 00:00:00 2001 From: owjs3901 Date: Sat, 3 Oct 2026 23:06:52 +0900 Subject: [PATCH 3/9] demo: an artificial 3 s delay and a WASM budget below the size (throwaway, must fail) Ultraworked with [Sisyphus](https://github.com/code-yeongyu/oh-my-openagent) Co-authored-by: Sisyphus --- benchmark/next-devup-ui-single-turbo/package.json | 2 +- bindings/devup-ui-wasm/size-budget.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/benchmark/next-devup-ui-single-turbo/package.json b/benchmark/next-devup-ui-single-turbo/package.json index 8ec760d0f..ba9c08aa3 100644 --- a/benchmark/next-devup-ui-single-turbo/package.json +++ b/benchmark/next-devup-ui-single-turbo/package.json @@ -5,7 +5,7 @@ "private": true, "scripts": { "dev": "next dev", - "build": "next build", + "build": "node -e \"setTimeout(() => {}, 3000)\" && next build", "start": "next start", "lint": "eslint" }, diff --git a/bindings/devup-ui-wasm/size-budget.json b/bindings/devup-ui-wasm/size-budget.json index a0a9703c6..95b5a7198 100644 --- a/bindings/devup-ui-wasm/size-budget.json +++ b/bindings/devup-ui-wasm/size-budget.json @@ -1,6 +1,6 @@ { "wasm": { - "raw": 5479424, + "raw": 5200000, "gzip": 1970176 }, "glue": { From fac1d7d4ade07db15349e39500fd97f9f2e212a7 Mon Sep 17 00:00:00 2001 From: owjs3901 Date: Thu, 8 Oct 2026 19:43:41 +0900 Subject: [PATCH 4/9] ci(benchmark): recalibrate for main Tailwind v4 output Main 11790bef intentionally emits complete Tailwind v4 font line-height declarations and property registrations, so CI 37758611477 produced 1491 CSS bytes for tailwind-turbo-devup-ui against the former 530-byte baseline. Follow the existing intentional-growth policy using that actual 18-target, three-sample artifact: append each measured seventh control ratio while retaining all six prior entries, regenerate baseline/results and both README benchmark blocks. Keep every gate, threshold formula, workflow step and WASM size budget unchanged; all nine artifact comparisons, provenance/calibration assertions, README renderer check and fresh WASM/root builds pass. Refs #683 Ultraworked with [Sisyphus](https://github.com/code-yeongyu/oh-my-openagent) Co-authored-by: Sisyphus --- README.md | 28 ++++----- README_ko.md | 28 ++++----- benchmark-baseline.json | 83 +++++++++++++++----------- benchmark-results.json | 126 ++++++++++++++++++++-------------------- 4 files changed, 139 insertions(+), 126 deletions(-) diff --git a/README.md b/README.md index bc9602e35..daf761b6a 100644 --- a/README.md +++ b/README.md @@ -80,7 +80,7 @@ npm install @devup-ui/webpack-plugin ## Comparison Benchmarks -[Latest CI benchmark](https://github.com/dev-five-git/devup-ui/actions/runs/37127187436) (commit `49b33534`, Devup UI 1.0.44) on `ubuntu-24.04` with Next.js 16.3.3. Every row is the median of 3 cold builds that ran in rotating order, shown with the range of the samples. All Next.js builds use the native TypeScript 7 CLI for type checking. The English and Korean tables are generated from the same checked result file, `benchmark-results.json`. +[Latest CI benchmark](https://github.com/dev-five-git/devup-ui/actions/runs/37758611477) (commit `467a379a`, Devup UI 1.0.44) on `ubuntu-24.04` with Next.js 16.3.3. Every row is the median of 3 cold builds that ran in rotating order, shown with the range of the samples. All Next.js builds use the native TypeScript 7 CLI for type checking. The English and Korean tables are generated from the same checked result file, `benchmark-results.json`. Build Size is every byte of the build output directory (`.next`, or `dist` for vinext); CSS Size is the emitted `.css` files alone, measured separately. @@ -88,25 +88,25 @@ Webpack: | Library | Version | Median Build Time | Range | Build Size | CSS Size | | --------------------------- | ------- | ----------------- | --------------- | ----------------- | ------------ | -| tailwindcss | 4.3.3 | 12.57s | 12.53s - 13.07s | 67,337,100 bytes | 5,852 bytes | -| styleX | 0.19.0 | 28.43s | 28.39s - 28.90s | 96,290,977 bytes | 425 bytes | -| vanilla-extract | 1.21.2 | 12.18s | 12.10s - 12.49s | 68,577,913 bytes | 294 bytes | -| kuma-ui | 1.6.4 | 13.50s | 13.23s - 14.56s | 75,627,639 bytes | 340 bytes | -| panda-css | 1.12.0 | 13.61s | 13.44s - 14.35s | 71,844,396 bytes | 15,615 bytes | -| chakra-ui | 3.37.0 | 20.19s | 20.16s - 20.75s | 207,420,203 bytes | 0 bytes | -| mui | 9.4.0 | 14.18s | 14.03s - 14.44s | 101,478,873 bytes | 0 bytes | -| **devup-ui (per-file CSS)** | 1.0.44 | **10.97s** | 10.83s - 11.25s | 67,438,054 bytes | 562 bytes | -| **devup-ui (single CSS)** | 1.0.44 | **10.98s** | 10.90s - 11.03s | 67,415,063 bytes | 790 bytes | +| tailwindcss | 4.3.3 | 13.02s | 12.91s - 13.20s | 67,339,250 bytes | 5,852 bytes | +| styleX | 0.19.0 | 29.01s | 28.72s - 29.50s | 96,290,533 bytes | 425 bytes | +| vanilla-extract | 1.21.2 | 12.34s | 12.22s - 12.35s | 68,565,511 bytes | 294 bytes | +| kuma-ui | 1.6.4 | 13.55s | 13.49s - 13.57s | 75,627,429 bytes | 340 bytes | +| panda-css | 1.12.0 | 13.44s | 13.29s - 13.58s | 71,849,243 bytes | 15,615 bytes | +| chakra-ui | 3.37.0 | 20.29s | 19.95s - 21.06s | 207,425,028 bytes | 0 bytes | +| mui | 9.4.0 | 14.12s | 13.78s - 14.26s | 101,476,098 bytes | 0 bytes | +| **devup-ui (per-file CSS)** | 1.0.44 | **10.95s** | 10.87s - 11.09s | 67,427,695 bytes | 562 bytes | +| **devup-ui (single CSS)** | 1.0.44 | **11.04s** | 10.88s - 11.40s | 67,436,336 bytes | 790 bytes | Turbopack: | Library | Version | Median Build Time | Range | Build Size | CSS Size | | -------------------------------------- | ------- | ----------------- | ------------- | ---------------- | ----------- | -| tailwindcss | 4.3.3 | 5.83s | 5.55s - 5.90s | 38,422,653 bytes | 6,197 bytes | -| **devup-ui (direct APIs, single CSS)** | 1.0.44 | **5.66s** | 5.58s - 5.68s | 36,545,972 bytes | 327 bytes | -| **devup-ui (static `.css.ts`)** | 1.0.44 | **5.52s** | 5.51s - 5.65s | 36,571,452 bytes | 268 bytes | +| tailwindcss | 4.3.3 | 5.68s | 5.51s - 5.73s | 38,440,013 bytes | 6,197 bytes | +| **devup-ui (direct APIs, single CSS)** | 1.0.44 | **5.79s** | 5.77s - 5.87s | 36,538,765 bytes | 327 bytes | +| **devup-ui (static `.css.ts`)** | 1.0.44 | **5.71s** | 5.67s - 5.99s | 36,580,207 bytes | 268 bytes | -The Turbopack ranges overlap, so the direct-API median is 0.17s (2.9%) ahead of Tailwind on this fixture and the two are effectively at parity. The 3 cold samples were Tailwind `5.90s, 5.55s, 5.83s`, direct Devup UI `5.66s, 5.58s, 5.68s`, and static `.css.ts` `5.65s, 5.52s, 5.51s`. The fixtures have comparable app shapes, not pixel-identical styling: Tailwind styles the leading paragraph and button more heavily, while Devup UI exercises typed component/style props. Treat these as build-pipeline results rather than a per-rule microbenchmark. Every `.css.ts` module runs on the full Boa evaluator, which adds about 20 ms for the first module and under 1 ms for each further one. CI compares the Devup UI rows with a checked baseline (see `benchmark-gate.js` for the rule) and fails on a regression beyond the measured noise. +The Turbopack ranges do not overlap, so the direct-API median is 0.12s (2.1%) behind Tailwind on this fixture. The 3 cold samples were Tailwind `5.68s, 5.73s, 5.51s`, direct Devup UI `5.77s, 5.79s, 5.87s`, and static `.css.ts` `5.71s, 5.99s, 5.67s`. The fixtures have comparable app shapes, not pixel-identical styling: Tailwind styles the leading paragraph and button more heavily, while Devup UI exercises typed component/style props. Treat these as build-pipeline results rather than a per-rule microbenchmark. Every `.css.ts` module runs on the full Boa evaluator, which adds about 20 ms for the first module and under 1 ms for each further one. CI compares the Devup UI rows with a checked baseline (see `benchmark-gate.js` for the rule) and fails on a regression beyond the measured noise. ## How it works diff --git a/README_ko.md b/README_ko.md index 81ffa9e99..1d4c199c8 100644 --- a/README_ko.md +++ b/README_ko.md @@ -80,7 +80,7 @@ npm install @devup-ui/webpack-plugin ## 비교 벤치마크 -[최신 CI 벤치마크](https://github.com/dev-five-git/devup-ui/actions/runs/37127187436) (커밋 `49b33534`, Devup UI 1.0.44)는 `ubuntu-24.04`에서 Next.js 16.3.3로 실행했습니다. 모든 행은 순서를 돌려 가며 측정한 cold build 3회의 중앙값이며 샘플의 범위를 함께 보여줍니다. 모든 Next.js 빌드는 네이티브 TypeScript 7 CLI로 타입 검사를 수행합니다. 영어와 한국어 표는 같은 결과 파일 `benchmark-results.json`에서 생성합니다. +[최신 CI 벤치마크](https://github.com/dev-five-git/devup-ui/actions/runs/37758611477) (커밋 `467a379a`, Devup UI 1.0.44)는 `ubuntu-24.04`에서 Next.js 16.3.3로 실행했습니다. 모든 행은 순서를 돌려 가며 측정한 cold build 3회의 중앙값이며 샘플의 범위를 함께 보여줍니다. 모든 Next.js 빌드는 네이티브 TypeScript 7 CLI로 타입 검사를 수행합니다. 영어와 한국어 표는 같은 결과 파일 `benchmark-results.json`에서 생성합니다. 빌드 사이즈는 빌드 출력 디렉터리(`.next`, vinext는 `dist`)의 모든 바이트이고, CSS 사이즈는 출력된 `.css` 파일만 따로 잰 값입니다. @@ -88,25 +88,25 @@ Webpack: | 라이브러리 | 버전 | 빌드 시간 중앙값 | 범위 | 빌드 사이즈 | CSS 사이즈 | | --------------------------- | ------ | ---------- | --------------- | ----------------- | ------------ | -| tailwindcss | 4.3.3 | 12.57s | 12.53s - 13.07s | 67,337,100 bytes | 5,852 bytes | -| styleX | 0.19.0 | 28.43s | 28.39s - 28.90s | 96,290,977 bytes | 425 bytes | -| vanilla-extract | 1.21.2 | 12.18s | 12.10s - 12.49s | 68,577,913 bytes | 294 bytes | -| kuma-ui | 1.6.4 | 13.50s | 13.23s - 14.56s | 75,627,639 bytes | 340 bytes | -| panda-css | 1.12.0 | 13.61s | 13.44s - 14.35s | 71,844,396 bytes | 15,615 bytes | -| chakra-ui | 3.37.0 | 20.19s | 20.16s - 20.75s | 207,420,203 bytes | 0 bytes | -| mui | 9.4.0 | 14.18s | 14.03s - 14.44s | 101,478,873 bytes | 0 bytes | -| **devup-ui (per-file CSS)** | 1.0.44 | **10.97s** | 10.83s - 11.25s | 67,438,054 bytes | 562 bytes | -| **devup-ui (single CSS)** | 1.0.44 | **10.98s** | 10.90s - 11.03s | 67,415,063 bytes | 790 bytes | +| tailwindcss | 4.3.3 | 13.02s | 12.91s - 13.20s | 67,339,250 bytes | 5,852 bytes | +| styleX | 0.19.0 | 29.01s | 28.72s - 29.50s | 96,290,533 bytes | 425 bytes | +| vanilla-extract | 1.21.2 | 12.34s | 12.22s - 12.35s | 68,565,511 bytes | 294 bytes | +| kuma-ui | 1.6.4 | 13.55s | 13.49s - 13.57s | 75,627,429 bytes | 340 bytes | +| panda-css | 1.12.0 | 13.44s | 13.29s - 13.58s | 71,849,243 bytes | 15,615 bytes | +| chakra-ui | 3.37.0 | 20.29s | 19.95s - 21.06s | 207,425,028 bytes | 0 bytes | +| mui | 9.4.0 | 14.12s | 13.78s - 14.26s | 101,476,098 bytes | 0 bytes | +| **devup-ui (per-file CSS)** | 1.0.44 | **10.95s** | 10.87s - 11.09s | 67,427,695 bytes | 562 bytes | +| **devup-ui (single CSS)** | 1.0.44 | **11.04s** | 10.88s - 11.40s | 67,436,336 bytes | 790 bytes | Turbopack: | 라이브러리 | 버전 | 빌드 시간 중앙값 | 범위 | 빌드 사이즈 | CSS 사이즈 | | -------------------------------------- | ------ | --------- | ------------- | ---------------- | ----------- | -| tailwindcss | 4.3.3 | 5.83s | 5.55s - 5.90s | 38,422,653 bytes | 6,197 bytes | -| **devup-ui (direct APIs, single CSS)** | 1.0.44 | **5.66s** | 5.58s - 5.68s | 36,545,972 bytes | 327 bytes | -| **devup-ui (static `.css.ts`)** | 1.0.44 | **5.52s** | 5.51s - 5.65s | 36,571,452 bytes | 268 bytes | +| tailwindcss | 4.3.3 | 5.68s | 5.51s - 5.73s | 38,440,013 bytes | 6,197 bytes | +| **devup-ui (direct APIs, single CSS)** | 1.0.44 | **5.79s** | 5.77s - 5.87s | 36,538,765 bytes | 327 bytes | +| **devup-ui (static `.css.ts`)** | 1.0.44 | **5.71s** | 5.67s - 5.99s | 36,580,207 bytes | 268 bytes | -Turbopack 측정 범위는 서로 겹치므로 direct API 중앙값은 이 fixture에서 Tailwind보다 0.17s(2.9%) 빠르며 사실상 동률입니다. cold 샘플 3개는 Tailwind `5.90s, 5.55s, 5.83s`, direct Devup UI `5.66s, 5.58s, 5.68s`, static `.css.ts` `5.65s, 5.52s, 5.51s`입니다. fixture는 앱 구조만 비슷하고 스타일이 픽셀 단위로 같지는 않습니다. Tailwind는 첫 문단과 버튼에 더 많은 스타일을 주고, Devup UI는 타입이 있는 컴포넌트/스타일 props를 사용합니다. 규칙 단위 마이크로벤치마크가 아니라 빌드 파이프라인 결과로 보세요. 모든 `.css.ts`는 full Boa evaluator로 처리하며, 첫 모듈에 약 20ms, 이후 모듈마다 1ms 미만이 더 듭니다. CI는 Devup UI 행을 체크인된 기준값과 비교해 측정된 노이즈를 넘는 회귀에서 실패합니다(규칙은 `benchmark-gate.js`). +Turbopack 측정 범위는 겹치지 않으므로 direct API 중앙값은 이 fixture에서 Tailwind보다 0.12s(2.1%) 느리며. cold 샘플 3개는 Tailwind `5.68s, 5.73s, 5.51s`, direct Devup UI `5.77s, 5.79s, 5.87s`, static `.css.ts` `5.71s, 5.99s, 5.67s`입니다. fixture는 앱 구조만 비슷하고 스타일이 픽셀 단위로 같지는 않습니다. Tailwind는 첫 문단과 버튼에 더 많은 스타일을 주고, Devup UI는 타입이 있는 컴포넌트/스타일 props를 사용합니다. 규칙 단위 마이크로벤치마크가 아니라 빌드 파이프라인 결과로 보세요. 모든 `.css.ts`는 full Boa evaluator로 처리하며, 첫 모듈에 약 20ms, 이후 모듈마다 1ms 미만이 더 듭니다. CI는 Devup UI 행을 체크인된 기준값과 비교해 측정된 노이즈를 넘는 회귀에서 실패합니다(규칙은 `benchmark-gate.js`). ## 작동 원리 diff --git a/benchmark-baseline.json b/benchmark-baseline.json index c28493adf..216544677 100644 --- a/benchmark-baseline.json +++ b/benchmark-baseline.json @@ -3,88 +3,101 @@ "rows": { "stylex-turbo-devup-ui": { "control": "tailwind-turbo", - "ratio": 1.349, + "ratio": 1.3509, "calibration": [ - 1.3509, 1.3471, 1.2939, 1.3096, 1.4427, 1.417800722121156 + 1.3509, 1.3471, 1.2939, 1.3096, 1.4427, 1.417800722121156, + 1.4176477939108723 ], - "timeTolerance": 0.22060785767234994, - "bytes": 31227953, + "timeTolerance": 0.22029757939151684, + "bytes": 31230251, "cssBytes": 321 }, "devup-ui": { "control": "tailwind", - "ratio": 0.8645789338279827, + "ratio": 0.856, "calibration": [ - 0.856, 0.8538, 0.9013, 0.8735, 0.8515, 0.8731578676559654 + 0.856, 0.8538, 0.9013, 0.8735, 0.8515, 0.8731578676559654, + 0.8408134770157684 ], - "timeTolerance": 0.1152005862079175, - "bytes": 67438054, + "timeTolerance": 0.14132365183231688, + "bytes": 67427695, "cssBytes": 562 }, "devup-ui-single": { "control": "tailwind", - "ratio": 0.87155, - "calibration": [0.85, 0.8727, 0.9024, 0.8704, 0.859, 0.874014974821355], - "timeTolerance": 0.12024553955596351, - "bytes": 67415063, + "ratio": 0.8704, + "calibration": [ + 0.85, 0.8727, 0.9024, 0.8704, 0.859, 0.874014974821355, + 0.848021750757629 + ], + "timeTolerance": 0.12495002123706572, + "bytes": 67436336, "cssBytes": 790 }, "devup-ui-single-turbo": { "control": "tailwind-turbo", - "ratio": 0.9753609130605841, + "ratio": 0.9794, "calibration": [ - 0.9702, 0.9794, 0.9701, 1.003, 0.9981, 0.9713218261211682 + 0.9702, 0.9794, 0.9701, 1.003, 0.9981, 0.9713218261211682, + 1.0209678104409452 ], - "timeTolerance": 0.1, - "bytes": 36545972, + "timeTolerance": 0.10387545526025166, + "bytes": 36538765, "cssBytes": 327 }, "devup-ui-turbo": { "control": "tailwind-turbo", - "ratio": 0.9893000000000001, - "calibration": [0.9901, 1, 0.9753, 1.0089, 0.9885, 0.9761607019540564], + "ratio": 0.9901, + "calibration": [ + 0.9901, 1, 0.9753, 1.0089, 0.9885, 0.9761607019540564, + 1.0183680359603602 + ], "timeTolerance": 0.1, - "bytes": 36539932, + "bytes": 36541732, "cssBytes": 358 }, "vanilla-extract-devup-ui": { "control": "tailwind-turbo", - "ratio": 0.9874499999999999, + "ratio": 0.9882, "calibration": [ - 0.9929, 0.9882, 0.9428, 0.9867, 1.0439, 0.9471234772686924 + 0.9929, 0.9882, 0.9428, 0.9867, 1.0439, 0.9471234772686924, + 1.0068141946392295 ], - "timeTolerance": 0.20476986176515283, - "bytes": 36571452, + "timeTolerance": 0.20461445051609004, + "bytes": 36580207, "cssBytes": 268 }, "tailwind-turbo-devup-ui": { "control": "tailwind-turbo", - "ratio": 0.9978, + "ratio": 0.9986, "calibration": [ - 0.9986, 1.0059, 0.9532, 0.997, 1.0115, 0.9732950875989826 + 0.9986, 1.0059, 0.9532, 0.997, 1.0115, 0.9732950875989826, + 1.0262587718287204 ], - "timeTolerance": 0.11685708558829429, - "bytes": 36558611, - "cssBytes": 530 + "timeTolerance": 0.1463223950104553, + "bytes": 36593833, + "cssBytes": 1491 }, "vinext-devup-ui": { "control": "tailwind", - "ratio": 0.19785000000000003, + "ratio": 0.1962, "calibration": [ - 0.1962, 0.2066, 0.2104, 0.1995, 0.1868, 0.1894402619634694 + 0.1962, 0.2066, 0.2104, 0.1995, 0.1868, 0.1894402619634694, + 0.1880300744715731 ], - "timeTolerance": 0.23856456911801877, + "timeTolerance": 0.2405708460754333, "bytes": 1198750, "cssBytes": 29746 }, "devup-ui-collapse": { "control": "tailwind-turbo", - "ratio": 1.00665, + "ratio": 1.0118, "calibration": [ - 1.0384, 1.0118, 0.9519, 1.0015, 1.021, 0.9659282187960732 + 1.0384, 1.0118, 0.9519, 1.0015, 1.021, 0.9659282187960732, + 1.0242688078238835 ], - "timeTolerance": 0.17185714995281381, - "bytes": 31851810, + "timeTolerance": 0.17098240759043293, + "bytes": 31849350, "cssBytes": 1286 } } diff --git a/benchmark-results.json b/benchmark-results.json index d614f59b7..6546d9ae5 100644 --- a/benchmark-results.json +++ b/benchmark-results.json @@ -1,8 +1,8 @@ { "run": { - "id": "37127187436", - "url": "https://github.com/dev-five-git/devup-ui/actions/runs/37127187436", - "commit": "49b33534", + "id": "37758611477", + "url": "https://github.com/dev-five-git/devup-ui/actions/runs/37758611477", + "commit": "467a379a", "runner": "ubuntu-24.04", "next": "16.3.3", "devupUi": "1.0.44" @@ -14,11 +14,11 @@ "target": "tailwind", "label": "tailwindcss", "version": "4.3.3", - "samples": [13.073913441999998, 12.53008934299997, 12.567363142999994], - "median": 12.567363142999994, - "min": 12.53008934299997, - "max": 13.073913441999998, - "bytes": 67337100, + "samples": [13.19597508, 13.021540712999995, 12.911684594999999], + "median": 13.021540712999995, + "min": 12.911684594999999, + "max": 13.19597508, + "bytes": 67339250, "cssBytes": 5852 }, { @@ -26,11 +26,11 @@ "target": "stylex", "label": "styleX", "version": "0.19.0", - "samples": [28.899089018999998, 28.39018241399998, 28.426503754000006], - "median": 28.426503754000006, - "min": 28.39018241399998, - "max": 28.899089018999998, - "bytes": 96290977, + "samples": [29.49553623, 28.717898384, 29.01065776699997], + "median": 29.01065776699997, + "min": 28.717898384, + "max": 29.49553623, + "bytes": 96290533, "cssBytes": 425 }, { @@ -38,11 +38,11 @@ "target": "vanilla-extract", "label": "vanilla-extract", "version": "1.21.2", - "samples": [12.486133507000005, 12.17654423, 12.095946322000003], - "median": 12.17654423, - "min": 12.095946322000003, - "max": 12.486133507000005, - "bytes": 68577913, + "samples": [12.34335704699999, 12.346740351999994, 12.215919330999954], + "median": 12.34335704699999, + "min": 12.215919330999954, + "max": 12.346740351999994, + "bytes": 68565511, "cssBytes": 294 }, { @@ -50,11 +50,11 @@ "target": "kuma-ui", "label": "kuma-ui", "version": "1.6.4", - "samples": [14.556600542999993, 13.22699236699997, 13.500704490000032], - "median": 13.500704490000032, - "min": 13.22699236699997, - "max": 14.556600542999993, - "bytes": 75627639, + "samples": [13.566745991999996, 13.491478220999998, 13.550664844000014], + "median": 13.550664844000014, + "min": 13.491478220999998, + "max": 13.566745991999996, + "bytes": 75627429, "cssBytes": 340 }, { @@ -62,11 +62,11 @@ "target": "panda-css", "label": "panda-css", "version": "1.12.0", - "samples": [14.353009642999998, 13.608209632000014, 13.440839359000034], - "median": 13.608209632000014, - "min": 13.440839359000034, - "max": 14.353009642999998, - "bytes": 71844396, + "samples": [13.581986012000009, 13.292706686999997, 13.438533212000038], + "median": 13.438533212000038, + "min": 13.292706686999997, + "max": 13.581986012000009, + "bytes": 71849243, "cssBytes": 15615 }, { @@ -74,11 +74,11 @@ "target": "chakra-ui", "label": "chakra-ui", "version": "3.37.0", - "samples": [20.753550944000004, 20.160875633999996, 20.191317410000018], - "median": 20.191317410000018, - "min": 20.160875633999996, - "max": 20.753550944000004, - "bytes": 207420203, + "samples": [21.061893298000008, 19.949156122000016, 20.292663101000013], + "median": 20.292663101000013, + "min": 19.949156122000016, + "max": 21.061893298000008, + "bytes": 207425028, "cssBytes": 0 }, { @@ -86,11 +86,11 @@ "target": "mui", "label": "mui", "version": "9.4.0", - "samples": [14.440934410000002, 14.029546139999992, 14.179219555000076], - "median": 14.179219555000076, - "min": 14.029546139999992, - "max": 14.440934410000002, - "bytes": 101478873, + "samples": [14.264408890999999, 13.777297842, 14.11791713899991], + "median": 14.11791713899991, + "min": 13.777297842, + "max": 14.264408890999999, + "bytes": 101476098, "cssBytes": 0 }, { @@ -99,11 +99,11 @@ "label": "devup-ui (per-file CSS)", "version": "1.0.44", "devup": true, - "samples": [11.254953350000026, 10.825207491000008, 10.973292004000045], - "median": 10.973292004000045, - "min": 10.825207491000008, - "max": 11.254953350000026, - "bytes": 67438054, + "samples": [11.089230005999998, 10.872500918999984, 10.948686922999913], + "median": 10.948686922999913, + "min": 10.872500918999984, + "max": 11.089230005999998, + "bytes": 67427695, "cssBytes": 562 }, { @@ -112,11 +112,11 @@ "label": "devup-ui (single CSS)", "version": "1.0.44", "devup": true, - "samples": [11.032363211999996, 10.896242967, 10.984063580999965], - "median": 10.984063580999965, - "min": 10.896242967, - "max": 11.032363211999996, - "bytes": 67415063, + "samples": [11.042549753, 11.396381741000019, 10.884128520000028], + "median": 11.042549753, + "min": 10.884128520000028, + "max": 11.396381741000019, + "bytes": 67436336, "cssBytes": 790 }, { @@ -124,11 +124,11 @@ "target": "tailwind-turbo", "label": "tailwindcss", "version": "4.3.3", - "samples": [5.900064186999982, 5.547877848000004, 5.831188684000051], - "median": 5.831188684000051, - "min": 5.547877848000004, - "max": 5.900064186999982, - "bytes": 38422653, + "samples": [5.6750599369999835, 5.72619077299998, 5.509298593999935], + "median": 5.6750599369999835, + "min": 5.509298593999935, + "max": 5.72619077299998, + "bytes": 38440013, "cssBytes": 6197 }, { @@ -137,11 +137,11 @@ "label": "devup-ui (direct APIs, single CSS)", "version": "1.0.44", "devup": true, - "samples": [5.663960841000022, 5.579608844000031, 5.6779602329999905], - "median": 5.663960841000022, - "min": 5.579608844000031, - "max": 5.6779602329999905, - "bytes": 36545972, + "samples": [5.765447661000013, 5.794053518000001, 5.874383794999972], + "median": 5.794053518000001, + "min": 5.765447661000013, + "max": 5.874383794999972, + "bytes": 36538765, "cssBytes": 327 }, { @@ -150,11 +150,11 @@ "label": "devup-ui (static `.css.ts`)", "version": "1.0.44", "devup": true, - "samples": [5.647029271000006, 5.522855702999979, 5.507302220999962], - "median": 5.522855702999979, - "min": 5.507302220999962, - "max": 5.647029271000006, - "bytes": 36571452, + "samples": [5.713730899999995, 5.988215032999986, 5.674502772999985], + "median": 5.713730899999995, + "min": 5.674502772999985, + "max": 5.988215032999986, + "bytes": 36580207, "cssBytes": 268 } ] From a85a8b1beaed04825cd491c437e26cd284ead27a Mon Sep 17 00:00:00 2001 From: owjs3901 Date: Thu, 15 Oct 2026 10:18:39 +0900 Subject: [PATCH 5/9] fix(benchmark): measure shipped CSS without compiler caches MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Refs #683 2026-10-15: main #743의 정렬된 쓰기로 보존된 450바이트 컴파일러 캐시는 브라우저 출력이 아니므로 측정에서 제외하고, 이전 #746의 동일한 실제 CSS 494바이트로 전용 writer를 실행해 기준만 보정했습니다. 기존 32바이트 여유(한도 526), 빌드 크기와 시간 규칙, 모든 보정 이력을 유지했습니다. Ultraworked with [Sisyphus](https://github.com/code-yeongyu/oh-my-openagent) Co-authored-by: Sisyphus --- README.md | 2 +- README_ko.md | 2 +- benchmark-baseline.json | 2 +- benchmark-check.js | 23 ++- benchmark-gate.js | 149 +++++++++++++++++- benchmark-output.js | 30 ++++ benchmark.js | 46 +----- .../__tests__/benchmark-css-baseline.test.ts | 135 ++++++++++++++++ .../src/__tests__/benchmark-output.test.ts | 140 ++++++++++++++++ render-benchmark-readme.js | 4 +- 10 files changed, 482 insertions(+), 51 deletions(-) create mode 100644 benchmark-output.js create mode 100644 packages/plugin-utils/src/__tests__/benchmark-css-baseline.test.ts create mode 100644 packages/plugin-utils/src/__tests__/benchmark-output.test.ts diff --git a/README.md b/README.md index b8836312e..6f8abc8cd 100644 --- a/README.md +++ b/README.md @@ -82,7 +82,7 @@ npm install @devup-ui/webpack-plugin [Latest CI benchmark](https://github.com/dev-five-git/devup-ui/actions/runs/37758611477) (commit `467a379a`, Devup UI 1.0.44) on `ubuntu-24.04` with Next.js 16.3.3. Every row is the median of 3 cold builds that ran in rotating order, shown with the range of the samples. All Next.js builds use the native TypeScript 7 CLI for type checking. The English and Korean tables are generated from the same checked result file, `benchmark-results.json`. -Build Size is every byte of the build output directory (`.next`, or `dist` for vinext); CSS Size is the emitted `.css` files alone, measured separately. +Build Size is every byte of the build output directory (`.next`, or `dist` for vinext), including caches and server output. The checked table retains its original historical CSS measurements, which recursively counted `.css` files throughout that output. New runs count only browser-shipped CSS: `.next/static/**/*.css` (Webpack `static/css` and Turbopack `static/chunks`) or vinext `dist/client/**/*.css`, excluding caches and server CSS. A missing browser-assets directory is an error; an existing directory with no CSS counts zero. For an approved per-file CSS measurement correction, `bun benchmark-check.js --write-css-baseline devup-ui` updates only that existing row's CSS bytes; unlike `--write-baseline`, it preserves sample count, build bytes, ratios, tolerances, all other rows and calibration history. Webpack: diff --git a/README_ko.md b/README_ko.md index 95a1b4f23..0636d5e84 100644 --- a/README_ko.md +++ b/README_ko.md @@ -82,7 +82,7 @@ npm install @devup-ui/webpack-plugin [최신 CI 벤치마크](https://github.com/dev-five-git/devup-ui/actions/runs/37758611477) (커밋 `467a379a`, Devup UI 1.0.44)는 `ubuntu-24.04`에서 Next.js 16.3.3로 실행했습니다. 모든 행은 순서를 돌려 가며 측정한 cold build 3회의 중앙값이며 샘플의 범위를 함께 보여줍니다. 모든 Next.js 빌드는 네이티브 TypeScript 7 CLI로 타입 검사를 수행합니다. 영어와 한국어 표는 같은 결과 파일 `benchmark-results.json`에서 생성합니다. -빌드 사이즈는 빌드 출력 디렉터리(`.next`, vinext는 `dist`)의 모든 바이트이고, CSS 사이즈는 출력된 `.css` 파일만 따로 잰 값입니다. +빌드 사이즈는 캐시와 서버 출력을 포함한 빌드 출력 디렉터리(`.next`, vinext는 `dist`)의 모든 바이트입니다. 체크인된 표는 출력 전체의 `.css` 파일을 재귀적으로 합산한 기존 CSS 측정값을 그대로 보존합니다. 새 실행은 브라우저에 제공되는 CSS만 측정합니다: `.next/static/**/*.css`(Webpack `static/css`, Turbopack `static/chunks`) 또는 vinext `dist/client/**/*.css`. 캐시와 서버 CSS는 제외합니다. 브라우저 에셋 디렉터리가 없으면 오류이며, 존재하지만 CSS가 없으면 0입니다. 승인된 per-file CSS 측정 보정에는 `bun benchmark-check.js --write-css-baseline devup-ui`를 사용합니다. `--write-baseline`과 달리 기존 해당 행의 CSS 바이트만 바꾸고 샘플 수, 빌드 바이트, 비율, 허용 오차, 다른 모든 행과 보정 이력을 보존합니다. Webpack: diff --git a/benchmark-baseline.json b/benchmark-baseline.json index 216544677..ee0e0e799 100644 --- a/benchmark-baseline.json +++ b/benchmark-baseline.json @@ -21,7 +21,7 @@ ], "timeTolerance": 0.14132365183231688, "bytes": 67427695, - "cssBytes": 562 + "cssBytes": 494 }, "devup-ui-single": { "control": "tailwind", diff --git a/benchmark-check.js b/benchmark-check.js index 6dfa3b61b..872c4dd3d 100644 --- a/benchmark-check.js +++ b/benchmark-check.js @@ -3,6 +3,7 @@ // // bun benchmark-check.js compare, exit 1 on a regression // bun benchmark-check.js --write-baseline add this run to the calibration and rewrite benchmark-baseline.json +// bun benchmark-check.js --write-css-baseline devup-ui replace only the existing per-file CSS baseline; keep all calibration, time and build-size fields // bun benchmark-check.js --write-results rewrite benchmark-results.json (the README tables) from this run // // A change that makes a build slower or larger on purpose runs the two writes @@ -10,7 +11,12 @@ // request. import { readFileSync, writeFileSync } from 'node:fs' -import { evaluate, nextBaseline, report } from './benchmark-gate.js' +import { + evaluate, + nextBaseline, + nextCssBaseline, + report, +} from './benchmark-gate.js' const read = (file) => JSON.parse(readFileSync(file, 'utf8')) const write = (file, value) => @@ -19,7 +25,20 @@ const write = (file, value) => const manifest = read('benchmark-manifest.json') const run = read('benchmark-run.json') -if (process.argv.includes('--write-baseline')) { +if (process.argv.includes('--write-css-baseline')) { + const target = process.argv[process.argv.indexOf('--write-css-baseline') + 1] + if ( + target !== 'devup-ui' || + process.argv.includes('--write-baseline') || + process.argv.includes('--write-results') + ) + throw new TypeError( + 'Use --write-css-baseline devup-ui without other writer modes', + ) + const previous = read('benchmark-baseline.json') + write('benchmark-baseline.json', nextCssBaseline(run, previous)) + console.info('benchmark-baseline.json devup-ui CSS baseline rewritten') +} else if (process.argv.includes('--write-baseline')) { let previous try { previous = read('benchmark-baseline.json') diff --git a/benchmark-gate.js b/benchmark-gate.js index 2de619703..ecd9604a5 100644 --- a/benchmark-gate.js +++ b/benchmark-gate.js @@ -1 +1,148 @@ -// The regression rule behind `benchmark-check.js`, kept free of file and process // access so it can be tested. // // Only the Devup UI rows of benchmark-manifest.json (`gate: true`) can fail; // every other library's row is informational. // // Build time is compared as a ratio to a control row measured in the same run // (Tailwind on the same bundler), because the shared CI runner drifts by about // 25% between runs while the ratio drifts by a few percent. A row fails when // ratio > baseline ratio * (1 + timeTolerance) // where the tolerance is at least 10% and at least twice the spread the ratio // showed over the calibration runs stored with the baseline, so ordinary noise // stays well inside it and a real slowdown does not. // // Output sizes are deterministic to about 0.05% between runs. A row fails when // its bytes grow beyond baseline * 1.01, or its CSS bytes beyond the baseline // plus the larger of 5% and 32 bytes. A change that grows a size on purpose // updates benchmark-baseline.json in the same pull request. export const MIN_TIME_TOLERANCE = 0.1 export const SPREAD_FACTOR = 2 export const BYTES_TOLERANCE = 0.01 export const CSS_TOLERANCE = 0.05 export const CSS_MIN_SLACK = 32 export const KEPT_CALIBRATION_RUNS = 10 export function median(values) { const sorted = values.toSorted((a, b) => a - b) const middle = Math.floor(sorted.length / 2) return sorted.length % 2 ? sorted[middle] : (sorted[middle - 1] + sorted[middle]) / 2 } /** The targets in the order of one round: the list, started ound steps of its length / rounds further on */ export function roundOrder(names, round, rounds) { const offset = Math.floor((names.length * round) / rounds) return [...names.slice(offset), ...names.slice(0, offset)] } /** (max - min) / median */ export function relativeSpread(values) { return (Math.max(...values) - Math.min(...values)) / median(values) } /** The time of a row as a ratio to the control of its group in the same run */ export function timeRatio(run, target, control) { return run.rows[target].median / run.rows[control].median } export function timeTolerance(calibration) { return Math.max( MIN_TIME_TOLERANCE, SPREAD_FACTOR * relativeSpread(calibration), ) } /** Checks the run against the baseline: one entry per gated row, and whether any failed */ export function evaluate(run, baseline) { const results = [] if (run.samples !== baseline.samples) results.push({ target: '(policy)', failures: [ `the run took ${run.samples} samples per row, the baseline was made with ${baseline.samples}`, ], }) for (const [target, expected] of Object.entries(baseline.rows)) { const row = run.rows[target] const failures = [] if (!row || !run.rows[expected.control]) { failures.push('missing from the run') } else { const ratio = timeRatio(run, target, expected.control) const limit = expected.ratio * (1 + expected.timeTolerance) if (ratio > limit) failures.push( `build time is ${ratio.toFixed(3)}x ${expected.control}, over the limit ${limit.toFixed(3)}x (baseline ${expected.ratio.toFixed(3)}x + ${(expected.timeTolerance * 100).toFixed(0)}%)`, ) const bytesLimit = Math.floor(expected.bytes * (1 + BYTES_TOLERANCE)) if (row.bytes > bytesLimit) failures.push( `output is ${row.bytes.toLocaleString()} bytes, over the limit ${bytesLimit.toLocaleString()} (baseline ${expected.bytes.toLocaleString()} + 1%)`, ) const cssLimit = expected.cssBytes + Math.max(Math.floor(expected.cssBytes * CSS_TOLERANCE), CSS_MIN_SLACK) if (row.cssBytes > cssLimit) failures.push( `CSS is ${row.cssBytes.toLocaleString()} bytes, over the limit ${cssLimit.toLocaleString()} (baseline ${expected.cssBytes.toLocaleString()})`, ) } results.push({ target, failures }) } return { results, failed: results.some((entry) => entry.failures.length > 0) } } /** The baseline for this run: the stored calibration runs plus this one, and what they say */ export function nextBaseline(run, manifest, previous) { const gated = manifest.targets.filter((target) => target.gate) const rows = {} for (const { name, group } of gated) { const control = manifest.controls[group] const calibration = [ ...(previous?.rows[name]?.calibration ?? []), timeRatio(run, name, control), ].slice(-KEPT_CALIBRATION_RUNS) rows[name] = { control, ratio: median(calibration), calibration, timeTolerance: timeTolerance(calibration), bytes: run.rows[name].bytes, cssBytes: run.rows[name].cssBytes, } } return { samples: run.samples, rows } } /** One line per gated row for the job log */ export function report(run, baseline, evaluation) { return evaluation.results .map(({ target, failures }) => { const expected = baseline.rows[target] const row = run.rows[target] const status = failures.length ? 'FAIL' : 'ok ' if (!expected || !row) return `${status} ${target}: ${failures.join('; ')}` const ratio = timeRatio(run, target, expected.control) return `${status} ${target}: ${row.median.toFixed(2)}s (${ratio.toFixed(3)}x ${expected.control}, baseline ${expected.ratio.toFixed(3)}x ±${(expected.timeTolerance * 100).toFixed(0)}%), ${row.bytes.toLocaleString()} bytes, css ${row.cssBytes.toLocaleString()} bytes${failures.length ? '\n ' + failures.join('\n ') : ''}` }) .join('\n') } \ No newline at end of file +// The regression rule behind `benchmark-check.js`, kept free of file and process +// access so it can be tested. +// +// Only the Devup UI rows of benchmark-manifest.json (`gate: true`) can fail; +// every other library's row is informational. +// +// Build time is compared as a ratio to a control row measured in the same run +// (Tailwind on the same bundler), because the shared CI runner drifts by about +// 25% between runs while the ratio drifts by a few percent. A row fails when +// ratio > baseline ratio * (1 + timeTolerance) +// where the tolerance is at least 10% and at least twice the spread the ratio +// showed over the calibration runs stored with the baseline, so ordinary noise +// stays well inside it and a real slowdown does not. +// +// Output sizes are deterministic to about 0.05% between runs. A row fails when +// its bytes grow beyond baseline * 1.01, or its CSS bytes beyond the baseline +// plus the larger of 5% and 32 bytes. A change that grows a size on purpose +// updates benchmark-baseline.json in the same pull request. + +export const MIN_TIME_TOLERANCE = 0.1 +export const SPREAD_FACTOR = 2 +export const BYTES_TOLERANCE = 0.01 +export const CSS_TOLERANCE = 0.05 +export const CSS_MIN_SLACK = 32 +export const KEPT_CALIBRATION_RUNS = 10 + +export function median(values) { + const sorted = values.toSorted((a, b) => a - b) + const middle = Math.floor(sorted.length / 2) + return sorted.length % 2 + ? sorted[middle] + : (sorted[middle - 1] + sorted[middle]) / 2 +} + +/** The targets in the order of one round: the list, started +ound steps of its length / rounds further on */ +export function roundOrder(names, round, rounds) { + const offset = Math.floor((names.length * round) / rounds) + return [...names.slice(offset), ...names.slice(0, offset)] +} + +/** (max - min) / median */ +export function relativeSpread(values) { + return (Math.max(...values) - Math.min(...values)) / median(values) +} + +/** The time of a row as a ratio to the control of its group in the same run */ +export function timeRatio(run, target, control) { + return run.rows[target].median / run.rows[control].median +} + +export function timeTolerance(calibration) { + return Math.max( + MIN_TIME_TOLERANCE, + SPREAD_FACTOR * relativeSpread(calibration), + ) +} + +/** Checks the run against the baseline: one entry per gated row, and whether any failed */ +export function evaluate(run, baseline) { + const results = [] + if (run.samples !== baseline.samples) + results.push({ + target: '(policy)', + failures: [ + `the run took ${run.samples} samples per row, the baseline was made with ${baseline.samples}`, + ], + }) + for (const [target, expected] of Object.entries(baseline.rows)) { + const row = run.rows[target] + const failures = [] + if (!row || !run.rows[expected.control]) { + failures.push('missing from the run') + } else { + const ratio = timeRatio(run, target, expected.control) + const limit = expected.ratio * (1 + expected.timeTolerance) + if (ratio > limit) + failures.push( + `build time is ${ratio.toFixed(3)}x ${expected.control}, over the limit ${limit.toFixed(3)}x (baseline ${expected.ratio.toFixed(3)}x + ${(expected.timeTolerance * 100).toFixed(0)}%)`, + ) + const bytesLimit = Math.floor(expected.bytes * (1 + BYTES_TOLERANCE)) + if (row.bytes > bytesLimit) + failures.push( + `output is ${row.bytes.toLocaleString()} bytes, over the limit ${bytesLimit.toLocaleString()} (baseline ${expected.bytes.toLocaleString()} + 1%)`, + ) + const cssLimit = + expected.cssBytes + + Math.max(Math.floor(expected.cssBytes * CSS_TOLERANCE), CSS_MIN_SLACK) + if (row.cssBytes > cssLimit) + failures.push( + `CSS is ${row.cssBytes.toLocaleString()} bytes, over the limit ${cssLimit.toLocaleString()} (baseline ${expected.cssBytes.toLocaleString()})`, + ) + } + results.push({ target, failures }) + } + return { results, failed: results.some((entry) => entry.failures.length > 0) } +} + +/** The baseline for this run: the stored calibration runs plus this one, and what they say */ +export function nextBaseline(run, manifest, previous) { + const gated = manifest.targets.filter((target) => target.gate) + const rows = {} + for (const { name, group } of gated) { + const control = manifest.controls[group] + const calibration = [ + ...(previous?.rows[name]?.calibration ?? []), + timeRatio(run, name, control), + ].slice(-KEPT_CALIBRATION_RUNS) + rows[name] = { + control, + ratio: median(calibration), + calibration, + timeTolerance: timeTolerance(calibration), + bytes: run.rows[name].bytes, + cssBytes: run.rows[name].cssBytes, + } + } + return { samples: run.samples, rows } +} + +/** Replace only the existing per-file Devup UI CSS baseline; never recalibrate. */ +export function nextCssBaseline(run, previous) { + const expected = previous.rows['devup-ui'] + if (!expected) + throw new TypeError('CSS-only baseline requires an existing devup-ui row') + return { + ...previous, + rows: { + ...previous.rows, + 'devup-ui': { ...expected, cssBytes: run.rows['devup-ui'].cssBytes }, + }, + } +} + +/** One line per gated row for the job log */ +export function report(run, baseline, evaluation) { + return evaluation.results + .map(({ target, failures }) => { + const expected = baseline.rows[target] + const row = run.rows[target] + const status = failures.length ? 'FAIL' : 'ok ' + if (!expected || !row) + return `${status} ${target}: ${failures.join('; ')}` + const ratio = timeRatio(run, target, expected.control) + return `${status} ${target}: ${row.median.toFixed(2)}s (${ratio.toFixed(3)}x ${expected.control}, baseline ${expected.ratio.toFixed(3)}x ±${(expected.timeTolerance * 100).toFixed(0)}%), ${row.bytes.toLocaleString()} bytes, css ${row.cssBytes.toLocaleString()} bytes${failures.length ? '\n ' + failures.join('\n ') : ''}` + }) + .join('\n') +} diff --git a/benchmark-output.js b/benchmark-output.js new file mode 100644 index 000000000..efc9c28f2 --- /dev/null +++ b/benchmark-output.js @@ -0,0 +1,30 @@ +// Measure the fixture layouts actually emitted by Next.js and vinext. +import { existsSync, readdirSync, statSync } from 'node:fs' +import { join } from 'node:path' + +/** Sum all files, or only CSS files, recursively. Missing directories throw. */ +function directoryBytes(directory, cssOnly = false) { + let bytes = 0 + for (const entry of readdirSync(directory)) { + const path = join(directory, entry) + const stat = statSync(path) + if (stat.isDirectory()) bytes += directoryBytes(path, cssOnly) + else if (!cssOnly || path.endsWith('.css')) bytes += stat.size + } + return bytes +} + +/** Whole-build bytes include caches/server output; CSS bytes are shipped assets only. */ +export function measureOutput(benchmarkDir) { + // vinext also leaves a vestigial .next stub: keep preferring dist. + const distDir = join(benchmarkDir, 'dist') + const hasDist = existsSync(distDir) + const outputDir = hasDist ? distDir : join(benchmarkDir, '.next') + const browserDir = join(outputDir, hasDist ? 'client' : 'static') + return { + bytes: directoryBytes(outputDir), + // Next Webpack uses static/css; Turbopack uses static/chunks. + // An existing empty browser directory counts zero; a missing one throws. + cssBytes: directoryBytes(browserDir, true), + } +} diff --git a/benchmark.js b/benchmark.js index 6c9005dbc..27c2228b8 100644 --- a/benchmark.js +++ b/benchmark.js @@ -9,19 +9,13 @@ // The machine-readable run goes to benchmark-run.json: `benchmark-check.js` // compares it with the checked baseline, and the README tables are written from // the checked results (benchmark-results.json, render-benchmark-readme.js). -import { - existsSync, - readdirSync, - readFileSync, - rmSync, - statSync, - writeFileSync, -} from 'node:fs' +import { existsSync, readFileSync, rmSync, writeFileSync } from 'node:fs' import { join } from 'node:path' import { execSync } from 'child_process' import { median, roundOrder } from './benchmark-gate.js' +import { measureOutput } from './benchmark-output.js' const manifest = JSON.parse(readFileSync('benchmark-manifest.json', 'utf8')) @@ -33,31 +27,6 @@ function clearBuildFile(dir) { } } -function checkDirSize(path, filter) { - let totalSize = 0 - - function calculateSize(directory) { - const entries = readdirSync(directory) - for (const entry of entries) { - const entryPath = join(directory, entry) - if (statSync(entryPath).isDirectory()) { - calculateSize(entryPath) - } else if (!filter || filter(entryPath)) { - totalSize += statSync(entryPath).size - } - } - } - - calculateSize(path) - return totalSize -} - -// Sum only the size of emitted CSS files. Build-size totals are dominated by -// JS/assets and hide CSS-only differences (e.g. single-importer collapse). -function checkCssSize(path) { - return checkDirSize(path, (p) => p.endsWith('.css')) -} - let benchmarkRun = 0 function benchmark(target) { @@ -77,18 +46,9 @@ function benchmark(target) { performance.measure(run, run + '-start', run + '-end') const benchmarkDir = join('./benchmark', dir) - // Resolve the real build-output dir. Next.js emits to `.next`; Vite emits to - // `dist`. vinext (Next-on-Vite) emits its real artifacts to `dist` but ALSO - // leaves a tiny vestigial `.next` stub (~988 B, no CSS) - so checking `.next` - // first measured the empty stub and reported "988 bytes (css 0 bytes)" even - // though dist held ~1.28 MB incl. the extracted CSS. Prefer `dist` when it - // exists; fall back to `.next` for pure Next.js apps (which never emit dist). - const distDir = join(benchmarkDir, 'dist') - const outputDir = existsSync(distDir) ? distDir : join(benchmarkDir, '.next') return { seconds: performance.getEntriesByName(run)[0].duration / 1000, - bytes: checkDirSize(outputDir), - cssBytes: checkCssSize(outputDir), + ...measureOutput(benchmarkDir), } } diff --git a/packages/plugin-utils/src/__tests__/benchmark-css-baseline.test.ts b/packages/plugin-utils/src/__tests__/benchmark-css-baseline.test.ts new file mode 100644 index 000000000..d35831ac4 --- /dev/null +++ b/packages/plugin-utils/src/__tests__/benchmark-css-baseline.test.ts @@ -0,0 +1,135 @@ +import { spawnSync } from 'node:child_process' +import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' + +import { describe, expect, it } from 'bun:test' + +import storedBaseline from '../../../../benchmark-baseline.json' +import { nextCssBaseline } from '../../../../benchmark-gate.js' + +const baseline = { + ...storedBaseline, + revision: 'baseline-only', + rows: { + ...storedBaseline.rows, + 'devup-ui': { ...storedBaseline.rows['devup-ui'], cssBytes: 562 }, + }, +} + +// Deliberately disagree with every stored field, including the history. +const run = { + revision: 'run-only', + samples: 9, + rows: Object.fromEntries( + Object.entries(baseline.rows).map(([target, row]) => [ + target, + { + control: 'different-control', + ratio: row.ratio + 1, + calibration: [9], + timeTolerance: row.timeTolerance + 1, + bytes: row.bytes + 123, + cssBytes: target === 'devup-ui' ? 494 : row.cssBytes + 456, + samples: [99], + median: 99, + min: 98, + max: 100, + }, + ]), + ), +} +const expected = { + ...baseline, + rows: { + ...baseline.rows, + 'devup-ui': { ...baseline.rows['devup-ui'], cssBytes: 494 }, + }, +} + +describe('CSS-only benchmark baseline', () => { + it('preserves every other field when the run disagrees with the baseline', () => { + const previous = structuredClone(baseline) + const input = structuredClone(run) + + const result = nextCssBaseline(input, previous) + + expect(result).toEqual(expected) + expect(previous).toEqual(baseline) + expect(input).toEqual(run) + }) + + it('preserves the baseline when CSS bytes are unchanged', () => { + const input = { + ...run, + rows: { + ...run.rows, + 'devup-ui': { cssBytes: baseline.rows['devup-ui'].cssBytes }, + }, + } + + const result = nextCssBaseline(input, baseline) + + expect(result).toEqual(baseline) + }) + + it('rejects a baseline when the existing per-file row is absent', () => { + const previous = { samples: 3, rows: {} } + + expect(() => nextCssBaseline(run, previous)).toThrow(TypeError) + }) + + it('rejects a run when the per-file measurement is absent', () => { + const input = { samples: 3, rows: {} } + + expect(() => nextCssBaseline(input, baseline)).toThrow(TypeError) + }) +}) + +describe('CSS-only baseline CLI', () => { + it.each([ + { args: ['--write-css-baseline', 'devup-ui'], success: true }, + { args: ['--write-css-baseline', 'devup-ui-turbo'], success: false }, + { args: ['--write-css-baseline'], success: false }, + { + args: ['--write-css-baseline', 'devup-ui', '--write-baseline'], + success: false, + }, + { + args: ['--write-css-baseline', 'devup-ui', '--write-results'], + success: false, + }, + ])( + 'writes only per-file CSS when invoked with $args', + ({ args, success }) => { + const directory = mkdtempSync(join(tmpdir(), 'devup-css-baseline-')) + const stored = JSON.stringify(baseline, null, 2) + '\n' + try { + writeFileSync(join(directory, 'benchmark-baseline.json'), stored) + writeFileSync(join(directory, 'benchmark-manifest.json'), '{}\n') + writeFileSync( + join(directory, 'benchmark-run.json'), + JSON.stringify(run), + ) + writeFileSync(join(directory, 'benchmark-results.json'), '{}\n') + + const result = spawnSync( + process.execPath, + [join(import.meta.dir, '../../../../benchmark-check.js'), ...args], + { cwd: directory, encoding: 'utf8', timeout: 10_000 }, + ) + + expect(result.error).toBeUndefined() + expect(result.status).toBe(success ? 0 : 1) + expect( + readFileSync(join(directory, 'benchmark-baseline.json'), 'utf8'), + ).toBe(success ? JSON.stringify(expected, null, 2) + '\n' : stored) + expect( + readFileSync(join(directory, 'benchmark-results.json'), 'utf8'), + ).toBe('{}\n') + } finally { + rmSync(directory, { recursive: true, force: true }) + } + }, + ) +}) diff --git a/packages/plugin-utils/src/__tests__/benchmark-output.test.ts b/packages/plugin-utils/src/__tests__/benchmark-output.test.ts new file mode 100644 index 000000000..a7193eaa9 --- /dev/null +++ b/packages/plugin-utils/src/__tests__/benchmark-output.test.ts @@ -0,0 +1,140 @@ +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { dirname, join } from 'node:path' + +import { afterEach, beforeEach, describe, expect, it } from 'bun:test' + +import { evaluate } from '../../../../benchmark-gate.js' +import { measureOutput } from '../../../../benchmark-output.js' + +describe('benchmark output measurement', () => { + let directory: string + + beforeEach(() => { + directory = mkdtempSync(join(tmpdir(), 'devup-benchmark-output-')) + }) + + afterEach(() => { + rmSync(directory, { recursive: true, force: true }) + }) + + function file(path: string, bytes: number) { + const destination = join(directory, path) + mkdirSync(dirname(destination), { recursive: true }) + writeFileSync(destination, Buffer.alloc(bytes, 'x')) + } + + it.each(['static/css/page.css', 'static/chunks/nested/page.css'])( + 'counts only shipped CSS when Next emits %s alongside compiler caches', + (path) => { + file('.next/' + path, 494) + file('.next/cache/devup-ui_build/page.css', 450) + file('.next/cache/devup-ui_build/base.css', 68) + file('.next/server/app/page.css', 83) + file('.next/static/chunks/app.js', 101) + + const result = measureOutput(directory) + + expect(result).toEqual({ bytes: 1196, cssBytes: 494 }) + }, + ) + + it('prefers dist and counts client CSS when vinext also emits a Next stub', () => { + file('.next/static/css/stub.css', 999) + file('dist/client/assets/page.css', 31) + file('dist/client/assets/nested/shared.css', 17) + file('dist/client/assets/app.js', 101) + file('dist/server/assets/page.css', 67) + file('dist/server/app.js', 83) + + const result = measureOutput(directory) + + expect(result).toEqual({ bytes: 299, cssBytes: 48 }) + }) + + it('rejects measurement when neither output directory exists', () => { + expect(() => measureOutput(directory)).toThrow() + }) + + it.each(['.next', 'dist'])( + 'rejects measurement when %s has no required browser-assets directory', + (output) => { + file(output + '/server/page.css', 123) + + expect(() => measureOutput(directory)).toThrow() + }, + ) + + it('does not fall back to Next when the preferred dist lacks client assets', () => { + file('.next/static/css/page.css', 494) + file('dist/server/page.css', 123) + + expect(() => measureOutput(directory)).toThrow() + }) + + it.each(['.next/static', 'dist/client'])( + 'counts zero CSS when the existing browser-assets directory %s is empty', + (browser) => { + mkdirSync(join(directory, browser), { recursive: true }) + + const result = measureOutput(directory) + + expect(result).toEqual({ bytes: 0, cssBytes: 0 }) + }, + ) + + it.each(['.next/static', 'dist/client'])( + 'counts zero CSS when %s contains only non-CSS assets', + (browser) => { + file(browser + '/assets/app.js', 41) + file(browser + '/assets/page.css.map', 23) + + const result = measureOutput(directory) + + expect(result).toEqual({ bytes: 64, cssBytes: 0 }) + }, + ) + + it.each([ + { cssBytes: 494, failed: false }, + { cssBytes: 526, failed: false }, + { cssBytes: 527, failed: true }, + ])( + 'enforces shipped CSS headroom when the asset is $cssBytes bytes', + ({ cssBytes, failed }) => { + file('.next/static/css/page.css', cssBytes) + file('.next/cache/devup-ui_build/page.css', 450) + const baseline = { + samples: 3, + rows: { + 'devup-ui': { + control: 'tailwind', + ratio: 1, + timeTolerance: 0.1, + bytes: 1000, + cssBytes: 494, + }, + }, + } + const run = { + samples: 3, + rows: { + tailwind: { median: 1 }, + 'devup-ui': { median: 1, ...measureOutput(directory) }, + }, + } + + const evaluation = evaluate(run, baseline) + + expect(evaluation.failed).toBe(failed) + expect(evaluation.results).toEqual([ + { + target: 'devup-ui', + failures: failed + ? ['CSS is 527 bytes, over the limit 526 (baseline 494)'] + : [], + }, + ]) + }, + ) +}) diff --git a/render-benchmark-readme.js b/render-benchmark-readme.js index b45a0a288..4d6786f76 100644 --- a/render-benchmark-readme.js +++ b/render-benchmark-readme.js @@ -24,7 +24,7 @@ const TEXT = { intro: ({ run, samples }) => `[Latest CI benchmark](${run.url}) (commit \`${run.commit}\`, Devup UI ${run.devupUi}) on \`${run.runner}\` with Next.js ${run.next}. Every row is the median of ${samples} cold builds that ran in rotating order, shown with the range of the samples. All Next.js builds use the native TypeScript 7 CLI for type checking. The English and Korean tables are generated from the same checked result file, \`benchmark-results.json\`.`, sizes: - 'Build Size is every byte of the build output directory (`.next`, or `dist` for vinext); CSS Size is the emitted `.css` files alone, measured separately.', + "Build Size is every byte of the build output directory (`.next`, or `dist` for vinext), including caches and server output. The checked table retains its original historical CSS measurements, which recursively counted `.css` files throughout that output. New runs count only browser-shipped CSS: `.next/static/**/*.css` (Webpack `static/css` and Turbopack `static/chunks`) or vinext `dist/client/**/*.css`, excluding caches and server CSS. A missing browser-assets directory is an error; an existing directory with no CSS counts zero. For an approved per-file CSS measurement correction, `bun benchmark-check.js --write-css-baseline devup-ui` updates only that existing row's CSS bytes; unlike `--write-baseline`, it preserves sample count, build bytes, ratios, tolerances, all other rows and calibration history.", webpack: 'Webpack:', turbopack: 'Turbopack:', columns: { @@ -49,7 +49,7 @@ const TEXT = { intro: ({ run, samples }) => `[최신 CI 벤치마크](${run.url}) (커밋 \`${run.commit}\`, Devup UI ${run.devupUi})는 \`${run.runner}\`에서 Next.js ${run.next}로 실행했습니다. 모든 행은 순서를 돌려 가며 측정한 cold build ${samples}회의 중앙값이며 샘플의 범위를 함께 보여줍니다. 모든 Next.js 빌드는 네이티브 TypeScript 7 CLI로 타입 검사를 수행합니다. 영어와 한국어 표는 같은 결과 파일 \`benchmark-results.json\`에서 생성합니다.`, sizes: - '빌드 사이즈는 빌드 출력 디렉터리(`.next`, vinext는 `dist`)의 모든 바이트이고, CSS 사이즈는 출력된 `.css` 파일만 따로 잰 값입니다.', + '빌드 사이즈는 캐시와 서버 출력을 포함한 빌드 출력 디렉터리(`.next`, vinext는 `dist`)의 모든 바이트입니다. 체크인된 표는 출력 전체의 `.css` 파일을 재귀적으로 합산한 기존 CSS 측정값을 그대로 보존합니다. 새 실행은 브라우저에 제공되는 CSS만 측정합니다: `.next/static/**/*.css`(Webpack `static/css`, Turbopack `static/chunks`) 또는 vinext `dist/client/**/*.css`. 캐시와 서버 CSS는 제외합니다. 브라우저 에셋 디렉터리가 없으면 오류이며, 존재하지만 CSS가 없으면 0입니다. 승인된 per-file CSS 측정 보정에는 `bun benchmark-check.js --write-css-baseline devup-ui`를 사용합니다. `--write-baseline`과 달리 기존 해당 행의 CSS 바이트만 바꾸고 샘플 수, 빌드 바이트, 비율, 허용 오차, 다른 모든 행과 보정 이력을 보존합니다.', webpack: 'Webpack:', turbopack: 'Turbopack:', columns: { From c2c988defb797cef798a7eea71e721ca31705bd0 Mon Sep 17 00:00:00 2001 From: owjs3901 Date: Thu, 15 Oct 2026 10:28:55 +0900 Subject: [PATCH 6/9] ci(wasm): align size budget with attributed parent growth MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Refs #683 2026-10-15: main의 #743 결정론적 ABI·상태 초기화, #716 Emotion 처리와 #734/#739 Tailwind 확장 및 #744에 통합된 styled 캡처·렌더링 수정으로 발생한 WASM 증가를 Linux run 38655364885 측정으로 기존 writer를 실행해 반영했습니다. #744와 #746 바이너리는 동일하며 #746 자체의 제품 코드 증가는 0입니다. glue gzip은 기존 7168바이트 예산에서 이미 통과하므로 원래 glue 예산을 그대로 보존했고 WASM 두 필드만 변경했습니다. Ultraworked with [Sisyphus](https://github.com/code-yeongyu/oh-my-openagent) Co-authored-by: Sisyphus --- bindings/devup-ui-wasm/size-budget.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bindings/devup-ui-wasm/size-budget.json b/bindings/devup-ui-wasm/size-budget.json index a0a9703c6..965109740 100644 --- a/bindings/devup-ui-wasm/size-budget.json +++ b/bindings/devup-ui-wasm/size-budget.json @@ -1,7 +1,7 @@ { "wasm": { - "raw": 5479424, - "gzip": 1970176 + "raw": 5697536, + "gzip": 2046976 }, "glue": { "raw": 41984, From 6ef98ddbbdee1873c990887aac01abc03a86c738 Mon Sep 17 00:00:00 2001 From: owjs3901 Date: Thu, 15 Oct 2026 11:41:51 +0900 Subject: [PATCH 7/9] fix(benchmark): tighten single and vinext CSS baselines MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Refs #683 2026-10-15: 이전 #746 fac1d7d4의 새 WASM/root 빌드로 측정한 실제 제공 CSS는 devup-ui-single 395바이트(기존 790에는 캐시 사본 포함), vinext-devup-ui 14873바이트(기존 29746에는 서버 사본 포함)입니다. 같은 이름 제한 writer로 이 두 행의 CSS 기준만 보정해 기존 공식의 한도 427/15616을 적용했고, 다른 모든 필드·시간 샘플·전체 빌드 크기 및 7회 보정 이력을 바이트 단위로 보존했습니다. Ultraworked with [Sisyphus](https://github.com/code-yeongyu/oh-my-openagent) Co-authored-by: Sisyphus --- README.md | 2 +- README_ko.md | 2 +- benchmark-baseline.json | 4 +- benchmark-check.js | 20 ++-- benchmark-gate.js | 30 +++-- .../__tests__/benchmark-css-baseline.test.ts | 106 +++++++++++++++++- .../src/__tests__/benchmark-output.test.ts | 57 ++++++++++ render-benchmark-readme.js | 4 +- 8 files changed, 195 insertions(+), 30 deletions(-) diff --git a/README.md b/README.md index 6f8abc8cd..eceb7305d 100644 --- a/README.md +++ b/README.md @@ -82,7 +82,7 @@ npm install @devup-ui/webpack-plugin [Latest CI benchmark](https://github.com/dev-five-git/devup-ui/actions/runs/37758611477) (commit `467a379a`, Devup UI 1.0.44) on `ubuntu-24.04` with Next.js 16.3.3. Every row is the median of 3 cold builds that ran in rotating order, shown with the range of the samples. All Next.js builds use the native TypeScript 7 CLI for type checking. The English and Korean tables are generated from the same checked result file, `benchmark-results.json`. -Build Size is every byte of the build output directory (`.next`, or `dist` for vinext), including caches and server output. The checked table retains its original historical CSS measurements, which recursively counted `.css` files throughout that output. New runs count only browser-shipped CSS: `.next/static/**/*.css` (Webpack `static/css` and Turbopack `static/chunks`) or vinext `dist/client/**/*.css`, excluding caches and server CSS. A missing browser-assets directory is an error; an existing directory with no CSS counts zero. For an approved per-file CSS measurement correction, `bun benchmark-check.js --write-css-baseline devup-ui` updates only that existing row's CSS bytes; unlike `--write-baseline`, it preserves sample count, build bytes, ratios, tolerances, all other rows and calibration history. +Build Size is every byte of the build output directory (`.next`, or `dist` for vinext), including caches and server output. The checked table retains its original historical CSS measurements, which recursively counted `.css` files throughout that output. New runs count only browser-shipped CSS: `.next/static/**/*.css` (Webpack `static/css` and Turbopack `static/chunks`) or vinext `dist/client/**/*.css`, excluding caches and server CSS. A missing browser-assets directory is an error; an existing directory with no CSS counts zero. Approved CSS measurement corrections are limited to explicitly named `devup-ui`, `devup-ui-single` and `vinext-devup-ui` rows. `bun benchmark-check.js --write-css-baseline devup-ui` retains the per-file mode; `bun benchmark-check.js --write-css-baseline devup-ui-single,vinext-devup-ui` selects only the other two rows in one comma-separated argument. Only selected existing rows' CSS bytes change; unlike `--write-baseline`, this preserves sample count, build bytes, ratios, tolerances, all unselected rows and calibration history. Missing or unsupported names and conflicting writer modes are rejected. Webpack: diff --git a/README_ko.md b/README_ko.md index 0636d5e84..85d3d2a83 100644 --- a/README_ko.md +++ b/README_ko.md @@ -82,7 +82,7 @@ npm install @devup-ui/webpack-plugin [최신 CI 벤치마크](https://github.com/dev-five-git/devup-ui/actions/runs/37758611477) (커밋 `467a379a`, Devup UI 1.0.44)는 `ubuntu-24.04`에서 Next.js 16.3.3로 실행했습니다. 모든 행은 순서를 돌려 가며 측정한 cold build 3회의 중앙값이며 샘플의 범위를 함께 보여줍니다. 모든 Next.js 빌드는 네이티브 TypeScript 7 CLI로 타입 검사를 수행합니다. 영어와 한국어 표는 같은 결과 파일 `benchmark-results.json`에서 생성합니다. -빌드 사이즈는 캐시와 서버 출력을 포함한 빌드 출력 디렉터리(`.next`, vinext는 `dist`)의 모든 바이트입니다. 체크인된 표는 출력 전체의 `.css` 파일을 재귀적으로 합산한 기존 CSS 측정값을 그대로 보존합니다. 새 실행은 브라우저에 제공되는 CSS만 측정합니다: `.next/static/**/*.css`(Webpack `static/css`, Turbopack `static/chunks`) 또는 vinext `dist/client/**/*.css`. 캐시와 서버 CSS는 제외합니다. 브라우저 에셋 디렉터리가 없으면 오류이며, 존재하지만 CSS가 없으면 0입니다. 승인된 per-file CSS 측정 보정에는 `bun benchmark-check.js --write-css-baseline devup-ui`를 사용합니다. `--write-baseline`과 달리 기존 해당 행의 CSS 바이트만 바꾸고 샘플 수, 빌드 바이트, 비율, 허용 오차, 다른 모든 행과 보정 이력을 보존합니다. +빌드 사이즈는 캐시와 서버 출력을 포함한 빌드 출력 디렉터리(`.next`, vinext는 `dist`)의 모든 바이트입니다. 체크인된 표는 출력 전체의 `.css` 파일을 재귀적으로 합산한 기존 CSS 측정값을 그대로 보존합니다. 새 실행은 브라우저에 제공되는 CSS만 측정합니다: `.next/static/**/*.css`(Webpack `static/css`, Turbopack `static/chunks`) 또는 vinext `dist/client/**/*.css`. 캐시와 서버 CSS는 제외합니다. 브라우저 에셋 디렉터리가 없으면 오류이며, 존재하지만 CSS가 없으면 0입니다. 승인된 CSS 측정 보정은 명시적으로 지정한 `devup-ui`, `devup-ui-single`, `vinext-devup-ui` 행으로 제한됩니다. `bun benchmark-check.js --write-css-baseline devup-ui`는 기존 per-file 모드를 유지하며, `bun benchmark-check.js --write-css-baseline devup-ui-single,vinext-devup-ui`는 쉼표로 구분한 인자 하나로 나머지 두 행만 선택합니다. 선택한 기존 행의 CSS 바이트만 바꾸고, `--write-baseline`과 달리 샘플 수, 빌드 바이트, 비율, 허용 오차, 선택하지 않은 모든 행과 보정 이력을 보존합니다. 이름 누락, 지원하지 않는 이름, 다른 쓰기 모드와의 혼용은 오류입니다. Webpack: diff --git a/benchmark-baseline.json b/benchmark-baseline.json index ee0e0e799..ee9594b82 100644 --- a/benchmark-baseline.json +++ b/benchmark-baseline.json @@ -32,7 +32,7 @@ ], "timeTolerance": 0.12495002123706572, "bytes": 67436336, - "cssBytes": 790 + "cssBytes": 395 }, "devup-ui-single-turbo": { "control": "tailwind-turbo", @@ -87,7 +87,7 @@ ], "timeTolerance": 0.2405708460754333, "bytes": 1198750, - "cssBytes": 29746 + "cssBytes": 14873 }, "devup-ui-collapse": { "control": "tailwind-turbo", diff --git a/benchmark-check.js b/benchmark-check.js index 872c4dd3d..37d3897c6 100644 --- a/benchmark-check.js +++ b/benchmark-check.js @@ -3,7 +3,10 @@ // // bun benchmark-check.js compare, exit 1 on a regression // bun benchmark-check.js --write-baseline add this run to the calibration and rewrite benchmark-baseline.json -// bun benchmark-check.js --write-css-baseline devup-ui replace only the existing per-file CSS baseline; keep all calibration, time and build-size fields +// bun benchmark-check.js --write-css-baseline devup-ui replace only the existing per-file CSS baseline (original mode) +// bun benchmark-check.js --write-css-baseline devup-ui-single,vinext-devup-ui replace only these two CSS fields +// CSS-only targets are limited to devup-ui, devup-ui-single and vinext-devup-ui; +// keep all calibration, time and build-size fields, never select rows implicitly. // bun benchmark-check.js --write-results rewrite benchmark-results.json (the README tables) from this run // // A change that makes a build slower or larger on purpose runs the two writes @@ -27,17 +30,16 @@ const run = read('benchmark-run.json') if (process.argv.includes('--write-css-baseline')) { const target = process.argv[process.argv.indexOf('--write-css-baseline') + 1] - if ( - target !== 'devup-ui' || - process.argv.includes('--write-baseline') || - process.argv.includes('--write-results') - ) + if (!target || process.argv.length !== 4) throw new TypeError( - 'Use --write-css-baseline devup-ui without other writer modes', + 'Use --write-css-baseline with one comma-separated target argument and no other writer modes', ) const previous = read('benchmark-baseline.json') - write('benchmark-baseline.json', nextCssBaseline(run, previous)) - console.info('benchmark-baseline.json devup-ui CSS baseline rewritten') + write( + 'benchmark-baseline.json', + nextCssBaseline(run, previous, target.split(',')), + ) + console.info('benchmark-baseline.json named CSS baselines rewritten') } else if (process.argv.includes('--write-baseline')) { let previous try { diff --git a/benchmark-gate.js b/benchmark-gate.js index ecd9604a5..07066a47f 100644 --- a/benchmark-gate.js +++ b/benchmark-gate.js @@ -118,18 +118,26 @@ export function nextBaseline(run, manifest, previous) { return { samples: run.samples, rows } } -/** Replace only the existing per-file Devup UI CSS baseline; never recalibrate. */ -export function nextCssBaseline(run, previous) { - const expected = previous.rows['devup-ui'] - if (!expected) - throw new TypeError('CSS-only baseline requires an existing devup-ui row') - return { - ...previous, - rows: { - ...previous.rows, - 'devup-ui': { ...expected, cssBytes: run.rows['devup-ui'].cssBytes }, - }, +/** Replace only named, approved CSS baselines; default to per-file, never recalibrate. */ +export function nextCssBaseline(run, previous, targets = ['devup-ui']) { + if ( + targets.length === 0 || + targets.some( + (target) => + !['devup-ui', 'devup-ui-single', 'vinext-devup-ui'].includes(target), + ) + ) + throw new TypeError('CSS-only baseline requires approved named targets') + const rows = { ...previous.rows } + for (const target of targets) { + const expected = previous.rows[target] + if (!expected) + throw new TypeError( + `CSS-only baseline requires an existing ${target} row`, + ) + rows[target] = { ...expected, cssBytes: run.rows[target].cssBytes } } + return { ...previous, rows } } /** One line per gated row for the job log */ diff --git a/packages/plugin-utils/src/__tests__/benchmark-css-baseline.test.ts b/packages/plugin-utils/src/__tests__/benchmark-css-baseline.test.ts index d35831ac4..a87ba6c4a 100644 --- a/packages/plugin-utils/src/__tests__/benchmark-css-baseline.test.ts +++ b/packages/plugin-utils/src/__tests__/benchmark-css-baseline.test.ts @@ -14,6 +14,14 @@ const baseline = { rows: { ...storedBaseline.rows, 'devup-ui': { ...storedBaseline.rows['devup-ui'], cssBytes: 562 }, + 'devup-ui-single': { + ...storedBaseline.rows['devup-ui-single'], + cssBytes: 790, + }, + 'vinext-devup-ui': { + ...storedBaseline.rows['vinext-devup-ui'], + cssBytes: 29746, + }, }, } @@ -46,8 +54,78 @@ const expected = { 'devup-ui': { ...baseline.rows['devup-ui'], cssBytes: 494 }, }, } +const pairBaseline = expected +const pairRun = { + ...run, + rows: { + ...run.rows, + 'devup-ui': { ...run.rows['devup-ui'], cssBytes: 617 }, + 'devup-ui-single': { ...run.rows['devup-ui-single'], cssBytes: 395 }, + 'vinext-devup-ui': { ...run.rows['vinext-devup-ui'], cssBytes: 14873 }, + }, +} +const pairExpected = { + ...pairBaseline, + rows: { + ...pairBaseline.rows, + 'devup-ui-single': { + ...pairBaseline.rows['devup-ui-single'], + cssBytes: 395, + }, + 'vinext-devup-ui': { + ...pairBaseline.rows['vinext-devup-ui'], + cssBytes: 14873, + }, + }, +} describe('CSS-only benchmark baseline', () => { + it('changes only the two named CSS fields when every run field disagrees', () => { + const previous = structuredClone(pairBaseline) + const input = structuredClone(pairRun) + + const result = nextCssBaseline(input, previous, [ + 'devup-ui-single', + 'vinext-devup-ui', + ]) + + expect(result).toEqual(pairExpected) + expect(previous).toEqual(pairBaseline) + expect(input).toEqual(pairRun) + }) + + it.each([ + { targets: [] }, + { targets: ['devup-ui-turbo'] }, + { targets: ['devup-ui-single', ''] }, + { targets: ['unknown'] }, + ])( + 'rejects unsupported or missing named targets when given $targets', + ({ targets }) => { + expect(() => nextCssBaseline(pairRun, pairBaseline, targets)).toThrow( + TypeError, + ) + }, + ) + + it.each(['devup-ui-single', 'vinext-devup-ui'])( + 'rejects missing selected rows when updating %s', + (target) => { + expect(() => nextCssBaseline(pairRun, { rows: {} }, [target])).toThrow( + TypeError, + ) + }, + ) + + it.each(['devup-ui-single', 'vinext-devup-ui'])( + 'rejects missing selected measurements when updating %s', + (target) => { + expect(() => + nextCssBaseline({ rows: {} }, pairBaseline, [target]), + ).toThrow(TypeError) + }, + ) + it('preserves every other field when the run disagrees with the baseline', () => { const previous = structuredClone(baseline) const input = structuredClone(run) @@ -89,7 +167,21 @@ describe('CSS-only benchmark baseline', () => { describe('CSS-only baseline CLI', () => { it.each([ { args: ['--write-css-baseline', 'devup-ui'], success: true }, + { + args: ['--write-css-baseline', 'devup-ui-single,vinext-devup-ui'], + success: true, + }, { args: ['--write-css-baseline', 'devup-ui-turbo'], success: false }, + { + args: ['--write-css-baseline', 'devup-ui-single,devup-ui-turbo'], + success: false, + }, + { args: ['--write-css-baseline', ''], success: false }, + { args: ['--write-css-baseline', 'devup-ui-single,'], success: false }, + { + args: ['--write-css-baseline', 'devup-ui-single', 'vinext-devup-ui'], + success: false, + }, { args: ['--write-css-baseline'], success: false }, { args: ['--write-css-baseline', 'devup-ui', '--write-baseline'], @@ -100,16 +192,18 @@ describe('CSS-only baseline CLI', () => { success: false, }, ])( - 'writes only per-file CSS when invoked with $args', + 'writes only explicitly named CSS when invoked with $args', ({ args, success }) => { const directory = mkdtempSync(join(tmpdir(), 'devup-css-baseline-')) - const stored = JSON.stringify(baseline, null, 2) + '\n' + const paired = args[1] === 'devup-ui-single,vinext-devup-ui' + const stored = + JSON.stringify(paired ? pairBaseline : baseline, null, 2) + '\n' try { writeFileSync(join(directory, 'benchmark-baseline.json'), stored) writeFileSync(join(directory, 'benchmark-manifest.json'), '{}\n') writeFileSync( join(directory, 'benchmark-run.json'), - JSON.stringify(run), + JSON.stringify(paired ? pairRun : run), ) writeFileSync(join(directory, 'benchmark-results.json'), '{}\n') @@ -123,7 +217,11 @@ describe('CSS-only baseline CLI', () => { expect(result.status).toBe(success ? 0 : 1) expect( readFileSync(join(directory, 'benchmark-baseline.json'), 'utf8'), - ).toBe(success ? JSON.stringify(expected, null, 2) + '\n' : stored) + ).toBe( + success + ? JSON.stringify(paired ? pairExpected : expected, null, 2) + '\n' + : stored, + ) expect( readFileSync(join(directory, 'benchmark-results.json'), 'utf8'), ).toBe('{}\n') diff --git a/packages/plugin-utils/src/__tests__/benchmark-output.test.ts b/packages/plugin-utils/src/__tests__/benchmark-output.test.ts index a7193eaa9..d6726e9e0 100644 --- a/packages/plugin-utils/src/__tests__/benchmark-output.test.ts +++ b/packages/plugin-utils/src/__tests__/benchmark-output.test.ts @@ -138,3 +138,60 @@ describe('benchmark output measurement', () => { }, ) }) + +describe('approved CSS baseline headroom', () => { + it.each([ + { + target: 'devup-ui-single', + baselineCss: 395, + cssBytes: 427, + failed: false, + }, + { + target: 'devup-ui-single', + baselineCss: 395, + cssBytes: 428, + failed: true, + }, + { + target: 'vinext-devup-ui', + baselineCss: 14873, + cssBytes: 15616, + failed: false, + }, + { + target: 'vinext-devup-ui', + baselineCss: 14873, + cssBytes: 15617, + failed: true, + }, + ])( + 'enforces the unchanged CSS gate for $target at $cssBytes bytes', + ({ target, baselineCss, cssBytes, failed }) => { + const baseline = { + samples: 3, + rows: { + [target]: { + control: 'tailwind', + ratio: 1, + timeTolerance: 0.1, + bytes: 1000, + cssBytes: baselineCss, + }, + }, + } + const run = { + samples: 3, + rows: { + tailwind: { median: 1 }, + [target]: { median: 1, bytes: 1000, cssBytes }, + }, + } + + const evaluation = evaluate(run, baseline) + + expect(evaluation.failed).toBe(failed) + expect(evaluation.results[0]?.failures).toHaveLength(failed ? 1 : 0) + }, + ) +}) diff --git a/render-benchmark-readme.js b/render-benchmark-readme.js index 4d6786f76..09fbdabb1 100644 --- a/render-benchmark-readme.js +++ b/render-benchmark-readme.js @@ -24,7 +24,7 @@ const TEXT = { intro: ({ run, samples }) => `[Latest CI benchmark](${run.url}) (commit \`${run.commit}\`, Devup UI ${run.devupUi}) on \`${run.runner}\` with Next.js ${run.next}. Every row is the median of ${samples} cold builds that ran in rotating order, shown with the range of the samples. All Next.js builds use the native TypeScript 7 CLI for type checking. The English and Korean tables are generated from the same checked result file, \`benchmark-results.json\`.`, sizes: - "Build Size is every byte of the build output directory (`.next`, or `dist` for vinext), including caches and server output. The checked table retains its original historical CSS measurements, which recursively counted `.css` files throughout that output. New runs count only browser-shipped CSS: `.next/static/**/*.css` (Webpack `static/css` and Turbopack `static/chunks`) or vinext `dist/client/**/*.css`, excluding caches and server CSS. A missing browser-assets directory is an error; an existing directory with no CSS counts zero. For an approved per-file CSS measurement correction, `bun benchmark-check.js --write-css-baseline devup-ui` updates only that existing row's CSS bytes; unlike `--write-baseline`, it preserves sample count, build bytes, ratios, tolerances, all other rows and calibration history.", + "Build Size is every byte of the build output directory (`.next`, or `dist` for vinext), including caches and server output. The checked table retains its original historical CSS measurements, which recursively counted `.css` files throughout that output. New runs count only browser-shipped CSS: `.next/static/**/*.css` (Webpack `static/css` and Turbopack `static/chunks`) or vinext `dist/client/**/*.css`, excluding caches and server CSS. A missing browser-assets directory is an error; an existing directory with no CSS counts zero. Approved CSS measurement corrections are limited to explicitly named `devup-ui`, `devup-ui-single` and `vinext-devup-ui` rows. `bun benchmark-check.js --write-css-baseline devup-ui` retains the per-file mode; `bun benchmark-check.js --write-css-baseline devup-ui-single,vinext-devup-ui` selects only the other two rows in one comma-separated argument. Only selected existing rows' CSS bytes change; unlike `--write-baseline`, this preserves sample count, build bytes, ratios, tolerances, all unselected rows and calibration history. Missing or unsupported names and conflicting writer modes are rejected.", webpack: 'Webpack:', turbopack: 'Turbopack:', columns: { @@ -49,7 +49,7 @@ const TEXT = { intro: ({ run, samples }) => `[최신 CI 벤치마크](${run.url}) (커밋 \`${run.commit}\`, Devup UI ${run.devupUi})는 \`${run.runner}\`에서 Next.js ${run.next}로 실행했습니다. 모든 행은 순서를 돌려 가며 측정한 cold build ${samples}회의 중앙값이며 샘플의 범위를 함께 보여줍니다. 모든 Next.js 빌드는 네이티브 TypeScript 7 CLI로 타입 검사를 수행합니다. 영어와 한국어 표는 같은 결과 파일 \`benchmark-results.json\`에서 생성합니다.`, sizes: - '빌드 사이즈는 캐시와 서버 출력을 포함한 빌드 출력 디렉터리(`.next`, vinext는 `dist`)의 모든 바이트입니다. 체크인된 표는 출력 전체의 `.css` 파일을 재귀적으로 합산한 기존 CSS 측정값을 그대로 보존합니다. 새 실행은 브라우저에 제공되는 CSS만 측정합니다: `.next/static/**/*.css`(Webpack `static/css`, Turbopack `static/chunks`) 또는 vinext `dist/client/**/*.css`. 캐시와 서버 CSS는 제외합니다. 브라우저 에셋 디렉터리가 없으면 오류이며, 존재하지만 CSS가 없으면 0입니다. 승인된 per-file CSS 측정 보정에는 `bun benchmark-check.js --write-css-baseline devup-ui`를 사용합니다. `--write-baseline`과 달리 기존 해당 행의 CSS 바이트만 바꾸고 샘플 수, 빌드 바이트, 비율, 허용 오차, 다른 모든 행과 보정 이력을 보존합니다.', + '빌드 사이즈는 캐시와 서버 출력을 포함한 빌드 출력 디렉터리(`.next`, vinext는 `dist`)의 모든 바이트입니다. 체크인된 표는 출력 전체의 `.css` 파일을 재귀적으로 합산한 기존 CSS 측정값을 그대로 보존합니다. 새 실행은 브라우저에 제공되는 CSS만 측정합니다: `.next/static/**/*.css`(Webpack `static/css`, Turbopack `static/chunks`) 또는 vinext `dist/client/**/*.css`. 캐시와 서버 CSS는 제외합니다. 브라우저 에셋 디렉터리가 없으면 오류이며, 존재하지만 CSS가 없으면 0입니다. 승인된 CSS 측정 보정은 명시적으로 지정한 `devup-ui`, `devup-ui-single`, `vinext-devup-ui` 행으로 제한됩니다. `bun benchmark-check.js --write-css-baseline devup-ui`는 기존 per-file 모드를 유지하며, `bun benchmark-check.js --write-css-baseline devup-ui-single,vinext-devup-ui`는 쉼표로 구분한 인자 하나로 나머지 두 행만 선택합니다. 선택한 기존 행의 CSS 바이트만 바꾸고, `--write-baseline`과 달리 샘플 수, 빌드 바이트, 비율, 허용 오차, 선택하지 않은 모든 행과 보정 이력을 보존합니다. 이름 누락, 지원하지 않는 이름, 다른 쓰기 모드와의 혼용은 오류입니다.', webpack: 'Webpack:', turbopack: 'Turbopack:', columns: { From 1194ab36401da30343f4205f90913669ed9826b5 Mon Sep 17 00:00:00 2001 From: owjs3901 Date: Fri, 16 Oct 2026 19:34:10 +0900 Subject: [PATCH 8/9] ci(wasm): attribute size budget to updated main compiler MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Refs #683 main의 #759, #749, #714, #717 통합 증가량을 동일 조건 로컬 비교와 Linux 실측으로 확인했으며 #744와 #746 바이너리는 동일합니다. Ultraworked with [Sisyphus](https://github.com/code-yeongyu/oh-my-openagent) Co-authored-by: Sisyphus --- bindings/devup-ui-wasm/size-budget.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bindings/devup-ui-wasm/size-budget.json b/bindings/devup-ui-wasm/size-budget.json index 965109740..b8b4ae776 100644 --- a/bindings/devup-ui-wasm/size-budget.json +++ b/bindings/devup-ui-wasm/size-budget.json @@ -1,7 +1,7 @@ { "wasm": { - "raw": 5697536, - "gzip": 2046976 + "raw": 5779456, + "gzip": 2075648 }, "glue": { "raw": 41984, From 64dd95699b05c516f5b172e23648b46d73c0eb19 Mon Sep 17 00:00:00 2001 From: owjs3901 Date: Sun, 18 Oct 2026 22:36:24 +0900 Subject: [PATCH 9/9] ci(wasm): match bindgen tools to each committed lock Refs #683 Ultraworked with [Sisyphus](https://github.com/code-yeongyu/oh-my-openagent) Co-authored-by: Sisyphus --- .github/workflows/publish.yml | 16 +- .../__tests__/wasm-bindgen-version.test.ts | 137 +++++++++++++++++ .../src/__tests__/wasm-size-bindgen.test.ts | 142 ++++++++++++++++++ .../src/__tests__/wasm-size-fixture.ts | 56 +++++++ .../src/__tests__/wasm-size.test.ts | 61 +------- resolve-wasm-bindgen.js | 16 ++ wasm-bindgen-version.js | 21 +++ wasm-size.js | 23 ++- 8 files changed, 409 insertions(+), 63 deletions(-) create mode 100644 packages/plugin-utils/src/__tests__/wasm-bindgen-version.test.ts create mode 100644 packages/plugin-utils/src/__tests__/wasm-size-bindgen.test.ts create mode 100644 packages/plugin-utils/src/__tests__/wasm-size-fixture.ts create mode 100644 resolve-wasm-bindgen.js create mode 100644 wasm-bindgen-version.js diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 1006f26e2..ad3837116 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -129,18 +129,17 @@ jobs: ~/.cargo/registry/cache/ ~/.cargo/git/db/ key: cargo-wasm-size-${{ runner.os }}-${{ hashFiles('**/Cargo.lock') }} - - name: Install the latest shared binding and optimizer tools once + - name: Install the latest shared installer and optimizer once env: GH_TOKEN: ${{ github.token }} run: | set -euo pipefail curl -L --proto '=https' --tlsv1.2 -sSf https://raw.githubusercontent.com/cargo-bins/cargo-binstall/main/install-from-binstall-release.sh | bash - cargo binstall wasm-bindgen-cli --no-confirm --force mkdir -p "$RUNNER_TEMP/wasm-size-tools/binaryen" gh release download --repo WebAssembly/binaryen --pattern '*-x86_64-linux.tar.gz' --dir "$RUNNER_TEMP/wasm-size-tools" tar -xzf "$RUNNER_TEMP"/wasm-size-tools/*.tar.gz --strip-components=1 -C "$RUNNER_TEMP/wasm-size-tools/binaryen" echo "$RUNNER_TEMP/wasm-size-tools/binaryen/bin" >> "$GITHUB_PATH" - - name: Build the exact base and tested candidate with the same binaries + - name: Build exact sources with shared tools and lock-matched bindgen run: | set -euo pipefail export SIZE_ROOT="$RUNNER_TEMP/wasm-size" @@ -181,14 +180,23 @@ jobs: metadata[side] = { commit: run('git', ['rev-parse', `${sha}^{commit}`]), tree: run('git', ['rev-parse', `${sha}^{tree}`]), + wasmBindgenVersion: process.env.WASM_BINDGEN_VERSION, packageDir: join(process.env.SIZE_ROOT, side, 'bindings/devup-ui-wasm/pkg'), tools, } writeFileSync(file, JSON.stringify(metadata, null, 2) + '\n') NODE + shared_path="$PATH" for side in base candidate; do stage="$side" - node "$SIZE_ROOT/provenance.mjs" "$side" ( + WASM_BINDGEN_VERSION="$(bun resolve-wasm-bindgen.js "$SIZE_ROOT/$side/Cargo.lock")" + export WASM_BINDGEN_VERSION + bindgen_root="$SIZE_ROOT/abi-tools/$WASM_BINDGEN_VERSION" + if [[ ! -x "$bindgen_root/bin/wasm-bindgen" ]]; then + cargo binstall wasm-bindgen-cli --version "=$WASM_BINDGEN_VERSION" --root "$bindgen_root" --no-confirm --force + fi + export PATH="$bindgen_root/bin:$shared_path" + node "$SIZE_ROOT/provenance.mjs" "$side" cd "$SIZE_ROOT/$side/bindings/devup-ui-wasm" export CARGO_TARGET_DIR="$SIZE_ROOT/$side-target" cargo metadata --locked --format-version 1 > /dev/null diff --git a/packages/plugin-utils/src/__tests__/wasm-bindgen-version.test.ts b/packages/plugin-utils/src/__tests__/wasm-bindgen-version.test.ts new file mode 100644 index 000000000..c450b3da1 --- /dev/null +++ b/packages/plugin-utils/src/__tests__/wasm-bindgen-version.test.ts @@ -0,0 +1,137 @@ +import { spawnSync } from 'node:child_process' +import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { fileURLToPath } from 'node:url' + +import { afterEach, describe, expect, it } from 'bun:test' + +import { resolveWasmBindgenVersion } from '../../../../wasm-bindgen-version.js' + +const resolver = fileURLToPath( + new URL('../../../../resolve-wasm-bindgen.js', import.meta.url), +) +const committedLock = fileURLToPath( + new URL('../../../../Cargo.lock', import.meta.url), +) +const committedVersion = readFileSync(committedLock, 'utf8').match( + /\[\[package\]\]\r?\nname = "wasm-bindgen"\r?\nversion = "([^"]+)"/, +)?.[1] +const roots: string[] = [] +const packageEntry = (version: string) => + `[[package]]\nname = "wasm-bindgen"\nversion = ${JSON.stringify(version)}\n` + +afterEach(() => { + for (const root of roots.splice(0)) + rmSync(root, { recursive: true, force: true }) +}) + +describe('committed wasm-bindgen version resolution', () => { + it.each(['0.2.128', '0.2.129', '1.0.0-rc.1+build.42'])( + 'preserves the exact committed semver when it is %s', + (version) => { + const lock = `version = 4\n${packageEntry(version)}` + const result = resolveWasmBindgenVersion(lock) + expect(result).toBe(version) + }, + ) + + it('deduplicates equal versions when package sources differ', () => { + const lock = `${packageEntry('0.2.128')}source = "registry+fixture"\n${packageEntry('0.2.128')}source = "git+fixture"\n` + const result = resolveWasmBindgenVersion(lock) + expect(result).toBe('0.2.128') + }) + + it('ignores other packages and misleading comments when resolving', () => { + const lock = `# name = "wasm-bindgen"\n[[package]]\nname = "wasm-bindgen-cli"\nversion = "9.0.0"\n${packageEntry('0.2.128')}` + const result = resolveWasmBindgenVersion(lock) + expect(result).toBe('0.2.128') + }) + + it.each([ + { lock: '' }, + { lock: 'version = 4' }, + { lock: '[package]\nname = "wasm-bindgen"\nversion = "0.2.128"' }, + { lock: 'package = ["wasm-bindgen", 128]' }, + { lock: '[[package]]\nname = "other"\nversion = "0.2.128"' }, + { lock: '[[package]]\nname = "wasm-bindgen"' }, + { lock: '[[package]]\nname = "wasm-bindgen"\nversion = 128' }, + { lock: packageEntry('0.2.128') + packageEntry('0.2.129') }, + { lock: packageEntry('0.2.128') + packageEntry('invalid') }, + { lock: '[[package]\nname = "wasm-bindgen"' }, + { lock: '[[package]]\nname = "wasm-bindgen"\nversion = "unterminated' }, + ])('rejects invalid, missing or ambiguous lock packages %j', ({ lock }) => { + expect(() => resolveWasmBindgenVersion(lock)).toThrow() + }) + + it.each([ + '', + 'latest', + 'v0.2.128', + '^0.2.128', + '0.2', + '00.2.128', + '0.02.128', + '0.2.0128', + '0.2.128-01', + '0.2.128+', + '0.2.128-', + ' 0.2.128', + '0.2.128\n', + ])('rejects an invalid committed version when it is %j', (version) => { + const lock = packageEntry(version) + expect(() => resolveWasmBindgenVersion(lock)).toThrow() + }) + + it('resolves the real committed Cargo.lock when used as library input', () => { + const lock = readFileSync(committedLock, 'utf8') + const version = resolveWasmBindgenVersion(lock) + expect(committedVersion).toBeDefined() + expect(version).toBe(committedVersion) + }) +}) + +describe('wasm-bindgen resolver CLI', () => { + it('prints only the locked version when given the real Cargo.lock', () => { + const result = spawnSync(process.execPath, [resolver, committedLock], { + encoding: 'utf8', + }) + expect(result.status).toBe(0) + expect(committedVersion).toBeDefined() + expect(result.stdout).toBe(`${committedVersion}\n`) + expect(result.stderr).toBe('') + }) + + it.each([ + { args: [] }, + { args: ['Cargo.lock'] }, + { args: [committedLock, committedLock] }, + { args: [join(committedLock, 'absent')] }, + ])( + 'fails without version output when arguments are invalid %j', + ({ args }) => { + const result = spawnSync(process.execPath, [resolver, ...args], { + encoding: 'utf8', + }) + expect(result.status).toBe(1) + expect(result.stdout).toBe('') + expect(result.stderr).not.toBe('') + }, + ) + + it.each(['version = 4', '[[package]', packageEntry('bad')])( + 'fails without version output when the lock is invalid %j', + (lock) => { + const root = mkdtempSync(join(tmpdir(), 'wasm-bindgen-version-')) + roots.push(root) + const file = join(root, 'Cargo.lock') + writeFileSync(file, lock) + const result = spawnSync(process.execPath, [resolver, file], { + encoding: 'utf8', + }) + expect(result.status).toBe(1) + expect(result.stdout).toBe('') + expect(result.stderr).not.toBe('') + }, + ) +}) diff --git a/packages/plugin-utils/src/__tests__/wasm-size-bindgen.test.ts b/packages/plugin-utils/src/__tests__/wasm-size-bindgen.test.ts new file mode 100644 index 000000000..6e718e5d0 --- /dev/null +++ b/packages/plugin-utils/src/__tests__/wasm-size-bindgen.test.ts @@ -0,0 +1,142 @@ +import { readFileSync, rmSync, writeFileSync } from 'node:fs' +import { join } from 'node:path' + +import { afterEach, describe, expect, it } from 'bun:test' + +import { cli, validateMetadata } from '../../../../wasm-size.js' +import { sizeFixture } from './wasm-size-fixture' + +const roots: string[] = [] + +function fixture() { + const input = sizeFixture(roots) + input.metadata.base.tools.wasmBindgen = + 'wasm-bindgen 0.2.128\n/base/bin/wasm-bindgen\nsha256:base' + input.metadata.candidate.wasmBindgenVersion = '0.2.129' + input.metadata.candidate.tools.wasmBindgen = + 'wasm-bindgen 0.2.129\n/candidate/bin/wasm-bindgen\nsha256:candidate' + return input +} + +afterEach(() => { + for (const root of roots.splice(0)) + rmSync(root, { recursive: true, force: true }) +}) + +describe('per-tree bindgen ABI provenance', () => { + it('retains both CLI versions in passing reports when locked ABIs differ', () => { + const input = fixture() + const manifest = join(input.root, 'metadata.json') + const report = join(input.root, 'report.json') + const markdown = join(input.root, 'summary.md') + writeFileSync(manifest, JSON.stringify(input.metadata)) + const status = cli([ + '--base-pkg', + input.base, + '--candidate-pkg', + input.candidate, + '--metadata', + manifest, + '--report', + report, + '--summary', + markdown, + ]) + expect(status).toBe(0) + const result: unknown = JSON.parse(readFileSync(report, 'utf8')) + expect(result).toMatchObject({ passed: true, metadata: input.metadata }) + const summary = readFileSync(markdown, 'utf8') + expect(summary).toContain( + JSON.stringify(input.metadata.base.tools.wasmBindgen), + ) + expect(summary).toContain( + JSON.stringify(input.metadata.candidate.tools.wasmBindgen), + ) + }) + + it('accepts identical CLI records with matching equal ABIs and CRLF provenance', () => { + const input = fixture() + input.metadata.candidate.wasmBindgenVersion = '0.2.128' + input.metadata.base.tools.wasmBindgen = + 'wasm-bindgen 0.2.128\r\n/shared/bin/wasm-bindgen\r\nsha256:shared' + input.metadata.candidate.tools.wasmBindgen = + input.metadata.base.tools.wasmBindgen + const result = validateMetadata(input.metadata, input) + expect(result).toEqual(input.metadata) + }) + + it('rejects different CLI records when the committed ABI is equal', () => { + const input = fixture() + input.metadata.candidate.wasmBindgenVersion = '0.2.128' + input.metadata.candidate.tools.wasmBindgen = + 'wasm-bindgen 0.2.128\n/candidate/bin/wasm-bindgen\nsha256:candidate' + expect(() => validateMetadata(input.metadata, input)).toThrow( + 'tool mismatch: wasmBindgen', + ) + }) + + it.each(['base', 'candidate'] as const)( + 'rejects a CLI version inconsistent with the %s committed lock', + (side) => { + const input = fixture() + input.metadata[side].wasmBindgenVersion = '0.2.130' + expect(() => validateMetadata(input.metadata, input)).toThrow( + 'tool mismatch', + ) + }, + ) + + it.each(['', 'latest', '0.2', '0.2.129\n'])( + 'rejects invalid declared versions when given %j', + (version) => { + const input = fixture() + input.metadata.candidate.wasmBindgenVersion = version + expect(() => validateMetadata(input.metadata, input)).toThrow() + }, + ) + + it('rejects missing lock provenance when metadata uses the old schema', () => { + const input = fixture() + const { commit, tree, packageDir, tools } = input.metadata.candidate + const candidate = { commit, tree, packageDir, tools } + const metadata = { ...input.metadata, candidate } + expect(() => validateMetadata(metadata, input)).toThrow('keys') + }) + + it.each([ + '', + 'wasm-bindgen 0.2.128', + 'path\nwasm-bindgen 0.2.129', + 'wasm-bindgen 0.2.129 extra', + 'wasm-bindgen 0.2.1290', + ])( + 'rejects missing or wrong actual CLI version records when given %j', + (record) => { + const input = fixture() + input.metadata.candidate.tools.wasmBindgen = record + expect(() => validateMetadata(input.metadata, input)).toThrow() + }, + ) + + it.each(['rustc', 'wasmPack', 'wasmOpt', 'bun', 'node'] as const)( + 'still rejects %s mismatches when bindgen ABIs differ', + (tool) => { + const input = fixture() + input.metadata.candidate.tools[tool] = 'different binary' + expect(() => validateMetadata(input.metadata, input)).toThrow( + `tool mismatch: ${tool}`, + ) + }, + ) + + it.each(['rustc', 'wasmPack', 'wasmOpt', 'bun', 'node'] as const)( + 'still rejects blank %s provenance when bindgen ABIs differ', + (tool) => { + const input = fixture() + input.metadata.candidate.tools[tool] = ' ' + expect(() => validateMetadata(input.metadata, input)).toThrow( + `missing candidate tool ${tool}`, + ) + }, + ) +}) diff --git a/packages/plugin-utils/src/__tests__/wasm-size-fixture.ts b/packages/plugin-utils/src/__tests__/wasm-size-fixture.ts new file mode 100644 index 000000000..81386d306 --- /dev/null +++ b/packages/plugin-utils/src/__tests__/wasm-size-fixture.ts @@ -0,0 +1,56 @@ +import { mkdirSync, mkdtempSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' + +export function sizeFixture(roots: string[]) { + const root = mkdtempSync(join(tmpdir(), 'wasm-size-')) + roots.push(root) + const base = join(root, 'base') + const candidate = join(root, 'candidate') + for (const directory of [base, candidate]) { + mkdirSync(directory) + writeFileSync(join(directory, 'index_bg.wasm'), 'abc') + writeFileSync(join(directory, 'index.js'), 'export const answer = 42\n') + } + const tools = { + rustc: 'rustc fixture', + wasmPack: 'wasm-pack fixture', + wasmBindgen: 'wasm-bindgen 0.2.128', + wasmOpt: 'wasm-opt fixture', + bun: 'bun fixture', + node: 'node fixture', + } + const metadata = { + jobId: 'fixture-job', + buildCommand: 'fixture release build and postprocess', + base: { + commit: 'a'.repeat(40), + tree: 'b'.repeat(40), + packageDir: base, + wasmBindgenVersion: '0.2.128', + tools: { ...tools }, + }, + candidate: { + commit: 'c'.repeat(40), + tree: 'd'.repeat(40), + packageDir: candidate, + wasmBindgenVersion: '0.2.128', + tools: { ...tools }, + }, + } + const manifest = join(root, 'metadata.json') + writeFileSync(manifest, JSON.stringify(metadata)) + const report = join(root, 'report.json') + const markdown = join(root, 'summary.md') + const args = [ + '--base-pkg', + base, + '--candidate-pkg', + candidate, + '--metadata', + manifest, + '--report', + report, + ] + return { root, base, candidate, metadata, manifest, report, markdown, args } +} diff --git a/packages/plugin-utils/src/__tests__/wasm-size.test.ts b/packages/plugin-utils/src/__tests__/wasm-size.test.ts index 7a25ec37a..a0f1c871d 100644 --- a/packages/plugin-utils/src/__tests__/wasm-size.test.ts +++ b/packages/plugin-utils/src/__tests__/wasm-size.test.ts @@ -1,12 +1,5 @@ import { spawnSync } from 'node:child_process' -import { - mkdirSync, - mkdtempSync, - readFileSync, - rmSync, - writeFileSync, -} from 'node:fs' -import { tmpdir } from 'node:os' +import { readFileSync, rmSync, writeFileSync } from 'node:fs' import { join } from 'node:path' import { fileURLToPath } from 'node:url' import { gzipSync } from 'node:zlib' @@ -21,62 +14,14 @@ import { summary, validateMetadata, } from '../../../../wasm-size.js' +import { sizeFixture } from './wasm-size-fixture' const roots: string[] = [] const checker = fileURLToPath( new URL('../../../../verify-wasm-size.js', import.meta.url), ) -function fixture() { - const root = mkdtempSync(join(tmpdir(), 'wasm-size-')) - roots.push(root) - const base = join(root, 'base') - const candidate = join(root, 'candidate') - for (const directory of [base, candidate]) { - mkdirSync(directory) - writeFileSync(join(directory, 'index_bg.wasm'), 'abc') - writeFileSync(join(directory, 'index.js'), 'export const answer = 42\n') - } - const tools = { - rustc: 'rustc fixture', - wasmPack: 'wasm-pack fixture', - wasmBindgen: 'wasm-bindgen fixture', - wasmOpt: 'wasm-opt fixture', - bun: 'bun fixture', - node: 'node fixture', - } - const metadata = { - jobId: 'fixture-job', - buildCommand: 'fixture release build and postprocess', - base: { - commit: 'a'.repeat(40), - tree: 'b'.repeat(40), - packageDir: base, - tools: { ...tools }, - }, - candidate: { - commit: 'c'.repeat(40), - tree: 'd'.repeat(40), - packageDir: candidate, - tools: { ...tools }, - }, - } - const manifest = join(root, 'metadata.json') - writeFileSync(manifest, JSON.stringify(metadata)) - const report = join(root, 'report.json') - const markdown = join(root, 'summary.md') - const args = [ - '--base-pkg', - base, - '--candidate-pkg', - candidate, - '--metadata', - manifest, - '--report', - report, - ] - return { root, base, candidate, metadata, manifest, report, markdown, args } -} +const fixture = () => sizeFixture(roots) afterEach(() => { for (const root of roots.splice(0)) diff --git a/resolve-wasm-bindgen.js b/resolve-wasm-bindgen.js new file mode 100644 index 000000000..0fe219ca4 --- /dev/null +++ b/resolve-wasm-bindgen.js @@ -0,0 +1,16 @@ +// bun resolve-wasm-bindgen.js /absolute/path/to/Cargo.lock +import { readFileSync } from 'node:fs' +import { isAbsolute } from 'node:path' + +import { resolveWasmBindgenVersion } from './wasm-bindgen-version.js' + +try { + const args = process.argv.slice(2) + if (args.length !== 1 || !isAbsolute(args[0])) + throw new Error('expected exactly one absolute Cargo.lock path') + console.info(resolveWasmBindgenVersion(readFileSync(args[0], 'utf8'))) +} catch (error) { + if (!(error instanceof Error)) throw error + console.error('resolve-wasm-bindgen: failed', error.message) + process.exitCode = 1 +} diff --git a/wasm-bindgen-version.js b/wasm-bindgen-version.js new file mode 100644 index 000000000..47d4f0508 --- /dev/null +++ b/wasm-bindgen-version.js @@ -0,0 +1,21 @@ +export const wasmBindgenVersionPattern = + /^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(?:-(?:0|[1-9]\d*|\d*[A-Za-z-][0-9A-Za-z-]*)(?:\.(?:0|[1-9]\d*|\d*[A-Za-z-][0-9A-Za-z-]*))*)?(?:\+[0-9A-Za-z-]+(?:\.[0-9A-Za-z-]+)*)?(?![\s\S])/ + +export function resolveWasmBindgenVersion(lockText) { + const lock = globalThis.Bun.TOML.parse(lockText) + if (!Array.isArray(lock.package)) + throw new Error('resolve-wasm-bindgen: missing package array') + const versions = new Set() + for (const entry of lock.package) { + if (entry?.name !== 'wasm-bindgen') continue + const version = entry.version + if (typeof version !== 'string' || !wasmBindgenVersionPattern.test(version)) + throw new Error('resolve-wasm-bindgen: invalid wasm-bindgen version') + versions.add(version) + } + if (versions.size !== 1) + throw new Error( + 'resolve-wasm-bindgen: expected one unique wasm-bindgen version', + ) + return [...versions][0] +} diff --git a/wasm-size.js b/wasm-size.js index 73c8e9a3f..76118ebd8 100644 --- a/wasm-size.js +++ b/wasm-size.js @@ -9,6 +9,8 @@ import { join, resolve } from 'node:path' import { parseArgs } from 'node:util' import { gzipSync } from 'node:zlib' +import { wasmBindgenVersionPattern } from './wasm-bindgen-version.js' + const artifacts = { wasm: 'index_bg.wasm', glue: 'index.js' } const tools = ['rustc', 'wasmPack', 'wasmBindgen', 'wasmOpt', 'bun', 'node'] const readJson = (file) => JSON.parse(readFileSync(file, 'utf8')) @@ -48,7 +50,11 @@ export function validateMetadata(metadata, directories) { requireInput(provenance, 'missing job/build provenance') for (const side of ['base', 'candidate']) { const source = metadata[side] - keys(source, ['commit', 'tree', 'packageDir', 'tools'], `metadata.${side}`) + keys( + source, + ['commit', 'tree', 'packageDir', 'wasmBindgenVersion', 'tools'], + `metadata.${side}`, + ) for (const id of ['commit', 'tree']) { const valid = typeof source[id] === 'string' && /^[a-f0-9]{40}$/.test(source[id]) @@ -61,9 +67,24 @@ export function validateMetadata(metadata, directories) { keys(source.tools, tools, `${side}.tools`) for (const tool of tools) { requireInput(text(source.tools[tool]), `missing ${side} tool ${tool}`) + if ( + tool === 'wasmBindgen' && + source.wasmBindgenVersion !== metadata.base.wasmBindgenVersion + ) + continue const matches = source.tools[tool] === metadata.base.tools[tool] requireInput(matches, `tool mismatch: ${tool}`) } + const version = source.wasmBindgenVersion + requireInput( + typeof version === 'string' && wasmBindgenVersionPattern.test(version), + `invalid ${side}.wasmBindgenVersion`, + ) + requireInput( + source.tools.wasmBindgen.split(/\r?\n/, 1)[0] === + `wasm-bindgen ${version}`, + `tool mismatch: ${side} wasmBindgen does not match committed lock version`, + ) } const distinct = realpathSync(directories.base) !== realpathSync(directories.candidate)