Compare benchmark top lists using median ops/s

.github/workflows/ci.yml

@@ -108,6 +108,7 @@ jobs:
   # machines, and that host delta dominates the 8% regression threshold.
   # Per-case time is longer than the local default so tinybench gets enough samples;
   # compare treats the median change as the verdict, not the per-case count.
+  # Per-case lists also compare median ops/s (tinybench p50), not mean throughput.
   benchmark-pr:
     name: Benchmark (PR)
     needs: changes

README.ja.md

@@ -428,4 +428,4 @@ pnpm run bench:compare -- --head tmp/bench/head.json --base tmp/bench/base.json
 - GitHub Actions では、`devel` / `main` 向け PR で base/head 比較を PR comment として投稿します
 - GitHub Actions では、`devel` / `main` への push でも直前 revision 比較を実行し、対象 commit へ commit comment を投稿します
 - CI は比較前に base と head を同一 runner で計測し、ホスト間ノイズが delta を支配しないようにします
-- CI はローカル default より長い per-case time を使い、比較シグナルは median change です。ケース単位の一覧は点検用です
+- CI はローカル default より長い per-case time を使い、比較シグナルは median change です。ケース単位の一覧も mean ではなく median ops/s で比較します

README.md

@@ -428,4 +428,4 @@ pnpm run bench:compare -- --head tmp/bench/head.json --base tmp/bench/base.json
 - In GitHub Actions, post base/head comparison as PR comment in PR for `devel` / `main`
 - GitHub Actions also performs the previous revision comparison when pushing to `devel` / `main` and posts a commit comment to the target commit.
 - CI measures base and head on the same runner before comparing, so host-to-host noise does not dominate the delta.
-- CI uses a longer per-case time than the local default, and treats the median change as the comparison signal. Per-case lists are for inspection.
+- CI uses a longer per-case time than the local default, and treats the median change as the comparison signal. Per-case lists compare median ops/s, not mean.

scripts/bench/aggregate-results.test.ts

@@ -38,7 +38,9 @@ function createSnapshot(
     results: {
       'utils.clamp': {
         hz,
+        medianHz: hz,
         meanMs,
+        p50Ms: meanMs,
         p75Ms: meanMs + 1,
         p99Ms: meanMs + 2,
         minMs: meanMs - 1,
@@ -65,10 +67,72 @@ describe('aggregateBenchmarkSnapshots', () => {
       runCount: 3,
     });
     expect(aggregated.results['utils.clamp'].hz).toBe(120);
+    expect(aggregated.results['utils.clamp'].medianHz).toBe(120);
     expect(aggregated.results['utils.clamp'].meanMs).toBe(9);
+    expect(aggregated.results['utils.clamp'].p50Ms).toBe(9);
     expect(aggregated.results['utils.clamp'].p99Ms).toBe(11);
   });
 
+  it('aggregates medianHz independently from mean hz', () => {
+    const aggregated = aggregateBenchmarkSnapshots([
+      createSnapshot(80, 12, {
+        results: {
+          'utils.clamp': {
+            hz: 80,
+            medianHz: 100,
+            meanMs: 12,
+            p50Ms: 10,
+            p75Ms: 13,
+            p99Ms: 14,
+            minMs: 9,
+            maxMs: 15,
+            rmePercent: 1,
+            sampleCount: 100,
+            totalTimeMs: 200,
+          },
+        },
+      }),
+      createSnapshot(200, 5, {
+        results: {
+          'utils.clamp': {
+            hz: 200,
+            medianHz: 140,
+            meanMs: 5,
+            p50Ms: 8,
+            p75Ms: 6,
+            p99Ms: 7,
+            minMs: 4,
+            maxMs: 8,
+            rmePercent: 1,
+            sampleCount: 100,
+            totalTimeMs: 200,
+          },
+        },
+      }),
+      createSnapshot(90, 11, {
+        results: {
+          'utils.clamp': {
+            hz: 90,
+            medianHz: 120,
+            meanMs: 11,
+            p50Ms: 9,
+            p75Ms: 12,
+            p99Ms: 13,
+            minMs: 8,
+            maxMs: 14,
+            rmePercent: 1,
+            sampleCount: 100,
+            totalTimeMs: 200,
+          },
+        },
+      }),
+    ]);
+
+    expect(aggregated.results['utils.clamp'].hz).toBe(90);
+    expect(aggregated.results['utils.clamp'].medianHz).toBe(120);
+    expect(aggregated.results['utils.clamp'].p50Ms).toBe(9);
+  });
+
   it('rejects incompatible snapshots', () => {
     expect(() =>
       aggregateBenchmarkSnapshots([

scripts/bench/aggregate-results.ts

@@ -3,6 +3,7 @@ import { dirname, resolve } from 'node:path';
 import { fileURLToPath } from 'node:url';
 import { loadExportsBenchmarkSnapshot, resolveCliValue, runCliMain } from './cli-utils.ts';
 import type { BenchmarkTaskStats, ExportsBenchmarkSnapshot } from './exports.types.ts';
+import { median } from './task-stats.ts';
 
 interface CliDefaults {
   outputPath: string;
@@ -22,7 +23,9 @@ const DEFAULTS: CliDefaults = {
 
 const BENCHMARK_RESULT_FIELDS = [
   'hz',
+  'medianHz',
   'meanMs',
+  'p50Ms',
   'p75Ms',
   'p99Ms',
   'minMs',
@@ -120,19 +123,6 @@ function aggregateTaskStats(values: readonly BenchmarkTaskStats[]): BenchmarkTas
   return aggregated;
 }
 
-function median(values: readonly number[]): number {
-  if (values.length === 0) {
-    return 0;
-  }
-
-  const sorted = [...values].sort((left, right) => left - right);
-  const middleIndex = Math.floor(sorted.length / 2);
-  if (sorted.length % 2 === 1) {
-    return sorted[middleIndex];
-  }
-  return (sorted[middleIndex - 1] + sorted[middleIndex]) / 2;
-}
-
 function parseArgs(args: string[], defaults: CliDefaults): CliOptions {
   const options: CliOptions = {
     inputPaths: [],

scripts/bench/cli-utils.test.ts

@@ -57,7 +57,9 @@ function createSnapshot(overrides: Partial<ExportsBenchmarkSnapshot> = {}): Expo
     results: {
       'utils.clamp': {
         hz: 100,
+        medianHz: 100,
         meanMs: 10,
+        p50Ms: 10,
         p75Ms: 11,
         p99Ms: 12,
         minMs: 9,

scripts/bench/compare-results.test.ts

@@ -49,7 +49,9 @@ function createSnapshot(
         key,
         {
           hz,
+          medianHz: hz,
           meanMs: hz === 0 ? 0 : 1000 / hz,
+          p50Ms: hz === 0 ? 0 : 1000 / hz,
           p75Ms: 11,
           p99Ms: 12,
           minMs: 9,
@@ -93,6 +95,37 @@ describe('compareSnapshots', () => {
     expect(rows[0]?.headHz).toBe(110);
     expect(rows[0]?.deltaPercent).toBeCloseTo(10);
   });
+
+  it('compares median ops/s when mean hz disagrees', () => {
+    const rows = compareSnapshots(
+      createSnapshot({ 'utils.case': 100 }),
+      createSnapshot(
+        { 'utils.case': 100 },
+        {
+          results: {
+            'utils.case': {
+              hz: 50,
+              medianHz: 110,
+              meanMs: 20,
+              p50Ms: 9.091,
+              p75Ms: 11,
+              p99Ms: 12,
+              minMs: 9,
+              maxMs: 40,
+              rmePercent: 1,
+              sampleCount: 100,
+              totalTimeMs: 200,
+            },
+          },
+        },
+      ),
+    );
+
+    expect(rows).toHaveLength(1);
+    expect(rows[0]?.baseHz).toBe(100);
+    expect(rows[0]?.headHz).toBe(110);
+    expect(rows[0]?.deltaPercent).toBeCloseTo(10);
+  });
 });
 
 describe('summarizeRows', () => {
@@ -165,6 +198,73 @@ describe('buildDiffMarkdown', () => {
     expect(markdown).toContain('| Median change | +1.00% |');
     expect(markdown).toContain('| Cases regressed (<= -threshold) | 1 |');
     expect(markdown).toContain('Overall verdict uses the median change');
+    expect(markdown).toContain('Per-case lists compare median ops/s, not mean.');
     expect(markdown).toContain('`utils.noisy`');
   });
+
+  it('ranks top lists by median ops/s, not mean hz', () => {
+    const markdown = buildDiffMarkdown(
+      createSnapshot({
+        'utils.meanOnly': 100,
+        'utils.medianReal': 100,
+        'utils.stable': 100,
+      }),
+      createSnapshot(
+        {
+          'utils.meanOnly': 100,
+          'utils.medianReal': 100,
+          'utils.stable': 100,
+        },
+        {
+          results: {
+            'utils.meanOnly': {
+              hz: 50,
+              medianHz: 99,
+              meanMs: 20,
+              p50Ms: 10.101,
+              p75Ms: 11,
+              p99Ms: 12,
+              minMs: 9,
+              maxMs: 40,
+              rmePercent: 1,
+              sampleCount: 100,
+              totalTimeMs: 200,
+            },
+            'utils.medianReal': {
+              hz: 99,
+              medianHz: 50,
+              meanMs: 10.101,
+              p50Ms: 20,
+              p75Ms: 21,
+              p99Ms: 22,
+              minMs: 9,
+              maxMs: 40,
+              rmePercent: 1,
+              sampleCount: 100,
+              totalTimeMs: 200,
+            },
+            'utils.stable': {
+              hz: 101,
+              medianHz: 101,
+              meanMs: 9.901,
+              p50Ms: 9.901,
+              p75Ms: 11,
+              p99Ms: 12,
+              minMs: 9,
+              maxMs: 13,
+              rmePercent: 1,
+              sampleCount: 100,
+              totalTimeMs: 200,
+            },
+          },
+        },
+      ),
+      8,
+      12,
+    );
+
+    expect(markdown).toContain('`utils.medianReal`');
+    expect(markdown).toContain('| `utils.medianReal` | 100.00 | 50.00 | -50.00% |');
+    expect(markdown).not.toContain('`utils.meanOnly`');
+  });
 });

scripts/bench/compare-results.ts

@@ -10,6 +10,7 @@ import {
   runCliMain,
 } from './cli-utils.ts';
 import type { ExportsBenchmarkSnapshot } from './exports.types.ts';
+import { resolveComparisonHz } from './task-stats.ts';
 
 interface CliDefaults {
   outputPath: string;
@@ -130,7 +131,8 @@ export function buildDiffMarkdown(
   lines.push(`- Head SHA: \`${formatSha(headSnapshot.gitSha)}\``);
   lines.push(`- Comparable cases: \`${summary.comparableCaseCount}\``);
   lines.push(`- Regression threshold: \`${thresholdPercent.toFixed(2)}%\``);
-  lines.push(`- Overall verdict uses the median change. Per-case lists are for inspection.`);
+  lines.push(`- Overall verdict uses the median change across cases.`);
+  lines.push(`- Per-case lists compare median ops/s, not mean.`);
   lines.push(`- Base runs: \`${formatRunCount(baseSnapshot)}\``);
   lines.push(`- Head runs: \`${formatRunCount(headSnapshot)}\``);
   lines.push('');
@@ -151,7 +153,7 @@ export function buildDiffMarkdown(
   if (regressions.length === 0) {
     lines.push('No regression over threshold.');
   } else {
-    lines.push('| API | Base ops/s | Head ops/s | Change |');
+    lines.push('| API | Base median ops/s | Head median ops/s | Change |');
     lines.push('| --- | ---: | ---: | ---: |');
     for (const row of regressions) {
       lines.push(
@@ -165,7 +167,7 @@ export function buildDiffMarkdown(
   if (improvements.length === 0) {
     lines.push('No improvement over threshold.');
   } else {
-    lines.push('| API | Base ops/s | Head ops/s | Change |');
+    lines.push('| API | Base median ops/s | Head median ops/s | Change |');
     lines.push('| --- | ---: | ---: | ---: |');
     for (const row of improvements) {
       lines.push(
@@ -188,7 +190,11 @@ export function buildDiffMarkdown(
 
 function buildHeadOnlyMarkdown(headSnapshot: ExportsBenchmarkSnapshot, topCount: number, basePath?: string): string {
   const slowest = Object.entries(headSnapshot.results)
-    .map(([key, value]) => ({ key, hz: value.hz, meanMs: value.meanMs }))
+    .map(([key, value]) => ({
+      key,
+      hz: resolveComparisonHz(value),
+      meanMs: value.p50Ms ?? value.meanMs,
+    }))
     .sort((left, right) => left.hz - right.hz)
     .slice(0, topCount);
 
@@ -209,7 +215,7 @@ function buildHeadOnlyMarkdown(headSnapshot: ExportsBenchmarkSnapshot, topCount:
   if (slowest.length === 0) {
     lines.push('No benchmark result found.');
   } else {
-    lines.push('| API | ops/s | mean (ms) |');
+    lines.push('| API | median ops/s | median (ms) |');
     lines.push('| --- | ---: | ---: |');
     for (const row of slowest) {
       lines.push(`| \`${row.key}\` | ${formatOps(row.hz)} | ${row.meanMs.toFixed(4)} |`);
@@ -225,25 +231,27 @@ export function compareSnapshots(
   const rows: ComparedRow[] = [];
   for (const [key, headResult] of Object.entries(headSnapshot.results)) {
     const baseResult = baseSnapshot.results[key];
-    if (!baseResult || !Number.isFinite(baseResult.hz) || baseResult.hz <= 0) {
+    if (!baseResult) {
+      continue;
+    }
+    const baseHz = resolveComparisonHz(baseResult);
+    const headHz = resolveComparisonHz(headResult);
+    if (baseHz <= 0 || headHz <= 0) {
       continue;
     }
-    const deltaPercent = (headResult.hz / baseResult.hz - 1) * 100;
+    const deltaPercent = (headHz / baseHz - 1) * 100;
     rows.push({
       key,
-      baseHz: baseResult.hz,
-      headHz: headResult.hz,
+      baseHz,
+      headHz,
       deltaPercent,
     });
   }
   rows.sort((left, right) => left.key.localeCompare(right.key));
   return rows;
 }
 
-export function resolveOverallVerdict(
-  medianDeltaPercent: number,
-  thresholdPercent: number,
-): BenchmarkOverallVerdict {
+export function resolveOverallVerdict(medianDeltaPercent: number, thresholdPercent: number): BenchmarkOverallVerdict {
   if (medianDeltaPercent >= thresholdPercent) {
     return 'improved';
   }