From 8dab9753397548d57139ef83d05ba5f26342a827 Mon Sep 17 00:00:00 2001 From: Luke Karrys Date: Tue, 15 Sep 2026 17:44:16 -0700 Subject: [PATCH 1/2] Rank registry leaderboard by equally weighted fixture wins --- .github/workflows/test.yml | 24 +++++++ README.md | 15 ++++ app/src/components/header.tsx | 82 +++++++++++++++------ app/src/lib/registry-leaderboard.ts | 68 ++++++++++++++++++ app/src/lib/utils.ts | 57 ++++++++------- scripts/registry-leaderboard.test.js | 103 +++++++++++++++++++++++++++ 6 files changed, 302 insertions(+), 47 deletions(-) create mode 100644 .github/workflows/test.yml create mode 100644 app/src/lib/registry-leaderboard.ts create mode 100644 scripts/registry-leaderboard.test.js diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000000..52c14048a0 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,24 @@ +name: Data and App Tests +on: + pull_request: + push: + branches: [main] +permissions: + contents: read +jobs: + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + - uses: actions/setup-node@v6 + with: + node-version: "24" + package-manager-cache: false + - run: npm install --ignore-scripts --no-package-lock + working-directory: app + - run: node --test scripts/*.test.js + - name: Test app data semantics + run: | + if [ -d app/tests ]; then node --test app/tests/*.test.*; fi + - run: npm run build + working-directory: app diff --git a/README.md b/README.md index b4dc5610e0..cf9c7ba9d9 100644 --- a/README.md +++ b/README.md @@ -229,6 +229,21 @@ The generated charts show: ### View Results Online +Registry leaderboard cards rank registries by **fixture wins**, not an average +installation time. Each selected fixture/variation pair counts once, so a large +fixture has the same influence as a small fixture. The `average` tab counts pairs +from both registry variations. Cards show wins out of the number of selected +pairs; filtering fixtures recomputes both values. Tool toggles only hide cards: +wins are always measured against all registries in the data. + +The fastest complete positive timing earns a win; exact ties each earn a win +and share a rank (ordered alphabetically). Failed (DNF), partial, missing, and +invalid results earn no wins and receive no invented timing penalty. They remain +in the common denominator, including pairs where every registry failed. A card +with incomplete coverage also shows its number of complete results. This summary +describes how often a registry wins, not the magnitude of its speed advantage; +the fixture charts provide the underlying timings. + Results are automatically deployed to GitHub Pages when running on the main branch: diff --git a/app/src/components/header.tsx b/app/src/components/header.tsx index 31de21d3bd..8a4ae5c9ee 100644 --- a/app/src/components/header.tsx +++ b/app/src/components/header.tsx @@ -307,7 +307,8 @@ HeaderSection.displayName = "HeaderSection"; interface LeaderBoardItemProps { packageManager: PackageManager; - averageTime: number; + averageTime?: number; + registryStats?: { wins: number; totalTests: number; completedTests: number }; idx: number; unit?: string; isBaseline?: boolean; @@ -315,7 +316,8 @@ interface LeaderBoardItemProps { const LeaderBoardItem = ({ packageManager, - averageTime, + averageTime = 0, + registryStats, idx, unit = "ms/pkg", isBaseline = false, @@ -362,7 +364,18 @@ const LeaderBoardItem = ({ )}

-

{formattedTime}

+

+ {registryStats + ? `${registryStats.wins}/${registryStats.totalTests} wins` + : formattedTime} +

+ {registryStats && + registryStats.completedTests < registryStats.totalTests && ( +

+ {registryStats.completedTests}/{registryStats.totalTests}{" "} + complete +

+ )} @@ -404,25 +417,50 @@ const HeaderLeaderboard = forwardRef>( if (leaderboard && leaderboard.length === 0) return null; return ( - -
- {leaderboard && - leaderboard.map((item, idx) => ( - - ))} -
- - +
+ {isRegistryRoute && ( +

+ Fixture wins: each selected fixture and variation counts equally. + Fastest complete result wins; tied fastest results each earn a win. + Failed, partial, or missing results earn no wins. Compared across + all registries. +

+ )} + +
+ {leaderboard && + leaderboard.map((item, idx) => ( + entry.wins === item.wins, + ) + : idx + } + averageTime={item.averageTime} + registryStats={ + isRegistryRoute + ? { + wins: item.wins, + totalTests: item.totalTests, + completedTests: item.completedTests ?? 0, + } + : undefined + } + packageManager={item.packageManager} + unit={unit} + isBaseline={isBaselinePackageManager( + item.packageManager, + isRegistryRoute, + )} + /> + ))} +
+ + +
); }, ); diff --git a/app/src/lib/registry-leaderboard.ts b/app/src/lib/registry-leaderboard.ts new file mode 100644 index 0000000000..94fdfc1414 --- /dev/null +++ b/app/src/lib/registry-leaderboard.ts @@ -0,0 +1,68 @@ +import type { + BenchmarkChartData, + Fixture, + PackageManager, + Variation, +} from "@/types/chart-data"; + +export interface RegistryRanking { + packageManager: PackageManager; + wins: number; + totalTests: number; + completedTests: number; +} + +/** Each selected fixture/variation is one contest, independent of its duration. */ +export function calculateRegistryLeaderboard( + chartData: BenchmarkChartData, + specificVariation?: Variation, + enabledFixtures?: Set, +): RegistryRanking[] { + const source = chartData.registryChartData; + if (!source) return []; + const variations = + specificVariation && specificVariation !== "average" + ? [specificVariation] + : source.variations.filter((variation) => variation !== "average"); + const rankings = source.packageManagers.map((packageManager) => ({ + packageManager, + wins: 0, + totalTests: 0, + completedTests: 0, + })); + + for (const variation of variations) { + for (const fixture of source.data[variation] ?? []) { + if (enabledFixtures && !enabledFixtures.has(fixture.fixture)) continue; + // Missing, DNF and partial samples never earn wins. Preserve a common + // denominator even when a registry failed or the whole contest failed. + const metadata = fixture as typeof fixture & Record; + const times = rankings.map(({ packageManager: pm }) => { + const value = fixture[pm]; + return metadata[`${pm}_dnf`] !== true && + metadata[`${pm}_partial`] !== true && + typeof value === "number" && + Number.isFinite(value) && + value > 0 + ? value + : undefined; + }); + const best = Math.min(...times.filter((time) => time !== undefined)); + rankings.forEach((ranking, index) => { + ranking.totalTests++; + const time = times[index]; + if (time === undefined) return; + ranking.completedTests++; + // Exact ties each receive a win; no registry gets arbitrary priority. + if (time === best) ranking.wins++; + }); + } + } + + return rankings + .filter((ranking) => ranking.totalTests > 0) + .sort( + (a, b) => + b.wins - a.wins || a.packageManager.localeCompare(b.packageManager), + ); +} diff --git a/app/src/lib/utils.ts b/app/src/lib/utils.ts index 1e615c0b82..35e178cfac 100644 --- a/app/src/lib/utils.ts +++ b/app/src/lib/utils.ts @@ -1,5 +1,6 @@ import { clsx, type ClassValue } from "clsx"; import { twMerge } from "tailwind-merge"; +import { calculateRegistryLeaderboard } from "./registry-leaderboard.ts"; import type { Variation, Fixture, @@ -216,14 +217,13 @@ export const getVariationCategories = ( interface RankingData { packageManager: PackageManager; wins: number; - averageTime: number; + averageTime?: number; totalTests: number; + completedTests?: number; } export type LeaderboardRoute = - | "package-managers" - | "task-runners" - | "registries"; + "package-managers" | "task-runners" | "registries"; export const calculateLeaderboard = ( chartData: BenchmarkChartData, @@ -231,6 +231,13 @@ export const calculateLeaderboard = ( route?: LeaderboardRoute, enabledFixtures?: Set, ): RankingData[] => { + if ((route ?? "registries") === "registries") { + return calculateRegistryLeaderboard( + chartData, + specificVariation, + enabledFixtures, + ); + } const categories = getVariationCategories(chartData.chartData.variations); const effectiveRoute = route ?? "registries"; @@ -362,30 +369,30 @@ export const calculateLeaderboard = ( }); // Calculate final rankings - const leaderboard: RankingData[] = ( - availablePackageManagers as PackageManager[] - ).map((pm) => { - const stats = packageManagerStats[pm]; - if (!stats) { + const leaderboard = (availablePackageManagers as PackageManager[]).map( + (pm) => { + const stats = packageManagerStats[pm]; + if (!stats) { + return { + packageManager: pm, + wins: 0, + averageTime: Number.MAX_SAFE_INTEGER, + totalTests: 0, + }; + } + const averageTime = + stats.testCount > 0 + ? stats.totalTime / stats.testCount + : Number.MAX_SAFE_INTEGER; + return { packageManager: pm, - wins: 0, - averageTime: Number.MAX_SAFE_INTEGER, - totalTests: 0, + wins: stats.wins, + averageTime, + totalTests: stats.testCount, }; - } - const averageTime = - stats.testCount > 0 - ? stats.totalTime / stats.testCount - : Number.MAX_SAFE_INTEGER; - - return { - packageManager: pm, - wins: stats.wins, - averageTime, - totalTests: stats.testCount, - }; - }); + }, + ); // Filter out PMs with no data, then sort by average time (lower is // better) so card order matches the displayed values, wins as tiebreaker diff --git a/scripts/registry-leaderboard.test.js b/scripts/registry-leaderboard.test.js new file mode 100644 index 0000000000..b50e99c8fa --- /dev/null +++ b/scripts/registry-leaderboard.test.js @@ -0,0 +1,103 @@ +const assert = require("node:assert/strict"); +const test = require("node:test"); +const { + calculateRegistryLeaderboard, +} = require("../app/src/lib/registry-leaderboard.ts"); + +function chart(data) { + return { + registryChartData: { + variations: Object.keys(data), + packageManagers: ["npm", "vlt", "aws"], + data, + }, + }; +} + +test("five fixture wins outweigh one arbitrarily large fixture", () => { + const data = chart({ + "registry-clean": [ + ...["next", "astro", "vue", "svelte", "large"].map((fixture) => ({ + fixture, + npm: 10, + vlt: 6, + aws: 20, + })), + { fixture: "babylon", npm: 100, vlt: 10000, aws: 20000 }, + ], + }); + assert.deepEqual( + calculateRegistryLeaderboard(data).map( + ({ packageManager, wins, totalTests }) => ({ + packageManager, + wins, + totalTests, + }), + ), + [ + { packageManager: "vlt", wins: 5, totalTests: 6 }, + { packageManager: "npm", wins: 1, totalTests: 6 }, + { packageManager: "aws", wins: 0, totalTests: 6 }, + ], + ); +}); + +test("DNF, partial, missing and non-finite values earn no wins or completions", () => { + const data = chart({ + "registry-clean": [ + { fixture: "next", npm: 3, vlt: 1, vlt_dnf: true }, + { fixture: "astro", npm: 3, vlt: 1, vlt_partial: true, aws: Infinity }, + { fixture: "vue", npm: 3, vlt: 1, vlt_dnf: true, npm_dnf: true }, + { fixture: "svelte", npm: 0, vlt: -1, aws: NaN }, + ], + }); + assert.deepEqual(calculateRegistryLeaderboard(data), [ + { packageManager: "npm", wins: 2, totalTests: 4, completedTests: 2 }, + { packageManager: "aws", wins: 0, totalTests: 4, completedTests: 0 }, + { packageManager: "vlt", wins: 0, totalTests: 4, completedTests: 0 }, + ]); +}); + +test("ties each earn a win and sort alphabetically without timing tiebreakers", () => { + const data = chart({ + "registry-clean": [{ fixture: "next", npm: 4, vlt: 4, aws: 1000 }], + }); + assert.deepEqual( + calculateRegistryLeaderboard(data).map(({ packageManager, wins }) => [ + packageManager, + wins, + ]), + [ + ["npm", 1], + ["vlt", 1], + ["aws", 0], + ], + ); +}); + +test("fixture and variation filters recompute wins and common denominators", () => { + const data = chart({ + "registry-clean": [ + { fixture: "next", npm: 4, vlt: 3 }, + { fixture: "astro", npm: 4, vlt: 5 }, + ], + "registry-lockfile": [{ fixture: "next", npm: 4, vlt: 5 }], + }); + const filtered = calculateRegistryLeaderboard( + data, + "registry-clean", + new Set(["next"]), + ); + assert.equal(filtered[0].packageManager, "vlt"); + assert.equal(filtered[0].wins, 1); + assert.equal(filtered[0].totalTests, 1); + const average = calculateRegistryLeaderboard(data, "average"); + assert.equal(average[0].packageManager, "npm"); + assert.equal(average[0].wins, 2); + assert.equal(average[0].totalTests, 3); + assert.deepEqual( + calculateRegistryLeaderboard(data, "average", new Set()), + [], + ); + assert.deepEqual(calculateRegistryLeaderboard({}), []); +}); From 1b46b739fc8b2e3ea6459bd3ec4925a61eb38d8c Mon Sep 17 00:00:00 2001 From: Luke Karrys Date: Tue, 15 Sep 2026 17:50:05 -0700 Subject: [PATCH 2/2] Clarify partial results in registry win rankings --- README.md | 8 +++++--- app/src/components/variation/chart.tsx | 5 ++++- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index c8d345b9e0..579c0096c6 100644 --- a/README.md +++ b/README.md @@ -218,8 +218,10 @@ Chart data adds optional `_partial`, `_attempted_runs`, per-package datasets. Tables and chart notices label partial results with their success counts. Synthetic averages retain that warning and omit the affected command's value. Commands with any -partial result in the selected comparisons are excluded from rankings; history -omits that command's affected daily variation and category average. +partial result in the selected comparisons are excluded from timing rankings; +registry win rankings retain the common denominator and award no wins for +incomplete fixtures. History omits that command's affected daily variation and +category average. This is an additive schema change. Historical JSON without these fields remains readable, but its completeness is unknown: failed attempts discarded by older @@ -249,7 +251,7 @@ same median multiplied by 1000 and divided by package count (ms/package). Failed attempts are excluded from timing statistics but retained in completeness metadata. Partial survivor samples are labeled and excluded from averages, history, leaderboard timing/win calculations, and registry speed alerts. -All-failed results remain DNF; the existing leaderboard DNF penalty uses the +All-failed results remain DNF; timing leaderboards' DNF penalty uses the slowest successful median for that fixture. Average views and leaderboard timing values are arithmetic averages of diff --git a/app/src/components/variation/chart.tsx b/app/src/components/variation/chart.tsx index a3de50f14d..0e1a3602dd 100644 --- a/app/src/components/variation/chart.tsx +++ b/app/src/components/variation/chart.tsx @@ -341,7 +341,10 @@ export const VariationChart = ({

Partial results include successful runs only and may appear faster because failed or timed-out runs were dropped. Commands with partial - results are excluded from rankings and history averages. + results are excluded from history averages.{" "} + {isRegistry + ? "Incomplete fixture results earn no leaderboard wins." + : "They are also excluded from timing rankings."}

    {partialResults.map((label) =>
  • {label}
    • )}