From f16a2d2cdff2336f70d4e23b26e29f4d7268eb55 Mon Sep 17 00:00:00 2001
From: Diogo Esteves <diogo.esteves.goncalves@gmail.com>
Date: Thu, 22 Oct 2026 14:23:15 +0100
Subject: [PATCH] docs(benchmark): remove the withdrawn earlier benchmark

The benchmark the site and the docs report is bench/v2's 1,200 runs. The
earlier, smaller one (bench/agents) is no longer rendered or quoted:

- /benchmark drops its "The earlier benchmark" section, the component
  that drew it, and the bench.json that generate.mjs wrote for it.
- bench/agents/README.md keeps only what the directory holds and why
  bench/v2 still uses it; its results tables are gone.
- bench/v2/README.md replaces "What changes from v1" with one sentence,
  and drops the other v1 mentions the site renders.
- A test checks that /benchmark shows no figure from it.
---
 bench/agents/README.md         |  62 +--------
 bench/v2/README.md             |  21 +--
 site/app/benchmark/page.tsx    |  40 ------
 site/components/benchmark.tsx  | 227 ---------------------------------
 site/lib/data.ts               |  34 -----
 site/scripts/generate.mjs      |  56 +-------
 site/test/benchmark.test.ts    |   8 ++
 site/test/first-screen.test.ts |   2 +-
 test/bench-v2-claims.test.ts   |   2 +-
 test/readme.test.ts            |   7 +-
 10 files changed, 24 insertions(+), 435 deletions(-)
 delete mode 100644 site/components/benchmark.tsx

diff --git a/bench/agents/README.md b/bench/agents/README.md
index c2500662..ddc729a2 100644
--- a/bench/agents/README.md
+++ b/bench/agents/README.md
@@ -1,59 +1,9 @@
-# Agent benchmark
+# Agent benchmark v1
 
-Does an agent write UI that fits the design system more often with onsystem than without it? This runs Claude Code on the same UI tasks twice, once as it ships and once with the [Claude Code plugin](https://onsystem.vercel.app/docs/plugin), and scores what it wrote with `onsystem check`.
+An earlier, smaller benchmark, replaced by [bench/v2](../v2).
 
-The runs below were made on 2026-10-08, before the rename, with `@dgesteves/design-system-mcp` 0.1 and its `design-system` plugin.
+Its files stay here because bench/v2 builds on them: v2's task prompts end with the same instruction as [run.mjs](run.mjs), its chatbot tasks reuse [tasks.json](tasks.json), and its cost estimate ([config/usage-model.json](../v2/config/usage-model.json)) is measured from these runs.
 
-A larger benchmark, across three providers, five repositories and five conditions, and scored with `tsc`, onsystem and a blind-labelled sample as well, is set up in [bench/v2](../v2).
-
-## Setup
-
-- **Project:** [vercel/ai-chatbot](https://github.com/vercel/ai-chatbot) at `c2f8235`, a real shadcn/ui app with 23 components and 52 tokens. Dependencies are not installed.
-- **Tasks:** ten components a chat product needs ([tasks.json](tasks.json)), for example a usage banner with a dismiss button, a delete-all-chats confirmation dialog, a settings card with a green "Connected" label, and a message toolbar of icon buttons. Each prompt ends with the same instruction: use the project's existing UI components and design tokens, create only that file, and do not run commands.
-- **Conditions:**
-  - `base`: `claude -p` with no plugin.
-  - `plugin`: the same, plus `--plugin-dir` with the plugin (MCP server, skill and hook) and the MCP tools allowed.
-  - Both use `--permission-mode acceptEdits` and a fresh copy of the project per run.
-- **Models:** Claude Haiku 4.5 and Claude Opus 5, one run per task, condition and model, so 40 runs in all.
-- **Scoring:** `onsystem check` on every file the run created or changed, against a baseline of the untouched project, so only findings the run introduced count. A task is clean when it has no errors.
-
-## Results
-
-| Model            | Condition | Clean tasks | Errors |  Cost | Avg. time | Avg. turns |
-| ---------------- | --------- | ----------: | -----: | ----: | --------: | ---------: |
-| Claude Haiku 4.5 | base      |      6 / 10 |     13 | $1.31 |      46 s |       13.8 |
-| Claude Haiku 4.5 | plugin    | **10 / 10** |  **0** | $1.17 |      42 s |       16.0 |
-| Claude Opus 5    | base      |      8 / 10 |      5 | $7.74 |     108 s |       15.8 |
-| Claude Opus 5    | plugin    | **10 / 10** |  **0** | $8.31 |     141 s |       21.1 |
-
-No run produced a warning.
-
-What the base runs got wrong:
-
-| Task             | Haiku 4.5                                                                        | Opus 5                                                                |
-| ---------------- | -------------------------------------------------------------------------------- | --------------------------------------------------------------------- |
-| rate-limit-alert | 9 × `no-hardcoded-color` (Tailwind's red palette)                                | none                                                                  |
-| settings-card    | 2 × `no-hardcoded-color`                                                         | 4 × `no-hardcoded-color` (`emerald` for "Connected")                  |
-| profile-card     | none                                                                             | 1 × `no-hardcoded-color` (an `oklch()` literal for the avatar circle) |
-| share-dialog     | 1 × `prefer-design-system-component` (native `<label>`; the project has `Label`) | none                                                                  |
-| usage-banner     | 1 × `icon-button-accessible-name`                                                | none                                                                  |
-
-With the plugin, the agents called the design-system tools 5.6 (Haiku) and 7.3 (Opus) times per task on average: `search_components`, `get_component`, `get_tokens` and `check_ui`. The lint-on-edit hook never had to report an error back. Looking the design system up before writing was enough, so the hook acted as a safety net that was never needed in these runs. A [separate run](https://onsystem.vercel.app/docs/plugin), where Claude was asked to write hardcoded values on purpose, shows the hook catching them and Claude fixing them.
-
-## Reading it honestly
-
-- **Frontier models already do well here.** Opus 5 reads the codebase and copies its patterns. Its misses were colors the prompt named ("green", "colored circle") that the project has no token for. Smaller, faster models miss more, and they gained the most: on Haiku the plugin also cost less and finished sooner.
-- **The sample is small.** It's one project and ten tasks, with one run per cell. Treat the numbers as a direction, not a rate.
-- **The score is the tool's own.** It only counts what `check` can see: hardcoded values, invented components, props and variants, native elements, and icon buttons without names. It does not judge whether the UI is good.
-
-The files each run produced are in [output/](output/), and the per-run results, with every finding, cost and turn count, are in [results/](results/).
-
-## Reproduce
-
-```sh
-git clone https://github.com/vercel/ai-chatbot && git -C ai-chatbot checkout c2f8235
-pnpm build   # in this repository
-node bench/agents/run.mjs ./ai-chatbot . ./bench-out --concurrency 5 --model haiku
-```
-
-Each run calls Claude Code headless and is billed to your account: about $0.06 to $0.25 per run with Haiku 4.5 and $0.45 to $1.30 with Opus 5.
+- [run.mjs](run.mjs): the harness, which runs Claude Code headless on each task with and without the plugin and scores the files with `onsystem check`.
+- [tasks.json](tasks.json): the tasks.
+- [results/](results/) and [output/](output/): the record of each run and the files it wrote.
diff --git a/bench/v2/README.md b/bench/v2/README.md
index 7aa63a24..78c2ceee 100644
--- a/bench/v2/README.md
+++ b/bench/v2/README.md
@@ -4,20 +4,7 @@ Does onsystem make coding agents write UI that fits a project's design system, a
 
 **Results (2026-10-12, onsystem 0.5.0):** 1,200 runs, 6 models from 3 providers, 5 real repositories. With onsystem's skill, MCP server and hook together, design-system errors per run, as onsystem 0.5.0 counts them, fell from 0.85 to 0.00, with no measurable change in `tsc` pass rate (93% → 92%, Δ 0 [−4, +3]) and the same cost per run. The MCP server and the hook each do most of the work on their own; the gain is largest for small models. That count is onsystem's own, of the errors its MCP server and hook show the agent; [Threats to validity](#threats-to-validity) says what it cannot show: dub's colors in 0.5.0, chart tokens used for status colors, and a blind sample too small to confirm it. Full tables with confidence intervals: [results/standard/results.md](results/standard/results.md); per-run data in `results/standard/runs/` and `runs.csv`; transcripts on the [bench-v2-2026-10-12 release](https://github.com/dgesteves/onsystem/releases/tag/bench-v2-2026-10-12). A stratified 60-file sample was labelled blind by an independent AI reviewer ([LABELLER.md](results/standard/labels/LABELLER.md)).
 
-## What changes from v1
-
-[v1](../agents) was one repository, Claude only, 40 runs, and onsystem scored its own output. It showed a direction, not a result.
-
-|              | v1 (2026-10-08)                                           | v2                                                                                                                                        |
-| ------------ | --------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------- |
-| Agent        | Claude Code headless, with the user's own settings loaded | One neutral agent loop over each provider's API, nothing else loaded                                                                      |
-| Models       | Claude Haiku 4.5, Claude Opus 5                           | Small and mid tiers from Anthropic, OpenAI and Google; frontier tier optional                                                             |
-| Repositories | 1 (vercel/chatbot)                                        | 5 from `corpus/`, pinned, installed with scripts off                                                                                      |
-| Tasks        | 10                                                        | 20 (4 per repository), fixed prompts                                                                                                      |
-| Conditions   | base, plugin                                              | none, skill, mcp, hook, all                                                                                                               |
-| Repeats      | 1                                                         | 1 to 3 per cell                                                                                                                           |
-| Scoring      | `onsystem check`                                          | `tsc` in each project's own setup, `onsystem check`, blind labels on a stratified sample, and @shadcn/lint (added after the standard run) |
-| Statistics   | counts                                                    | Wilson intervals for rates, cluster bootstrap for means and paired differences                                                            |
+An earlier, smaller benchmark, [v1](../agents), was replaced by this one.
 
 ## Method
 
@@ -41,7 +28,7 @@ Each arm adds shipped onsystem files to the same loop, unchanged:
 
 | Arm     | What the agent gets on top of the file tools                                                                                                                                                                                                                                                                           |
 | ------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| `none`  | Nothing. The prompt still asks it to use the project's components and tokens, as v1's did.                                                                                                                                                                                                                             |
+| `none`  | Nothing. The prompt still asks it to use the project's components and tokens.                                                                                                                                                                                                                                          |
 | `skill` | The Claude Code plugin's `SKILL.md`, the way Agent Skills work: its name and description in the system prompt and a `load_skill` tool that returns the whole file. The skill names the MCP tools, which this arm does not have; its other instructions (tokens, variants, native elements, icon buttons) still apply.  |
 | `mcp`   | The onsystem MCP server (this checkout's `dist/cli.js`), started over stdio in the run's copy of the repository. Its five tools, with their own descriptions and schemas, as `mcp__onsystem__<tool>`, and its instructions in the system prompt, as MCP clients do.                                                    |
 | `hook`  | The plugin's hook script run after every write or edit, with the PostToolUse input Claude Code gives it. On exit 2 its stderr goes back to the model with the tool result; on exit 0 its `additionalContext` is added as a note. Its message points at `get_component` and `get_tokens`, which this arm does not have. |
@@ -123,7 +110,7 @@ Rates get 95% [Wilson score intervals](https://en.wikipedia.org/wiki/Binomial_pr
 
 - **Not a product test.** The loop is not Claude Code, Codex or Gemini CLI. Real clients add their own prompts and tools, which can change how much onsystem helps.
 - **One reasoning level.** `medium` means different things to different providers; other levels may change the picture.
-- **The prompt asks for the design system.** Every arm is told to use the project's components and tokens. A prompt that does not ask would likely show larger effects; this one is the conservative case, and the same as v1.
+- **The prompt asks for the design system.** Every arm is told to use the project's components and tokens. A prompt that does not ask would likely show larger effects; this one is the conservative case.
 - **The scorers' blind spots.** `tsc` sees invented APIs, not hardcoded values. onsystem sees both but is the tool under test; its precision on the corpus is in `corpus/labels.json`. @shadcn/lint, added after the standard run, sees hardcoded values and restyled components, not invented APIs, and reads less in some repositories (its own section, Independent scorer: @shadcn/lint, says where). No tool judges whether the UI does what was asked; only the blind labels do, and only on a sample.
 - **onsystem counts what it feeds back.** The MCP server's `check_ui` and the hook give the agent the same errors that `onsystem check` counts afterwards, so the arms with either end close to zero partly by construction. No run added a suppression comment or changed a file other than its own.
 - **dub, as of onsystem 0.5.0.** 0.5.0 found no color tokens in dub, so the color rules stayed quiet there in every arm. dub keeps its tokens in a shared Tailwind config package, which onsystem reads since [#107](https://github.com/dgesteves/onsystem/pull/107), merged after 0.6.0, and re-scoring these outputs with it raises dub's count in every arm. A rerun on the released version is planned.
@@ -166,7 +153,7 @@ node bench/v2/run.mjs score --with shadcn-lint --out bench/v2/results/standard
 | **`standard`** | **1,200** | `small` plus mid models, 5 arms, 20 tasks, 1 repeat              |  **$45** |    **$214** | $17 / $17 / $11                        |
 | `large`        |     2,100 | small and mid models with 3 repeats, plus frontier models with 1 |     $256 |      $1,178 | $79 / $136 / $41                       |
 
-**How the estimate is made.** Per run, the number of requests, the context a run adds by reading and writing, and the output tokens come from v1's 40 transcripts ([config/usage-model.json](config/usage-model.json)): Claude Haiku 4.5's runs for small models, Claude Opus 5's for mid and frontier models, v1's base condition for `none` and the larger of its two conditions for the other arms. v1's own fixed prompt (Claude Code's, about 34,000 tokens per request) is replaced by this harness's (about 700 tokens, computed from its text; the skill adds about 170, plus 570 once loaded; the MCP server about 1,570). _Expected_ assumes each request reads the earlier context from the prompt cache and writes only what is new. _Pessimistic_ assumes no cache hits at all, 50% more requests and context, and 50% more output. As a check, the same formula given v1's setup gives $0.17 per Haiku 4.5 run where v1 measured $0.13: it errs high. Token counts use 4 characters per token, and other providers' tokenizers differ by tens of percent. The canary measures real cost per run, and `plan --usage-from <canary output>` re-estimates from it.
+**How the estimate is made.** Per run, the number of requests, the context a run adds by reading and writing, and the output tokens come from v1's transcripts ([config/usage-model.json](config/usage-model.json)): Claude Haiku 4.5's runs for small models, Claude Opus 5's for mid and frontier models, v1's base condition for `none` and the larger of its two conditions for the other arms. v1's own fixed prompt (Claude Code's, about 34,000 tokens per request) is replaced by this harness's (about 700 tokens, computed from its text; the skill adds about 170, plus 570 once loaded; the MCP server about 1,570). _Expected_ assumes each request reads the earlier context from the prompt cache and writes only what is new. _Pessimistic_ assumes no cache hits at all, 50% more requests and context, and 50% more output. As a check, the same formula given v1's setup gives $0.17 per Haiku 4.5 run where v1 measured $0.13: it errs high. Token counts use 4 characters per token, and other providers' tokenizers differ by tens of percent. The canary measures real cost per run, and `plan --usage-from <canary output>` re-estimates from it.
 
 **What each size can conclude.** Half-widths of the 95% Wilson interval for one cell (one model, one arm): ±17 to ±20 points with 20 runs, ±10 to ±12 with 60, ±6 to ±7 with 180 (for rates between 80% and 50%). For differences between an arm and `none`, a simulation with this harness's own interval code (20 tasks whose difficulty varies a lot, as v1's did; the arm removes a share of each task's failures; 300 simulated benchmarks per point) gives the smallest effect found 80% of the time:
 
diff --git a/site/app/benchmark/page.tsx b/site/app/benchmark/page.tsx
index dd728586..6b936bc7 100644
--- a/site/app/benchmark/page.tsx
+++ b/site/app/benchmark/page.tsx
@@ -1,7 +1,6 @@
 import type { Metadata } from 'next';
 import type { ReactNode } from 'react';
 
-import { Benchmark as EarlierBenchmark } from '@/components/benchmark';
 import { Headline, HeadlineStats } from '@/components/benchmark-summary';
 import { MarkdownBlocks } from '@/components/markdown-blocks';
 import { Eyebrow, InlineCode, TextLink } from '@/components/section';
@@ -30,7 +29,6 @@ import {
   words,
 } from '@/lib/benchmark';
 import {
-  bench,
   benchmark,
   type BenchDelta,
   type BenchRate,
@@ -528,7 +526,6 @@ const SECTIONS = [
   ['method', 'Method'],
   ['threats', 'Threats to validity'],
   ['reproduce', 'Reproduce it'],
-  ['v1', 'The earlier benchmark'],
 ] as const;
 
 /** A task's prompt, with the file it names and the component it exports set as code. */
@@ -558,9 +555,6 @@ export default function BenchmarkPage() {
   const wrong = labels.precision.n - labels.precision.successes;
   const tiers = [...new Set(benchmark.models.map((m) => m.tier))];
   const atLimit = Object.entries(benchmark.statuses).filter(([status]) => status !== 'done');
-  const v1Calls = bench.models.map((m) => m.plugin.toolCalls).sort((a, b) => a - b);
-  const [v1Small] = bench.models;
-  const v1Runs = bench.models.reduce((n, m) => n + m.base.runs + m.plugin.runs, 0);
 
   return (
     <main id="main" className="relative">
@@ -980,40 +974,6 @@ export default function BenchmarkPage() {
               </p>
             </div>
           </Block>
-
-          <Block id="v1" title="The earlier benchmark">
-            <div className="max-w-3xl">
-              <MarkdownBlocks markdown={benchmark.method.v1} />
-            </div>
-            <div className="mt-8">
-              <EarlierBenchmark bench={bench} />
-            </div>
-            <div className="mt-8 grid gap-6 text-[15px] leading-relaxed text-muted md:grid-cols-2 md:gap-10">
-              <p>
-                Claude Code built {words(bench.tasks.length)} components for{' '}
-                <TextLink href="https://github.com/vercel/ai-chatbot">vercel/ai-chatbot</TextLink>,
-                once as it ships and once with the plugin, and{' '}
-                <InlineCode>onsystem check</InlineCode> scored every file. Without the plugin, the
-                misses were raw colors for things the prompt described, a native{' '}
-                <InlineCode>&lt;label&gt;</InlineCode> where the project has{' '}
-                <InlineCode>Label</InlineCode>, and an icon button nobody could name with a screen
-                reader. With it, the agent looked components and tokens up before writing (
-                {v1Calls.join(' to ')} design-system tool calls per task).
-              </p>
-              <p>
-                It is one project and {count(v1Runs)} runs, one per task, model and condition, so
-                read it as a direction rather than a rate
-                {v1Small && v1Small.costDelta < 0
-                  ? `; ${v1Small.name} cost less with the plugin than without`
-                  : ''}
-                .{' '}
-                <TextLink href={repoLink('bench/agents', 'tree')}>
-                  The method, per-run results and every generated file
-                </TextLink>{' '}
-                are in the repository.
-              </p>
-            </div>
-          </Block>
         </div>
       </div>
     </main>
diff --git a/site/components/benchmark.tsx b/site/components/benchmark.tsx
deleted file mode 100644
index a39201ef..00000000
--- a/site/components/benchmark.tsx
+++ /dev/null
@@ -1,227 +0,0 @@
-import { CheckIcon } from '@/components/icons';
-import { TableScroll } from '@/components/table-scroll';
-import type { Bench, BenchModel } from '@/lib/data';
-
-const WITHOUT = '#5d6875';
-const WITH = 'var(--color-cyan)';
-
-function taskLabel(id: string): string {
-  const words = id.replace(/-/g, ' ');
-  return words.charAt(0).toUpperCase() + words.slice(1);
-}
-
-function money(value: number): string {
-  return `$${value.toFixed(2)}`;
-}
-
-function signed(value: number): string {
-  return value > 0 ? `+${value}%` : value < 0 ? `−${Math.abs(value)}%` : '0%';
-}
-
-/** A labelled horizontal bar on a shared scale. */
-function Bar({
-  label,
-  value,
-  max,
-  color,
-  text,
-}: {
-  label: string;
-  value: number;
-  max: number;
-  color: string;
-  text: string;
-}) {
-  const width = max === 0 ? 0 : (value / max) * 100;
-  return (
-    <div className="grid grid-cols-[4.75rem_minmax(0,1fr)_4.5rem] items-center gap-3 text-[13px]">
-      <span className="text-muted">{label}</span>
-      <span className="relative h-2.5 rounded-full bg-fg/[0.05]">
-        <span
-          className="absolute inset-y-0 left-0 rounded-full"
-          style={{ width: `${width}%`, background: color, minWidth: value > 0 ? 6 : 0 }}
-        />
-      </span>
-      <span className="text-right font-mono text-fg tabular-nums">{text}</span>
-    </div>
-  );
-}
-
-function ModelCard({ model, maxErrors }: { model: BenchModel; maxErrors: number }) {
-  const { base, plugin } = model;
-  return (
-    <figure className="rounded-2xl border border-line bg-raised p-5 sm:p-6">
-      <figcaption className="flex items-baseline justify-between gap-3">
-        <span className="text-base font-semibold text-fg">{model.name}</span>
-        <span className="font-mono text-xs text-subtle">{base.runs + plugin.runs} runs</span>
-      </figcaption>
-
-      <div className="mt-5">
-        <p className="text-[13px] font-medium text-fg-soft">
-          Components with no design-system errors
-        </p>
-        <div className="mt-2.5 grid gap-2">
-          <Bar
-            label="Without"
-            value={base.clean}
-            max={base.runs}
-            color={WITHOUT}
-            text={`${base.clean} / ${base.runs}`}
-          />
-          <Bar
-            label="With plugin"
-            value={plugin.clean}
-            max={plugin.runs}
-            color={WITH}
-            text={`${plugin.clean} / ${plugin.runs}`}
-          />
-        </div>
-      </div>
-
-      <div className="mt-5">
-        <p className="text-[13px] font-medium text-fg-soft">Design-system errors, all ten tasks</p>
-        <div className="mt-2.5 grid gap-2">
-          <Bar
-            label="Without"
-            value={base.errors}
-            max={maxErrors}
-            color={WITHOUT}
-            text={String(base.errors)}
-          />
-          <Bar
-            label="With plugin"
-            value={plugin.errors}
-            max={maxErrors}
-            color={WITH}
-            text={String(plugin.errors)}
-          />
-        </div>
-      </div>
-
-      <dl className="mt-6 grid grid-cols-3 gap-3 border-t border-line pt-4 text-[13px]">
-        <div>
-          <dt className="text-muted">Cost</dt>
-          <dd className="mt-1 font-mono text-fg tabular-nums">{signed(model.costDelta)}</dd>
-          <dd className="font-mono text-[11px] text-subtle tabular-nums">
-            {money(base.cost)} → {money(plugin.cost)}
-          </dd>
-        </div>
-        <div>
-          <dt className="text-muted">Avg. time</dt>
-          <dd className="mt-1 font-mono text-fg tabular-nums">
-            {base.seconds} → {plugin.seconds} s
-          </dd>
-        </div>
-        <div>
-          <dt className="text-muted">Tool calls</dt>
-          <dd className="mt-1 font-mono whitespace-nowrap text-fg tabular-nums">
-            {plugin.toolCalls} per task
-          </dd>
-        </div>
-      </dl>
-    </figure>
-  );
-}
-
-function Cell({ errors, byRule }: { errors: number; byRule: Record<string, number> }) {
-  if (errors === 0) {
-    return (
-      <span className="inline-flex items-center gap-1.5 text-muted">
-        <CheckIcon className="size-3.5 text-cyan" />
-        clean
-      </span>
-    );
-  }
-  return (
-    <span className="block">
-      <span className="font-medium text-error">
-        {errors} {errors === 1 ? 'error' : 'errors'}
-      </span>
-      {Object.entries(byRule).map(([rule, count]) => (
-        <span key={rule} className="block font-mono text-[11px] text-subtle">
-          {count} × {rule}
-        </span>
-      ))}
-    </span>
-  );
-}
-
-export function Benchmark({ bench }: { bench: Bench }) {
-  const maxErrors = Math.max(...bench.models.map((m) => m.base.errors), 1);
-  return (
-    <div>
-      <div
-        className="mb-4 flex flex-wrap items-center gap-x-5 gap-y-2 text-[13px] text-muted"
-        aria-hidden="true"
-      >
-        <span className="inline-flex items-center gap-2">
-          <span className="size-2.5 rounded-full" style={{ background: WITHOUT }} />
-          Claude Code as it ships
-        </span>
-        <span className="inline-flex items-center gap-2">
-          <span className="size-2.5 rounded-full" style={{ background: WITH }} />
-          With the onsystem plugin
-        </span>
-      </div>
-      <div className="grid gap-4 md:grid-cols-2">
-        {bench.models.map((model) => (
-          <ModelCard key={model.id} model={model} maxErrors={maxErrors} />
-        ))}
-      </div>
-
-      <details className="group mt-4 rounded-2xl border border-line bg-raised/60">
-        <summary className="flex cursor-pointer list-none items-center justify-between gap-3 px-5 py-4 text-sm font-medium text-fg-soft hover:text-fg [&::-webkit-details-marker]:hidden">
-          Every task, every run
-          <span className="font-mono text-xs text-subtle group-open:hidden">Show</span>
-          <span className="hidden font-mono text-xs text-subtle group-open:inline">Hide</span>
-        </summary>
-        <TableScroll label="Every task, every run" className="border-t border-line">
-          <table className="w-full min-w-[640px] text-left text-[13px]">
-            <caption className="sr-only">
-              Design-system errors per task, for each model, without and with the plugin
-            </caption>
-            <thead>
-              <tr className="border-b border-line text-muted">
-                <th scope="col" className="px-5 py-3 font-medium">
-                  Task
-                </th>
-                {bench.models.flatMap((m) =>
-                  (['without', 'with plugin'] as const).map((condition) => (
-                    <th key={`${m.id}-${condition}`} scope="col" className="px-3 py-3 font-medium">
-                      {m.name.replace('Claude ', '')}
-                      <span className="block font-normal text-subtle">{condition}</span>
-                    </th>
-                  )),
-                )}
-              </tr>
-            </thead>
-            <tbody>
-              {bench.tasks.map((task, row) => (
-                <tr key={task.id} className="border-b border-line/60 align-top last:border-0">
-                  <th scope="row" className="px-5 py-3 font-normal">
-                    <span className="text-fg-soft">{taskLabel(task.id)}</span>
-                    <span className="mt-0.5 block max-w-[22rem] text-[12px] leading-snug text-subtle">
-                      {task.prompt.replace(/^Create \S+ exporting \w+: /, '')}
-                    </span>
-                  </th>
-                  {bench.models.flatMap((m) => {
-                    const result = m.tasks[row];
-                    if (!result) return [];
-                    return [
-                      <td key={`${m.id}-base`} className="px-3 py-3">
-                        <Cell {...result.base} />
-                      </td>,
-                      <td key={`${m.id}-plugin`} className="px-3 py-3">
-                        <Cell {...result.plugin} />
-                      </td>,
-                    ];
-                  })}
-                </tr>
-              ))}
-            </tbody>
-          </table>
-        </TableScroll>
-      </details>
-    </div>
-  );
-}
diff --git a/site/lib/data.ts b/site/lib/data.ts
index e0a47ae3..91ecdf07 100644
--- a/site/lib/data.ts
+++ b/site/lib/data.ts
@@ -2,7 +2,6 @@
  * Typed access to generated/*.json, which scripts/generate.mjs writes at build time from the
  * library, the demo design system, the benchmark results, docs/*.md and the README.
  */
-import benchJson from '@/generated/bench.json';
 import benchmarkJson from '@/generated/benchmark.json';
 import corpusJson from '@/generated/corpus.json';
 import demoJson from '@/generated/demo.json';
@@ -44,37 +43,6 @@ export interface Demo {
   checkMs: number;
 }
 
-export interface RunSummary {
-  runs: number;
-  clean: number;
-  errors: number;
-  warnings: number;
-  cost: number;
-  seconds: number;
-  turns: number;
-  toolCalls: number;
-  hookBlocks: number;
-}
-
-export interface BenchModel {
-  id: string;
-  name: string;
-  base: RunSummary;
-  plugin: RunSummary;
-  costDelta: number;
-  tasks: {
-    id: string;
-    base: { errors: number; byRule: Record<string, number> };
-    plugin: { errors: number; byRule: Record<string, number> };
-  }[];
-}
-
-export interface Bench {
-  project: string;
-  tasks: { id: string; file: string; prompt: string }[];
-  models: BenchModel[];
-}
-
 /** A rate with its 95% Wilson interval, as bench/v2's results.json has it (fractions). */
 export interface BenchRate {
   successes: number;
@@ -201,7 +169,6 @@ export interface Benchmark {
   };
   /** Sections of bench/v2/README.md, as Markdown with links relative to the repository root. */
   method: {
-    v1: string;
     arms: string;
     scoring: string;
     statistics: string;
@@ -363,7 +330,6 @@ function typed<T>(json: unknown): T {
 }
 
 export const demo = typed<Demo>(demoJson);
-export const bench = typed<Bench>(benchJson);
 export const benchmark = typed<Benchmark>(benchmarkJson);
 export const corpus = typed<Corpus>(corpusJson);
 export const tools = typed<Tools>(toolsJson);
diff --git a/site/scripts/generate.mjs b/site/scripts/generate.mjs
index de904885..0c898e68 100644
--- a/site/scripts/generate.mjs
+++ b/site/scripts/generate.mjs
@@ -1,5 +1,5 @@
 // Generates the data the site renders, from the library itself, the demo design system in
-// examples/shadcn-demo, the benchmark results in bench/agents and the README, so what the
+// examples/shadcn-demo, the benchmark results in bench/v2 and the README, so what the
 // site shows is what the tool does at this commit. `pnpm build` and `pnpm dev` run it first.
 // It needs the library built: `pnpm --dir .. build`.
 import assert from 'node:assert/strict';
@@ -374,59 +374,6 @@ write('design-system.json', {
   ),
 });
 
-// ─── The benchmark ──────────────────────────────────────────────────────────
-
-const bench = path.join(repo, 'bench/agents');
-const tasks = JSON.parse(fs.readFileSync(path.join(bench, 'tasks.json'), 'utf8'));
-const MODEL_NAMES = { 'haiku-4-5': 'Claude Haiku 4.5', 'opus-5': 'Claude Opus 5' };
-const models = fs
-  .readdirSync(path.join(bench, 'results'))
-  .filter((dir) => fs.statSync(path.join(bench, 'results', dir)).isDirectory())
-  .sort()
-  .map((dir) => {
-    const runs = fs
-      .readdirSync(path.join(bench, 'results', dir))
-      .filter((f) => f.endsWith('.json'))
-      .map((f) => JSON.parse(fs.readFileSync(path.join(bench, 'results', dir, f), 'utf8')));
-    const summary = (condition) => {
-      const list = runs.filter((r) => r.condition === condition);
-      assert.equal(list.length, tasks.length, `${dir}/${condition} should have a run per task`);
-      const sum = (key) => list.reduce((total, r) => total + r[key], 0);
-      return {
-        runs: list.length,
-        clean: list.filter((r) => r.errors === 0).length,
-        errors: sum('errors'),
-        warnings: sum('warnings'),
-        cost: Math.round(sum('cost') * 100) / 100,
-        seconds: Math.round(sum('seconds') / list.length),
-        turns: Math.round((sum('turns') / list.length) * 10) / 10,
-        toolCalls: Math.round((sum('mcpCalls') / list.length) * 10) / 10,
-        hookBlocks: sum('hookBlocks'),
-      };
-    };
-    const base = summary('base');
-    const plugin = summary('plugin');
-    return {
-      id: dir,
-      name: MODEL_NAMES[dir] ?? dir,
-      base,
-      plugin,
-      costDelta: Math.round(((plugin.cost - base.cost) / base.cost) * 100),
-      tasks: tasks.map((task) => {
-        const of = (condition) => {
-          const run = runs.find((r) => r.task === task.id && r.condition === condition);
-          return { errors: run.errors, byRule: run.byRule };
-        };
-        return { id: task.id, base: of('base'), plugin: of('plugin') };
-      }),
-    };
-  });
-write('bench.json', {
-  project: 'vercel/ai-chatbot',
-  tasks: tasks.map((t) => ({ id: t.id, file: t.file, prompt: t.prompt })),
-  models,
-});
-
 // ─── The benchmark, v2 ──────────────────────────────────────────────────────
 
 // /benchmark renders bench/v2's standard run from the files `run.mjs report` writes: the tables
@@ -699,7 +646,6 @@ write('benchmark.json', {
     precision: labelPrecision,
   },
   method: {
-    v1: benchLinks(section(v2Readme, '## What changes from v1', 'bench/v2/README.md')),
     arms: benchLinks(section(v2Readme, '### Arms', 'bench/v2/README.md')),
     scoring: benchLinks(section(v2Readme, '### Scoring', 'bench/v2/README.md')),
     statistics: benchLinks(section(v2Readme, '### Statistics', 'bench/v2/README.md')),
diff --git a/site/test/benchmark.test.ts b/site/test/benchmark.test.ts
index 23fc21c7..4471da9c 100644
--- a/site/test/benchmark.test.ts
+++ b/site/test/benchmark.test.ts
@@ -553,6 +553,14 @@ describe('/benchmark', () => {
     );
     expect(text(html)).toContain('node bench/v2/run.mjs run --matrix standard');
   });
+
+  it('shows no figure from the earlier, smaller benchmark it replaced', () => {
+    expect(html).not.toContain('id="v1"');
+    expect(html).not.toContain('bench/agents');
+    expect(text(html)).not.toMatch(
+      /earlier benchmark|\d ?\/ ?10\b|[−-]11 ?%|\b40 runs|ai-chatbot/i,
+    );
+  });
 });
 
 describe('the rest of the site', () => {
diff --git a/site/test/first-screen.test.ts b/site/test/first-screen.test.ts
index fede735d..5e6d7bf8 100644
--- a/site/test/first-screen.test.ts
+++ b/site/test/first-screen.test.ts
@@ -96,7 +96,7 @@ describe('the first screen', () => {
     const copy = text(match?.[2] ?? '');
     expect(headline).toBe('Keeps coding agents on your design system.');
     expect(words(`${headline} ${copy}`)).toBeLessThanOrEqual(45);
-    // The three invention rules fired 0 times in the benchmark's 1,240 runs: they come last.
+    // The three invention rules fired 0 times in the benchmark's 1,200 runs: they come last.
     expect(copy).toMatch(/palette colors, native elements and APIs your design system lacks/);
     expect(copy).toContain('every edit and pull request');
     expect(copy).toContain(
diff --git a/test/bench-v2-claims.test.ts b/test/bench-v2-claims.test.ts
index b63b45b7..931cfbbe 100644
--- a/test/bench-v2-claims.test.ts
+++ b/test/bench-v2-claims.test.ts
@@ -99,7 +99,7 @@ describe('bench/v2 results', () => {
   });
 
   it('names the onsystem version that scored the runs, and what changed in dub since', () => {
-    const results = flat(readme.slice(0, readme.indexOf('## What changes from v1')));
+    const results = flat(readme.slice(0, readme.indexOf('\n## Method\n')));
     expect(results).toContain(`**Results (2026-10-12, onsystem ${info.onsystem}):**`);
     expect(results).toContain(`as onsystem ${info.onsystem} counts them`);
     expect(flat(notes)).toContain(`They are onsystem ${info.onsystem}'s own count`);
diff --git a/test/readme.test.ts b/test/readme.test.ts
index 825c3e98..23c6bee9 100644
--- a/test/readme.test.ts
+++ b/test/readme.test.ts
@@ -649,15 +649,14 @@ describe('the README', () => {
       `${count(measures.length)} measures, ${count(measures.filter((m) => m.independent).length).toLowerCase()} of them independent of onsystem.`,
     );
 
-    // Anywhere else it counts them, it says the same number; and the two places that list the
-    // scorers, the v2 column of the table of what changes from v1 and their blind spots under
-    // Threats to validity, name every measure (`onsystem check` as onsystem).
+    // Anywhere else it counts them, it says the same number; and where it lists the scorers,
+    // their blind spots under Threats to validity, it names every measure (`onsystem check` as
+    // onsystem).
     for (const m of v2.matchAll(/\b(one|two|three|four|five|six) (?:scorers|measures)\b/gi))
       expect(m[1]?.toLowerCase(), m[0]).toBe(count(measures.length).toLowerCase());
     const lines = v2.split('\n');
     const blindSpots = "- **The scorers' blind spots.** ";
     const lists = {
-      'What changes from v1': lines.find((l) => /^\| Scoring +\|/.test(l))?.split('|')[3],
       'Threats to validity': lines.find((l) => l.startsWith(blindSpots))?.slice(blindSpots.length),
     };
     const named = (name: string) =>