feat: explain benchmark model choices

CONTRIBUTING.md

@@ -64,9 +64,9 @@ Include refreshed results for PRs with new/changed evals so a reviewer can inspe
 
 ## Publishing and consuming results
 
-To publish a benchmark, upload complete runs with `--runs` and update the experiment IDs in [fetch-braintrust.ts](apps/web/scripts/fetch-braintrust.ts). Include every experiment in `experiments/benchmark/`, using its name as the key.
+To publish a benchmark, upload complete runs with `--runs` and update the pins and rationales in [publication.ts](apps/web/src/data/publication.ts). Include every experiment in `experiments/benchmark/`.
 
-For local UI work with a partial run, temporarily pin only the uploaded experiment IDs and start the server with `EVALS_ALLOW_PARTIAL=1 pnpm web`.
+For local UI work with a partial run, temporarily keep only the uploaded experiment IDs and their rationales in the catalog and start the server with `EVALS_ALLOW_PARTIAL=1 pnpm web`.
 
 The published benchmark is at https://supabase.com/evals/eval-results.json. For other suites or history, query the Evals Braintrust project. For example, replace `EXPERIMENT_ID` below to read an experiment's scored runs:
 

apps/web/scripts/fetch-braintrust.test.ts

@@ -2,14 +2,17 @@ import { test } from "vitest"
 import assert from "node:assert/strict"
 import {
   assemble,
+  attachRationales,
   createBraintrustRequest,
   fetchExperiments,
+  publicationPins,
   retryAfterMs,
 } from "./fetch-braintrust.ts"
 
 const id = "63331944-8a8c-4396-8add-d6fbe88f80d3"
 const experiment: Parameters<typeof assemble>[0][number] = {
   id,
+  created: "2026-10-28T18:53:01.928Z",
   metadata: {
     experiment: "example",
     expected_runs: 1,
@@ -59,6 +62,45 @@ const usage: Parameters<typeof assemble>[3][number] = {
 const make = (roots = [root], children = [score, task], usages = [usage]) =>
   assemble([experiment], roots, children, usages, ["example-eval"])
 
+test("carries the pinned experiment's rationale into published results", () => {
+  const rationale = {
+    summary: ["A ", { text: "model", url: "https://example.com" }, "."],
+  }
+  const withoutSkills = "d2e33ddc-64f1-4392-892c-d4db9f3d0146"
+  const entries = [
+    { experiment: "example", withSkills: id, withoutSkills, rationale },
+  ]
+  const [pinned, unskilled] = attachRationales(
+    [experiment, { ...experiment, id: withoutSkills }],
+    entries
+  )
+  assert.deepEqual(unskilled.rationale, rationale)
+  const [result] = assemble(
+    [pinned],
+    [root],
+    [score, task],
+    [usage],
+    ["example-eval"]
+  )
+  assert.deepEqual(result.experimentRationale, rationale)
+  assert.equal(result.experimentCreatedAt, experiment.created)
+  assert.throws(() => attachRationales([experiment], []), /Missing rationale/)
+})
+
+test("requires both variants outside partial mode and rejects duplicate names", () => {
+  const rationale = { summary: ["A model."] }
+  const entry = { experiment: "example", withSkills: id, rationale }
+  assert.throws(
+    () => publicationPins([entry], false),
+    /Missing pinned variants/
+  )
+  assert.deepEqual(publicationPins([entry], true), { example: id })
+  assert.throws(
+    () => publicationPins([entry, entry], true),
+    /distinct experiment names/
+  )
+})
+
 test("joins failed runs, scorer checks, task duration, and trace usage", () => {
   const [result] = make()
   assert.equal(result.passed, false)

apps/web/scripts/fetch-braintrust.ts

@@ -4,33 +4,27 @@ import { fileURLToPath } from "node:url"
 import { setTimeout as delay } from "node:timers/promises"
 import { z } from "zod"
 import pRetry, { AbortError } from "p-retry"
-import { evalResultSchema } from "@supabase-evals/core/eval-metadata"
+import {
+  evalResultSchema,
+  experimentRationaleSchema,
+} from "@supabase-evals/core/eval-metadata"
+import { publication } from "../src/data/publication.ts"
 
 const app = resolve(dirname(fileURLToPath(import.meta.url)), "..")
 const root = resolve(app, "../..")
 const output = resolve(app, "src/data/braintrust-results.json")
-const publication = {
-  "claude-code-opus-5.5": "c8e0f865-ca59-4703-a19d-507a234e100a",
-  "claude-code-opus-5.5-no-skills": "e0fd16ad-e117-4165-8e30-9346ebe7e835",
-  "claude-code-sonnet-5.5": "ede84738-9f79-428f-bdfe-9d8adb2a6489",
-  "claude-code-sonnet-5.5-no-skills": "15a067ef-f04e-44f5-9f94-7b8c0b0e10f1",
-  "codex-gpt-6-luna": "b4a9c5ce-60fc-4604-bc99-595269b7fd7d",
-  "codex-gpt-6-luna-no-skills": "123ecf1b-1004-458c-98cf-b17450b65c92",
-  "codex-gpt-6.1-sol": "d2e33ddc-64f1-4392-892c-d4db9f3d0146",
-  "codex-gpt-6.1-sol-no-skills": "f2a8dad6-2d8b-4088-8f5f-165d00a24058",
-  "grok-4.7": "e986dead-a1d3-4523-b47d-33a82ff1348b",
-  "grok-4.7-no-skills": "545924a6-a02c-482c-9ef7-b4dfc1a993c7",
-  "opencode-kimi-k3": "66f84377-4b0d-4be2-8f41-8e0756cbc40a",
-  "opencode-kimi-k3-no-skills": "63331944-8a8c-4396-8add-d6fbe88f80d3",
-}
 const uuid = z.uuid()
 const metadataSchema = z
   .object({
     experiment: z.string(),
     expected_runs: z.number().int().positive().optional(),
   })
   .passthrough()
-const experimentSchema = z.object({ id: uuid, metadata: metadataSchema })
+const experimentSchema = z.object({
+  id: uuid,
+  created: z.iso.datetime(),
+  metadata: metadataSchema,
+})
 const rootSchema = z.object({
   id: z.string(),
   experiment_id: uuid,
@@ -64,7 +58,41 @@ const usageSchema = z.object({
   input_tokens: z.number().nullable(),
   output_tokens: z.number().nullable(),
 })
-type Experiment = z.infer<typeof experimentSchema>
+type Experiment = z.infer<typeof experimentSchema> & {
+  rationale?: z.infer<typeof experimentRationaleSchema>
+}
+
+export function attachRationales(
+  experiments: Experiment[],
+  entries: typeof publication
+) {
+  return experiments.map((experiment) => {
+    const entry = entries.find(
+      ({ withSkills, withoutSkills }) =>
+        withSkills === experiment.id || withoutSkills === experiment.id
+    )
+    if (!entry) throw new Error(`Missing rationale for ${experiment.id}`)
+    return { ...experiment, rationale: entry.rationale }
+  })
+}
+
+export function publicationPins(
+  entries: typeof publication,
+  allowPartial: boolean
+) {
+  const pinEntries: [string, string][] = []
+  for (const { experiment, withSkills, withoutSkills } of entries) {
+    if ((!withSkills || !withoutSkills) && !allowPartial)
+      throw new Error(`Missing pinned variants: ${experiment}`)
+    if (withSkills) pinEntries.push([experiment, withSkills])
+    if (withoutSkills)
+      pinEntries.push([`${experiment}-no-skills`, withoutSkills])
+  }
+  const pins = Object.fromEntries(pinEntries)
+  if (Object.keys(pins).length !== pinEntries.length)
+    throw new Error("Publication must contain distinct experiment names")
+  return pins
+}
 type Root = z.infer<typeof rootSchema>
 type Child = z.infer<typeof childSchema>
 type Usage = z.infer<typeof usageSchema>
@@ -154,6 +182,8 @@ export function assemble(
           }
     const parsed = evalResultSchema.safeParse({
       experiment: em.experiment,
+      experimentRationale: exp.rationale,
+      experimentCreatedAt: exp.created,
       experimentDisplay: {
         agent: em.agent,
         modelProvider: em.model_provider,
@@ -419,37 +449,24 @@ async function main() {
     "Content-Type": "application/json",
   }
   const request = createBraintrustRequest(headers, fetch)
-  const experiments = await fetchExperiments(request, projectId, publication)
+  const pins = publicationPins(publication, allowPartial)
+  const experiments = attachRationales(
+    await fetchExperiments(request, projectId, pins),
+    publication
+  )
   const experimentNames = (
     await readdir(resolve(root, "experiments/benchmark"))
   )
-    .filter((f) => f.endsWith(".experiment.ts"))
-    .map((f) => f.slice(0, -".experiment.ts".length))
-    .sort()
-  const selectedNames = experiments.map((e) => e.metadata.experiment).sort()
-  if (
-    !allowPartial &&
-    JSON.stringify(experimentNames) !== JSON.stringify(selectedNames)
-  ) {
-    const missing = experimentNames.filter(
-      (name) => !selectedNames.includes(name)
-    )
-    const unexpected = selectedNames.filter(
-      (name) => !experimentNames.includes(name)
-    )
-    throw new Error(
-      [
-        "Pinned experiments do not match active benchmark experiments.",
-        ...(missing.length ? [`Missing: ${missing.join(", ")}`] : []),
-        ...(unexpected.length ? [`Unexpected: ${unexpected.join(", ")}`] : []),
-        ...(process.argv.includes("--dev")
-          ? [
-              "To run with partial results locally, use EVALS_ALLOW_PARTIAL=1 pnpm web.",
-            ]
-          : []),
-      ].join("\n")
+    .filter((name) => name.endsWith(".experiment.ts"))
+    .map((name) => name.slice(0, -".experiment.ts".length))
+  const missing = experimentNames.filter((name) => !(name in pins))
+  const unexpected = Object.keys(pins).filter(
+    (name) => !experimentNames.includes(name)
+  )
+  if (!allowPartial && (missing.length || unexpected.length))
+    console.warn(
+      `Published experiments differ from current benchmark. Missing: ${missing.join(", ") || "none"}. Unexpected: ${unexpected.join(", ") || "none"}.`
     )
-  }
   const evalIds = []
   for (const dir of await readdir(resolve(root, "evals/benchmark"), {
     withFileTypes: true,

apps/web/scripts/prerender.ts

@@ -12,8 +12,6 @@ const htmlFile = path.join(appDir, "dist", "index.html")
 const serverEntryFile = path.join(appDir, "dist-ssr", "entry-server.js")
 
 const ROOT_DIV = '<div id="root"></div>'
-/** The results table renders one header row above its data rows. */
-const HEADER_ROWS = 1
 /** Rendered by App when the results file parses to nothing. */
 const EMPTY_STATE = "No result files found"
 
@@ -73,15 +71,10 @@ if (markup.includes(EMPTY_STATE)) {
   fail("the app rendered its empty state, so the results data did not load")
 }
 
-const expectedRows = expectedTableRows + HEADER_ROWS
-const tableRows = markup.match(/<tr\b/g)?.length ?? 0
-if (!markup.includes("<table")) {
-  fail("the rendered markup has no results table")
-}
-
-if (tableRows !== expectedRows) {
+const renderedResultRowCount = markup.split('data-result-row=""').length - 1
+if (renderedResultRowCount !== expectedTableRows) {
   fail(
-    `the rendered table has ${tableRows} rows, expected ${expectedRows} (${expectedTableRows} data rows plus a header)`
+    `rendered ${renderedResultRowCount} results rows, expected ${expectedTableRows}`
   )
 }
 
@@ -104,5 +97,5 @@ copyFileSync(
 )
 writeFileSync(htmlFile, prerendered)
 console.log(
-  `prerender: wrote ${path.relative(appDir, htmlFile)} (${tableRows} table rows, ${markup.length} bytes of markup)`
+  `prerender: wrote ${path.relative(appDir, htmlFile)} (${renderedResultRowCount} result rows, ${markup.length} bytes of markup)`
 )

apps/web/src/App.tsx

@@ -1,6 +1,7 @@
 import { useQueryStates } from "nuqs"
 
 import { EvalOverviewCards } from "@/components/eval-overview-cards"
+import { ExperimentRationale } from "@/components/experiment-rationale"
 import { PageContainer } from "@/components/page-container"
 import { ResultsTable } from "@/components/results/results-table"
 import { SiteFooter } from "@/components/site-footer"
@@ -43,9 +44,12 @@ export function App() {
                 }}
               />
             </PageContainer>
-            <PageContainer className="pb-24 md:pb-28">
+            <PageContainer className="pb-10 md:pb-12">
               <EvalOverviewCards />
             </PageContainer>
+            <PageContainer className="pb-24 md:pb-28">
+              <ExperimentRationale />
+            </PageContainer>
             <SiteFooter />
           </>
         ) : (

apps/web/src/components/experiment-rationale.test.tsx

@@ -0,0 +1,32 @@
+import { renderToStaticMarkup } from "react-dom/server"
+import { describe, expect, it, vi } from "vitest"
+
+vi.mock("@/lib/eval-results", () => ({
+  getSuiteResults: () => [],
+  getVisibleExperiments: () => ["test-experiment"],
+  getExperimentDisplay: () => undefined,
+  getExperimentResults: () => [
+    { experimentCreatedAt: "2026-10-28T18:53:01.928Z" },
+  ],
+  getExperimentRationale: () => ({
+    summary: [
+      "See the ",
+      { text: "model", url: "https://example.com/model" },
+      ".",
+    ],
+  }),
+}))
+
+import { ExperimentRationale } from "./experiment-rationale"
+
+describe("inline rationale links", () => {
+  it("links the claim inside the sentence", () => {
+    const html = renderToStaticMarkup(<ExperimentRationale />)
+    expect(html).toContain('See the <a href="https://example.com/model"')
+    expect(html).toContain(">model</a>.")
+    expect(html).toContain("Experiment upload dates")
+    expect(html).toContain(
+      'dateTime="2026-10-28T18:53:01.928Z">2026-10-28</time>'
+    )
+  })
+})

apps/web/src/components/experiment-rationale.tsx

@@ -0,0 +1,81 @@
+import { ExperimentLabel } from "@/components/results/experiment-label"
+import {
+  getExperimentRationale,
+  getExperimentResults,
+  getSuiteResults,
+  getVisibleExperiments,
+} from "@/lib/eval-results"
+
+export function ExperimentRationale() {
+  const experiments = getVisibleExperiments(
+    getSuiteResults("benchmark")
+  ).filter((experiment) => getExperimentRationale(experiment))
+
+  if (!experiments.length) {
+    return null
+  }
+
+  return (
+    <section aria-labelledby="experiment-rationale-title">
+      <h2
+        id="experiment-rationale-title"
+        className="text-base font-medium text-foreground"
+      >
+        Why these experiments
+      </h2>
+      <ul className="mt-4 divide-y divide-border rounded-lg border border-border bg-card">
+        {experiments.map((experiment) => {
+          const rationale = getExperimentRationale(experiment)
+          return (
+            <li key={experiment} className="flex flex-col gap-2 px-5 py-5">
+              <span className="text-sm font-medium text-foreground [&_span]:text-foreground">
+                <ExperimentLabel experiment={experiment} />
+              </span>
+              <p className="text-[13px] leading-6 text-muted-foreground">
+                {rationale?.summary.map((part, index) =>
+                  typeof part === "string" ? (
+                    part
+                  ) : (
+                    <a
+                      key={index}
+                      href={part.url}
+                      target="_blank"
+                      rel="noreferrer"
+                      className="underline decoration-muted-foreground/40 underline-offset-4 hover:text-foreground hover:decoration-foreground"
+                    >
+                      {part.text}
+                    </a>
+                  )
+                )}
+              </p>
+            </li>
+          )
+        })}
+      </ul>
+      <div className="mt-6 text-xs text-muted-foreground">
+        <h3 className="text-sm font-medium text-foreground">
+          Experiment upload dates
+        </h3>
+        <table className="mt-3 text-left">
+          <caption className="sr-only">Experiment upload dates</caption>
+          <tbody>
+            {experiments.map((experiment) => {
+              const createdAt =
+                getExperimentResults(experiment)[0]?.experimentCreatedAt
+              return (
+                <tr key={experiment}>
+                  <th scope="row" className="py-1 pr-6 font-normal">
+                    <ExperimentLabel experiment={experiment} />
+                  </th>
+                  <td className="py-1 align-top whitespace-nowrap tabular-nums">
+                    <time dateTime={createdAt}>{createdAt?.slice(0, 10)}</time>
+                  </td>
+                </tr>
+              )
+            })}
+          </tbody>
+        </table>
+      </div>
+    </section>
+  )
+}