diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index dcb03762..1b725930 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -64,9 +64,9 @@ Include refreshed results for PRs with new/changed evals so a reviewer can inspe ## Publishing and consuming results -To publish a benchmark, upload complete runs with `--runs` and update the experiment IDs in [fetch-braintrust.ts](apps/web/scripts/fetch-braintrust.ts). Include every experiment in `experiments/benchmark/`, using its name as the key. +To publish a benchmark, upload complete runs with `--runs` and update the pins and rationales in [publication.ts](apps/web/src/data/publication.ts). Include every experiment in `experiments/benchmark/`. -For local UI work with a partial run, temporarily pin only the uploaded experiment IDs and start the server with `EVALS_ALLOW_PARTIAL=1 pnpm web`. +For local UI work with a partial run, temporarily keep only the uploaded experiment IDs and their rationales in the catalog and start the server with `EVALS_ALLOW_PARTIAL=1 pnpm web`. The published benchmark is at https://supabase.com/evals/eval-results.json. For other suites or history, query the Evals Braintrust project. For example, replace `EXPERIMENT_ID` below to read an experiment's scored runs: diff --git a/apps/web/scripts/fetch-braintrust.test.ts b/apps/web/scripts/fetch-braintrust.test.ts index 163e79a6..b88521a1 100644 --- a/apps/web/scripts/fetch-braintrust.test.ts +++ b/apps/web/scripts/fetch-braintrust.test.ts @@ -2,14 +2,17 @@ import { test } from "vitest" import assert from "node:assert/strict" import { assemble, + attachRationales, createBraintrustRequest, fetchExperiments, + publicationPins, retryAfterMs, } from "./fetch-braintrust.ts" const id = "63331944-8a8c-4396-8add-d6fbe88f80d3" const experiment: Parameters[0][number] = { id, + created: "2026-10-28T18:53:01.928Z", metadata: { experiment: "example", expected_runs: 1, @@ -59,6 +62,45 @@ const usage: Parameters[3][number] = { const make = (roots = [root], children = [score, task], usages = [usage]) => assemble([experiment], roots, children, usages, ["example-eval"]) +test("carries the pinned experiment's rationale into published results", () => { + const rationale = { + summary: ["A ", { text: "model", url: "https://example.com" }, "."], + } + const withoutSkills = "d2e33ddc-64f1-4392-892c-d4db9f3d0146" + const entries = [ + { experiment: "example", withSkills: id, withoutSkills, rationale }, + ] + const [pinned, unskilled] = attachRationales( + [experiment, { ...experiment, id: withoutSkills }], + entries + ) + assert.deepEqual(unskilled.rationale, rationale) + const [result] = assemble( + [pinned], + [root], + [score, task], + [usage], + ["example-eval"] + ) + assert.deepEqual(result.experimentRationale, rationale) + assert.equal(result.experimentCreatedAt, experiment.created) + assert.throws(() => attachRationales([experiment], []), /Missing rationale/) +}) + +test("requires both variants outside partial mode and rejects duplicate names", () => { + const rationale = { summary: ["A model."] } + const entry = { experiment: "example", withSkills: id, rationale } + assert.throws( + () => publicationPins([entry], false), + /Missing pinned variants/ + ) + assert.deepEqual(publicationPins([entry], true), { example: id }) + assert.throws( + () => publicationPins([entry, entry], true), + /distinct experiment names/ + ) +}) + test("joins failed runs, scorer checks, task duration, and trace usage", () => { const [result] = make() assert.equal(result.passed, false) diff --git a/apps/web/scripts/fetch-braintrust.ts b/apps/web/scripts/fetch-braintrust.ts index c255d461..1f046d8d 100644 --- a/apps/web/scripts/fetch-braintrust.ts +++ b/apps/web/scripts/fetch-braintrust.ts @@ -4,25 +4,15 @@ import { fileURLToPath } from "node:url" import { setTimeout as delay } from "node:timers/promises" import { z } from "zod" import pRetry, { AbortError } from "p-retry" -import { evalResultSchema } from "@supabase-evals/core/eval-metadata" +import { + evalResultSchema, + experimentRationaleSchema, +} from "@supabase-evals/core/eval-metadata" +import { publication } from "../src/data/publication.ts" const app = resolve(dirname(fileURLToPath(import.meta.url)), "..") const root = resolve(app, "../..") const output = resolve(app, "src/data/braintrust-results.json") -const publication = { - "claude-code-opus-5.5": "c8e0f865-ca59-4703-a19d-507a234e100a", - "claude-code-opus-5.5-no-skills": "e0fd16ad-e117-4165-8e30-9346ebe7e835", - "claude-code-sonnet-5.5": "ede84738-9f79-428f-bdfe-9d8adb2a6489", - "claude-code-sonnet-5.5-no-skills": "15a067ef-f04e-44f5-9f94-7b8c0b0e10f1", - "codex-gpt-6-luna": "b4a9c5ce-60fc-4604-bc99-595269b7fd7d", - "codex-gpt-6-luna-no-skills": "123ecf1b-1004-458c-98cf-b17450b65c92", - "codex-gpt-6.1-sol": "d2e33ddc-64f1-4392-892c-d4db9f3d0146", - "codex-gpt-6.1-sol-no-skills": "f2a8dad6-2d8b-4088-8f5f-165d00a24058", - "grok-4.7": "e986dead-a1d3-4523-b47d-33a82ff1348b", - "grok-4.7-no-skills": "545924a6-a02c-482c-9ef7-b4dfc1a993c7", - "opencode-kimi-k3": "66f84377-4b0d-4be2-8f41-8e0756cbc40a", - "opencode-kimi-k3-no-skills": "63331944-8a8c-4396-8add-d6fbe88f80d3", -} const uuid = z.uuid() const metadataSchema = z .object({ @@ -30,7 +20,11 @@ const metadataSchema = z expected_runs: z.number().int().positive().optional(), }) .passthrough() -const experimentSchema = z.object({ id: uuid, metadata: metadataSchema }) +const experimentSchema = z.object({ + id: uuid, + created: z.iso.datetime(), + metadata: metadataSchema, +}) const rootSchema = z.object({ id: z.string(), experiment_id: uuid, @@ -64,7 +58,41 @@ const usageSchema = z.object({ input_tokens: z.number().nullable(), output_tokens: z.number().nullable(), }) -type Experiment = z.infer +type Experiment = z.infer & { + rationale?: z.infer +} + +export function attachRationales( + experiments: Experiment[], + entries: typeof publication +) { + return experiments.map((experiment) => { + const entry = entries.find( + ({ withSkills, withoutSkills }) => + withSkills === experiment.id || withoutSkills === experiment.id + ) + if (!entry) throw new Error(`Missing rationale for ${experiment.id}`) + return { ...experiment, rationale: entry.rationale } + }) +} + +export function publicationPins( + entries: typeof publication, + allowPartial: boolean +) { + const pinEntries: [string, string][] = [] + for (const { experiment, withSkills, withoutSkills } of entries) { + if ((!withSkills || !withoutSkills) && !allowPartial) + throw new Error(`Missing pinned variants: ${experiment}`) + if (withSkills) pinEntries.push([experiment, withSkills]) + if (withoutSkills) + pinEntries.push([`${experiment}-no-skills`, withoutSkills]) + } + const pins = Object.fromEntries(pinEntries) + if (Object.keys(pins).length !== pinEntries.length) + throw new Error("Publication must contain distinct experiment names") + return pins +} type Root = z.infer type Child = z.infer type Usage = z.infer @@ -154,6 +182,8 @@ export function assemble( } const parsed = evalResultSchema.safeParse({ experiment: em.experiment, + experimentRationale: exp.rationale, + experimentCreatedAt: exp.created, experimentDisplay: { agent: em.agent, modelProvider: em.model_provider, @@ -419,37 +449,24 @@ async function main() { "Content-Type": "application/json", } const request = createBraintrustRequest(headers, fetch) - const experiments = await fetchExperiments(request, projectId, publication) + const pins = publicationPins(publication, allowPartial) + const experiments = attachRationales( + await fetchExperiments(request, projectId, pins), + publication + ) const experimentNames = ( await readdir(resolve(root, "experiments/benchmark")) ) - .filter((f) => f.endsWith(".experiment.ts")) - .map((f) => f.slice(0, -".experiment.ts".length)) - .sort() - const selectedNames = experiments.map((e) => e.metadata.experiment).sort() - if ( - !allowPartial && - JSON.stringify(experimentNames) !== JSON.stringify(selectedNames) - ) { - const missing = experimentNames.filter( - (name) => !selectedNames.includes(name) - ) - const unexpected = selectedNames.filter( - (name) => !experimentNames.includes(name) - ) - throw new Error( - [ - "Pinned experiments do not match active benchmark experiments.", - ...(missing.length ? [`Missing: ${missing.join(", ")}`] : []), - ...(unexpected.length ? [`Unexpected: ${unexpected.join(", ")}`] : []), - ...(process.argv.includes("--dev") - ? [ - "To run with partial results locally, use EVALS_ALLOW_PARTIAL=1 pnpm web.", - ] - : []), - ].join("\n") + .filter((name) => name.endsWith(".experiment.ts")) + .map((name) => name.slice(0, -".experiment.ts".length)) + const missing = experimentNames.filter((name) => !(name in pins)) + const unexpected = Object.keys(pins).filter( + (name) => !experimentNames.includes(name) + ) + if (!allowPartial && (missing.length || unexpected.length)) + console.warn( + `Published experiments differ from current benchmark. Missing: ${missing.join(", ") || "none"}. Unexpected: ${unexpected.join(", ") || "none"}.` ) - } const evalIds = [] for (const dir of await readdir(resolve(root, "evals/benchmark"), { withFileTypes: true, diff --git a/apps/web/scripts/prerender.ts b/apps/web/scripts/prerender.ts index d2ada590..5b1d0ac0 100644 --- a/apps/web/scripts/prerender.ts +++ b/apps/web/scripts/prerender.ts @@ -12,8 +12,6 @@ const htmlFile = path.join(appDir, "dist", "index.html") const serverEntryFile = path.join(appDir, "dist-ssr", "entry-server.js") const ROOT_DIV = '
' -/** The results table renders one header row above its data rows. */ -const HEADER_ROWS = 1 /** Rendered by App when the results file parses to nothing. */ const EMPTY_STATE = "No result files found" @@ -73,15 +71,10 @@ if (markup.includes(EMPTY_STATE)) { fail("the app rendered its empty state, so the results data did not load") } -const expectedRows = expectedTableRows + HEADER_ROWS -const tableRows = markup.match(/ - + + + + ) : ( diff --git a/apps/web/src/components/experiment-rationale.test.tsx b/apps/web/src/components/experiment-rationale.test.tsx new file mode 100644 index 00000000..627f9341 --- /dev/null +++ b/apps/web/src/components/experiment-rationale.test.tsx @@ -0,0 +1,32 @@ +import { renderToStaticMarkup } from "react-dom/server" +import { describe, expect, it, vi } from "vitest" + +vi.mock("@/lib/eval-results", () => ({ + getSuiteResults: () => [], + getVisibleExperiments: () => ["test-experiment"], + getExperimentDisplay: () => undefined, + getExperimentResults: () => [ + { experimentCreatedAt: "2026-10-28T18:53:01.928Z" }, + ], + getExperimentRationale: () => ({ + summary: [ + "See the ", + { text: "model", url: "https://example.com/model" }, + ".", + ], + }), +})) + +import { ExperimentRationale } from "./experiment-rationale" + +describe("inline rationale links", () => { + it("links the claim inside the sentence", () => { + const html = renderToStaticMarkup() + expect(html).toContain('See the model.") + expect(html).toContain("Experiment upload dates") + expect(html).toContain( + 'dateTime="2026-10-28T18:53:01.928Z">2026-10-28' + ) + }) +}) diff --git a/apps/web/src/components/experiment-rationale.tsx b/apps/web/src/components/experiment-rationale.tsx new file mode 100644 index 00000000..e622c2d9 --- /dev/null +++ b/apps/web/src/components/experiment-rationale.tsx @@ -0,0 +1,81 @@ +import { ExperimentLabel } from "@/components/results/experiment-label" +import { + getExperimentRationale, + getExperimentResults, + getSuiteResults, + getVisibleExperiments, +} from "@/lib/eval-results" + +export function ExperimentRationale() { + const experiments = getVisibleExperiments( + getSuiteResults("benchmark") + ).filter((experiment) => getExperimentRationale(experiment)) + + if (!experiments.length) { + return null + } + + return ( +
+

+ Why these experiments +

+
    + {experiments.map((experiment) => { + const rationale = getExperimentRationale(experiment) + return ( +
  • + + + +

    + {rationale?.summary.map((part, index) => + typeof part === "string" ? ( + part + ) : ( + + {part.text} + + ) + )} +

    +
  • + ) + })} +
+
+

+ Experiment upload dates +

+ + + + {experiments.map((experiment) => { + const createdAt = + getExperimentResults(experiment)[0]?.experimentCreatedAt + return ( + + + + + ) + })} + +
Experiment upload dates
+ + + +
+
+
+ ) +} diff --git a/apps/web/src/components/results/results-table.tsx b/apps/web/src/components/results/results-table.tsx index 65f680d1..bbc2bd61 100644 --- a/apps/web/src/components/results/results-table.tsx +++ b/apps/web/src/components/results/results-table.tsx @@ -302,6 +302,7 @@ export function ResultsTable({ return ( [number] + +// Review the rationale whenever either pinned experiment ID changes. +const entries = [ + { + experiment: "claude-code-opus-5.5", + withSkills: "c8e0f865-ca59-4703-a19d-507a234e100a", + withoutSkills: "e0fd16ad-e117-4165-8e30-9346ebe7e835", + rationale: { + summary: [ + "Claude Code's ", + { + text: "default model", + url: "https://code.claude.com/docs/en/model-config#default-model-setting", + }, + " on most plans, at its ", + { + text: "default effort", + url: "https://code.claude.com/docs/en/model-config#adjust-effort-level", + }, + ".", + ], + }, + }, + { + experiment: "claude-code-sonnet-5.5", + withSkills: "ede84738-9f79-428f-bdfe-9d8adb2a6489", + withoutSkills: "15a067ef-f04e-44f5-9f94-7b8c0b0e10f1", + rationale: { + summary: [ + "Claude Code's ", + { + text: "Sonnet model for daily coding tasks", + url: "https://code.claude.com/docs/en/model-config", + }, + ", at its ", + { + text: "default effort", + url: "https://code.claude.com/docs/en/model-config#adjust-effort-level", + }, + ".", + ], + }, + }, + { + experiment: "codex-gpt-6-luna", + withSkills: "b4a9c5ce-60fc-4604-bc99-595269b7fd7d", + withoutSkills: "123ecf1b-1004-458c-98cf-b17450b65c92", + rationale: { + summary: [ + "A cost-effective model for everyday tasks, and the one behind the ", + { + text: "Supabase dashboard assistant", + url: "https://github.com/supabase/supabase/blob/d98c277790766d2ceb905b675eb0a718f25e6345/apps/studio/lib/ai/model.utils.ts#L92", + }, + ".", + ], + }, + }, + { + experiment: "codex-gpt-6.1-sol", + withSkills: "d2e33ddc-64f1-4392-892c-d4db9f3d0146", + withoutSkills: "f2a8dad6-2d8b-4088-8f5f-165d00a24058", + rationale: { + summary: [ + { + text: "OpenAI's recommended Codex model for paid ChatGPT plans", + url: "https://learn.chatgpt.com/docs/models?surface=cli#gpt-55-retirement", + }, + ", run at medium effort.", + ], + }, + }, + { + experiment: "grok-4.7", + withSkills: "e986dead-a1d3-4523-b47d-33a82ff1348b", + withoutSkills: "545924a6-a02c-482c-9ef7-b4dfc1a993c7", + rationale: { + summary: [ + "Grok Build's ", + { + text: "default model", + url: "https://docs.x.ai/developers/grok-4-7", + }, + ", at its ", + { + text: "default effort", + url: "https://github.com/xai-org/grok-build/blob/2bdd1d6a6369de0e8c68132ea4539e9abd9e14a8/crates/codegen/xai-grok-shell/src/remote/client.rs#L832-L862", + }, + ".", + ], + }, + }, + { + experiment: "opencode-kimi-k3", + withSkills: "66f84377-4b0d-4be2-8f41-8e0756cbc40a", + withoutSkills: "63331944-8a8c-4396-8add-d6fbe88f80d3", + rationale: { + summary: [ + "Included as an ", + { + text: "open-weight option", + url: "https://opencode.ai/data/moonshotai/kimi-k3", + }, + " in OpenCode.", + ], + }, + }, +] satisfies Publication[] + +export const publication = publicationSchema.parse(entries) diff --git a/apps/web/src/lib/eval-results.test.ts b/apps/web/src/lib/eval-results.test.ts index a9bd03fa..5cf41f43 100644 --- a/apps/web/src/lib/eval-results.test.ts +++ b/apps/web/src/lib/eval-results.test.ts @@ -7,6 +7,7 @@ import { JOURNEY_STAGES, UNASSIGNED_PRODUCT, getEvalIds, + getExperimentRationale, getProductKeys, getProductResults, getVisibleExperiments, @@ -15,6 +16,16 @@ import { sortedResults, } from "@/lib/eval-results" +it("reads the rationale carried by the published result", () => { + const rationale = { summary: ["Published choice."] } + const result = makeResult({ + experiment: "pinned", + experimentRationale: rationale, + }) + expect(getExperimentRationale("pinned", [result])).toEqual(rationale) + expect(getExperimentRationale("other", [result])).toBeUndefined() +}) + describe("sortResults", () => { const order = (results: ReturnType[]) => [...results].sort(sortResults).map((result) => result.eval) diff --git a/apps/web/src/lib/eval-results.ts b/apps/web/src/lib/eval-results.ts index 455081ba..6b7d9131 100644 --- a/apps/web/src/lib/eval-results.ts +++ b/apps/web/src/lib/eval-results.ts @@ -126,6 +126,14 @@ export function getExperimentDisplay(experiment: string) { ?.experimentDisplay } +export function getExperimentRationale( + experiment: string, + sourceResults = sortedResults +) { + return sourceResults.find((result) => result.experiment === experiment) + ?.experimentRationale +} + export function getExperimentResults( experiment: string, sourceResults = sortedResults diff --git a/packages/core/src/eval-metadata.ts b/packages/core/src/eval-metadata.ts index 0762689f..e3b11934 100644 --- a/packages/core/src/eval-metadata.ts +++ b/packages/core/src/eval-metadata.ts @@ -457,9 +457,22 @@ const evalResultShape = { // Raw result files may carry extra fields we don't model; tolerate them. export const rawEvalResultSchema = z.looseObject(evalResultShape); +export const experimentRationaleSchema = z.object({ + summary: z + .array( + z.union([ + z.string().min(1), + z.object({ text: z.string().min(1), url: z.url() }), + ]) + ) + .min(1), +}); + // Web-facing result; a clean strict object so its inferred type stays usable. export const evalResultSchema = z.object({ ...evalResultShape, + experimentRationale: experimentRationaleSchema.optional(), + experimentCreatedAt: z.iso.datetime().optional(), passed: z.boolean(), prompt: z.string().optional(), promptSourcePath: z.string().optional(),