From 92cea6ba3acfd1db44d248c06ab8b52f53e85410 Mon Sep 17 00:00:00 2001 From: Matt Rossman <22670878+mattrossman@users.noreply.github.com> Date: Mon, 26 Oct 2026 10:49:25 -0400 Subject: [PATCH 01/18] chore(evals): align Claude Code benchmark presets with current defaults --- README.md | 6 +++--- .../README.md | 2 +- ...de-code-sonnet-5.5-no-skills.experiment.ts} | 4 ++-- ...ts => claude-code-sonnet-5.5.experiment.ts} | 4 ++-- experiments/presets.ts | 8 ++++---- .../core/src/agents/claude-code/runner.test.ts | 12 ++++++------ packages/core/src/agents/claude-code/runner.ts | 2 +- .../core/src/agents/opencode/runner.test.ts | 18 +++++++++--------- packages/core/src/index.test.ts | 4 ++-- 9 files changed, 30 insertions(+), 30 deletions(-) rename experiments/ai/{claude-code-sonnet-5-no-skills.experiment.ts => claude-code-sonnet-5.5-no-skills.experiment.ts} (63%) rename experiments/ai/{claude-code-sonnet-5.experiment.ts => claude-code-sonnet-5.5.experiment.ts} (60%) diff --git a/README.md b/README.md index 521c0f499..f6994a5e9 100644 --- a/README.md +++ b/README.md @@ -42,7 +42,7 @@ Running evals executes experiment x eval pairs and writes local result files und Run a single eval with one experiment: ```bash -pnpm eval -- --eval resolve-dataapi-001-empty-results --experiment claude-code-sonnet-5 +pnpm eval -- --eval resolve-dataapi-001-empty-results --experiment claude-code-sonnet-5.5 ``` @@ -50,7 +50,7 @@ Run selected evals across multiple experiments: ```bash pnpm eval -- \ - --experiment claude-code-sonnet-5 \ + --experiment claude-code-sonnet-5.5 \ --experiment claude-code-opus-5.5 \ --eval resolve-dataapi-001-empty-results \ --eval investigate-auth-001-deleted-user-access @@ -64,7 +64,7 @@ Each run writes to `results///run-/result.json`. Add ```bash pnpm eval -- --eval resolve-dataapi-001-empty-results \ - --experiment claude-code-sonnet-5 \ + --experiment claude-code-sonnet-5.5 \ --runs 3 \ --skip-existing ``` diff --git a/evals/docs/build-docs-001-homepage-quickstart/README.md b/evals/docs/build-docs-001-homepage-quickstart/README.md index 67210b556..0622e31e7 100644 --- a/evals/docs/build-docs-001-homepage-quickstart/README.md +++ b/evals/docs/build-docs-001-homepage-quickstart/README.md @@ -6,6 +6,6 @@ This guide explains how to add an eval for another AI setup prompt from the docs 2. Copy the target prompt verbatim into `PROMPT.md`. Don't paraphrase it or add detail that the prompt doesn't already give. Keep `skills: []` and `skipCliInstall: true` in the frontmatter if the prompt has the agent install its own tooling. 3. Seed `local/` with a minimal, unmodified starter that matches the guide's context. Don't include a `supabase/` directory, so the "is Supabase already initialized" step has something real to check. 4. Reuse this eval's `EVAL.ts` checks as a starting point. Adjust the checks to match what the new guide's prompt requires, including the judge rubric's wording. -5. Validate the eval before you open a PR. Run `pnpm eval:dry -- --eval --experiment claude-code-sonnet-5` to check the setup, and then run `pnpm eval -- --eval --experiment claude-code-sonnet-5` for a full pass. +5. Validate the eval before you open a PR. Run `pnpm eval:dry -- --eval --experiment claude-code-sonnet-5.5` to check the setup, and then run `pnpm eval -- --eval --experiment claude-code-sonnet-5.5` for a full pass. For the full eval-authoring workflow, including frontmatter fields, suite selection, and submitting for review, see `CONTRIBUTING.md`. diff --git a/experiments/ai/claude-code-sonnet-5-no-skills.experiment.ts b/experiments/ai/claude-code-sonnet-5.5-no-skills.experiment.ts similarity index 63% rename from experiments/ai/claude-code-sonnet-5-no-skills.experiment.ts rename to experiments/ai/claude-code-sonnet-5.5-no-skills.experiment.ts index df5f5b70a..3e4f69924 100644 --- a/experiments/ai/claude-code-sonnet-5-no-skills.experiment.ts +++ b/experiments/ai/claude-code-sonnet-5.5-no-skills.experiment.ts @@ -1,8 +1,8 @@ import { defineExperiment } from '@supabase-evals/core'; -import { claudeCodeSonnet5 } from '../presets.js'; +import { claudeCodeSonnet55 } from '../presets.js'; export default defineExperiment({ - ...claudeCodeSonnet5, + ...claudeCodeSonnet55, suite: ['no-skills'], skills: [], }); diff --git a/experiments/ai/claude-code-sonnet-5.experiment.ts b/experiments/ai/claude-code-sonnet-5.5.experiment.ts similarity index 60% rename from experiments/ai/claude-code-sonnet-5.experiment.ts rename to experiments/ai/claude-code-sonnet-5.5.experiment.ts index da35fac1f..6eee1d6f3 100644 --- a/experiments/ai/claude-code-sonnet-5.experiment.ts +++ b/experiments/ai/claude-code-sonnet-5.5.experiment.ts @@ -1,7 +1,7 @@ import { defineExperiment } from '@supabase-evals/core'; -import { claudeCodeSonnet5 } from '../presets.js'; +import { claudeCodeSonnet55 } from '../presets.js'; export default defineExperiment({ - ...claudeCodeSonnet5, + ...claudeCodeSonnet55, suite: ['benchmark'], }); diff --git a/experiments/presets.ts b/experiments/presets.ts index 0e62f846c..991c68d93 100644 --- a/experiments/presets.ts +++ b/experiments/presets.ts @@ -25,17 +25,17 @@ function defaultRuntime() { export const claudeCodeOpus55 = { agent: claudeCodeAgent({ model: 'claude-opus-5-5', - reasoningEffort: 'high', + reasoningEffort: 'medium', }), runtime: defaultRuntime(), localStack: localStackRuntime(), skills, } satisfies ExperimentPreset; -export const claudeCodeSonnet5 = { +export const claudeCodeSonnet55 = { agent: claudeCodeAgent({ - model: 'claude-sonnet-5', - reasoningEffort: 'high', + model: 'claude-sonnet-5-5', + reasoningEffort: 'medium', }), runtime: defaultRuntime(), localStack: localStackRuntime(), diff --git a/packages/core/src/agents/claude-code/runner.test.ts b/packages/core/src/agents/claude-code/runner.test.ts index 075f0f19d..a8ba68ced 100644 --- a/packages/core/src/agents/claude-code/runner.test.ts +++ b/packages/core/src/agents/claude-code/runner.test.ts @@ -97,7 +97,7 @@ describe('claudeCodeRunner.extractUsage', () => { subtype: 'success', usage: { input_tokens: 10, output_tokens: 40 }, modelUsage: { - 'claude-sonnet-5': { + 'claude-sonnet-5-5': { inputTokens: 10, cacheCreationInputTokens: 200, cacheReadInputTokens: 3000, @@ -114,9 +114,9 @@ describe('claudeCodeRunner.extractUsage', () => { }, }), ].join('\n'); - expect(extract(raw, 'claude-sonnet-5')).toEqual([ + expect(extract(raw, 'claude-sonnet-5-5')).toEqual([ { - model: 'claude-sonnet-5', + model: 'claude-sonnet-5-5', inputTokens: 3210, cacheReadInputTokens: 3000, cacheWriteInputTokens: 200, @@ -133,12 +133,12 @@ describe('claudeCodeRunner.extractUsage', () => { }); it('returns undefined without modelUsage', () => { - expect(extract(undefined, 'claude-sonnet-5')).toBeUndefined(); - expect(extract('not json\n', 'claude-sonnet-5')).toBeUndefined(); + expect(extract(undefined, 'claude-sonnet-5-5')).toBeUndefined(); + expect(extract('not json\n', 'claude-sonnet-5-5')).toBeUndefined(); expect( extract( JSON.stringify({ type: 'result', subtype: 'success' }), - 'claude-sonnet-5' + 'claude-sonnet-5-5' ) ).toBeUndefined(); }); diff --git a/packages/core/src/agents/claude-code/runner.ts b/packages/core/src/agents/claude-code/runner.ts index dd37dbf60..cba2928ea 100644 --- a/packages/core/src/agents/claude-code/runner.ts +++ b/packages/core/src/agents/claude-code/runner.ts @@ -26,7 +26,7 @@ export const claudeCodeRunner: AgentRunner = { cliPackage: '@anthropic-ai/claude-code', // Pinned: Claude Code's transcript format evolves; bump deliberately and // re-check the parser. See ./parser.ts. - defaultCliVersion: '2.1.280', + defaultCliVersion: '2.1.284', defaultModel: 'claude-sonnet-4-6', sessionDir: '"$HOME/.claude/projects"', diff --git a/packages/core/src/agents/opencode/runner.test.ts b/packages/core/src/agents/opencode/runner.test.ts index a40bfef53..f2fff3781 100644 --- a/packages/core/src/agents/opencode/runner.test.ts +++ b/packages/core/src/agents/opencode/runner.test.ts @@ -21,7 +21,7 @@ const SESSION = [ describe('opencode runner', () => { it("resolves the results-metadata provider from the slug's vendor prefix", () => { - expect(providerForModel('anthropic/claude-sonnet-5')).toBe('anthropic'); + expect(providerForModel('anthropic/claude-sonnet-5-5')).toBe('anthropic'); expect(providerForModel('openai/gpt-5.4')).toBe('openai'); expect(providerForModel('moonshotai/kimi-k3')).toBe('moonshotai'); }); @@ -40,7 +40,7 @@ describe('opencode runner', () => { }); it('deriveStopReason reads the terminal step_finish reason', () => { - const runner = createOpencodeRunner('anthropic/claude-sonnet-5'); + const runner = createOpencodeRunner('anthropic/claude-sonnet-5-5'); const ok = { ok: true, exitCode: 0, stdout: '', stderr: '' }; expect(runner.deriveStopReason!(SESSION, ok)).toBe('stop'); // A non-stop terminal reason is surfaced verbatim. @@ -84,7 +84,7 @@ describe('opencode runner', () => { }); describe('opencode runner extractUsage', () => { - const extract = createOpencodeRunner('anthropic/claude-sonnet-5') + const extract = createOpencodeRunner('anthropic/claude-sonnet-5-5') .extractUsage!; it('adds cache into input and reasoning into output', () => { @@ -117,9 +117,9 @@ describe('opencode runner extractUsage', () => { }, }), ].join('\n'); - expect(extract(raw, 'anthropic/claude-sonnet-5')).toEqual([ + expect(extract(raw, 'anthropic/claude-sonnet-5-5')).toEqual([ { - model: 'anthropic/claude-sonnet-5', + model: 'anthropic/claude-sonnet-5-5', inputTokens: 440, cacheReadInputTokens: 130, cacheWriteInputTokens: 10, @@ -129,15 +129,15 @@ describe('opencode runner extractUsage', () => { }); it('returns undefined without usage', () => { - expect(extract(undefined, 'anthropic/claude-sonnet-5')).toBeUndefined(); - expect(extract('not json\n', 'anthropic/claude-sonnet-5')).toBeUndefined(); + expect(extract(undefined, 'anthropic/claude-sonnet-5-5')).toBeUndefined(); + expect(extract('not json\n', 'anthropic/claude-sonnet-5-5')).toBeUndefined(); expect( extract( JSON.stringify({ type: 'step_finish', part: { type: 'step-finish', reason: 'stop' }, }), - 'anthropic/claude-sonnet-5' + 'anthropic/claude-sonnet-5-5' ) ).toBeUndefined(); }); @@ -213,7 +213,7 @@ describe('opencode runner exec routing', () => { }); describe('opencode runner extractStepCount', () => { - const extract = createOpencodeRunner('anthropic/claude-sonnet-5') + const extract = createOpencodeRunner('anthropic/claude-sonnet-5-5') .extractStepCount!; it('counts step_finish records', () => { diff --git a/packages/core/src/index.test.ts b/packages/core/src/index.test.ts index 64e05d3a8..c7a8ca89d 100644 --- a/packages/core/src/index.test.ts +++ b/packages/core/src/index.test.ts @@ -9,7 +9,7 @@ describe('aiSdkAgent usage', () => { vi.stubEnv('ANTHROPIC_API_KEY', 'test'); // to satisfy `assertProviderReady()` const model = new MockLanguageModelV3({ provider: 'anthropic.messages', - modelId: 'claude-sonnet-5', + modelId: 'claude-sonnet-5-5', doGenerate: async () => ({ content: [{ type: 'text', text: 'Done.' }], finishReason: { unified: 'stop', raw: 'stop' }, @@ -34,7 +34,7 @@ describe('aiSdkAgent usage', () => { expect(run.usage).toEqual([ { - model: 'claude-sonnet-5', + model: 'claude-sonnet-5-5', inputTokens: 3210, cacheReadInputTokens: 3000, cacheWriteInputTokens: 200, From 24491af7fedbbfe2f984c31c7a708e51222487a2 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 26 Oct 2026 14:57:23 +0000 Subject: [PATCH 02/18] chore: refresh eval results --- apps/web/src/data/eval-results.json | 309 ++++++++++++---------------- 1 file changed, 127 insertions(+), 182 deletions(-) diff --git a/apps/web/src/data/eval-results.json b/apps/web/src/data/eval-results.json index ce7cde21e..8b233cbff 100644 --- a/apps/web/src/data/eval-results.json +++ b/apps/web/src/data/eval-results.json @@ -5697,7 +5697,7 @@ "agent": "claude-code", "modelProvider": "anthropic", "modelId": "claude-opus-5-5", - "reasoningEffort": "high" + "reasoningEffort": "medium" }, "eval": "resolve-dataapi-001-empty-results", "stage": "resolve", @@ -5741,7 +5741,7 @@ { "name": "diagnosed RLS and added owner-scoped policies", "passed": true, - "judgeNotes": "Diagnosed RLS deny-all, kept RLS enabled, and created authenticated SELECT and INSERT policies scoped to each user's bookmarks." + "judgeNotes": "Correctly diagnoses RLS enabled without policies as deny-all, creates authenticated-only owner-scoped SELECT and INSERT policies using auth.uid(), and leaves RLS enabled." } ], "skills": { @@ -5749,9 +5749,7 @@ "supabase", "supabase-postgres-best-practices" ], - "loaded": [ - "supabase" - ] + "loaded": [] }, "docs": { "calls": [] @@ -5759,193 +5757,29 @@ "usage": [ { "model": "claude-opus-5-5", - "inputTokens": 291639, - "cacheReadInputTokens": 270261, - "cacheWriteInputTokens": 21356, - "outputTokens": 5474 + "inputTokens": 296229, + "cacheReadInputTokens": 279491, + "cacheWriteInputTokens": 16712, + "outputTokens": 4205 } ], - "stepCount": 17, + "stepCount": 16, "toolCallCount": 15, - "agentRunDurationMs": 60110, + "agentRunDurationMs": 49585, "sandboxUsage": { - "memory": 8192 + "activeCpuDurationMs": 155711, + "duration": 169764, + "memory": 8192, + "networkTransfer": { + "ingress": 721169174, + "egress": 2570080 + } }, "prompt": "Our app lets signed-in users save bookmarks and view them on their dashboard. Bookmarks are stored in the `bookmarks` table and are private — a user must only ever see their own. \nUsers also need to be able to save new bookmarks from the app.\n\nI can see the rows when I query the table directly, but the dashboard shows an empty list for every user.\n\nFind out why the Data API returns nothing and fix it.", "promptSourcePath": "evals/benchmark/resolve-dataapi-001-empty-results/PROMPT.md", "run": 1, "sourcePath": "claude-code-opus-5.5/resolve-dataapi-001-empty-results/run-1/result.json" }, - { - "experiment": "claude-code-opus-5.5", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "claude-code", - "modelProvider": "anthropic", - "modelId": "claude-opus-5-5", - "reasoningEffort": "high" - }, - "eval": "resolve-dataapi-001-empty-results", - "stage": "resolve", - "product": [ - "data-api", - "database", - "auth" - ], - "topic": [ - "rls", - "sdk" - ], - "suite": "benchmark", - "interface": "mcp", - "passed": true, - "checks": [ - { - "name": "RLS still enabled on bookmarks", - "passed": true - }, - { - "name": "user A reads own bookmarks", - "passed": true - }, - { - "name": "user B cannot read user A bookmarks", - "passed": true - }, - { - "name": "anon reads no bookmarks", - "passed": true - }, - { - "name": "user A can save a new bookmark", - "passed": true - }, - { - "name": "user B cannot insert a bookmark as user A", - "passed": true - }, - { - "name": "diagnosed RLS and added owner-scoped policies", - "passed": true, - "judgeNotes": "Diagnosed RLS with no policies as the cause of empty Data API results, created owner-scoped SELECT and INSERT policies for authenticated users, kept RLS enabled, and verified access." - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase" - ] - }, - "docs": { - "calls": [] - }, - "usage": [ - { - "model": "claude-opus-5-5", - "inputTokens": 315506, - "cacheReadInputTokens": 293993, - "cacheWriteInputTokens": 21489, - "outputTokens": 5299 - } - ], - "stepCount": 18, - "toolCallCount": 16, - "agentRunDurationMs": 66314, - "sandboxUsage": { - "memory": 8192 - }, - "prompt": "Our app lets signed-in users save bookmarks and view them on their dashboard. Bookmarks are stored in the `bookmarks` table and are private — a user must only ever see their own. \nUsers also need to be able to save new bookmarks from the app.\n\nI can see the rows when I query the table directly, but the dashboard shows an empty list for every user.\n\nFind out why the Data API returns nothing and fix it.", - "promptSourcePath": "evals/benchmark/resolve-dataapi-001-empty-results/PROMPT.md", - "run": 2, - "sourcePath": "claude-code-opus-5.5/resolve-dataapi-001-empty-results/run-2/result.json" - }, - { - "experiment": "claude-code-opus-5.5", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "claude-code", - "modelProvider": "anthropic", - "modelId": "claude-opus-5-5", - "reasoningEffort": "high" - }, - "eval": "resolve-dataapi-001-empty-results", - "stage": "resolve", - "product": [ - "data-api", - "database", - "auth" - ], - "topic": [ - "rls", - "sdk" - ], - "suite": "benchmark", - "interface": "mcp", - "passed": true, - "checks": [ - { - "name": "RLS still enabled on bookmarks", - "passed": true - }, - { - "name": "user A reads own bookmarks", - "passed": true - }, - { - "name": "user B cannot read user A bookmarks", - "passed": true - }, - { - "name": "anon reads no bookmarks", - "passed": true - }, - { - "name": "user A can save a new bookmark", - "passed": true - }, - { - "name": "user B cannot insert a bookmark as user A", - "passed": true - }, - { - "name": "diagnosed RLS and added owner-scoped policies", - "passed": true, - "judgeNotes": "Correctly diagnosed deny-all RLS, created authenticated-only owner-scoped SELECT and INSERT policies, and kept RLS enabled." - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [] - }, - "docs": { - "calls": [] - }, - "usage": [ - { - "model": "claude-opus-5-5", - "inputTokens": 323324, - "cacheReadInputTokens": 305485, - "cacheWriteInputTokens": 17811, - "outputTokens": 5414 - } - ], - "stepCount": 17, - "toolCallCount": 16, - "agentRunDurationMs": 59898, - "sandboxUsage": { - "memory": 8192 - }, - "prompt": "Our app lets signed-in users save bookmarks and view them on their dashboard. Bookmarks are stored in the `bookmarks` table and are private — a user must only ever see their own. \nUsers also need to be able to save new bookmarks from the app.\n\nI can see the rows when I query the table directly, but the dashboard shows an empty list for every user.\n\nFind out why the Data API returns nothing and fix it.", - "promptSourcePath": "evals/benchmark/resolve-dataapi-001-empty-results/PROMPT.md", - "run": 3, - "sourcePath": "claude-code-opus-5.5/resolve-dataapi-001-empty-results/run-3/result.json" - }, { "experiment": "claude-code-opus-5.5", "experimentSuite": "benchmark", @@ -24644,6 +24478,117 @@ "run": 3, "sourcePath": "claude-code-sonnet-5-no-skills/resolve-security-002-rls-cross-tenant-leak/run-3/result.json" }, + { + "experiment": "claude-code-sonnet-5.5", + "experimentSuite": "benchmark", + "experimentDisplay": { + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-sonnet-5-5", + "reasoningEffort": "medium" + }, + "eval": "resolve-dataapi-001-empty-results", + "stage": "resolve", + "product": [ + "data-api", + "database", + "auth" + ], + "topic": [ + "rls", + "sdk" + ], + "suite": "benchmark", + "interface": "mcp", + "passed": true, + "checks": [ + { + "name": "RLS still enabled on bookmarks", + "passed": true + }, + { + "name": "user A reads own bookmarks", + "passed": true + }, + { + "name": "user B cannot read user A bookmarks", + "passed": true + }, + { + "name": "anon reads no bookmarks", + "passed": true + }, + { + "name": "user A can save a new bookmark", + "passed": true + }, + { + "name": "user B cannot insert a bookmark as user A", + "passed": true + }, + { + "name": "diagnosed RLS and added owner-scoped policies", + "passed": true, + "judgeNotes": "Diagnosed RLS enabled without policies as deny-all, created authenticated-only owner-scoped SELECT and INSERT policies, and kept RLS enabled. Also tested ownership isolation and rejection of inserts for another user." + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase" + ] + }, + "docs": { + "calls": [ + { + "source": "web_fetch", + "query": "Summarize debugging guidance for empty results / RLS / permissions", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/monitoring-and-debugging.md" + } + ], + "resultChars": 1392 + } + ] + }, + "usage": [ + { + "model": "claude-sonnet-5-5", + "inputTokens": 164108, + "cacheReadInputTokens": 147998, + "cacheWriteInputTokens": 16096, + "outputTokens": 2430 + }, + { + "model": "claude-haiku-4-5-20251001", + "inputTokens": 1091, + "cacheReadInputTokens": 0, + "cacheWriteInputTokens": 0, + "outputTokens": 292 + } + ], + "stepCount": 12, + "toolCallCount": 10, + "agentRunDurationMs": 23457, + "sandboxUsage": { + "activeCpuDurationMs": 117336, + "duration": 121806, + "memory": 8192, + "networkTransfer": { + "ingress": 723349609, + "egress": 3870800 + } + }, + "prompt": "Our app lets signed-in users save bookmarks and view them on their dashboard. Bookmarks are stored in the `bookmarks` table and are private — a user must only ever see their own. \nUsers also need to be able to save new bookmarks from the app.\n\nI can see the rows when I query the table directly, but the dashboard shows an empty list for every user.\n\nFind out why the Data API returns nothing and fix it.", + "promptSourcePath": "evals/benchmark/resolve-dataapi-001-empty-results/PROMPT.md", + "run": 1, + "sourcePath": "claude-code-sonnet-5.5/resolve-dataapi-001-empty-results/run-1/result.json" + }, { "experiment": "codex-gpt-6-luna", "experimentSuite": "benchmark", From 6d1d4126c6d215a6d2587835caa65ba789b9f4b6 Mon Sep 17 00:00:00 2001 From: Matt Rossman <22670878+mattrossman@users.noreply.github.com> Date: Mon, 26 Oct 2026 10:58:10 -0400 Subject: [PATCH 03/18] Revert "chore: refresh eval results" This reverts commit 24491af7fedbbfe2f984c31c7a708e51222487a2. --- apps/web/src/data/eval-results.json | 309 ++++++++++++++++------------ 1 file changed, 182 insertions(+), 127 deletions(-) diff --git a/apps/web/src/data/eval-results.json b/apps/web/src/data/eval-results.json index 8b233cbff..ce7cde21e 100644 --- a/apps/web/src/data/eval-results.json +++ b/apps/web/src/data/eval-results.json @@ -5697,7 +5697,7 @@ "agent": "claude-code", "modelProvider": "anthropic", "modelId": "claude-opus-5-5", - "reasoningEffort": "medium" + "reasoningEffort": "high" }, "eval": "resolve-dataapi-001-empty-results", "stage": "resolve", @@ -5741,7 +5741,7 @@ { "name": "diagnosed RLS and added owner-scoped policies", "passed": true, - "judgeNotes": "Correctly diagnoses RLS enabled without policies as deny-all, creates authenticated-only owner-scoped SELECT and INSERT policies using auth.uid(), and leaves RLS enabled." + "judgeNotes": "Diagnosed RLS deny-all, kept RLS enabled, and created authenticated SELECT and INSERT policies scoped to each user's bookmarks." } ], "skills": { @@ -5749,7 +5749,9 @@ "supabase", "supabase-postgres-best-practices" ], - "loaded": [] + "loaded": [ + "supabase" + ] }, "docs": { "calls": [] @@ -5757,29 +5759,193 @@ "usage": [ { "model": "claude-opus-5-5", - "inputTokens": 296229, - "cacheReadInputTokens": 279491, - "cacheWriteInputTokens": 16712, - "outputTokens": 4205 + "inputTokens": 291639, + "cacheReadInputTokens": 270261, + "cacheWriteInputTokens": 21356, + "outputTokens": 5474 } ], - "stepCount": 16, + "stepCount": 17, "toolCallCount": 15, - "agentRunDurationMs": 49585, + "agentRunDurationMs": 60110, "sandboxUsage": { - "activeCpuDurationMs": 155711, - "duration": 169764, - "memory": 8192, - "networkTransfer": { - "ingress": 721169174, - "egress": 2570080 - } + "memory": 8192 }, "prompt": "Our app lets signed-in users save bookmarks and view them on their dashboard. Bookmarks are stored in the `bookmarks` table and are private — a user must only ever see their own. \nUsers also need to be able to save new bookmarks from the app.\n\nI can see the rows when I query the table directly, but the dashboard shows an empty list for every user.\n\nFind out why the Data API returns nothing and fix it.", "promptSourcePath": "evals/benchmark/resolve-dataapi-001-empty-results/PROMPT.md", "run": 1, "sourcePath": "claude-code-opus-5.5/resolve-dataapi-001-empty-results/run-1/result.json" }, + { + "experiment": "claude-code-opus-5.5", + "experimentSuite": "benchmark", + "experimentDisplay": { + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-opus-5-5", + "reasoningEffort": "high" + }, + "eval": "resolve-dataapi-001-empty-results", + "stage": "resolve", + "product": [ + "data-api", + "database", + "auth" + ], + "topic": [ + "rls", + "sdk" + ], + "suite": "benchmark", + "interface": "mcp", + "passed": true, + "checks": [ + { + "name": "RLS still enabled on bookmarks", + "passed": true + }, + { + "name": "user A reads own bookmarks", + "passed": true + }, + { + "name": "user B cannot read user A bookmarks", + "passed": true + }, + { + "name": "anon reads no bookmarks", + "passed": true + }, + { + "name": "user A can save a new bookmark", + "passed": true + }, + { + "name": "user B cannot insert a bookmark as user A", + "passed": true + }, + { + "name": "diagnosed RLS and added owner-scoped policies", + "passed": true, + "judgeNotes": "Diagnosed RLS with no policies as the cause of empty Data API results, created owner-scoped SELECT and INSERT policies for authenticated users, kept RLS enabled, and verified access." + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase" + ] + }, + "docs": { + "calls": [] + }, + "usage": [ + { + "model": "claude-opus-5-5", + "inputTokens": 315506, + "cacheReadInputTokens": 293993, + "cacheWriteInputTokens": 21489, + "outputTokens": 5299 + } + ], + "stepCount": 18, + "toolCallCount": 16, + "agentRunDurationMs": 66314, + "sandboxUsage": { + "memory": 8192 + }, + "prompt": "Our app lets signed-in users save bookmarks and view them on their dashboard. Bookmarks are stored in the `bookmarks` table and are private — a user must only ever see their own. \nUsers also need to be able to save new bookmarks from the app.\n\nI can see the rows when I query the table directly, but the dashboard shows an empty list for every user.\n\nFind out why the Data API returns nothing and fix it.", + "promptSourcePath": "evals/benchmark/resolve-dataapi-001-empty-results/PROMPT.md", + "run": 2, + "sourcePath": "claude-code-opus-5.5/resolve-dataapi-001-empty-results/run-2/result.json" + }, + { + "experiment": "claude-code-opus-5.5", + "experimentSuite": "benchmark", + "experimentDisplay": { + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-opus-5-5", + "reasoningEffort": "high" + }, + "eval": "resolve-dataapi-001-empty-results", + "stage": "resolve", + "product": [ + "data-api", + "database", + "auth" + ], + "topic": [ + "rls", + "sdk" + ], + "suite": "benchmark", + "interface": "mcp", + "passed": true, + "checks": [ + { + "name": "RLS still enabled on bookmarks", + "passed": true + }, + { + "name": "user A reads own bookmarks", + "passed": true + }, + { + "name": "user B cannot read user A bookmarks", + "passed": true + }, + { + "name": "anon reads no bookmarks", + "passed": true + }, + { + "name": "user A can save a new bookmark", + "passed": true + }, + { + "name": "user B cannot insert a bookmark as user A", + "passed": true + }, + { + "name": "diagnosed RLS and added owner-scoped policies", + "passed": true, + "judgeNotes": "Correctly diagnosed deny-all RLS, created authenticated-only owner-scoped SELECT and INSERT policies, and kept RLS enabled." + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [] + }, + "docs": { + "calls": [] + }, + "usage": [ + { + "model": "claude-opus-5-5", + "inputTokens": 323324, + "cacheReadInputTokens": 305485, + "cacheWriteInputTokens": 17811, + "outputTokens": 5414 + } + ], + "stepCount": 17, + "toolCallCount": 16, + "agentRunDurationMs": 59898, + "sandboxUsage": { + "memory": 8192 + }, + "prompt": "Our app lets signed-in users save bookmarks and view them on their dashboard. Bookmarks are stored in the `bookmarks` table and are private — a user must only ever see their own. \nUsers also need to be able to save new bookmarks from the app.\n\nI can see the rows when I query the table directly, but the dashboard shows an empty list for every user.\n\nFind out why the Data API returns nothing and fix it.", + "promptSourcePath": "evals/benchmark/resolve-dataapi-001-empty-results/PROMPT.md", + "run": 3, + "sourcePath": "claude-code-opus-5.5/resolve-dataapi-001-empty-results/run-3/result.json" + }, { "experiment": "claude-code-opus-5.5", "experimentSuite": "benchmark", @@ -24478,117 +24644,6 @@ "run": 3, "sourcePath": "claude-code-sonnet-5-no-skills/resolve-security-002-rls-cross-tenant-leak/run-3/result.json" }, - { - "experiment": "claude-code-sonnet-5.5", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "claude-code", - "modelProvider": "anthropic", - "modelId": "claude-sonnet-5-5", - "reasoningEffort": "medium" - }, - "eval": "resolve-dataapi-001-empty-results", - "stage": "resolve", - "product": [ - "data-api", - "database", - "auth" - ], - "topic": [ - "rls", - "sdk" - ], - "suite": "benchmark", - "interface": "mcp", - "passed": true, - "checks": [ - { - "name": "RLS still enabled on bookmarks", - "passed": true - }, - { - "name": "user A reads own bookmarks", - "passed": true - }, - { - "name": "user B cannot read user A bookmarks", - "passed": true - }, - { - "name": "anon reads no bookmarks", - "passed": true - }, - { - "name": "user A can save a new bookmark", - "passed": true - }, - { - "name": "user B cannot insert a bookmark as user A", - "passed": true - }, - { - "name": "diagnosed RLS and added owner-scoped policies", - "passed": true, - "judgeNotes": "Diagnosed RLS enabled without policies as deny-all, created authenticated-only owner-scoped SELECT and INSERT policies, and kept RLS enabled. Also tested ownership isolation and rejection of inserts for another user." - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase" - ] - }, - "docs": { - "calls": [ - { - "source": "web_fetch", - "query": "Summarize debugging guidance for empty results / RLS / permissions", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging.md" - } - ], - "resultChars": 1392 - } - ] - }, - "usage": [ - { - "model": "claude-sonnet-5-5", - "inputTokens": 164108, - "cacheReadInputTokens": 147998, - "cacheWriteInputTokens": 16096, - "outputTokens": 2430 - }, - { - "model": "claude-haiku-4-5-20251001", - "inputTokens": 1091, - "cacheReadInputTokens": 0, - "cacheWriteInputTokens": 0, - "outputTokens": 292 - } - ], - "stepCount": 12, - "toolCallCount": 10, - "agentRunDurationMs": 23457, - "sandboxUsage": { - "activeCpuDurationMs": 117336, - "duration": 121806, - "memory": 8192, - "networkTransfer": { - "ingress": 723349609, - "egress": 3870800 - } - }, - "prompt": "Our app lets signed-in users save bookmarks and view them on their dashboard. Bookmarks are stored in the `bookmarks` table and are private — a user must only ever see their own. \nUsers also need to be able to save new bookmarks from the app.\n\nI can see the rows when I query the table directly, but the dashboard shows an empty list for every user.\n\nFind out why the Data API returns nothing and fix it.", - "promptSourcePath": "evals/benchmark/resolve-dataapi-001-empty-results/PROMPT.md", - "run": 1, - "sourcePath": "claude-code-sonnet-5.5/resolve-dataapi-001-empty-results/run-1/result.json" - }, { "experiment": "codex-gpt-6-luna", "experimentSuite": "benchmark", From 47cb1b7611bc6b44cab71319529e204813d410d3 Mon Sep 17 00:00:00 2001 From: Matt Rossman <22670878+mattrossman@users.noreply.github.com> Date: Mon, 26 Oct 2026 11:16:30 -0400 Subject: [PATCH 04/18] chore(evals): bump pinned Claude Code CLI to 2.1.310 --- packages/core/src/agents/claude-code/runner.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/core/src/agents/claude-code/runner.ts b/packages/core/src/agents/claude-code/runner.ts index cba2928ea..ed031e932 100644 --- a/packages/core/src/agents/claude-code/runner.ts +++ b/packages/core/src/agents/claude-code/runner.ts @@ -26,7 +26,7 @@ export const claudeCodeRunner: AgentRunner = { cliPackage: '@anthropic-ai/claude-code', // Pinned: Claude Code's transcript format evolves; bump deliberately and // re-check the parser. See ./parser.ts. - defaultCliVersion: '2.1.284', + defaultCliVersion: '2.1.310', defaultModel: 'claude-sonnet-4-6', sessionDir: '"$HOME/.claude/projects"', From 1d09dd8b74f75d88c0ef40c7c96cc9f2b4dc9fc6 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 26 Oct 2026 15:20:55 +0000 Subject: [PATCH 05/18] chore: refresh eval results --- apps/web/src/data/eval-results.json | 268 +++++++++------------------- 1 file changed, 85 insertions(+), 183 deletions(-) diff --git a/apps/web/src/data/eval-results.json b/apps/web/src/data/eval-results.json index ce7cde21e..d8f3c5df7 100644 --- a/apps/web/src/data/eval-results.json +++ b/apps/web/src/data/eval-results.json @@ -5697,7 +5697,7 @@ "agent": "claude-code", "modelProvider": "anthropic", "modelId": "claude-opus-5-5", - "reasoningEffort": "high" + "reasoningEffort": "medium" }, "eval": "resolve-dataapi-001-empty-results", "stage": "resolve", @@ -5741,7 +5741,7 @@ { "name": "diagnosed RLS and added owner-scoped policies", "passed": true, - "judgeNotes": "Diagnosed RLS deny-all, kept RLS enabled, and created authenticated SELECT and INSERT policies scoped to each user's bookmarks." + "judgeNotes": "Diagnosed RLS enabled with no policies as deny-all. Created authenticated-only SELECT USING and INSERT WITH CHECK policies matching user_id to auth.uid(), and kept RLS enabled." } ], "skills": { @@ -5749,9 +5749,7 @@ "supabase", "supabase-postgres-best-practices" ], - "loaded": [ - "supabase" - ] + "loaded": [] }, "docs": { "calls": [] @@ -5759,193 +5757,29 @@ "usage": [ { "model": "claude-opus-5-5", - "inputTokens": 291639, - "cacheReadInputTokens": 270261, - "cacheWriteInputTokens": 21356, - "outputTokens": 5474 + "inputTokens": 279324, + "cacheReadInputTokens": 263102, + "cacheWriteInputTokens": 16198, + "outputTokens": 4214 } ], - "stepCount": 17, - "toolCallCount": 15, - "agentRunDurationMs": 60110, + "stepCount": 15, + "toolCallCount": 14, + "agentRunDurationMs": 47003, "sandboxUsage": { - "memory": 8192 + "activeCpuDurationMs": 145124, + "duration": 161920, + "memory": 8192, + "networkTransfer": { + "ingress": 746108858, + "egress": 3825012 + } }, "prompt": "Our app lets signed-in users save bookmarks and view them on their dashboard. Bookmarks are stored in the `bookmarks` table and are private — a user must only ever see their own. \nUsers also need to be able to save new bookmarks from the app.\n\nI can see the rows when I query the table directly, but the dashboard shows an empty list for every user.\n\nFind out why the Data API returns nothing and fix it.", "promptSourcePath": "evals/benchmark/resolve-dataapi-001-empty-results/PROMPT.md", "run": 1, "sourcePath": "claude-code-opus-5.5/resolve-dataapi-001-empty-results/run-1/result.json" }, - { - "experiment": "claude-code-opus-5.5", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "claude-code", - "modelProvider": "anthropic", - "modelId": "claude-opus-5-5", - "reasoningEffort": "high" - }, - "eval": "resolve-dataapi-001-empty-results", - "stage": "resolve", - "product": [ - "data-api", - "database", - "auth" - ], - "topic": [ - "rls", - "sdk" - ], - "suite": "benchmark", - "interface": "mcp", - "passed": true, - "checks": [ - { - "name": "RLS still enabled on bookmarks", - "passed": true - }, - { - "name": "user A reads own bookmarks", - "passed": true - }, - { - "name": "user B cannot read user A bookmarks", - "passed": true - }, - { - "name": "anon reads no bookmarks", - "passed": true - }, - { - "name": "user A can save a new bookmark", - "passed": true - }, - { - "name": "user B cannot insert a bookmark as user A", - "passed": true - }, - { - "name": "diagnosed RLS and added owner-scoped policies", - "passed": true, - "judgeNotes": "Diagnosed RLS with no policies as the cause of empty Data API results, created owner-scoped SELECT and INSERT policies for authenticated users, kept RLS enabled, and verified access." - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase" - ] - }, - "docs": { - "calls": [] - }, - "usage": [ - { - "model": "claude-opus-5-5", - "inputTokens": 315506, - "cacheReadInputTokens": 293993, - "cacheWriteInputTokens": 21489, - "outputTokens": 5299 - } - ], - "stepCount": 18, - "toolCallCount": 16, - "agentRunDurationMs": 66314, - "sandboxUsage": { - "memory": 8192 - }, - "prompt": "Our app lets signed-in users save bookmarks and view them on their dashboard. Bookmarks are stored in the `bookmarks` table and are private — a user must only ever see their own. \nUsers also need to be able to save new bookmarks from the app.\n\nI can see the rows when I query the table directly, but the dashboard shows an empty list for every user.\n\nFind out why the Data API returns nothing and fix it.", - "promptSourcePath": "evals/benchmark/resolve-dataapi-001-empty-results/PROMPT.md", - "run": 2, - "sourcePath": "claude-code-opus-5.5/resolve-dataapi-001-empty-results/run-2/result.json" - }, - { - "experiment": "claude-code-opus-5.5", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "claude-code", - "modelProvider": "anthropic", - "modelId": "claude-opus-5-5", - "reasoningEffort": "high" - }, - "eval": "resolve-dataapi-001-empty-results", - "stage": "resolve", - "product": [ - "data-api", - "database", - "auth" - ], - "topic": [ - "rls", - "sdk" - ], - "suite": "benchmark", - "interface": "mcp", - "passed": true, - "checks": [ - { - "name": "RLS still enabled on bookmarks", - "passed": true - }, - { - "name": "user A reads own bookmarks", - "passed": true - }, - { - "name": "user B cannot read user A bookmarks", - "passed": true - }, - { - "name": "anon reads no bookmarks", - "passed": true - }, - { - "name": "user A can save a new bookmark", - "passed": true - }, - { - "name": "user B cannot insert a bookmark as user A", - "passed": true - }, - { - "name": "diagnosed RLS and added owner-scoped policies", - "passed": true, - "judgeNotes": "Correctly diagnosed deny-all RLS, created authenticated-only owner-scoped SELECT and INSERT policies, and kept RLS enabled." - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [] - }, - "docs": { - "calls": [] - }, - "usage": [ - { - "model": "claude-opus-5-5", - "inputTokens": 323324, - "cacheReadInputTokens": 305485, - "cacheWriteInputTokens": 17811, - "outputTokens": 5414 - } - ], - "stepCount": 17, - "toolCallCount": 16, - "agentRunDurationMs": 59898, - "sandboxUsage": { - "memory": 8192 - }, - "prompt": "Our app lets signed-in users save bookmarks and view them on their dashboard. Bookmarks are stored in the `bookmarks` table and are private — a user must only ever see their own. \nUsers also need to be able to save new bookmarks from the app.\n\nI can see the rows when I query the table directly, but the dashboard shows an empty list for every user.\n\nFind out why the Data API returns nothing and fix it.", - "promptSourcePath": "evals/benchmark/resolve-dataapi-001-empty-results/PROMPT.md", - "run": 3, - "sourcePath": "claude-code-opus-5.5/resolve-dataapi-001-empty-results/run-3/result.json" - }, { "experiment": "claude-code-opus-5.5", "experimentSuite": "benchmark", @@ -24644,6 +24478,74 @@ "run": 3, "sourcePath": "claude-code-sonnet-5-no-skills/resolve-security-002-rls-cross-tenant-leak/run-3/result.json" }, + { + "experiment": "claude-code-sonnet-5.5", + "experimentSuite": "benchmark", + "experimentDisplay": { + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-sonnet-5-5", + "reasoningEffort": "medium" + }, + "eval": "resolve-dataapi-001-empty-results", + "stage": "resolve", + "product": [ + "data-api", + "database", + "auth" + ], + "topic": [ + "rls", + "sdk" + ], + "suite": "benchmark", + "interface": "mcp", + "passed": false, + "checks": [ + { + "name": "created auth sessions", + "passed": false, + "notes": "Internal server error" + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase" + ] + }, + "docs": { + "calls": [] + }, + "usage": [ + { + "model": "claude-sonnet-5-5", + "inputTokens": 407471, + "cacheReadInputTokens": 387145, + "cacheWriteInputTokens": 20296, + "outputTokens": 4168 + } + ], + "stepCount": 19, + "toolCallCount": 17, + "agentRunDurationMs": 32505, + "sandboxUsage": { + "activeCpuDurationMs": 190445, + "duration": 176643, + "memory": 8192, + "networkTransfer": { + "ingress": 744557150, + "egress": 2844827 + } + }, + "prompt": "Our app lets signed-in users save bookmarks and view them on their dashboard. Bookmarks are stored in the `bookmarks` table and are private — a user must only ever see their own. \nUsers also need to be able to save new bookmarks from the app.\n\nI can see the rows when I query the table directly, but the dashboard shows an empty list for every user.\n\nFind out why the Data API returns nothing and fix it.", + "promptSourcePath": "evals/benchmark/resolve-dataapi-001-empty-results/PROMPT.md", + "run": 1, + "sourcePath": "claude-code-sonnet-5.5/resolve-dataapi-001-empty-results/run-1/result.json" + }, { "experiment": "codex-gpt-6-luna", "experimentSuite": "benchmark", From fa8111e87877e80f58b96ae5db1c088de301ed91 Mon Sep 17 00:00:00 2001 From: Matt Rossman <22670878+mattrossman@users.noreply.github.com> Date: Mon, 26 Oct 2026 11:21:27 -0400 Subject: [PATCH 06/18] Revert "chore: refresh eval results" This reverts commit 1d09dd8b74f75d88c0ef40c7c96cc9f2b4dc9fc6. --- apps/web/src/data/eval-results.json | 268 +++++++++++++++++++--------- 1 file changed, 183 insertions(+), 85 deletions(-) diff --git a/apps/web/src/data/eval-results.json b/apps/web/src/data/eval-results.json index d8f3c5df7..ce7cde21e 100644 --- a/apps/web/src/data/eval-results.json +++ b/apps/web/src/data/eval-results.json @@ -5697,7 +5697,7 @@ "agent": "claude-code", "modelProvider": "anthropic", "modelId": "claude-opus-5-5", - "reasoningEffort": "medium" + "reasoningEffort": "high" }, "eval": "resolve-dataapi-001-empty-results", "stage": "resolve", @@ -5741,7 +5741,7 @@ { "name": "diagnosed RLS and added owner-scoped policies", "passed": true, - "judgeNotes": "Diagnosed RLS enabled with no policies as deny-all. Created authenticated-only SELECT USING and INSERT WITH CHECK policies matching user_id to auth.uid(), and kept RLS enabled." + "judgeNotes": "Diagnosed RLS deny-all, kept RLS enabled, and created authenticated SELECT and INSERT policies scoped to each user's bookmarks." } ], "skills": { @@ -5749,7 +5749,9 @@ "supabase", "supabase-postgres-best-practices" ], - "loaded": [] + "loaded": [ + "supabase" + ] }, "docs": { "calls": [] @@ -5757,29 +5759,193 @@ "usage": [ { "model": "claude-opus-5-5", - "inputTokens": 279324, - "cacheReadInputTokens": 263102, - "cacheWriteInputTokens": 16198, - "outputTokens": 4214 + "inputTokens": 291639, + "cacheReadInputTokens": 270261, + "cacheWriteInputTokens": 21356, + "outputTokens": 5474 } ], - "stepCount": 15, - "toolCallCount": 14, - "agentRunDurationMs": 47003, + "stepCount": 17, + "toolCallCount": 15, + "agentRunDurationMs": 60110, "sandboxUsage": { - "activeCpuDurationMs": 145124, - "duration": 161920, - "memory": 8192, - "networkTransfer": { - "ingress": 746108858, - "egress": 3825012 - } + "memory": 8192 }, "prompt": "Our app lets signed-in users save bookmarks and view them on their dashboard. Bookmarks are stored in the `bookmarks` table and are private — a user must only ever see their own. \nUsers also need to be able to save new bookmarks from the app.\n\nI can see the rows when I query the table directly, but the dashboard shows an empty list for every user.\n\nFind out why the Data API returns nothing and fix it.", "promptSourcePath": "evals/benchmark/resolve-dataapi-001-empty-results/PROMPT.md", "run": 1, "sourcePath": "claude-code-opus-5.5/resolve-dataapi-001-empty-results/run-1/result.json" }, + { + "experiment": "claude-code-opus-5.5", + "experimentSuite": "benchmark", + "experimentDisplay": { + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-opus-5-5", + "reasoningEffort": "high" + }, + "eval": "resolve-dataapi-001-empty-results", + "stage": "resolve", + "product": [ + "data-api", + "database", + "auth" + ], + "topic": [ + "rls", + "sdk" + ], + "suite": "benchmark", + "interface": "mcp", + "passed": true, + "checks": [ + { + "name": "RLS still enabled on bookmarks", + "passed": true + }, + { + "name": "user A reads own bookmarks", + "passed": true + }, + { + "name": "user B cannot read user A bookmarks", + "passed": true + }, + { + "name": "anon reads no bookmarks", + "passed": true + }, + { + "name": "user A can save a new bookmark", + "passed": true + }, + { + "name": "user B cannot insert a bookmark as user A", + "passed": true + }, + { + "name": "diagnosed RLS and added owner-scoped policies", + "passed": true, + "judgeNotes": "Diagnosed RLS with no policies as the cause of empty Data API results, created owner-scoped SELECT and INSERT policies for authenticated users, kept RLS enabled, and verified access." + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase" + ] + }, + "docs": { + "calls": [] + }, + "usage": [ + { + "model": "claude-opus-5-5", + "inputTokens": 315506, + "cacheReadInputTokens": 293993, + "cacheWriteInputTokens": 21489, + "outputTokens": 5299 + } + ], + "stepCount": 18, + "toolCallCount": 16, + "agentRunDurationMs": 66314, + "sandboxUsage": { + "memory": 8192 + }, + "prompt": "Our app lets signed-in users save bookmarks and view them on their dashboard. Bookmarks are stored in the `bookmarks` table and are private — a user must only ever see their own. \nUsers also need to be able to save new bookmarks from the app.\n\nI can see the rows when I query the table directly, but the dashboard shows an empty list for every user.\n\nFind out why the Data API returns nothing and fix it.", + "promptSourcePath": "evals/benchmark/resolve-dataapi-001-empty-results/PROMPT.md", + "run": 2, + "sourcePath": "claude-code-opus-5.5/resolve-dataapi-001-empty-results/run-2/result.json" + }, + { + "experiment": "claude-code-opus-5.5", + "experimentSuite": "benchmark", + "experimentDisplay": { + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-opus-5-5", + "reasoningEffort": "high" + }, + "eval": "resolve-dataapi-001-empty-results", + "stage": "resolve", + "product": [ + "data-api", + "database", + "auth" + ], + "topic": [ + "rls", + "sdk" + ], + "suite": "benchmark", + "interface": "mcp", + "passed": true, + "checks": [ + { + "name": "RLS still enabled on bookmarks", + "passed": true + }, + { + "name": "user A reads own bookmarks", + "passed": true + }, + { + "name": "user B cannot read user A bookmarks", + "passed": true + }, + { + "name": "anon reads no bookmarks", + "passed": true + }, + { + "name": "user A can save a new bookmark", + "passed": true + }, + { + "name": "user B cannot insert a bookmark as user A", + "passed": true + }, + { + "name": "diagnosed RLS and added owner-scoped policies", + "passed": true, + "judgeNotes": "Correctly diagnosed deny-all RLS, created authenticated-only owner-scoped SELECT and INSERT policies, and kept RLS enabled." + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [] + }, + "docs": { + "calls": [] + }, + "usage": [ + { + "model": "claude-opus-5-5", + "inputTokens": 323324, + "cacheReadInputTokens": 305485, + "cacheWriteInputTokens": 17811, + "outputTokens": 5414 + } + ], + "stepCount": 17, + "toolCallCount": 16, + "agentRunDurationMs": 59898, + "sandboxUsage": { + "memory": 8192 + }, + "prompt": "Our app lets signed-in users save bookmarks and view them on their dashboard. Bookmarks are stored in the `bookmarks` table and are private — a user must only ever see their own. \nUsers also need to be able to save new bookmarks from the app.\n\nI can see the rows when I query the table directly, but the dashboard shows an empty list for every user.\n\nFind out why the Data API returns nothing and fix it.", + "promptSourcePath": "evals/benchmark/resolve-dataapi-001-empty-results/PROMPT.md", + "run": 3, + "sourcePath": "claude-code-opus-5.5/resolve-dataapi-001-empty-results/run-3/result.json" + }, { "experiment": "claude-code-opus-5.5", "experimentSuite": "benchmark", @@ -24478,74 +24644,6 @@ "run": 3, "sourcePath": "claude-code-sonnet-5-no-skills/resolve-security-002-rls-cross-tenant-leak/run-3/result.json" }, - { - "experiment": "claude-code-sonnet-5.5", - "experimentSuite": "benchmark", - "experimentDisplay": { - "agent": "claude-code", - "modelProvider": "anthropic", - "modelId": "claude-sonnet-5-5", - "reasoningEffort": "medium" - }, - "eval": "resolve-dataapi-001-empty-results", - "stage": "resolve", - "product": [ - "data-api", - "database", - "auth" - ], - "topic": [ - "rls", - "sdk" - ], - "suite": "benchmark", - "interface": "mcp", - "passed": false, - "checks": [ - { - "name": "created auth sessions", - "passed": false, - "notes": "Internal server error" - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase" - ] - }, - "docs": { - "calls": [] - }, - "usage": [ - { - "model": "claude-sonnet-5-5", - "inputTokens": 407471, - "cacheReadInputTokens": 387145, - "cacheWriteInputTokens": 20296, - "outputTokens": 4168 - } - ], - "stepCount": 19, - "toolCallCount": 17, - "agentRunDurationMs": 32505, - "sandboxUsage": { - "activeCpuDurationMs": 190445, - "duration": 176643, - "memory": 8192, - "networkTransfer": { - "ingress": 744557150, - "egress": 2844827 - } - }, - "prompt": "Our app lets signed-in users save bookmarks and view them on their dashboard. Bookmarks are stored in the `bookmarks` table and are private — a user must only ever see their own. \nUsers also need to be able to save new bookmarks from the app.\n\nI can see the rows when I query the table directly, but the dashboard shows an empty list for every user.\n\nFind out why the Data API returns nothing and fix it.", - "promptSourcePath": "evals/benchmark/resolve-dataapi-001-empty-results/PROMPT.md", - "run": 1, - "sourcePath": "claude-code-sonnet-5.5/resolve-dataapi-001-empty-results/run-1/result.json" - }, { "experiment": "codex-gpt-6-luna", "experimentSuite": "benchmark", From 0ec0de6f27ccbecc534afc5e8b7a8a8ca01804d4 Mon Sep 17 00:00:00 2001 From: Matt Rossman <22670878+mattrossman@users.noreply.github.com> Date: Mon, 26 Oct 2026 11:33:48 -0400 Subject: [PATCH 07/18] style(evals): format opencode runner test --- packages/core/src/agents/opencode/runner.test.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/core/src/agents/opencode/runner.test.ts b/packages/core/src/agents/opencode/runner.test.ts index f2fff3781..1eee6bbdd 100644 --- a/packages/core/src/agents/opencode/runner.test.ts +++ b/packages/core/src/agents/opencode/runner.test.ts @@ -130,7 +130,9 @@ describe('opencode runner extractUsage', () => { it('returns undefined without usage', () => { expect(extract(undefined, 'anthropic/claude-sonnet-5-5')).toBeUndefined(); - expect(extract('not json\n', 'anthropic/claude-sonnet-5-5')).toBeUndefined(); + expect( + extract('not json\n', 'anthropic/claude-sonnet-5-5') + ).toBeUndefined(); expect( extract( JSON.stringify({ From 3ec9967e3795cc3aef0626ab5e9d43633d1c13cd Mon Sep 17 00:00:00 2001 From: Matt Rossman <22670878+mattrossman@users.noreply.github.com> Date: Mon, 26 Oct 2026 16:45:00 -0400 Subject: [PATCH 08/18] feat(evals): explain each benchmark experiment's model and effort on the results page --- apps/framework/lib/result-files.ts | 3 + apps/framework/scripts/export-results.test.ts | 36 +++++++++ apps/framework/scripts/export-results.ts | 17 ++++ apps/web/src/App.tsx | 6 +- .../src/components/experiment-rationale.tsx | 65 ++++++++++++++++ apps/web/src/lib/eval-results.ts | 5 ++ experiments/presets.ts | 77 ++++++++++++++++++- packages/core/src/eval-metadata.ts | 8 ++ packages/core/src/index.ts | 5 ++ 9 files changed, 220 insertions(+), 2 deletions(-) create mode 100644 apps/web/src/components/experiment-rationale.tsx diff --git a/apps/framework/lib/result-files.ts b/apps/framework/lib/result-files.ts index dbc79323c..80ad64922 100644 --- a/apps/framework/lib/result-files.ts +++ b/apps/framework/lib/result-files.ts @@ -11,6 +11,7 @@ import { getExperimentDisplayMetadata, type ExperimentConfig, type ExperimentDisplayMetadata, + type ExperimentRationale, } from '@supabase-evals/core'; import type { ExperimentSuite } from '@supabase-evals/core/eval-metadata'; import { discoverExperimentFiles } from './experiment-files.js'; @@ -23,6 +24,7 @@ const EXPERIMENTS_DIR = join(ROOT, 'experiments'); export interface ExperimentMetadata { display: ExperimentDisplayMetadata; + rationale?: ExperimentRationale; suites: ExperimentSuite[]; } @@ -35,6 +37,7 @@ export async function loadExperimentMetadata(): Promise< const config: ExperimentConfig = mod.default; map.set(experiment.name, { display: getExperimentDisplayMetadata(config), + rationale: config.rationale, suites: config.suite ?? [], }); } diff --git a/apps/framework/scripts/export-results.test.ts b/apps/framework/scripts/export-results.test.ts index d850fc541..e01e22898 100644 --- a/apps/framework/scripts/export-results.test.ts +++ b/apps/framework/scripts/export-results.test.ts @@ -32,3 +32,39 @@ describe('toEvalResult cliVersion precedence', () => { expect(result.cliVersion).toBe('2.109.1'); }); }); + +describe('toEvalResult experimentRationale', () => { + const display = { + agent: 'claude-code', + modelProvider: 'anthropic', + modelId: 'claude-opus-5-5', + reasoningEffort: 'medium', + } as const; + const experimentData = { + display, + rationale: { summary: 'Default model.', sources: [] }, + suites: [], + }; + + const build = async (experimentDisplay: Record) => + toEvalResult( + rawEvalResultSchema.parse({ + experiment: 'test-experiment', + eval: PINNED_EVAL_ID, + interface: 'cli', + experimentDisplay, + }), + 'test-experiment/run-1/result.json', + await readPrompt(PINNED_EVAL_ID), + experimentData + ); + + it('attaches the rationale when the run matches the config', async () => { + expect((await build(display)).experimentRationale).toBeDefined(); + }); + + it('drops the rationale when the run used a different effort', async () => { + const result = await build({ ...display, reasoningEffort: 'high' }); + expect(result.experimentRationale).toBeUndefined(); + }); +}); diff --git a/apps/framework/scripts/export-results.ts b/apps/framework/scripts/export-results.ts index 04c9c6f42..11b61db2b 100644 --- a/apps/framework/scripts/export-results.ts +++ b/apps/framework/scripts/export-results.ts @@ -7,6 +7,7 @@ import { rawEvalResultSchema } from '@supabase-evals/core/eval-metadata'; import type { EvalResult, EvalSuite, + ExperimentDisplayMetadata, ExperimentSuite, } from '@supabase-evals/core/eval-metadata'; import { @@ -54,6 +55,14 @@ const EXPECTED_RUNS_FLAG = readFlag(rawArgs, 'runs'); const OUTPUT_FLAG = readRepeatedFlag(rawArgs, 'output')[0]; const outputPath = OUTPUT_FLAG ? resolve(ROOT, OUTPUT_FLAG) : OUTPUT_PATH; +/** True when the run used the model and effort the experiment config names today. */ +function sameModelAndEffort( + a: ExperimentDisplayMetadata, + b: ExperimentDisplayMetadata | undefined +) { + return a.modelId === b?.modelId && a.reasoningEffort === b?.reasoningEffort; +} + export function toEvalResult( parsedResult: ReturnType, sourcePath: string, @@ -70,6 +79,14 @@ export function toEvalResult( experimentSuite, experimentDisplay: parsedResult.experimentDisplay ?? experimentData?.display, + experimentRationale: + !parsedResult.experimentDisplay || + sameModelAndEffort( + parsedResult.experimentDisplay, + experimentData?.display + ) + ? experimentData?.rationale + : undefined, eval: parsedResult.eval, stage: promptData?.stage ?? parsedResult.stage, product: promptData?.product ?? parsedResult.product, diff --git a/apps/web/src/App.tsx b/apps/web/src/App.tsx index 8f3f154ef..e2bdd072f 100644 --- a/apps/web/src/App.tsx +++ b/apps/web/src/App.tsx @@ -1,6 +1,7 @@ import { useQueryStates } from "nuqs" import { EvalOverviewCards } from "@/components/eval-overview-cards" +import { ExperimentRationale } from "@/components/experiment-rationale" import { PageContainer } from "@/components/page-container" import { ResultsTable } from "@/components/results/results-table" import { SiteFooter } from "@/components/site-footer" @@ -43,9 +44,12 @@ export function App() { }} /> - + + + + ) : ( diff --git a/apps/web/src/components/experiment-rationale.tsx b/apps/web/src/components/experiment-rationale.tsx new file mode 100644 index 000000000..6abf79c34 --- /dev/null +++ b/apps/web/src/components/experiment-rationale.tsx @@ -0,0 +1,65 @@ +import { ExperimentLabel } from "@/components/results/experiment-label" +import { + getExperimentRationale, + getSuiteResults, + getVisibleExperiments, +} from "@/lib/eval-results" + +/** Why each benchmark experiment uses its model and effort, with links to check it. */ +export function ExperimentRationale() { + const experiments = getVisibleExperiments( + getSuiteResults("benchmark") + ).filter((experiment) => getExperimentRationale(experiment)) + + if (!experiments.length) { + return null + } + + return ( +
+

+ Why these experiments +

+

+ Where an agent has a default model and effort, we use it. Where we + picked one ourselves, we say why. +

+
    + {experiments.map((experiment) => { + const rationale = getExperimentRationale(experiment) + return ( +
  • +
    + + + + + {rationale?.sources.map((source) => ( + + {source.label} + + ))} + +
    +

    + {rationale?.summary} +

    +
    • + ) + })} +
+
+ ) +} diff --git a/apps/web/src/lib/eval-results.ts b/apps/web/src/lib/eval-results.ts index a25674c72..a10da0f47 100644 --- a/apps/web/src/lib/eval-results.ts +++ b/apps/web/src/lib/eval-results.ts @@ -123,6 +123,11 @@ export function getExperimentDisplay(experiment: string) { ?.experimentDisplay } +export function getExperimentRationale(experiment: string) { + return sortedResults.find((result) => result.experiment === experiment) + ?.experimentRationale +} + export function getExperimentResults( experiment: string, sourceResults = sortedResults diff --git a/experiments/presets.ts b/experiments/presets.ts index 991c68d93..8cb416364 100644 --- a/experiments/presets.ts +++ b/experiments/presets.ts @@ -11,7 +11,7 @@ import { localStackRuntime } from '@supabase-evals/sandbox'; type ExperimentPreset = Pick< ExperimentConfig, - 'agent' | 'runtime' | 'localStack' | 'skills' + 'agent' | 'rationale' | 'runtime' | 'localStack' | 'skills' >; const skills = ['supabase', 'supabase-postgres-best-practices']; @@ -27,6 +27,20 @@ export const claudeCodeOpus55 = { model: 'claude-opus-5-5', reasoningEffort: 'medium', }), + rationale: { + summary: + "Claude Code's default model on most plans, at its default effort.", + sources: [ + { + label: 'Default model', + url: 'https://code.claude.com/docs/en/model-config#default-model-setting', + }, + { + label: 'Default effort', + url: 'https://code.claude.com/docs/en/model-config#adjust-effort-level', + }, + ], + }, runtime: defaultRuntime(), localStack: localStackRuntime(), skills, @@ -37,6 +51,20 @@ export const claudeCodeSonnet55 = { model: 'claude-sonnet-5-5', reasoningEffort: 'medium', }), + rationale: { + summary: + "Claude Code's Sonnet model for daily coding tasks, at its default effort.", + sources: [ + { + label: 'Model aliases', + url: 'https://code.claude.com/docs/en/model-config', + }, + { + label: 'Default effort', + url: 'https://code.claude.com/docs/en/model-config#adjust-effort-level', + }, + ], + }, runtime: defaultRuntime(), localStack: localStackRuntime(), skills, @@ -47,6 +75,16 @@ export const codexGpt6Luna = { model: 'gpt-6-luna', reasoningEffort: 'medium', }), + rationale: { + summary: + 'A cost-effective model for everyday tasks, and the one behind the Supabase dashboard assistant.', + sources: [ + { + label: 'Dashboard assistant default model', + url: 'https://github.com/supabase/supabase/blob/d98c277790766d2ceb905b675eb0a718f25e6345/apps/studio/lib/ai/model.utils.ts#L92', + }, + ], + }, runtime: defaultRuntime(), localStack: localStackRuntime(), skills, @@ -57,6 +95,19 @@ export const codexGpt61Sol = { model: 'gpt-6.1-sol', reasoningEffort: 'medium', }), + rationale: { + summary: "Codex's default model, at its default effort.", + sources: [ + { + label: 'Where the catalog comes from', + url: 'https://github.com/openai/codex/blob/03748ad5e18a3c9c1f0828561dcf1f3f0d7b2f77/codex-rs/models-manager/src/manager.rs#L394-L436', + }, + { + label: 'How the default is picked', + url: 'https://github.com/openai/codex/blob/7d4c7a0767cdf81ff76c8fa5a05c22929568e3c6/codex-rs/models-manager/src/manager.rs#L845-L851', + }, + ], + }, runtime: defaultRuntime(), localStack: localStackRuntime(), skills, @@ -67,6 +118,19 @@ export const grok47 = { model: 'grok-4.7', reasoningEffort: 'high', }), + rationale: { + summary: "Grok Build's default model, at its default effort.", + sources: [ + { + label: 'Grok 4.7 and Grok Build', + url: 'https://docs.x.ai/developers/grok-4-7', + }, + { + label: 'Default effort', + url: 'https://docs.x.ai/developers/models/grok-4.7', + }, + ], + }, runtime: defaultRuntime(), localStack: localStackRuntime(), skills, @@ -76,6 +140,17 @@ export const opencodeKimiK3 = { agent: opencodeAgent({ model: 'moonshotai/kimi-k3', }), + rationale: { + summary: + 'An open-weight model we picked to represent open-weight options in OpenCode.', + sources: [ + { label: 'OpenCode usage data', url: 'https://opencode.ai/data/' }, + { + label: 'Kimi K3 usage on OpenCode', + url: 'https://opencode.ai/data/moonshotai/kimi-k3', + }, + ], + }, runtime: defaultRuntime(), localStack: localStackRuntime(), skills, diff --git a/packages/core/src/eval-metadata.ts b/packages/core/src/eval-metadata.ts index d6f4d16a1..f18fd0e3e 100644 --- a/packages/core/src/eval-metadata.ts +++ b/packages/core/src/eval-metadata.ts @@ -96,6 +96,13 @@ export type ExperimentDisplayMetadata = z.infer< typeof experimentDisplayMetadataSchema >; +/** Why an experiment uses its model and effort, with links to check it. */ +export const experimentRationaleSchema = z.object({ + summary: z.string(), + sources: z.array(z.object({ label: z.string(), url: z.string().url() })), +}); +export type ExperimentRationale = z.infer; + /** * Interface(s) the agent uses to act on Supabase — a benchmark dimension * (cross-team KPI), not the runtime switch. `mcp` = the platform-lite MCP/tool @@ -424,6 +431,7 @@ const evalResultShape = { experimentSuite: experimentSuiteSchema.optional(), profile: experimentSuiteSchema.optional(), experimentDisplay: experimentDisplayMetadataSchema.optional(), + experimentRationale: experimentRationaleSchema.optional(), eval: z.string(), stage: evalStageSchema.optional(), product: z.array(evalProductSchema).optional(), diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index ef8bae3ee..7bf86cf7d 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -46,6 +46,7 @@ import type { EvalMetadata, EvalSuite, ExperimentDisplayMetadata, + ExperimentRationale, ExperimentSuite, JudgeCall, ModelProvider, @@ -93,6 +94,7 @@ export { evalSuiteSchema, experimentSuiteSchema, experimentDisplayMetadataSchema, + experimentRationaleSchema, modelProviderSchema, rawEvalResultSchema, reasoningEffortSchema, @@ -144,6 +146,7 @@ export type { EvalSuite, EvalStage, ExperimentDisplayMetadata, + ExperimentRationale, ExperimentSuite, ModelProvider, ParsedEvalMarkdown, @@ -645,6 +648,8 @@ export type ExperimentConfig = { /** Named experiment suites this experiment belongs to. */ suite?: ExperimentSuite[]; agent: AgentHarness; + /** Why this model and effort, shown next to the experiment on the results page. */ + rationale?: ExperimentRationale; runtime: EvalRuntime; /** Local-stack environment (e.g. localStackRuntime() from @supabase-evals/sandbox). */ localStack?: LocalStackRuntime; From e27d80aa4c2d2be5ca07b4c50b9d5d046a6ea2a4 Mon Sep 17 00:00:00 2001 From: Matt Rossman <22670878+mattrossman@users.noreply.github.com> Date: Wed, 28 Oct 2026 10:27:42 -0400 Subject: [PATCH 09/18] refactor: keep benchmark rationales in website catalog --- apps/framework/lib/result-files.ts | 3 - .../scripts/check-experiment-rationales.ts | 7 + apps/framework/scripts/export-results.test.ts | 36 ---- apps/framework/scripts/export-results.ts | 17 -- apps/web/package.json | 8 +- .../src/components/experiment-rationale.tsx | 8 +- apps/web/src/data/experiment-rationales.json | 158 ++++++++++++++++++ apps/web/src/lib/eval-results.test.ts | 77 +++++++++ apps/web/src/lib/eval-results.ts | 25 ++- apps/web/src/lib/experiment-rationales.ts | 51 ++++++ experiments/presets.ts | 77 +-------- packages/core/src/eval-metadata.ts | 1 - packages/core/src/index.ts | 5 - 13 files changed, 326 insertions(+), 147 deletions(-) create mode 100644 apps/framework/scripts/check-experiment-rationales.ts create mode 100644 apps/web/src/data/experiment-rationales.json create mode 100644 apps/web/src/lib/experiment-rationales.ts diff --git a/apps/framework/lib/result-files.ts b/apps/framework/lib/result-files.ts index 80ad64922..dbc79323c 100644 --- a/apps/framework/lib/result-files.ts +++ b/apps/framework/lib/result-files.ts @@ -11,7 +11,6 @@ import { getExperimentDisplayMetadata, type ExperimentConfig, type ExperimentDisplayMetadata, - type ExperimentRationale, } from '@supabase-evals/core'; import type { ExperimentSuite } from '@supabase-evals/core/eval-metadata'; import { discoverExperimentFiles } from './experiment-files.js'; @@ -24,7 +23,6 @@ const EXPERIMENTS_DIR = join(ROOT, 'experiments'); export interface ExperimentMetadata { display: ExperimentDisplayMetadata; - rationale?: ExperimentRationale; suites: ExperimentSuite[]; } @@ -37,7 +35,6 @@ export async function loadExperimentMetadata(): Promise< const config: ExperimentConfig = mod.default; map.set(experiment.name, { display: getExperimentDisplayMetadata(config), - rationale: config.rationale, suites: config.suite ?? [], }); } diff --git a/apps/framework/scripts/check-experiment-rationales.ts b/apps/framework/scripts/check-experiment-rationales.ts new file mode 100644 index 000000000..a6189f1f9 --- /dev/null +++ b/apps/framework/scripts/check-experiment-rationales.ts @@ -0,0 +1,7 @@ +import { loadExperimentMetadata } from '../lib/result-files.js'; +import { missingBenchmarkRationales } from '../../web/src/lib/experiment-rationales.js'; + +const missing = missingBenchmarkRationales(await loadExperimentMetadata()); +if (missing.length) { + throw new Error(`Missing benchmark rationales: ${missing.join(', ')}`); +} diff --git a/apps/framework/scripts/export-results.test.ts b/apps/framework/scripts/export-results.test.ts index e01e22898..d850fc541 100644 --- a/apps/framework/scripts/export-results.test.ts +++ b/apps/framework/scripts/export-results.test.ts @@ -32,39 +32,3 @@ describe('toEvalResult cliVersion precedence', () => { expect(result.cliVersion).toBe('2.109.1'); }); }); - -describe('toEvalResult experimentRationale', () => { - const display = { - agent: 'claude-code', - modelProvider: 'anthropic', - modelId: 'claude-opus-5-5', - reasoningEffort: 'medium', - } as const; - const experimentData = { - display, - rationale: { summary: 'Default model.', sources: [] }, - suites: [], - }; - - const build = async (experimentDisplay: Record) => - toEvalResult( - rawEvalResultSchema.parse({ - experiment: 'test-experiment', - eval: PINNED_EVAL_ID, - interface: 'cli', - experimentDisplay, - }), - 'test-experiment/run-1/result.json', - await readPrompt(PINNED_EVAL_ID), - experimentData - ); - - it('attaches the rationale when the run matches the config', async () => { - expect((await build(display)).experimentRationale).toBeDefined(); - }); - - it('drops the rationale when the run used a different effort', async () => { - const result = await build({ ...display, reasoningEffort: 'high' }); - expect(result.experimentRationale).toBeUndefined(); - }); -}); diff --git a/apps/framework/scripts/export-results.ts b/apps/framework/scripts/export-results.ts index 11b61db2b..04c9c6f42 100644 --- a/apps/framework/scripts/export-results.ts +++ b/apps/framework/scripts/export-results.ts @@ -7,7 +7,6 @@ import { rawEvalResultSchema } from '@supabase-evals/core/eval-metadata'; import type { EvalResult, EvalSuite, - ExperimentDisplayMetadata, ExperimentSuite, } from '@supabase-evals/core/eval-metadata'; import { @@ -55,14 +54,6 @@ const EXPECTED_RUNS_FLAG = readFlag(rawArgs, 'runs'); const OUTPUT_FLAG = readRepeatedFlag(rawArgs, 'output')[0]; const outputPath = OUTPUT_FLAG ? resolve(ROOT, OUTPUT_FLAG) : OUTPUT_PATH; -/** True when the run used the model and effort the experiment config names today. */ -function sameModelAndEffort( - a: ExperimentDisplayMetadata, - b: ExperimentDisplayMetadata | undefined -) { - return a.modelId === b?.modelId && a.reasoningEffort === b?.reasoningEffort; -} - export function toEvalResult( parsedResult: ReturnType, sourcePath: string, @@ -79,14 +70,6 @@ export function toEvalResult( experimentSuite, experimentDisplay: parsedResult.experimentDisplay ?? experimentData?.display, - experimentRationale: - !parsedResult.experimentDisplay || - sameModelAndEffort( - parsedResult.experimentDisplay, - experimentData?.display - ) - ? experimentData?.rationale - : undefined, eval: parsedResult.eval, stage: promptData?.stage ?? parsedResult.stage, product: promptData?.product ?? parsedResult.product, diff --git a/apps/web/package.json b/apps/web/package.json index c02742a1b..42d6c0944 100644 --- a/apps/web/package.json +++ b/apps/web/package.json @@ -4,13 +4,13 @@ "version": "0.0.1", "type": "module", "scripts": { - "dev": "vite", - "build": "node -e \"console.log('Node.js version:', process.version)\" && tsc -b && vite build && vite build --ssr src/entry-server.tsx --outDir dist-ssr && node scripts/prerender.ts", + "dev": "pnpm --filter @supabase-evals/framework exec node --import tsx/esm scripts/check-experiment-rationales.ts && vite", + "build": "pnpm --filter @supabase-evals/framework exec node --import tsx/esm scripts/check-experiment-rationales.ts && node -e \"console.log('Node.js version:', process.version)\" && tsc -b && vite build && vite build --ssr src/entry-server.tsx --outDir dist-ssr && node scripts/prerender.ts", "lint": "eslint .", "format": "prettier --write \"**/*.{ts,tsx}\"", "format:check": "prettier --check \"**/*.{ts,tsx}\"", - "typecheck": "tsc -b --noEmit", - "test": "vitest run", + "typecheck": "pnpm --filter @supabase-evals/framework exec node --import tsx/esm scripts/check-experiment-rationales.ts && tsc -b --noEmit", + "test": "pnpm --filter @supabase-evals/framework exec node --import tsx/esm scripts/check-experiment-rationales.ts && vitest run", "preview": "vite preview" }, "dependencies": { diff --git a/apps/web/src/components/experiment-rationale.tsx b/apps/web/src/components/experiment-rationale.tsx index 6abf79c34..b9e830549 100644 --- a/apps/web/src/components/experiment-rationale.tsx +++ b/apps/web/src/components/experiment-rationale.tsx @@ -24,8 +24,7 @@ export function ExperimentRationale() { Why these experiments

- Where an agent has a default model and effort, we use it. Where we - picked one ourselves, we say why. + Why we chose the model and effort for each benchmark experiment.

    {experiments.map((experiment) => { @@ -56,6 +55,11 @@ export function ExperimentRationale() {

    {rationale?.summary}

    + {rationale && ( +

    + As of +

    + )} ) })} diff --git a/apps/web/src/data/experiment-rationales.json b/apps/web/src/data/experiment-rationales.json new file mode 100644 index 000000000..0bbf03913 --- /dev/null +++ b/apps/web/src/data/experiment-rationales.json @@ -0,0 +1,158 @@ +[ + { + "experiment": "claude-code-opus-5.5", + "display": { + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-opus-5-5", + "reasoningEffort": "medium" + }, + "rationale": { + "summary": "Claude Code's default model on most plans, at its default effort.", + "sources": [ + { + "label": "Default model", + "url": "https://code.claude.com/docs/en/model-config#default-model-setting" + }, + { + "label": "Default effort", + "url": "https://code.claude.com/docs/en/model-config#adjust-effort-level" + } + ], + "asOf": "2026-10-26" + } + }, + { + "experiment": "claude-code-sonnet-5", + "display": { + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-sonnet-5", + "reasoningEffort": "high" + }, + "rationale": { + "summary": "Anthropic's latest Sonnet model for coding, at Claude Code's default high effort.", + "sources": [ + { + "label": "Sonnet 5 announcement", + "url": "https://www.anthropic.com/news/claude-sonnet-5" + }, + { + "label": "Default effort", + "url": "https://platform.claude.com/docs/en/models/sonnet-5/overview" + } + ], + "asOf": "2026-07-02" + } + }, + { + "experiment": "claude-code-sonnet-5.5", + "display": { + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-sonnet-5-5", + "reasoningEffort": "medium" + }, + "rationale": { + "summary": "Claude Code's Sonnet model for daily coding tasks, at its default effort.", + "sources": [ + { + "label": "Model aliases", + "url": "https://code.claude.com/docs/en/model-config" + }, + { + "label": "Default effort", + "url": "https://code.claude.com/docs/en/model-config#adjust-effort-level" + } + ], + "asOf": "2026-10-26" + } + }, + { + "experiment": "codex-gpt-6-luna", + "display": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-6-luna", + "reasoningEffort": "medium" + }, + "rationale": { + "summary": "A cost-effective model for everyday tasks, and the one behind the Supabase dashboard assistant.", + "sources": [ + { + "label": "Dashboard assistant default model", + "url": "https://github.com/supabase/supabase/blob/d98c277790766d2ceb905b675eb0a718f25e6345/apps/studio/lib/ai/model.utils.ts#L92" + } + ], + "asOf": "2026-10-26" + } + }, + { + "experiment": "codex-gpt-6.1-sol", + "display": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-6.1-sol", + "reasoningEffort": "medium" + }, + "rationale": { + "summary": "Codex's default model, at its default effort.", + "sources": [ + { + "label": "Where the catalog comes from", + "url": "https://github.com/openai/codex/blob/03748ad5e18a3c9c1f0828561dcf1f3f0d7b2f77/codex-rs/models-manager/src/manager.rs#L394-L436" + }, + { + "label": "How the default is picked", + "url": "https://github.com/openai/codex/blob/7d4c7a0767cdf81ff76c8fa5a05c22929568e3c6/codex-rs/models-manager/src/manager.rs#L845-L851" + } + ], + "asOf": "2026-10-26" + } + }, + { + "experiment": "grok-4.7", + "display": { + "agent": "grok", + "modelProvider": "spacexai", + "modelId": "grok-4.7", + "reasoningEffort": "high" + }, + "rationale": { + "summary": "Grok Build's default model, at its default effort.", + "sources": [ + { + "label": "Grok 4.7 and Grok Build", + "url": "https://docs.x.ai/developers/grok-4-7" + }, + { + "label": "Default effort", + "url": "https://docs.x.ai/developers/models/grok-4.7" + } + ], + "asOf": "2026-10-26" + } + }, + { + "experiment": "opencode-kimi-k3", + "display": { + "agent": "opencode", + "modelProvider": "moonshotai", + "modelId": "moonshotai/kimi-k3" + }, + "rationale": { + "summary": "An open-weight model we picked to represent open-weight options in OpenCode.", + "sources": [ + { + "label": "OpenCode usage data", + "url": "https://opencode.ai/data/" + }, + { + "label": "Kimi K3 usage on OpenCode", + "url": "https://opencode.ai/data/moonshotai/kimi-k3" + } + ], + "asOf": "2026-10-26" + } + } +] diff --git a/apps/web/src/lib/eval-results.test.ts b/apps/web/src/lib/eval-results.test.ts index 67e05ed5a..208ea8066 100644 --- a/apps/web/src/lib/eval-results.test.ts +++ b/apps/web/src/lib/eval-results.test.ts @@ -1,11 +1,13 @@ import { describe, expect, it } from "vitest" import { makeResult } from "@/lib/eval-results.fixture" +import { missingBenchmarkRationales } from "@/lib/experiment-rationales" import { EXPERIMENT_SUITES, JOURNEY_STAGES, UNASSIGNED_PRODUCT, getEvalIds, + getExperimentRationale, getProductKeys, getProductResults, getVisibleExperiments, @@ -218,3 +220,78 @@ describe("the exported results", () => { expect(inconsistent).toEqual([]) }) }) + +describe("getExperimentRationale", () => { + const display = { + agent: "opencode", + modelProvider: "moonshotai", + modelId: "moonshotai/kimi-k3", + } as const + const match = makeResult({ + experiment: "opencode-kimi-k3", + experimentDisplay: display, + }) + + it("shows current rationale for matching results", () => { + expect( + getExperimentRationale(match.experiment, [match])?.summary + ).toContain("open-weight") + }) + + it("hides rationale for mixed or missing metadata", () => { + const other = makeResult({ + experiment: match.experiment, + experimentDisplay: { ...display, modelId: "other" }, + }) + const missing = makeResult({ experiment: match.experiment }) + expect( + getExperimentRationale(match.experiment, [match, other]) + ).toBeUndefined() + expect( + getExperimentRationale(match.experiment, [match, missing]) + ).toBeUndefined() + expect( + getExperimentRationale("removed-experiment", [match]) + ).toBeUndefined() + }) +}) + +describe("benchmark rationale coverage", () => { + const kimi = { + agent: "opencode", + modelProvider: "moonshotai", + modelId: "moonshotai/kimi-k3", + } as const + + it("requires an entry matching each current benchmark model and effort", () => { + expect( + missingBenchmarkRationales([ + ["opencode-kimi-k3", { display: kimi, suites: ["benchmark"] }], + ["new-experiment", { display: kimi, suites: ["benchmark"] }], + [ + "opencode-kimi-k3", + { + display: { ...kimi, reasoningEffort: "high" }, + suites: ["benchmark"], + }, + ], + ["other", { display: kimi, suites: ["no-skills"] }], + ]) + ).toEqual(["new-experiment", "opencode-kimi-k3"]) + }) + + it("shows the historical Sonnet rationale when its results are present", () => { + const old = makeResult({ + experiment: "claude-code-sonnet-5", + experimentDisplay: { + agent: "claude-code", + modelProvider: "anthropic", + modelId: "claude-sonnet-5", + reasoningEffort: "high", + }, + }) + expect(getExperimentRationale(old.experiment, [old])?.asOf).toBe( + "2026-07-02" + ) + }) +}) diff --git a/apps/web/src/lib/eval-results.ts b/apps/web/src/lib/eval-results.ts index a10da0f47..55efddcd2 100644 --- a/apps/web/src/lib/eval-results.ts +++ b/apps/web/src/lib/eval-results.ts @@ -4,6 +4,10 @@ import { type EvalResult, } from "@supabase-evals/core/eval-metadata" import rawResults from "@/data/eval-results.json" +import { + experimentRationales, + sameExperimentDisplay, +} from "@/lib/experiment-rationales" /** * The exported eval results and the queries the site runs over them. Everything @@ -123,9 +127,24 @@ export function getExperimentDisplay(experiment: string) { ?.experimentDisplay } -export function getExperimentRationale(experiment: string) { - return sortedResults.find((result) => result.experiment === experiment) - ?.experimentRationale +export function getExperimentRationale( + experiment: string, + sourceResults = sortedResults +) { + const results = sourceResults.filter( + (result) => result.experiment === experiment + ) + if (!results.length) return undefined + + return experimentRationales.find( + (entry) => + entry.experiment === experiment && + results.every( + ({ experimentDisplay }) => + experimentDisplay && + sameExperimentDisplay(experimentDisplay, entry.display) + ) + )?.rationale } export function getExperimentResults( diff --git a/apps/web/src/lib/experiment-rationales.ts b/apps/web/src/lib/experiment-rationales.ts new file mode 100644 index 000000000..fb2c6daf9 --- /dev/null +++ b/apps/web/src/lib/experiment-rationales.ts @@ -0,0 +1,51 @@ +import { z } from "zod" +import { + experimentDisplayMetadataSchema, + experimentRationaleSchema, + type ExperimentDisplayMetadata, + type ExperimentSuite, +} from "@supabase-evals/core/eval-metadata" +import rawRationales from "../data/experiment-rationales.json" with { type: "json" } + +export const experimentRationales = z + .array( + z.object({ + experiment: z.string(), + display: experimentDisplayMetadataSchema, + rationale: experimentRationaleSchema.extend({ + summary: z.string().trim().min(1), + asOf: z.iso.date(), + }), + }) + ) + .parse(rawRationales) + +export function sameExperimentDisplay( + a: ExperimentDisplayMetadata, + b: ExperimentDisplayMetadata +) { + return ( + a.agent === b.agent && + a.modelProvider === b.modelProvider && + a.modelId === b.modelId && + a.reasoningEffort === b.reasoningEffort + ) +} + +export function missingBenchmarkRationales( + experiments: Iterable< + [string, { display: ExperimentDisplayMetadata; suites: ExperimentSuite[] }] + > +) { + return Array.from(experiments) + .filter(([, config]) => config.suites.includes("benchmark")) + .filter( + ([name, config]) => + !experimentRationales.some( + (entry) => + entry.experiment === name && + sameExperimentDisplay(entry.display, config.display) + ) + ) + .map(([name]) => name) +} diff --git a/experiments/presets.ts b/experiments/presets.ts index 8cb416364..991c68d93 100644 --- a/experiments/presets.ts +++ b/experiments/presets.ts @@ -11,7 +11,7 @@ import { localStackRuntime } from '@supabase-evals/sandbox'; type ExperimentPreset = Pick< ExperimentConfig, - 'agent' | 'rationale' | 'runtime' | 'localStack' | 'skills' + 'agent' | 'runtime' | 'localStack' | 'skills' >; const skills = ['supabase', 'supabase-postgres-best-practices']; @@ -27,20 +27,6 @@ export const claudeCodeOpus55 = { model: 'claude-opus-5-5', reasoningEffort: 'medium', }), - rationale: { - summary: - "Claude Code's default model on most plans, at its default effort.", - sources: [ - { - label: 'Default model', - url: 'https://code.claude.com/docs/en/model-config#default-model-setting', - }, - { - label: 'Default effort', - url: 'https://code.claude.com/docs/en/model-config#adjust-effort-level', - }, - ], - }, runtime: defaultRuntime(), localStack: localStackRuntime(), skills, @@ -51,20 +37,6 @@ export const claudeCodeSonnet55 = { model: 'claude-sonnet-5-5', reasoningEffort: 'medium', }), - rationale: { - summary: - "Claude Code's Sonnet model for daily coding tasks, at its default effort.", - sources: [ - { - label: 'Model aliases', - url: 'https://code.claude.com/docs/en/model-config', - }, - { - label: 'Default effort', - url: 'https://code.claude.com/docs/en/model-config#adjust-effort-level', - }, - ], - }, runtime: defaultRuntime(), localStack: localStackRuntime(), skills, @@ -75,16 +47,6 @@ export const codexGpt6Luna = { model: 'gpt-6-luna', reasoningEffort: 'medium', }), - rationale: { - summary: - 'A cost-effective model for everyday tasks, and the one behind the Supabase dashboard assistant.', - sources: [ - { - label: 'Dashboard assistant default model', - url: 'https://github.com/supabase/supabase/blob/d98c277790766d2ceb905b675eb0a718f25e6345/apps/studio/lib/ai/model.utils.ts#L92', - }, - ], - }, runtime: defaultRuntime(), localStack: localStackRuntime(), skills, @@ -95,19 +57,6 @@ export const codexGpt61Sol = { model: 'gpt-6.1-sol', reasoningEffort: 'medium', }), - rationale: { - summary: "Codex's default model, at its default effort.", - sources: [ - { - label: 'Where the catalog comes from', - url: 'https://github.com/openai/codex/blob/03748ad5e18a3c9c1f0828561dcf1f3f0d7b2f77/codex-rs/models-manager/src/manager.rs#L394-L436', - }, - { - label: 'How the default is picked', - url: 'https://github.com/openai/codex/blob/7d4c7a0767cdf81ff76c8fa5a05c22929568e3c6/codex-rs/models-manager/src/manager.rs#L845-L851', - }, - ], - }, runtime: defaultRuntime(), localStack: localStackRuntime(), skills, @@ -118,19 +67,6 @@ export const grok47 = { model: 'grok-4.7', reasoningEffort: 'high', }), - rationale: { - summary: "Grok Build's default model, at its default effort.", - sources: [ - { - label: 'Grok 4.7 and Grok Build', - url: 'https://docs.x.ai/developers/grok-4-7', - }, - { - label: 'Default effort', - url: 'https://docs.x.ai/developers/models/grok-4.7', - }, - ], - }, runtime: defaultRuntime(), localStack: localStackRuntime(), skills, @@ -140,17 +76,6 @@ export const opencodeKimiK3 = { agent: opencodeAgent({ model: 'moonshotai/kimi-k3', }), - rationale: { - summary: - 'An open-weight model we picked to represent open-weight options in OpenCode.', - sources: [ - { label: 'OpenCode usage data', url: 'https://opencode.ai/data/' }, - { - label: 'Kimi K3 usage on OpenCode', - url: 'https://opencode.ai/data/moonshotai/kimi-k3', - }, - ], - }, runtime: defaultRuntime(), localStack: localStackRuntime(), skills, diff --git a/packages/core/src/eval-metadata.ts b/packages/core/src/eval-metadata.ts index f18fd0e3e..97d2c0326 100644 --- a/packages/core/src/eval-metadata.ts +++ b/packages/core/src/eval-metadata.ts @@ -431,7 +431,6 @@ const evalResultShape = { experimentSuite: experimentSuiteSchema.optional(), profile: experimentSuiteSchema.optional(), experimentDisplay: experimentDisplayMetadataSchema.optional(), - experimentRationale: experimentRationaleSchema.optional(), eval: z.string(), stage: evalStageSchema.optional(), product: z.array(evalProductSchema).optional(), diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 7bf86cf7d..ef8bae3ee 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -46,7 +46,6 @@ import type { EvalMetadata, EvalSuite, ExperimentDisplayMetadata, - ExperimentRationale, ExperimentSuite, JudgeCall, ModelProvider, @@ -94,7 +93,6 @@ export { evalSuiteSchema, experimentSuiteSchema, experimentDisplayMetadataSchema, - experimentRationaleSchema, modelProviderSchema, rawEvalResultSchema, reasoningEffortSchema, @@ -146,7 +144,6 @@ export type { EvalSuite, EvalStage, ExperimentDisplayMetadata, - ExperimentRationale, ExperimentSuite, ModelProvider, ParsedEvalMarkdown, @@ -648,8 +645,6 @@ export type ExperimentConfig = { /** Named experiment suites this experiment belongs to. */ suite?: ExperimentSuite[]; agent: AgentHarness; - /** Why this model and effort, shown next to the experiment on the results page. */ - rationale?: ExperimentRationale; runtime: EvalRuntime; /** Local-stack environment (e.g. localStackRuntime() from @supabase-evals/sandbox). */ localStack?: LocalStackRuntime; From f7c2341adc027be1f0fe6ebc2ee28f661eb0fc3e Mon Sep 17 00:00:00 2001 From: Matt Rossman <22670878+mattrossman@users.noreply.github.com> Date: Mon, 9 Nov 2026 12:47:40 -0500 Subject: [PATCH 10/18] refactor: link benchmark rationale claims inline --- .../src/components/experiment-rationale.tsx | 29 +++---- apps/web/src/data/experiment-rationales.json | 85 +++++++++---------- apps/web/src/lib/eval-results.test.ts | 16 +++- apps/web/src/lib/experiment-rationales.ts | 12 ++- packages/core/src/eval-metadata.ts | 7 -- 5 files changed, 80 insertions(+), 69 deletions(-) diff --git a/apps/web/src/components/experiment-rationale.tsx b/apps/web/src/components/experiment-rationale.tsx index b9e830549..5b88a3e8a 100644 --- a/apps/web/src/components/experiment-rationale.tsx +++ b/apps/web/src/components/experiment-rationale.tsx @@ -34,26 +34,25 @@ export function ExperimentRationale() { key={experiment} className="flex flex-col gap-1 px-5 py-3 text-sm" > -
    - - - - - {rationale?.sources.map((source) => ( + + + +

    + {rationale?.summary.map((part, index) => + typeof part === "string" ? ( + part + ) : ( - {source.label} + {part.text} - ))} - -

    -

    - {rationale?.summary} + ) + )}

    {rationale && (

    diff --git a/apps/web/src/data/experiment-rationales.json b/apps/web/src/data/experiment-rationales.json index f0449bea3..9dba1adec 100644 --- a/apps/web/src/data/experiment-rationales.json +++ b/apps/web/src/data/experiment-rationales.json @@ -8,16 +8,18 @@ "reasoningEffort": "medium" }, "rationale": { - "summary": "Claude Code's default model on most plans, at its default effort.", - "sources": [ + "summary": [ + "Claude Code's ", { - "label": "Default model", + "text": "default model", "url": "https://code.claude.com/docs/en/model-config#default-model-setting" }, + " on most plans, at its ", { - "label": "Default effort", + "text": "default effort", "url": "https://code.claude.com/docs/en/model-config#adjust-effort-level" - } + }, + "." ], "asOf": "2026-10-26" } @@ -31,16 +33,18 @@ "reasoningEffort": "high" }, "rationale": { - "summary": "Anthropic's latest Sonnet model for coding, at Claude Code's default high effort.", - "sources": [ + "summary": [ + "Anthropic's ", { - "label": "Sonnet 5 announcement", + "text": "latest Sonnet model for coding", "url": "https://www.anthropic.com/news/claude-sonnet-5" }, + ", at Claude Code's ", { - "label": "Default effort", + "text": "default high effort", "url": "https://platform.claude.com/docs/en/models/sonnet-5/overview" - } + }, + "." ], "asOf": "2026-07-02" } @@ -54,16 +58,18 @@ "reasoningEffort": "medium" }, "rationale": { - "summary": "Claude Code's Sonnet model for daily coding tasks, at its default effort.", - "sources": [ + "summary": [ + "Claude Code's ", { - "label": "Model aliases", + "text": "Sonnet model for daily coding tasks", "url": "https://code.claude.com/docs/en/model-config" }, + ", at its ", { - "label": "Default effort", + "text": "default effort", "url": "https://code.claude.com/docs/en/model-config#adjust-effort-level" - } + }, + "." ], "asOf": "2026-10-26" } @@ -77,12 +83,13 @@ "reasoningEffort": "medium" }, "rationale": { - "summary": "A cost-effective model for everyday tasks, and the one behind the Supabase dashboard assistant.", - "sources": [ + "summary": [ + "A cost-effective model for everyday tasks, and the one behind the ", { - "label": "Dashboard assistant default model", + "text": "Supabase dashboard assistant", "url": "https://github.com/supabase/supabase/blob/d98c277790766d2ceb905b675eb0a718f25e6345/apps/studio/lib/ai/model.utils.ts#L92" - } + }, + "." ], "asOf": "2026-10-26" } @@ -96,20 +103,13 @@ "reasoningEffort": "medium" }, "rationale": { - "summary": "Codex's default model for ChatGPT accounts, at its default effort.", - "sources": [ + "summary": [ + "Codex's ", { - "label": "Model and effort guidance", + "text": "default model for ChatGPT accounts, at its default effort", "url": "https://learn.chatgpt.com/docs/models?surface=app#pick-a-reasoning-effort" }, - { - "label": "Where the catalog comes from", - "url": "https://github.com/openai/codex/blob/03748ad5e18a3c9c1f0828561dcf1f3f0d7b2f77/codex-rs/models-manager/src/manager.rs#L394-L436" - }, - { - "label": "How the default is picked", - "url": "https://github.com/openai/codex/blob/7d4c7a0767cdf81ff76c8fa5a05c22929568e3c6/codex-rs/models-manager/src/manager.rs#L845-L851" - } + "." ], "asOf": "2026-11-09" } @@ -123,16 +123,18 @@ "reasoningEffort": "high" }, "rationale": { - "summary": "Chosen to match Grok Build's default model and effort when selected.", - "sources": [ + "summary": [ + "Grok Build's ", { - "label": "Grok 4.7 and Grok Build", + "text": "default model", "url": "https://docs.x.ai/developers/grok-4-7" }, + ", at its ", { - "label": "Model catalog", + "text": "default effort", "url": "https://github.com/xai-org/grok-build/blob/2bdd1d6a6369de0e8c68132ea4539e9abd9e14a8/crates/codegen/xai-grok-shell/src/remote/client.rs#L832-L862" - } + }, + "." ], "asOf": "2026-10-26" } @@ -145,16 +147,13 @@ "modelId": "moonshotai/kimi-k3" }, "rationale": { - "summary": "An open-weight model we picked to represent open-weight options in OpenCode.", - "sources": [ + "summary": [ + "Included as an ", { - "label": "OpenCode usage data", - "url": "https://opencode.ai/data/" - }, - { - "label": "Kimi K3 usage on OpenCode", + "text": "open-weight option", "url": "https://opencode.ai/data/moonshotai/kimi-k3" - } + }, + " in OpenCode." ], "asOf": "2026-10-26" } diff --git a/apps/web/src/lib/eval-results.test.ts b/apps/web/src/lib/eval-results.test.ts index c1b19035c..5196e12ce 100644 --- a/apps/web/src/lib/eval-results.test.ts +++ b/apps/web/src/lib/eval-results.test.ts @@ -1,3 +1,6 @@ +import { createElement } from "react" +import { renderToStaticMarkup } from "react-dom/server" +import { ExperimentRationale } from "@/components/experiment-rationale" import { describe, expect, it } from "vitest" import { makeResult } from "@/lib/eval-results.fixture" @@ -235,7 +238,7 @@ describe("getExperimentRationale", () => { it("shows current rationale for matching results", () => { expect( getExperimentRationale(match.experiment, [match])?.summary - ).toContain("open-weight") + ).toContainEqual(expect.objectContaining({ text: "open-weight option" })) }) it("hides rationale for mixed or missing metadata", () => { @@ -295,3 +298,14 @@ describe("benchmark rationale coverage", () => { ) }) }) + +describe("inline rationale links", () => { + it("links the claim inside the sentence", () => { + const html = renderToStaticMarkup(createElement(ExperimentRationale)) + expect(html).toContain( + 'href="https://code.claude.com/docs/en/model-config#default-model-setting"' + ) + expect(html).toContain(">default model on most plans, at its ") + expect(html).toContain(">default effort.") + }) +}) diff --git a/apps/web/src/lib/experiment-rationales.ts b/apps/web/src/lib/experiment-rationales.ts index eec6d588d..cef155ea0 100644 --- a/apps/web/src/lib/experiment-rationales.ts +++ b/apps/web/src/lib/experiment-rationales.ts @@ -1,7 +1,6 @@ import { z } from "zod" import { experimentDisplayMetadataSchema, - experimentRationaleSchema, type ExperimentDisplayMetadata, } from "@supabase-evals/core/eval-metadata" import rawRationales from "../data/experiment-rationales.json" with { type: "json" } @@ -11,8 +10,15 @@ export const experimentRationales = z z.object({ experiment: z.string(), display: experimentDisplayMetadataSchema, - rationale: experimentRationaleSchema.extend({ - summary: z.string().trim().min(1), + rationale: z.object({ + summary: z + .array( + z.union([ + z.string().min(1), + z.object({ text: z.string().min(1), url: z.url() }), + ]) + ) + .min(1), asOf: z.iso.date(), }), }) diff --git a/packages/core/src/eval-metadata.ts b/packages/core/src/eval-metadata.ts index e3aeb1255..0762689f1 100644 --- a/packages/core/src/eval-metadata.ts +++ b/packages/core/src/eval-metadata.ts @@ -87,13 +87,6 @@ export type ExperimentDisplayMetadata = z.infer< typeof experimentDisplayMetadataSchema >; -/** Why an experiment uses its model and effort, with links to check it. */ -export const experimentRationaleSchema = z.object({ - summary: z.string(), - sources: z.array(z.object({ label: z.string(), url: z.string().url() })), -}); -export type ExperimentRationale = z.infer; - /** * Interface(s) the agent uses to act on Supabase — a benchmark dimension * (cross-team KPI), not the runtime switch. `mcp` = the platform-lite MCP/tool From e3cdcc8085614b4a355071d6a817cb1be1a3b197 Mon Sep 17 00:00:00 2001 From: Matt Rossman <22670878+mattrossman@users.noreply.github.com> Date: Mon, 9 Nov 2026 14:13:06 -0500 Subject: [PATCH 11/18] style: clarify benchmark rationale hierarchy --- .../src/components/experiment-rationale.tsx | 39 +++++++++++++------ 1 file changed, 27 insertions(+), 12 deletions(-) diff --git a/apps/web/src/components/experiment-rationale.tsx b/apps/web/src/components/experiment-rationale.tsx index 5b88a3e8a..994ce57ca 100644 --- a/apps/web/src/components/experiment-rationale.tsx +++ b/apps/web/src/components/experiment-rationale.tsx @@ -30,14 +30,11 @@ export function ExperimentRationale() { {experiments.map((experiment) => { const rationale = getExperimentRationale(experiment) return ( -

  • - +
  • + -

    +

    {rationale?.summary.map((part, index) => typeof part === "string" ? ( part @@ -47,22 +44,40 @@ export function ExperimentRationale() { href={part.url} target="_blank" rel="noreferrer" - className="underline underline-offset-2 hover:text-foreground" + className="underline decoration-muted-foreground/40 underline-offset-4 hover:text-foreground hover:decoration-foreground" > {part.text} ) )}

    - {rationale && ( -

    - As of -

    - )}
    • ) })}
+
+ + Source review dates + +

+ When the sources supporting each model and effort choice were last + reviewed. +

+
    + {experiments.map((experiment) => { + const rationale = getExperimentRationale(experiment) + return ( +
  • + + +
    • + ) + })} +
+
) } From 3b210aaea7e7ca46f37840fb1c5400bef08f5d86 Mon Sep 17 00:00:00 2001 From: Matt Rossman <22670878+mattrossman@users.noreply.github.com> Date: Mon, 9 Nov 2026 14:16:49 -0500 Subject: [PATCH 12/18] style: align source review dates --- .../src/components/experiment-rationale.tsx | 34 +++++++++++-------- 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/apps/web/src/components/experiment-rationale.tsx b/apps/web/src/components/experiment-rationale.tsx index 994ce57ca..d910ced17 100644 --- a/apps/web/src/components/experiment-rationale.tsx +++ b/apps/web/src/components/experiment-rationale.tsx @@ -63,20 +63,26 @@ export function ExperimentRationale() { When the sources supporting each model and effort choice were last reviewed.

-
    - {experiments.map((experiment) => { - const rationale = getExperimentRationale(experiment) - return ( -
  • - - -
    • - ) - })} -
+ + + + {experiments.map((experiment) => { + const rationale = getExperimentRationale(experiment) + return ( + + + + + ) + })} + +
+ Source review dates by experiment +
+ + + +
) From bde2808d2c77ea0d595b530d990ab7aa3c15ecc0 Mon Sep 17 00:00:00 2001 From: Matt Rossman <22670878+mattrossman@users.noreply.github.com> Date: Mon, 9 Nov 2026 15:07:41 -0500 Subject: [PATCH 13/18] fix: scope prerender count to results rows --- apps/web/scripts/prerender.ts | 15 ++++----------- apps/web/src/components/results/results-table.tsx | 1 + 2 files changed, 5 insertions(+), 11 deletions(-) diff --git a/apps/web/scripts/prerender.ts b/apps/web/scripts/prerender.ts index d2ada590e..5b1d0ac0d 100644 --- a/apps/web/scripts/prerender.ts +++ b/apps/web/scripts/prerender.ts @@ -12,8 +12,6 @@ const htmlFile = path.join(appDir, "dist", "index.html") const serverEntryFile = path.join(appDir, "dist-ssr", "entry-server.js") const ROOT_DIV = '
' -/** The results table renders one header row above its data rows. */ -const HEADER_ROWS = 1 /** Rendered by App when the results file parses to nothing. */ const EMPTY_STATE = "No result files found" @@ -73,15 +71,10 @@ if (markup.includes(EMPTY_STATE)) { fail("the app rendered its empty state, so the results data did not load") } -const expectedRows = expectedTableRows + HEADER_ROWS -const tableRows = markup.match(/ Date: Mon, 9 Nov 2026 15:31:48 -0500 Subject: [PATCH 14/18] refactor: attach rationales to published experiments --- CONTRIBUTING.md | 4 +- .../scripts/check-experiment-rationales.ts | 7 - apps/web/package.json | 8 +- apps/web/scripts/fetch-braintrust.test.ts | 41 +++++ apps/web/scripts/fetch-braintrust.ts | 98 ++++++----- .../components/experiment-rationale.test.tsx | 26 +++ .../src/components/experiment-rationale.tsx | 1 - apps/web/src/data/experiment-rationales.json | 161 ------------------ apps/web/src/data/publication.ts | 130 ++++++++++++++ apps/web/src/lib/eval-results.test.ts | 100 ++--------- apps/web/src/lib/eval-results.ts | 20 +-- apps/web/src/lib/experiment-rationales.ts | 55 ------ packages/core/src/eval-metadata.ts | 13 ++ 13 files changed, 283 insertions(+), 381 deletions(-) delete mode 100644 apps/framework/scripts/check-experiment-rationales.ts create mode 100644 apps/web/src/components/experiment-rationale.test.tsx delete mode 100644 apps/web/src/data/experiment-rationales.json create mode 100644 apps/web/src/data/publication.ts delete mode 100644 apps/web/src/lib/experiment-rationales.ts diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index dcb037620..1b725930c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -64,9 +64,9 @@ Include refreshed results for PRs with new/changed evals so a reviewer can inspe ## Publishing and consuming results -To publish a benchmark, upload complete runs with `--runs` and update the experiment IDs in [fetch-braintrust.ts](apps/web/scripts/fetch-braintrust.ts). Include every experiment in `experiments/benchmark/`, using its name as the key. +To publish a benchmark, upload complete runs with `--runs` and update the pins and rationales in [publication.ts](apps/web/src/data/publication.ts). Include every experiment in `experiments/benchmark/`. -For local UI work with a partial run, temporarily pin only the uploaded experiment IDs and start the server with `EVALS_ALLOW_PARTIAL=1 pnpm web`. +For local UI work with a partial run, temporarily keep only the uploaded experiment IDs and their rationales in the catalog and start the server with `EVALS_ALLOW_PARTIAL=1 pnpm web`. The published benchmark is at https://supabase.com/evals/eval-results.json. For other suites or history, query the Evals Braintrust project. For example, replace `EXPERIMENT_ID` below to read an experiment's scored runs: diff --git a/apps/framework/scripts/check-experiment-rationales.ts b/apps/framework/scripts/check-experiment-rationales.ts deleted file mode 100644 index a6189f1f9..000000000 --- a/apps/framework/scripts/check-experiment-rationales.ts +++ /dev/null @@ -1,7 +0,0 @@ -import { loadExperimentMetadata } from '../lib/result-files.js'; -import { missingBenchmarkRationales } from '../../web/src/lib/experiment-rationales.js'; - -const missing = missingBenchmarkRationales(await loadExperimentMetadata()); -if (missing.length) { - throw new Error(`Missing benchmark rationales: ${missing.join(', ')}`); -} diff --git a/apps/web/package.json b/apps/web/package.json index a93390533..edd1c4655 100644 --- a/apps/web/package.json +++ b/apps/web/package.json @@ -4,13 +4,13 @@ "version": "0.0.1", "type": "module", "scripts": { - "dev": "pnpm --filter @supabase-evals/framework exec node --import tsx/esm scripts/check-experiment-rationales.ts && node --env-file-if-exists=../../.env scripts/fetch-braintrust.ts --dev && vite", - "build": "pnpm --filter @supabase-evals/framework exec node --import tsx/esm scripts/check-experiment-rationales.ts && node --env-file-if-exists=../../.env scripts/fetch-braintrust.ts && tsc -b && vite build && vite build --ssr src/entry-server.tsx --outDir dist-ssr && node scripts/prerender.ts", + "dev": "node --env-file-if-exists=../../.env scripts/fetch-braintrust.ts --dev && vite", + "build": "node --env-file-if-exists=../../.env scripts/fetch-braintrust.ts && tsc -b && vite build && vite build --ssr src/entry-server.tsx --outDir dist-ssr && node scripts/prerender.ts", "lint": "eslint .", "format": "prettier --write \"**/*.{ts,tsx}\"", "format:check": "prettier --check \"**/*.{ts,tsx}\"", - "typecheck": "pnpm --filter @supabase-evals/framework exec node --import tsx/esm scripts/check-experiment-rationales.ts && tsc -b --noEmit", - "test": "pnpm --filter @supabase-evals/framework exec node --import tsx/esm scripts/check-experiment-rationales.ts && vitest run", + "typecheck": "tsc -b --noEmit", + "test": "vitest run", "preview": "vite preview" }, "dependencies": { diff --git a/apps/web/scripts/fetch-braintrust.test.ts b/apps/web/scripts/fetch-braintrust.test.ts index 163e79a6d..9244b7d86 100644 --- a/apps/web/scripts/fetch-braintrust.test.ts +++ b/apps/web/scripts/fetch-braintrust.test.ts @@ -2,8 +2,10 @@ import { test } from "vitest" import assert from "node:assert/strict" import { assemble, + attachRationales, createBraintrustRequest, fetchExperiments, + publicationPins, retryAfterMs, } from "./fetch-braintrust.ts" @@ -59,6 +61,45 @@ const usage: Parameters[3][number] = { const make = (roots = [root], children = [score, task], usages = [usage]) => assemble([experiment], roots, children, usages, ["example-eval"]) +test("carries the pinned experiment's rationale into published results", () => { + const rationale = { + summary: ["A ", { text: "model", url: "https://example.com" }, "."], + asOf: "2026-01-01", + } + const withoutSkills = "d2e33ddc-64f1-4392-892c-d4db9f3d0146" + const entries = [ + { experiment: "example", withSkills: id, withoutSkills, rationale }, + ] + const [pinned, unskilled] = attachRationales( + [experiment, { ...experiment, id: withoutSkills }], + entries + ) + assert.deepEqual(unskilled.rationale, rationale) + const [result] = assemble( + [pinned], + [root], + [score, task], + [usage], + ["example-eval"] + ) + assert.deepEqual(result.experimentRationale, rationale) + assert.throws(() => attachRationales([experiment], []), /Missing rationale/) +}) + +test("requires both variants outside partial mode and rejects duplicate names", () => { + const rationale = { summary: ["A model."], asOf: "2026-01-01" } + const entry = { experiment: "example", withSkills: id, rationale } + assert.throws( + () => publicationPins([entry], false), + /Missing pinned variants/ + ) + assert.deepEqual(publicationPins([entry], true), { example: id }) + assert.throws( + () => publicationPins([entry, entry], true), + /distinct experiment names/ + ) +}) + test("joins failed runs, scorer checks, task duration, and trace usage", () => { const [result] = make() assert.equal(result.passed, false) diff --git a/apps/web/scripts/fetch-braintrust.ts b/apps/web/scripts/fetch-braintrust.ts index c255d4612..002afd229 100644 --- a/apps/web/scripts/fetch-braintrust.ts +++ b/apps/web/scripts/fetch-braintrust.ts @@ -4,25 +4,15 @@ import { fileURLToPath } from "node:url" import { setTimeout as delay } from "node:timers/promises" import { z } from "zod" import pRetry, { AbortError } from "p-retry" -import { evalResultSchema } from "@supabase-evals/core/eval-metadata" +import { + evalResultSchema, + experimentRationaleSchema, +} from "@supabase-evals/core/eval-metadata" +import { publication } from "../src/data/publication.ts" const app = resolve(dirname(fileURLToPath(import.meta.url)), "..") const root = resolve(app, "../..") const output = resolve(app, "src/data/braintrust-results.json") -const publication = { - "claude-code-opus-5.5": "c8e0f865-ca59-4703-a19d-507a234e100a", - "claude-code-opus-5.5-no-skills": "e0fd16ad-e117-4165-8e30-9346ebe7e835", - "claude-code-sonnet-5.5": "ede84738-9f79-428f-bdfe-9d8adb2a6489", - "claude-code-sonnet-5.5-no-skills": "15a067ef-f04e-44f5-9f94-7b8c0b0e10f1", - "codex-gpt-6-luna": "b4a9c5ce-60fc-4604-bc99-595269b7fd7d", - "codex-gpt-6-luna-no-skills": "123ecf1b-1004-458c-98cf-b17450b65c92", - "codex-gpt-6.1-sol": "d2e33ddc-64f1-4392-892c-d4db9f3d0146", - "codex-gpt-6.1-sol-no-skills": "f2a8dad6-2d8b-4088-8f5f-165d00a24058", - "grok-4.7": "e986dead-a1d3-4523-b47d-33a82ff1348b", - "grok-4.7-no-skills": "545924a6-a02c-482c-9ef7-b4dfc1a993c7", - "opencode-kimi-k3": "66f84377-4b0d-4be2-8f41-8e0756cbc40a", - "opencode-kimi-k3-no-skills": "63331944-8a8c-4396-8add-d6fbe88f80d3", -} const uuid = z.uuid() const metadataSchema = z .object({ @@ -64,7 +54,41 @@ const usageSchema = z.object({ input_tokens: z.number().nullable(), output_tokens: z.number().nullable(), }) -type Experiment = z.infer +type Experiment = z.infer & { + rationale?: z.infer +} + +export function attachRationales( + experiments: Experiment[], + entries: typeof publication +) { + return experiments.map((experiment) => { + const entry = entries.find( + ({ withSkills, withoutSkills }) => + withSkills === experiment.id || withoutSkills === experiment.id + ) + if (!entry) throw new Error(`Missing rationale for ${experiment.id}`) + return { ...experiment, rationale: entry.rationale } + }) +} + +export function publicationPins( + entries: typeof publication, + allowPartial: boolean +) { + const pinEntries: [string, string][] = [] + for (const { experiment, withSkills, withoutSkills } of entries) { + if ((!withSkills || !withoutSkills) && !allowPartial) + throw new Error(`Missing pinned variants: ${experiment}`) + if (withSkills) pinEntries.push([experiment, withSkills]) + if (withoutSkills) + pinEntries.push([`${experiment}-no-skills`, withoutSkills]) + } + const pins = Object.fromEntries(pinEntries) + if (Object.keys(pins).length !== pinEntries.length) + throw new Error("Publication must contain distinct experiment names") + return pins +} type Root = z.infer type Child = z.infer type Usage = z.infer @@ -154,6 +178,7 @@ export function assemble( } const parsed = evalResultSchema.safeParse({ experiment: em.experiment, + experimentRationale: exp.rationale, experimentDisplay: { agent: em.agent, modelProvider: em.model_provider, @@ -419,37 +444,24 @@ async function main() { "Content-Type": "application/json", } const request = createBraintrustRequest(headers, fetch) - const experiments = await fetchExperiments(request, projectId, publication) + const pins = publicationPins(publication, allowPartial) + const experiments = attachRationales( + await fetchExperiments(request, projectId, pins), + publication + ) const experimentNames = ( await readdir(resolve(root, "experiments/benchmark")) ) - .filter((f) => f.endsWith(".experiment.ts")) - .map((f) => f.slice(0, -".experiment.ts".length)) - .sort() - const selectedNames = experiments.map((e) => e.metadata.experiment).sort() - if ( - !allowPartial && - JSON.stringify(experimentNames) !== JSON.stringify(selectedNames) - ) { - const missing = experimentNames.filter( - (name) => !selectedNames.includes(name) - ) - const unexpected = selectedNames.filter( - (name) => !experimentNames.includes(name) - ) - throw new Error( - [ - "Pinned experiments do not match active benchmark experiments.", - ...(missing.length ? [`Missing: ${missing.join(", ")}`] : []), - ...(unexpected.length ? [`Unexpected: ${unexpected.join(", ")}`] : []), - ...(process.argv.includes("--dev") - ? [ - "To run with partial results locally, use EVALS_ALLOW_PARTIAL=1 pnpm web.", - ] - : []), - ].join("\n") + .filter((name) => name.endsWith(".experiment.ts")) + .map((name) => name.slice(0, -".experiment.ts".length)) + const missing = experimentNames.filter((name) => !(name in pins)) + const unexpected = Object.keys(pins).filter( + (name) => !experimentNames.includes(name) + ) + if (!allowPartial && (missing.length || unexpected.length)) + console.warn( + `Published experiments differ from current benchmark. Missing: ${missing.join(", ") || "none"}. Unexpected: ${unexpected.join(", ") || "none"}.` ) - } const evalIds = [] for (const dir of await readdir(resolve(root, "evals/benchmark"), { withFileTypes: true, diff --git a/apps/web/src/components/experiment-rationale.test.tsx b/apps/web/src/components/experiment-rationale.test.tsx new file mode 100644 index 000000000..144b591ea --- /dev/null +++ b/apps/web/src/components/experiment-rationale.test.tsx @@ -0,0 +1,26 @@ +import { renderToStaticMarkup } from "react-dom/server" +import { describe, expect, it, vi } from "vitest" + +vi.mock("@/lib/eval-results", () => ({ + getSuiteResults: () => [], + getVisibleExperiments: () => ["test-experiment"], + getExperimentDisplay: () => undefined, + getExperimentRationale: () => ({ + summary: [ + "See the ", + { text: "model", url: "https://example.com/model" }, + ".", + ], + asOf: "2026-01-01", + }), +})) + +import { ExperimentRationale } from "./experiment-rationale" + +describe("inline rationale links", () => { + it("links the claim inside the sentence", () => { + const html = renderToStaticMarkup() + expect(html).toContain('See the model.") + }) +}) diff --git a/apps/web/src/components/experiment-rationale.tsx b/apps/web/src/components/experiment-rationale.tsx index d910ced17..15155e2c2 100644 --- a/apps/web/src/components/experiment-rationale.tsx +++ b/apps/web/src/components/experiment-rationale.tsx @@ -5,7 +5,6 @@ import { getVisibleExperiments, } from "@/lib/eval-results" -/** Why each benchmark experiment uses its model and effort, with links to check it. */ export function ExperimentRationale() { const experiments = getVisibleExperiments( getSuiteResults("benchmark") diff --git a/apps/web/src/data/experiment-rationales.json b/apps/web/src/data/experiment-rationales.json deleted file mode 100644 index 9dba1adec..000000000 --- a/apps/web/src/data/experiment-rationales.json +++ /dev/null @@ -1,161 +0,0 @@ -[ - { - "experiment": "claude-code-opus-5.5", - "display": { - "agent": "claude-code", - "modelProvider": "anthropic", - "modelId": "claude-opus-5-5", - "reasoningEffort": "medium" - }, - "rationale": { - "summary": [ - "Claude Code's ", - { - "text": "default model", - "url": "https://code.claude.com/docs/en/model-config#default-model-setting" - }, - " on most plans, at its ", - { - "text": "default effort", - "url": "https://code.claude.com/docs/en/model-config#adjust-effort-level" - }, - "." - ], - "asOf": "2026-10-26" - } - }, - { - "experiment": "claude-code-sonnet-5", - "display": { - "agent": "claude-code", - "modelProvider": "anthropic", - "modelId": "claude-sonnet-5", - "reasoningEffort": "high" - }, - "rationale": { - "summary": [ - "Anthropic's ", - { - "text": "latest Sonnet model for coding", - "url": "https://www.anthropic.com/news/claude-sonnet-5" - }, - ", at Claude Code's ", - { - "text": "default high effort", - "url": "https://platform.claude.com/docs/en/models/sonnet-5/overview" - }, - "." - ], - "asOf": "2026-07-02" - } - }, - { - "experiment": "claude-code-sonnet-5.5", - "display": { - "agent": "claude-code", - "modelProvider": "anthropic", - "modelId": "claude-sonnet-5-5", - "reasoningEffort": "medium" - }, - "rationale": { - "summary": [ - "Claude Code's ", - { - "text": "Sonnet model for daily coding tasks", - "url": "https://code.claude.com/docs/en/model-config" - }, - ", at its ", - { - "text": "default effort", - "url": "https://code.claude.com/docs/en/model-config#adjust-effort-level" - }, - "." - ], - "asOf": "2026-10-26" - } - }, - { - "experiment": "codex-gpt-6-luna", - "display": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-6-luna", - "reasoningEffort": "medium" - }, - "rationale": { - "summary": [ - "A cost-effective model for everyday tasks, and the one behind the ", - { - "text": "Supabase dashboard assistant", - "url": "https://github.com/supabase/supabase/blob/d98c277790766d2ceb905b675eb0a718f25e6345/apps/studio/lib/ai/model.utils.ts#L92" - }, - "." - ], - "asOf": "2026-10-26" - } - }, - { - "experiment": "codex-gpt-6.1-sol", - "display": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-6.1-sol", - "reasoningEffort": "medium" - }, - "rationale": { - "summary": [ - "Codex's ", - { - "text": "default model for ChatGPT accounts, at its default effort", - "url": "https://learn.chatgpt.com/docs/models?surface=app#pick-a-reasoning-effort" - }, - "." - ], - "asOf": "2026-11-09" - } - }, - { - "experiment": "grok-4.7", - "display": { - "agent": "grok", - "modelProvider": "spacexai", - "modelId": "grok-4.7", - "reasoningEffort": "high" - }, - "rationale": { - "summary": [ - "Grok Build's ", - { - "text": "default model", - "url": "https://docs.x.ai/developers/grok-4-7" - }, - ", at its ", - { - "text": "default effort", - "url": "https://github.com/xai-org/grok-build/blob/2bdd1d6a6369de0e8c68132ea4539e9abd9e14a8/crates/codegen/xai-grok-shell/src/remote/client.rs#L832-L862" - }, - "." - ], - "asOf": "2026-10-26" - } - }, - { - "experiment": "opencode-kimi-k3", - "display": { - "agent": "opencode", - "modelProvider": "moonshotai", - "modelId": "moonshotai/kimi-k3" - }, - "rationale": { - "summary": [ - "Included as an ", - { - "text": "open-weight option", - "url": "https://opencode.ai/data/moonshotai/kimi-k3" - }, - " in OpenCode." - ], - "asOf": "2026-10-26" - } - } -] diff --git a/apps/web/src/data/publication.ts b/apps/web/src/data/publication.ts new file mode 100644 index 000000000..7a93bbbb1 --- /dev/null +++ b/apps/web/src/data/publication.ts @@ -0,0 +1,130 @@ +import { z } from "zod" +import { experimentRationaleSchema } from "@supabase-evals/core/eval-metadata" + +const publicationSchema = z.array( + z.object({ + experiment: z.string(), + withSkills: z.uuid().optional(), + withoutSkills: z.uuid().optional(), + rationale: experimentRationaleSchema, + }) +) + +type Publication = z.infer[number] + +// Review the rationale whenever either pinned experiment ID changes. +const entries = [ + { + experiment: "claude-code-opus-5.5", + withSkills: "c8e0f865-ca59-4703-a19d-507a234e100a", + withoutSkills: "e0fd16ad-e117-4165-8e30-9346ebe7e835", + rationale: { + summary: [ + "Claude Code's ", + { + text: "default model", + url: "https://code.claude.com/docs/en/model-config#default-model-setting", + }, + " on most plans, at its ", + { + text: "default effort", + url: "https://code.claude.com/docs/en/model-config#adjust-effort-level", + }, + ".", + ], + asOf: "2026-10-26", + }, + }, + { + experiment: "claude-code-sonnet-5.5", + withSkills: "ede84738-9f79-428f-bdfe-9d8adb2a6489", + withoutSkills: "15a067ef-f04e-44f5-9f94-7b8c0b0e10f1", + rationale: { + summary: [ + "Claude Code's ", + { + text: "Sonnet model for daily coding tasks", + url: "https://code.claude.com/docs/en/model-config", + }, + ", at its ", + { + text: "default effort", + url: "https://code.claude.com/docs/en/model-config#adjust-effort-level", + }, + ".", + ], + asOf: "2026-10-26", + }, + }, + { + experiment: "codex-gpt-6-luna", + withSkills: "b4a9c5ce-60fc-4604-bc99-595269b7fd7d", + withoutSkills: "123ecf1b-1004-458c-98cf-b17450b65c92", + rationale: { + summary: [ + "A cost-effective model for everyday tasks, and the one behind the ", + { + text: "Supabase dashboard assistant", + url: "https://github.com/supabase/supabase/blob/d98c277790766d2ceb905b675eb0a718f25e6345/apps/studio/lib/ai/model.utils.ts#L92", + }, + ".", + ], + asOf: "2026-10-26", + }, + }, + { + experiment: "codex-gpt-6.1-sol", + withSkills: "d2e33ddc-64f1-4392-892c-d4db9f3d0146", + withoutSkills: "f2a8dad6-2d8b-4088-8f5f-165d00a24058", + rationale: { + summary: [ + "Codex's ", + { + text: "default model for ChatGPT accounts, at its default effort", + url: "https://learn.chatgpt.com/docs/models?surface=app#pick-a-reasoning-effort", + }, + ".", + ], + asOf: "2026-11-09", + }, + }, + { + experiment: "grok-4.7", + withSkills: "e986dead-a1d3-4523-b47d-33a82ff1348b", + withoutSkills: "545924a6-a02c-482c-9ef7-b4dfc1a993c7", + rationale: { + summary: [ + "Grok Build's ", + { + text: "default model", + url: "https://docs.x.ai/developers/grok-4-7", + }, + ", at its ", + { + text: "default effort", + url: "https://github.com/xai-org/grok-build/blob/2bdd1d6a6369de0e8c68132ea4539e9abd9e14a8/crates/codegen/xai-grok-shell/src/remote/client.rs#L832-L862", + }, + ".", + ], + asOf: "2026-10-26", + }, + }, + { + experiment: "opencode-kimi-k3", + withSkills: "66f84377-4b0d-4be2-8f41-8e0756cbc40a", + withoutSkills: "63331944-8a8c-4396-8add-d6fbe88f80d3", + rationale: { + summary: [ + "Included as an ", + { + text: "open-weight option", + url: "https://opencode.ai/data/moonshotai/kimi-k3", + }, + " in OpenCode.", + ], + asOf: "2026-10-26", + }, + }, +] satisfies Publication[] + +export const publication = publicationSchema.parse(entries) diff --git a/apps/web/src/lib/eval-results.test.ts b/apps/web/src/lib/eval-results.test.ts index 5196e12ce..bf3e851d0 100644 --- a/apps/web/src/lib/eval-results.test.ts +++ b/apps/web/src/lib/eval-results.test.ts @@ -1,10 +1,6 @@ -import { createElement } from "react" -import { renderToStaticMarkup } from "react-dom/server" -import { ExperimentRationale } from "@/components/experiment-rationale" import { describe, expect, it } from "vitest" import { makeResult } from "@/lib/eval-results.fixture" -import { missingBenchmarkRationales } from "@/lib/experiment-rationales" import { EXPERIMENT_SUITES, getSuiteResults, @@ -20,6 +16,16 @@ import { sortedResults, } from "@/lib/eval-results" +it("reads the rationale carried by the published result", () => { + const rationale = { summary: ["Published choice."], asOf: "2026-01-01" } + const result = makeResult({ + experiment: "pinned", + experimentRationale: rationale, + }) + expect(getExperimentRationale("pinned", [result])).toEqual(rationale) + expect(getExperimentRationale("other", [result])).toBeUndefined() +}) + describe("sortResults", () => { const order = (results: ReturnType[]) => [...results].sort(sortResults).map((result) => result.eval) @@ -223,89 +229,3 @@ describe("the exported results", () => { expect(inconsistent).toEqual([]) }) }) - -describe("getExperimentRationale", () => { - const display = { - agent: "opencode", - modelProvider: "moonshotai", - modelId: "moonshotai/kimi-k3", - } as const - const match = makeResult({ - experiment: "opencode-kimi-k3", - experimentDisplay: display, - }) - - it("shows current rationale for matching results", () => { - expect( - getExperimentRationale(match.experiment, [match])?.summary - ).toContainEqual(expect.objectContaining({ text: "open-weight option" })) - }) - - it("hides rationale for mixed or missing metadata", () => { - const other = makeResult({ - experiment: match.experiment, - experimentDisplay: { ...display, modelId: "other" }, - }) - const missing = makeResult({ experiment: match.experiment }) - expect( - getExperimentRationale(match.experiment, [match, other]) - ).toBeUndefined() - expect( - getExperimentRationale(match.experiment, [match, missing]) - ).toBeUndefined() - expect( - getExperimentRationale("removed-experiment", [match]) - ).toBeUndefined() - }) -}) - -describe("benchmark rationale coverage", () => { - const kimi = { - agent: "opencode", - modelProvider: "moonshotai", - modelId: "moonshotai/kimi-k3", - } as const - - it("requires an entry matching each current benchmark model and effort", () => { - expect( - missingBenchmarkRationales([ - ["benchmark/opencode-kimi-k3", { display: kimi }], - ["benchmark/opencode-kimi-k3-no-skills", { display: kimi }], - ["benchmark/new-experiment", { display: kimi }], - [ - "benchmark/opencode-kimi-k3", - { - display: { ...kimi, reasoningEffort: "high" }, - }, - ], - ["cli/other", { display: kimi }], - ]) - ).toEqual(["benchmark/new-experiment", "benchmark/opencode-kimi-k3"]) - }) - - it("shows the historical Sonnet rationale when its results are present", () => { - const old = makeResult({ - experiment: "claude-code-sonnet-5", - experimentDisplay: { - agent: "claude-code", - modelProvider: "anthropic", - modelId: "claude-sonnet-5", - reasoningEffort: "high", - }, - }) - expect(getExperimentRationale(old.experiment, [old])?.asOf).toBe( - "2026-07-02" - ) - }) -}) - -describe("inline rationale links", () => { - it("links the claim inside the sentence", () => { - const html = renderToStaticMarkup(createElement(ExperimentRationale)) - expect(html).toContain( - 'href="https://code.claude.com/docs/en/model-config#default-model-setting"' - ) - expect(html).toContain(">default model on most plans, at its ") - expect(html).toContain(">default effort.") - }) -}) diff --git a/apps/web/src/lib/eval-results.ts b/apps/web/src/lib/eval-results.ts index 65fec25b5..6b7d91318 100644 --- a/apps/web/src/lib/eval-results.ts +++ b/apps/web/src/lib/eval-results.ts @@ -4,10 +4,6 @@ import { type EvalResult, } from "@supabase-evals/core/eval-metadata" import rawResults from "@/data/eval-results.json" -import { - experimentRationales, - sameExperimentDisplay, -} from "@/lib/experiment-rationales" /** * The exported eval results and the queries the site runs over them. Everything @@ -134,20 +130,8 @@ export function getExperimentRationale( experiment: string, sourceResults = sortedResults ) { - const results = sourceResults.filter( - (result) => result.experiment === experiment - ) - if (!results.length) return undefined - - return experimentRationales.find( - (entry) => - entry.experiment === experiment.replace(/-no-skills$/, "") && - results.every( - ({ experimentDisplay }) => - experimentDisplay && - sameExperimentDisplay(experimentDisplay, entry.display) - ) - )?.rationale + return sourceResults.find((result) => result.experiment === experiment) + ?.experimentRationale } export function getExperimentResults( diff --git a/apps/web/src/lib/experiment-rationales.ts b/apps/web/src/lib/experiment-rationales.ts deleted file mode 100644 index cef155ea0..000000000 --- a/apps/web/src/lib/experiment-rationales.ts +++ /dev/null @@ -1,55 +0,0 @@ -import { z } from "zod" -import { - experimentDisplayMetadataSchema, - type ExperimentDisplayMetadata, -} from "@supabase-evals/core/eval-metadata" -import rawRationales from "../data/experiment-rationales.json" with { type: "json" } - -export const experimentRationales = z - .array( - z.object({ - experiment: z.string(), - display: experimentDisplayMetadataSchema, - rationale: z.object({ - summary: z - .array( - z.union([ - z.string().min(1), - z.object({ text: z.string().min(1), url: z.url() }), - ]) - ) - .min(1), - asOf: z.iso.date(), - }), - }) - ) - .parse(rawRationales) - -export function sameExperimentDisplay( - a: ExperimentDisplayMetadata, - b: ExperimentDisplayMetadata -) { - return ( - a.agent === b.agent && - a.modelProvider === b.modelProvider && - a.modelId === b.modelId && - a.reasoningEffort === b.reasoningEffort - ) -} - -export function missingBenchmarkRationales( - experiments: Iterable<[string, { display: ExperimentDisplayMetadata }]> -) { - return Array.from(experiments) - .filter(([name]) => name.startsWith("benchmark/")) - .filter( - ([name, config]) => - !experimentRationales.some( - (entry) => - entry.experiment === - name.slice("benchmark/".length).replace(/-no-skills$/, "") && - sameExperimentDisplay(entry.display, config.display) - ) - ) - .map(([name]) => name) -} diff --git a/packages/core/src/eval-metadata.ts b/packages/core/src/eval-metadata.ts index 0762689f1..1d42c58d9 100644 --- a/packages/core/src/eval-metadata.ts +++ b/packages/core/src/eval-metadata.ts @@ -457,9 +457,22 @@ const evalResultShape = { // Raw result files may carry extra fields we don't model; tolerate them. export const rawEvalResultSchema = z.looseObject(evalResultShape); +export const experimentRationaleSchema = z.object({ + summary: z + .array( + z.union([ + z.string().min(1), + z.object({ text: z.string().min(1), url: z.url() }), + ]) + ) + .min(1), + asOf: z.iso.date(), +}); + // Web-facing result; a clean strict object so its inferred type stays usable. export const evalResultSchema = z.object({ ...evalResultShape, + experimentRationale: experimentRationaleSchema.optional(), passed: z.boolean(), prompt: z.string().optional(), promptSourcePath: z.string().optional(), From 2809237228066287f27e35019cd3c3b398fb1679 Mon Sep 17 00:00:00 2001 From: Matt Rossman <22670878+mattrossman@users.noreply.github.com> Date: Mon, 9 Nov 2026 16:22:14 -0500 Subject: [PATCH 15/18] feat: show experiment upload dates --- apps/web/scripts/fetch-braintrust.test.ts | 5 +++-- apps/web/scripts/fetch-braintrust.ts | 7 ++++++- .../src/components/experiment-rationale.test.tsx | 8 +++++++- apps/web/src/components/experiment-rationale.tsx | 16 ++++++---------- apps/web/src/data/publication.ts | 6 ------ apps/web/src/lib/eval-results.test.ts | 2 +- packages/core/src/eval-metadata.ts | 2 +- 7 files changed, 24 insertions(+), 22 deletions(-) diff --git a/apps/web/scripts/fetch-braintrust.test.ts b/apps/web/scripts/fetch-braintrust.test.ts index 9244b7d86..b88521a18 100644 --- a/apps/web/scripts/fetch-braintrust.test.ts +++ b/apps/web/scripts/fetch-braintrust.test.ts @@ -12,6 +12,7 @@ import { const id = "63331944-8a8c-4396-8add-d6fbe88f80d3" const experiment: Parameters[0][number] = { id, + created: "2026-10-28T18:53:01.928Z", metadata: { experiment: "example", expected_runs: 1, @@ -64,7 +65,6 @@ const make = (roots = [root], children = [score, task], usages = [usage]) => test("carries the pinned experiment's rationale into published results", () => { const rationale = { summary: ["A ", { text: "model", url: "https://example.com" }, "."], - asOf: "2026-01-01", } const withoutSkills = "d2e33ddc-64f1-4392-892c-d4db9f3d0146" const entries = [ @@ -83,11 +83,12 @@ test("carries the pinned experiment's rationale into published results", () => { ["example-eval"] ) assert.deepEqual(result.experimentRationale, rationale) + assert.equal(result.experimentCreatedAt, experiment.created) assert.throws(() => attachRationales([experiment], []), /Missing rationale/) }) test("requires both variants outside partial mode and rejects duplicate names", () => { - const rationale = { summary: ["A model."], asOf: "2026-01-01" } + const rationale = { summary: ["A model."] } const entry = { experiment: "example", withSkills: id, rationale } assert.throws( () => publicationPins([entry], false), diff --git a/apps/web/scripts/fetch-braintrust.ts b/apps/web/scripts/fetch-braintrust.ts index 002afd229..1f046d8df 100644 --- a/apps/web/scripts/fetch-braintrust.ts +++ b/apps/web/scripts/fetch-braintrust.ts @@ -20,7 +20,11 @@ const metadataSchema = z expected_runs: z.number().int().positive().optional(), }) .passthrough() -const experimentSchema = z.object({ id: uuid, metadata: metadataSchema }) +const experimentSchema = z.object({ + id: uuid, + created: z.iso.datetime(), + metadata: metadataSchema, +}) const rootSchema = z.object({ id: z.string(), experiment_id: uuid, @@ -179,6 +183,7 @@ export function assemble( const parsed = evalResultSchema.safeParse({ experiment: em.experiment, experimentRationale: exp.rationale, + experimentCreatedAt: exp.created, experimentDisplay: { agent: em.agent, modelProvider: em.model_provider, diff --git a/apps/web/src/components/experiment-rationale.test.tsx b/apps/web/src/components/experiment-rationale.test.tsx index 144b591ea..627f9341f 100644 --- a/apps/web/src/components/experiment-rationale.test.tsx +++ b/apps/web/src/components/experiment-rationale.test.tsx @@ -5,13 +5,15 @@ vi.mock("@/lib/eval-results", () => ({ getSuiteResults: () => [], getVisibleExperiments: () => ["test-experiment"], getExperimentDisplay: () => undefined, + getExperimentResults: () => [ + { experimentCreatedAt: "2026-10-28T18:53:01.928Z" }, + ], getExperimentRationale: () => ({ summary: [ "See the ", { text: "model", url: "https://example.com/model" }, ".", ], - asOf: "2026-01-01", }), })) @@ -22,5 +24,9 @@ describe("inline rationale links", () => { const html = renderToStaticMarkup() expect(html).toContain('See the model.") + expect(html).toContain("Experiment upload dates") + expect(html).toContain( + 'dateTime="2026-10-28T18:53:01.928Z">2026-10-28' + ) }) }) diff --git a/apps/web/src/components/experiment-rationale.tsx b/apps/web/src/components/experiment-rationale.tsx index 15155e2c2..3cb467c06 100644 --- a/apps/web/src/components/experiment-rationale.tsx +++ b/apps/web/src/components/experiment-rationale.tsx @@ -1,6 +1,7 @@ import { ExperimentLabel } from "@/components/results/experiment-label" import { getExperimentRationale, + getExperimentResults, getSuiteResults, getVisibleExperiments, } from "@/lib/eval-results" @@ -56,26 +57,21 @@ export function ExperimentRationale() {
- Source review dates + Experiment upload dates -

- When the sources supporting each model and effort choice were last - reviewed. -

- + {experiments.map((experiment) => { - const rationale = getExperimentRationale(experiment) + const createdAt = + getExperimentResults(experiment)[0]?.experimentCreatedAt return ( ) diff --git a/apps/web/src/data/publication.ts b/apps/web/src/data/publication.ts index 7a93bbbb1..12ee3ae20 100644 --- a/apps/web/src/data/publication.ts +++ b/apps/web/src/data/publication.ts @@ -32,7 +32,6 @@ const entries = [ }, ".", ], - asOf: "2026-10-26", }, }, { @@ -53,7 +52,6 @@ const entries = [ }, ".", ], - asOf: "2026-10-26", }, }, { @@ -69,7 +67,6 @@ const entries = [ }, ".", ], - asOf: "2026-10-26", }, }, { @@ -85,7 +82,6 @@ const entries = [ }, ".", ], - asOf: "2026-11-09", }, }, { @@ -106,7 +102,6 @@ const entries = [ }, ".", ], - asOf: "2026-10-26", }, }, { @@ -122,7 +117,6 @@ const entries = [ }, " in OpenCode.", ], - asOf: "2026-10-26", }, }, ] satisfies Publication[] diff --git a/apps/web/src/lib/eval-results.test.ts b/apps/web/src/lib/eval-results.test.ts index bf3e851d0..5cf41f43e 100644 --- a/apps/web/src/lib/eval-results.test.ts +++ b/apps/web/src/lib/eval-results.test.ts @@ -17,7 +17,7 @@ import { } from "@/lib/eval-results" it("reads the rationale carried by the published result", () => { - const rationale = { summary: ["Published choice."], asOf: "2026-01-01" } + const rationale = { summary: ["Published choice."] } const result = makeResult({ experiment: "pinned", experimentRationale: rationale, diff --git a/packages/core/src/eval-metadata.ts b/packages/core/src/eval-metadata.ts index 1d42c58d9..e3b119341 100644 --- a/packages/core/src/eval-metadata.ts +++ b/packages/core/src/eval-metadata.ts @@ -466,13 +466,13 @@ export const experimentRationaleSchema = z.object({ ]) ) .min(1), - asOf: z.iso.date(), }); // Web-facing result; a clean strict object so its inferred type stays usable. export const evalResultSchema = z.object({ ...evalResultShape, experimentRationale: experimentRationaleSchema.optional(), + experimentCreatedAt: z.iso.datetime().optional(), passed: z.boolean(), prompt: z.string().optional(), promptSourcePath: z.string().optional(), From bf27a7634cf4c14c12cb3e768100d41f3df3e190 Mon Sep 17 00:00:00 2001 From: Matt Rossman <22670878+mattrossman@users.noreply.github.com> Date: Mon, 9 Nov 2026 16:30:50 -0500 Subject: [PATCH 16/18] style: trim experiment section introduction --- apps/web/src/components/experiment-rationale.tsx | 3 --- 1 file changed, 3 deletions(-) diff --git a/apps/web/src/components/experiment-rationale.tsx b/apps/web/src/components/experiment-rationale.tsx index 3cb467c06..a21f0d87b 100644 --- a/apps/web/src/components/experiment-rationale.tsx +++ b/apps/web/src/components/experiment-rationale.tsx @@ -23,9 +23,6 @@ export function ExperimentRationale() { > Why these experiments -

- Why we chose the model and effort for each benchmark experiment. -

    {experiments.map((experiment) => { const rationale = getExperimentRationale(experiment) From 410bc4edcff7e1e5a97e96039e88c26c74bfbd6b Mon Sep 17 00:00:00 2001 From: Matt Rossman <22670878+mattrossman@users.noreply.github.com> Date: Mon, 9 Nov 2026 17:00:43 -0500 Subject: [PATCH 17/18] fix: cite Codex model recommendation --- apps/web/src/data/publication.ts | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/apps/web/src/data/publication.ts b/apps/web/src/data/publication.ts index 12ee3ae20..d79cd0c98 100644 --- a/apps/web/src/data/publication.ts +++ b/apps/web/src/data/publication.ts @@ -75,12 +75,11 @@ const entries = [ withoutSkills: "f2a8dad6-2d8b-4088-8f5f-165d00a24058", rationale: { summary: [ - "Codex's ", { - text: "default model for ChatGPT accounts, at its default effort", - url: "https://learn.chatgpt.com/docs/models?surface=app#pick-a-reasoning-effort", + text: "OpenAI's recommended Codex model for paid ChatGPT plans", + url: "https://learn.chatgpt.com/docs/models?surface=cli#gpt-55-retirement", }, - ".", + ", run at medium effort.", ], }, }, From 2d6b8846f4b7af9d8dab2ceaaf7afa5b1115c9b9 Mon Sep 17 00:00:00 2001 From: Matt Rossman <22670878+mattrossman@users.noreply.github.com> Date: Wed, 11 Nov 2026 13:38:38 -0500 Subject: [PATCH 18/18] fix: keep experiment upload dates visible --- apps/web/src/components/experiment-rationale.tsx | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/apps/web/src/components/experiment-rationale.tsx b/apps/web/src/components/experiment-rationale.tsx index a21f0d87b..e622c2d92 100644 --- a/apps/web/src/components/experiment-rationale.tsx +++ b/apps/web/src/components/experiment-rationale.tsx @@ -52,11 +52,11 @@ export function ExperimentRationale() { ) })}
-
- +
+

Experiment upload dates -

-
- Source review dates by experiment - Experiment upload dates
- +
+ +
{experiments.map((experiment) => { @@ -75,7 +75,7 @@ export function ExperimentRationale() { })}
Experiment upload dates
-
+ ) }