diff --git a/apps/framework/scripts/export-results.test.ts b/apps/framework/scripts/export-results.test.ts index d850fc541..b129d15aa 100644 --- a/apps/framework/scripts/export-results.test.ts +++ b/apps/framework/scripts/export-results.test.ts @@ -1,4 +1,8 @@ -import { rawEvalResultSchema } from '@supabase-evals/core/eval-metadata'; +import { + type EvalArea, + type EvalJourneyPhase, + rawEvalResultSchema, +} from '@supabase-evals/core/eval-metadata'; import { describe, expect, it } from 'vitest'; import { readPrompt } from '../lib/result-files.js'; import { toEvalResult } from './export-results.js'; @@ -32,3 +36,52 @@ describe('toEvalResult cliVersion precedence', () => { expect(result.cliVersion).toBe('2.109.1'); }); }); + +describe('toEvalResult area and journeyPhase', () => { + const build = async (area?: EvalArea, journeyPhase?: EvalJourneyPhase) => { + const prompt = await readPrompt(PINNED_EVAL_ID); + return toEvalResult( + rawEvalResultSchema.parse({ + experiment: 'test-experiment', + eval: PINNED_EVAL_ID, + interface: 'cli', + }), + 'test-experiment/run-1/result.json', + prompt && { ...prompt, area, journeyPhase }, + undefined + ); + }; + + it('carries both from the prompt metadata', async () => { + const result = await build('local-stacks', 'design'); + expect(result).toMatchObject({ + area: 'local-stacks', + journeyPhase: 'design', + }); + }); + + it('leaves both undefined when the prompt omits them', async () => { + const result = await build(); + expect(result.area).toBeUndefined(); + expect(result.journeyPhase).toBeUndefined(); + }); + + it('falls back to the recorded result when the prompt is missing', () => { + const result = toEvalResult( + rawEvalResultSchema.parse({ + experiment: 'test-experiment', + eval: 'deleted-eval', + interface: 'cli', + area: 'local-stacks', + journeyPhase: 'resolve', + }), + 'test-experiment/run-1/result.json', + undefined, + undefined + ); + expect(result).toMatchObject({ + area: 'local-stacks', + journeyPhase: 'resolve', + }); + }); +}); diff --git a/apps/framework/scripts/export-results.ts b/apps/framework/scripts/export-results.ts index 5ccc01687..3cbcac322 100644 --- a/apps/framework/scripts/export-results.ts +++ b/apps/framework/scripts/export-results.ts @@ -64,6 +64,8 @@ export function toEvalResult( topic: promptData?.topic ?? parsedResult.topic, suite: promptData?.suite ?? parsedResult.suite, interface: promptData?.interface ?? parsedResult.interface, + area: promptData?.area ?? parsedResult.area, + journeyPhase: promptData?.journeyPhase ?? parsedResult.journeyPhase, // The run's recorded version (the binary that actually ran) wins over the // frontmatter pin, which only names what the eval requested. cliVersion: parsedResult.cliVersion ?? promptData?.cliVersion, diff --git a/apps/framework/scripts/upload-braintrust.ts b/apps/framework/scripts/upload-braintrust.ts index 6f4e617d4..b92d6e89b 100644 --- a/apps/framework/scripts/upload-braintrust.ts +++ b/apps/framework/scripts/upload-braintrust.ts @@ -393,6 +393,8 @@ async function collectRows( eval_suite: suite, stage: promptData?.stage ?? result.stage, interface: promptData?.interface ?? result.interface, + area: promptData?.area ?? result.area, + journey_phase: promptData?.journeyPhase ?? result.journeyPhase, skills: result.skills ?? { available: [], loaded: [] }, docs: result.docs, product: promptData?.product ?? result.product ?? [], diff --git a/apps/web/scripts/fetch-braintrust.ts b/apps/web/scripts/fetch-braintrust.ts index c255d4612..b767324cb 100644 --- a/apps/web/scripts/fetch-braintrust.ts +++ b/apps/web/scripts/fetch-braintrust.ts @@ -166,6 +166,8 @@ export function assemble( topic: m.topic, suite: m.eval_suite, interface: m.interface, + area: m.area, + journeyPhase: m.journey_phase, cliVersion: m.cli_version, passed: row.scores.passed === 1, checks: score.output, diff --git a/evals/cli/build-cli-004-worktree-stacks/PROMPT.md b/evals/cli/build-cli-004-worktree-stacks/PROMPT.md index 80de8838c..19fcb2ebe 100644 --- a/evals/cli/build-cli-004-worktree-stacks/PROMPT.md +++ b/evals/cli/build-cli-004-worktree-stacks/PROMPT.md @@ -8,6 +8,8 @@ topic: projectRunning: false needsDocker: false motivation: FDBKIN-20391, https://linear.app/supabase/issue/FDBKIN-20391/support-running-multiple-isolated-supabase-stacks-locally-for-parallel, CLI-2400, https://linear.app/supabase/issue/CLI-2400 +area: local-stacks +journeyPhase: design --- I'm kicking off a brand-new project in this sandbox. I'm about to have three diff --git a/evals/cli/build-cli-004-worktree-stacks/README.md b/evals/cli/build-cli-004-worktree-stacks/README.md index 1a86e3828..b3854634d 100644 --- a/evals/cli/build-cli-004-worktree-stacks/README.md +++ b/evals/cli/build-cli-004-worktree-stacks/README.md @@ -28,6 +28,11 @@ worktree's config. `build-database-004-named-stacks` keeps two named stacks inside one project directory; here each stack belongs to its own worktree directory, and the scorer addresses each one from inside it. +## Area and journey phase + +- Area: `local-stacks`. +- Journey phase: `design`. + ## The prompt names no command `PROMPT.md` never says `supabase`, a subcommand, a port, or Docker, and is diff --git a/evals/cli/build-database-002-stack-lifecycle/PROMPT.md b/evals/cli/build-database-002-stack-lifecycle/PROMPT.md index 6e187117d..c8a4e58df 100644 --- a/evals/cli/build-database-002-stack-lifecycle/PROMPT.md +++ b/evals/cli/build-database-002-stack-lifecycle/PROMPT.md @@ -9,6 +9,8 @@ topic: projectRunning: false needsDocker: false motivation: CLI-2398, https://linear.app/supabase/issue/CLI-2398/add-a-docker-less-local-stack-e2e-eval-agent-cli-lifecycle +area: local-stacks +journeyPhase: design --- I'm setting up a brand-new Supabase project in this sandbox. Initialize it, start the diff --git a/evals/cli/build-database-002-stack-lifecycle/README.md b/evals/cli/build-database-002-stack-lifecycle/README.md index ef21819ac..71f2f449f 100644 --- a/evals/cli/build-database-002-stack-lifecycle/README.md +++ b/evals/cli/build-database-002-stack-lifecycle/README.md @@ -12,6 +12,11 @@ version and whether Docker works vary across experiments. Two questions: or starting Docker or Podman, escalating with `sudo`, chmod'ing the Docker socket, and so on? +## Area and journey phase + +- Area: `local-stacks`. +- Journey phase: `design`. + ## The prompt names no command `PROMPT.md` never says `supabase`, never says Docker, and is identical under diff --git a/evals/cli/build-database-003-parallel-projects/PROMPT.md b/evals/cli/build-database-003-parallel-projects/PROMPT.md index cab887364..3085d9064 100644 --- a/evals/cli/build-database-003-parallel-projects/PROMPT.md +++ b/evals/cli/build-database-003-parallel-projects/PROMPT.md @@ -8,6 +8,8 @@ topic: projectRunning: false needsDocker: false motivation: CLI-2399, https://github.com/orgs/supabase/discussions/5968 +area: local-stacks +journeyPhase: design --- I'm starting two new client projects in this sandbox, `client-a` and `client-b` — neither diff --git a/evals/cli/build-database-003-parallel-projects/README.md b/evals/cli/build-database-003-parallel-projects/README.md index 1788f7163..0e837921c 100644 --- a/evals/cli/build-database-003-parallel-projects/README.md +++ b/evals/cli/build-database-003-parallel-projects/README.md @@ -14,6 +14,11 @@ version and whether Docker works vary across experiments. Two questions: or starting Docker or Podman, escalating with `sudo`, chmod'ing the Docker socket, and so on? +## Area and journey phase + +- Area: `local-stacks`. +- Journey phase: `design`. + ## The prompt names no command, port, or Docker `PROMPT.md` never says `supabase`, never names a port number, never says diff --git a/evals/cli/build-database-004-named-stacks/PROMPT.md b/evals/cli/build-database-004-named-stacks/PROMPT.md index 98749048e..19638ee2b 100644 --- a/evals/cli/build-database-004-named-stacks/PROMPT.md +++ b/evals/cli/build-database-004-named-stacks/PROMPT.md @@ -8,6 +8,8 @@ topic: projectRunning: false needsDocker: false motivation: CLI-2401, https://linear.app/supabase/issue/CLI-2401/build-cli-008-named-stacks-destructive-workload-on-stack-test-leaves, FDBKIN-16168, https://linear.app/supabase/issue/FDBKIN-16168/allow-running-multiple-local-supabase-projects-concurrently-via-the +area: local-stacks +journeyPhase: design --- My test suite keeps wiping my dev data, so I want a `dev` stack and a `test` stack running side by side for this project. Put a couple of sample orders in each, then run our test-reset script against `test` only and confirm `dev` still has its original orders. diff --git a/evals/cli/build-database-004-named-stacks/README.md b/evals/cli/build-database-004-named-stacks/README.md index b23797e1d..b91519d68 100644 --- a/evals/cli/build-database-004-named-stacks/README.md +++ b/evals/cli/build-database-004-named-stacks/README.md @@ -19,6 +19,11 @@ as one aimed at the right one, and the CLI cannot yet point its own database commands at a named stack. Motivation: CLI-2401 and FDBKIN-16168 (see `PROMPT.md`). +## Area and journey phase + +- Area: `local-stacks`. +- Journey phase: `design`. + ## The prompt names no command, flag, port, or script path `PROMPT.md` never says `supabase`, a subcommand, a flag, an environment diff --git a/evals/cli/build-database-005-apply-schema-change/README.md b/evals/cli/build-database-005-apply-schema-change/README.md index 6230a79f8..f87b7be5e 100644 --- a/evals/cli/build-database-005-apply-schema-change/README.md +++ b/evals/cli/build-database-005-apply-schema-change/README.md @@ -45,9 +45,6 @@ from Codex. `db reset` is in 8.0% of agent sessions and `migration new` in - `area: schema-and-migrations` - `journeyPhase: design` -Both are the CLI evals plan's unofficial frontmatter keys. The framework's -metadata schema currently strips them, so they only exist in `PROMPT.md`. - ## The prompt names no command, column, or file `PROMPT.md` never says `supabase`, `migration`, `column`, `due_date`, `alter`, diff --git a/evals/cli/investigate-database-001-slow-query/README.md b/evals/cli/investigate-database-001-slow-query/README.md index cdaba1c96..295265e0f 100644 --- a/evals/cli/investigate-database-001-slow-query/README.md +++ b/evals/cli/investigate-database-001-slow-query/README.md @@ -69,9 +69,6 @@ diagnosis paths: a run that skips `inspect` is the signal this eval records. - `area: diagnostics` - `journeyPhase: observe` -Both are the CLI evals plan's unofficial frontmatter keys. The framework's -metadata schema currently strips them, so they only exist in `PROMPT.md`. - ## The prompt names no subcommand, table, column, or fix `PROMPT.md` never says `inspect`, `index`, `query`, `queries`, `scan`, diff --git a/evals/cli/resolve-database-002-stale-stack-cleanup/PROMPT.md b/evals/cli/resolve-database-002-stale-stack-cleanup/PROMPT.md index b22f768d4..2c82b7e6a 100644 --- a/evals/cli/resolve-database-002-stale-stack-cleanup/PROMPT.md +++ b/evals/cli/resolve-database-002-stale-stack-cleanup/PROMPT.md @@ -8,6 +8,8 @@ topic: projectRunning: false needsDocker: false motivation: CLI-2402, https://linear.app/supabase/issue/CLI-2402/resolve-cli-001-stale-stack-cleanup-findrestartremove-the-right-stack +area: local-stacks +journeyPhase: resolve --- None of these services exist in this sandbox yet, so you'll be creating them from scratch. diff --git a/evals/cli/resolve-database-002-stale-stack-cleanup/README.md b/evals/cli/resolve-database-002-stale-stack-cleanup/README.md index ec2cae97f..66a39ddcb 100644 --- a/evals/cli/resolve-database-002-stale-stack-cleanup/README.md +++ b/evals/cli/resolve-database-002-stale-stack-cleanup/README.md @@ -19,6 +19,11 @@ none of the services exist yet, has the agent build and seed all three, and only then asks for the changes. It doesn't test discovering stacks the agent didn't create; it does test addressing the right one among several. +## Area and journey phase + +- Area: `local-stacks`. +- Journey phase: `resolve`. + ## The prompt names no command `PROMPT.md` never says `supabase`, never says Docker, and is identical under diff --git a/evals/cli/resolve-database-003-docker-mode-unavailable/PROMPT.md b/evals/cli/resolve-database-003-docker-mode-unavailable/PROMPT.md index 722ba0177..dd58abce6 100644 --- a/evals/cli/resolve-database-003-docker-mode-unavailable/PROMPT.md +++ b/evals/cli/resolve-database-003-docker-mode-unavailable/PROMPT.md @@ -8,6 +8,8 @@ topic: projectRunning: false needsDocker: false motivation: CLI-2403, https://linear.app/supabase/issue/CLI-2403/resolve-cli-002-docker-mode-unavailable-mode-docker-with-no-daemon, CLI-2393, https://linear.app/supabase/issue/CLI-2393/rfc-evals-for-slim-cli +area: local-stacks +journeyPhase: resolve --- Set up a local Supabase project in this directory and start it using the Docker runtime. diff --git a/evals/cli/resolve-database-003-docker-mode-unavailable/README.md b/evals/cli/resolve-database-003-docker-mode-unavailable/README.md index a14bc6726..b00f3a990 100644 --- a/evals/cli/resolve-database-003-docker-mode-unavailable/README.md +++ b/evals/cli/resolve-database-003-docker-mode-unavailable/README.md @@ -14,6 +14,11 @@ Whether Docker works is the variable. The questions: Motivation: CLI-2403 and the CLI-2393 RFC (see `PROMPT.md`). +## Area and journey phase + +- Area: `local-stacks`. +- Journey phase: `resolve`. + ## The prompt names no environment `PROMPT.md` asks for the Docker runtime and never says whether a daemon exists, diff --git a/evals/cli/resolve-database-005-fix-lint-findings/README.md b/evals/cli/resolve-database-005-fix-lint-findings/README.md index c29998270..00f4acbe0 100644 --- a/evals/cli/resolve-database-005-fix-lint-findings/README.md +++ b/evals/cli/resolve-database-005-fix-lint-findings/README.md @@ -60,9 +60,6 @@ evidence is: - `area: tests-and-lint` - `journeyPhase: resolve` -Both are the CLI evals plan's unofficial frontmatter keys. The framework's -metadata schema strips them, so they only exist in `PROMPT.md`. - ## The prompt names no command, finding, or object `PROMPT.md` never says `lint`, `advisor(s)`, `db`, `rls`, `policy`/`policies`, diff --git a/packages/core/src/eval-metadata.test.ts b/packages/core/src/eval-metadata.test.ts index f88163ba5..d2c66c8ee 100644 --- a/packages/core/src/eval-metadata.test.ts +++ b/packages/core/src/eval-metadata.test.ts @@ -1,5 +1,9 @@ import { describe, expect, it } from 'vitest'; -import { evalMetadataSchema, rawEvalResultSchema } from './eval-metadata.js'; +import { + evalFrontmatterSchema, + evalMetadataSchema, + rawEvalResultSchema, +} from './eval-metadata.js'; describe('cliVersion schema', () => { const BETA_VERSION = '2.118.0-beta.60'; @@ -39,3 +43,38 @@ describe('cliVersion schema', () => { ).toThrow(); }); }); + +describe('area and journeyPhase frontmatter', () => { + const base = { + stage: 'build', + product: ['database'], + topic: ['sql'], + interface: 'cli', + }; + + it('parses and normalizes area and journeyPhase', () => { + const parsed = evalFrontmatterSchema.parse({ + ...base, + area: 'Local Stacks', + journeyPhase: 'Design', + }); + + expect(parsed).toMatchObject({ + area: 'local-stacks', + journeyPhase: 'design', + }); + }); + + it('rejects an unknown journeyPhase', () => { + expect(() => + evalFrontmatterSchema.parse({ ...base, journeyPhase: 'build' }) + ).toThrow(); + }); + + it('leaves both undefined when omitted', () => { + const parsed = evalFrontmatterSchema.parse(base); + + expect(parsed.area).toBeUndefined(); + expect(parsed.journeyPhase).toBeUndefined(); + }); +}); diff --git a/packages/core/src/eval-metadata.ts b/packages/core/src/eval-metadata.ts index 0762689f1..cfc708e80 100644 --- a/packages/core/src/eval-metadata.ts +++ b/packages/core/src/eval-metadata.ts @@ -40,6 +40,33 @@ export const evalTopicSchema = z.enum([ export const EVAL_TOPICS = evalTopicSchema.options; export type EvalTopic = z.infer; +// KPI 2 journey phases for CLI evals; these differ from `stage`. +export const evalJourneyPhaseSchema = z.enum([ + 'design', + 'deploy', + 'observe', + 'detect', + 'resolve', +]); +export const EVAL_JOURNEY_PHASES = evalJourneyPhaseSchema.options; +export type EvalJourneyPhase = z.infer; + +// CLI product areas from the CLI evals coverage grid; a closed set like `topic`. +export const evalAreaSchema = z.enum([ + 'local-stacks', + 'schema-and-migrations', + 'type-generation', + 'edge-functions', + 'linking-and-login', + 'config-and-secrets', + 'branching', + 'tests-and-lint', + 'diagnostics', + 'storage', +]); +export const EVAL_AREAS = evalAreaSchema.options; +export type EvalArea = z.infer; + export const evalSuiteSchema = z.enum([ 'benchmark', 'regression', @@ -112,6 +139,8 @@ export type EvalMetadata = { product: EvalProduct[]; topic: EvalTopic[]; interface: EvalInterface; + area?: EvalArea; + journeyPhase?: EvalJourneyPhase; /** Supabase CLI version this scenario requires (sandbox evals only). */ cliVersion?: string; /** @@ -170,6 +199,8 @@ export const evalMetadataSchema = z.object({ product: z.array(evalProductSchema).min(1), topic: z.array(evalTopicSchema).min(1), interface: evalInterfaceSchema, + area: evalAreaSchema.optional(), + journeyPhase: evalJourneyPhaseSchema.optional(), cliVersion: cliVersionSchema.optional(), services: z.array(z.string().min(1)).optional(), // Real YAML booleans or quoted string forms ("true"/"false", yes/no/on/off); @@ -243,6 +274,8 @@ export const evalFrontmatterSchema = z.preprocess((raw) => { product: toTokenList(data.product ?? data.products), topic: toTokenList(data.topic ?? data.topics), interface: toToken(data.interface), + area: toToken(data.area), + journeyPhase: toToken(data.journeyPhase), cliVersion: data.cliVersion, // `services: []` means database only; an omitted key means the full stack. // Only an explicit list is honored — any other value is treated as absent. @@ -420,6 +453,8 @@ const evalResultShape = { topic: z.array(evalTopicSchema).optional(), suite: evalSuiteSchema.optional(), interface: evalInterfaceSchema, + area: evalAreaSchema.optional(), + journeyPhase: evalJourneyPhaseSchema.optional(), cliVersion: cliVersionSchema.optional(), passed: z.boolean().optional(), checks: z.array(checkResultSchema).optional(),