diff --git a/.changeset/sparkly-waves-post.md b/.changeset/sparkly-waves-post.md new file mode 100644 index 00000000..6c314326 --- /dev/null +++ b/.changeset/sparkly-waves-post.md @@ -0,0 +1,8 @@ +--- +'@primer/agent-eval': minor +--- + +Add support for running benchmark experiments with the `benchmark` option. + +Calls to `getExperiment` and `listExperiments` now require `benchmarksDirectory`, +including when loading only scenario-backed experiments. diff --git a/AGENTS.md b/AGENTS.md index bf2e3b1d..91256abe 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -31,6 +31,16 @@ `SKILL.md` concise, put detailed guidance in references, and use npm/npx in skill examples. +## Evidence for decisions + +- When a decision, recommendation, or review finding relies on a resource, + include the specific evidence and explain how it supports the conclusion. +- Cite repository files with paths and line numbers. For external resources, + link to the relevant section or comment and quote the supporting passage + when line numbers are unavailable. +- Read the cited resource before relying on it. Distinguish what it explicitly + states from your inference, and state uncertainty or conflicting evidence. + ## Code - Parse, don't validate. Use the type system to gurantee correctness. @@ -123,7 +133,7 @@ When adding an experiment, follow these rules: it, when appropriate - New experiments are added at `./experiments` as a new `*.ts` file - Provide an appropriate name for the experiment as its file name -- Export an `experiment` named export created with `defineConfig` +- Use a default export created with `defineConfig` - Provide an appropriate name and description for what we are testing - Select appropriate models and scenarios based on the experiment - For treatments, make sure to include the fewest number possible to meet the diff --git a/docs/experiments.md b/docs/experiments.md index ec85d13d..59d1dd02 100644 --- a/docs/experiments.md +++ b/docs/experiments.md @@ -2,9 +2,9 @@ Experiments are used to determine how different setups (or treatments) affect the performance of an agent on a given task. By default, they live in an `experiments` folder in your project. -Each experiment defines a set of scenarios that are used to evaluate the -performance of each treatment. An experiment also defines a set of models to -use. +Each experiment defines either a set of scenarios or a benchmark whose +capabilities and scenarios are used to evaluate each treatment. An experiment +also defines a set of models to use. As an example, you may want to design an experiment comparing the performance of your MCP server to a skills based approach. With this in mind, you create two @@ -55,3 +55,37 @@ export default defineConfig({ You can interact with experiments using the `experiments` subcommand of the `agent-eval` CLI. This sub-command gives you access to run experiments, create run plans to use for sharding, or merge the results of a plan. Use `agent-eval experiments --help` to see the available commands and options. + +## Reports + +Experiment runs print a summary grouped by treatment, scenario, and model +variant. When the experiment selects a `benchmark` instead of `scenarios`, the +report adds a `Capability` column and capability totals between each treatment +and its scenarios: + +```text +treatment + capability + scenario + model / reasoning effort +``` + +Treatment totals include all completed capabilities. Scenarios and model +variants are grouped separately within each capability, even when a scenario +appears in more than one capability. Reports keep runner results separate and +show a `Runner` column when SDK trials are present. + +Each level includes run counts, check summaries, and implementation-agent usage +totals. Partial or sharded runs show only the groups with completed results. +Scenario-backed experiments retain their existing report format. + +On the website, experiment pages show overall treatment summaries, then +capability summaries with their scenario results. Runner, model, and reasoning +effort groups remain separate. Website resource usage is averaged per trial. +Scenario output links open the matching capability and scenario on the run +detail page, which also supports filtering by capability. + +Benchmark-backed result bundles preserve capability metadata in `output.json` +and a `capabilityId` on each trial. Older experiment bundles without this +metadata remain readable, but their results stay grouped by scenario because +capability membership cannot be inferred reliably. diff --git a/experiments/benchmark.ts b/experiments/benchmark.ts new file mode 100644 index 00000000..e68ba3fb --- /dev/null +++ b/experiments/benchmark.ts @@ -0,0 +1,16 @@ +import {defineConfig} from '@primer/agent-eval/experiment' + +export default defineConfig({ + name: 'benchmark', + description: 'An end-to-end test for running experiments against benchmarks', + models: ['gpt-5.6-luna'], + benchmark: 'noop', + treatments: [ + { + name: 'noop', + async setup() { + console.log('local setup') + }, + }, + ], +}) diff --git a/packages/agent-eval/src/__snapshots__/package.test.ts.snap b/packages/agent-eval/src/__snapshots__/package.test.ts.snap index 3929a7bc..f435da64 100644 --- a/packages/agent-eval/src/__snapshots__/package.test.ts.snap +++ b/packages/agent-eval/src/__snapshots__/package.test.ts.snap @@ -11,6 +11,7 @@ exports[`@primer/agent-eval exports 1`] = ` "listExperiments", "ExperimentOutputFileSchema", "ExperimentTrialOutputSchema", + "parseExperimentTrialOutput", "getScenario", "listScenarios", "addCheckResults", diff --git a/packages/agent-eval/src/benchmark/output.ts b/packages/agent-eval/src/benchmark/output.ts index bce7f540..f4899fa0 100644 --- a/packages/agent-eval/src/benchmark/output.ts +++ b/packages/agent-eval/src/benchmark/output.ts @@ -399,10 +399,11 @@ async function listBenchmarkOutputFiles({ export { BenchmarkOutputFileSchema, BenchmarkTrialOutputSchema, + CapabilityOutputSchema, parseBenchmarkTrialOutput, createBenchmarkOutput, listBenchmarkOutputFiles, mergeBenchmarkOutputFiles, writeBenchmarkOutput, } -export type {BenchmarkOutput, BenchmarkTrialOutput} +export type {BenchmarkOutput, BenchmarkTrialOutput, CapabilityOutput} diff --git a/packages/agent-eval/src/cli/commands/experiment.ts b/packages/agent-eval/src/cli/commands/experiment.ts index 68988f0b..7e683bf4 100644 --- a/packages/agent-eval/src/cli/commands/experiment.ts +++ b/packages/agent-eval/src/cli/commands/experiment.ts @@ -14,6 +14,7 @@ import {logger} from '../../logger' import {createPlanFromManifest, selectPlanTrials, runPlan} from '../../plan' import {parseShard} from '../../shard' import { + benchmarksOption, copilotConcurrencyOption, containerConcurrencyOption, experimentsOption, @@ -82,6 +83,7 @@ const experimentCommand = defineCommand({ description: 'Create an experiment plan', }, args: { + benchmarks: benchmarksOption, experiments: experimentsOption, name: { type: 'positional', @@ -98,25 +100,35 @@ const experimentCommand = defineCommand({ async run({args}) { logger.info('Planning experiment: %s', args.name) + const benchmarksDirectory = path.resolve(args.benchmarks) const experimentsDirectory = path.resolve(args.experiments) const scenariosDirectory = path.resolve(args.scenarios) const outputPath = path.resolve(args['output-path']) logger.debug({ + benchmarksDirectory, experimentsDirectory, scenariosDirectory, outputPath, }) const experiment = await getExperiment({ + benchmarksDirectory, experimentsDirectory, scenariosDirectory, name: args.name, }) - const plan = createExperimentPlan({experiment}) - const manifest = createExperimentPlanManifest({experiment, plan}) + const plan = createExperimentPlan({ + experiment, + }) + const manifest = createExperimentPlanManifest({ + experiment, + plan, + }) - await host.fs.mkdir(path.dirname(outputPath), {recursive: true}) + await host.fs.mkdir(path.dirname(outputPath), { + recursive: true, + }) await host.fs.writeFile(outputPath, JSON.stringify(manifest, null, 2), 'utf-8') logger.info('Wrote experiment plan to: %s', path.relative(process.cwd(), outputPath)) @@ -128,6 +140,7 @@ const experimentCommand = defineCommand({ description: 'Run an experiment plan from a manifest', }, args: { + benchmarks: benchmarksOption, experiments: experimentsOption, 'copilot-concurrency': copilotConcurrencyOption, 'container-concurrency': containerConcurrencyOption, @@ -143,6 +156,7 @@ const experimentCommand = defineCommand({ token: githubCopilotTokenOption, }, async run({args}) { + const benchmarksDirectory = path.resolve(args.benchmarks) const experimentsDirectory = path.resolve(args.experiments) const copilotConcurrency = getConcurrencyValue(args['copilot-concurrency'], 'copilot-concurrency') const containerConcurrency = getConcurrencyValue(args['container-concurrency'], 'container-concurrency') @@ -156,6 +170,7 @@ const experimentCommand = defineCommand({ logger.debug({ artifactsDirectory, + benchmarksDirectory, experimentsDirectory, copilotConcurrency, containerConcurrency, @@ -172,6 +187,7 @@ const experimentCommand = defineCommand({ const contents = await host.fs.readFile(planPath, 'utf-8') const manifest = await parseExperimentPlanManifest({ + benchmarksDirectory, experimentsDirectory, contents, scenariosDirectory, @@ -210,6 +226,7 @@ const experimentCommand = defineCommand({ description: 'Run an experiment', }, args: { + benchmarks: benchmarksOption, experiments: experimentsOption, 'copilot-concurrency': copilotConcurrencyOption, 'container-concurrency': containerConcurrencyOption, @@ -225,6 +242,7 @@ const experimentCommand = defineCommand({ async run({args}) { logger.info('Running experiment: %s', args.name) + const benchmarksDirectory = path.resolve(args.benchmarks) const experimentsDirectory = path.resolve(args.experiments) const copilotConcurrency = getConcurrencyValue(args['copilot-concurrency'], 'copilot-concurrency') const containerConcurrency = getConcurrencyValue(args['container-concurrency'], 'container-concurrency') @@ -236,6 +254,7 @@ const experimentCommand = defineCommand({ logger.debug({ artifactsDirectory, + benchmarksDirectory, experimentsDirectory, copilotConcurrency, containerConcurrency, @@ -245,6 +264,7 @@ const experimentCommand = defineCommand({ }) const experiment = await getExperiment({ + benchmarksDirectory, experimentsDirectory, scenariosDirectory, name: args.name, diff --git a/packages/agent-eval/src/experiment/config.ts b/packages/agent-eval/src/experiment/config.ts index ef8ba503..284f311d 100644 --- a/packages/agent-eval/src/experiment/config.ts +++ b/packages/agent-eval/src/experiment/config.ts @@ -4,7 +4,35 @@ import {ModelVariantConfigSchema} from '../model' import {ControlTreatment, TreatmentConfigSchema, TreatmentSetupSchema} from '../treatment' import {InlineScenarioConfigSchema} from '../scenario/config' -const ExperimentConfigSchema = z.strictObject({ +const BenchmarkExperimentConfig = z.strictObject({ + name: z.string(), + description: z.string(), + models: z.array(ModelVariantConfigSchema), + runners: CopilotRunnerConfigSchema, + benchmark: z.string().check(z.minLength(1)), + setup: z.optional(TreatmentSetupSchema), + treatments: z.array(TreatmentConfigSchema).check( + z.refine( + treatments => { + const names = new Set([ControlTreatment.name, 'Benchmark']) + for (const treatment of treatments) { + if (names.has(treatment.name)) { + return false + } + names.add(treatment.name) + } + return true + }, + { + message: 'Treatment names must be unique and cannot use the reserved names "Control" or "Benchmark"', + }, + ), + ), +}) + +type BenchmarkExperimentConfig = z.infer + +const ScenarioExperimentConfig = z.strictObject({ name: z.string(), description: z.string(), models: z.array(ModelVariantConfigSchema), @@ -30,6 +58,10 @@ const ExperimentConfigSchema = z.strictObject({ ), }) +type ScenarioExperimentConfig = z.infer + +const ExperimentConfigSchema = z.union([ScenarioExperimentConfig, BenchmarkExperimentConfig]) + type ExperimentConfig = z.infer function defineConfig(config: z.input): ExperimentConfig { diff --git a/packages/agent-eval/src/experiment/experiment.ts b/packages/agent-eval/src/experiment/experiment.ts index 41f03c7c..0e064dd2 100644 --- a/packages/agent-eval/src/experiment/experiment.ts +++ b/packages/agent-eval/src/experiment/experiment.ts @@ -2,8 +2,23 @@ import type {ExperimentConfig} from './config' import type {ModelVariant} from '../model' import type {Scenario} from '../scenario/scenario' import type {Treatment, TreatmentSetup} from '../treatment' +import type {Benchmark} from '../benchmark/benchmark' -type Experiment = { +type BenchmarkExperiment = { + type: 'benchmark' + id: string + filepath: string + name: ExperimentConfig['name'] + description: ExperimentConfig['description'] + models: Array + runners: ExperimentConfig['runners'] + benchmark: Benchmark + setup?: TreatmentSetup + treatments: Array +} + +type ScenarioExperiment = { + type: 'scenario' id: string filepath: string name: ExperimentConfig['name'] @@ -15,4 +30,6 @@ type Experiment = { treatments: Array } -export type {Experiment} +type Experiment = ScenarioExperiment | BenchmarkExperiment + +export type {Experiment, ScenarioExperiment, BenchmarkExperiment} diff --git a/packages/agent-eval/src/experiment/get.ts b/packages/agent-eval/src/experiment/get.ts index e03c10e7..509a258d 100644 --- a/packages/agent-eval/src/experiment/get.ts +++ b/packages/agent-eval/src/experiment/get.ts @@ -3,6 +3,7 @@ import type {Experiment} from './experiment' import {listExperiments} from './list' type GetExperimentOptions = { + benchmarksDirectory: string experimentsDirectory: string host?: Host scenariosDirectory: string @@ -10,14 +11,16 @@ type GetExperimentOptions = { } async function getExperiment({ + benchmarksDirectory, experimentsDirectory, host = DefaultHost, name, scenariosDirectory, }: GetExperimentOptions): Promise { const experiments = await listExperiments({ - host, + benchmarksDirectory, experimentsDirectory, + host, scenariosDirectory, }) const experiment = experiments.find(candidate => { diff --git a/packages/agent-eval/src/experiment/list.ts b/packages/agent-eval/src/experiment/list.ts index 7d728ea5..735cb055 100644 --- a/packages/agent-eval/src/experiment/list.ts +++ b/packages/agent-eval/src/experiment/list.ts @@ -8,6 +8,8 @@ import {loadScenario} from '../scenario/load' import {createTreatment} from '../treatment' import {ExperimentConfigSchema} from './config' import type {Experiment} from './experiment' +import {exhaustiveCheck} from '../exhaustive' +import {getBenchmark} from '../benchmark/get' const EXPERIMENT_FILE_EXTENSIONS = new Set(['.cjs', '.js', '.mjs', '.ts']) @@ -17,12 +19,14 @@ type ExperimentModule = { } type ListExperimentsOptions = { + benchmarksDirectory: string experimentsDirectory: string host?: Host scenariosDirectory: string } async function listExperiments({ + benchmarksDirectory, experimentsDirectory, host = DefaultHost, scenariosDirectory, @@ -62,30 +66,58 @@ async function listExperiments({ } const {data: config} = parseResult - const scenarios = await Promise.all( - config.scenarios.map(scenario => { - if (typeof scenario === 'string') { - return getScenario({host, directory: scenariosDirectory, name: scenario}) - } - - const directory = path.resolve(scenario.path) - return loadScenario({host, directory, name: scenario.name}) - }), - ) - - experiments.push({ - id: getExperimentId(filename), - filepath, - name: config.name, - description: config.description, - models: getModelVariants(config.models), - runners: config.runners, - scenarios, - setup: config.setup, - treatments: config.treatments.map(treatment => { - return createTreatment(treatment) - }), - }) + + if ('scenarios' in config) { + const scenarios = await Promise.all( + config.scenarios.map(scenario => { + if (typeof scenario === 'string') { + return getScenario({host, directory: scenariosDirectory, name: scenario}) + } + + const directory = path.resolve(scenario.path) + return loadScenario({host, directory, name: scenario.name}) + }), + ) + + experiments.push({ + type: 'scenario', + id: getExperimentId(filename), + filepath, + name: config.name, + description: config.description, + models: getModelVariants(config.models), + runners: config.runners, + scenarios, + setup: config.setup, + treatments: config.treatments.map(treatment => { + return createTreatment(treatment) + }), + }) + } else if ('benchmark' in config) { + const benchmark = await getBenchmark({ + benchmarksDirectory, + host, + scenariosDirectory, + name: config.benchmark, + }) + + experiments.push({ + type: 'benchmark', + id: getExperimentId(filename), + filepath, + name: config.name, + description: config.description, + models: getModelVariants(config.models), + runners: config.runners, + benchmark, + setup: config.setup, + treatments: config.treatments.map(treatment => { + return createTreatment(treatment) + }), + }) + } else { + exhaustiveCheck(config) + } } return experiments diff --git a/packages/agent-eval/src/experiment/output.test.ts b/packages/agent-eval/src/experiment/output.test.ts index e8b10c52..36263c42 100644 --- a/packages/agent-eval/src/experiment/output.test.ts +++ b/packages/agent-eval/src/experiment/output.test.ts @@ -1,10 +1,17 @@ import {expect, expectTypeOf, test} from 'vitest' import type {CopilotRunner} from '../copilot-runner' import {VirtualHost} from '../host' +import {RunTrialResultSchema} from '../trial/run' +import {getExperiment} from './get' +import {createExperimentPlan} from './plan' import { ExperimentOutputFileSchema, ExperimentTrialOutputSchema, + createExperimentOutput, + listExperimentOutputFiles, mergeExperimentOutputFiles, + parseExperimentTrialOutput, + writeExperimentOutput, type ExperimentTrialOutput, } from './output' @@ -61,6 +68,7 @@ test.each([null, 'unknown'])('rejects an invalid experiment output runner: %s', function createOutputFile(trials: Record = {}) { return { id: 'example', + capabilities: {}, scenarios: { example: { id: 'example', @@ -261,3 +269,103 @@ test('merges compatible experiment shards with repeated metadata and distinct tr expect(output.scenarios.size).toBe(1) expect(output.treatments.size).toBe(1) }) + +test('preserves benchmark capabilities and trial membership through writing and merging shards', async () => { + const host = VirtualHost.create({ + '/experiments/example.ts': + 'export default {name: "Example", description: "Compare a benchmark", models: ["gpt-5.5"], benchmark: "example", treatments: []}', + '/benchmarks/example.ts': + 'export default {name: "Example", description: "Shared scenarios", models: ["gpt-5.5"], capabilities: [{name: "Components", scenarios: ["example"]}, {name: "Layout", scenarios: ["example"]}]}', + '/scenarios/example/package.json': '{}', + '/scenarios/example/scenario.config.ts': 'export default {prompt: "Build a page"}', + }) + const experiment = await getExperiment({ + host, + name: 'example', + experimentsDirectory: '/experiments', + benchmarksDirectory: '/benchmarks', + scenariosDirectory: '/scenarios', + }) + const plan = createExperimentPlan({experiment}) + const results = plan.trials.map(entry => { + return { + trial: entry, + result: RunTrialResultSchema.parse({ + ...trial, + trial: entry, + artifacts: { + ...trial.artifacts, + directory: `/results/artifacts/${entry.id}`, + }, + }), + } + }) + const expected = createExperimentOutput({experiment, runPlanResult: {results}}) + + for (const [index, shard] of [results.slice(0, 2), results.slice(2)].entries()) { + await writeExperimentOutput({ + host, + output: createExperimentOutput({experiment, runPlanResult: {results: shard}}), + outputPath: `/results/output-${index + 1}.json`, + }) + } + const files = await listExperimentOutputFiles({host, outputDirectory: '/results'}) + const merged = await mergeExperimentOutputFiles({ + host, + outputDirectory: '/results', + outputs: files.map(([output]) => { + return output + }), + }) + + expect(merged).toEqual(expected) + expect( + [...merged.capabilities.values()] + .map(capability => { + return capability.name + }) + .sort(), + ).toEqual(['Components', 'Layout']) + expect( + [...merged.trials.values()].every(entry => { + return entry.capabilityId !== undefined && merged.capabilities.has(entry.capabilityId) + }), + ).toBe(true) +}) + +test('defaults legacy scenario manifests to no capabilities', () => { + const legacy: Record = createOutputFile() + delete legacy.capabilities + + expect(ExperimentOutputFileSchema.parse(legacy).capabilities).toEqual({}) +}) + +test.each([ + {capabilityId: undefined, scenarioId: 'example'}, + {capabilityId: 'unknown', scenarioId: 'example'}, + {capabilityId: 'components', scenarioId: 'wrong-scenario'}, +])('rejects invalid benchmark experiment capability membership: %j', fields => { + expect(() => { + parseExperimentTrialOutput( + {...trial, ...fields}, + new Map([['components', {id: 'components', name: 'Components', scenarioIds: ['example']}]]), + ) + }).toThrow('Invalid capability') +}) + +test('rejects conflicting capability metadata before reading experiment shard artifacts', async () => { + const capability = {id: 'components', name: 'Components', scenarioIds: ['example']} + + await expect( + mergeExperimentOutputFiles({ + host: VirtualHost.create(), + outputDirectory: '/results', + outputs: [ + {...createOutputFile({trial: 'missing.json'}), capabilities: {components: capability}}, + {...createOutputFile(), capabilities: {components: {...capability, name: 'Renamed'}}}, + ], + }), + ).rejects.toMatchObject({ + issues: [{path: [1, 'capabilities', 'components']}], + }) +}) diff --git a/packages/agent-eval/src/experiment/output.ts b/packages/agent-eval/src/experiment/output.ts index 8b70ef1e..af4979e7 100644 --- a/packages/agent-eval/src/experiment/output.ts +++ b/packages/agent-eval/src/experiment/output.ts @@ -2,6 +2,7 @@ import path from 'node:path' import {isDeepStrictEqual} from 'node:util' import * as z from 'zod/mini' import {CopilotRunnerSchema} from '../copilot-runner' +import {CapabilityOutputSchema, type CapabilityOutput} from '../benchmark/output' import {DefaultHost, type Host} from '../host' import {ModelVariantSchema} from '../model' import type {RunPlanResult} from '../plan' @@ -21,6 +22,7 @@ import type {ExperimentTrial} from './plan' const ExperimentTrialOutputSchema = z.object({ agent: TrialAgentSchema, artifacts: TrialArtifactsSchema, + capabilityId: z.optional(z.string()), checks: z._default(TrialChecksSchema, []), id: z.string(), judges: TrialJudgesSchema, @@ -53,6 +55,21 @@ type TreatmentOutput = z.infer const ExperimentOutputFileSchema = z.object({ id: z.string(), + capabilities: z._default( + z.record(z.string(), CapabilityOutputSchema).check(ctx => { + for (const [key, capability] of Object.entries(ctx.value)) { + if (capability.id !== key) { + ctx.issues.push({ + code: 'custom', + message: `Manifest capability ID "${capability.id}" does not match key "${key}"`, + path: [key, 'id'], + input: capability.id, + }) + } + } + }), + {}, + ), scenarios: z.record(z.string(), ScenarioOutputSchema).check(ctx => { for (const [key, scenario] of Object.entries(ctx.value)) { if (scenario.id !== key) { @@ -87,6 +104,7 @@ const ExperimentOutputFilesSchema = z .check(z.minLength(1, 'Cannot merge experiment output files: no outputs provided'), ctx => { const id = ctx.value[0]?.id const trialIds = new Set() + const capabilities = new Map() const scenarios = new Map() const treatments = new Map() @@ -100,6 +118,18 @@ const ExperimentOutputFilesSchema = z }) } + for (const [key, capability] of Object.entries(output.capabilities)) { + if (capabilities.has(key) && !isDeepStrictEqual(capabilities.get(key), capability)) { + ctx.issues.push({ + code: 'custom', + message: `Cannot merge conflicting capability metadata for id: ${key}`, + path: [index, 'capabilities', key], + input: capability, + }) + } + capabilities.set(key, capability) + } + for (const [key, scenario] of Object.entries(output.scenarios)) { if (scenarios.has(key) && !isDeepStrictEqual(scenarios.get(key), scenario)) { ctx.issues.push({ @@ -140,11 +170,29 @@ const ExperimentOutputFilesSchema = z type ExperimentOutput = { id: string + capabilities: Map scenarios: Map treatments: Map trials: Map } +function parseExperimentTrialOutput( + json: unknown, + capabilities: ExperimentOutput['capabilities'], +): ExperimentTrialOutput { + const trial = ExperimentTrialOutputSchema.parse(json) + if (trial.capabilityId === undefined && capabilities.size === 0) { + return trial + } + const capability = trial.capabilityId === undefined ? undefined : capabilities.get(trial.capabilityId) + if (!capability || capability.id !== trial.capabilityId || !capability.scenarioIds.includes(trial.scenarioId)) { + throw new Error( + `Invalid capability "${trial.capabilityId}" for scenario "${trial.scenarioId}" in trial "${trial.id}"`, + ) + } + return trial +} + type CreateExperimentOutputOptions = { experiment: Experiment runPlanResult: RunPlanResult @@ -153,12 +201,26 @@ type CreateExperimentOutputOptions = { function createExperimentOutput({experiment, runPlanResult}: CreateExperimentOutputOptions): ExperimentOutput { const result: ExperimentOutput = { id: experiment.id, + capabilities: new Map(), scenarios: new Map(), treatments: new Map(), trials: new Map(), } for (const {trial, result: trialResult} of runPlanResult.results) { + if (experiment.type === 'benchmark' && !('capability' in trial)) { + throw new Error(`Capability not found for benchmark experiment trial: ${trial.id}`) + } + const capability = 'capability' in trial ? trial.capability : undefined + if (capability && !result.capabilities.has(capability.id)) { + result.capabilities.set(capability.id, { + id: capability.id, + name: capability.name, + scenarioIds: capability.scenarios.map(scenario => { + return scenario.id + }), + }) + } if (!result.scenarios.has(trial.scenario.id)) { result.scenarios.set(trial.scenario.id, ScenarioOutputSchema.parse(trial.scenario)) } @@ -173,6 +235,7 @@ function createExperimentOutput({experiment, runPlanResult}: CreateExperimentOut result.trials.set(trial.id, { agent: trialResult.agent, artifacts: trialResult.artifacts, + ...(capability ? {capabilityId: capability.id} : {}), checks: trialResult.checks, id: trial.id, judges: trialResult.judges, @@ -199,12 +262,16 @@ async function mergeExperimentOutputFiles({ outputDirectory, }: MergeExperimentOutputFilesOptions): Promise { const files = ExperimentOutputFilesSchema.parse(outputs) + const capabilities = new Map() const scenarios = new Map() const treatments = new Map() const trials = new Map() const id = files[0].id for (const output of files) { + for (const [key, value] of Object.entries(output.capabilities)) { + capabilities.set(key, value) + } for (const [key, value] of Object.entries(output.scenarios)) { scenarios.set(key, value) } @@ -216,7 +283,7 @@ async function mergeExperimentOutputFiles({ const filepath = await resolveTrialArtifactsPath(host, outputDirectory, value) const contents = await host.fs.readFile(filepath, 'utf-8') - const trialOutput = ExperimentTrialOutputSchema.parse(JSON.parse(contents)) + const trialOutput = parseExperimentTrialOutput(JSON.parse(contents), new Map(Object.entries(output.capabilities))) if (trialOutput.id !== key) { throw new Error(`Cannot merge experiment output files: mismatched trial ID for: ${key}`) } @@ -225,7 +292,7 @@ async function mergeExperimentOutputFiles({ } } - return {id, scenarios, treatments, trials} + return {id, capabilities, scenarios, treatments, trials} } type WriteExperimentOutputOptions = { @@ -249,6 +316,7 @@ async function writeExperimentOutput({host = DefaultHost, output, outputPath}: W const experimentFile: ExperimentOutputFile = { id: output.id, + capabilities: Object.fromEntries(output.capabilities), scenarios: Object.fromEntries(output.scenarios), treatments: Object.fromEntries(output.treatments), trials: Object.fromEntries(trials), @@ -290,6 +358,7 @@ async function listExperimentOutputFiles({ export { ExperimentOutputFileSchema, ExperimentTrialOutputSchema, + parseExperimentTrialOutput, createExperimentOutput, listExperimentOutputFiles, mergeExperimentOutputFiles, diff --git a/packages/agent-eval/src/experiment/plan.test.ts b/packages/agent-eval/src/experiment/plan.test.ts index 501f63d5..bd2484b0 100644 --- a/packages/agent-eval/src/experiment/plan.test.ts +++ b/packages/agent-eval/src/experiment/plan.test.ts @@ -1,6 +1,7 @@ import {expect, expectTypeOf, test} from 'vitest' import {VirtualHost} from '../host' import type {CopilotRunner} from '../copilot-runner' +import {VirtualSandbox} from '../sandbox' import {ExperimentConfigSchema} from './config' import {getExperiment} from './get' import {createExperimentPlan, createExperimentPlanManifest, parseExperimentPlanManifest} from './plan' @@ -28,7 +29,12 @@ test.each([ 'expands and restores the runner dimension: $runners', async ({runners, expected}) => { const host = createHost(runners) - const options = {host, experimentsDirectory: '/experiments', scenariosDirectory: '/scenarios'} + const options = { + host, + benchmarksDirectory: '/benchmarks', + experimentsDirectory: '/experiments', + scenariosDirectory: '/scenarios', + } const experiment = await getExperiment({...options, name: 'example'}) const plan = createExperimentPlan({experiment}) expectTypeOf(plan.trials[0].runner).toEqualTypeOf() @@ -55,7 +61,12 @@ test.each([ ) test('restores legacy plans without a runner as CLI trials', async () => { - const options = {host: createHost(), experimentsDirectory: '/experiments', scenariosDirectory: '/scenarios'} + const options = { + host: createHost(), + benchmarksDirectory: '/benchmarks', + experimentsDirectory: '/experiments', + scenariosDirectory: '/scenarios', + } const experiment = await getExperiment({...options, name: 'example'}) const plan = createExperimentPlan({experiment}) const manifest = createExperimentPlanManifest({experiment, plan}) @@ -70,41 +81,47 @@ test('restores legacy plans without a runner as CLI trials', async () => { expect(parsed.trials).toEqual(plan.trials) }) -test('rejects duplicate experiment trial IDs before loading configuration', async () => { - const trial = { - id: 'duplicate', - model: { - name: 'gpt-5.5', - reasoningEffort: 'medium', - }, - scenarioId: 'example', - treatmentId: 'control', - } +test.each([undefined, 'components'])( + 'rejects duplicate experiment trial IDs before loading configuration: %s', + async capabilityId => { + const trial = { + id: 'duplicate', + model: { + name: 'gpt-5.5', + reasoningEffort: 'medium', + }, + scenarioId: 'example', + treatmentId: 'control', + capabilityId, + } - await expect( - parseExperimentPlanManifest({ - host: VirtualHost.create(), - experimentsDirectory: '/experiments', - scenariosDirectory: '/scenarios', - contents: JSON.stringify({ - id: 'missing', - name: 'Example', - trials: [trial, trial], + await expect( + parseExperimentPlanManifest({ + host: VirtualHost.create(), + benchmarksDirectory: '/benchmarks', + experimentsDirectory: '/experiments', + scenariosDirectory: '/scenarios', + contents: JSON.stringify({ + id: 'missing', + name: 'Example', + trials: [trial, trial], + }), }), - }), - ).rejects.toMatchObject({ - issues: [ - { - path: ['trials', 1, 'id'], - message: 'Duplicate trial ID in experiment plan: duplicate', - }, - ], - }) -}) + ).rejects.toMatchObject({ + issues: [ + { + path: ['trials', 1, 'id'], + message: 'Duplicate trial ID in experiment plan: duplicate', + }, + ], + }) + }, +) test('preserves repeated experiment trial combinations with distinct IDs', async () => { const options = { host: createHost(), + benchmarksDirectory: '/benchmarks', experimentsDirectory: '/experiments', scenariosDirectory: '/scenarios', } @@ -127,6 +144,137 @@ test('preserves repeated experiment trial combinations with distinct IDs', async ]) }) +test('expands and restores benchmark experiment control, benchmark, and custom treatments', async () => { + const host = VirtualHost.create({ + '/experiments/example.ts': `export default ${JSON.stringify({ + name: 'Example', + description: 'Compare benchmark treatments', + models: ['gpt-5.5'], + runners: ['copilot-cli', 'copilot-sdk'], + benchmark: 'design-system', + treatments: [{name: 'Skill'}], + })}`, + '/benchmarks/design-system.ts': `export default ${JSON.stringify({ + name: 'Design System', + description: 'Example benchmark', + models: ['gpt-5.5'], + capabilities: [{name: 'Components', scenarios: ['example']}], + })}`, + '/scenarios/example/package.json': '{}', + '/scenarios/example/scenario.config.ts': 'export default {prompt: "Create a page"}', + }) + const options = { + host, + benchmarksDirectory: '/benchmarks', + experimentsDirectory: '/experiments', + scenariosDirectory: '/scenarios', + } + const experiment = await getExperiment({...options, name: 'example'}) + const plan = createExperimentPlan({experiment}) + + expect(plan.trials).toHaveLength(6) + for (const runner of ['copilot-cli', 'copilot-sdk']) { + expect( + plan.trials + .filter(trial => { + return trial.runner === runner + }) + .map(trial => { + return trial.treatment.name + }) + .sort(), + ).toEqual(['Benchmark', 'Control', 'Skill']) + } + + const manifest = createExperimentPlanManifest({experiment, plan}) + const parsed = await parseExperimentPlanManifest({...options, contents: JSON.stringify(manifest)}) + + expect(parsed.trials).toEqual( + plan.trials.map(trial => { + return { + ...trial, + setup: expect.any(Function), + } + }), + ) +}) + +test('preserves shared setup order and treatment isolation in created and restored benchmark experiment plans', async () => { + const host = VirtualHost.create({ + '/experiments/example.ts': `export default { + name: 'Example', + description: 'Compare benchmark treatments', + models: ['gpt-5.5'], + benchmark: 'design-system', + async setup({sandbox}) { + await sandbox.writeFile('/setup.txt', 'experiment') + }, + treatments: [{ + name: 'Skill', + async setup({sandbox}) { + await sandbox.writeFile('/setup.txt', await sandbox.readFile('/setup.txt') + ',skill') + }, + }], + }`, + '/benchmarks/design-system.ts': `export default { + name: 'Design System', + description: 'Example benchmark', + models: ['gpt-5.5'], + async setup({sandbox}) { + await sandbox.writeFile('/setup.txt', await sandbox.readFile('/setup.txt') + ',benchmark') + }, + capabilities: ['Components', 'Layouts'].map(name => { + return { + name, + scenarios: ['example'], + async setup({sandbox}) { + await sandbox.writeFile('/setup.txt', await sandbox.readFile('/setup.txt') + ',' + name) + }, + } + }), + }`, + '/scenarios/example/package.json': '{}', + '/scenarios/example/scenario.config.ts': 'export default {prompt: "Create a page"}', + }) + const options = { + host, + benchmarksDirectory: '/benchmarks', + experimentsDirectory: '/experiments', + scenariosDirectory: '/scenarios', + } + const experiment = await getExperiment({...options, name: 'example'}) + const plan = createExperimentPlan({experiment}) + const manifest = createExperimentPlanManifest({experiment, plan}) + const parsed = await parseExperimentPlanManifest({...options, contents: JSON.stringify(manifest)}) + const plans: Array = [plan.trials, parsed.trials] + + for (const trials of plans) { + expect(trials).toHaveLength(6) + for (const trial of trials) { + if (!('capability' in trial)) { + throw new Error('Expected a benchmark trial') + } + await using sandbox = await VirtualSandbox.create() + const sharedSetup = `experiment,${trial.capability.name}` + + await trial.setup?.({sandbox}) + + expect(await sandbox.readFile('/setup.txt')).toBe(sharedSetup) + + await trial.treatment.setup?.({sandbox}) + + if (trial.treatment.name === 'Control') { + expect(await sandbox.readFile('/setup.txt')).toBe(sharedSetup) + } else if (trial.treatment.name === 'Benchmark') { + expect(await sandbox.readFile('/setup.txt')).toBe(`${sharedSetup},benchmark`) + } else { + expect(trial.treatment.name).toBe('Skill') + expect(await sandbox.readFile('/setup.txt')).toBe(`${sharedSetup},skill`) + } + } + } +}) + test.each([{runners: []}, {runners: ['sdk']}, {runners: ['unknown']}])( 'rejects invalid runner configuration: %j', ({runners}) => { diff --git a/packages/agent-eval/src/experiment/plan.ts b/packages/agent-eval/src/experiment/plan.ts index 7edb4375..7bb33eea 100644 --- a/packages/agent-eval/src/experiment/plan.ts +++ b/packages/agent-eval/src/experiment/plan.ts @@ -4,56 +4,82 @@ import {CopilotRunnerSchema} from '../copilot-runner' import type {Host} from '../host' import {ModelVariantSchema} from '../model' import {createPlan, type Plan} from '../plan' -import {ControlTreatment} from '../treatment' +import {composeTreatmentSetup, ControlTreatment, createTreatment} from '../treatment' import type {Trial} from '../trial/trial' import type {Experiment} from './experiment' import {getExperiment} from './get' +import {exhaustiveCheck} from '../exhaustive' +import type {Capability} from '../benchmark/benchmark' -type ExperimentTrial = Trial - -type CreateExperimentPlanOptions = { - experiment: Experiment +type BenchmarkTrial = Trial & { + capability: Capability } -function getExperimentTreatments(experiment: Experiment) { - const treatments = new Map([[ControlTreatment.id, ControlTreatment]]) - const names = new Set([ControlTreatment.name]) +type ScenarioTrial = Trial - for (const treatment of experiment.treatments) { - if (treatments.has(treatment.id) || names.has(treatment.name)) { - throw new Error(`Experiment "${experiment.id}" contains duplicate treatment: ${treatment.name}`) - } - treatments.set(treatment.id, treatment) - names.add(treatment.name) - } +type ExperimentTrial = Trial | BenchmarkTrial - return treatments +type CreateExperimentPlanOptions = { + experiment: Experiment } -function createExperimentPlan({experiment}: CreateExperimentPlanOptions): Plan { - const treatments = [...getExperimentTreatments(experiment).values()] - - return createPlan({ - trials: experiment.models.flatMap(model => { - return experiment.scenarios.flatMap(scenario => { +function createExperimentPlan({experiment}: CreateExperimentPlanOptions): Plan | Plan { + if (experiment.type === 'scenario') { + const treatments = [ControlTreatment, ...experiment.treatments] + return createPlan({ + trials: experiment.models.flatMap(model => { + return experiment.scenarios.flatMap(scenario => { + return experiment.runners.flatMap(runner => { + return treatments.map(treatment => { + return { + id: randomUUID(), + scenario, + treatment, + model, + runner, + setup: experiment.setup, + } + }) + }) + }) + }), + }) + } else if (experiment.type === 'benchmark') { + return createPlan({ + trials: experiment.models.flatMap(model => { return experiment.runners.flatMap(runner => { - return treatments.map(treatment => { - return { - id: randomUUID(), - scenario, - treatment, - model, - runner, - setup: experiment.setup, - } + return experiment.benchmark.capabilities.flatMap(capability => { + const treatments = [ + ControlTreatment, + createTreatment({ + name: 'Benchmark', + setup: experiment.benchmark.setup, + }), + ...experiment.treatments, + ] + return capability.scenarios.flatMap(scenario => { + return treatments.map(treatment => { + return { + id: randomUUID(), + scenario, + treatment, + model, + runner, + capability, + setup: composeTreatmentSetup(experiment.setup, capability.setup), + } + }) + }) }) }) - }) - }), - }) + }), + }) + } else { + exhaustiveCheck(experiment) + } } -const ExperimentPlanManifestFileSchema = z.object({ +const ExperimentPlanManifestFileSchema = z.strictObject({ id: z.string(), name: z.string(), trials: z @@ -64,6 +90,7 @@ const ExperimentPlanManifestFileSchema = z.object({ runner: z._default(CopilotRunnerSchema, 'copilot-cli'), scenarioId: z.string(), treatmentId: z.string(), + capabilityId: z.optional(z.string()), }), ) .check(ctx => { @@ -97,12 +124,25 @@ function createExperimentPlanManifest({ id: experiment.id, name: experiment.name, trials: plan.trials.map(trial => { - return { - id: trial.id, - model: trial.model, - runner: trial.runner, - scenarioId: trial.scenario.id, - treatmentId: trial.treatment.id, + if ('capability' in trial) { + return { + id: trial.id, + model: trial.model, + runner: trial.runner, + scenarioId: trial.scenario.id, + capabilityId: trial.capability.id, + treatmentId: trial.treatment.id, + } + } else if ('scenario' in trial) { + return { + id: trial.id, + model: trial.model, + runner: trial.runner, + scenarioId: trial.scenario.id, + treatmentId: trial.treatment.id, + } + } else { + exhaustiveCheck(trial) } }), } @@ -114,6 +154,7 @@ type ExperimentPlanManifest = { } type ParseExperimentPlanManifestOptions = { + benchmarksDirectory: string experimentsDirectory: string contents: string host?: Host @@ -121,6 +162,7 @@ type ParseExperimentPlanManifestOptions = { } async function parseExperimentPlanManifest({ + benchmarksDirectory, experimentsDirectory, contents, host, @@ -128,47 +170,113 @@ async function parseExperimentPlanManifest({ }: ParseExperimentPlanManifestOptions): Promise { const result = ExperimentPlanManifestFileSchema.parse(JSON.parse(contents)) const experiment = await getExperiment({ + benchmarksDirectory, experimentsDirectory, host, name: result.id, scenariosDirectory, }) - const scenarios = new Map( - experiment.scenarios.map(scenario => { - return [scenario.id, scenario] - }), - ) - const treatments = getExperimentTreatments(experiment) - return { - experiment, - trials: result.trials.map(trial => { - const scenario = scenarios.get(trial.scenarioId) - if (!scenario) { - throw new Error(`Scenario not found for trial: ${trial.id}`) - } + if (experiment.type === 'scenario') { + const scenarios = new Map( + experiment.scenarios.map(scenario => { + return [scenario.id, scenario] + }), + ) + const treatments = [ControlTreatment, ...experiment.treatments] + return { + experiment, + trials: result.trials.map(trial => { + const scenario = scenarios.get(trial.scenarioId) + if (!scenario) { + throw new Error(`Scenario not found for trial: ${trial.id}`) + } - const treatment = treatments.get(trial.treatmentId) - if (!treatment) { - throw new Error(`Treatment not found for trial: ${trial.id}`) - } + const treatment = treatments.find(candidate => { + return candidate.id === trial.treatmentId + }) + if (!treatment) { + throw new Error(`Treatment not found for trial: ${trial.id}`) + } - const model = experiment.models.find(candidate => { - return candidate.name === trial.model.name && candidate.reasoningEffort === trial.model.reasoningEffort - }) - if (!model) { - throw new Error(`Model variant not found for trial: ${trial.id}`) - } + const model = experiment.models.find(candidate => { + return candidate.name === trial.model.name && candidate.reasoningEffort === trial.model.reasoningEffort + }) + if (!model) { + throw new Error(`Model variant not found for trial: ${trial.id}`) + } - return { - id: trial.id, - model, - runner: trial.runner, - scenario, - treatment, - setup: experiment.setup, - } - }), + return { + id: trial.id, + model, + runner: trial.runner, + scenario, + treatment, + setup: experiment.setup, + } + }), + } + } else if (experiment.type === 'benchmark') { + const capabilities = new Map( + experiment.benchmark.capabilities.map(capability => { + return [capability.id, capability] + }), + ) + + return { + experiment, + trials: result.trials.map(trial => { + if (trial.capabilityId === undefined) { + throw new Error(`Capability not found for trial: ${trial.id}`) + } + + const capability = capabilities.get(trial.capabilityId) + if (!capability) { + throw new Error(`Capability not found for trial: ${trial.id}`) + } + + const scenario = capability.scenarios.find(candidate => { + return candidate.id === trial.scenarioId + }) + if (!scenario) { + throw new Error(`Scenario not found for trial: ${trial.id}`) + } + + const model = experiment.models.find(candidate => { + return candidate.name === trial.model.name && candidate.reasoningEffort === trial.model.reasoningEffort + }) + if (!model) { + throw new Error(`Model variant not found for trial: ${trial.id}`) + } + + const treatments = [ + ControlTreatment, + createTreatment({ + name: 'Benchmark', + setup: experiment.benchmark.setup, + }), + ...experiment.treatments, + ] + const treatment = treatments.find(candidate => { + return candidate.id === trial.treatmentId + }) + if (!treatment) { + throw new Error(`Treatment not found for trial: ${trial.id}`) + } + + return { + id: trial.id, + model, + runner: trial.runner, + scenario, + capability, + treatment, + setup: composeTreatmentSetup(experiment.setup, capability.setup), + } + }), + } + } else { + exhaustiveCheck(experiment) } } diff --git a/packages/agent-eval/src/experiment/report.test.ts b/packages/agent-eval/src/experiment/report.test.ts new file mode 100644 index 00000000..9cb606c1 --- /dev/null +++ b/packages/agent-eval/src/experiment/report.test.ts @@ -0,0 +1,283 @@ +import {describe, expect, test} from 'vitest' +import type {CopilotRunner} from '../copilot-runner' +import {VirtualHost} from '../host' +import type {RunPlanResult} from '../plan' +import type {Experiment} from './experiment' +import {getExperiment} from './get' +import {createExperimentPlan, type ExperimentTrial} from './plan' +import {createExperimentReport} from './report' + +async function createExperiment(type: Experiment['type'], runners: Array = ['copilot-cli']) { + const host = VirtualHost.create({ + '/experiments/example.ts': `export default ${JSON.stringify({ + name: 'Example', + description: 'Compare treatments', + models: [{name: 'gpt-5.5', reasoningEfforts: ['medium', 'high']}], + runners, + ...(type === 'benchmark' ? {benchmark: 'design-system'} : {scenarios: ['example']}), + treatments: [{name: 'Skill'}], + })}`, + '/benchmarks/design-system.ts': `export default ${JSON.stringify({ + name: 'Design System', + description: 'Evaluate capabilities', + models: ['gpt-5.5'], + capabilities: [ + {name: 'Components', scenarios: ['example']}, + {name: 'Layout', scenarios: ['example']}, + ], + })}`, + '/scenarios/example/package.json': '{}', + '/scenarios/example/scenario.config.ts': 'export default {prompt: "Create a page"}', + }) + + return getExperiment({ + host, + benchmarksDirectory: '/benchmarks', + experimentsDirectory: '/experiments', + scenariosDirectory: '/scenarios', + name: 'example', + }) +} + +function createResult( + trial: ExperimentTrial, + outputTokens: number, + passed = true, +): RunPlanResult['results'][number] { + return { + trial, + result: { + trial, + agent: { + sessions: [ + { + turns: 1, + outputTokens, + premiumRequests: 1, + sessionDurationMs: 1000, + totalApiDurationMs: 500, + tools: {}, + messages: [], + }, + ], + }, + artifacts: { + directory: '/artifacts', + copilotConfigDirectory: '/artifacts/copilot', + skillsConfigDirectory: '/artifacts/skills', + walkthroughDirectory: '/artifacts/walkthrough', + workspaceDirectory: '/artifacts/workspace', + }, + checks: [ + { + check: { + name: 'Quality', + files: [], + }, + result: { + type: 'outcomes', + outcomes: [ + { + type: 'outcome', + status: passed ? 'passed' : 'failed', + }, + ], + }, + }, + ], + judges: [], + walkthrough: { + type: 'Unavailable', + }, + }, + } +} + +function readReportRows(report: string): Array> { + const [header, separator, ...lines] = report.split('\n\n')[0].split('\n') + const columns = [...separator.matchAll(/-+/g)].map(match => { + return { + name: header.slice(match.index, match.index + match[0].length).trim(), + start: match.index, + end: match.index + match[0].length, + } + }) + + return lines.map(line => { + return Object.fromEntries( + columns.map(column => { + return [column.name, line.slice(column.start, column.end).trim()] + }), + ) + }) +} + +describe('createExperimentReport', () => { + test('preserves scenario experiment rows without capability or runner columns', async () => { + const experiment = await createExperiment('scenario') + const plan = createExperimentPlan({experiment}) + const runPlanResult = { + results: plan.trials.map(trial => { + return createResult(trial, trial.model.reasoningEffort === 'medium' ? 10 : 20) + }), + } + + const report = createExperimentReport({experiment, runPlanResult}) + const rows = readReportRows(report) + + expect(rows).toHaveLength(8) + expect(Object.keys(rows[0])).not.toContain('Capability') + expect(Object.keys(rows[0])).not.toContain('Runner') + expect( + rows + .filter(row => { + return row.Experiment === 'Example' + }) + .map(row => { + return { + treatment: row.Treatment, + runs: row.Runs, + checks: row.Checks, + tokens: row['Output Tokens'], + } + }), + ).toEqual([ + {treatment: 'Control', runs: '2', checks: '100.0%', tokens: '30'}, + {treatment: 'Skill', runs: '2', checks: '100.0%', tokens: '30'}, + ]) + expect(report).toContain(' example') + expect(report).toContain(' gpt-5.5') + }) + + test('groups benchmark experiments by treatment, runner, capability, scenario, and model', async () => { + const experiment = await createExperiment('benchmark', ['copilot-cli', 'copilot-sdk']) + const plan = createExperimentPlan({experiment}) + const weights = new Map([ + ['Control', 1], + ['Benchmark', 2], + ['Skill', 3], + ]) + const runPlanResult = { + results: plan.trials.map(trial => { + if (!('capability' in trial)) { + throw new Error('Expected a benchmark trial') + } + const capabilityWeight = trial.capability.name === 'Components' ? 10 : 100 + const runnerWeight = trial.runner === 'copilot-cli' ? 1 : 2 + const modelWeight = trial.model.reasoningEffort === 'medium' ? 1 : 2 + const treatmentWeight = weights.get(trial.treatment.name) + if (treatmentWeight === undefined) { + throw new Error(`Unexpected treatment: ${trial.treatment.name}`) + } + return createResult( + trial, + capabilityWeight * runnerWeight * modelWeight * treatmentWeight, + trial.capability.name === 'Components', + ) + }), + } + + const report = createExperimentReport({experiment, runPlanResult}) + const rows = readReportRows(report) + const treatmentRows = rows.filter(row => { + return row.Experiment === 'Example' + }) + + expect(rows).toHaveLength(54) + expect(treatmentRows).toHaveLength(6) + for (const treatment of treatmentRows) { + const treatmentWeight = weights.get(treatment.Treatment) + if (treatmentWeight === undefined) { + throw new Error(`Unexpected treatment: ${treatment.Treatment}`) + } + const weight = treatmentWeight * (treatment.Runner === 'copilot-cli' ? 1 : 2) + const start = rows.indexOf(treatment) + const group = rows.slice(start, start + 9).map(row => { + return [ + row.Capability, + row.Scenario, + row.Model, + row['Reasoning Effort'], + row.Runs, + row.Checks, + row['Output Tokens'], + ] + }) + + expect(group).toEqual([ + ['All capabilities', 'All scenarios', 'All models', '', '4', '50.0%', (330 * weight).toLocaleString('en-US')], + ['Components', 'All scenarios', 'All models', '', '2', '100.0%', (30 * weight).toLocaleString('en-US')], + ['', 'example', 'All models', '', '2', '100.0%', (30 * weight).toLocaleString('en-US')], + ['', '', 'gpt-5.5', 'medium', '1', '100.0%', (10 * weight).toLocaleString('en-US')], + ['', '', 'gpt-5.5', 'high', '1', '100.0%', (20 * weight).toLocaleString('en-US')], + ['Layout', 'All scenarios', 'All models', '', '2', '0.0%', (300 * weight).toLocaleString('en-US')], + ['', 'example', 'All models', '', '2', '0.0%', (300 * weight).toLocaleString('en-US')], + ['', '', 'gpt-5.5', 'medium', '1', '0.0%', (100 * weight).toLocaleString('en-US')], + ['', '', 'gpt-5.5', 'high', '1', '0.0%', (200 * weight).toLocaleString('en-US')], + ]) + } + }) + + test('reports only completed benchmark groups for an SDK-only run without checks', async () => { + const experiment = await createExperiment('benchmark', ['copilot-cli', 'copilot-sdk']) + const plan = createExperimentPlan({experiment}) + const selected = plan.trials.filter((trial: ExperimentTrial) => { + return ( + 'capability' in trial && + trial.capability.name === 'Components' && + trial.treatment.name === 'Benchmark' && + trial.runner === 'copilot-sdk' && + trial.model.reasoningEffort === 'high' + ) + }) + const runPlanResult = { + results: selected.map(trial => { + const entry = createResult(trial, 10) + entry.result.checks = [] + return entry + }), + } + + const report = createExperimentReport({experiment, runPlanResult}) + const rows = readReportRows(report) + + expect(rows).toHaveLength(4) + expect(rows[0]).toMatchObject({ + Treatment: 'Benchmark', + Runner: 'copilot-sdk', + Capability: 'All capabilities', + Runs: '1', + 'Output Tokens': '10', + 'AI Credits': 'N/A', + }) + expect(rows[1].Capability).toBe('Components') + expect(rows[2].Scenario).toBe('example') + expect(rows[3]['Reasoning Effort']).toBe('high') + expect(Object.keys(rows[0])).not.toContain('Checks') + expect(report).not.toContain('Layout') + expect(report).not.toContain('Control') + }) + + test('rejects a benchmark experiment result without capability metadata', async () => { + const experiment = await createExperiment('benchmark') + const scenarioExperiment = await createExperiment('scenario') + const plan = createExperimentPlan({experiment: scenarioExperiment}) + + expect(() => { + createExperimentReport({ + experiment, + runPlanResult: { + results: [createResult(plan.trials[0], 10)], + }, + }) + }).toThrow(`Capability not found for benchmark experiment trial: ${plan.trials[0].id}`) + }) + + test.each(['scenario', 'benchmark'] as const)('reports no results for an empty %s experiment run', async type => { + const experiment = await createExperiment(type) + + expect(createExperimentReport({experiment, runPlanResult: {results: []}})).toBe( + 'Experiment: Example\nNo trial results.', + ) + }) +}) diff --git a/packages/agent-eval/src/experiment/report.ts b/packages/agent-eval/src/experiment/report.ts index e27b74fb..c95915d8 100644 --- a/packages/agent-eval/src/experiment/report.ts +++ b/packages/agent-eval/src/experiment/report.ts @@ -1,5 +1,6 @@ import type {ModelVariant} from '../model' import type {CopilotRunner} from '../copilot-runner' +import type {Capability} from '../benchmark/benchmark' import type {RunPlanResult} from '../plan' import {formatTable, type TableRow} from '../report/format' import {getCheckDimensions, type CheckDimension} from '../report/checks' @@ -19,6 +20,7 @@ import type {ExperimentTrial} from './plan' type ExperimentSummary = TrialSummary & { treatmentId: string treatment: string + capability?: Capability scenario?: string model?: ModelVariant runner: CopilotRunner @@ -33,6 +35,7 @@ function compareExperimentNames(a: ExperimentSummary, b: ExperimentSummary): num return ( a.treatment.localeCompare(b.treatment) || a.runner.localeCompare(b.runner) || + (a.capability?.name ?? '').localeCompare(b.capability?.name ?? '') || (a.scenario ?? '').localeCompare(b.scenario ?? '') || (a.model?.name ?? '').localeCompare(b.model?.name ?? '') || (a.model?.reasoningEffort ?? '').localeCompare(b.model?.reasoningEffort ?? '') @@ -49,16 +52,30 @@ function sortExperimentSummaries(summaries: Array): Array, showRunner: boolean, ): TableRow { + const showCapability = experiment.type === 'benchmark' + const scenarioIndent = showCapability ? ' ' : ' ' + const modelIndent = showCapability ? ' ' : ' ' return { Experiment: level === 'treatment' ? experiment.name : '', Treatment: level === 'treatment' ? summary.treatment : '', ...(showRunner ? {Runner: level === 'treatment' ? summary.runner : ''} : {}), - Scenario: level === 'treatment' ? 'All scenarios' : level === 'scenario' ? ` ${summary.scenario}` : '', - Model: level === 'model' ? ` ${summary.model?.name}` : 'All models', + ...(showCapability + ? { + Capability: + level === 'treatment' ? 'All capabilities' : level === 'capability' ? ` ${summary.capability?.name}` : '', + } + : {}), + Scenario: + level === 'treatment' || level === 'capability' + ? 'All scenarios' + : level === 'scenario' + ? `${scenarioIndent}${summary.scenario}` + : '', + Model: level === 'model' ? `${modelIndent}${summary.model?.name}` : 'All models', 'Reasoning Effort': level === 'model' ? (summary.model?.reasoningEffort ?? '') : '', ...formatTrialSummary(summary, dimensions), } @@ -70,13 +87,19 @@ function createExperimentReport({experiment, runPlanResult}: CreateExperimentRep } const treatmentSummaries = new Map() + const capabilitySummaries = new Map() const scenarioSummaries = new Map() const modelSummaries = new Map() + const showCapability = experiment.type === 'benchmark' const showRunner = runPlanResult.results.some(({trial}) => { return trial.runner === 'copilot-sdk' }) for (const {trial, result} of runPlanResult.results) { + if (showCapability && !('capability' in trial)) { + throw new Error(`Capability not found for benchmark experiment trial: ${trial.id}`) + } + const capability = showCapability && 'capability' in trial ? trial.capability : undefined const values = { treatmentId: trial.treatment.id, treatment: trial.treatment.name, @@ -87,8 +110,19 @@ function createExperimentReport({experiment, runPlanResult}: CreateExperimentRep addTrialResultToSummary(treatmentSummary, result) treatmentSummaries.set(treatmentKey, treatmentSummary) - const scenarioValues = {...values, scenario: trial.scenario.id} - const scenarioKey = JSON.stringify([trial.treatment.id, values.runner, trial.scenario.id]) + if (capability) { + const capabilityKey = JSON.stringify([trial.treatment.id, values.runner, capability.id]) + const capabilitySummary = capabilitySummaries.get(capabilityKey) ?? { + ...createTrialSummary(), + ...values, + capability, + } + addTrialResultToSummary(capabilitySummary, result) + capabilitySummaries.set(capabilityKey, capabilitySummary) + } + + const scenarioValues = {...values, capability, scenario: trial.scenario.id} + const scenarioKey = JSON.stringify([trial.treatment.id, values.runner, capability?.id, trial.scenario.id]) const scenarioSummary = scenarioSummaries.get(scenarioKey) ?? {...createTrialSummary(), ...scenarioValues} addTrialResultToSummary(scenarioSummary, result) scenarioSummaries.set(scenarioKey, scenarioSummary) @@ -96,6 +130,7 @@ function createExperimentReport({experiment, runPlanResult}: CreateExperimentRep const modelKey = JSON.stringify([ trial.treatment.id, values.runner, + capability?.id, trial.scenario.id, trial.model.name, trial.model.reasoningEffort, @@ -113,21 +148,39 @@ function createExperimentReport({experiment, runPlanResult}: CreateExperimentRep const dimensions = getCheckDimensions([...treatmentSummaries.values()]) for (const treatment of sortExperimentSummaries([...treatmentSummaries.values()])) { rows.push(formatExperimentSummary(experiment, treatment, 'treatment', dimensions, showRunner)) - const scenarios = [...scenarioSummaries.values()].filter(summary => { - return summary.treatmentId === treatment.treatmentId && summary.runner === treatment.runner - }) + const groups = showCapability + ? sortExperimentSummaries( + [...capabilitySummaries.values()].filter(summary => { + return summary.treatmentId === treatment.treatmentId && summary.runner === treatment.runner + }), + ) + : [treatment] - for (const scenario of sortExperimentSummaries(scenarios)) { - rows.push(formatExperimentSummary(experiment, scenario, 'scenario', dimensions, showRunner)) - const models = [...modelSummaries.values()].filter(summary => { + for (const group of groups) { + if (showCapability) { + rows.push(formatExperimentSummary(experiment, group, 'capability', dimensions, showRunner)) + } + const scenarios = [...scenarioSummaries.values()].filter(summary => { return ( summary.treatmentId === treatment.treatmentId && summary.runner === treatment.runner && - summary.scenario === scenario.scenario + summary.capability?.id === group.capability?.id ) }) - for (const model of sortExperimentSummaries(models)) { - rows.push(formatExperimentSummary(experiment, model, 'model', dimensions, showRunner)) + + for (const scenario of sortExperimentSummaries(scenarios)) { + rows.push(formatExperimentSummary(experiment, scenario, 'scenario', dimensions, showRunner)) + const models = [...modelSummaries.values()].filter(summary => { + return ( + summary.treatmentId === treatment.treatmentId && + summary.runner === treatment.runner && + summary.capability?.id === group.capability?.id && + summary.scenario === scenario.scenario + ) + }) + for (const model of sortExperimentSummaries(models)) { + rows.push(formatExperimentSummary(experiment, model, 'model', dimensions, showRunner)) + } } } } @@ -137,6 +190,7 @@ function createExperimentReport({experiment, runPlanResult}: CreateExperimentRep 'Experiment', 'Treatment', ...(showRunner ? ['Runner'] : []), + ...(showCapability ? ['Capability'] : []), 'Scenario', 'Model', 'Reasoning Effort', diff --git a/packages/agent-eval/src/index.ts b/packages/agent-eval/src/index.ts index bace22db..a5d4da68 100644 --- a/packages/agent-eval/src/index.ts +++ b/packages/agent-eval/src/index.ts @@ -7,7 +7,7 @@ export {getExperiment} from './experiment/get' export {listExperiments} from './experiment/list' export type {Experiment} from './experiment/experiment' export type {CopilotRunner} from './copilot-runner' -export {ExperimentOutputFileSchema, ExperimentTrialOutputSchema} from './experiment/output' +export {ExperimentOutputFileSchema, ExperimentTrialOutputSchema, parseExperimentTrialOutput} from './experiment/output' export type {ExperimentOutput, ExperimentTrialOutput} from './experiment/output' export type {Logger} from './logger' export type {Sandbox} from './sandbox' diff --git a/packages/agent-eval/src/treatment.ts b/packages/agent-eval/src/treatment.ts index 6861b0ff..5a6f910c 100644 --- a/packages/agent-eval/src/treatment.ts +++ b/packages/agent-eval/src/treatment.ts @@ -45,5 +45,23 @@ function getTreatmentId(name: string): string { return hash(`Treatment:${name}`) } -export {ControlTreatment, TreatmentConfigSchema, TreatmentSchema, TreatmentSetupSchema, createTreatment, getTreatmentId} +function composeTreatmentSetup(...setups: Array): TreatmentSetup { + return async function composedSetup(options: TreatmentSetupOptions): Promise { + for (const setup of setups) { + if (setup) { + await setup(options) + } + } + } +} + +export { + ControlTreatment, + TreatmentConfigSchema, + TreatmentSchema, + TreatmentSetupSchema, + composeTreatmentSetup, + createTreatment, + getTreatmentId, +} export type {TreatmentConfig, Treatment, TreatmentSetup} diff --git a/skills/agent-eval/SKILL.md b/skills/agent-eval/SKILL.md index 805abd24..07505d63 100644 --- a/skills/agent-eval/SKILL.md +++ b/skills/agent-eval/SKILL.md @@ -13,22 +13,27 @@ to their environment improves the result. Read only the references needed for the current task. The overview is context, not a prerequisite for every operation. -| Task | Read | -| :------------------------------------------------------ | :--------------------------------------------------------------------------------- | -| Install and complete a first evaluation | [Getting started](references/getting-started.md) | -| Understand project structure and evaluation methodology | [Overview](references/overview.md) | -| Establish a capability baseline | [Benchmarks](references/benchmarks.md), [capabilities](references/capabilities.md) | -| Compare instructions, skills, agents, MCP, or plugins | [Experiments](references/experiments.md), [treatments](references/treatments.md) | -| Create a task, starting workspace, or custom image | [Scenarios](references/scenarios.md) | -| Add deterministic grading or a subjective rubric | [Checks](references/checks.md) or [judges](references/judges.md) | -| Select models, effort, or execution backends | [Models and runners](references/models-and-runners.md) | -| Install resources or run commands in a trial | [Sandbox](references/sandbox.md) | -| Inspect a run matrix, shard, or merge | [Plans](references/plans.md) | -| Read results or diagnose agent behavior | [Trials and results](references/trials-and-results.md) | -| Find a command, flag, or runtime failure | [CLI](references/cli.md) | +| Task | Read | +| :------------------------------------------------------ | :-------------------------------------------------------------------------------------------------------------- | +| Install and complete a first evaluation | [Getting started](references/getting-started.md) | +| Understand project structure and evaluation methodology | [Overview](references/overview.md) | +| Establish a capability baseline | [Benchmarks](references/benchmarks.md), [capabilities](references/capabilities.md) | +| Compare instructions, skills, agents, MCP, or plugins | [Experiments](references/experiments.md), [treatments](references/treatments.md) | +| Create a task, starting workspace, or custom image | [Scenarios](references/scenarios.md) | +| Add deterministic grading or a subjective rubric | [Checks](references/checks.md) or [judges](references/judges.md) | +| Select models, effort, or execution backends | [Models and runners](references/models-and-runners.md) | +| Install resources or run commands in a trial | [Sandbox](references/sandbox.md) | +| Inspect a run matrix, shard, or merge | [Plans](references/plans.md) | +| Read results or diagnose agent behavior | [Trials and results](references/trials-and-results.md), [experiment reports](references/experiments.md#reports) | +| Find a command, flag, or runtime failure | [CLI](references/cli.md) | ## Execution rules +- When a decision or recommendation relies on a resource, include the specific + evidence and explain how it supports the conclusion. Cite file paths and line + numbers, or link to the relevant external section or comment and quote the + supporting passage. Read the resource first and distinguish evidence from + inference. - For a first comparison, use one scenario, one model/effort, and one treatment beyond the automatic control. - Keep prompts and fixtures treatment-blind. Put the knowledge being tested in diff --git a/skills/agent-eval/references/benchmarks.md b/skills/agent-eval/references/benchmarks.md index 9d7d7e5d..ad5eebc2 100644 --- a/skills/agent-eval/references/benchmarks.md +++ b/skills/agent-eval/references/benchmarks.md @@ -17,6 +17,9 @@ time. Use an [experiment](experiments.md) to compare multiple interventions. Benchmark-level `setup` runs only for `Benchmark`. Capability-level setup runs for **both** treatments, before treatment setup. Put neutral prerequisites in capability setup and the resource being measured in benchmark setup. +When an experiment selects this benchmark, capability setup runs for every +treatment, including custom treatments, after experiment setup. Benchmark-level +setup still runs only for `Benchmark`. ## Minimal example diff --git a/skills/agent-eval/references/capabilities.md b/skills/agent-eval/references/capabilities.md index c1b3cefb..8bc40f2e 100644 --- a/skills/agent-eval/references/capabilities.md +++ b/skills/agent-eval/references/capabilities.md @@ -1,8 +1,8 @@ # Capabilities **Use when:** grouping benchmark scenarios under a behavior such as navigation, -forms, or API migration. Experiments select scenarios directly, without -capability groups. +forms, or API migration. Benchmark-backed experiments reuse the benchmark's +capability groups; scenario-backed experiments select scenarios directly. ## Contract @@ -13,10 +13,12 @@ capability groups. | Optional field | `setup` | | Identity | Name-derived ID; names must be unique within the benchmark | | Scenario references | Folder names under the selected scenarios directory | -| Setup scope | Both `Control` and `Benchmark`, before treatment setup | +| Setup scope | Every treatment in the capability, before treatment setup | Put only shared prerequisites in capability setup. Install the resource being -measured in the benchmark's top-level setup. +measured in the benchmark's top-level setup, which runs only for `Benchmark`. +In benchmark-backed experiments, capability setup also applies to custom +treatments, after experiment setup and before treatment setup. ## Configuration fragment @@ -37,9 +39,9 @@ Prefer pinned dependencies in the fixture manifest when no setup hook is needed. ## Run and verify -Create and run the enclosing [benchmark](benchmarks.md), not the capability -itself. Inspect its plan for the expected capability/scenario memberships and -both treatment conditions. +Create and run the enclosing [benchmark](benchmarks.md) or a benchmark-backed +[experiment](experiments.md), not the capability itself. Inspect its plan for +the expected capability/scenario memberships and all treatment conditions. Group related tasks without making one scenario cover every behavior. Keep granular scenario checks so the capability summary does not hide failures. diff --git a/skills/agent-eval/references/cli.md b/skills/agent-eval/references/cli.md index 716666b4..f0e240ae 100644 --- a/skills/agent-eval/references/cli.md +++ b/skills/agent-eval/references/cli.md @@ -42,22 +42,22 @@ npx agent-eval scenario image clean --help ## Options by scope -| Option | Scope and default | -| :-------------------------------- | :------------------------------------------------ | -| `--benchmarks ` | Benchmark run and plan commands; `./benchmarks` | -| `--experiments ` | Experiment run and plan commands; `./experiments` | -| `--scenarios ` | Run, plan, and image build; `./scenarios` | -| `--output-dir ` | Run, plan run, and merge; `./results` | -| `--output-path ` | Plan create only; `plan.json` | -| `--plan-path ` | Plan run only; `./plan.json` | -| `--shard /` | Plan run only; omitted means all trials | -| `--runner ` | Scenario run and plan run; see below | -| `--check ` | Scenario run only; selects one configured check | -| `--copilot-concurrency `, `-c` | Run and plan run; `1` | -| `--container-concurrency ` | Run and plan run; `5` | -| `--docker-image ` | Run and plan run; package default Node image | -| `--token ` | Run and plan run; prefer `COPILOT_GITHUB_TOKEN` | -| `--log-level ` | Root option; `info` | +| Option | Scope and default | +| :-------------------------------- | :--------------------------------------------------------- | +| `--benchmarks ` | Benchmark and experiment run/plan commands; `./benchmarks` | +| `--experiments ` | Experiment run and plan commands; `./experiments` | +| `--scenarios ` | Run, plan, and image build; `./scenarios` | +| `--output-dir ` | Run, plan run, and merge; `./results` | +| `--output-path ` | Plan create only; `plan.json` | +| `--plan-path ` | Plan run only; `./plan.json` | +| `--shard /` | Plan run only; omitted means all trials | +| `--runner ` | Scenario run and plan run; see below | +| `--check ` | Scenario run only; selects one configured check | +| `--copilot-concurrency `, `-c` | Run and plan run; `1` | +| `--container-concurrency ` | Run and plan run; `5` | +| `--docker-image ` | Run and plan run; package default Node image | +| `--token ` | Run and plan run; prefer `COPILOT_GITHUB_TOKEN` | +| `--log-level ` | Root option; `info` | Concurrency values must be positive integers. Copilot concurrency limits active sessions; container concurrency limits active trial containers. Both apply per diff --git a/skills/agent-eval/references/experiments.md b/skills/agent-eval/references/experiments.md index 3b6ce2ec..7c219c4d 100644 --- a/skills/agent-eval/references/experiments.md +++ b/skills/agent-eval/references/experiments.md @@ -6,22 +6,48 @@ plugins, or execution backends on the same tasks. Use a ## Contract -| Item | Rule | -| :------------------ | :------------------------------------------------------------------------- | -| File and import | `experiments/.ts`; `defineConfig` from `@primer/agent-eval/experiment` | -| Export | Prefer named `experiment`; default also supported | -| Required fields | `name`, `description`, `models`, `scenarios`, `treatments` | -| Optional fields | `setup`, `runners` | -| CLI identifier | Filename without extension, not display `name` | -| Scenario references | Folder IDs; alternatively `{path, name?}` | -| Runner default | `copilot-cli` | +| Item | Rule | +| :------------------ | :----------------------------------------------------------------------------------- | +| File and import | `experiments/.ts`; `defineConfig` from `@primer/agent-eval/experiment` | +| Export | Use a default export; named `experiment` also supported | +| Required fields | `name`, `description`, `models`, `treatments`, and either `scenarios` or `benchmark` | +| Optional fields | `setup`, `runners` | +| CLI identifier | Filename without extension, not display `name` | +| Scenario references | Folder IDs; alternatively `{path, name?}` | +| Benchmark reference | Benchmark filename ID; selects its capabilities and scenarios | +| Runner default | `copilot-cli` | Top-level `setup` runs for **all trials, including control**. Put only neutral prerequisites there. Treatment setup runs afterward and installs the intervention. Do not put the tested resource in shared setup or the fixture. The harness adds `Control` automatically. Treatment names must be unique and -cannot be `Control`. `treatments: []` runs control only. +cannot be `Control`. For scenario-backed experiments, `treatments: []` runs +control only. Benchmark-backed experiments also add a `Benchmark` treatment +using only benchmark-level setup; `Benchmark` is also a reserved name. +Capability setup runs for every treatment in that capability, including +`Control` and custom treatments, after experiment setup and before treatment +setup. + +## Reports + +CLI reports group scenario experiments by treatment, scenario, and model/effort. +Benchmark-backed experiments add capability totals under each treatment: +treatment, capability, scenario, then model/effort. Shared scenarios stay +separate across capabilities. Runner groups remain separate, with a `Runner` +column when SDK trials are present. + +Each level shows run counts, check summaries, and implementation-agent usage. +Partial runs show only completed groups, not empty configured capabilities. + +Website experiment pages show treatment summaries, capability summaries, and +scenario results within each capability. Usage is averaged per trial, with +runner/model/effort groups kept separate. Output links target the matching +capability and scenario on the run detail page. + +New benchmark-backed experiment bundles include capability metadata and trial +membership. Older bundles lacking that metadata remain readable as flat +scenario results; do not infer membership from current configuration. ## Minimal example @@ -31,7 +57,7 @@ Save this complete configuration as `experiments/verification.ts`: ```ts import {defineConfig} from '@primer/agent-eval/experiment' -export const experiment = defineConfig({ +export default defineConfig({ name: 'Verification instructions', description: 'Test whether verification instructions improve correctness on a small transformation task', models: [{name: 'gpt-5.4', reasoningEfforts: ['low']}], @@ -80,6 +106,10 @@ The matrix expands as follows: trials = model variants * scenarios * unique runners * (configured treatments + 1) ``` +For benchmark-backed experiments, sum scenario counts across capabilities and +use `(configured treatments + 2)` for the automatic `Control` and `Benchmark` +treatments. A scenario shared by two capabilities runs once per capability. + `runners: ['copilot-cli', 'copilot-sdk']` adds a backend comparison. Changing both runner and resource confounds their effects; see [models and runners](models-and-runners.md). diff --git a/skills/agent-eval/references/trials-and-results.md b/skills/agent-eval/references/trials-and-results.md index e496ea76..7892d6ea 100644 --- a/skills/agent-eval/references/trials-and-results.md +++ b/skills/agent-eval/references/trials-and-results.md @@ -41,7 +41,7 @@ artifacts/ `output.json` is a manifest. Its `trials` object maps IDs to relative trial JSON paths, not embedded trial results. It includes scenario and treatment metadata; -benchmark output also includes capability metadata. +benchmark and benchmark-backed experiment output also include capability metadata. Resolve each trial file path relative to the output directory. Trial files include model, runner, treatment/scenario identifiers, checks, judges, agent @@ -49,6 +49,12 @@ sessions, walkthrough information, and artifact locations. Preserve the whole bundle when sharing or archiving. Embedded artifact paths can be absolute; do not assume every metadata path relocates automatically. +Benchmark-backed experiment trials include `capabilityId`. The package root +exposes `parseExperimentTrialOutput(json, capabilities)` to check membership +against the bundle's capability map. Scenario-only and legacy experiment +bundles default to an empty capability map; missing historical membership is +not inferred. + Standalone `scenario run` writes a different output: `{id, results}`, with entries containing `trial: {id}` and an embedded `result`. Do not parse it as a benchmark/experiment manifest. @@ -89,6 +95,9 @@ verified pricing basis. - Support causal explanations with saved workspace and session evidence. Compare like-for-like scenario/model/effort/runner groups across treatments. +For benchmark-backed experiments, CLI reports include capability totals under +each treatment. Compare scenario/model groups within the same capability; +shared scenarios are reported separately for each capability. Look at granular regressions before an aggregate score. Explain failures using saved source and sessions rather than speculating from a summary. diff --git a/website/src/app/components/ExperimentResults.tsx b/website/src/app/components/ExperimentResults.tsx index 5e6aebde..67e4849c 100644 --- a/website/src/app/components/ExperimentResults.tsx +++ b/website/src/app/components/ExperimentResults.tsx @@ -25,6 +25,9 @@ function formatCredits(value: number): string { function TreatmentResultsTable({results, label}: {results: Array; label: string}) { const labelId = useId() + const showRunner = results.some(result => { + return result.runner === 'copilot-sdk' + }) return ( @@ -35,6 +38,15 @@ function TreatmentResultsTable({results, label}: {results: Array Checks show per-trial pass percentages or measurement means, averaged across trials. Skips, errors, and missing - values are reported separately. Resource usage is the average per trial. Treatments are grouped by model and - reasoning effort; compare scenario and trial counts before comparing performance. + values are reported separately. Resource usage is the average per trial. Treatments are grouped by runner, model, + and reasoning effort; compare scenario and trial counts before comparing performance.

) } @@ -130,7 +142,7 @@ export function ExperimentsOverview({experiments}: {experiments: Array - . View scenario results and run history + . View detailed results and run history

) : null} {experiment.treatments.length > 0 ? ( @@ -179,28 +191,66 @@ export function LatestExperimentResults({id, results}: {id: string; results: Exp No trial results were recorded in the latest run. )} -

Scenario results

- {results.scenarios.map(scenario => { + {results.capabilities.length > 0 ?

Capability results

: null} + {results.capabilities.map(capability => { return ( -
-

{scenario.id}

- {scenario.treatments.length > 0 ? ( - <> - -

- - View output for {scenario.id} - -

- +
+

{capability.name}

+ {capability.treatments.length > 0 ? ( + ) : ( -

No trial results were recorded for this scenario.

+

No trial results were recorded for this capability.

)} +
) })} + {results.capabilities.length === 0 ?

Scenario results

: null} +
) } + +function ScenarioResults({ + id, + date, + scenarios, + capability, +}: { + id: string + date: string + scenarios: ExperimentResults['scenarios'] + capability?: {id: string; name: string} +}) { + return scenarios.map(scenario => { + const label = capability ? `${capability.name} / ${scenario.id}` : scenario.id + return ( +
+ {capability ? ( +
{scenario.id}
+ ) : ( +

{scenario.id}

+ )} + {scenario.treatments.length > 0 ? ( + <> + +

+ + View output for {label} + +

+ + ) : ( +

No trial results were recorded for this scenario.

+ )} +
+ ) + }) +} diff --git a/website/src/app/components/RunDetailsView.tsx b/website/src/app/components/RunDetailsView.tsx index 1bff91ee..467fb604 100644 --- a/website/src/app/components/RunDetailsView.tsx +++ b/website/src/app/components/RunDetailsView.tsx @@ -444,7 +444,7 @@ function ScenarioResults({group, index}: {group: ScenarioResultGroup; index: num

diff --git a/website/src/app/components/ScenarioAnchors.test.tsx b/website/src/app/components/ScenarioAnchors.test.tsx index 8bfc869e..6e60d100 100644 --- a/website/src/app/components/ScenarioAnchors.test.tsx +++ b/website/src/app/components/ScenarioAnchors.test.tsx @@ -19,6 +19,7 @@ test.each(['001-button', 'space / literal%20 # caf\u00e9'])( const overview = renderToStaticMarkup( , ) + const details = renderToStaticMarkup( { + const results = [ + createResult({id: 'first', capabilityId: 'a'}), + createResult({id: 'sdk', capabilityId: 'a', runner: 'copilot-sdk'}), + createResult({id: 'second', capabilityId: 'b'}), + ] + const output = createRun(results).output + output.capabilities = new Map([ + ['a', {id: 'a', name: 'Components', scenarioIds: ['scenario-a']}], + ['b', {id: 'b', name: 'Layout', scenarioIds: ['scenario-a']}], + ]) + const run = await createExperimentRunDetails('2026-09-10', output) + const summary = getExperimentResults({...createRun(results), output}) + const overview = renderToStaticMarkup() + const details = renderToStaticMarkup( + , + ) + const links = [...overview.matchAll(/href="([^"]+#capability-scenario-[^"]+)"/g)] + const targets = [...details.matchAll(/]+id="([^"]+)"/g)].map(match => { + return match[1] + }) + + expect(overview).toContain('Capability results') + expect(overview).toContain('copilot-sdk') + expect(overview).toContain('Runner') + expect(links).toHaveLength(2) + expect(targets).toHaveLength(2) + expect(new Set(targets).size).toBe(2) + for (const [, href] of links) { + expect(targets).toContain(new URL(href, 'https://example.test').hash.slice(1)) + } + expect(details).toContain('All capabilities') + expect(details).toContain('Components / scenario-a') + expect(details).toContain('Layout / scenario-a') +}) diff --git a/website/src/experiment-results.test.ts b/website/src/experiment-results.test.ts index 60fe4e48..a7e81fe8 100644 --- a/website/src/experiment-results.test.ts +++ b/website/src/experiment-results.test.ts @@ -8,6 +8,7 @@ test('distinguishes missing runs from runs with no trials', () => { expect(getExperimentResults(createRun([]))).toEqual({ date: '2026-09-10', treatments: [], + capabilities: [], scenarios: [ {id: 'scenario-a', treatments: []}, {id: 'scenario-b', treatments: []}, @@ -56,8 +57,9 @@ test('averages checks and implementation usage per trial, not per scenario or ou expect(summary?.treatments).toEqual([ { - id: JSON.stringify(['control', 'gpt-5.6-sol', 'medium']), + id: JSON.stringify(['control', 'copilot-cli', 'gpt-5.6-sol', 'medium']), treatment: 'Control', + runner: 'copilot-cli', model: 'gpt-5.6-sol', reasoningEffort: 'medium', trials: 3, @@ -121,3 +123,48 @@ test('preserves recorded scenario and treatment IDs even when metadata is absent treatments: [{treatment: 'historical-treatment'}], }) }) + +test('keeps shared scenarios separate across capabilities and runners', () => { + const first = createResult({id: 'components-cli', capabilityId: 'components'}) + const run = createRun([ + first, + createResult({ + id: 'components-sdk', + capabilityId: 'components', + runner: 'copilot-sdk', + agent: {sessions: [{...first.agent.sessions[0], outputTokens: 300}]}, + }), + createResult({ + id: 'layout-cli', + capabilityId: 'layout', + agent: {sessions: [{...first.agent.sessions[0], outputTokens: 900}]}, + }), + ]) + run.output.capabilities = new Map([ + ['layout', {id: 'layout', name: 'Layout', scenarioIds: ['scenario-a']}], + ['components', {id: 'components', name: 'Components', scenarioIds: ['scenario-a']}], + ]) + + const summary = getExperimentResults(run) + + expect(summary?.scenarios).toEqual([]) + expect(summary?.treatments).toMatchObject([ + {runner: 'copilot-cli', trials: 2, outputTokens: 500}, + {runner: 'copilot-sdk', trials: 1, outputTokens: 300}, + ]) + expect(summary?.capabilities).toMatchObject([ + { + id: 'components', + treatments: [ + {runner: 'copilot-cli', trials: 1, outputTokens: 100}, + {runner: 'copilot-sdk', trials: 1, outputTokens: 300}, + ], + scenarios: [{id: 'scenario-a', treatments: [{outputTokens: 100}, {outputTokens: 300}]}], + }, + { + id: 'layout', + treatments: [{runner: 'copilot-cli', trials: 1, outputTokens: 900}], + scenarios: [{id: 'scenario-a', treatments: [{outputTokens: 900}]}], + }, + ]) +}) diff --git a/website/src/experiment-results.ts b/website/src/experiment-results.ts index e9241139..3fb6ed2b 100644 --- a/website/src/experiment-results.ts +++ b/website/src/experiment-results.ts @@ -1,10 +1,11 @@ -import type {ExperimentTrialOutput} from '@primer/agent-eval' +import type {CopilotRunner, ExperimentTrialOutput} from '@primer/agent-eval' import {formatChecks, summarizeTrials} from './check-results' import type {Run} from './runs' export type TreatmentResult = { id: string treatment: string + runner: CopilotRunner model: string reasoningEffort: string trials: number @@ -24,13 +25,19 @@ export type ExperimentResults = { id: string treatments: Array }> + capabilities: Array<{ + id: string + name: string + treatments: Array + scenarios: ExperimentResults['scenarios'] + }> } function summarizeTreatments(run: Run, results: Array): Array { const groups = new Map>() for (const result of results) { - const key = JSON.stringify([result.treatmentId, result.model.name, result.model.reasoningEffort]) + const key = JSON.stringify([result.treatmentId, result.runner, result.model.name, result.model.reasoningEffort]) const group = groups.get(key) if (group) { group.push(result) @@ -46,6 +53,7 @@ function summarizeTreatments(run: Run, results: Array): A return { id, treatment: run.output.treatments.get(first.treatmentId)?.name ?? first.treatmentId, + runner: first.runner, model: first.model.name, reasoningEffort: first.model.reasoningEffort, trials: trials.length, @@ -65,22 +73,23 @@ function summarizeTreatments(run: Run, results: Array): A return ( first.model.localeCompare(second.model) || first.reasoningEffort.localeCompare(second.reasoningEffort) || + first.runner.localeCompare(second.runner) || first.treatment.localeCompare(second.treatment) || first.id.localeCompare(second.id) ) }) } -export function getExperimentResults(run: Run | undefined): ExperimentResults | null { - if (!run) { - return null - } - +function summarizeScenarios( + run: Run, + results: Array, + scenarioIds: Iterable, +): ExperimentResults['scenarios'] { const scenarios = new Map>() - for (const scenario of run.output.scenarios.values()) { - scenarios.set(scenario.id, []) + for (const id of scenarioIds) { + scenarios.set(id, []) } - for (const result of run.output.trials.values()) { + for (const result of results) { const group = scenarios.get(result.scenarioId) if (group) { group.push(result) @@ -89,13 +98,37 @@ export function getExperimentResults(run: Run | undefined): ExperimentResults | } } + return Array.from(scenarios, ([id, trials]) => { + return {id, treatments: summarizeTreatments(run, trials)} + }).toSorted((first, second) => { + return first.id.localeCompare(second.id) + }) +} + +export function getExperimentResults(run: Run | undefined): ExperimentResults | null { + if (!run) { + return null + } + const trials = [...run.output.trials.values()] + return { date: run.name, - treatments: summarizeTreatments(run, [...run.output.trials.values()]), - scenarios: Array.from(scenarios, ([id, results]) => { - return {id, treatments: summarizeTreatments(run, results)} - }).toSorted((first, second) => { - return first.id.localeCompare(second.id) - }), + treatments: summarizeTreatments(run, trials), + scenarios: run.output.capabilities.size === 0 ? summarizeScenarios(run, trials, run.output.scenarios.keys()) : [], + capabilities: [...run.output.capabilities.values()] + .map(capability => { + const selected = trials.filter(trial => { + return trial.capabilityId === capability.id + }) + return { + id: capability.id, + name: capability.name, + treatments: summarizeTreatments(run, selected), + scenarios: summarizeScenarios(run, selected, capability.scenarioIds), + } + }) + .toSorted((first, second) => { + return first.name.localeCompare(second.name) || first.id.localeCompare(second.id) + }), } } diff --git a/website/src/experiments.ts b/website/src/experiments.ts index 5cc0ca87..d5897f57 100644 --- a/website/src/experiments.ts +++ b/website/src/experiments.ts @@ -7,6 +7,7 @@ const {listExperiments, getExperiment} = await import( ) const EXPERIMENTS_DIR = path.resolve(process.cwd(), '..', 'experiments') +const BENCHMARKS_DIR = path.resolve(process.cwd(), '..', 'benchmarks') const SCENARIOS_DIR = path.resolve(process.cwd(), '..', 'scenarios') export type Experiment = Pick & { @@ -16,39 +17,41 @@ export type Experiment = Pick> { const experiments = await listExperiments({ + benchmarksDirectory: BENCHMARKS_DIR, experimentsDirectory: EXPERIMENTS_DIR, scenariosDirectory: SCENARIOS_DIR, }) return experiments.map(experiment => { - return { - id: experiment.id, - name: experiment.name, - description: experiment.description, - models: experiment.models, - scenarios: experiment.scenarios.map(scenario => { - return {id: scenario.id} - }), - treatments: experiment.treatments.map(treatment => { - return {name: treatment.name} - }), - } + return toExperiment(experiment) }) } export async function get(id: string): Promise { const experiment = await getExperiment({ + benchmarksDirectory: BENCHMARKS_DIR, experimentsDirectory: EXPERIMENTS_DIR, scenariosDirectory: SCENARIOS_DIR, name: id, }) + return toExperiment(experiment) +} + +function toExperiment(experiment: AgentEvalExperiment): Experiment { + const scenarios = + experiment.type === 'scenario' + ? experiment.scenarios + : experiment.benchmark.capabilities.flatMap(capability => { + return capability.scenarios + }) + return { id: experiment.id, name: experiment.name, description: experiment.description, models: experiment.models, - scenarios: experiment.scenarios.map(scenario => { + scenarios: scenarios.map(scenario => { return {id: scenario.id} }), treatments: experiment.treatments.map(treatment => { diff --git a/website/src/result-files.test.ts b/website/src/result-files.test.ts index 0cdc8a73..822ee2fe 100644 --- a/website/src/result-files.test.ts +++ b/website/src/result-files.test.ts @@ -67,6 +67,51 @@ test.each(['benchmark', 'experiment'] as const)( }, ) +test('reads benchmark-backed experiment capability metadata without losing trial membership', async () => { + const directory = await createDirectory() + const output = createBenchmarkOutput([ + {...createTrial(), capabilityId: 'a'}, + {...createTrial({id: 'trial-2', runner: 'copilot-sdk'}), capabilityId: 'b'}, + ]) + output.id = 'test-experiment' + const filepath = await writeBundle(directory, output) + + const parsed = await readExperimentOutput(filepath) + + expect(parsed).toEqual(output) + if (!parsed) { + throw new Error('Expected an experiment bundle') + } + const details = await createExperimentRunDetails('2026-09-15', parsed) + expect( + details.results.map(result => { + return result.capability + }), + ).toEqual([ + {id: 'a', name: 'First capability'}, + {id: 'b', name: 'Overlapping capability'}, + ]) +}) + +test('reads a legacy experiment manifest without inventing capability membership', async () => { + const directory = await createDirectory() + const output = createExperimentOutput() + const filepath = await writeBundle(directory, output) + const manifest = JSON.parse(await fs.readFile(filepath, 'utf8')) + delete manifest.capabilities + await fs.writeFile(filepath, JSON.stringify(manifest)) + + expect(await readExperimentOutput(filepath)).toEqual(output) +}) + +test('rejects experiment trial capability membership that conflicts with the manifest', async () => { + const directory = await createDirectory() + const output = createBenchmarkOutput([{...createTrial(), capabilityId: 'unknown'}]) + const filepath = await writeBundle(directory, output) + + await expect(readExperimentOutput(filepath)).rejects.toThrow('Invalid capability "unknown"') +}) + test.each([ {key: 'benchmarkId', read: readBenchmarkOutput}, {key: 'experimentId', read: readExperimentOutput}, diff --git a/website/src/result-files.ts b/website/src/result-files.ts index 1b6861d2..002cec52 100644 --- a/website/src/result-files.ts +++ b/website/src/result-files.ts @@ -8,6 +8,7 @@ const { parseBenchmarkTrialOutput, ExperimentOutputFileSchema, ExperimentTrialOutputSchema, + parseExperimentTrialOutput, } = await import( /* turbopackIgnore: true */ '@primer/agent-eval' @@ -96,8 +97,13 @@ async function readExperimentOutput(filepath: string): Promise 0) && !capability) { + throw new Error(`Unknown capability for trial "${result.id}"`) + } const baseUrl = getTrialDataUrl(collection, output.id, date, result.id) const tools = new Map() for (const session of result.agent.sessions) { @@ -389,6 +393,7 @@ async function createExperimentRunDetails( results.push({ id: result.id, scenarioId: result.scenarioId, + ...(capability ? {capability: {id: capability.id, name: capability.name}} : {}), treatment, model: result.model.name, reasoningEffort: result.model.reasoningEffort, @@ -445,19 +450,7 @@ function createReferenceFiles(files: CheckOutput['check']['files']): CheckOutput } async function createBenchmarkRunDetails(run: BenchmarkRun): Promise { - const output = run.output - const details = await createExperimentRunDetails(run.name, output, 'benchmarks', run.directory) - return { - ...details, - results: details.results.map(result => { - const trial = output.trials.get(result.id) - const capability = trial ? output.capabilities.get(trial.capabilityId) : undefined - if (!capability) { - throw new Error(`Unknown capability for benchmark trial "${result.id}"`) - } - return {...result, capability: {id: capability.id, name: capability.name}} - }), - } + return createExperimentRunDetails(run.name, run.output, 'benchmarks', run.directory) } export { diff --git a/website/src/runs.test.ts b/website/src/runs.test.ts index b24658b3..5d6fd97d 100644 --- a/website/src/runs.test.ts +++ b/website/src/runs.test.ts @@ -25,6 +25,7 @@ beforeEach(async () => { } vi.mocked(read).mockResolvedValue({ id: 'example', + capabilities: new Map(), scenarios: new Map(), treatments: new Map(), trials: new Map(), @@ -83,6 +84,7 @@ test('rejects a bundle belonging to a different experiment without reading older const latest = await createRunDirectory('2026-09-10') vi.mocked(read).mockResolvedValueOnce({ id: 'other', + capabilities: new Map(), scenarios: new Map(), treatments: new Map(), trials: new Map(), diff --git a/website/src/scenario-anchor.ts b/website/src/scenario-anchor.ts index 7748c062..aa9acb96 100644 --- a/website/src/scenario-anchor.ts +++ b/website/src/scenario-anchor.ts @@ -1,6 +1,8 @@ -export function getScenarioAnchor(scenarioId: string): {id: string; fragment: string} { +export function getScenarioAnchor(scenarioId: string, capabilityId?: string): {id: string; fragment: string} { // Percent-free IDs avoid differences between native and Next.js fragment lookups. - const id = `scenario-${encodeURIComponent(scenarioId).replaceAll('_', '%5F').replaceAll('%', '_')}` + const key = capabilityId === undefined ? scenarioId : JSON.stringify([capabilityId, scenarioId]) + const prefix = capabilityId === undefined ? 'scenario' : 'capability-scenario' + const id = `${prefix}-${encodeURIComponent(key).replaceAll('_', '%5F').replaceAll('%', '_')}` return { id, fragment: `#${id}`, diff --git a/website/src/test-fixtures.ts b/website/src/test-fixtures.ts index a1ed5bf9..0bc94f84 100644 --- a/website/src/test-fixtures.ts +++ b/website/src/test-fixtures.ts @@ -103,6 +103,7 @@ function createTrial(overrides: Partial = {}): Experiment function createExperimentOutput(trials: Array = [createTrial()]): ExperimentOutput { return { id: 'test-experiment', + capabilities: new Map(), scenarios: new Map([ [ 'empty-state',