From 38620da6fe64166d94bd0616de26dd9f68343eed Mon Sep 17 00:00:00 2001
From: Josh Black <joshblack@github.com>
Date: Fri, 9 Oct 2026 12:52:24 -0500
Subject: [PATCH 1/7] feat: add support for benchmarks to experiments

---
 experiments/benchmark.ts                      |  16 +
 .../agent-eval/src/cli/commands/experiment.ts |  18 +-
 packages/agent-eval/src/experiment/config.ts  |  34 +-
 .../agent-eval/src/experiment/experiment.ts   |  21 +-
 packages/agent-eval/src/experiment/get.ts     |   5 +-
 packages/agent-eval/src/experiment/list.ts    |  80 +++--
 packages/agent-eval/src/experiment/plan.ts    | 293 +++++++++++++-----
 packages/agent-eval/src/treatment.ts          |  20 +-
 8 files changed, 376 insertions(+), 111 deletions(-)
 create mode 100644 experiments/benchmark.ts

diff --git a/experiments/benchmark.ts b/experiments/benchmark.ts
new file mode 100644
index 00000000..e68ba3fb
--- /dev/null
+++ b/experiments/benchmark.ts
@@ -0,0 +1,16 @@
+import {defineConfig} from '@primer/agent-eval/experiment'
+
+export default defineConfig({
+  name: 'benchmark',
+  description: 'An end-to-end test for running experiments against benchmarks',
+  models: ['gpt-5.6-luna'],
+  benchmark: 'noop',
+  treatments: [
+    {
+      name: 'noop',
+      async setup() {
+        console.log('local setup')
+      },
+    },
+  ],
+})
diff --git a/packages/agent-eval/src/cli/commands/experiment.ts b/packages/agent-eval/src/cli/commands/experiment.ts
index 239206ab..92bfa1ae 100644
--- a/packages/agent-eval/src/cli/commands/experiment.ts
+++ b/packages/agent-eval/src/cli/commands/experiment.ts
@@ -14,6 +14,7 @@ import {logger} from '../../logger'
 import {createPlanFromManifest, runPlan} from '../../plan'
 import {parseShard} from '../../shard'
 import {
+  benchmarksOption,
   copilotConcurrencyOption,
   containerConcurrencyOption,
   experimentsOption,
@@ -82,6 +83,7 @@ const experimentCommand = defineCommand({
             description: 'Create an experiment plan',
           },
           args: {
+            benchmarks: benchmarksOption,
             experiments: experimentsOption,
             name: {
               type: 'positional',
@@ -98,25 +100,35 @@ const experimentCommand = defineCommand({
           async run({args}) {
             logger.info('Planning experiment: %s', args.name)
 
+            const benchmarksDirectory = path.resolve(args.benchmarks)
             const experimentsDirectory = path.resolve(args.experiments)
             const scenariosDirectory = path.resolve(args.scenarios)
             const outputPath = path.resolve(args['output-path'])
 
             logger.debug({
+              benchmarksDirectory,
               experimentsDirectory,
               scenariosDirectory,
               outputPath,
             })
 
             const experiment = await getExperiment({
+              benchmarksDirectory,
               experimentsDirectory,
               scenariosDirectory,
               name: args.name,
             })
-            const plan = createExperimentPlan({experiment})
-            const manifest = createExperimentPlanManifest({experiment, plan})
+            const plan = createExperimentPlan({
+              experiment,
+            })
+            const manifest = createExperimentPlanManifest({
+              experiment,
+              plan,
+            })
 
-            await host.fs.mkdir(path.dirname(outputPath), {recursive: true})
+            await host.fs.mkdir(path.dirname(outputPath), {
+              recursive: true,
+            })
             await host.fs.writeFile(outputPath, JSON.stringify(manifest, null, 2), 'utf-8')
 
             logger.info('Wrote experiment plan to: %s', path.relative(process.cwd(), outputPath))
diff --git a/packages/agent-eval/src/experiment/config.ts b/packages/agent-eval/src/experiment/config.ts
index ef8ba503..284f311d 100644
--- a/packages/agent-eval/src/experiment/config.ts
+++ b/packages/agent-eval/src/experiment/config.ts
@@ -4,7 +4,35 @@ import {ModelVariantConfigSchema} from '../model'
 import {ControlTreatment, TreatmentConfigSchema, TreatmentSetupSchema} from '../treatment'
 import {InlineScenarioConfigSchema} from '../scenario/config'
 
-const ExperimentConfigSchema = z.strictObject({
+const BenchmarkExperimentConfig = z.strictObject({
+  name: z.string(),
+  description: z.string(),
+  models: z.array(ModelVariantConfigSchema),
+  runners: CopilotRunnerConfigSchema,
+  benchmark: z.string().check(z.minLength(1)),
+  setup: z.optional(TreatmentSetupSchema),
+  treatments: z.array(TreatmentConfigSchema).check(
+    z.refine(
+      treatments => {
+        const names = new Set<string>([ControlTreatment.name, 'Benchmark'])
+        for (const treatment of treatments) {
+          if (names.has(treatment.name)) {
+            return false
+          }
+          names.add(treatment.name)
+        }
+        return true
+      },
+      {
+        message: 'Treatment names must be unique and cannot use the reserved names "Control" or "Benchmark"',
+      },
+    ),
+  ),
+})
+
+type BenchmarkExperimentConfig = z.infer<typeof BenchmarkExperimentConfig>
+
+const ScenarioExperimentConfig = z.strictObject({
   name: z.string(),
   description: z.string(),
   models: z.array(ModelVariantConfigSchema),
@@ -30,6 +58,10 @@ const ExperimentConfigSchema = z.strictObject({
   ),
 })
 
+type ScenarioExperimentConfig = z.infer<typeof ScenarioExperimentConfig>
+
+const ExperimentConfigSchema = z.union([ScenarioExperimentConfig, BenchmarkExperimentConfig])
+
 type ExperimentConfig = z.infer<typeof ExperimentConfigSchema>
 
 function defineConfig(config: z.input<typeof ExperimentConfigSchema>): ExperimentConfig {
diff --git a/packages/agent-eval/src/experiment/experiment.ts b/packages/agent-eval/src/experiment/experiment.ts
index 41f03c7c..0e064dd2 100644
--- a/packages/agent-eval/src/experiment/experiment.ts
+++ b/packages/agent-eval/src/experiment/experiment.ts
@@ -2,8 +2,23 @@ import type {ExperimentConfig} from './config'
 import type {ModelVariant} from '../model'
 import type {Scenario} from '../scenario/scenario'
 import type {Treatment, TreatmentSetup} from '../treatment'
+import type {Benchmark} from '../benchmark/benchmark'
 
-type Experiment = {
+type BenchmarkExperiment = {
+  type: 'benchmark'
+  id: string
+  filepath: string
+  name: ExperimentConfig['name']
+  description: ExperimentConfig['description']
+  models: Array<ModelVariant>
+  runners: ExperimentConfig['runners']
+  benchmark: Benchmark
+  setup?: TreatmentSetup
+  treatments: Array<Treatment>
+}
+
+type ScenarioExperiment = {
+  type: 'scenario'
   id: string
   filepath: string
   name: ExperimentConfig['name']
@@ -15,4 +30,6 @@ type Experiment = {
   treatments: Array<Treatment>
 }
 
-export type {Experiment}
+type Experiment = ScenarioExperiment | BenchmarkExperiment
+
+export type {Experiment, ScenarioExperiment, BenchmarkExperiment}
diff --git a/packages/agent-eval/src/experiment/get.ts b/packages/agent-eval/src/experiment/get.ts
index e03c10e7..509a258d 100644
--- a/packages/agent-eval/src/experiment/get.ts
+++ b/packages/agent-eval/src/experiment/get.ts
@@ -3,6 +3,7 @@ import type {Experiment} from './experiment'
 import {listExperiments} from './list'
 
 type GetExperimentOptions = {
+  benchmarksDirectory: string
   experimentsDirectory: string
   host?: Host
   scenariosDirectory: string
@@ -10,14 +11,16 @@ type GetExperimentOptions = {
 }
 
 async function getExperiment({
+  benchmarksDirectory,
   experimentsDirectory,
   host = DefaultHost,
   name,
   scenariosDirectory,
 }: GetExperimentOptions): Promise<Experiment> {
   const experiments = await listExperiments({
-    host,
+    benchmarksDirectory,
     experimentsDirectory,
+    host,
     scenariosDirectory,
   })
   const experiment = experiments.find(candidate => {
diff --git a/packages/agent-eval/src/experiment/list.ts b/packages/agent-eval/src/experiment/list.ts
index 7d728ea5..735cb055 100644
--- a/packages/agent-eval/src/experiment/list.ts
+++ b/packages/agent-eval/src/experiment/list.ts
@@ -8,6 +8,8 @@ import {loadScenario} from '../scenario/load'
 import {createTreatment} from '../treatment'
 import {ExperimentConfigSchema} from './config'
 import type {Experiment} from './experiment'
+import {exhaustiveCheck} from '../exhaustive'
+import {getBenchmark} from '../benchmark/get'
 
 const EXPERIMENT_FILE_EXTENSIONS = new Set(['.cjs', '.js', '.mjs', '.ts'])
 
@@ -17,12 +19,14 @@ type ExperimentModule = {
 }
 
 type ListExperimentsOptions = {
+  benchmarksDirectory: string
   experimentsDirectory: string
   host?: Host
   scenariosDirectory: string
 }
 
 async function listExperiments({
+  benchmarksDirectory,
   experimentsDirectory,
   host = DefaultHost,
   scenariosDirectory,
@@ -62,30 +66,58 @@ async function listExperiments({
     }
 
     const {data: config} = parseResult
-    const scenarios = await Promise.all(
-      config.scenarios.map(scenario => {
-        if (typeof scenario === 'string') {
-          return getScenario({host, directory: scenariosDirectory, name: scenario})
-        }
-
-        const directory = path.resolve(scenario.path)
-        return loadScenario({host, directory, name: scenario.name})
-      }),
-    )
-
-    experiments.push({
-      id: getExperimentId(filename),
-      filepath,
-      name: config.name,
-      description: config.description,
-      models: getModelVariants(config.models),
-      runners: config.runners,
-      scenarios,
-      setup: config.setup,
-      treatments: config.treatments.map(treatment => {
-        return createTreatment(treatment)
-      }),
-    })
+
+    if ('scenarios' in config) {
+      const scenarios = await Promise.all(
+        config.scenarios.map(scenario => {
+          if (typeof scenario === 'string') {
+            return getScenario({host, directory: scenariosDirectory, name: scenario})
+          }
+
+          const directory = path.resolve(scenario.path)
+          return loadScenario({host, directory, name: scenario.name})
+        }),
+      )
+
+      experiments.push({
+        type: 'scenario',
+        id: getExperimentId(filename),
+        filepath,
+        name: config.name,
+        description: config.description,
+        models: getModelVariants(config.models),
+        runners: config.runners,
+        scenarios,
+        setup: config.setup,
+        treatments: config.treatments.map(treatment => {
+          return createTreatment(treatment)
+        }),
+      })
+    } else if ('benchmark' in config) {
+      const benchmark = await getBenchmark({
+        benchmarksDirectory,
+        host,
+        scenariosDirectory,
+        name: config.benchmark,
+      })
+
+      experiments.push({
+        type: 'benchmark',
+        id: getExperimentId(filename),
+        filepath,
+        name: config.name,
+        description: config.description,
+        models: getModelVariants(config.models),
+        runners: config.runners,
+        benchmark,
+        setup: config.setup,
+        treatments: config.treatments.map(treatment => {
+          return createTreatment(treatment)
+        }),
+      })
+    } else {
+      exhaustiveCheck(config)
+    }
   }
 
   return experiments
diff --git a/packages/agent-eval/src/experiment/plan.ts b/packages/agent-eval/src/experiment/plan.ts
index 82161a4d..140e7a75 100644
--- a/packages/agent-eval/src/experiment/plan.ts
+++ b/packages/agent-eval/src/experiment/plan.ts
@@ -4,69 +4,131 @@ import {CopilotRunnerSchema} from '../copilot-runner'
 import type {Host} from '../host'
 import {ModelVariantSchema} from '../model'
 import {createPlan, type Plan} from '../plan'
-import {ControlTreatment} from '../treatment'
+import {composeTreatmentSetup, ControlTreatment, createTreatment} from '../treatment'
 import type {Trial} from '../trial/trial'
 import type {Experiment} from './experiment'
 import {getExperiment} from './get'
+import {exhaustiveCheck} from '../exhaustive'
+import type {Capability} from '../benchmark/benchmark'
 
-type ExperimentTrial = Trial
-
-type CreateExperimentPlanOptions = {
-  experiment: Experiment
+type BenchmarkTrial = Trial & {
+  capability: Capability
 }
 
-function getExperimentTreatments(experiment: Experiment) {
-  const treatments = new Map([[ControlTreatment.id, ControlTreatment]])
-  const names = new Set([ControlTreatment.name])
+type ScenarioTrial = Trial
 
-  for (const treatment of experiment.treatments) {
-    if (treatments.has(treatment.id) || names.has(treatment.name)) {
-      throw new Error(`Experiment "${experiment.id}" contains duplicate treatment: ${treatment.name}`)
-    }
-    treatments.set(treatment.id, treatment)
-    names.add(treatment.name)
-  }
+type ExperimentTrial = Trial | BenchmarkTrial
 
-  return treatments
+type CreateExperimentPlanOptions = {
+  experiment: Experiment
 }
 
-function createExperimentPlan({experiment}: CreateExperimentPlanOptions): Plan<ExperimentTrial> {
-  const treatments = [...getExperimentTreatments(experiment).values()]
-
-  return createPlan({
-    trials: experiment.models.flatMap(model => {
-      return experiment.scenarios.flatMap(scenario => {
+function createExperimentPlan({experiment}: CreateExperimentPlanOptions): Plan<ScenarioTrial> | Plan<BenchmarkTrial> {
+  if (experiment.type === 'scenario') {
+    return createPlan({
+      trials: experiment.models.flatMap(model => {
+        return experiment.scenarios.flatMap(scenario => {
+          return experiment.runners.flatMap(runner => {
+            return experiment.treatments.map(treatment => {
+              return {
+                id: randomUUID(),
+                scenario,
+                treatment,
+                model,
+                runner,
+                setup: experiment.setup,
+              }
+            })
+          })
+        })
+      }),
+    })
+  } else if (experiment.type === 'benchmark') {
+    return createPlan({
+      trials: experiment.models.flatMap(model => {
         return experiment.runners.flatMap(runner => {
-          return treatments.map(treatment => {
-            return {
-              id: randomUUID(),
-              scenario,
-              treatment,
-              model,
-              runner,
-              setup: experiment.setup,
-            }
+          return experiment.benchmark.capabilities.flatMap(capability => {
+            return capability.scenarios.flatMap(scenario => {
+              return [
+                // Control
+                {
+                  id: randomUUID(),
+                  scenario,
+                  treatment: ControlTreatment,
+                  model,
+                  runner,
+                  capability,
+                },
+
+                // Benchmark
+                {
+                  id: randomUUID(),
+                  scenario,
+                  treatment: createTreatment({
+                    name: 'Benchmark',
+                    setup: composeTreatmentSetup(experiment.benchmark.setup, capability.setup),
+                  }),
+                  model,
+                  runner,
+                  capability,
+                },
+
+                // Treatments
+                ...experiment.treatments.map(treatment => {
+                  return {
+                    id: randomUUID(),
+                    scenario,
+                    treatment,
+                    model,
+                    runner,
+                    capability,
+                    setup: experiment.setup,
+                  }
+                }),
+              ]
+            })
           })
         })
-      })
-    }),
-  })
+      }),
+    })
+  } else {
+    exhaustiveCheck(experiment)
+  }
 }
 
-const ExperimentPlanManifestFileSchema = z.object({
+const BenchmarkExperimentPlanManifestFileSchema = z.strictObject({
   id: z.string(),
   name: z.string(),
   trials: z.array(
     z.object({
       id: z.string(),
       model: ModelVariantSchema,
-      runner: z._default(CopilotRunnerSchema, 'copilot-cli'),
+      runner: CopilotRunnerSchema,
+      scenarioId: z.string(),
+      capabilityId: z.string(),
+    }),
+  ),
+})
+
+const ScenarioExperimentPlanManifestFileSchema = z.strictObject({
+  id: z.string(),
+  name: z.string(),
+  trials: z.array(
+    z.object({
+      id: z.string(),
+      model: ModelVariantSchema,
+      runner: CopilotRunnerSchema,
       scenarioId: z.string(),
       treatmentId: z.string(),
     }),
   ),
 })
 
+const ExperimentPlanManifestFileSchema = z.union([
+  BenchmarkExperimentPlanManifestFileSchema,
+  ScenarioExperimentPlanManifestFileSchema,
+])
+
 type ExperimentPlanManifestFile = z.infer<typeof ExperimentPlanManifestFileSchema>
 
 type CreateExperimentPlanManifestOptions = {
@@ -82,12 +144,25 @@ function createExperimentPlanManifest({
     id: experiment.id,
     name: experiment.name,
     trials: plan.trials.map(trial => {
-      return {
-        id: trial.id,
-        model: trial.model,
-        runner: trial.runner ?? 'copilot-cli',
-        scenarioId: trial.scenario.id,
-        treatmentId: trial.treatment.id,
+      if ('capability' in trial) {
+        return {
+          id: trial.id,
+          model: trial.model,
+          runner: trial.runner ?? 'copilot-cli',
+          scenarioId: trial.scenario.id,
+          capabilityId: trial.capability.id,
+          treatmentId: trial.treatment.id,
+        }
+      } else if ('scenario' in trial) {
+        return {
+          id: trial.id,
+          model: trial.model,
+          runner: trial.runner ?? 'copilot-cli',
+          scenarioId: trial.scenario.id,
+          treatmentId: trial.treatment.id,
+        }
+      } else {
+        exhaustiveCheck(trial)
       }
     }),
   }
@@ -99,6 +174,7 @@ type ExperimentPlanManifest = {
 }
 
 type ParseExperimentPlanManifestOptions = {
+  benchmarksDirectory: string
   experimentsDirectory: string
   contents: string
   host?: Host
@@ -106,6 +182,7 @@ type ParseExperimentPlanManifestOptions = {
 }
 
 async function parseExperimentPlanManifest({
+  benchmarksDirectory,
   experimentsDirectory,
   contents,
   host,
@@ -113,53 +190,111 @@ async function parseExperimentPlanManifest({
 }: ParseExperimentPlanManifestOptions): Promise<ExperimentPlanManifest> {
   const result = ExperimentPlanManifestFileSchema.parse(JSON.parse(contents))
   const experiment = await getExperiment({
+    benchmarksDirectory,
     experimentsDirectory,
     host,
     name: result.id,
     scenariosDirectory,
   })
-  const scenarios = new Map(
-    experiment.scenarios.map(scenario => {
-      return [scenario.id, scenario]
-    }),
-  )
-  const treatments = getExperimentTreatments(experiment)
-  const trialIds = new Set<string>()
+  if (experiment.type === 'scenario') {
+    const scenarios = new Map(
+      experiment.scenarios.map(scenario => {
+        return [scenario.id, scenario]
+      }),
+    )
+    const treatments = getExperimentTreatments(experiment)
+    const trialIds = new Set<string>()
 
-  return {
-    experiment,
-    trials: result.trials.map(trial => {
-      if (trialIds.has(trial.id)) {
-        throw new Error(`Duplicate trial ID in experiment plan: ${trial.id}`)
-      }
-      trialIds.add(trial.id)
+    return {
+      experiment,
+      trials: result.trials.map(trial => {
+        if (trialIds.has(trial.id)) {
+          throw new Error(`Duplicate trial ID in experiment plan: ${trial.id}`)
+        }
+        trialIds.add(trial.id)
 
-      const scenario = scenarios.get(trial.scenarioId)
-      if (!scenario) {
-        throw new Error(`Scenario not found for trial: ${trial.id}`)
-      }
+        const scenario = scenarios.get(trial.scenarioId)
+        if (!scenario) {
+          throw new Error(`Scenario not found for trial: ${trial.id}`)
+        }
 
-      const treatment = treatments.get(trial.treatmentId)
-      if (!treatment) {
-        throw new Error(`Treatment not found for trial: ${trial.id}`)
-      }
+        const treatment = treatments.get(trial.treatmentId)
+        if (!treatment) {
+          throw new Error(`Treatment not found for trial: ${trial.id}`)
+        }
 
-      const model = experiment.models.find(candidate => {
-        return candidate.name === trial.model.name && candidate.reasoningEffort === trial.model.reasoningEffort
-      })
-      if (!model) {
-        throw new Error(`Model variant not found for trial: ${trial.id}`)
-      }
+        const model = experiment.models.find(candidate => {
+          return candidate.name === trial.model.name && candidate.reasoningEffort === trial.model.reasoningEffort
+        })
+        if (!model) {
+          throw new Error(`Model variant not found for trial: ${trial.id}`)
+        }
 
-      return {
-        id: trial.id,
-        model,
-        runner: trial.runner,
-        scenario,
-        treatment,
-        setup: experiment.setup,
-      }
-    }),
+        return {
+          id: trial.id,
+          model,
+          runner: trial.runner,
+          scenario,
+          treatment,
+          setup: experiment.setup,
+        }
+      }),
+    }
+  } else if (experiment.type === 'benchmark') {
+    const capabilities = new Map(
+      experiment.benchmark.capabilities.map(capability => {
+        return [capability.id, capability]
+      }),
+    )
+    const trialIds = new Set<string>()
+
+    return {
+      experiment,
+      trials: result.trials.map(trial => {
+        if (trialIds.has(trial.id)) {
+          throw new Error(`Duplicate trial ID in experiment plan: ${trial.id}`)
+        }
+        trialIds.add(trial.id)
+
+        const capability = capabilities.get(trial.capabilityId)
+        if (!capability) {
+          throw new Error(`Capability not found for trial: ${trial.id}`)
+        }
+
+        const scenario = capability.scenarios.find(candidate => {
+          return candidate.id === trial.scenarioId
+        })
+        if (!scenario) {
+          throw new Error(`Scenario not found for trial: ${trial.id}`)
+        }
+
+        const model = experiment.models.find(candidate => {
+          return candidate.name === trial.model.name && candidate.reasoningEffort === trial.model.reasoningEffort
+        })
+        if (!model) {
+          throw new Error(`Model variant not found for trial: ${trial.id}`)
+        }
+
+        const treatment = experiment.treatments.find(candidate => {
+          return candidate.id === trial.treatmentId
+        })
+        if (!treatment) {
+          throw new Error(`Treatment not found for trial: ${trial.id}`)
+        }
+
+        return {
+          id: trial.id,
+          model,
+          runner: trial.runner,
+          scenario,
+          capability,
+          treatment,
+          setup: experiment.setup,
+        }
+      }),
+    }
+  } else {
+    exhaustiveCheck(experiment)
   }
 }
 
diff --git a/packages/agent-eval/src/treatment.ts b/packages/agent-eval/src/treatment.ts
index 6861b0ff..5a6f910c 100644
--- a/packages/agent-eval/src/treatment.ts
+++ b/packages/agent-eval/src/treatment.ts
@@ -45,5 +45,23 @@ function getTreatmentId(name: string): string {
   return hash(`Treatment:${name}`)
 }
 
-export {ControlTreatment, TreatmentConfigSchema, TreatmentSchema, TreatmentSetupSchema, createTreatment, getTreatmentId}
+function composeTreatmentSetup(...setups: Array<TreatmentSetup | undefined>): TreatmentSetup {
+  return async function composedSetup(options: TreatmentSetupOptions): Promise<void> {
+    for (const setup of setups) {
+      if (setup) {
+        await setup(options)
+      }
+    }
+  }
+}
+
+export {
+  ControlTreatment,
+  TreatmentConfigSchema,
+  TreatmentSchema,
+  TreatmentSetupSchema,
+  composeTreatmentSetup,
+  createTreatment,
+  getTreatmentId,
+}
 export type {TreatmentConfig, Treatment, TreatmentSetup}

From cdb1f45909e55a1bffc65f158f43ec1784666e8a Mon Sep 17 00:00:00 2001
From: Josh Black <joshblack@github.com>
Date: Fri, 9 Oct 2026 13:22:25 -0500
Subject: [PATCH 2/7] refactor: inline getExperimentTreatments

---
 packages/agent-eval/src/experiment/plan.ts | 59 ++++++++++------------
 1 file changed, 28 insertions(+), 31 deletions(-)

diff --git a/packages/agent-eval/src/experiment/plan.ts b/packages/agent-eval/src/experiment/plan.ts
index 89a4865e..757fc0da 100644
--- a/packages/agent-eval/src/experiment/plan.ts
+++ b/packages/agent-eval/src/experiment/plan.ts
@@ -23,35 +23,9 @@ type CreateExperimentPlanOptions = {
   experiment: Experiment
 }
 
-function getExperimentTreatments(experiment: Experiment, capability?: Capability) {
-  const treatments = new Map([[ControlTreatment.id, ControlTreatment]])
-  if (experiment.type === 'benchmark' && capability) {
-    const treatment = createTreatment({
-      name: 'Benchmark',
-      setup: composeTreatmentSetup(experiment.benchmark.setup, capability.setup),
-    })
-    treatments.set(treatment.id, treatment)
-  }
-  const names = new Set(
-    [...treatments.values()].map(treatment => {
-      return treatment.name
-    }),
-  )
-
-  for (const treatment of experiment.treatments) {
-    if (treatments.has(treatment.id) || names.has(treatment.name)) {
-      throw new Error(`Experiment "${experiment.id}" contains duplicate treatment: ${treatment.name}`)
-    }
-    treatments.set(treatment.id, treatment)
-    names.add(treatment.name)
-  }
-
-  return treatments
-}
-
 function createExperimentPlan({experiment}: CreateExperimentPlanOptions): Plan<ScenarioTrial> | Plan<BenchmarkTrial> {
   if (experiment.type === 'scenario') {
-    const treatments = [...getExperimentTreatments(experiment).values()]
+    const treatments = [ControlTreatment, ...experiment.treatments]
     return createPlan({
       trials: experiment.models.flatMap(model => {
         return experiment.scenarios.flatMap(scenario => {
@@ -75,8 +49,16 @@ function createExperimentPlan({experiment}: CreateExperimentPlanOptions): Plan<S
       trials: experiment.models.flatMap(model => {
         return experiment.runners.flatMap(runner => {
           return experiment.benchmark.capabilities.flatMap(capability => {
+            const treatments = [
+              ControlTreatment,
+              createTreatment({
+                name: 'Benchmark',
+                setup: composeTreatmentSetup(experiment.benchmark.setup, capability.setup),
+              }),
+              ...experiment.treatments,
+            ]
             return capability.scenarios.flatMap(scenario => {
-              return [...getExperimentTreatments(experiment, capability).values()].map(treatment => {
+              return treatments.map(treatment => {
                 return {
                   id: randomUUID(),
                   scenario,
@@ -194,13 +176,14 @@ async function parseExperimentPlanManifest({
     name: result.id,
     scenariosDirectory,
   })
+
   if (experiment.type === 'scenario') {
     const scenarios = new Map(
       experiment.scenarios.map(scenario => {
         return [scenario.id, scenario]
       }),
     )
-    const treatments = getExperimentTreatments(experiment)
+    const treatments = [ControlTreatment, ...experiment.treatments]
     return {
       experiment,
       trials: result.trials.map(trial => {
@@ -209,7 +192,9 @@ async function parseExperimentPlanManifest({
           throw new Error(`Scenario not found for trial: ${trial.id}`)
         }
 
-        const treatment = treatments.get(trial.treatmentId)
+        const treatment = treatments.find(candidate => {
+          return candidate.id === trial.treatmentId
+        })
         if (!treatment) {
           throw new Error(`Treatment not found for trial: ${trial.id}`)
         }
@@ -237,12 +222,14 @@ async function parseExperimentPlanManifest({
         return [capability.id, capability]
       }),
     )
+
     return {
       experiment,
       trials: result.trials.map(trial => {
         if (trial.capabilityId === undefined) {
           throw new Error(`Capability not found for trial: ${trial.id}`)
         }
+
         const capability = capabilities.get(trial.capabilityId)
         if (!capability) {
           throw new Error(`Capability not found for trial: ${trial.id}`)
@@ -262,7 +249,17 @@ async function parseExperimentPlanManifest({
           throw new Error(`Model variant not found for trial: ${trial.id}`)
         }
 
-        const treatment = getExperimentTreatments(experiment, capability).get(trial.treatmentId)
+        const treatments = [
+          ControlTreatment,
+          createTreatment({
+            name: 'Benchmark',
+            setup: composeTreatmentSetup(experiment.benchmark.setup, capability.setup),
+          }),
+          ...experiment.treatments,
+        ]
+        const treatment = treatments.find(candidate => {
+          return candidate.id === trial.treatmentId
+        })
         if (!treatment) {
           throw new Error(`Treatment not found for trial: ${trial.id}`)
         }

From 8bb7586cd4df6022464b04387661d0ce93a05a3d Mon Sep 17 00:00:00 2001
From: Josh Black <joshblack@github.com>
Date: Fri, 9 Oct 2026 13:40:19 -0500
Subject: [PATCH 3/7] feat: add support for benchmark experiments to result

---
 docs/experiments.md                           |  29 +-
 .../agent-eval/src/experiment/report.test.ts  | 283 ++++++++++++++++++
 packages/agent-eval/src/experiment/report.ts  |  82 ++++-
 skills/agent-eval/SKILL.md                    |  26 +-
 skills/agent-eval/references/experiments.md   |  38 ++-
 .../references/trials-and-results.md          |   3 +
 6 files changed, 421 insertions(+), 40 deletions(-)
 create mode 100644 packages/agent-eval/src/experiment/report.test.ts

diff --git a/docs/experiments.md b/docs/experiments.md
index ec85d13d..4083a23c 100644
--- a/docs/experiments.md
+++ b/docs/experiments.md
@@ -2,9 +2,9 @@
 
 Experiments are used to determine how different setups (or treatments) affect the performance of an agent on a given task. By default, they live in an `experiments` folder in your project.
 
-Each experiment defines a set of scenarios that are used to evaluate the
-performance of each treatment. An experiment also defines a set of models to
-use.
+Each experiment defines either a set of scenarios or a benchmark whose
+capabilities and scenarios are used to evaluate each treatment. An experiment
+also defines a set of models to use.
 
 As an example, you may want to design an experiment comparing the performance of
 your MCP server to a skills based approach. With this in mind, you create two
@@ -55,3 +55,26 @@ export default defineConfig({
 You can interact with experiments using the `experiments` subcommand of the `agent-eval` CLI. This sub-command gives you access to run experiments, create run plans to use for sharding, or merge the results of a plan.
 
 Use `agent-eval experiments --help` to see the available commands and options.
+
+## Reports
+
+Experiment runs print a summary grouped by treatment, scenario, and model
+variant. When the experiment selects a `benchmark` instead of `scenarios`, the
+report adds a `Capability` column and capability totals between each treatment
+and its scenarios:
+
+```text
+treatment
+  capability
+    scenario
+      model / reasoning effort
+```
+
+Treatment totals include all completed capabilities. Scenarios and model
+variants are grouped separately within each capability, even when a scenario
+appears in more than one capability. Reports keep runner results separate and
+show a `Runner` column when SDK trials are present.
+
+Each level includes run counts, check summaries, and implementation-agent usage
+totals. Partial or sharded runs show only the groups with completed results.
+Scenario-backed experiments retain their existing report format.
diff --git a/packages/agent-eval/src/experiment/report.test.ts b/packages/agent-eval/src/experiment/report.test.ts
new file mode 100644
index 00000000..9cb606c1
--- /dev/null
+++ b/packages/agent-eval/src/experiment/report.test.ts
@@ -0,0 +1,283 @@
+import {describe, expect, test} from 'vitest'
+import type {CopilotRunner} from '../copilot-runner'
+import {VirtualHost} from '../host'
+import type {RunPlanResult} from '../plan'
+import type {Experiment} from './experiment'
+import {getExperiment} from './get'
+import {createExperimentPlan, type ExperimentTrial} from './plan'
+import {createExperimentReport} from './report'
+
+async function createExperiment(type: Experiment['type'], runners: Array<CopilotRunner> = ['copilot-cli']) {
+  const host = VirtualHost.create({
+    '/experiments/example.ts': `export default ${JSON.stringify({
+      name: 'Example',
+      description: 'Compare treatments',
+      models: [{name: 'gpt-5.5', reasoningEfforts: ['medium', 'high']}],
+      runners,
+      ...(type === 'benchmark' ? {benchmark: 'design-system'} : {scenarios: ['example']}),
+      treatments: [{name: 'Skill'}],
+    })}`,
+    '/benchmarks/design-system.ts': `export default ${JSON.stringify({
+      name: 'Design System',
+      description: 'Evaluate capabilities',
+      models: ['gpt-5.5'],
+      capabilities: [
+        {name: 'Components', scenarios: ['example']},
+        {name: 'Layout', scenarios: ['example']},
+      ],
+    })}`,
+    '/scenarios/example/package.json': '{}',
+    '/scenarios/example/scenario.config.ts': 'export default {prompt: "Create a page"}',
+  })
+
+  return getExperiment({
+    host,
+    benchmarksDirectory: '/benchmarks',
+    experimentsDirectory: '/experiments',
+    scenariosDirectory: '/scenarios',
+    name: 'example',
+  })
+}
+
+function createResult(
+  trial: ExperimentTrial,
+  outputTokens: number,
+  passed = true,
+): RunPlanResult<ExperimentTrial>['results'][number] {
+  return {
+    trial,
+    result: {
+      trial,
+      agent: {
+        sessions: [
+          {
+            turns: 1,
+            outputTokens,
+            premiumRequests: 1,
+            sessionDurationMs: 1000,
+            totalApiDurationMs: 500,
+            tools: {},
+            messages: [],
+          },
+        ],
+      },
+      artifacts: {
+        directory: '/artifacts',
+        copilotConfigDirectory: '/artifacts/copilot',
+        skillsConfigDirectory: '/artifacts/skills',
+        walkthroughDirectory: '/artifacts/walkthrough',
+        workspaceDirectory: '/artifacts/workspace',
+      },
+      checks: [
+        {
+          check: {
+            name: 'Quality',
+            files: [],
+          },
+          result: {
+            type: 'outcomes',
+            outcomes: [
+              {
+                type: 'outcome',
+                status: passed ? 'passed' : 'failed',
+              },
+            ],
+          },
+        },
+      ],
+      judges: [],
+      walkthrough: {
+        type: 'Unavailable',
+      },
+    },
+  }
+}
+
+function readReportRows(report: string): Array<Record<string, string>> {
+  const [header, separator, ...lines] = report.split('\n\n')[0].split('\n')
+  const columns = [...separator.matchAll(/-+/g)].map(match => {
+    return {
+      name: header.slice(match.index, match.index + match[0].length).trim(),
+      start: match.index,
+      end: match.index + match[0].length,
+    }
+  })
+
+  return lines.map(line => {
+    return Object.fromEntries(
+      columns.map(column => {
+        return [column.name, line.slice(column.start, column.end).trim()]
+      }),
+    )
+  })
+}
+
+describe('createExperimentReport', () => {
+  test('preserves scenario experiment rows without capability or runner columns', async () => {
+    const experiment = await createExperiment('scenario')
+    const plan = createExperimentPlan({experiment})
+    const runPlanResult = {
+      results: plan.trials.map(trial => {
+        return createResult(trial, trial.model.reasoningEffort === 'medium' ? 10 : 20)
+      }),
+    }
+
+    const report = createExperimentReport({experiment, runPlanResult})
+    const rows = readReportRows(report)
+
+    expect(rows).toHaveLength(8)
+    expect(Object.keys(rows[0])).not.toContain('Capability')
+    expect(Object.keys(rows[0])).not.toContain('Runner')
+    expect(
+      rows
+        .filter(row => {
+          return row.Experiment === 'Example'
+        })
+        .map(row => {
+          return {
+            treatment: row.Treatment,
+            runs: row.Runs,
+            checks: row.Checks,
+            tokens: row['Output Tokens'],
+          }
+        }),
+    ).toEqual([
+      {treatment: 'Control', runs: '2', checks: '100.0%', tokens: '30'},
+      {treatment: 'Skill', runs: '2', checks: '100.0%', tokens: '30'},
+    ])
+    expect(report).toContain('  example')
+    expect(report).toContain('    gpt-5.5')
+  })
+
+  test('groups benchmark experiments by treatment, runner, capability, scenario, and model', async () => {
+    const experiment = await createExperiment('benchmark', ['copilot-cli', 'copilot-sdk'])
+    const plan = createExperimentPlan({experiment})
+    const weights = new Map([
+      ['Control', 1],
+      ['Benchmark', 2],
+      ['Skill', 3],
+    ])
+    const runPlanResult = {
+      results: plan.trials.map(trial => {
+        if (!('capability' in trial)) {
+          throw new Error('Expected a benchmark trial')
+        }
+        const capabilityWeight = trial.capability.name === 'Components' ? 10 : 100
+        const runnerWeight = trial.runner === 'copilot-cli' ? 1 : 2
+        const modelWeight = trial.model.reasoningEffort === 'medium' ? 1 : 2
+        const treatmentWeight = weights.get(trial.treatment.name)
+        if (treatmentWeight === undefined) {
+          throw new Error(`Unexpected treatment: ${trial.treatment.name}`)
+        }
+        return createResult(
+          trial,
+          capabilityWeight * runnerWeight * modelWeight * treatmentWeight,
+          trial.capability.name === 'Components',
+        )
+      }),
+    }
+
+    const report = createExperimentReport({experiment, runPlanResult})
+    const rows = readReportRows(report)
+    const treatmentRows = rows.filter(row => {
+      return row.Experiment === 'Example'
+    })
+
+    expect(rows).toHaveLength(54)
+    expect(treatmentRows).toHaveLength(6)
+    for (const treatment of treatmentRows) {
+      const treatmentWeight = weights.get(treatment.Treatment)
+      if (treatmentWeight === undefined) {
+        throw new Error(`Unexpected treatment: ${treatment.Treatment}`)
+      }
+      const weight = treatmentWeight * (treatment.Runner === 'copilot-cli' ? 1 : 2)
+      const start = rows.indexOf(treatment)
+      const group = rows.slice(start, start + 9).map(row => {
+        return [
+          row.Capability,
+          row.Scenario,
+          row.Model,
+          row['Reasoning Effort'],
+          row.Runs,
+          row.Checks,
+          row['Output Tokens'],
+        ]
+      })
+
+      expect(group).toEqual([
+        ['All capabilities', 'All scenarios', 'All models', '', '4', '50.0%', (330 * weight).toLocaleString('en-US')],
+        ['Components', 'All scenarios', 'All models', '', '2', '100.0%', (30 * weight).toLocaleString('en-US')],
+        ['', 'example', 'All models', '', '2', '100.0%', (30 * weight).toLocaleString('en-US')],
+        ['', '', 'gpt-5.5', 'medium', '1', '100.0%', (10 * weight).toLocaleString('en-US')],
+        ['', '', 'gpt-5.5', 'high', '1', '100.0%', (20 * weight).toLocaleString('en-US')],
+        ['Layout', 'All scenarios', 'All models', '', '2', '0.0%', (300 * weight).toLocaleString('en-US')],
+        ['', 'example', 'All models', '', '2', '0.0%', (300 * weight).toLocaleString('en-US')],
+        ['', '', 'gpt-5.5', 'medium', '1', '0.0%', (100 * weight).toLocaleString('en-US')],
+        ['', '', 'gpt-5.5', 'high', '1', '0.0%', (200 * weight).toLocaleString('en-US')],
+      ])
+    }
+  })
+
+  test('reports only completed benchmark groups for an SDK-only run without checks', async () => {
+    const experiment = await createExperiment('benchmark', ['copilot-cli', 'copilot-sdk'])
+    const plan = createExperimentPlan({experiment})
+    const selected = plan.trials.filter((trial: ExperimentTrial) => {
+      return (
+        'capability' in trial &&
+        trial.capability.name === 'Components' &&
+        trial.treatment.name === 'Benchmark' &&
+        trial.runner === 'copilot-sdk' &&
+        trial.model.reasoningEffort === 'high'
+      )
+    })
+    const runPlanResult = {
+      results: selected.map(trial => {
+        const entry = createResult(trial, 10)
+        entry.result.checks = []
+        return entry
+      }),
+    }
+
+    const report = createExperimentReport({experiment, runPlanResult})
+    const rows = readReportRows(report)
+
+    expect(rows).toHaveLength(4)
+    expect(rows[0]).toMatchObject({
+      Treatment: 'Benchmark',
+      Runner: 'copilot-sdk',
+      Capability: 'All capabilities',
+      Runs: '1',
+      'Output Tokens': '10',
+      'AI Credits': 'N/A',
+    })
+    expect(rows[1].Capability).toBe('Components')
+    expect(rows[2].Scenario).toBe('example')
+    expect(rows[3]['Reasoning Effort']).toBe('high')
+    expect(Object.keys(rows[0])).not.toContain('Checks')
+    expect(report).not.toContain('Layout')
+    expect(report).not.toContain('Control')
+  })
+
+  test('rejects a benchmark experiment result without capability metadata', async () => {
+    const experiment = await createExperiment('benchmark')
+    const scenarioExperiment = await createExperiment('scenario')
+    const plan = createExperimentPlan({experiment: scenarioExperiment})
+
+    expect(() => {
+      createExperimentReport({
+        experiment,
+        runPlanResult: {
+          results: [createResult(plan.trials[0], 10)],
+        },
+      })
+    }).toThrow(`Capability not found for benchmark experiment trial: ${plan.trials[0].id}`)
+  })
+
+  test.each(['scenario', 'benchmark'] as const)('reports no results for an empty %s experiment run', async type => {
+    const experiment = await createExperiment(type)
+
+    expect(createExperimentReport({experiment, runPlanResult: {results: []}})).toBe(
+      'Experiment: Example\nNo trial results.',
+    )
+  })
+})
diff --git a/packages/agent-eval/src/experiment/report.ts b/packages/agent-eval/src/experiment/report.ts
index e27b74fb..c95915d8 100644
--- a/packages/agent-eval/src/experiment/report.ts
+++ b/packages/agent-eval/src/experiment/report.ts
@@ -1,5 +1,6 @@
 import type {ModelVariant} from '../model'
 import type {CopilotRunner} from '../copilot-runner'
+import type {Capability} from '../benchmark/benchmark'
 import type {RunPlanResult} from '../plan'
 import {formatTable, type TableRow} from '../report/format'
 import {getCheckDimensions, type CheckDimension} from '../report/checks'
@@ -19,6 +20,7 @@ import type {ExperimentTrial} from './plan'
 type ExperimentSummary = TrialSummary & {
   treatmentId: string
   treatment: string
+  capability?: Capability
   scenario?: string
   model?: ModelVariant
   runner: CopilotRunner
@@ -33,6 +35,7 @@ function compareExperimentNames(a: ExperimentSummary, b: ExperimentSummary): num
   return (
     a.treatment.localeCompare(b.treatment) ||
     a.runner.localeCompare(b.runner) ||
+    (a.capability?.name ?? '').localeCompare(b.capability?.name ?? '') ||
     (a.scenario ?? '').localeCompare(b.scenario ?? '') ||
     (a.model?.name ?? '').localeCompare(b.model?.name ?? '') ||
     (a.model?.reasoningEffort ?? '').localeCompare(b.model?.reasoningEffort ?? '')
@@ -49,16 +52,30 @@ function sortExperimentSummaries(summaries: Array<ExperimentSummary>): Array<Exp
 function formatExperimentSummary(
   experiment: Experiment,
   summary: ExperimentSummary,
-  level: 'treatment' | 'scenario' | 'model',
+  level: 'treatment' | 'capability' | 'scenario' | 'model',
   dimensions: Array<CheckDimension>,
   showRunner: boolean,
 ): TableRow {
+  const showCapability = experiment.type === 'benchmark'
+  const scenarioIndent = showCapability ? '    ' : '  '
+  const modelIndent = showCapability ? '      ' : '    '
   return {
     Experiment: level === 'treatment' ? experiment.name : '',
     Treatment: level === 'treatment' ? summary.treatment : '',
     ...(showRunner ? {Runner: level === 'treatment' ? summary.runner : ''} : {}),
-    Scenario: level === 'treatment' ? 'All scenarios' : level === 'scenario' ? `  ${summary.scenario}` : '',
-    Model: level === 'model' ? `    ${summary.model?.name}` : 'All models',
+    ...(showCapability
+      ? {
+          Capability:
+            level === 'treatment' ? 'All capabilities' : level === 'capability' ? `  ${summary.capability?.name}` : '',
+        }
+      : {}),
+    Scenario:
+      level === 'treatment' || level === 'capability'
+        ? 'All scenarios'
+        : level === 'scenario'
+          ? `${scenarioIndent}${summary.scenario}`
+          : '',
+    Model: level === 'model' ? `${modelIndent}${summary.model?.name}` : 'All models',
     'Reasoning Effort': level === 'model' ? (summary.model?.reasoningEffort ?? '') : '',
     ...formatTrialSummary(summary, dimensions),
   }
@@ -70,13 +87,19 @@ function createExperimentReport({experiment, runPlanResult}: CreateExperimentRep
   }
 
   const treatmentSummaries = new Map<string, ExperimentSummary>()
+  const capabilitySummaries = new Map<string, ExperimentSummary>()
   const scenarioSummaries = new Map<string, ExperimentSummary>()
   const modelSummaries = new Map<string, ExperimentSummary>()
+  const showCapability = experiment.type === 'benchmark'
   const showRunner = runPlanResult.results.some(({trial}) => {
     return trial.runner === 'copilot-sdk'
   })
 
   for (const {trial, result} of runPlanResult.results) {
+    if (showCapability && !('capability' in trial)) {
+      throw new Error(`Capability not found for benchmark experiment trial: ${trial.id}`)
+    }
+    const capability = showCapability && 'capability' in trial ? trial.capability : undefined
     const values = {
       treatmentId: trial.treatment.id,
       treatment: trial.treatment.name,
@@ -87,8 +110,19 @@ function createExperimentReport({experiment, runPlanResult}: CreateExperimentRep
     addTrialResultToSummary(treatmentSummary, result)
     treatmentSummaries.set(treatmentKey, treatmentSummary)
 
-    const scenarioValues = {...values, scenario: trial.scenario.id}
-    const scenarioKey = JSON.stringify([trial.treatment.id, values.runner, trial.scenario.id])
+    if (capability) {
+      const capabilityKey = JSON.stringify([trial.treatment.id, values.runner, capability.id])
+      const capabilitySummary = capabilitySummaries.get(capabilityKey) ?? {
+        ...createTrialSummary(),
+        ...values,
+        capability,
+      }
+      addTrialResultToSummary(capabilitySummary, result)
+      capabilitySummaries.set(capabilityKey, capabilitySummary)
+    }
+
+    const scenarioValues = {...values, capability, scenario: trial.scenario.id}
+    const scenarioKey = JSON.stringify([trial.treatment.id, values.runner, capability?.id, trial.scenario.id])
     const scenarioSummary = scenarioSummaries.get(scenarioKey) ?? {...createTrialSummary(), ...scenarioValues}
     addTrialResultToSummary(scenarioSummary, result)
     scenarioSummaries.set(scenarioKey, scenarioSummary)
@@ -96,6 +130,7 @@ function createExperimentReport({experiment, runPlanResult}: CreateExperimentRep
     const modelKey = JSON.stringify([
       trial.treatment.id,
       values.runner,
+      capability?.id,
       trial.scenario.id,
       trial.model.name,
       trial.model.reasoningEffort,
@@ -113,21 +148,39 @@ function createExperimentReport({experiment, runPlanResult}: CreateExperimentRep
   const dimensions = getCheckDimensions([...treatmentSummaries.values()])
   for (const treatment of sortExperimentSummaries([...treatmentSummaries.values()])) {
     rows.push(formatExperimentSummary(experiment, treatment, 'treatment', dimensions, showRunner))
-    const scenarios = [...scenarioSummaries.values()].filter(summary => {
-      return summary.treatmentId === treatment.treatmentId && summary.runner === treatment.runner
-    })
+    const groups = showCapability
+      ? sortExperimentSummaries(
+          [...capabilitySummaries.values()].filter(summary => {
+            return summary.treatmentId === treatment.treatmentId && summary.runner === treatment.runner
+          }),
+        )
+      : [treatment]
 
-    for (const scenario of sortExperimentSummaries(scenarios)) {
-      rows.push(formatExperimentSummary(experiment, scenario, 'scenario', dimensions, showRunner))
-      const models = [...modelSummaries.values()].filter(summary => {
+    for (const group of groups) {
+      if (showCapability) {
+        rows.push(formatExperimentSummary(experiment, group, 'capability', dimensions, showRunner))
+      }
+      const scenarios = [...scenarioSummaries.values()].filter(summary => {
         return (
           summary.treatmentId === treatment.treatmentId &&
           summary.runner === treatment.runner &&
-          summary.scenario === scenario.scenario
+          summary.capability?.id === group.capability?.id
         )
       })
-      for (const model of sortExperimentSummaries(models)) {
-        rows.push(formatExperimentSummary(experiment, model, 'model', dimensions, showRunner))
+
+      for (const scenario of sortExperimentSummaries(scenarios)) {
+        rows.push(formatExperimentSummary(experiment, scenario, 'scenario', dimensions, showRunner))
+        const models = [...modelSummaries.values()].filter(summary => {
+          return (
+            summary.treatmentId === treatment.treatmentId &&
+            summary.runner === treatment.runner &&
+            summary.capability?.id === group.capability?.id &&
+            summary.scenario === scenario.scenario
+          )
+        })
+        for (const model of sortExperimentSummaries(models)) {
+          rows.push(formatExperimentSummary(experiment, model, 'model', dimensions, showRunner))
+        }
       }
     }
   }
@@ -137,6 +190,7 @@ function createExperimentReport({experiment, runPlanResult}: CreateExperimentRep
       'Experiment',
       'Treatment',
       ...(showRunner ? ['Runner'] : []),
+      ...(showCapability ? ['Capability'] : []),
       'Scenario',
       'Model',
       'Reasoning Effort',
diff --git a/skills/agent-eval/SKILL.md b/skills/agent-eval/SKILL.md
index 805abd24..96dc7c6d 100644
--- a/skills/agent-eval/SKILL.md
+++ b/skills/agent-eval/SKILL.md
@@ -13,19 +13,19 @@ to their environment improves the result.
 Read only the references needed for the current task. The overview is context,
 not a prerequisite for every operation.
 
-| Task                                                    | Read                                                                               |
-| :------------------------------------------------------ | :--------------------------------------------------------------------------------- |
-| Install and complete a first evaluation                 | [Getting started](references/getting-started.md)                                   |
-| Understand project structure and evaluation methodology | [Overview](references/overview.md)                                                 |
-| Establish a capability baseline                         | [Benchmarks](references/benchmarks.md), [capabilities](references/capabilities.md) |
-| Compare instructions, skills, agents, MCP, or plugins   | [Experiments](references/experiments.md), [treatments](references/treatments.md)   |
-| Create a task, starting workspace, or custom image      | [Scenarios](references/scenarios.md)                                               |
-| Add deterministic grading or a subjective rubric        | [Checks](references/checks.md) or [judges](references/judges.md)                   |
-| Select models, effort, or execution backends            | [Models and runners](references/models-and-runners.md)                             |
-| Install resources or run commands in a trial            | [Sandbox](references/sandbox.md)                                                   |
-| Inspect a run matrix, shard, or merge                   | [Plans](references/plans.md)                                                       |
-| Read results or diagnose agent behavior                 | [Trials and results](references/trials-and-results.md)                             |
-| Find a command, flag, or runtime failure                | [CLI](references/cli.md)                                                           |
+| Task                                                    | Read                                                                                                            |
+| :------------------------------------------------------ | :-------------------------------------------------------------------------------------------------------------- |
+| Install and complete a first evaluation                 | [Getting started](references/getting-started.md)                                                                |
+| Understand project structure and evaluation methodology | [Overview](references/overview.md)                                                                              |
+| Establish a capability baseline                         | [Benchmarks](references/benchmarks.md), [capabilities](references/capabilities.md)                              |
+| Compare instructions, skills, agents, MCP, or plugins   | [Experiments](references/experiments.md), [treatments](references/treatments.md)                                |
+| Create a task, starting workspace, or custom image      | [Scenarios](references/scenarios.md)                                                                            |
+| Add deterministic grading or a subjective rubric        | [Checks](references/checks.md) or [judges](references/judges.md)                                                |
+| Select models, effort, or execution backends            | [Models and runners](references/models-and-runners.md)                                                          |
+| Install resources or run commands in a trial            | [Sandbox](references/sandbox.md)                                                                                |
+| Inspect a run matrix, shard, or merge                   | [Plans](references/plans.md)                                                                                    |
+| Read results or diagnose agent behavior                 | [Trials and results](references/trials-and-results.md), [experiment reports](references/experiments.md#reports) |
+| Find a command, flag, or runtime failure                | [CLI](references/cli.md)                                                                                        |
 
 ## Execution rules
 
diff --git a/skills/agent-eval/references/experiments.md b/skills/agent-eval/references/experiments.md
index 3b6ce2ec..f50f313a 100644
--- a/skills/agent-eval/references/experiments.md
+++ b/skills/agent-eval/references/experiments.md
@@ -6,22 +6,36 @@ plugins, or execution backends on the same tasks. Use a
 
 ## Contract
 
-| Item                | Rule                                                                       |
-| :------------------ | :------------------------------------------------------------------------- |
-| File and import     | `experiments/<id>.ts`; `defineConfig` from `@primer/agent-eval/experiment` |
-| Export              | Prefer named `experiment`; default also supported                          |
-| Required fields     | `name`, `description`, `models`, `scenarios`, `treatments`                 |
-| Optional fields     | `setup`, `runners`                                                         |
-| CLI identifier      | Filename without extension, not display `name`                             |
-| Scenario references | Folder IDs; alternatively `{path, name?}`                                  |
-| Runner default      | `copilot-cli`                                                              |
+| Item                | Rule                                                                                 |
+| :------------------ | :----------------------------------------------------------------------------------- |
+| File and import     | `experiments/<id>.ts`; `defineConfig` from `@primer/agent-eval/experiment`           |
+| Export              | Prefer named `experiment`; default also supported                                    |
+| Required fields     | `name`, `description`, `models`, `treatments`, and either `scenarios` or `benchmark` |
+| Optional fields     | `setup`, `runners`                                                                   |
+| CLI identifier      | Filename without extension, not display `name`                                       |
+| Scenario references | Folder IDs; alternatively `{path, name?}`                                            |
+| Benchmark reference | Benchmark filename ID; selects its capabilities and scenarios                        |
+| Runner default      | `copilot-cli`                                                                        |
 
 Top-level `setup` runs for **all trials, including control**. Put only neutral
 prerequisites there. Treatment setup runs afterward and installs the intervention.
 Do not put the tested resource in shared setup or the fixture.
 
 The harness adds `Control` automatically. Treatment names must be unique and
-cannot be `Control`. `treatments: []` runs control only.
+cannot be `Control`. For scenario-backed experiments, `treatments: []` runs
+control only. Benchmark-backed experiments also add a `Benchmark` treatment
+using the benchmark and capability setup; `Benchmark` is also a reserved name.
+
+## Reports
+
+CLI reports group scenario experiments by treatment, scenario, and model/effort.
+Benchmark-backed experiments add capability totals under each treatment:
+treatment, capability, scenario, then model/effort. Shared scenarios stay
+separate across capabilities. Runner groups remain separate, with a `Runner`
+column when SDK trials are present.
+
+Each level shows run counts, check summaries, and implementation-agent usage.
+Partial runs show only completed groups, not empty configured capabilities.
 
 ## Minimal example
 
@@ -80,6 +94,10 @@ The matrix expands as follows:
 trials = model variants * scenarios * unique runners * (configured treatments + 1)
 ```
 
+For benchmark-backed experiments, sum scenario counts across capabilities and
+use `(configured treatments + 2)` for the automatic `Control` and `Benchmark`
+treatments. A scenario shared by two capabilities runs once per capability.
+
 `runners: ['copilot-cli', 'copilot-sdk']` adds a backend comparison. Changing both
 runner and resource confounds their effects; see [models and runners](models-and-runners.md).
 
diff --git a/skills/agent-eval/references/trials-and-results.md b/skills/agent-eval/references/trials-and-results.md
index e496ea76..d605de98 100644
--- a/skills/agent-eval/references/trials-and-results.md
+++ b/skills/agent-eval/references/trials-and-results.md
@@ -89,6 +89,9 @@ verified pricing basis.
 - Support causal explanations with saved workspace and session evidence.
 
 Compare like-for-like scenario/model/effort/runner groups across treatments.
+For benchmark-backed experiments, CLI reports include capability totals under
+each treatment. Compare scenario/model groups within the same capability;
+shared scenarios are reported separately for each capability.
 Look at granular regressions before an aggregate score. Explain failures using
 saved source and sessions rather than speculating from a summary.
 

From 9362f5325735e60b43b3db18036f85f1a82e3830 Mon Sep 17 00:00:00 2001
From: Josh Black <joshblack@github.com>
Date: Fri, 9 Oct 2026 14:41:53 -0500
Subject: [PATCH 4/7] feat(website): add support for visualizing benchmark
 experiment results

---
 docs/experiments.md                           |  11 ++
 .../src/__snapshots__/package.test.ts.snap    |   1 +
 packages/agent-eval/src/benchmark/output.ts   |   3 +-
 .../agent-eval/src/experiment/output.test.ts  | 108 ++++++++++++++++++
 packages/agent-eval/src/experiment/output.ts  |  73 +++++++++++-
 packages/agent-eval/src/index.ts              |   2 +-
 skills/agent-eval/references/experiments.md   |   9 ++
 .../references/trials-and-results.md          |   8 +-
 .../src/app/components/ExperimentResults.tsx  |  88 +++++++++++---
 website/src/app/components/RunDetailsView.tsx |   2 +-
 .../app/components/ScenarioAnchors.test.tsx   |  46 ++++++++
 website/src/experiment-results.test.ts        |  49 +++++++-
 website/src/experiment-results.ts             |  65 ++++++++---
 website/src/result-files.test.ts              |  45 ++++++++
 website/src/result-files.ts                   |   6 +
 website/src/run-details.ts                    |  19 +--
 website/src/runs.test.ts                      |   2 +
 website/src/scenario-anchor.ts                |   6 +-
 website/src/test-fixtures.ts                  |   1 +
 19 files changed, 487 insertions(+), 57 deletions(-)

diff --git a/docs/experiments.md b/docs/experiments.md
index 4083a23c..59d1dd02 100644
--- a/docs/experiments.md
+++ b/docs/experiments.md
@@ -78,3 +78,14 @@ show a `Runner` column when SDK trials are present.
 Each level includes run counts, check summaries, and implementation-agent usage
 totals. Partial or sharded runs show only the groups with completed results.
 Scenario-backed experiments retain their existing report format.
+
+On the website, experiment pages show overall treatment summaries, then
+capability summaries with their scenario results. Runner, model, and reasoning
+effort groups remain separate. Website resource usage is averaged per trial.
+Scenario output links open the matching capability and scenario on the run
+detail page, which also supports filtering by capability.
+
+Benchmark-backed result bundles preserve capability metadata in `output.json`
+and a `capabilityId` on each trial. Older experiment bundles without this
+metadata remain readable, but their results stay grouped by scenario because
+capability membership cannot be inferred reliably.
diff --git a/packages/agent-eval/src/__snapshots__/package.test.ts.snap b/packages/agent-eval/src/__snapshots__/package.test.ts.snap
index 3929a7bc..f435da64 100644
--- a/packages/agent-eval/src/__snapshots__/package.test.ts.snap
+++ b/packages/agent-eval/src/__snapshots__/package.test.ts.snap
@@ -11,6 +11,7 @@ exports[`@primer/agent-eval exports 1`] = `
   "listExperiments",
   "ExperimentOutputFileSchema",
   "ExperimentTrialOutputSchema",
+  "parseExperimentTrialOutput",
   "getScenario",
   "listScenarios",
   "addCheckResults",
diff --git a/packages/agent-eval/src/benchmark/output.ts b/packages/agent-eval/src/benchmark/output.ts
index bce7f540..f4899fa0 100644
--- a/packages/agent-eval/src/benchmark/output.ts
+++ b/packages/agent-eval/src/benchmark/output.ts
@@ -399,10 +399,11 @@ async function listBenchmarkOutputFiles({
 export {
   BenchmarkOutputFileSchema,
   BenchmarkTrialOutputSchema,
+  CapabilityOutputSchema,
   parseBenchmarkTrialOutput,
   createBenchmarkOutput,
   listBenchmarkOutputFiles,
   mergeBenchmarkOutputFiles,
   writeBenchmarkOutput,
 }
-export type {BenchmarkOutput, BenchmarkTrialOutput}
+export type {BenchmarkOutput, BenchmarkTrialOutput, CapabilityOutput}
diff --git a/packages/agent-eval/src/experiment/output.test.ts b/packages/agent-eval/src/experiment/output.test.ts
index e8b10c52..36263c42 100644
--- a/packages/agent-eval/src/experiment/output.test.ts
+++ b/packages/agent-eval/src/experiment/output.test.ts
@@ -1,10 +1,17 @@
 import {expect, expectTypeOf, test} from 'vitest'
 import type {CopilotRunner} from '../copilot-runner'
 import {VirtualHost} from '../host'
+import {RunTrialResultSchema} from '../trial/run'
+import {getExperiment} from './get'
+import {createExperimentPlan} from './plan'
 import {
   ExperimentOutputFileSchema,
   ExperimentTrialOutputSchema,
+  createExperimentOutput,
+  listExperimentOutputFiles,
   mergeExperimentOutputFiles,
+  parseExperimentTrialOutput,
+  writeExperimentOutput,
   type ExperimentTrialOutput,
 } from './output'
 
@@ -61,6 +68,7 @@ test.each([null, 'unknown'])('rejects an invalid experiment output runner: %s',
 function createOutputFile(trials: Record<string, string> = {}) {
   return {
     id: 'example',
+    capabilities: {},
     scenarios: {
       example: {
         id: 'example',
@@ -261,3 +269,103 @@ test('merges compatible experiment shards with repeated metadata and distinct tr
   expect(output.scenarios.size).toBe(1)
   expect(output.treatments.size).toBe(1)
 })
+
+test('preserves benchmark capabilities and trial membership through writing and merging shards', async () => {
+  const host = VirtualHost.create({
+    '/experiments/example.ts':
+      'export default {name: "Example", description: "Compare a benchmark", models: ["gpt-5.5"], benchmark: "example", treatments: []}',
+    '/benchmarks/example.ts':
+      'export default {name: "Example", description: "Shared scenarios", models: ["gpt-5.5"], capabilities: [{name: "Components", scenarios: ["example"]}, {name: "Layout", scenarios: ["example"]}]}',
+    '/scenarios/example/package.json': '{}',
+    '/scenarios/example/scenario.config.ts': 'export default {prompt: "Build a page"}',
+  })
+  const experiment = await getExperiment({
+    host,
+    name: 'example',
+    experimentsDirectory: '/experiments',
+    benchmarksDirectory: '/benchmarks',
+    scenariosDirectory: '/scenarios',
+  })
+  const plan = createExperimentPlan({experiment})
+  const results = plan.trials.map(entry => {
+    return {
+      trial: entry,
+      result: RunTrialResultSchema.parse({
+        ...trial,
+        trial: entry,
+        artifacts: {
+          ...trial.artifacts,
+          directory: `/results/artifacts/${entry.id}`,
+        },
+      }),
+    }
+  })
+  const expected = createExperimentOutput({experiment, runPlanResult: {results}})
+
+  for (const [index, shard] of [results.slice(0, 2), results.slice(2)].entries()) {
+    await writeExperimentOutput({
+      host,
+      output: createExperimentOutput({experiment, runPlanResult: {results: shard}}),
+      outputPath: `/results/output-${index + 1}.json`,
+    })
+  }
+  const files = await listExperimentOutputFiles({host, outputDirectory: '/results'})
+  const merged = await mergeExperimentOutputFiles({
+    host,
+    outputDirectory: '/results',
+    outputs: files.map(([output]) => {
+      return output
+    }),
+  })
+
+  expect(merged).toEqual(expected)
+  expect(
+    [...merged.capabilities.values()]
+      .map(capability => {
+        return capability.name
+      })
+      .sort(),
+  ).toEqual(['Components', 'Layout'])
+  expect(
+    [...merged.trials.values()].every(entry => {
+      return entry.capabilityId !== undefined && merged.capabilities.has(entry.capabilityId)
+    }),
+  ).toBe(true)
+})
+
+test('defaults legacy scenario manifests to no capabilities', () => {
+  const legacy: Record<string, unknown> = createOutputFile()
+  delete legacy.capabilities
+
+  expect(ExperimentOutputFileSchema.parse(legacy).capabilities).toEqual({})
+})
+
+test.each([
+  {capabilityId: undefined, scenarioId: 'example'},
+  {capabilityId: 'unknown', scenarioId: 'example'},
+  {capabilityId: 'components', scenarioId: 'wrong-scenario'},
+])('rejects invalid benchmark experiment capability membership: %j', fields => {
+  expect(() => {
+    parseExperimentTrialOutput(
+      {...trial, ...fields},
+      new Map([['components', {id: 'components', name: 'Components', scenarioIds: ['example']}]]),
+    )
+  }).toThrow('Invalid capability')
+})
+
+test('rejects conflicting capability metadata before reading experiment shard artifacts', async () => {
+  const capability = {id: 'components', name: 'Components', scenarioIds: ['example']}
+
+  await expect(
+    mergeExperimentOutputFiles({
+      host: VirtualHost.create(),
+      outputDirectory: '/results',
+      outputs: [
+        {...createOutputFile({trial: 'missing.json'}), capabilities: {components: capability}},
+        {...createOutputFile(), capabilities: {components: {...capability, name: 'Renamed'}}},
+      ],
+    }),
+  ).rejects.toMatchObject({
+    issues: [{path: [1, 'capabilities', 'components']}],
+  })
+})
diff --git a/packages/agent-eval/src/experiment/output.ts b/packages/agent-eval/src/experiment/output.ts
index 8b70ef1e..af4979e7 100644
--- a/packages/agent-eval/src/experiment/output.ts
+++ b/packages/agent-eval/src/experiment/output.ts
@@ -2,6 +2,7 @@ import path from 'node:path'
 import {isDeepStrictEqual} from 'node:util'
 import * as z from 'zod/mini'
 import {CopilotRunnerSchema} from '../copilot-runner'
+import {CapabilityOutputSchema, type CapabilityOutput} from '../benchmark/output'
 import {DefaultHost, type Host} from '../host'
 import {ModelVariantSchema} from '../model'
 import type {RunPlanResult} from '../plan'
@@ -21,6 +22,7 @@ import type {ExperimentTrial} from './plan'
 const ExperimentTrialOutputSchema = z.object({
   agent: TrialAgentSchema,
   artifacts: TrialArtifactsSchema,
+  capabilityId: z.optional(z.string()),
   checks: z._default(TrialChecksSchema, []),
   id: z.string(),
   judges: TrialJudgesSchema,
@@ -53,6 +55,21 @@ type TreatmentOutput = z.infer<typeof TreatmentOutputSchema>
 
 const ExperimentOutputFileSchema = z.object({
   id: z.string(),
+  capabilities: z._default(
+    z.record(z.string(), CapabilityOutputSchema).check(ctx => {
+      for (const [key, capability] of Object.entries(ctx.value)) {
+        if (capability.id !== key) {
+          ctx.issues.push({
+            code: 'custom',
+            message: `Manifest capability ID "${capability.id}" does not match key "${key}"`,
+            path: [key, 'id'],
+            input: capability.id,
+          })
+        }
+      }
+    }),
+    {},
+  ),
   scenarios: z.record(z.string(), ScenarioOutputSchema).check(ctx => {
     for (const [key, scenario] of Object.entries(ctx.value)) {
       if (scenario.id !== key) {
@@ -87,6 +104,7 @@ const ExperimentOutputFilesSchema = z
   .check(z.minLength(1, 'Cannot merge experiment output files: no outputs provided'), ctx => {
     const id = ctx.value[0]?.id
     const trialIds = new Set<string>()
+    const capabilities = new Map<string, CapabilityOutput>()
     const scenarios = new Map<string, ScenarioOutput>()
     const treatments = new Map<string, TreatmentOutput>()
 
@@ -100,6 +118,18 @@ const ExperimentOutputFilesSchema = z
         })
       }
 
+      for (const [key, capability] of Object.entries(output.capabilities)) {
+        if (capabilities.has(key) && !isDeepStrictEqual(capabilities.get(key), capability)) {
+          ctx.issues.push({
+            code: 'custom',
+            message: `Cannot merge conflicting capability metadata for id: ${key}`,
+            path: [index, 'capabilities', key],
+            input: capability,
+          })
+        }
+        capabilities.set(key, capability)
+      }
+
       for (const [key, scenario] of Object.entries(output.scenarios)) {
         if (scenarios.has(key) && !isDeepStrictEqual(scenarios.get(key), scenario)) {
           ctx.issues.push({
@@ -140,11 +170,29 @@ const ExperimentOutputFilesSchema = z
 
 type ExperimentOutput = {
   id: string
+  capabilities: Map<string, CapabilityOutput>
   scenarios: Map<string, ScenarioOutput>
   treatments: Map<string, TreatmentOutput>
   trials: Map<string, ExperimentTrialOutput>
 }
 
+function parseExperimentTrialOutput(
+  json: unknown,
+  capabilities: ExperimentOutput['capabilities'],
+): ExperimentTrialOutput {
+  const trial = ExperimentTrialOutputSchema.parse(json)
+  if (trial.capabilityId === undefined && capabilities.size === 0) {
+    return trial
+  }
+  const capability = trial.capabilityId === undefined ? undefined : capabilities.get(trial.capabilityId)
+  if (!capability || capability.id !== trial.capabilityId || !capability.scenarioIds.includes(trial.scenarioId)) {
+    throw new Error(
+      `Invalid capability "${trial.capabilityId}" for scenario "${trial.scenarioId}" in trial "${trial.id}"`,
+    )
+  }
+  return trial
+}
+
 type CreateExperimentOutputOptions = {
   experiment: Experiment
   runPlanResult: RunPlanResult<ExperimentTrial>
@@ -153,12 +201,26 @@ type CreateExperimentOutputOptions = {
 function createExperimentOutput({experiment, runPlanResult}: CreateExperimentOutputOptions): ExperimentOutput {
   const result: ExperimentOutput = {
     id: experiment.id,
+    capabilities: new Map(),
     scenarios: new Map(),
     treatments: new Map(),
     trials: new Map(),
   }
 
   for (const {trial, result: trialResult} of runPlanResult.results) {
+    if (experiment.type === 'benchmark' && !('capability' in trial)) {
+      throw new Error(`Capability not found for benchmark experiment trial: ${trial.id}`)
+    }
+    const capability = 'capability' in trial ? trial.capability : undefined
+    if (capability && !result.capabilities.has(capability.id)) {
+      result.capabilities.set(capability.id, {
+        id: capability.id,
+        name: capability.name,
+        scenarioIds: capability.scenarios.map(scenario => {
+          return scenario.id
+        }),
+      })
+    }
     if (!result.scenarios.has(trial.scenario.id)) {
       result.scenarios.set(trial.scenario.id, ScenarioOutputSchema.parse(trial.scenario))
     }
@@ -173,6 +235,7 @@ function createExperimentOutput({experiment, runPlanResult}: CreateExperimentOut
     result.trials.set(trial.id, {
       agent: trialResult.agent,
       artifacts: trialResult.artifacts,
+      ...(capability ? {capabilityId: capability.id} : {}),
       checks: trialResult.checks,
       id: trial.id,
       judges: trialResult.judges,
@@ -199,12 +262,16 @@ async function mergeExperimentOutputFiles({
   outputDirectory,
 }: MergeExperimentOutputFilesOptions): Promise<ExperimentOutput> {
   const files = ExperimentOutputFilesSchema.parse(outputs)
+  const capabilities = new Map<string, CapabilityOutput>()
   const scenarios = new Map<string, ScenarioOutput>()
   const treatments = new Map<string, TreatmentOutput>()
   const trials = new Map<string, ExperimentTrialOutput>()
   const id = files[0].id
 
   for (const output of files) {
+    for (const [key, value] of Object.entries(output.capabilities)) {
+      capabilities.set(key, value)
+    }
     for (const [key, value] of Object.entries(output.scenarios)) {
       scenarios.set(key, value)
     }
@@ -216,7 +283,7 @@ async function mergeExperimentOutputFiles({
       const filepath = await resolveTrialArtifactsPath(host, outputDirectory, value)
 
       const contents = await host.fs.readFile(filepath, 'utf-8')
-      const trialOutput = ExperimentTrialOutputSchema.parse(JSON.parse(contents))
+      const trialOutput = parseExperimentTrialOutput(JSON.parse(contents), new Map(Object.entries(output.capabilities)))
       if (trialOutput.id !== key) {
         throw new Error(`Cannot merge experiment output files: mismatched trial ID for: ${key}`)
       }
@@ -225,7 +292,7 @@ async function mergeExperimentOutputFiles({
     }
   }
 
-  return {id, scenarios, treatments, trials}
+  return {id, capabilities, scenarios, treatments, trials}
 }
 
 type WriteExperimentOutputOptions = {
@@ -249,6 +316,7 @@ async function writeExperimentOutput({host = DefaultHost, output, outputPath}: W
 
   const experimentFile: ExperimentOutputFile = {
     id: output.id,
+    capabilities: Object.fromEntries(output.capabilities),
     scenarios: Object.fromEntries(output.scenarios),
     treatments: Object.fromEntries(output.treatments),
     trials: Object.fromEntries(trials),
@@ -290,6 +358,7 @@ async function listExperimentOutputFiles({
 export {
   ExperimentOutputFileSchema,
   ExperimentTrialOutputSchema,
+  parseExperimentTrialOutput,
   createExperimentOutput,
   listExperimentOutputFiles,
   mergeExperimentOutputFiles,
diff --git a/packages/agent-eval/src/index.ts b/packages/agent-eval/src/index.ts
index bace22db..a5d4da68 100644
--- a/packages/agent-eval/src/index.ts
+++ b/packages/agent-eval/src/index.ts
@@ -7,7 +7,7 @@ export {getExperiment} from './experiment/get'
 export {listExperiments} from './experiment/list'
 export type {Experiment} from './experiment/experiment'
 export type {CopilotRunner} from './copilot-runner'
-export {ExperimentOutputFileSchema, ExperimentTrialOutputSchema} from './experiment/output'
+export {ExperimentOutputFileSchema, ExperimentTrialOutputSchema, parseExperimentTrialOutput} from './experiment/output'
 export type {ExperimentOutput, ExperimentTrialOutput} from './experiment/output'
 export type {Logger} from './logger'
 export type {Sandbox} from './sandbox'
diff --git a/skills/agent-eval/references/experiments.md b/skills/agent-eval/references/experiments.md
index f50f313a..6414c3f7 100644
--- a/skills/agent-eval/references/experiments.md
+++ b/skills/agent-eval/references/experiments.md
@@ -37,6 +37,15 @@ column when SDK trials are present.
 Each level shows run counts, check summaries, and implementation-agent usage.
 Partial runs show only completed groups, not empty configured capabilities.
 
+Website experiment pages show treatment summaries, capability summaries, and
+scenario results within each capability. Usage is averaged per trial, with
+runner/model/effort groups kept separate. Output links target the matching
+capability and scenario on the run detail page.
+
+New benchmark-backed experiment bundles include capability metadata and trial
+membership. Older bundles lacking that metadata remain readable as flat
+scenario results; do not infer membership from current configuration.
+
 ## Minimal example
 
 Prerequisite: create `001-labels` from [getting started](getting-started.md).
diff --git a/skills/agent-eval/references/trials-and-results.md b/skills/agent-eval/references/trials-and-results.md
index d605de98..7892d6ea 100644
--- a/skills/agent-eval/references/trials-and-results.md
+++ b/skills/agent-eval/references/trials-and-results.md
@@ -41,7 +41,7 @@ artifacts/
 
 `output.json` is a manifest. Its `trials` object maps IDs to relative trial JSON
 paths, not embedded trial results. It includes scenario and treatment metadata;
-benchmark output also includes capability metadata.
+benchmark and benchmark-backed experiment output also include capability metadata.
 
 Resolve each trial file path relative to the output directory. Trial files
 include model, runner, treatment/scenario identifiers, checks, judges, agent
@@ -49,6 +49,12 @@ sessions, walkthrough information, and artifact locations. Preserve the whole
 bundle when sharing or archiving. Embedded artifact paths can be absolute;
 do not assume every metadata path relocates automatically.
 
+Benchmark-backed experiment trials include `capabilityId`. The package root
+exposes `parseExperimentTrialOutput(json, capabilities)` to check membership
+against the bundle's capability map. Scenario-only and legacy experiment
+bundles default to an empty capability map; missing historical membership is
+not inferred.
+
 Standalone `scenario run` writes a different output: `{id, results}`, with
 entries containing `trial: {id}` and an embedded `result`. Do not parse it as a
 benchmark/experiment manifest.
diff --git a/website/src/app/components/ExperimentResults.tsx b/website/src/app/components/ExperimentResults.tsx
index 5e6aebde..67e4849c 100644
--- a/website/src/app/components/ExperimentResults.tsx
+++ b/website/src/app/components/ExperimentResults.tsx
@@ -25,6 +25,9 @@ function formatCredits(value: number): string {
 
 function TreatmentResultsTable({results, label}: {results: Array<TreatmentResult>; label: string}) {
   const labelId = useId()
+  const showRunner = results.some(result => {
+    return result.runner === 'copilot-sdk'
+  })
   return (
     <Table.Container>
       <span className="sr-only" id={labelId}>
@@ -35,6 +38,15 @@ function TreatmentResultsTable({results, label}: {results: Array<TreatmentResult
         cellPadding="condensed"
         columns={[
           {id: 'treatment', header: 'Treatment', field: 'treatment', rowHeader: true},
+          ...(showRunner
+            ? [
+                {
+                  id: 'runner',
+                  header: 'Runner',
+                  field: 'runner' as const,
+                },
+              ]
+            : []),
           {id: 'model', header: 'Model', field: 'model'},
           {id: 'effort', header: 'Effort', field: 'reasoningEffort'},
           {id: 'trials', header: 'Trials', field: 'trials', align: 'end'},
@@ -101,8 +113,8 @@ function MetricsDescription() {
   return (
     <p className="text-muted">
       Checks show per-trial pass percentages or measurement means, averaged across trials. Skips, errors, and missing
-      values are reported separately. Resource usage is the average per trial. Treatments are grouped by model and
-      reasoning effort; compare scenario and trial counts before comparing performance.
+      values are reported separately. Resource usage is the average per trial. Treatments are grouped by runner, model,
+      and reasoning effort; compare scenario and trial counts before comparing performance.
     </p>
   )
 }
@@ -130,7 +142,7 @@ export function ExperimentsOverview({experiments}: {experiments: Array<Experimen
                 <Link href={`/experiments/${experiment.id}/runs/${experiment.date}` as Route}>
                   <time dateTime={experiment.date}>{experiment.date}</time>
                 </Link>
-                . <Link href={`/experiments/${experiment.id}`}>View scenario results and run history</Link>
+                . <Link href={`/experiments/${experiment.id}`}>View detailed results and run history</Link>
               </p>
             ) : null}
             {experiment.treatments.length > 0 ? (
@@ -179,28 +191,66 @@ export function LatestExperimentResults({id, results}: {id: string; results: Exp
           <Blankslate.Description>No trial results were recorded in the latest run.</Blankslate.Description>
         </Blankslate>
       )}
-      <h3 className="text-title-small">Scenario results</h3>
-      {results.scenarios.map(scenario => {
+      {results.capabilities.length > 0 ? <h3 className="text-title-small">Capability results</h3> : null}
+      {results.capabilities.map(capability => {
         return (
-          <section className="flex flex-col gap-3" key={scenario.id}>
-            <h4 className="text-title-small">{scenario.id}</h4>
-            {scenario.treatments.length > 0 ? (
-              <>
-                <TreatmentResultsTable label={`Treatment results for ${scenario.id}`} results={scenario.treatments} />
-                <p>
-                  <Link
-                    href={`/experiments/${id}/runs/${results.date}${getScenarioAnchor(scenario.id).fragment}` as Route}
-                  >
-                    View output for {scenario.id}
-                  </Link>
-                </p>
-              </>
+          <section className="flex flex-col gap-4" key={capability.id}>
+            <h4 className="text-title-small">{capability.name}</h4>
+            {capability.treatments.length > 0 ? (
+              <TreatmentResultsTable
+                label={`Treatment results for ${capability.name}`}
+                results={capability.treatments}
+              />
             ) : (
-              <p>No trial results were recorded for this scenario.</p>
+              <p>No trial results were recorded for this capability.</p>
             )}
+            <ScenarioResults id={id} date={results.date} scenarios={capability.scenarios} capability={capability} />
           </section>
         )
       })}
+      {results.capabilities.length === 0 ? <h3 className="text-title-small">Scenario results</h3> : null}
+      <ScenarioResults id={id} date={results.date} scenarios={results.scenarios} />
     </section>
   )
 }
+
+function ScenarioResults({
+  id,
+  date,
+  scenarios,
+  capability,
+}: {
+  id: string
+  date: string
+  scenarios: ExperimentResults['scenarios']
+  capability?: {id: string; name: string}
+}) {
+  return scenarios.map(scenario => {
+    const label = capability ? `${capability.name} / ${scenario.id}` : scenario.id
+    return (
+      <section className="flex flex-col gap-3" key={scenario.id}>
+        {capability ? (
+          <h5 className="text-title-small">{scenario.id}</h5>
+        ) : (
+          <h4 className="text-title-small">{scenario.id}</h4>
+        )}
+        {scenario.treatments.length > 0 ? (
+          <>
+            <TreatmentResultsTable label={`Treatment results for ${label}`} results={scenario.treatments} />
+            <p>
+              <Link
+                href={
+                  `/experiments/${id}/runs/${date}${getScenarioAnchor(scenario.id, capability?.id).fragment}` as Route
+                }
+              >
+                View output for {label}
+              </Link>
+            </p>
+          </>
+        ) : (
+          <p>No trial results were recorded for this scenario.</p>
+        )}
+      </section>
+    )
+  })
+}
diff --git a/website/src/app/components/RunDetailsView.tsx b/website/src/app/components/RunDetailsView.tsx
index 1bff91ee..467fb604 100644
--- a/website/src/app/components/RunDetailsView.tsx
+++ b/website/src/app/components/RunDetailsView.tsx
@@ -444,7 +444,7 @@ function ScenarioResults({group, index}: {group: ScenarioResultGroup; index: num
     <article
       aria-labelledby={resultHeadingId}
       className="flex flex-col gap-4"
-      id={group.capability ? `capability-scenario-${index}` : getScenarioAnchor(group.scenarioId).id}
+      id={getScenarioAnchor(group.scenarioId, group.capability?.id).id}
     >
       <header className="border-b border-default pb-3 flex flex-col sm:flex-row sm:items-end sm:justify-between gap-4">
         <h2 className="text-title-medium m-0" id={resultHeadingId}>
diff --git a/website/src/app/components/ScenarioAnchors.test.tsx b/website/src/app/components/ScenarioAnchors.test.tsx
index 8bfc869e..6e60d100 100644
--- a/website/src/app/components/ScenarioAnchors.test.tsx
+++ b/website/src/app/components/ScenarioAnchors.test.tsx
@@ -19,6 +19,7 @@ test.each(['001-button', 'space / literal%20 # caf\u00e9'])(
     const overview = renderToStaticMarkup(
       <LatestExperimentResults id="noop" results={getExperimentResults(createRun([result]))} />,
     )
+
     const details = renderToStaticMarkup(
       <RunDetailsPage
         resource={{
@@ -39,3 +40,48 @@ test.each(['001-button', 'space / literal%20 # caf\u00e9'])(
     expect(target).not.toMatch(/\s/)
   },
 )
+
+test('links shared experiment scenarios to distinct capability targets and shows separate runner rows', async () => {
+  const results = [
+    createResult({id: 'first', capabilityId: 'a'}),
+    createResult({id: 'sdk', capabilityId: 'a', runner: 'copilot-sdk'}),
+    createResult({id: 'second', capabilityId: 'b'}),
+  ]
+  const output = createRun(results).output
+  output.capabilities = new Map([
+    ['a', {id: 'a', name: 'Components', scenarioIds: ['scenario-a']}],
+    ['b', {id: 'b', name: 'Layout', scenarioIds: ['scenario-a']}],
+  ])
+  const run = await createExperimentRunDetails('2026-09-10', output)
+  const summary = getExperimentResults({...createRun(results), output})
+  const overview = renderToStaticMarkup(<LatestExperimentResults id="noop" results={summary} />)
+  const details = renderToStaticMarkup(
+    <RunDetailsPage
+      resource={{
+        id: 'noop',
+        name: 'Example experiment',
+        collectionLabel: 'Experiments',
+        collectionHref: '/experiments',
+        href: '/experiments/noop' as Route,
+      }}
+      run={run}
+    />,
+  )
+  const links = [...overview.matchAll(/href="([^"]+#capability-scenario-[^"]+)"/g)]
+  const targets = [...details.matchAll(/<article[^>]+id="([^"]+)"/g)].map(match => {
+    return match[1]
+  })
+
+  expect(overview).toContain('Capability results')
+  expect(overview).toContain('copilot-sdk')
+  expect(overview).toContain('Runner')
+  expect(links).toHaveLength(2)
+  expect(targets).toHaveLength(2)
+  expect(new Set(targets).size).toBe(2)
+  for (const [, href] of links) {
+    expect(targets).toContain(new URL(href, 'https://example.test').hash.slice(1))
+  }
+  expect(details).toContain('All capabilities')
+  expect(details).toContain('Components / scenario-a')
+  expect(details).toContain('Layout / scenario-a')
+})
diff --git a/website/src/experiment-results.test.ts b/website/src/experiment-results.test.ts
index 60fe4e48..a7e81fe8 100644
--- a/website/src/experiment-results.test.ts
+++ b/website/src/experiment-results.test.ts
@@ -8,6 +8,7 @@ test('distinguishes missing runs from runs with no trials', () => {
   expect(getExperimentResults(createRun([]))).toEqual({
     date: '2026-09-10',
     treatments: [],
+    capabilities: [],
     scenarios: [
       {id: 'scenario-a', treatments: []},
       {id: 'scenario-b', treatments: []},
@@ -56,8 +57,9 @@ test('averages checks and implementation usage per trial, not per scenario or ou
 
   expect(summary?.treatments).toEqual([
     {
-      id: JSON.stringify(['control', 'gpt-5.6-sol', 'medium']),
+      id: JSON.stringify(['control', 'copilot-cli', 'gpt-5.6-sol', 'medium']),
       treatment: 'Control',
+      runner: 'copilot-cli',
       model: 'gpt-5.6-sol',
       reasoningEffort: 'medium',
       trials: 3,
@@ -121,3 +123,48 @@ test('preserves recorded scenario and treatment IDs even when metadata is absent
     treatments: [{treatment: 'historical-treatment'}],
   })
 })
+
+test('keeps shared scenarios separate across capabilities and runners', () => {
+  const first = createResult({id: 'components-cli', capabilityId: 'components'})
+  const run = createRun([
+    first,
+    createResult({
+      id: 'components-sdk',
+      capabilityId: 'components',
+      runner: 'copilot-sdk',
+      agent: {sessions: [{...first.agent.sessions[0], outputTokens: 300}]},
+    }),
+    createResult({
+      id: 'layout-cli',
+      capabilityId: 'layout',
+      agent: {sessions: [{...first.agent.sessions[0], outputTokens: 900}]},
+    }),
+  ])
+  run.output.capabilities = new Map([
+    ['layout', {id: 'layout', name: 'Layout', scenarioIds: ['scenario-a']}],
+    ['components', {id: 'components', name: 'Components', scenarioIds: ['scenario-a']}],
+  ])
+
+  const summary = getExperimentResults(run)
+
+  expect(summary?.scenarios).toEqual([])
+  expect(summary?.treatments).toMatchObject([
+    {runner: 'copilot-cli', trials: 2, outputTokens: 500},
+    {runner: 'copilot-sdk', trials: 1, outputTokens: 300},
+  ])
+  expect(summary?.capabilities).toMatchObject([
+    {
+      id: 'components',
+      treatments: [
+        {runner: 'copilot-cli', trials: 1, outputTokens: 100},
+        {runner: 'copilot-sdk', trials: 1, outputTokens: 300},
+      ],
+      scenarios: [{id: 'scenario-a', treatments: [{outputTokens: 100}, {outputTokens: 300}]}],
+    },
+    {
+      id: 'layout',
+      treatments: [{runner: 'copilot-cli', trials: 1, outputTokens: 900}],
+      scenarios: [{id: 'scenario-a', treatments: [{outputTokens: 900}]}],
+    },
+  ])
+})
diff --git a/website/src/experiment-results.ts b/website/src/experiment-results.ts
index e9241139..3fb6ed2b 100644
--- a/website/src/experiment-results.ts
+++ b/website/src/experiment-results.ts
@@ -1,10 +1,11 @@
-import type {ExperimentTrialOutput} from '@primer/agent-eval'
+import type {CopilotRunner, ExperimentTrialOutput} from '@primer/agent-eval'
 import {formatChecks, summarizeTrials} from './check-results'
 import type {Run} from './runs'
 
 export type TreatmentResult = {
   id: string
   treatment: string
+  runner: CopilotRunner
   model: string
   reasoningEffort: string
   trials: number
@@ -24,13 +25,19 @@ export type ExperimentResults = {
     id: string
     treatments: Array<TreatmentResult>
   }>
+  capabilities: Array<{
+    id: string
+    name: string
+    treatments: Array<TreatmentResult>
+    scenarios: ExperimentResults['scenarios']
+  }>
 }
 
 function summarizeTreatments(run: Run, results: Array<ExperimentTrialOutput>): Array<TreatmentResult> {
   const groups = new Map<string, Array<ExperimentTrialOutput>>()
 
   for (const result of results) {
-    const key = JSON.stringify([result.treatmentId, result.model.name, result.model.reasoningEffort])
+    const key = JSON.stringify([result.treatmentId, result.runner, result.model.name, result.model.reasoningEffort])
     const group = groups.get(key)
     if (group) {
       group.push(result)
@@ -46,6 +53,7 @@ function summarizeTreatments(run: Run, results: Array<ExperimentTrialOutput>): A
     return {
       id,
       treatment: run.output.treatments.get(first.treatmentId)?.name ?? first.treatmentId,
+      runner: first.runner,
       model: first.model.name,
       reasoningEffort: first.model.reasoningEffort,
       trials: trials.length,
@@ -65,22 +73,23 @@ function summarizeTreatments(run: Run, results: Array<ExperimentTrialOutput>): A
     return (
       first.model.localeCompare(second.model) ||
       first.reasoningEffort.localeCompare(second.reasoningEffort) ||
+      first.runner.localeCompare(second.runner) ||
       first.treatment.localeCompare(second.treatment) ||
       first.id.localeCompare(second.id)
     )
   })
 }
 
-export function getExperimentResults(run: Run | undefined): ExperimentResults | null {
-  if (!run) {
-    return null
-  }
-
+function summarizeScenarios(
+  run: Run,
+  results: Array<ExperimentTrialOutput>,
+  scenarioIds: Iterable<string>,
+): ExperimentResults['scenarios'] {
   const scenarios = new Map<string, Array<ExperimentTrialOutput>>()
-  for (const scenario of run.output.scenarios.values()) {
-    scenarios.set(scenario.id, [])
+  for (const id of scenarioIds) {
+    scenarios.set(id, [])
   }
-  for (const result of run.output.trials.values()) {
+  for (const result of results) {
     const group = scenarios.get(result.scenarioId)
     if (group) {
       group.push(result)
@@ -89,13 +98,37 @@ export function getExperimentResults(run: Run | undefined): ExperimentResults |
     }
   }
 
+  return Array.from(scenarios, ([id, trials]) => {
+    return {id, treatments: summarizeTreatments(run, trials)}
+  }).toSorted((first, second) => {
+    return first.id.localeCompare(second.id)
+  })
+}
+
+export function getExperimentResults(run: Run | undefined): ExperimentResults | null {
+  if (!run) {
+    return null
+  }
+  const trials = [...run.output.trials.values()]
+
   return {
     date: run.name,
-    treatments: summarizeTreatments(run, [...run.output.trials.values()]),
-    scenarios: Array.from(scenarios, ([id, results]) => {
-      return {id, treatments: summarizeTreatments(run, results)}
-    }).toSorted((first, second) => {
-      return first.id.localeCompare(second.id)
-    }),
+    treatments: summarizeTreatments(run, trials),
+    scenarios: run.output.capabilities.size === 0 ? summarizeScenarios(run, trials, run.output.scenarios.keys()) : [],
+    capabilities: [...run.output.capabilities.values()]
+      .map(capability => {
+        const selected = trials.filter(trial => {
+          return trial.capabilityId === capability.id
+        })
+        return {
+          id: capability.id,
+          name: capability.name,
+          treatments: summarizeTreatments(run, selected),
+          scenarios: summarizeScenarios(run, selected, capability.scenarioIds),
+        }
+      })
+      .toSorted((first, second) => {
+        return first.name.localeCompare(second.name) || first.id.localeCompare(second.id)
+      }),
   }
 }
diff --git a/website/src/result-files.test.ts b/website/src/result-files.test.ts
index 0cdc8a73..822ee2fe 100644
--- a/website/src/result-files.test.ts
+++ b/website/src/result-files.test.ts
@@ -67,6 +67,51 @@ test.each(['benchmark', 'experiment'] as const)(
   },
 )
 
+test('reads benchmark-backed experiment capability metadata without losing trial membership', async () => {
+  const directory = await createDirectory()
+  const output = createBenchmarkOutput([
+    {...createTrial(), capabilityId: 'a'},
+    {...createTrial({id: 'trial-2', runner: 'copilot-sdk'}), capabilityId: 'b'},
+  ])
+  output.id = 'test-experiment'
+  const filepath = await writeBundle(directory, output)
+
+  const parsed = await readExperimentOutput(filepath)
+
+  expect(parsed).toEqual(output)
+  if (!parsed) {
+    throw new Error('Expected an experiment bundle')
+  }
+  const details = await createExperimentRunDetails('2026-09-15', parsed)
+  expect(
+    details.results.map(result => {
+      return result.capability
+    }),
+  ).toEqual([
+    {id: 'a', name: 'First capability'},
+    {id: 'b', name: 'Overlapping capability'},
+  ])
+})
+
+test('reads a legacy experiment manifest without inventing capability membership', async () => {
+  const directory = await createDirectory()
+  const output = createExperimentOutput()
+  const filepath = await writeBundle(directory, output)
+  const manifest = JSON.parse(await fs.readFile(filepath, 'utf8'))
+  delete manifest.capabilities
+  await fs.writeFile(filepath, JSON.stringify(manifest))
+
+  expect(await readExperimentOutput(filepath)).toEqual(output)
+})
+
+test('rejects experiment trial capability membership that conflicts with the manifest', async () => {
+  const directory = await createDirectory()
+  const output = createBenchmarkOutput([{...createTrial(), capabilityId: 'unknown'}])
+  const filepath = await writeBundle(directory, output)
+
+  await expect(readExperimentOutput(filepath)).rejects.toThrow('Invalid capability "unknown"')
+})
+
 test.each([
   {key: 'benchmarkId', read: readBenchmarkOutput},
   {key: 'experimentId', read: readExperimentOutput},
diff --git a/website/src/result-files.ts b/website/src/result-files.ts
index 1b6861d2..002cec52 100644
--- a/website/src/result-files.ts
+++ b/website/src/result-files.ts
@@ -8,6 +8,7 @@ const {
   parseBenchmarkTrialOutput,
   ExperimentOutputFileSchema,
   ExperimentTrialOutputSchema,
+  parseExperimentTrialOutput,
 } = await import(
   /* turbopackIgnore: true */
   '@primer/agent-eval'
@@ -96,8 +97,13 @@ async function readExperimentOutput(filepath: string): Promise<ExperimentOutput
   if (trials === null) {
     return null
   }
+  const capabilities = new Map(Object.entries(file.capabilities))
+  for (const [id, trial] of trials) {
+    trials.set(id, parseExperimentTrialOutput(trial, capabilities))
+  }
   return {
     id: file.id,
+    capabilities,
     scenarios: new Map(Object.entries(file.scenarios)),
     treatments: new Map(Object.entries(file.treatments)),
     trials,
diff --git a/website/src/run-details.ts b/website/src/run-details.ts
index d68c9aba..e41aa1f9 100644
--- a/website/src/run-details.ts
+++ b/website/src/run-details.ts
@@ -379,6 +379,10 @@ async function createExperimentRunDetails(
     if (treatment === undefined) {
       throw new Error(`Unknown treatment "${result.treatmentId}" for trial "${result.id}"`)
     }
+    const capability = result.capabilityId === undefined ? undefined : output.capabilities.get(result.capabilityId)
+    if ((result.capabilityId !== undefined || output.capabilities.size > 0) && !capability) {
+      throw new Error(`Unknown capability for trial "${result.id}"`)
+    }
     const baseUrl = getTrialDataUrl(collection, output.id, date, result.id)
     const tools = new Map<string, number>()
     for (const session of result.agent.sessions) {
@@ -389,6 +393,7 @@ async function createExperimentRunDetails(
     results.push({
       id: result.id,
       scenarioId: result.scenarioId,
+      ...(capability ? {capability: {id: capability.id, name: capability.name}} : {}),
       treatment,
       model: result.model.name,
       reasoningEffort: result.model.reasoningEffort,
@@ -445,19 +450,7 @@ function createReferenceFiles(files: CheckOutput['check']['files']): CheckOutput
 }
 
 async function createBenchmarkRunDetails(run: BenchmarkRun): Promise<RunDetails> {
-  const output = run.output
-  const details = await createExperimentRunDetails(run.name, output, 'benchmarks', run.directory)
-  return {
-    ...details,
-    results: details.results.map(result => {
-      const trial = output.trials.get(result.id)
-      const capability = trial ? output.capabilities.get(trial.capabilityId) : undefined
-      if (!capability) {
-        throw new Error(`Unknown capability for benchmark trial "${result.id}"`)
-      }
-      return {...result, capability: {id: capability.id, name: capability.name}}
-    }),
-  }
+  return createExperimentRunDetails(run.name, run.output, 'benchmarks', run.directory)
 }
 
 export {
diff --git a/website/src/runs.test.ts b/website/src/runs.test.ts
index b24658b3..5d6fd97d 100644
--- a/website/src/runs.test.ts
+++ b/website/src/runs.test.ts
@@ -25,6 +25,7 @@ beforeEach(async () => {
   }
   vi.mocked(read).mockResolvedValue({
     id: 'example',
+    capabilities: new Map(),
     scenarios: new Map(),
     treatments: new Map(),
     trials: new Map(),
@@ -83,6 +84,7 @@ test('rejects a bundle belonging to a different experiment without reading older
   const latest = await createRunDirectory('2026-09-10')
   vi.mocked(read).mockResolvedValueOnce({
     id: 'other',
+    capabilities: new Map(),
     scenarios: new Map(),
     treatments: new Map(),
     trials: new Map(),
diff --git a/website/src/scenario-anchor.ts b/website/src/scenario-anchor.ts
index 7748c062..aa9acb96 100644
--- a/website/src/scenario-anchor.ts
+++ b/website/src/scenario-anchor.ts
@@ -1,6 +1,8 @@
-export function getScenarioAnchor(scenarioId: string): {id: string; fragment: string} {
+export function getScenarioAnchor(scenarioId: string, capabilityId?: string): {id: string; fragment: string} {
   // Percent-free IDs avoid differences between native and Next.js fragment lookups.
-  const id = `scenario-${encodeURIComponent(scenarioId).replaceAll('_', '%5F').replaceAll('%', '_')}`
+  const key = capabilityId === undefined ? scenarioId : JSON.stringify([capabilityId, scenarioId])
+  const prefix = capabilityId === undefined ? 'scenario' : 'capability-scenario'
+  const id = `${prefix}-${encodeURIComponent(key).replaceAll('_', '%5F').replaceAll('%', '_')}`
   return {
     id,
     fragment: `#${id}`,
diff --git a/website/src/test-fixtures.ts b/website/src/test-fixtures.ts
index a1ed5bf9..0bc94f84 100644
--- a/website/src/test-fixtures.ts
+++ b/website/src/test-fixtures.ts
@@ -103,6 +103,7 @@ function createTrial(overrides: Partial<ExperimentTrialOutput> = {}): Experiment
 function createExperimentOutput(trials: Array<ExperimentTrialOutput> = [createTrial()]): ExperimentOutput {
   return {
     id: 'test-experiment',
+    capabilities: new Map(),
     scenarios: new Map([
       [
         'empty-state',

From 68438bbc04a5c6adede5e2e4cf662d702d53e4db Mon Sep 17 00:00:00 2001
From: Josh Black <joshblack@github.com>
Date: Fri, 9 Oct 2026 14:43:16 -0500
Subject: [PATCH 5/7] chore: add changeset

---
 .changeset/sparkly-waves-post.md | 5 +++++
 1 file changed, 5 insertions(+)
 create mode 100644 .changeset/sparkly-waves-post.md

diff --git a/.changeset/sparkly-waves-post.md b/.changeset/sparkly-waves-post.md
new file mode 100644
index 00000000..c35420ae
--- /dev/null
+++ b/.changeset/sparkly-waves-post.md
@@ -0,0 +1,5 @@
+---
+'@primer/agent-eval': minor
+---
+
+Add support for running benchmark experiments with the 'benchmark' option

From e24311b1d7b948720def58c9b35cbcee445340b2 Mon Sep 17 00:00:00 2001
From: Josh Black <joshblack@github.com>
Date: Mon, 12 Oct 2026 11:10:21 -0500
Subject: [PATCH 6/7] Potential fix for pull request finding 'Compose
 capability setup into trial setup for all treatments'

Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com>
---
 packages/agent-eval/src/experiment/plan.ts | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/packages/agent-eval/src/experiment/plan.ts b/packages/agent-eval/src/experiment/plan.ts
index 757fc0da..824da419 100644
--- a/packages/agent-eval/src/experiment/plan.ts
+++ b/packages/agent-eval/src/experiment/plan.ts
@@ -53,7 +53,7 @@ function createExperimentPlan({experiment}: CreateExperimentPlanOptions): Plan<S
               ControlTreatment,
               createTreatment({
                 name: 'Benchmark',
-                setup: composeTreatmentSetup(experiment.benchmark.setup, capability.setup),
+                setup: experiment.benchmark.setup,
               }),
               ...experiment.treatments,
             ]
@@ -66,7 +66,7 @@ function createExperimentPlan({experiment}: CreateExperimentPlanOptions): Plan<S
                   model,
                   runner,
                   capability,
-                  setup: experiment.setup,
+                  setup: composeTreatmentSetup(experiment.setup, capability.setup),
                 }
               })
             })

From 3c09de94ad872e90c5f3b4a9fbf6d6ba6200dbd1 Mon Sep 17 00:00:00 2001
From: Josh Black <joshblack@github.com>
Date: Mon, 12 Oct 2026 11:23:43 -0500
Subject: [PATCH 7/7] fix: preserve setup when restoring benchmark experiments

Update experiment guidance to use default exports and require source evidence for resource-based decisions. Clarify capability setup scopes and the benchmarksDirectory migration.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
Copilot-Session: b79d13c1-4b88-4e32-9faa-d425f1e2b55d
---
 .changeset/sparkly-waves-post.md              |  5 +-
 AGENTS.md                                     | 12 ++-
 .../agent-eval/src/experiment/plan.test.ts    | 87 +++++++++++++++++--
 packages/agent-eval/src/experiment/plan.ts    |  4 +-
 skills/agent-eval/SKILL.md                    |  5 ++
 skills/agent-eval/references/benchmarks.md    |  3 +
 skills/agent-eval/references/capabilities.md  | 16 ++--
 skills/agent-eval/references/experiments.md   |  9 +-
 8 files changed, 119 insertions(+), 22 deletions(-)

diff --git a/.changeset/sparkly-waves-post.md b/.changeset/sparkly-waves-post.md
index c35420ae..6c314326 100644
--- a/.changeset/sparkly-waves-post.md
+++ b/.changeset/sparkly-waves-post.md
@@ -2,4 +2,7 @@
 '@primer/agent-eval': minor
 ---
 
-Add support for running benchmark experiments with the 'benchmark' option
+Add support for running benchmark experiments with the `benchmark` option.
+
+Calls to `getExperiment` and `listExperiments` now require `benchmarksDirectory`,
+including when loading only scenario-backed experiments.
diff --git a/AGENTS.md b/AGENTS.md
index bf2e3b1d..91256abe 100644
--- a/AGENTS.md
+++ b/AGENTS.md
@@ -31,6 +31,16 @@
   `SKILL.md` concise, put detailed guidance in references, and use npm/npx in
   skill examples.
 
+## Evidence for decisions
+
+- When a decision, recommendation, or review finding relies on a resource,
+  include the specific evidence and explain how it supports the conclusion.
+- Cite repository files with paths and line numbers. For external resources,
+  link to the relevant section or comment and quote the supporting passage
+  when line numbers are unavailable.
+- Read the cited resource before relying on it. Distinguish what it explicitly
+  states from your inference, and state uncertainty or conflicting evidence.
+
 ## Code
 
 - Parse, don't validate. Use the type system to gurantee correctness.
@@ -123,7 +133,7 @@ When adding an experiment, follow these rules:
   it, when appropriate
 - New experiments are added at `./experiments` as a new `*.ts` file
 - Provide an appropriate name for the experiment as its file name
-- Export an `experiment` named export created with `defineConfig`
+- Use a default export created with `defineConfig`
 - Provide an appropriate name and description for what we are testing
 - Select appropriate models and scenarios based on the experiment
 - For treatments, make sure to include the fewest number possible to meet the
diff --git a/packages/agent-eval/src/experiment/plan.test.ts b/packages/agent-eval/src/experiment/plan.test.ts
index 44ac7281..bd2484b0 100644
--- a/packages/agent-eval/src/experiment/plan.test.ts
+++ b/packages/agent-eval/src/experiment/plan.test.ts
@@ -1,6 +1,7 @@
 import {expect, expectTypeOf, test} from 'vitest'
 import {VirtualHost} from '../host'
 import type {CopilotRunner} from '../copilot-runner'
+import {VirtualSandbox} from '../sandbox'
 import {ExperimentConfigSchema} from './config'
 import {getExperiment} from './get'
 import {createExperimentPlan, createExperimentPlanManifest, parseExperimentPlanManifest} from './plan'
@@ -190,18 +191,88 @@ test('expands and restores benchmark experiment control, benchmark, and custom t
 
   expect(parsed.trials).toEqual(
     plan.trials.map(trial => {
-      if (trial.treatment.name === 'Benchmark') {
+      return {
+        ...trial,
+        setup: expect.any(Function),
+      }
+    }),
+  )
+})
+
+test('preserves shared setup order and treatment isolation in created and restored benchmark experiment plans', async () => {
+  const host = VirtualHost.create({
+    '/experiments/example.ts': `export default {
+      name: 'Example',
+      description: 'Compare benchmark treatments',
+      models: ['gpt-5.5'],
+      benchmark: 'design-system',
+      async setup({sandbox}) {
+        await sandbox.writeFile('/setup.txt', 'experiment')
+      },
+      treatments: [{
+        name: 'Skill',
+        async setup({sandbox}) {
+          await sandbox.writeFile('/setup.txt', await sandbox.readFile('/setup.txt') + ',skill')
+        },
+      }],
+    }`,
+    '/benchmarks/design-system.ts': `export default {
+      name: 'Design System',
+      description: 'Example benchmark',
+      models: ['gpt-5.5'],
+      async setup({sandbox}) {
+        await sandbox.writeFile('/setup.txt', await sandbox.readFile('/setup.txt') + ',benchmark')
+      },
+      capabilities: ['Components', 'Layouts'].map(name => {
         return {
-          ...trial,
-          treatment: {
-            ...trial.treatment,
-            setup: expect.any(Function),
+          name,
+          scenarios: ['example'],
+          async setup({sandbox}) {
+            await sandbox.writeFile('/setup.txt', await sandbox.readFile('/setup.txt') + ',' + name)
           },
         }
+      }),
+    }`,
+    '/scenarios/example/package.json': '{}',
+    '/scenarios/example/scenario.config.ts': 'export default {prompt: "Create a page"}',
+  })
+  const options = {
+    host,
+    benchmarksDirectory: '/benchmarks',
+    experimentsDirectory: '/experiments',
+    scenariosDirectory: '/scenarios',
+  }
+  const experiment = await getExperiment({...options, name: 'example'})
+  const plan = createExperimentPlan({experiment})
+  const manifest = createExperimentPlanManifest({experiment, plan})
+  const parsed = await parseExperimentPlanManifest({...options, contents: JSON.stringify(manifest)})
+  const plans: Array<typeof parsed.trials> = [plan.trials, parsed.trials]
+
+  for (const trials of plans) {
+    expect(trials).toHaveLength(6)
+    for (const trial of trials) {
+      if (!('capability' in trial)) {
+        throw new Error('Expected a benchmark trial')
       }
-      return trial
-    }),
-  )
+      await using sandbox = await VirtualSandbox.create()
+      const sharedSetup = `experiment,${trial.capability.name}`
+
+      await trial.setup?.({sandbox})
+
+      expect(await sandbox.readFile('/setup.txt')).toBe(sharedSetup)
+
+      await trial.treatment.setup?.({sandbox})
+
+      if (trial.treatment.name === 'Control') {
+        expect(await sandbox.readFile('/setup.txt')).toBe(sharedSetup)
+      } else if (trial.treatment.name === 'Benchmark') {
+        expect(await sandbox.readFile('/setup.txt')).toBe(`${sharedSetup},benchmark`)
+      } else {
+        expect(trial.treatment.name).toBe('Skill')
+        expect(await sandbox.readFile('/setup.txt')).toBe(`${sharedSetup},skill`)
+      }
+    }
+  }
 })
 
 test.each([{runners: []}, {runners: ['sdk']}, {runners: ['unknown']}])(
diff --git a/packages/agent-eval/src/experiment/plan.ts b/packages/agent-eval/src/experiment/plan.ts
index 824da419..7bb33eea 100644
--- a/packages/agent-eval/src/experiment/plan.ts
+++ b/packages/agent-eval/src/experiment/plan.ts
@@ -253,7 +253,7 @@ async function parseExperimentPlanManifest({
           ControlTreatment,
           createTreatment({
             name: 'Benchmark',
-            setup: composeTreatmentSetup(experiment.benchmark.setup, capability.setup),
+            setup: experiment.benchmark.setup,
           }),
           ...experiment.treatments,
         ]
@@ -271,7 +271,7 @@ async function parseExperimentPlanManifest({
           scenario,
           capability,
           treatment,
-          setup: experiment.setup,
+          setup: composeTreatmentSetup(experiment.setup, capability.setup),
         }
       }),
     }
diff --git a/skills/agent-eval/SKILL.md b/skills/agent-eval/SKILL.md
index 96dc7c6d..07505d63 100644
--- a/skills/agent-eval/SKILL.md
+++ b/skills/agent-eval/SKILL.md
@@ -29,6 +29,11 @@ not a prerequisite for every operation.
 
 ## Execution rules
 
+- When a decision or recommendation relies on a resource, include the specific
+  evidence and explain how it supports the conclusion. Cite file paths and line
+  numbers, or link to the relevant external section or comment and quote the
+  supporting passage. Read the resource first and distinguish evidence from
+  inference.
 - For a first comparison, use one scenario, one model/effort, and one treatment
   beyond the automatic control.
 - Keep prompts and fixtures treatment-blind. Put the knowledge being tested in
diff --git a/skills/agent-eval/references/benchmarks.md b/skills/agent-eval/references/benchmarks.md
index 9d7d7e5d..ad5eebc2 100644
--- a/skills/agent-eval/references/benchmarks.md
+++ b/skills/agent-eval/references/benchmarks.md
@@ -17,6 +17,9 @@ time. Use an [experiment](experiments.md) to compare multiple interventions.
 Benchmark-level `setup` runs only for `Benchmark`. Capability-level setup
 runs for **both** treatments, before treatment setup. Put neutral prerequisites
 in capability setup and the resource being measured in benchmark setup.
+When an experiment selects this benchmark, capability setup runs for every
+treatment, including custom treatments, after experiment setup. Benchmark-level
+setup still runs only for `Benchmark`.
 
 ## Minimal example
 
diff --git a/skills/agent-eval/references/capabilities.md b/skills/agent-eval/references/capabilities.md
index c1b3cefb..8bc40f2e 100644
--- a/skills/agent-eval/references/capabilities.md
+++ b/skills/agent-eval/references/capabilities.md
@@ -1,8 +1,8 @@
 # Capabilities
 
 **Use when:** grouping benchmark scenarios under a behavior such as navigation,
-forms, or API migration. Experiments select scenarios directly, without
-capability groups.
+forms, or API migration. Benchmark-backed experiments reuse the benchmark's
+capability groups; scenario-backed experiments select scenarios directly.
 
 ## Contract
 
@@ -13,10 +13,12 @@ capability groups.
 | Optional field      | `setup`                                                    |
 | Identity            | Name-derived ID; names must be unique within the benchmark |
 | Scenario references | Folder names under the selected scenarios directory        |
-| Setup scope         | Both `Control` and `Benchmark`, before treatment setup     |
+| Setup scope         | Every treatment in the capability, before treatment setup  |
 
 Put only shared prerequisites in capability setup. Install the resource being
-measured in the benchmark's top-level setup.
+measured in the benchmark's top-level setup, which runs only for `Benchmark`.
+In benchmark-backed experiments, capability setup also applies to custom
+treatments, after experiment setup and before treatment setup.
 
 ## Configuration fragment
 
@@ -37,9 +39,9 @@ Prefer pinned dependencies in the fixture manifest when no setup hook is needed.
 
 ## Run and verify
 
-Create and run the enclosing [benchmark](benchmarks.md), not the capability
-itself. Inspect its plan for the expected capability/scenario memberships and
-both treatment conditions.
+Create and run the enclosing [benchmark](benchmarks.md) or a benchmark-backed
+[experiment](experiments.md), not the capability itself. Inspect its plan for
+the expected capability/scenario memberships and all treatment conditions.
 
 Group related tasks without making one scenario cover every behavior. Keep
 granular scenario checks so the capability summary does not hide failures.
diff --git a/skills/agent-eval/references/experiments.md b/skills/agent-eval/references/experiments.md
index 6414c3f7..7c219c4d 100644
--- a/skills/agent-eval/references/experiments.md
+++ b/skills/agent-eval/references/experiments.md
@@ -9,7 +9,7 @@ plugins, or execution backends on the same tasks. Use a
 | Item                | Rule                                                                                 |
 | :------------------ | :----------------------------------------------------------------------------------- |
 | File and import     | `experiments/<id>.ts`; `defineConfig` from `@primer/agent-eval/experiment`           |
-| Export              | Prefer named `experiment`; default also supported                                    |
+| Export              | Use a default export; named `experiment` also supported                              |
 | Required fields     | `name`, `description`, `models`, `treatments`, and either `scenarios` or `benchmark` |
 | Optional fields     | `setup`, `runners`                                                                   |
 | CLI identifier      | Filename without extension, not display `name`                                       |
@@ -24,7 +24,10 @@ Do not put the tested resource in shared setup or the fixture.
 The harness adds `Control` automatically. Treatment names must be unique and
 cannot be `Control`. For scenario-backed experiments, `treatments: []` runs
 control only. Benchmark-backed experiments also add a `Benchmark` treatment
-using the benchmark and capability setup; `Benchmark` is also a reserved name.
+using only benchmark-level setup; `Benchmark` is also a reserved name.
+Capability setup runs for every treatment in that capability, including
+`Control` and custom treatments, after experiment setup and before treatment
+setup.
 
 ## Reports
 
@@ -54,7 +57,7 @@ Save this complete configuration as `experiments/verification.ts`:
 ```ts
 import {defineConfig} from '@primer/agent-eval/experiment'
 
-export const experiment = defineConfig({
+export default defineConfig({
   name: 'Verification instructions',
   description: 'Test whether verification instructions improve correctness on a small transformation task',
   models: [{name: 'gpt-5.4', reasoningEfforts: ['low']}],