diff --git a/src/benchmarks/benchmark-config.ts b/src/benchmarks/benchmark-config.ts index 28d0b06..9dba017 100644 --- a/src/benchmarks/benchmark-config.ts +++ b/src/benchmarks/benchmark-config.ts @@ -10,6 +10,7 @@ import { TAU3_BENCH_BANKING_META, TAU_BENCH_AIRLINE_META, } from "./benchmark-meta"; +import { EvalSpecSchema } from "./custom-eval/spec"; import { DEFAULT_STEP_LIMIT as DEEP_SWE_DEFAULT_STEP_LIMIT } from "./deep-swe/schema"; import { DracoPanelConfigSchema } from "./draco/schemas"; import { SearchLaneConfigSchema } from "./search/core/config"; @@ -164,6 +165,27 @@ export type IfStructBenchmarkConfig = z.infer< typeof IfStructBenchmarkConfigSchema >; +/** + * Declarative custom eval: the run config carries the full EvalSpec (dataset + + * prompt + deterministic scorer), so user evals are data, not registry code. + */ +export const CustomEvalOptionsSchema = z.object({ + spec: EvalSpecSchema, +}); + +export const CustomEvalBenchmarkConfigSchema = z.object({ + benchmarkId: z.literal("custom_eval"), + ...ModelBenchmarkBaseSchema.shape, + ...CustomEvalOptionsSchema.shape, +}); +export type CustomEvalBenchmarkConfig = z.infer< + typeof CustomEvalBenchmarkConfigSchema +>; + +/** + * Options shared by the Modal-backed agentic benchmarks (SWE-Atlas tracks and + * DeepSWE). `stepLimit` is declared per benchmark because the defaults differ. + */ const AgenticOptionsSchema = z.object({ taskSubset: z.array(z.string()).optional(), maxAgentTimeoutSec: z.number().positive().optional(), @@ -278,6 +300,7 @@ export const BenchmarkRunConfigSchema = z.discriminatedUnion("benchmarkId", [ TerminalBenchConfigSchema, DracoBenchmarkConfigSchema, IfStructBenchmarkConfigSchema, + CustomEvalBenchmarkConfigSchema, SweAtlasQaConfigSchema, SweAtlasTwConfigSchema, SweAtlasRfConfigSchema, @@ -314,6 +337,7 @@ export const BENCHMARK_OPTIONS_SCHEMAS = { mmmu_pro_vision: MmmuProVisionOptionsSchema, terminal_bench: TerminalBenchOptionsSchema, ifstruct: IfStructOptionsSchema, + custom_eval: CustomEvalOptionsSchema, swe_atlas_qa: SweAtlasOptionsSchema, swe_atlas_tw: SweAtlasOptionsSchema, swe_atlas_rf: SweAtlasOptionsSchema, diff --git a/src/benchmarks/benchmark-meta.ts b/src/benchmarks/benchmark-meta.ts index cf8ceeb..da30ce7 100644 --- a/src/benchmarks/benchmark-meta.ts +++ b/src/benchmarks/benchmark-meta.ts @@ -49,6 +49,13 @@ export const IFSTRUCT_META = { defaultEpochs: 1, } as const satisfies BenchmarkMeta; +/** Declarative customer/internal eval defined entirely by its run config's EvalSpec. */ +export const CUSTOM_EVAL_META = { + id: "custom_eval", + defaultEpochs: 1, +} as const satisfies BenchmarkMeta; + +/* defaultEpochs 3 mirrors the reference run scripts' `-k 3` (3 rollouts/task). */ export const SWE_ATLAS_QA_META = { id: "swe_atlas_qa", defaultEpochs: 3, @@ -103,6 +110,7 @@ const BENCHMARK_META: Readonly> = { [TERMINAL_BENCH_META.id]: TERMINAL_BENCH_META, [DRACO_META.id]: DRACO_META, [IFSTRUCT_META.id]: IFSTRUCT_META, + [CUSTOM_EVAL_META.id]: CUSTOM_EVAL_META, [SWE_ATLAS_QA_META.id]: SWE_ATLAS_QA_META, [SWE_ATLAS_TW_META.id]: SWE_ATLAS_TW_META, [SWE_ATLAS_RF_META.id]: SWE_ATLAS_RF_META, diff --git a/src/benchmarks/custom-eval/benchmark.ts b/src/benchmarks/custom-eval/benchmark.ts new file mode 100644 index 0000000..5293a14 --- /dev/null +++ b/src/benchmarks/custom-eval/benchmark.ts @@ -0,0 +1,176 @@ +import type { HttpClient } from "@effect/platform"; +import { fail as effectFail, gen } from "effect/Effect"; +import type { Layer } from "effect/Layer"; +import { + effect as layerEffect, + fail as layerFail, + mergeAll as layerMergeAll, + provide as layerProvide, + succeed as layerSucceed, +} from "effect/Layer"; +import { fail as streamFail } from "effect/Stream"; + +import { DatasetError } from "../../harness/core"; +/** + * `custom_eval` — one generic benchmark whose config carries a declarative + * {@link EvalSpec}. Customers (and internal users) define an eval as data — + * dataset + prompt + deterministic scorer — and it runs through the exact + * same solver/scorer/epochs/results pipeline as first-party benchmarks. The + * registry stays finite; user evals are rows, not code. + * + * Unlike `defineChatBenchmark` benchmarks, the dataset AND scorer both come + * from the run config, so `makeLayer` is hand-rolled (the tau3/draco pattern). + */ +import { Dataset } from "../../harness/dataset"; +import type { GenerateConfig, ModelService } from "../../harness/model"; +import { Model } from "../../harness/model"; +import type { Scorer as ScorerType } from "../../harness/scorer"; +import { Scorer } from "../../harness/scorer"; +import type { Solver as SolverType, SolverService } from "../../harness/solver"; +import { chain, generate, Solver, systemMessage } from "../../harness/solver"; +import { definedValues } from "../../internal/guards"; +import { makeOpenRouterModelLayer } from "../../providers/openrouter-model"; +import type { RetryConfig } from "../../runtime/retry"; +import type { + BenchmarkRunConfig, + CustomEvalBenchmarkConfig, +} from "../benchmark-config"; +import { CUSTOM_EVAL_META } from "../benchmark-meta"; +import type { Benchmark, BenchmarkRunInput } from "../types"; +import { makeCustomEvalDatasetLayer } from "./dataset"; +import { makeCustomEvalScorer } from "./scorer"; + +export const CUSTOM_EVAL_DEFAULT_TEMPERATURE = 0; + +export function renderPrompt( + template: string | undefined, + input: string +): string { + if (template === undefined) { + return input; + } + // replaceAll: a template may reference {input} more than once (e.g. quoted + // and restated); function form so literal `$` sequences in the input survive. + return template.replaceAll("{input}", () => input); +} + +export function customEvalSolver( + model: ModelService, + config: CustomEvalBenchmarkConfig +): SolverService { + const generateConfig: GenerateConfig = { + temperature: config.temperature ?? CUSTOM_EVAL_DEFAULT_TEMPERATURE, + ...definedValues({ + maxTokens: config.maxTokens, + reasoningEffort: config.reasoningEffort, + timeoutMs: config.timeoutMs, + sort: config.sort, + cloudflareVersion: config.cloudflareVersion, + }), + ...(config.endpointId !== undefined && { endpointId: config.endpointId }), + }; + /** Render the prompt template over the sample's input (the last user message). */ + const applyTemplate: SolverService = (state) => + generate( + model, + generateConfig + )({ + ...state, + messages: state.messages.map((message, index) => + index === state.messages.length - 1 && + typeof message.content === "string" + ? { + ...message, + content: renderPrompt( + config.spec.promptTemplate, + message.content + ), + } + : message + ), + }); + return config.spec.systemPrompt !== undefined + ? chain(systemMessage(config.spec.systemPrompt), applyTemplate) + : applyTemplate; +} + +function makeLayer( + input: BenchmarkRunInput +): Layer { + const config = input.benchmarkConfig; + if (config.benchmarkId !== "custom_eval") { + return layerFail( + new Error("custom_eval received mismatched benchmarkConfig") + ); + } + + const datasetLayer = makeCustomEvalDatasetLayer( + config.spec.dataset, + input.datasetRetry + ); + + const modelLayer = + input.modelLayer ?? + makeOpenRouterModelLayer({ + model: config.model, + apiKey: input.apiKey, + ...(input.baseUrl !== undefined && { baseUrl: input.baseUrl }), + sessionId: input.sessionId, + ...(input.modelRetry !== undefined && { retry: input.modelRetry }), + }); + + const solverLayer = layerEffect(Solver)( + gen(function* () { + const model = yield* Model; + return Solver.of(customEvalSolver(model, config)); + }) + ).pipe(layerProvide(modelLayer)); + + const scorerLayer = layerSucceed( + Scorer, + Scorer.of(makeCustomEvalScorer(config.spec.scorer)) + ); + + return layerMergeAll(datasetLayer, solverLayer, scorerLayer); +} + +/** + * The dataset comes from the run config, so the config-free + * `makeDatasetLayer` (used for registry-wide dataset-size probing of static + * benchmarks) has nothing real to return. Its size/stream FAIL rather than + * answering with a placeholder: an earlier stand-in dataset of size 1 made + * the orchestration chunk every custom eval to its first item and silently + * score one case. Size probing goes through `makeDatasetLayerForConfig`. + */ +function makeDatasetLayer(_retryConfig?: RetryConfig): Layer { + const noConfigError = new DatasetError({ + message: + "custom_eval has no config-free dataset; resolve size via the run config", + }); + return layerSucceed( + Dataset, + Dataset.of({ + stream: () => streamFail(noConfigError), + size: effectFail(noConfigError), + }) + ); +} + +function makeDatasetLayerForConfig( + config: BenchmarkRunConfig, + retryConfig?: RetryConfig +): Layer { + if (config.benchmarkId !== "custom_eval") { + return makeDatasetLayer(retryConfig); + } + return makeCustomEvalDatasetLayer(config.spec.dataset, retryConfig); +} + +export const CUSTOM_EVAL_BENCHMARK: Benchmark = { + id: CUSTOM_EVAL_META.id, + makeDatasetLayer, + makeDatasetLayerForConfig, + makeLayer, + temperature: CUSTOM_EVAL_DEFAULT_TEMPERATURE, + defaultEpochs: CUSTOM_EVAL_META.defaultEpochs, +}; diff --git a/src/benchmarks/custom-eval/custom-eval.test.ts b/src/benchmarks/custom-eval/custom-eval.test.ts new file mode 100644 index 0000000..29930e3 --- /dev/null +++ b/src/benchmarks/custom-eval/custom-eval.test.ts @@ -0,0 +1,181 @@ +import { describe, expect, it } from "bun:test"; + +import { Either } from "../../internal/either"; +import { assertLeft, assertRight } from "../../internal/testing"; +import { parseSchema } from "../../internal/zod"; +import { BenchmarkRunConfigSchema } from "../benchmark-config"; +import { renderPrompt } from "./benchmark"; +import { inlineCaseToSample } from "./dataset"; +import { extractLastNumber, scoreCompletion } from "./scorer"; +import { EvalSpecSchema } from "./spec"; + +const spec = (overrides: Record = {}): unknown => ({ + name: "my eval", + dataset: { + kind: "inline", + cases: [{ input: "What is 2+2?", target: "4" }], + }, + scorer: { kind: "exact" }, + ...overrides, +}); + +describe("EvalSpecSchema", () => { + it("accepts a minimal inline spec and applies defaults", () => { + const parsed = parseSchema(EvalSpecSchema, spec()); + expect(Either.isRight(parsed)).toBe(true); + if (Either.isRight(parsed)) { + expect(parsed.right.specVersion).toBe(1); + expect(parsed.right.scorer).toMatchObject({ + kind: "exact", + caseSensitive: false, + trim: true, + }); + } + }); + + it("accepts an HF dataset spec with field mapping", () => { + const parsed = parseSchema( + EvalSpecSchema, + spec({ + dataset: { + kind: "hf", + dataset: "openai/gsm8k", + split: "test", + inputField: "question", + targetField: "answer", + }, + scorer: { kind: "numeric", absoluteTolerance: 0 }, + }) + ); + expect(Either.isRight(parsed)).toBe(true); + }); + + it("rejects empty inline datasets and unknown scorers", () => { + expect( + Either.isLeft( + parseSchema( + EvalSpecSchema, + spec({ dataset: { kind: "inline", cases: [] } }) + ) + ) + ).toBe(true); + expect( + Either.isLeft( + parseSchema(EvalSpecSchema, spec({ scorer: { kind: "vibes" } })) + ) + ).toBe(true); + }); + + it("round-trips through the benchmark config union", () => { + const parsed = parseSchema(BenchmarkRunConfigSchema, { + benchmarkId: "custom_eval", + model: "openai/gpt-4o", + spec: spec(), + }); + expect(Either.isRight(parsed)).toBe(true); + if (Either.isRight(parsed) && parsed.right.benchmarkId === "custom_eval") { + expect(parsed.right.spec.name).toBe("my eval"); + } + }); +}); + +describe("scoreCompletion", () => { + it("exact: trims and case-folds by default", () => { + const scorer = { kind: "exact", caseSensitive: false, trim: true } as const; + expect(scoreCompletion(scorer, " Paris \n", "paris").value).toBe("C"); + expect(scoreCompletion(scorer, "London", "paris").value).toBe("I"); + }); + + it("contains: substring match", () => { + const scorer = { kind: "contains", caseSensitive: false } as const; + expect( + scoreCompletion(scorer, "The answer is Paris, France.", "paris").value + ).toBe("C"); + expect(scoreCompletion(scorer, "No idea.", "paris").value).toBe("I"); + }); + + it("regex: pattern match with case-insensitive default", () => { + const scorer = { + kind: "regex", + pattern: "answer:\\s*42", + caseSensitive: false, + } as const; + expect(scoreCompletion(scorer, "ANSWER: 42", "unused").value).toBe("C"); + expect(scoreCompletion(scorer, "answer: 41", "unused").value).toBe("I"); + }); + + it("choice: reuses MCQ letter extraction", () => { + const scorer = { kind: "choice" } as const; + expect(scoreCompletion(scorer, "Thinking...\nAnswer: B", "b").value).toBe( + "C" + ); + expect(scoreCompletion(scorer, "Answer: C", "B").value).toBe("I"); + }); + + it("numeric: exact and tolerant comparison", () => { + const exact = { + kind: "numeric", + absoluteTolerance: 0, + relativeTolerance: 0, + } as const; + expect(scoreCompletion(exact, "The total is 1,234.", "1234").value).toBe( + "C" + ); + expect(scoreCompletion(exact, "roughly 1233", "1234").value).toBe("I"); + const tolerant = { + kind: "numeric", + absoluteTolerance: 2, + relativeTolerance: 0, + } as const; + expect(scoreCompletion(tolerant, "roughly 1233", "1234").value).toBe("C"); + expect(scoreCompletion(exact, "no numbers here", "1").value).toBe("I"); + }); +}); + +describe("extractLastNumber", () => { + it("takes the final number, ignoring commas and signs", () => { + expect(extractLastNumber("First 3 then 4.5, answer -7")).toBe(-7); + expect(extractLastNumber("total: 1,234,567")).toBe(1_234_567); + expect(extractLastNumber("none")).toBeNull(); + }); +}); + +describe("renderPrompt / inlineCaseToSample", () => { + it("substitutes {input} and preserves literal $ in inputs", () => { + expect(renderPrompt("Q: {input}\nA:", "cost is $5")).toBe( + "Q: cost is $5\nA:" + ); + expect(renderPrompt(undefined, "raw")).toBe("raw"); + }); + + it("assigns stable ids to inline cases", () => { + expect(inlineCaseToSample({ input: "a", target: "b" }, 3).id).toBe( + "custom_eval-3" + ); + expect( + inlineCaseToSample({ id: "mine", input: "a", target: "b" }, 3).id + ).toBe("mine"); + }); +}); + +describe("EvalScorerSchema regex validation", () => { + it("rejects a malformed regex pattern at parse time, not scoring time", () => { + const parsed = parseSchema(EvalSpecSchema, { + specVersion: 1, + name: "bad regex", + dataset: { kind: "inline", cases: [{ input: "x", target: "y" }] }, + scorer: { kind: "regex", pattern: "[unclosed" }, + }); + assertLeft(parsed); + }); + + it("accepts a valid pattern", () => { + const parsed = parseSchema(EvalSpecSchema, { + specVersion: 1, + name: "good regex", + dataset: { kind: "inline", cases: [{ input: "x", target: "y" }] }, + scorer: { kind: "regex", pattern: String.raw`\d{4}-\d{2}-\d{2}` }, + }); + assertRight(parsed); + }); +}); diff --git a/src/benchmarks/custom-eval/dataset.ts b/src/benchmarks/custom-eval/dataset.ts new file mode 100644 index 0000000..e27a283 --- /dev/null +++ b/src/benchmarks/custom-eval/dataset.ts @@ -0,0 +1,73 @@ +import { succeed } from "effect/Effect"; +import type { Layer } from "effect/Layer"; +import { succeed as layerSucceed } from "effect/Layer"; +import type { Stream } from "effect/Stream"; +import { fromIterable } from "effect/Stream"; + +import { makeHfDatasetLayer } from "../../datasets/huggingface"; +/** + * Dataset layers for declarative custom evals: inline cases (materialized in + * the spec) or a HuggingFace dataset with declared input/target fields. + */ +import type { Sample } from "../../harness/core"; +import type { Dataset, DatasetStreamOptions } from "../../harness/dataset"; +import { Dataset as DatasetTag } from "../../harness/dataset"; +import type { RetryConfig } from "../../runtime/retry"; +import type { EvalDataset, InlineCase } from "./spec"; + +export function inlineCaseToSample( + evalCase: InlineCase, + index: number +): Sample { + return { + id: evalCase.id ?? `custom_eval-${index}`, + input: evalCase.input, + target: { text: evalCase.target }, + ...(evalCase.metadata !== undefined && { metadata: evalCase.metadata }), + }; +} + +function makeInlineDatasetLayer(cases: readonly InlineCase[]): Layer { + const samples = cases.map(inlineCaseToSample); + const stream = (opts?: DatasetStreamOptions): Stream => + fromIterable(samples.slice(opts?.start ?? 0, opts?.end ?? samples.length)); + return layerSucceed( + DatasetTag, + DatasetTag.of({ stream, size: succeed(samples.length) }) + ); +} + +function asString(value: unknown, field: string): string { + if (typeof value === "string") { + return value; + } + if (typeof value === "number" || typeof value === "boolean") { + return String(value); + } + throw new TypeError( + `custom_eval record field "${field}" is not a string/number/boolean` + ); +} + +export function makeCustomEvalDatasetLayer( + dataset: EvalDataset, + retryConfig?: RetryConfig +): Layer { + if (dataset.kind === "inline") { + return makeInlineDatasetLayer(dataset.cases); + } + return makeHfDatasetLayer({ + dataset: dataset.dataset, + config: dataset.config, + split: dataset.split, + ...(dataset.revision !== undefined && { revision: dataset.revision }), + recordToSample: (record, index) => ({ + id: `custom_eval-${index}`, + input: asString(record[dataset.inputField], dataset.inputField), + target: { + text: asString(record[dataset.targetField], dataset.targetField), + }, + }), + ...(retryConfig !== undefined && { retry: retryConfig }), + }); +} diff --git a/src/benchmarks/custom-eval/scorer.ts b/src/benchmarks/custom-eval/scorer.ts new file mode 100644 index 0000000..1ee4ab8 --- /dev/null +++ b/src/benchmarks/custom-eval/scorer.ts @@ -0,0 +1,119 @@ +import { sync } from "effect/Effect"; + +/** + * Deterministic scorer dispatch for declarative custom evals. Pure functions: + * every verdict is reproducible from the stored completion + spec, which is + * what lets rescoring reuse trajectories with zero model calls later. + */ +import type { Score } from "../../harness/core"; +import { ScoreValue } from "../../harness/core"; +import type { ScorerService } from "../../harness/scorer"; +import { extractMcqAnswer } from "../scorers/mcq/extract"; +import type { EvalScorer } from "./spec"; + +const score = ( + isCorrect: boolean, + answer: string | null, + explanation: string +): Score => ({ + value: isCorrect ? ScoreValue.Correct : ScoreValue.Incorrect, + answer, + explanation, +}); + +/** Last number in the text (handles commas and signs), or null. */ +export function extractLastNumber(text: string): number | null { + const matches = text.replaceAll(",", "").match(/-?\d+(?:\.\d+)?/g); + if (!matches || matches.length === 0) { + return null; + } + const last = Number(matches.at(-1)); + return Number.isFinite(last) ? last : null; +} + +export function scoreCompletion( + scorer: EvalScorer, + completion: string, + target: string +): Score { + switch (scorer.kind) { + case "exact": { + const normalize = (value: string): string => { + const trimmed = scorer.trim ? value.trim() : value; + return scorer.caseSensitive ? trimmed : trimmed.toLowerCase(); + }; + const ok = normalize(completion) === normalize(target); + return score( + ok, + completion.trim(), + ok ? "exact match" : `expected "${target}"` + ); + } + case "contains": { + const haystack = scorer.caseSensitive + ? completion + : completion.toLowerCase(); + const needle = scorer.caseSensitive ? target : target.toLowerCase(); + const ok = haystack.includes(needle); + return score( + ok, + null, + ok ? `completion contains "${target}"` : `missing "${target}"` + ); + } + case "regex": { + const flags = scorer.caseSensitive ? "u" : "iu"; + const ok = new RegExp(scorer.pattern, flags).test(completion); + return score( + ok, + null, + ok ? `matched /${scorer.pattern}/` : `no match for /${scorer.pattern}/` + ); + } + case "choice": { + const extracted = extractMcqAnswer(completion); + const expected = target.trim().toUpperCase(); + const ok = extracted !== null && extracted === expected; + return score( + ok, + extracted, + extracted + ? `extracted '${extracted}', target '${expected}'` + : "no answer letter found" + ); + } + case "numeric": { + const actual = extractLastNumber(completion); + const expected = Number(target.replaceAll(",", "")); + if (actual === null || !Number.isFinite(expected)) { + return score( + false, + actual === null ? null : String(actual), + "no comparable number found" + ); + } + const absOk = Math.abs(actual - expected) <= scorer.absoluteTolerance; + const relOk = + scorer.relativeTolerance > 0 && + Math.abs(actual - expected) <= + Math.abs(expected) * scorer.relativeTolerance; + const ok = actual === expected || absOk || relOk; + return score( + ok, + String(actual), + ok ? "within tolerance" : `expected ${expected}, got ${actual}` + ); + } + default: { + scorer satisfies never; + return score(false, null, "unknown scorer"); + } + } +} + +export function makeCustomEvalScorer(scorer: EvalScorer): ScorerService { + return (state, target) => + sync(() => + scoreCompletion(scorer, state.output?.completion ?? "", target.text) + ); +} diff --git a/src/benchmarks/custom-eval/spec.ts b/src/benchmarks/custom-eval/spec.ts new file mode 100644 index 0000000..b23fb4c --- /dev/null +++ b/src/benchmarks/custom-eval/spec.ts @@ -0,0 +1,117 @@ +/** + * EvalSpec v1: a fully declarative eval definition — dataset + prompt + + * deterministic scorer. User evals are **data, not code**, so they run on the + * existing worker fleet with no isolation concerns. This is the rung-1 surface + * of the evals product plan (docs/plans/2026-07-28-001-feat-evals-product- + * staging-plan.md); user-TypeScript evals are a later rung. + * + * Workflow-safe: schema-only module, no heavy imports. + */ +import { z } from "../../internal/zod"; + +/** One inline case: a prompt and its grading target. */ +export const InlineCaseSchema = z.object({ + id: z.string().optional(), + input: z.string(), + target: z.string(), + metadata: z.record(z.string(), z.unknown()).optional(), +}); +export type InlineCase = z.infer; + +export const InlineDatasetSchema = z.object({ + kind: z.literal("inline"), + cases: z.array(InlineCaseSchema).min(1).max(10_000), +}); + +export const HfDatasetSpecSchema = z.object({ + kind: z.literal("hf"), + /** Dataset repo id, e.g. "openai/gsm8k". */ + dataset: z + .string() + .regex( + /^[\w.-]+\/[\w.-]+$/, + "dataset must be a HuggingFace repo id (owner/name); no slashes beyond the separator" + ), + config: z.string().default("default"), + split: z.string().default("test"), + /** Record field used as the prompt input. */ + inputField: z.string(), + /** Record field used as the grading target. */ + targetField: z.string(), + /** + * Pinned dataset git revision (commit SHA). When set, the run fails closed + * if the upstream default branch has moved — an upstream dataset change is + * a new comparability series, never a silent score shift. + */ + revision: z.string().optional(), +}); + +export const EvalDatasetSchema = z.discriminatedUnion("kind", [ + InlineDatasetSchema, + HfDatasetSpecSchema, +]); +export type EvalDataset = z.infer; + +/** + * Deterministic scorers only, on purpose: every verdict is reproducible from + * the stored trajectory. An LLM-judge scorer is a deliberate follow-up — it + * needs judge pinning and `judge-dependent` labeling on results before + * customer exposure. + */ +export const EvalScorerSchema = z.discriminatedUnion("kind", [ + z.object({ + kind: z.literal("exact"), + caseSensitive: z.boolean().default(false), + trim: z.boolean().default(true), + }), + z.object({ + kind: z.literal("contains"), + caseSensitive: z.boolean().default(false), + }), + z.object({ + kind: z.literal("regex"), + /** Correct when the completion matches. Compiled with the `u` flag. */ + pattern: z.string().refine( + (value) => { + try { + void new RegExp(value, "u"); + return true; + } catch { + return false; + } + }, + { + message: + "pattern must be a valid regular expression (compiled with the u flag)", + } + ), + caseSensitive: z.boolean().default(false), + }), + z.object({ + /** MCQ letter extraction (shared with gpqa/mmlu): target is a letter A-Z. */ + kind: z.literal("choice"), + }), + z.object({ + /** Compares the last number in the completion to the numeric target. */ + kind: z.literal("numeric"), + absoluteTolerance: z.number().nonnegative().default(0), + relativeTolerance: z.number().nonnegative().default(0), + }), +]); +export type EvalScorer = z.infer; + +export const EvalSpecSchema = z.object({ + specVersion: z.literal(1).default(1), + /** Display name; not part of scoring identity. */ + name: z.string().min(1).max(200), + dataset: EvalDatasetSchema, + /** + * Prompt template applied to each case's input. `{input}` is replaced with + * the case input; omitted → the raw input is the user message. + */ + promptTemplate: z.string().optional(), + /** Optional system message prepended to every conversation. */ + systemPrompt: z.string().optional(), + scorer: EvalScorerSchema, +}); +export type EvalSpec = z.infer; diff --git a/src/benchmarks/registry.ts b/src/benchmarks/registry.ts index a7d0121..967e7a3 100644 --- a/src/benchmarks/registry.ts +++ b/src/benchmarks/registry.ts @@ -1,3 +1,4 @@ +import { CUSTOM_EVAL_BENCHMARK } from "./custom-eval/benchmark"; import { DEEP_SWE_BENCHMARK } from "./deep-swe/benchmark"; import { DRACO_BENCHMARK } from "./draco/benchmark"; import { GPQA_BENCHMARK } from "./gpqa"; @@ -28,6 +29,7 @@ const BENCHMARKS: Record = { [TERMINAL_BENCH_BENCHMARK.id]: TERMINAL_BENCH_BENCHMARK, [DRACO_BENCHMARK.id]: DRACO_BENCHMARK, [IFSTRUCT_BENCHMARK.id]: IFSTRUCT_BENCHMARK, + [CUSTOM_EVAL_BENCHMARK.id]: CUSTOM_EVAL_BENCHMARK, [SWE_ATLAS_QA_BENCHMARK.id]: SWE_ATLAS_QA_BENCHMARK, [SWE_ATLAS_TW_BENCHMARK.id]: SWE_ATLAS_TW_BENCHMARK, [SWE_ATLAS_RF_BENCHMARK.id]: SWE_ATLAS_RF_BENCHMARK, diff --git a/src/benchmarks/types.ts b/src/benchmarks/types.ts index ddf5b34..53feb27 100644 --- a/src/benchmarks/types.ts +++ b/src/benchmarks/types.ts @@ -17,6 +17,7 @@ export interface BenchmarkRunInput { readonly sessionId: string; readonly datasetRetry?: RetryConfig; readonly modelRetry?: RetryConfig; + /** Override the benchmark's default Model layer (e.g. TRINITY routing). */ readonly modelLayer?: Layer; readonly responsesModelLayer?: Layer< ResponsesModel, @@ -33,6 +34,16 @@ export interface BenchmarkPrimaryScore { export interface Benchmark { readonly id: string; readonly makeDatasetLayer: (retryConfig?: RetryConfig) => Layer; + /** + * Config-aware dataset layer for benchmarks whose dataset lives in the run + * config (custom_eval). Size probing MUST use this when defined — the + * config-free `makeDatasetLayer` cannot know the real dataset and must not + * be allowed to answer for it. + */ + readonly makeDatasetLayerForConfig?: ( + config: BenchmarkRunConfig, + retryConfig?: RetryConfig + ) => Layer; readonly makeLayer: ( input: BenchmarkRunInput ) => Layer; diff --git a/src/datasets/huggingface.test.ts b/src/datasets/huggingface.test.ts index 2c4eb96..88bd7c5 100644 --- a/src/datasets/huggingface.test.ts +++ b/src/datasets/huggingface.test.ts @@ -34,7 +34,7 @@ const headersByRequest: Record[] = []; let restoreFetch: (() => void) | undefined; -function stubFetch(response: unknown): void { +function stubFetch(response: unknown, datasetInfo?: unknown): void { const original = globalThis.fetch; const stub: typeof fetch = (input, init) => { const req = @@ -44,8 +44,11 @@ function stubFetch(response: unknown): void { headers[key] = value; }); headersByRequest.push(headers); + const body = req.url.includes("/api/datasets/") + ? (datasetInfo ?? { sha: "abc" }) + : response; return Promise.resolve( - new Response(JSON.stringify(response), { + new Response(JSON.stringify(body), { status: 200, headers: { "content-type": "application/json" }, }) @@ -111,6 +114,72 @@ describe("makeHfDatasetLayer", () => { expect(headersByRequest[0]?.["authorization"]).toBeUndefined(); }); }); + +describe("revision pinning", () => { + afterEach(() => { + restoreFetch?.(); + headersByRequest.length = 0; + }); + + it("streams normally when the pinned revision matches upstream", async () => { + stubFetch(rowsPage({ numRowsTotal: 3, rows: 1 }), { sha: "pinned-sha" }); + const layer = makeHfDatasetLayer({ + dataset: "o/d", + config: "default", + split: "test", + revision: "pinned-sha", + recordToSample: (_record, index) => ({ + id: String(index), + input: "", + target: { text: "" }, + }), + hfToken: "", + retry: { baseDelayMs: 0 }, + }); + expect(await fetchOnceWithLayer(layer)).toBe(3); + }); + + it("fails closed with both SHAs when upstream moved past the pinned revision", async () => { + stubFetch(rowsPage({ numRowsTotal: 3, rows: 1 }), { sha: "newer-sha" }); + const layer = makeHfDatasetLayer({ + dataset: "o/d", + config: "default", + split: "test", + revision: "pinned-sha", + recordToSample: (_record, index) => ({ + id: String(index), + input: "", + target: { text: "" }, + }), + hfToken: "", + retry: { baseDelayMs: 0 }, + }); + await expect(fetchOnceWithLayer(layer)).rejects.toThrow( + /pinned pinned-sha.*newer-sha/s + ); + }); + + it("skips verification entirely when no revision is pinned", async () => { + stubFetch(rowsPage({ numRowsTotal: 2, rows: 1 })); + const layer = makeHfDatasetLayer({ + dataset: "o/d", + config: "default", + split: "test", + recordToSample: (_record, index) => ({ + id: String(index), + input: "", + target: { text: "" }, + }), + hfToken: "", + retry: { baseDelayMs: 0 }, + }); + expect(await fetchOnceWithLayer(layer)).toBe(2); + const infoCalls = headersByRequest.length; + /* one /rows call only — no dataset-info request was made */ + expect(infoCalls).toBe(1); + }); +}); + describe("hfFetchRetrySchedule", () => { let restoreWarn: (() => void) | undefined; afterEach(() => { diff --git a/src/datasets/huggingface.ts b/src/datasets/huggingface.ts index 10fa8ed..b5d59eb 100644 --- a/src/datasets/huggingface.ts +++ b/src/datasets/huggingface.ts @@ -3,6 +3,7 @@ import { fromIterable } from "effect/Chunk"; import { map as configMap, option, string } from "effect/Config"; import type { Effect } from "effect/Effect"; import { + cached, fail, flatMap, gen, @@ -24,7 +25,11 @@ import { whileInput, } from "effect/Schedule"; import type { Stream } from "effect/Stream"; -import { paginateChunkEffect } from "effect/Stream"; +import { + flatMap as streamFlatMap, + fromEffect, + paginateChunkEffect, +} from "effect/Stream"; import type { Sample } from "../harness/core"; import { DatasetError } from "../harness/core"; @@ -38,6 +43,7 @@ import { withRetryAttemptLogging } from "../runtime/retry"; const HF_MAX_PAGE_SIZE = 100; const HF_ROWS_BASE_URL = "https://datasets-server.huggingface.co/rows"; +const HF_DATASET_INFO_BASE_URL = "https://huggingface.co/api/datasets"; export function hfFetchRetrySchedule( config: RetryConfig = {}, @@ -80,6 +86,14 @@ export interface HfDatasetConfig { readonly pageSize?: number; readonly retry?: RetryConfig; readonly hfToken?: string; + /** + * Expected dataset git revision (commit SHA). The Dataset Viewer /rows + * endpoint only serves the default branch, so pinning is enforced by + * VERIFICATION: before streaming, the dataset's current revision is fetched + * and compared; a mismatch fails closed with the observed SHA so the + * mismatch is explicit rather than silently scoring a different dataset. + */ + readonly revision?: string; } export const HfImageSchema = z.object({ @@ -211,6 +225,63 @@ export function paginateHfRows(opts: { ); } +/** + * Build a Dataset Layer backed by the HF Dataset Viewer /rows API. The stream + * is paginated and backpressured: pages are fetched only as the consumer pulls, + * so peak memory is one page plus whatever the run pipeline holds in flight. + */ +const HfDatasetInfoSchema = z.object({ sha: z.string() }); + +/** + * Fail closed when the dataset's current default-branch revision differs from + * the pinned one. The Dataset Viewer /rows endpoint always serves the default + * branch, so this check is what turns `revision` from documentation into an + * enforced comparability guarantee: an upstream dataset push (e.g. label + * fixes) fails the run with both SHAs instead of silently scoring different + * data under an unchanged identity digest. + */ +export function verifyHfRevision( + config: Pick, + client: HttpClient.HttpClient +): Effect { + const pinned = config.revision; + if (pinned === undefined) { + return succeed(undefined); + } + const fetchRetry = hfFetchRetrySchedule(config.retry); + return client.get(`${HF_DATASET_INFO_BASE_URL}/${config.dataset}`).pipe( + flatMap((response) => response.json), + retry(fetchRetry), + mapError( + (cause) => + new DatasetError({ + message: `HF dataset-info request failed for ${config.dataset}: ${String(cause)}`, + }) + ), + flatMap((body) => { + const parsed = parseSchema(HfDatasetInfoSchema, body); + if (Either.isLeft(parsed)) { + return fail( + new DatasetError({ + message: `HF dataset-info response failed validation for ${config.dataset}: ${parsed.left.message}`, + }) + ); + } + if (parsed.right.sha !== pinned) { + return fail( + new DatasetError({ + message: + `HF dataset ${config.dataset} revision mismatch: pinned ${pinned}, ` + + `upstream default branch is at ${parsed.right.sha}. The dataset changed ` + + `upstream; re-pin the revision (new comparability series) or investigate.`, + }) + ); + } + return succeed(undefined); + }) + ); +} + export function makeHfDatasetLayer(config: HfDatasetConfig): Layer { const pageSize = Math.min( config.pageSize ?? HF_MAX_PAGE_SIZE, @@ -219,20 +290,31 @@ export function makeHfDatasetLayer(config: HfDatasetConfig): Layer { const makeService = gen(function* () { const client = yield* HttpClient.HttpClient; const fetchPage = makeHfPageFetcher(config, client); - const sizeEffect: Effect = fetchPage(0, 1).pipe( + /* Verification runs inside size/stream — the Dataset error channel + * already carries DatasetError, while layer construction must stay + * infallible. `cached` memoizes the check so one run makes at most one + * dataset-info request no matter how many size/stream calls follow. */ + const verified = yield* cached(verifyHfRevision(config, client)); + + const sizeEffect: Effect = verified.pipe( + flatMap(() => fetchPage(0, 1)), map((page) => page.num_rows_total) ); const stream = ( opts?: DatasetStreamOptions ): Stream => { - return paginateHfRows({ - fetchPage, - pageSize, - dataset: config.dataset, - start: opts?.start, - end: opts?.end, - mapRow: (row, index) => config.recordToSample(row.row, index), - }); + return fromEffect(verified).pipe( + streamFlatMap(() => + paginateHfRows({ + fetchPage, + pageSize, + dataset: config.dataset, + start: opts?.start, + end: opts?.end, + mapRow: (row, index) => config.recordToSample(row.row, index), + }) + ) + ); }; return Dataset.of({ stream, size: sizeEffect }); }); diff --git a/src/runner/run-by-id.ts b/src/runner/run-by-id.ts index 6cebe7e..7a748d2 100644 --- a/src/runner/run-by-id.ts +++ b/src/runner/run-by-id.ts @@ -148,13 +148,31 @@ export function runBenchmarkById( } export function datasetSizeById( - benchmarkId: string + benchmarkId: string, + benchmarkConfig?: BenchmarkRunConfig ): AsyncEither { const benchmark = getBenchmark(benchmarkId); if (benchmark === undefined) { return Promise.resolve(Either.left(`Unknown benchmark "${benchmarkId}"`)); } - const datasetLayer = benchmark.makeDatasetLayer(); + /* Config-bound datasets (custom_eval) can only be sized through the run + config; probing the config-free layer would size a placeholder and + silently truncate the run to its first chunk. */ + if ( + benchmark.makeDatasetLayerForConfig !== undefined && + benchmarkConfig === undefined + ) { + return Promise.resolve( + Either.left( + `benchmark "${benchmarkId}" needs a run config to resolve its dataset size` + ) + ); + } + const datasetLayer = + benchmark.makeDatasetLayerForConfig !== undefined && + benchmarkConfig !== undefined + ? benchmark.makeDatasetLayerForConfig(benchmarkConfig) + : benchmark.makeDatasetLayer(); const program = Dataset.pipe(flatMap((d) => d.size)); return runHarnessPromise(program.pipe(provide(datasetLayer))) .then((size) => Either.right(size))