diff --git a/bun.lock b/bun.lock index 2ae760b..c0ff543 100644 --- a/bun.lock +++ b/bun.lock @@ -9,6 +9,7 @@ "@google-cloud/storage": "7.19.0", "@openrouter/sdk": "1.2.9", "change-case": "5.4.4", + "chess.js": "1.4.0", "chrono-node": "2.9.0", "cli-progress": "3.12.0", "csv-parse": "5.6.0", @@ -297,6 +298,8 @@ "change-case": ["change-case@5.4.4", "", {}, "sha512-HRQyTk2/YPEkt9TnUPbOpr64Uw3KOicFWPVBb+xiHvd6eBx/qPr9xqfBFDT8P2vWsvvz4jbEkfDe71W3VyNu2w=="], + "chess.js": ["chess.js@1.4.0", "", {}, "sha512-BBJgrrtKQOzFLonR0l+k64A98NLemPwNsCskwb+29bRwobUa4iTm51E1kwGPbWXAcfdDa18nad6vpPPKPWarqw=="], + "chrono-node": ["chrono-node@2.9.0", "", {}, "sha512-glI4YY2Jy6JII5l3d5FN6rcrIbKSQqKPhWsIRYPK2IK8Mm4Q1ZZFdYIaDqglUNf7gNwG+kWIzTn0omzzE0VkvQ=="], "citty": ["citty@0.2.2", "", {}, "sha512-+6vJA3L98yv+IdfKGZHBNiGW5KHn22e/JwID0Strsz8h4S/csAu/OuICwxrg44k5MRiZHWIo8XXuJgQTriRP4w=="], diff --git a/package.json b/package.json index b5e1ecf..30f4927 100644 --- a/package.json +++ b/package.json @@ -44,6 +44,7 @@ "@google-cloud/storage": "7.19.0", "@openrouter/sdk": "1.2.9", "change-case": "5.4.4", + "chess.js": "1.4.0", "chrono-node": "2.9.0", "cli-progress": "3.12.0", "csv-parse": "5.6.0", diff --git a/src/benchmarks/benchmark-config.ts b/src/benchmarks/benchmark-config.ts index 9dba017..7cda003 100644 --- a/src/benchmarks/benchmark-config.ts +++ b/src/benchmarks/benchmark-config.ts @@ -182,6 +182,18 @@ export type CustomEvalBenchmarkConfig = z.infer< typeof CustomEvalBenchmarkConfigSchema >; +/** Chess has no per-run options: the five tasks are fixed dataset rows. */ +export const ChessOptionsSchema = z.object({}); + +export const ChessBenchmarkConfigSchema = z.object({ + benchmarkId: z.literal("chess"), + /* Fixed-temperature base: the solver pins temperature 0 for move + * determinism, so the config must not accept (and silently drop) one. */ + ...FixedTemperatureBenchmarkBaseSchema.shape, + ...ChessOptionsSchema.shape, +}); +export type ChessBenchmarkConfig = z.infer; + /** * Options shared by the Modal-backed agentic benchmarks (SWE-Atlas tracks and * DeepSWE). `stepLimit` is declared per benchmark because the defaults differ. @@ -301,6 +313,7 @@ export const BenchmarkRunConfigSchema = z.discriminatedUnion("benchmarkId", [ DracoBenchmarkConfigSchema, IfStructBenchmarkConfigSchema, CustomEvalBenchmarkConfigSchema, + ChessBenchmarkConfigSchema, SweAtlasQaConfigSchema, SweAtlasTwConfigSchema, SweAtlasRfConfigSchema, @@ -338,6 +351,7 @@ export const BENCHMARK_OPTIONS_SCHEMAS = { terminal_bench: TerminalBenchOptionsSchema, ifstruct: IfStructOptionsSchema, custom_eval: CustomEvalOptionsSchema, + chess: ChessOptionsSchema, swe_atlas_qa: SweAtlasOptionsSchema, swe_atlas_tw: SweAtlasOptionsSchema, swe_atlas_rf: SweAtlasOptionsSchema, diff --git a/src/benchmarks/benchmark-meta.ts b/src/benchmarks/benchmark-meta.ts index da30ce7..115f184 100644 --- a/src/benchmarks/benchmark-meta.ts +++ b/src/benchmarks/benchmark-meta.ts @@ -55,6 +55,13 @@ export const CUSTOM_EVAL_META = { defaultEpochs: 1, } as const satisfies BenchmarkMeta; +/** Full games vs Stockfish; deterministic per-ply engine evals, temperature 0. */ +export const CHESS_META = { + id: "chess", + defaultEpochs: 1, + temperature: 0, +} as const satisfies BenchmarkMeta; + /* defaultEpochs 3 mirrors the reference run scripts' `-k 3` (3 rollouts/task). */ export const SWE_ATLAS_QA_META = { id: "swe_atlas_qa", @@ -111,6 +118,7 @@ const BENCHMARK_META: Readonly> = { [DRACO_META.id]: DRACO_META, [IFSTRUCT_META.id]: IFSTRUCT_META, [CUSTOM_EVAL_META.id]: CUSTOM_EVAL_META, + [CHESS_META.id]: CHESS_META, [SWE_ATLAS_QA_META.id]: SWE_ATLAS_QA_META, [SWE_ATLAS_TW_META.id]: SWE_ATLAS_TW_META, [SWE_ATLAS_RF_META.id]: SWE_ATLAS_RF_META, diff --git a/src/benchmarks/chess/benchmark.ts b/src/benchmarks/chess/benchmark.ts new file mode 100644 index 0000000..b0af660 --- /dev/null +++ b/src/benchmarks/chess/benchmark.ts @@ -0,0 +1,205 @@ +import { HttpClient } from "@effect/platform"; +import { gen, succeed } from "effect/Effect"; +import type { Layer } from "effect/Layer"; +import { + fail as layerFail, + effect as layerEffect, + provide as layerProvide, + mergeAll as layerMergeAll, + succeed as layerSucceed, +} from "effect/Layer"; +import type { Stream } from "effect/Stream"; +import { fromIterable } from "effect/Stream"; + +/** + * Chess benchmark — full games against a real UCI engine (Stockfish). + * + * Measures a model's ability to RETAIN THE POSITION across a complete + * back-and-forth game in SAN notation, with per-ply Stockfish evaluation and + * fully deterministic scoring (no judge model anywhere). Faithful port of the + * byo-benchmark chess bench: same protocol, extraction, adjudication rules, + * and scoring semantics. + * + * Five tasks: full game as White, as Black, with "check " probes, in + * strict-SAN mode, and a K+Q vs K endgame conversion (mate or fail). + * + * Requires a Stockfish binary (`brew install stockfish` / + * `apt-get install stockfish`, or STOCKFISH_PATH). The solver fails closed — + * per sample, before any model spend — when the engine is missing. + * + * Concurrency: every sample owns its engine pair; nothing is shared between + * games, so harness-level concurrency is bounded only by process headroom + * (2 single-threaded engine processes per in-flight game). + */ +import type { Sample } from "../../harness/core"; +import { ScoreValue } from "../../harness/core"; +import type { Dataset, DatasetStreamOptions } from "../../harness/dataset"; +import { Dataset as DatasetTag } from "../../harness/dataset"; +import { Model } from "../../harness/model"; +import type { RunResult } from "../../harness/run"; +import { Scorer } from "../../harness/scorer"; +import { Solver } from "../../harness/solver"; +import { Either } from "../../internal/either"; +import { parseSchema } from "../../internal/zod"; +import { makeOpenRouterModelLayer } from "../../providers/openrouter-model"; +import type { RetryConfig } from "../../runtime/retry"; +import { ChessBenchmarkConfigSchema } from "../benchmark-config"; +import { CHESS_META } from "../benchmark-meta"; +import type { Benchmark, BenchmarkRunInput } from "../types"; +import { CHESS_TASK_DEFINITIONS, CHESS_TASKS } from "./game"; +import { ChessGameRecordSchema } from "./schema"; +import { chessScorer } from "./scorer"; +import { chessSolver } from "./solver"; + +export const CHESS_TEMPERATURE = 0; + +//#region Dataset + +function chessTaskToSample(taskId: (typeof CHESS_TASKS)[number]): Sample { + const task = CHESS_TASK_DEFINITIONS[taskId]; + return { + id: `chess-${taskId}`, + input: `Full chess game vs Stockfish depth ${task.engineDepth} as ${ + task.modelColor === "w" ? "White" : "Black" + } (${taskId})`, + /* State-based scoring: the target is the game outcome, not a string. */ + target: { text: "" }, + metadata: { task }, + }; +} + +function makeChessDatasetLayer(_retryConfig?: RetryConfig): Layer { + const samples = CHESS_TASKS.map(chessTaskToSample); + const stream = (opts?: DatasetStreamOptions): Stream => + fromIterable(samples.slice(opts?.start ?? 0, opts?.end ?? samples.length)); + return layerSucceed( + DatasetTag, + DatasetTag.of({ stream, size: succeed(samples.length) }) + ); +} + +//#endregion + +//#region Run-level scores + +const CHESS_RUN_METRICS = [ + "points", + "acpl", + "illegalAttempts", + "blunders", + "gameLengthMoves", +] as const; + +/** + * Aggregate the deterministic per-game quality profile across the run: + * average points (chess score), ACPL, illegal attempts, blunders, and game + * length — the numbers a chess player would actually compare models by. + */ +export function chessRunLevelScores( + result: RunResult +): readonly { + name: string; + metrics: Readonly>; +}[] { + const games = result.sampleScores + .filter((sample) => sample.score.value !== ScoreValue.Skipped) + .flatMap((sample) => { + const parsed = parseSchema( + ChessGameRecordSchema, + sample.metadata?.["game"] + ); + return Either.isRight(parsed) ? [parsed.right] : []; + }); + if (games.length === 0) { + return []; + } + const average = (select: (game: (typeof games)[number]) => number): number => + games.reduce((sum, game) => sum + select(game), 0) / games.length; + const values: Record<(typeof CHESS_RUN_METRICS)[number], number> = { + points: average((game) => game.points), + acpl: average((game) => game.acpl), + illegalAttempts: average((game) => game.illegalAttempts), + blunders: average((game) => game.blunders), + gameLengthMoves: average((game) => game.gameLengthMoves), + }; + return [ + { + name: "chess", + metrics: Object.fromEntries( + CHESS_RUN_METRICS.map((key) => [key, { value: values[key] }]) + ), + }, + ]; +} + +//#endregion + +//#region Benchmark registration + +function makeLayer( + input: BenchmarkRunInput +): Layer { + const configParsed = parseSchema( + ChessBenchmarkConfigSchema, + input.benchmarkConfig + ); + if (Either.isLeft(configParsed)) { + return layerFail( + new Error( + `chess received invalid benchmarkConfig: ${configParsed.left.message}` + ) + ); + } + const config = configParsed.right; + + const modelLayer = + input.modelLayer ?? + makeOpenRouterModelLayer({ + apiKey: input.apiKey, + model: config.model, + ...(input.baseUrl !== undefined && { baseUrl: input.baseUrl }), + sessionId: input.sessionId, + ...(input.modelRetry !== undefined && { retry: input.modelRetry }), + }); + + const solverLayer = layerEffect( + Solver, + gen(function* () { + const model = yield* Model; + return chessSolver(model, { + temperature: CHESS_TEMPERATURE, + ...(config.endpointId !== undefined && { + endpointId: config.endpointId, + }), + ...(config.maxTokens !== undefined && { maxTokens: config.maxTokens }), + ...(config.reasoningEffort !== undefined && { + reasoningEffort: config.reasoningEffort, + }), + ...(config.timeoutMs !== undefined && { timeoutMs: config.timeoutMs }), + }); + }) + ).pipe(layerProvide(modelLayer)); + + return layerMergeAll( + makeChessDatasetLayer(input.datasetRetry), + solverLayer, + layerSucceed(Scorer, chessScorer) + ); +} + +export const CHESS_BENCHMARK: Benchmark = { + id: CHESS_META.id, + makeDatasetLayer: makeChessDatasetLayer, + makeLayer, + temperature: CHESS_TEMPERATURE, + defaultEpochs: CHESS_META.defaultEpochs, + /* + * A missing Stockfish binary or a UCI crash is a per-sample infrastructure + * failure, not a model failure: degrade to Incorrect rather than aborting + * the whole fan-out (mirrors terminal-bench / swe-atlas sandbox handling). + */ + degradeSolverErrors: true, + runLevelScores: chessRunLevelScores, +}; + +//#endregion diff --git a/src/benchmarks/chess/chess.test.ts b/src/benchmarks/chess/chess.test.ts new file mode 100644 index 0000000..23efb70 --- /dev/null +++ b/src/benchmarks/chess/chess.test.ts @@ -0,0 +1,248 @@ +import { describe, expect, it } from "bun:test"; + +import { Chess } from "chess.js"; + +import { ScoreValue } from "../../harness/core"; +import { runHarnessPromise } from "../../internal/effect-logger"; +import { assertRight } from "../../internal/testing"; +import { parseSchema } from "../../internal/zod"; +import { + boardResult, + capVerdict, + CHESS_TASK_DEFINITIONS, + CHESS_TASKS, + extractMove, + fromModelView, + tryMove, +} from "./game"; +import { ChessGameRecordSchema, ChessTaskSchema } from "./schema"; +import { chessScorer } from "./scorer"; + +describe("move extraction", () => { + const start = new Chess(); + + it("accepts a bare SAN move and normalizes it", () => { + expect(tryMove(start, "Nf3")).toBe("Nf3"); + expect(tryMove(start, "e4")).toBe("e4"); + }); + + it("rejects illegal moves without mutating the board", () => { + const fenBefore = start.fen(); + expect(tryMove(start, "Ke2")).toBeUndefined(); + expect(start.fen()).toBe(fenBefore); + }); + + it("extracts the first legal move from chatty replies", () => { + expect(extractMove(start, "I will play e4, controlling the center.")).toBe( + "e4" + ); + expect(extractMove(start, "My move: Nf3!")).toBe("Nf3"); + }); + + it("extracts castling and promotion notation", () => { + const castled = new Chess( + "r1bqkbnr/pppp1ppp/2n5/4p3/2B1P3/5N2/PPPP1PPP/RNBQK2R w KQkq - 4 4" + ); + expect(extractMove(castled, "O-O")).toBe("O-O"); + const promoting = new Chess("8/4P3/8/8/8/8/2k5/K7 w - - 0 1"); + expect(extractMove(promoting, "e8=Q")).toBe("e8=Q"); + }); + + it("returns undefined when no legal move is present", () => { + expect(extractMove(start, "I resign, you play too well.")).toBeUndefined(); + }); +}); + +describe("eval perspective", () => { + it("keeps sign when the side to move is the model", () => { + const start = new Chess(); + expect(fromModelView({ cp: 35 }, start.fen(), "w").cp).toBe(35); + }); + + it("flips sign when the opponent is to move", () => { + const start = new Chess(); + expect(fromModelView({ cp: 35 }, start.fen(), "b").cp).toBe(-35); + expect(fromModelView({ cp: -120, mateIn: -3 }, start.fen(), "b")).toEqual({ + cp: 120, + mateIn: 3, + }); + }); +}); + +describe("result classification", () => { + it("classifies checkmate for and against the model", () => { + /* Fool's mate: black mates white. */ + const mated = new Chess(); + for (const move of ["f3", "e5", "g4", "Qh4#"]) { + mated.move(move); + } + expect(boardResult(mated, "w")).toEqual({ + result: "checkmate-loss", + points: 0, + }); + expect(boardResult(mated, "b")).toEqual({ + result: "checkmate-win", + points: 1, + }); + }); + + it("classifies stalemate as a half point", () => { + const stalemate = new Chess("7k/5Q2/6K1/8/8/8/8/8 b - - 0 1"); + expect(boardResult(stalemate, "w")).toEqual({ + result: "stalemate", + points: 0.5, + }); + }); + + it("adjudicates the maxPlies cap by final eval", () => { + expect(capVerdict(450)).toEqual({ result: "adjudicated-win", points: 1 }); + expect(capVerdict(-450)).toEqual({ result: "adjudicated-loss", points: 0 }); + expect(capVerdict(120)).toEqual({ + result: "draw-agreed-adjudication", + points: 0.5, + }); + }); +}); + +describe("task definitions", () => { + it("every task parses through the metadata schema", () => { + for (const taskId of CHESS_TASKS) { + assertRight(parseSchema(ChessTaskSchema, CHESS_TASK_DEFINITIONS[taskId])); + } + }); + + it("the endgame task starts from the K+Q vs K FEN and requires mate", () => { + const endgame = CHESS_TASK_DEFINITIONS["endgame-conversion"]; + expect(endgame.fen).toBe("4k3/8/4K3/8/8/8/8/4Q3 w - - 0 1"); + expect(endgame.requireMate).toBe(true); + expect(endgame.maxPlies).toBe(30); + }); +}); + +function gameRecord( + overrides: Record = {} +): Record { + return { + taskId: "stockfish-full", + modelMoves: ["e4", "Nf3"], + plies: [], + illegalAttempts: 0, + strictViolations: 0, + checksUsed: 0, + forfeited: false, + result: "checkmate-win", + points: 1, + gameLengthMoves: 20, + acpl: 45, + worstCpLoss: 130, + blunders: 0, + finalEvalCp: 900, + finalFen: "8/8/8/8/8/8/8/K1k5 w - - 0 1", + pgn: "1. e4", + turnCosts: [ + { + iteration: 0, + turn: 0, + generationId: "gen-abc", + inputTokens: 120, + outputTokens: 4, + totalTokens: 124, + reasoningTokens: 0, + costUsd: 0.0004, + generationTimeMs: 800, + }, + { + iteration: 1, + turn: 1, + inputTokens: 160, + outputTokens: 4, + totalTokens: 164, + reasoningTokens: 0, + costUsd: 0.0005, + generationTimeMs: 750, + }, + ], + totalCostUsd: 0.0009, + ...overrides, + }; +} + +describe("chessScorer", () => { + const score = (metadata: Record) => + runHarnessPromise( + chessScorer( + { + sample: { id: "chess-x", input: "", target: { text: "" }, metadata }, + messages: [], + completed: true, + }, + { text: "" } + ) + ); + + it("scores a win Correct with the quality profile in the explanation", async () => { + const result = await score({ game: gameRecord() }); + expect(result.value).toBe(ScoreValue.Correct); + expect(result.explanation).toContain("ACPL 45cp"); + }); + + it("scores a draw Correct (points >= 0.5) on standard tasks", async () => { + const result = await score({ + game: gameRecord({ result: "stalemate", points: 0.5 }), + }); + expect(result.value).toBe(ScoreValue.Correct); + }); + + it("scores a loss and a forfeit Incorrect", async () => { + const loss = await score({ + game: gameRecord({ result: "checkmate-loss", points: 0 }), + }); + expect(loss.value).toBe(ScoreValue.Incorrect); + const forfeit = await score({ + game: gameRecord({ result: "forfeit", points: 0, forfeited: true }), + }); + expect(forfeit.value).toBe(ScoreValue.Incorrect); + }); + + it("endgame-conversion requires checkmate — a draw scores Incorrect", async () => { + const drawn = await score({ + game: gameRecord({ + taskId: "endgame-conversion", + result: "stalemate", + points: 0.5, + }), + }); + expect(drawn.value).toBe(ScoreValue.Incorrect); + expect(drawn.explanation).toContain("failed to convert"); + const mated = await score({ + game: gameRecord({ + taskId: "endgame-conversion", + result: "checkmate-win", + points: 1, + }), + }); + expect(mated.value).toBe(ScoreValue.Correct); + }); + + it("scores Incorrect when the game record is missing or malformed", async () => { + const missing = await score({}); + expect(missing.value).toBe(ScoreValue.Incorrect); + expect(missing.explanation).toContain("no game record"); + }); + + it("the record round-trips through its schema", () => { + assertRight(parseSchema(ChessGameRecordSchema, gameRecord())); + }); + + it("turn costs are the atomic spend ledger: totalCostUsd equals their sum", () => { + const parsed = parseSchema(ChessGameRecordSchema, gameRecord()); + assertRight(parsed); + const summed = parsed.right.turnCosts.reduce( + (acc, cost) => acc + cost.costUsd, + 0 + ); + expect(parsed.right.totalCostUsd).toBeCloseTo(summed, 10); + /* Each iteration is billing-joinable when the API returned an id. */ + expect(parsed.right.turnCosts[0]?.generationId).toBe("gen-abc"); + }); +}); diff --git a/src/benchmarks/chess/game.ts b/src/benchmarks/chess/game.ts new file mode 100644 index 0000000..7679bac --- /dev/null +++ b/src/benchmarks/chess/game.ts @@ -0,0 +1,263 @@ +import { Chess } from "chess.js"; + +/** + * Chess game state machine — everything deterministic about a game, kept + * separate from the solver so move extraction, result classification, and + * adjudication rules are unit-testable without an engine or a model. + * + * Faithful port of the byo-benchmark chess bench (examples/benchmarks/chess): + * same SAN extraction, same adjudication thresholds, same result taxonomy, + * same scoring semantics (points 1/0.5/0, ACPL, blunders). + */ +import type { ValueOf } from "../../internal/guards"; +import type { UciScore } from "./uci"; + +export const CHESS_TASKS = [ + "stockfish-full", + "stockfish-full-black", + "stockfish-full-validated", + "stockfish-full-strict", + "endgame-conversion", +] as const; +export type ChessTaskId = ValueOf; + +export interface ChessTask { + readonly id: ChessTaskId; + /** Opponent search depth (deterministic weak opponent at low depth). */ + readonly engineDepth: number; + /** Stockfish evaluation depth (position evals + best-move baseline). */ + readonly evalDepth: number; + /** Optional starting FEN (endgames). */ + readonly fen?: string; + /** Which side the model plays. */ + readonly modelColor: "w" | "b"; + /** Safety cap on model moves; hitting it adjudicates by final eval. */ + readonly maxPlies: number; + /** Allow "check " legality probes before committing a move. */ + readonly moveValidation?: boolean; + /** Reply must be exactly the SAN move — no lenient extraction. */ + readonly strict?: boolean; + /** endgame-conversion: only checkmate-win scores Correct. */ + readonly requireMate?: boolean; +} + +/** + * Full games vs a depth-2 opponent. maxPlies=120 is a cost/safety cap, not a + * target — depth-2 games typically resolve well before it (mate, draw rule, + * or the −900cp adjudication). + */ +const STANDARD = { + engineDepth: 2, + evalDepth: 10, + modelColor: "w", + maxPlies: 120, +} as const; + +export const CHESS_TASK_DEFINITIONS: Readonly> = + { + "stockfish-full": { id: "stockfish-full", ...STANDARD }, + "stockfish-full-black": { + id: "stockfish-full-black", + ...STANDARD, + modelColor: "b", + }, + "stockfish-full-validated": { + id: "stockfish-full-validated", + ...STANDARD, + moveValidation: true, + }, + "stockfish-full-strict": { + id: "stockfish-full-strict", + ...STANDARD, + strict: true, + }, + "endgame-conversion": { + id: "endgame-conversion", + ...STANDARD, + fen: "4k3/8/4K3/8/8/8/8/4Q3 w - - 0 1", + maxPlies: 30, + requireMate: true, + }, + }; + +// ---------- move extraction ---------- + +const SAN_RE = /O-O-O|O-O|[KQRBN]?[a-h]?[1-8]?x?[a-h][1-8](?:=[QRBN])?[+#]?/g; + +/** Try a move on a probe board; returns normalized SAN or undefined. */ +export function tryMove(game: Chess, move: string): string | undefined { + const probe = new Chess(game.fen()); + try { + return probe.move(move).san; + } catch { + return undefined; + } +} + +/** + * Strict mode: the reply must BE the canonical SAN for a legal move — + * chess.js's parser is lenient (accepts e2e4, Ng1f3, sloppy check marks), + * so legality alone would let non-SAN slip through unpenalized. The reply + * must round-trip to itself modulo an omitted +/# suffix. + */ +export function tryStrictMove(game: Chess, reply: string): string | undefined { + const san = tryMove(game, reply); + if (san === undefined) { + return undefined; + } + return reply === san || reply === san.replace(/[+#]$/, "") ? san : undefined; +} + +/** Lenient: first token in the reply that is a legal move. */ +export function extractMove(game: Chess, reply: string): string | undefined { + for (const candidate of [reply.trim(), ...(reply.match(SAN_RE) ?? [])]) { + const san = tryMove(game, candidate); + if (san !== undefined) { + return san; + } + } + return undefined; +} + +/** Convert side-to-move UCI score to the model's perspective. */ +export function fromModelView( + score: UciScore, + fen: string, + modelColor: "w" | "b" +): { cp: number; mateIn?: number } { + const sideToMove = fen.split(" ")[1]; + const sign = sideToMove === modelColor ? 1 : -1; + return { + cp: sign * score.cp, + ...(score.mateIn !== undefined && { mateIn: sign * score.mateIn }), + }; +} + +// ---------- adjudication (engine-tournament style) ---------- + +export const MAX_CHECKS_PER_TURN = 3; +/** Position must stay at/below this (model view) to accumulate hopeless turns. */ +export const ADJUDICATE_LOSS_CP = -900; +/** Consecutive hopeless model turns before adjudicating the loss. */ +export const ADJUDICATE_LOSS_TURNS = 3; +/** |eval| needed to call a decisive result when the maxPlies cap is hit. */ +export const ADJUDICATE_CAP_CP = 400; +/** A model move losing at least this many centipawns counts as a blunder. */ +export const BLUNDER_CP = 200; +/** cpLoss is clamped here so one catastrophe can't dominate ACPL unboundedly. */ +export const CP_LOSS_CLAMP = 1000; + +export const GAME_RESULTS = [ + "checkmate-win", + "checkmate-loss", + "stalemate", + "draw-repetition", + "draw-insufficient", + "draw-fifty-moves", + "adjudicated-win", + "adjudicated-loss", + "draw-agreed-adjudication", + "forfeit", +] as const; +export type GameResult = ValueOf; + +export interface GameVerdict { + readonly result: GameResult; + readonly points: number; +} + +/** Classify a finished (game-over) board from the model's perspective. */ +export function boardResult(game: Chess, modelColor: "w" | "b"): GameVerdict { + if (game.isCheckmate()) { + const modelMated = game.turn() === modelColor; + return { + result: modelMated ? "checkmate-loss" : "checkmate-win", + points: modelMated ? 0 : 1, + }; + } + if (game.isStalemate()) { + return { result: "stalemate", points: 0.5 }; + } + if (game.isThreefoldRepetition()) { + return { result: "draw-repetition", points: 0.5 }; + } + if (game.isInsufficientMaterial()) { + return { result: "draw-insufficient", points: 0.5 }; + } + // chess.js isDraw() covers the 50-move rule once the above are excluded. + return { result: "draw-fifty-moves", points: 0.5 }; +} + +/** Adjudicate a live game that hit the maxPlies safety cap, by final eval. */ +export function capVerdict(finalCp: number): GameVerdict { + if (finalCp >= ADJUDICATE_CAP_CP) { + return { result: "adjudicated-win", points: 1 }; + } + if (finalCp <= -ADJUDICATE_CAP_CP) { + return { result: "adjudicated-loss", points: 0 }; + } + return { result: "draw-agreed-adjudication", points: 0.5 }; +} + +// ---------- per-game record ---------- + +/** One ply as recorded in the output: validation + evaluation at every stage. */ +export interface PlyRecord { + readonly ply: number; + readonly by: "model" | "engine"; + readonly san: string; + readonly fenAfter: string; + /** Stockfish eval after this ply, centipawns from the MODEL's perspective. */ + readonly evalCp: number; + /** Mate distance if the evaluator sees one (signed, model perspective). */ + readonly mateIn?: number; + /** Model plies only: Stockfish's preferred move in this position (SAN). */ + readonly bestMove?: string; + /** Model plies only: centipawns lost vs playing bestMove (0 = perfect). */ + readonly cpLoss?: number; +} + +/** + * Cost/usage for ONE model call (iteration) in the game — the atomic unit of + * spend. `turn` is the game turn the call served; a turn can have several + * iterations (illegal-move retry, "check " probes). `generationId` + * joins to billing, so reported cost is auditable against actual charges + * per iteration rather than a self-reported total. + */ +export interface TurnCost { + readonly iteration: number; + readonly turn: number; + readonly generationId?: string; + readonly inputTokens: number; + readonly outputTokens: number; + readonly totalTokens: number; + readonly reasoningTokens: number; + readonly costUsd: number; + readonly generationTimeMs: number; +} + +export interface ChessGameRecord { + readonly taskId: ChessTaskId; + readonly modelMoves: readonly string[]; + readonly plies: readonly PlyRecord[]; + readonly illegalAttempts: number; + readonly strictViolations: number; + readonly checksUsed: number; + readonly forfeited: boolean; + readonly result: GameResult; + /** Chess scoring: 1 = model won, 0.5 = draw, 0 = loss/forfeit. */ + readonly points: number; + readonly gameLengthMoves: number; + /** Average centipawn loss across the model's moves (engine-perfect = 0). */ + readonly acpl: number; + readonly worstCpLoss: number; + /** Model moves losing ≥ BLUNDER_CP (blunder count). */ + readonly blunders: number; + readonly finalEvalCp: number; + readonly finalFen: string; + readonly pgn: string; + /** Per-iteration cost ledger: one row per model call, billing-joinable. */ + readonly turnCosts: readonly TurnCost[]; + /** Sum of turnCosts[].costUsd — must equal the sample's reported usage. */ + readonly totalCostUsd: number; +} diff --git a/src/benchmarks/chess/schema.ts b/src/benchmarks/chess/schema.ts new file mode 100644 index 0000000..2d6ad23 --- /dev/null +++ b/src/benchmarks/chess/schema.ts @@ -0,0 +1,66 @@ +import { z } from "../../internal/zod"; +/** + * Zod schemas for chess task metadata and the per-game record. The task + * schema round-trips through sample metadata (dataset → solver), and the + * game-record schema is what the scorer reads back — parse, don't cast. + */ +import type { ChessGameRecord, ChessTask, PlyRecord, TurnCost } from "./game"; +import { CHESS_TASKS, GAME_RESULTS } from "./game"; + +export const ChessTaskSchema = z.object({ + id: z.enum(CHESS_TASKS), + engineDepth: z.number().int().min(1), + evalDepth: z.number().int().min(1), + fen: z.string().optional(), + modelColor: z.enum(["w", "b"]), + maxPlies: z.number().int().min(1), + moveValidation: z.boolean().optional(), + strict: z.boolean().optional(), + requireMate: z.boolean().optional(), +}) satisfies z.ZodType; +export type ChessTaskParsed = z.infer; + +export const ChessPlyRecordSchema = z.object({ + ply: z.number().int(), + by: z.enum(["model", "engine"]), + san: z.string(), + fenAfter: z.string(), + evalCp: z.number(), + mateIn: z.number().optional(), + bestMove: z.string().optional(), + cpLoss: z.number().optional(), +}) satisfies z.ZodType; + +export const ChessTurnCostSchema = z.object({ + iteration: z.number().int(), + turn: z.number().int(), + generationId: z.string().optional(), + inputTokens: z.number(), + outputTokens: z.number(), + totalTokens: z.number(), + reasoningTokens: z.number(), + costUsd: z.number(), + generationTimeMs: z.number(), +}) satisfies z.ZodType; + +export const ChessGameRecordSchema = z.object({ + taskId: z.enum(CHESS_TASKS), + modelMoves: z.array(z.string()), + plies: z.array(ChessPlyRecordSchema), + illegalAttempts: z.number().int(), + strictViolations: z.number().int(), + checksUsed: z.number().int(), + forfeited: z.boolean(), + result: z.enum(GAME_RESULTS), + points: z.number(), + gameLengthMoves: z.number().int(), + acpl: z.number(), + worstCpLoss: z.number(), + blunders: z.number().int(), + finalEvalCp: z.number(), + finalFen: z.string(), + pgn: z.string(), + turnCosts: z.array(ChessTurnCostSchema), + totalCostUsd: z.number(), +}) satisfies z.ZodType; +export type ChessGameRecordParsed = z.infer; diff --git a/src/benchmarks/chess/scorer.ts b/src/benchmarks/chess/scorer.ts new file mode 100644 index 0000000..08d11b3 --- /dev/null +++ b/src/benchmarks/chess/scorer.ts @@ -0,0 +1,62 @@ +import { succeed } from "effect/Effect"; + +/** + * Chess scorer — fully deterministic, no judge model. + * + * Primary verdict: game POINTS (win 1 / draw 0.5 / loss 0), thresholded at + * a draw: a game the model didn't lose scores Correct. endgame-conversion is + * stricter — only checkmate-win counts, because K+Q vs K is trivially won + * and anything else means the model lost the position in its head. + * + * The full quality profile (ACPL, blunders, illegal attempts, worst move, + * result taxonomy) rides on the explanation and run-level scores so the + * numbers behind the verdict stay reviewable per sample. + */ +import type { Score } from "../../harness/core"; +import { ScoreValue } from "../../harness/core"; +import type { ScorerService } from "../../harness/scorer"; +import { Either } from "../../internal/either"; +import { parseSchema } from "../../internal/zod"; +import { ChessGameRecordSchema } from "./schema"; + +export const chessScorer: ScorerService = (state) => { + const parsed = parseSchema( + ChessGameRecordSchema, + state.sample.metadata?.["game"] + ); + if (Either.isLeft(parsed)) { + const score: Score = { + value: ScoreValue.Incorrect, + answer: null, + explanation: `no game record in sample metadata: ${parsed.left.message}`, + }; + return succeed(score); + } + const game = parsed.right; + + const requireMate = game.taskId === "endgame-conversion"; + const won = requireMate + ? game.result === "checkmate-win" + : game.points >= 0.5; + + const quality = + game.modelMoves.length > 0 + ? `ACPL ${game.acpl}cp, worst ${game.worstCpLoss}cp, ${game.blunders} blunder(s), ` + + `${game.illegalAttempts} illegal attempt(s) over ${game.modelMoves.length} move(s)` + : "no moves played"; + + let explanation = `${game.result}: ${quality}`; + if (requireMate) { + explanation = + game.result === "checkmate-win" + ? `mated in ${game.modelMoves.length} — ${quality}` + : `failed to convert a trivially won endgame (${game.result}) — ${quality}`; + } + + const score: Score = { + value: won ? ScoreValue.Correct : ScoreValue.Incorrect, + answer: `${game.result} (${game.points} pts, ${game.gameLengthMoves} moves)`, + explanation, + }; + return succeed(score); +}; diff --git a/src/benchmarks/chess/solver.ts b/src/benchmarks/chess/solver.ts new file mode 100644 index 0000000..cb3aea7 --- /dev/null +++ b/src/benchmarks/chess/solver.ts @@ -0,0 +1,478 @@ +import { Chess } from "chess.js"; +import type { Effect } from "effect/Effect"; +import { + either as effectEither, + ensuring, + fail as effectFail, + gen, + sync, + tryPromise, +} from "effect/Effect"; + +import type { ReasoningEffort } from "../../harness/constants"; +/** + * Chess solver: plays a full game against a real UCI engine (Stockfish), + * one model turn at a time. The model never sees a board or FEN after the + * first message — only the move history — so every legal move is evidence it + * still knows where the pieces are. + * + * Games run END TO END: checkmate, draw (50-move / repetition / stalemate / + * insufficient material), forfeit (two illegal replies on one turn), or + * adjudication. Engine-tournament adjudication keeps cost bounded: a position + * that stays hopeless (≤ −900cp) for 3 consecutive model turns is an + * adjudicated loss; hitting the maxPlies cap adjudicates by final eval. + * + * Every ply is validated with chess.js and evaluated by Stockfish at fixed + * depth (independent of the opponent's search, so evals are comparable across + * cases and runs). Scoring is fully deterministic — no judge model anywhere. + * + * Concurrency: each sample spawns its own opponent + evaluator engine pair + * (Threads=1 each) and quits them in ensuring(). Games share NOTHING — no + * engine handles, no boards, no mutable module state — so any harness + * concurrency level is safe; the practical bound is 2 processes per live game. + */ +import type { ChatMessage } from "../../harness/core"; +import { MessageRole, SolverError } from "../../harness/core"; +import type { ModelService } from "../../harness/model"; +import type { SolverService } from "../../harness/solver"; +import { Either } from "../../internal/either"; +import { unknownErrorToString } from "../../internal/errors"; +import { parseSchema } from "../../internal/zod"; +import type { GameVerdict, PlyRecord, TurnCost } from "./game"; +import { + ADJUDICATE_LOSS_CP, + ADJUDICATE_LOSS_TURNS, + BLUNDER_CP, + boardResult, + capVerdict, + CP_LOSS_CLAMP, + extractMove, + fromModelView, + MAX_CHECKS_PER_TURN, + tryMove, + tryStrictMove, +} from "./game"; +import { ChessTaskSchema } from "./schema"; +import { MATE_CP, UciEngine } from "./uci"; + +export interface ChessSolverOpts { + readonly temperature: number; + readonly endpointId?: string; + readonly maxTokens?: number; + readonly reasoningEffort?: ReasoningEffort; + readonly timeoutMs?: number; +} + +/** + * Engine interactions fail as typed SolverError, never as a defect: with + * degradeSolverErrors the run engine degrades that one game to Incorrect + * (missing binary, engine hang) instead of tearing down the whole fan-out. + */ +function engineCall(run: () => Promise): Effect { + return tryPromise({ + try: run, + catch: (error) => + new SolverError({ + message: unknownErrorToString(error), + }), + }); +} + +export function chessSolver( + model: ModelService, + opts: ChessSolverOpts +): SolverService { + return (taskState) => { + /* Declared OUTSIDE the effect and cleaned in an outer ensuring(): an + * interrupt landing between an engine's spawn and the inner cleanup + * installation must still quit whatever started. */ + const engines: UciEngine[] = []; + return gen(function* () { + const taskMeta = parseSchema( + ChessTaskSchema, + taskState.sample.metadata?.["task"] + ); + if (Either.isLeft(taskMeta)) { + return yield* new SolverError({ + message: `chess sample carries no valid task metadata: ${taskMeta.left.message}`, + }); + } + const task = taskMeta.right; + const game = new Chess(task.fen); + + /* Fail closed before any model spend: no Stockfish, no chess run. */ + const opponent = yield* engineCall(() => UciEngine.start()); + engines.push(opponent); + /* A failed evaluator start propagates as SolverError; the outer + * ensuring() quits the already-started opponent. */ + const evaluatorResult = yield* effectEither( + engineCall(() => UciEngine.start()) + ); + if (Either.isLeft(evaluatorResult)) { + return yield* evaluatorResult.left; + } + const evaluator = evaluatorResult.right; + engines.push(evaluator); + + return yield* gen(function* () { + opponent.newGame(); + evaluator.newGame(); + + const plies: PlyRecord[] = []; + const modelMoves: string[] = []; + let illegalAttempts = 0; + let strictViolations = 0; + let checksUsed = 0; + let forfeited = false; + const accUsage = { + inputTokens: 0, + outputTokens: 0, + totalTokens: 0, + reasoningTokens: 0, + totalCost: 0, + }; + let totalGenerationTimeMs = 0; + /* Atomic cost ledger: one row per model call (iteration), each with + * its generation id so cost joins to billing per iteration — never + * only a self-reported aggregate (the Terminal-Bench gap). */ + const turnCosts: TurnCost[] = []; + let currentTurn = 0; + + /* Eval from the model's view; terminal positions short-circuit. */ + const evalNow = engineCall( + async (): Promise<{ cp: number; mateIn?: number }> => { + if (game.isCheckmate()) { + const modelMated = game.turn() === task.modelColor; + return { cp: modelMated ? -MATE_CP : MATE_CP, mateIn: 0 }; + } + if (game.isDraw()) { + return { cp: 0 }; + } + const searched = await evaluator.search(game.fen(), task.evalDepth); + return fromModelView(searched.score, game.fen(), task.modelColor); + } + ); + + function record(input: { + readonly by: "model" | "engine"; + readonly san: string; + readonly extra?: Partial; + /* Pass the eval when the caller already searched this position — + * re-searching an identical fixed-depth position doubles engine + * time per model ply for a bit-identical result. */ + readonly evaluated?: { cp: number; mateIn?: number }; + }): Effect { + const { by, san, extra = {} } = input; + return gen(function* () { + const resolved = input.evaluated ?? (yield* evalNow); + plies.push({ + ply: plies.length, + by, + san, + fenAfter: game.fen(), + evalCp: resolved.cp, + ...(resolved.mateIn !== undefined && { mateIn: resolved.mateIn }), + ...extra, + }); + }); + } + + const colorName = task.modelColor === "w" ? "White" : "Black"; + const strictRule = + task.strict === true + ? "Reply with EXACTLY your move in standard algebraic notation (SAN) and nothing else — no punctuation, no commentary, no move numbers. Example replies: Nf3 | exd5 | O-O | e8=Q+" + : 'Reply with ONLY your next move in standard algebraic notation (SAN), e.g. "Nf3", "exd5", "O-O". No commentary, no move numbers.'; + const validationRule = + task.moveValidation === true + ? `Before committing, you may reply "check " (e.g. "check Nf3") to verify a move is legal without playing it. You get ${MAX_CHECKS_PER_TURN} checks per turn. A bare move commits it.` + : ""; + const rules = [ + `You are playing a chess game as ${colorName}.`, + strictRule, + validationRule, + ] + .filter((line) => line !== "") + .join("\n"); + const messages: ChatMessage[] = [ + { role: MessageRole.System, content: rules }, + ]; + + // Opening user message carries the full starting context; moves only after. + let situation = + task.fen === undefined ? "" : `Position (FEN): ${task.fen}\n`; + + // If it's not the model's turn, the engine opens. + if (game.turn() !== task.modelColor && !game.isGameOver()) { + const opening = yield* engineCall(() => + opponent.search(game.fen(), task.engineDepth) + ); + const san = game.move(opening.bestmove).san; + situation += `Your opponent played: ${san}\n`; + yield* record({ by: "engine", san }); + } + messages.push({ + role: MessageRole.User, + content: `${situation}Your move.`, + }); + + const generateTurn = gen(function* () { + const output = yield* model.generate(messages, { + temperature: opts.temperature, + ...(opts.endpointId !== undefined && { + endpointId: opts.endpointId, + }), + ...(opts.maxTokens !== undefined && { maxTokens: opts.maxTokens }), + ...(opts.reasoningEffort !== undefined && { + reasoningEffort: opts.reasoningEffort, + }), + ...(opts.timeoutMs !== undefined && { timeoutMs: opts.timeoutMs }), + }); + totalGenerationTimeMs += output.generationTimeMs ?? 0; + accUsage.inputTokens += output.usage?.inputTokens ?? 0; + accUsage.outputTokens += output.usage?.outputTokens ?? 0; + accUsage.totalTokens += output.usage?.totalTokens ?? 0; + accUsage.reasoningTokens += output.usage?.reasoningTokens ?? 0; + accUsage.totalCost += output.usage?.totalCost ?? 0; + turnCosts.push({ + iteration: turnCosts.length, + turn: currentTurn, + ...(output.generationId !== undefined && { + generationId: output.generationId, + }), + inputTokens: output.usage?.inputTokens ?? 0, + outputTokens: output.usage?.outputTokens ?? 0, + totalTokens: output.usage?.totalTokens ?? 0, + reasoningTokens: output.usage?.reasoningTokens ?? 0, + costUsd: output.usage?.totalCost ?? 0, + generationTimeMs: output.generationTimeMs ?? 0, + }); + return output.completion; + }); + + let hopelessTurns = 0; + let terminated: GameVerdict | undefined; + + for (let turn = 0; turn < task.maxPlies && !game.isGameOver(); turn++) { + currentTurn = turn; + // Stockfish's best move BEFORE the model moves — the cpLoss baseline. + const before = yield* engineCall(() => + evaluator.search(game.fen(), task.evalDepth) + ); + const bestSan = tryMove(game, before.bestmove); + const beforeCp = fromModelView( + before.score, + game.fen(), + task.modelColor + ).cp; + + // Adjudicate hopeless positions (engine-tournament style). + hopelessTurns = + beforeCp <= ADJUDICATE_LOSS_CP ? hopelessTurns + 1 : 0; + if (hopelessTurns >= ADJUDICATE_LOSS_TURNS) { + terminated = { result: "adjudicated-loss", points: 0 }; + break; + } + + let san: string | undefined; + let checksThisTurn = 0; + + // One retry on an illegal/unparseable reply; a second failure forfeits. + for (let attempt = 0; attempt < 2 && san === undefined;) { + const reply = yield* generateTurn; + messages.push({ role: MessageRole.Assistant, content: reply }); + const trimmed = reply.trim(); + + /* moveValidation probes don't commit and don't count as attempts + * — but only up to the per-turn limit. Past it, a probe consumes + * an attempt like any non-move reply, so a model that only ever + * probes forfeits after two more replies instead of looping the + * turn (and its spend) forever. */ + const probe = + task.moveValidation === true + ? trimmed.match(/^check[:\s]+(\S+)$/i) + : null; + if ( + probe?.[1] !== undefined && + checksThisTurn < MAX_CHECKS_PER_TURN + ) { + checksUsed++; + checksThisTurn++; + const legal = tryMove(game, probe[1]); + messages.push({ + role: MessageRole.User, + content: + legal === undefined + ? `"${probe[1]}" is NOT legal here.` + : `"${probe[1]}" is legal. Reply with a move to play it.`, + }); + continue; + } + if (probe?.[1] !== undefined) { + // Probe past the limit: burns an attempt. + attempt++; + illegalAttempts++; + if (attempt < 2) { + messages.push({ + role: MessageRole.User, + content: "Check limit reached. Reply with your move.", + }); + } + continue; + } + + san = + task.strict === true + ? tryStrictMove(game, trimmed) + : extractMove(game, trimmed); + if ( + task.strict === true && + san === undefined && + extractMove(game, trimmed) !== undefined + ) { + strictViolations++; + } + if (san === undefined) { + attempt++; + illegalAttempts++; + if (attempt < 2) { + messages.push({ + role: MessageRole.User, + content: `"${trimmed}" is not a legal move${ + task.strict === true + ? " (reply with the exact SAN move only)" + : "" + }. Legal moves: ${game.moves().join(", ")}. Your move.`, + }); + } + } + } + if (san === undefined) { + forfeited = true; + terminated = { result: "forfeit", points: 0 }; + break; + } + + game.move(san); + modelMoves.push(san); + // cpLoss: eval swing vs the evaluator's preferred move (model's view). + const afterEvaluated = yield* evalNow; + const cpLoss = Math.min( + CP_LOSS_CLAMP, + Math.max(0, beforeCp - afterEvaluated.cp) + ); + yield* record({ + by: "model", + san, + extra: { + ...(bestSan !== undefined && { bestMove: bestSan }), + cpLoss, + }, + evaluated: afterEvaluated, + }); + if (game.isGameOver()) { + break; + } + + const engineMove = yield* engineCall(() => + opponent.search(game.fen(), task.engineDepth) + ); + /* Validate before applying: a malformed/empty bestmove (engine + * killed mid-search, protocol hiccup) must degrade this game via + * SolverError, not throw an untyped chess.js exception. */ + const engineSan = tryMove(game, engineMove.bestmove); + if (engineSan === undefined) { + return yield* effectFail( + new SolverError({ + message: `engine returned an unplayable bestmove "${engineMove.bestmove}" at ${game.fen()}`, + }) + ); + } + game.move(engineSan); + yield* record({ by: "engine", san: engineSan }); + messages.push({ + role: MessageRole.User, + content: `Your opponent played: ${engineSan}. Your move.`, + }); + } + + let verdict: GameVerdict; + if (terminated !== undefined) { + verdict = terminated; + } else if (game.isGameOver()) { + verdict = boardResult(game, task.modelColor); + } else { + // maxPlies safety cap hit with a live game: adjudicate by final eval. + const finalEvaluated = yield* evalNow; + verdict = capVerdict(finalEvaluated.cp); + } + + const modelPlies = plies.filter((ply) => ply.by === "model"); + const acpl = + modelPlies.length > 0 + ? Math.round( + modelPlies.reduce((acc, ply) => acc + (ply.cpLoss ?? 0), 0) / + modelPlies.length + ) + : 0; + const worstCpLoss = modelPlies.reduce( + (acc, ply) => Math.max(acc, ply.cpLoss ?? 0), + 0 + ); + const blunders = modelPlies.filter( + (ply) => (ply.cpLoss ?? 0) >= BLUNDER_CP + ).length; + + const gameRecord = { + taskId: task.id, + modelMoves, + plies, + illegalAttempts, + strictViolations, + checksUsed, + forfeited, + result: verdict.result, + points: verdict.points, + gameLengthMoves: Math.ceil(game.history().length / 2), + acpl, + worstCpLoss, + blunders, + finalEvalCp: plies.at(-1)?.evalCp ?? 0, + finalFen: game.fen(), + pgn: game.pgn(), + turnCosts, + totalCostUsd: accUsage.totalCost, + }; + + const completion = JSON.stringify({ + result: verdict.result, + points: verdict.points, + moves: modelMoves.length, + }); + + return { + ...taskState, + sample: { + ...taskState.sample, + metadata: { ...taskState.sample.metadata, game: gameRecord }, + }, + messages, + output: { + completion, + message: { role: MessageRole.Assistant, content: completion }, + usage: accUsage, + generationTimeMs: totalGenerationTimeMs, + }, + completed: true, + }; + }); + }).pipe( + ensuring( + sync(() => { + for (const engine of engines) { + engine.quit(); + } + }) + ) + ); + }; +} diff --git a/src/benchmarks/chess/uci.ts b/src/benchmarks/chess/uci.ts new file mode 100644 index 0000000..3bf4975 --- /dev/null +++ b/src/benchmarks/chess/uci.ts @@ -0,0 +1,183 @@ +/** + * Minimal UCI engine driver for the chess benchmark (Stockfish). + * + * Determinism: Threads=1 and fixed-depth search make bestmove/eval + * reproducible for a given binary version — the evaluation baseline is + * independent of wall-clock, so scores are comparable across runs and + * across concurrent games. + * + * Concurrency: each UciEngine owns one child process; games spawn their own + * engine pair (opponent + evaluator) and quit them in a finally block, so + * concurrent games never share engine state. + */ +import type { ChildProcessByStdio } from "node:child_process"; +import { spawn } from "node:child_process"; +import type { Interface } from "node:readline"; +import { createInterface } from "node:readline"; +import type { Readable, Writable } from "node:stream"; + +/** Engine score for a position, from the side-to-move's perspective. */ +export interface UciScore { + /** Centipawns (positive = side to move is better). Mate maps to ±MATE_CP. */ + readonly cp: number; + /** Moves until mate, signed (positive = side to move mates). */ + readonly mateIn?: number; +} + +export interface UciSearchResult { + /** Best move in UCI long algebraic (e2e4). */ + readonly bestmove: string; + readonly score: UciScore; +} + +export const MATE_CP = 10_000; + +export function stockfishPath(): string { + return process.env["STOCKFISH_PATH"] ?? "stockfish"; +} + +const UCI_TIMEOUT_MS = 30_000; + +const SPAWN_FAILED_LINE = "\u0000uci-spawn-failed"; + +export class UciEngine { + readonly #proc: ChildProcessByStdio; + readonly #rl: Interface; + #listeners: ((line: string) => void)[] = []; + #spawnError: Error | undefined; + /* Set on a #send timeout: the engine may still emit lines for the + * timed-out search, which would be mis-delivered to the NEXT #send as + * answers to a different position. A timed-out engine is dead. */ + #dead: Error | undefined; + + private constructor(proc: ChildProcessByStdio) { + this.#proc = proc; + /* stdin errors (EPIPE on a dead engine, ERR_STREAM_DESTROYED after a + * failed spawn) must not become uncaught exceptions in the worker — + * the spawn/exit paths already surface the actionable error. */ + proc.stdin.on("error", () => undefined); + this.#rl = createInterface({ input: proc.stdout }); + this.#rl.on("line", (line) => { + for (const listener of this.#listeners) { + listener(line); + } + }); + } + + /** + * Spawn + UCI handshake. Fails closed with a remediation message when the + * binary is missing — a chess run must never silently score without its + * engine. + */ + static async start(): Promise { + const bin = stockfishPath(); + const proc = spawn(bin, [], { stdio: ["pipe", "pipe", "ignore"] }); + const engine = new UciEngine(proc); + /* A spawn failure (missing binary) rejects every in-flight #send by + * flushing its listeners with a poisoned line — no Promise.race needed, + * and no listener leak on the losing branch. */ + proc.on("error", (error) => { + engine.#spawnError = new Error( + `stockfish not executable at "${bin}" (${error.message}). ` + + "Install with `brew install stockfish` / `apt-get install stockfish`, or set STOCKFISH_PATH." + ); + for (const listener of engine.#listeners) { + listener(SPAWN_FAILED_LINE); + } + }); + try { + await engine.#send("uci", (line) => line === "uciok"); + engine.#write("setoption name Threads value 1"); + await engine.#send("isready", (line) => line === "readyok"); + } catch (error) { + engine.quit(); + throw error; + } + return engine; + } + + #write(cmd: string): void { + if (this.#proc.stdin.writable) { + this.#proc.stdin.write(`${cmd}\n`); + } + } + + /** Send a command; resolve with all lines up to the one matching `done`. */ + #send(cmd: string, done: (line: string) => boolean): Promise { + return new Promise((resolve, reject) => { + if (this.#dead !== undefined) { + reject(this.#dead); + return; + } + const lines: string[] = []; + const listener = (line: string): void => { + if (line === SPAWN_FAILED_LINE) { + this.#listeners = this.#listeners.filter((l) => l !== listener); + clearTimeout(timer); + reject(this.#spawnError ?? new Error("uci engine failed to spawn")); + return; + } + lines.push(line); + if (done(line)) { + this.#listeners = this.#listeners.filter((l) => l !== listener); + clearTimeout(timer); + resolve(lines); + } + }; + const timer = setTimeout(() => { + this.#listeners = this.#listeners.filter((l) => l !== listener); + /* Poison the engine: late lines from this search would otherwise be + * delivered to the next #send as answers to a different position. */ + this.#dead = new Error(`UCI timeout on "${cmd}" — engine marked dead`); + this.#proc.kill("SIGKILL"); + reject(this.#dead); + }, UCI_TIMEOUT_MS); + this.#listeners.push(listener); + this.#write(cmd); + }); + } + + /** Search a position to fixed depth: best move + score (side-to-move view). */ + async search(fen: string, depth: number): Promise { + this.#write(`position fen ${fen}`); + const lines = await this.#send(`go depth ${depth}`, (line) => + line.startsWith("bestmove") + ); + const last = lines.at(-1); + const bestmove = last === undefined ? "" : (last.split(/\s+/)[1] ?? ""); + // The last `info` line carrying a score wins (deepest completed iteration). + let score: UciScore = { cp: 0 }; + for (const line of lines) { + /* Skip aspiration-window bound lines: a lowerbound/upperbound score is + * a search bound, not the position's evaluation, and taking one as + * the final score skews cpLoss/adjudication. */ + if (/\b(lowerbound|upperbound)\b/.test(line)) { + continue; + } + const match = line.match(/\bscore (cp|mate) (-?\d+)/); + if (match?.[1] === undefined || match[2] === undefined) { + continue; + } + score = + match[1] === "cp" + ? { cp: Number(match[2]) } + : { + cp: Math.sign(Number(match[2])) * MATE_CP, + mateIn: Number(match[2]), + }; + } + return { bestmove, score }; + } + + newGame(): void { + this.#write("ucinewgame"); + } + + quit(): void { + this.#write("quit"); + // Belt and braces: never leave engine processes behind if quit is ignored. + const proc = this.#proc; + setTimeout(() => proc.kill("SIGKILL"), 2000).unref(); + this.#rl.close(); + } +} diff --git a/src/benchmarks/registry.ts b/src/benchmarks/registry.ts index 967e7a3..e819b11 100644 --- a/src/benchmarks/registry.ts +++ b/src/benchmarks/registry.ts @@ -1,3 +1,4 @@ +import { CHESS_BENCHMARK } from "./chess/benchmark"; import { CUSTOM_EVAL_BENCHMARK } from "./custom-eval/benchmark"; import { DEEP_SWE_BENCHMARK } from "./deep-swe/benchmark"; import { DRACO_BENCHMARK } from "./draco/benchmark"; @@ -30,6 +31,7 @@ const BENCHMARKS: Record = { [DRACO_BENCHMARK.id]: DRACO_BENCHMARK, [IFSTRUCT_BENCHMARK.id]: IFSTRUCT_BENCHMARK, [CUSTOM_EVAL_BENCHMARK.id]: CUSTOM_EVAL_BENCHMARK, + [CHESS_BENCHMARK.id]: CHESS_BENCHMARK, [SWE_ATLAS_QA_BENCHMARK.id]: SWE_ATLAS_QA_BENCHMARK, [SWE_ATLAS_TW_BENCHMARK.id]: SWE_ATLAS_TW_BENCHMARK, [SWE_ATLAS_RF_BENCHMARK.id]: SWE_ATLAS_RF_BENCHMARK, diff --git a/src/harness/core.ts b/src/harness/core.ts index a393ff3..f5164ec 100644 --- a/src/harness/core.ts +++ b/src/harness/core.ts @@ -136,6 +136,12 @@ export interface ModelOutput { readonly message: ChatMessage; readonly usage?: ModelUsage; readonly generationTimeMs?: number; + /** + * OpenRouter generation id for THIS call. Also recorded into the + * fiber-scoped run collector; surfaced here so multi-turn solvers can + * attribute cost to individual iterations (billing joins per turn). + */ + readonly generationId?: string; } export interface TaskState {