feat(harness): durable per-(sample, epoch) result store + fiber-scoped request context

package.json

@@ -23,7 +23,9 @@
     "./progress": "./src/harness/progress.ts",
     "./result-store": "./src/results/result-store.ts",
     "./internal/log": "./src/internal/log.ts",
-    "./internal/effect-logger": "./src/internal/effect-logger.ts"
+    "./internal/effect-logger": "./src/internal/effect-logger.ts",
+    "./sample-result-store": "./src/harness/sample-result-store.ts",
+    "./request-context": "./src/harness/request-context.ts"
   },
   "scripts": {
     "bench": "bun src/cli/index.ts",

src/benchmarks/wandr/scorer.test.ts

@@ -65,19 +65,51 @@ describe("WANDR score aggregation", () => {
     expect(wandrRunLevelScores(result)).toEqual([]);
     expect(wandrPrimaryScore(result)).toBeUndefined();
   });
-  it("includes degraded samples as zero rewards in the aggregate denominator", () => {
+
+  it("excludes skipped (infrastructure-failure) samples from the mean and weight", () => {
+    const result = resultWith([1, 0]);
+    const validSample = result.sampleScores[0]!;
+    const skippedSample = result.sampleScores[1]!;
+    const partialFailure = {
+      ...result,
+      sampleScores: [
+        validSample,
+        {
+          ...skippedSample,
+          score: {
+            ...skippedSample.score,
+            value: ScoreValue.Skipped,
+            explanation: "Solver error (skipped): timeout",
+          },
+        },
+      ],
+    };
+
+    expect(wandrRunLevelScores(partialFailure)).toEqual([
+      {
+        name: "wandr",
+        metrics: Object.fromEntries(
+          WANDR_REWARD_NAMES.map((name) => [name, { value: 1 }])
+        ),
+      },
+    ]);
+    expect(wandrPrimaryScore(partialFailure)).toEqual({ value: 1, weight: 1 });
+  });
+
+  it("counts evaluated samples without reward metadata as zero rewards", () => {
     const result = resultWith([1, 0]);
     const validSample = result.sampleScores[0]!;
-    const degradedSample = result.sampleScores[1]!;
+    const unparseableSample = result.sampleScores[1]!;
     const partialFailure = {
       ...result,
       sampleScores: [
         validSample,
         {
-          ...degradedSample,
+          ...unparseableSample,
           score: {
-            ...degradedSample.score,
-            explanation: "Solver error: timeout",
+            ...unparseableSample.score,
+            value: ScoreValue.Incorrect,
+            explanation: "not reward json",
           },
         },
       ],
@@ -95,4 +127,25 @@ describe("WANDR score aggregation", () => {
       weight: 2,
     });
   });
+
+  it("returns no aggregate when every sample was skipped", () => {
+    const result = resultWith([1]);
+    const skippedSample = result.sampleScores[0]!;
+    const allSkipped = {
+      ...result,
+      sampleScores: [
+        {
+          ...skippedSample,
+          score: {
+            ...skippedSample.score,
+            value: ScoreValue.Skipped,
+            explanation: "Solver error (skipped): sandbox down",
+          },
+        },
+      ],
+    };
+
+    expect(wandrRunLevelScores(allSkipped)).toEqual([]);
+    expect(wandrPrimaryScore(allSkipped)).toBeUndefined();
+  });
 });

src/benchmarks/wandr/scorer.ts

@@ -53,6 +53,18 @@ export const wandrScorer: ScorerService = (
   return succeed(score);
 };
 
+/**
+ * Samples that were actually evaluated. Skipped scores are infrastructure
+ * failures (degraded solver/model errors), excluded from the accuracy
+ * denominator by `aggregateScores` — the reward mean and primary-score weight
+ * must exclude them the same way.
+ */
+function scoredWandrSamples(result: RunResult): RunResult["sampleScores"] {
+  return result.sampleScores.filter(
+    (sample) => sample.score.value !== ScoreValue.Skipped
+  );
+}
+
 export function wandrRunLevelScores(result: RunResult): readonly {
   name: string;
   metrics: Readonly<
@@ -64,7 +76,7 @@ export function wandrRunLevelScores(result: RunResult): readonly {
     >
   >;
 }[] {
-  const rewards = result.sampleScores.flatMap((sample) => {
+  const rewards = scoredWandrSamples(result).flatMap((sample) => {
     const parsed = Either.try((): unknown =>
       JSON.parse(sample.score.explanation)
     );
@@ -97,12 +109,13 @@ export function wandrRunLevelScores(result: RunResult): readonly {
 export function wandrPrimaryScore(
   result: RunResult
 ): BenchmarkPrimaryScore | undefined {
-  if (result.sampleScores.length === 0) {
+  const weight = scoredWandrSamples(result).length;
+  if (weight === 0) {
     return undefined;
   }
   const metrics = wandrRunLevelScores(result)[0]?.metrics;
   return {
     value: metrics?.["soft_f1_full"]?.value ?? 0,
-    weight: result.sampleScores.length,
+    weight,
   };
 }

src/harness/request-context.ts

@@ -0,0 +1,29 @@
+import type { Effect } from "effect/Effect";
+import { get, set, unsafeMake } from "effect/FiberRef";
+
+/**
+ * Fiber-scoped (sample, epoch) identity of the evaluation currently issuing
+ * model calls. The model layer stamps it into each request's `x_bench`
+ * extension so the bench-gateway's request-coalescing hash can tell two
+ * epochs of the same sample apart — without it, identical bodies from
+ * different epochs coalesce and epoch 2 replays epoch 1's answer. Fiber-scoped
+ * for the same reason as `generationIdCollector`: each (sample, epoch) runs
+ * in its own child fiber under streamMapEffect concurrency.
+ */
+export interface BenchRequestContext {
+  readonly sampleId: string;
+  readonly epoch: number;
+}
+
+export const benchRequestContext = unsafeMake<BenchRequestContext | undefined>(
+  undefined
+);
+
+export function setBenchRequestContext(
+  context: BenchRequestContext
+): Effect<void> {
+  return set(benchRequestContext, context);
+}
+
+export const getBenchRequestContext: Effect<BenchRequestContext | undefined> =
+  get(benchRequestContext);