ci(cli): judge block-scoped benchmark timings by per-block spread

apps/cli/bench/workloads.full.json

@@ -48,7 +48,7 @@
       "runs": 10,
       "warmup": 1,
       "traceRuns": 2,
-      "estimatedSeconds": 90
+      "estimatedSeconds": 220
     },
     {
       "id": "db-query",
@@ -61,7 +61,7 @@
       "runs": 10,
       "warmup": 1,
       "traceRuns": 2,
-      "estimatedSeconds": 90
+      "estimatedSeconds": 220
     },
     {
       "id": "db-reset",
@@ -90,7 +90,7 @@
       "warmup": 1,
       "traceRuns": 1,
       "timeoutSeconds": 120,
-      "estimatedSeconds": 60
+      "estimatedSeconds": 180
     },
     {
       "id": "gen-types-small",
@@ -104,7 +104,7 @@
       "runs": 10,
       "warmup": 1,
       "traceRuns": 2,
-      "estimatedSeconds": 90
+      "estimatedSeconds": 220
     },
     {
       "id": "gen-types-medium",
@@ -118,7 +118,7 @@
       "runs": 10,
       "warmup": 1,
       "traceRuns": 2,
-      "estimatedSeconds": 90
+      "estimatedSeconds": 220
     },
     {
       "id": "gen-types-large",
@@ -132,7 +132,7 @@
       "runs": 10,
       "warmup": 1,
       "traceRuns": 2,
-      "estimatedSeconds": 100
+      "estimatedSeconds": 250
     },
     {
       "id": "db-diff-cold",
@@ -183,7 +183,7 @@
       "runs": 10,
       "warmup": 1,
       "traceRuns": 2,
-      "estimatedSeconds": 100
+      "estimatedSeconds": 250
     },
     {
       "id": "db-lint",
@@ -197,7 +197,7 @@
       "runs": 10,
       "warmup": 1,
       "traceRuns": 2,
-      "estimatedSeconds": 90
+      "estimatedSeconds": 220
     },
     {
       "id": "migration-up",
@@ -239,7 +239,7 @@
       "runs": 10,
       "warmup": 1,
       "traceRuns": 2,
-      "estimatedSeconds": 90
+      "estimatedSeconds": 220
     },
     {
       "id": "stack-start-docker",
@@ -266,7 +266,7 @@
       "runs": 10,
       "warmup": 1,
       "traceRuns": 2,
-      "estimatedSeconds": 60
+      "estimatedSeconds": 150
     },
     {
       "id": "stack-stop-docker",
@@ -294,7 +294,7 @@
       "runs": 10,
       "warmup": 1,
       "traceRuns": 2,
-      "estimatedSeconds": 60
+      "estimatedSeconds": 150
     },
     {
       "id": "stack-db-reset-docker",
@@ -336,7 +336,7 @@
       "runs": 10,
       "warmup": 1,
       "traceRuns": 2,
-      "estimatedSeconds": 60
+      "estimatedSeconds": 150
     },
     {
       "id": "stack-stop-native",
@@ -364,7 +364,7 @@
       "runs": 10,
       "warmup": 1,
       "traceRuns": 2,
-      "estimatedSeconds": 60
+      "estimatedSeconds": 150
     },
     {
       "id": "stack-db-reset-native",

apps/cli/docs/tracing-monitoring.md

@@ -178,9 +178,14 @@ the timing ranges, API calls per command, and the slowest phases. It reports:
   responses (the build did not follow the recorded scenario), non-zero or differing exit codes,
   missing traces, and too few runs for a timing verdict. Other count changes, such as 1 to 2 calls, appear in the API calls table with the
   changed endpoints but are not flagged.
-- Wall time: base ("before") and branch ("after") medians and ranges. A verdict of slower or faster needs the branch
-  median outside the base min-max range, a change of at least 5 percent and 5 ms, and at least 5
-  measured runs; otherwise it reads "within noise", or "not comparable" when a build failed.
+- Wall time: base ("before") and branch ("after") medians and ranges. A verdict of slower or faster needs a
+  change of at least 5 percent and 5 ms that is also larger than 1.5 times the wider build's
+  interquartile range, a branch median beyond the base min-max range for run-scoped workloads, and
+  at least 5 measured runs; otherwise it reads "within noise", or "not
+  comparable" when a build failed. Block-scoped workloads record a block index per launch and, from
+  three paired blocks, judge block medians instead: the median per-block change must pass the same
+  tests and more than half of the blocks must move the same way past the floor, so one outlier
+  stack session can neither flag nor hide a change.
   Thresholds are named constants in `bench-report.ts`.
 - API calls per workload, base and branch.
 - Phase deltas: the three largest span deltas for each workload with a slower or faster verdict.
@@ -297,7 +302,7 @@ described above:
 | Field                                | Meaning                                                                                                                |
 | ------------------------------------ | ---------------------------------------------------------------------------------------------------------------------- |
 | `setup`, `teardown`                  | Lists of CLI arguments run untimed with the build under test. `teardown` always runs, even after a failure or timeout. |
-| `setupScope`                         | `run` (default) repeats setup and teardown for every launch; `block` starts one stack per block of five launches.      |
+| `setupScope`                         | `run` (default) repeats setup and teardown for every launch; `block` starts one stack per block of two launches.       |
 | `stack`                              | Sets `SUPABASE_EXPERIMENTAL_STACK=1` on every launch of the workload, the only variable a workload can set.            |
 | `runtime`                            | `docker` or `native`. A stack workload must pass the same `--runtime` to every `stack start`.                          |
 | `fixture`                            | A directory under `apps/cli/bench/fixtures/` copied into the project's `supabase/` directory after `init`.             |

apps/cli/scripts/bench-cli.integration.test.ts

@@ -204,13 +204,21 @@ describe("runWorkloadBench", () => {
         expect(report.base).toHaveLength(6);
         expect(report.branch).toHaveLength(6);
         const boots = calls.filter((call) => call.command === "boot");
-        expect(boots.map((call) => call.build)).toEqual(["base", "branch", "branch", "base"]);
-        expect(calls.filter((call) => call.command === "halt")).toHaveLength(4);
-        // One warmup plus five measured launches share the first block's stack.
+        expect(boots.map((call) => call.build)).toEqual([
+          "base",
+          "branch",
+          "branch",
+          "base",
+          "base",
+          "branch",
+        ]);
+        expect(calls.filter((call) => call.command === "halt")).toHaveLength(6);
+        expect(report.base.map((sample) => sample.block)).toEqual([0, 0, 1, 1, 2, 2]);
+        // One warmup plus two measured launches share the first block's stack.
         const firstBlock = calls.filter((call) => call.home === boots[0]?.home);
         expect(firstBlock.map((call) => call.command)).toEqual([
           "boot",
-          ...Array.from({ length: 6 }, () => "work"),
+          ...Array.from({ length: 3 }, () => "work"),
           "halt",
         ]);
       }),

apps/cli/scripts/bench-cli.ts

@@ -463,11 +463,13 @@ interface LaunchOutcome {
 /** Planned units of work for one build: a run-scoped unit holds one launch, a block-scoped one many. */
 export interface PlannedUnit {
   readonly build: Build;
+  /** Block index for block-scoped units. */
+  readonly block?: number;
   readonly launches: ReadonlyArray<"warmup" | "timed">;
 }
 
 /** Measured launches per block for `block`-scoped workloads. */
-export const BLOCK_SIZE = 5;
+export const BLOCK_SIZE = 2;
 
 /**
  * Units in execution order. `run` scope reuses {@link buildRunPlan}, one launch per unit.
@@ -495,7 +497,7 @@ export function buildUnitPlan(
       ...Array.from({ length: measured }, () => "timed" as const),
     ];
     const order: ReadonlyArray<Build> = block % 2 === 0 ? ["base", "branch"] : ["branch", "base"];
-    for (const build of order) units.push({ build, launches });
+    for (const build of order) units.push({ build, block, launches });
   }
   return units;
 }
@@ -644,7 +646,7 @@ export const runWorkloadBench = (
     const plan = buildUnitPlan(runs, warmup, scope);
     const started = yield* Clock.currentTimeNanos;
 
-    const timed: Record<Build, Array<{ wallMs: number; exitCode: number }>> = {
+    const timed: Record<Build, Array<{ wallMs: number; exitCode: number; block?: number }>> = {
       base: [],
       branch: [],
     };
@@ -668,10 +670,14 @@ export const runWorkloadBench = (
           (options.cpuProf && build === "branch" && !alreadyProfiled),
       );
 
-    const record = (build: Build, outcomes: ReadonlyArray<LaunchOutcome>) => {
+    const record = (build: Build, outcomes: ReadonlyArray<LaunchOutcome>, block?: number) => {
       for (const outcome of outcomes) {
         if (outcome.kind === "timed") {
-          timed[build].push({ wallMs: outcome.wallMs, exitCode: outcome.exitCode });
+          timed[build].push({
+            wallMs: outcome.wallMs,
+            exitCode: outcome.exitCode,
+            ...(block === undefined ? {} : { block }),
+          });
         } else if (outcome.kind === "traced") {
           traced[build].push({ exitCode: outcome.exitCode, trace: outcome.trace });
         }
@@ -728,6 +734,7 @@ export const runWorkloadBench = (
           teardownFailures,
           onProfile,
         }),
+        planned.block,
       );
     }
 

apps/cli/scripts/bench-cli.unit.test.ts

@@ -137,6 +137,21 @@ describe("buildUnitPlan", () => {
   });
 });
 
+describe("buildUnitPlan block sizing", () => {
+  it("gives ten runs five blocks per build by default, each tagged with its block index", () => {
+    const plan = buildUnitPlan(10, 1, "block");
+
+    for (const build of ["base", "branch"] as const) {
+      expect(plan.filter((unit) => unit.build === build).map((unit) => unit.block)).toEqual([
+        0, 1, 2, 3, 4,
+      ]);
+    }
+    expect(
+      plan.every((unit) => unit.launches.filter((kind) => kind === "timed").length === 2),
+    ).toBe(true);
+  });
+});
+
 describe("parseBenchArgs for stack runs", () => {
   it("lets explicit --runs and --warmup override the manifest, and defaults otherwise", () => {
     expect(parseBenchArgs(required)).toMatchObject({

apps/cli/scripts/bench-report.ts

@@ -29,15 +29,21 @@ export interface TimingSummary {
   readonly min: number;
   readonly max: number;
   readonly p90: number;
+  readonly iqr: number;
 }
 
+/** Interquartile range by nearest rank, so one extreme value at either end cannot move it. */
+const iqr = (values: ReadonlyArray<number>): number =>
+  percentile(values, 75) - percentile(values, 25);
+
 export function summarizeTimings(values: ReadonlyArray<number>): TimingSummary {
   const sorted = [...values].sort((a, b) => a - b);
   return {
     median: round(median(sorted)),
     min: round(sorted[0] ?? 0),
     max: round(sorted[sorted.length - 1] ?? 0),
     p90: round(percentile(sorted, 90)),
+    iqr: round(iqr(sorted)),
   };
 }
 
@@ -289,6 +295,8 @@ const TraceSample = Schema.Struct({
 
 const RunSample = Schema.Struct({
   wallMs: Schema.Finite,
+  /** Block index of a launch from a block-scoped workload; absent for run-scoped launches. */
+  block: Schema.optionalKey(Schema.Finite),
   exitCode: Schema.Finite,
   /** Exit code of the traced launch, recorded even when no spans could be decoded. */
   tracedExitCode: Schema.optionalKey(Schema.Finite),
@@ -345,27 +353,88 @@ export const encodeBenchReport = Schema.encodeEffect(BenchReportJson);
 export const VERDICT_MIN_PCT = 5;
 export const VERDICT_MIN_MS = 5;
 export const VERDICT_MIN_SAMPLES = 5;
+/** Paired blocks needed before the block, not the launch, is the unit of evidence for a verdict. */
+export const VERDICT_MIN_BLOCKS = 3;
+export const VERDICT_IQR_FACTOR = 1.5;
 export const PHASE_ROWS = 3;
 
 export type TimingVerdict = "slower" | "faster" | "within noise" | "not comparable";
 
+/** Median wall time of one block for each build, paired by block index. */
+export interface BlockPair {
+  readonly base: number;
+  readonly branch: number;
+}
+
+const exceedsFloor = (delta: Delta): boolean =>
+  Math.abs(delta.pct) >= VERDICT_MIN_PCT && Math.abs(delta.ms) >= VERDICT_MIN_MS;
+
 /**
- * "slower" or "faster" only when the branch median is outside the base min-max range and the
- * change is at least {@link VERDICT_MIN_PCT} percent and {@link VERDICT_MIN_MS} ms.
+ * "slower" or "faster" only when the median change passes the {@link VERDICT_MIN_PCT} percent and
+ * {@link VERDICT_MIN_MS} ms floor and is larger than {@link VERDICT_IQR_FACTOR} times the wider
+ * build's interquartile range, so one extreme sample cannot fake a change. Run-scoped timings
+ * also need the branch median beyond the base range; with at least {@link VERDICT_MIN_BLOCKS}
+ * paired blocks the spread test runs on block medians instead, and a strict majority of blocks
+ * must pass the floor in the same direction.
  */
 export function timingVerdict(
   base: TimingSummary,
   branch: TimingSummary,
-  context: { readonly comparable: boolean; readonly samples: number },
+  context: {
+    readonly comparable: boolean;
+    readonly samples: number;
+    readonly blocks?: ReadonlyArray<BlockPair>;
+  },
 ): TimingVerdict {
   if (!context.comparable || context.samples < VERDICT_MIN_SAMPLES) return "not comparable";
+  const blocks = context.blocks;
+  if (blocks !== undefined && blocks.length >= VERDICT_MIN_BLOCKS) {
+    const baseMedians = blocks.map((block) => block.base);
+    const branchMedians = blocks.map((block) => block.branch);
+    const reference = median(baseMedians);
+    const ms = median(blocks.map((block) => block.branch - block.base));
+    const pct = reference === 0 ? 0 : (ms / reference) * 100;
+    if (!exceedsFloor({ ms: round(ms), pct: round(pct) })) return "within noise";
+    const agreeing = blocks.filter((block) => {
+      const blockDelta = deltaOf(block.base, block.branch);
+      return exceedsFloor(blockDelta) && blockDelta.ms > 0 === ms > 0;
+    }).length;
+    if (agreeing * 2 <= blocks.length) return "within noise";
+    if (Math.abs(ms) <= VERDICT_IQR_FACTOR * Math.max(iqr(baseMedians), iqr(branchMedians))) {
+      return "within noise";
+    }
+    return ms > 0 ? "slower" : "faster";
+  }
   const delta = deltaOf(base.median, branch.median);
-  if (Math.abs(delta.pct) < VERDICT_MIN_PCT || Math.abs(delta.ms) < VERDICT_MIN_MS) {
+  if (!exceedsFloor(delta)) return "within noise";
+  const outsideRange = delta.ms > 0 ? branch.median > base.max : branch.median < base.min;
+  if (!outsideRange) return "within noise";
+  if (Math.abs(delta.ms) <= VERDICT_IQR_FACTOR * Math.max(base.iqr, branch.iqr)) {
     return "within noise";
   }
-  if (branch.median > base.max) return "slower";
-  if (branch.median < base.min) return "faster";
-  return "within noise";
+  return delta.ms > 0 ? "slower" : "faster";
+}
+
+/** Pairs the per-block medians of both builds; empty unless every sample carries a block index. */
+export function pairBlocks(
+  base: ReadonlyArray<RunSample>,
+  branch: ReadonlyArray<RunSample>,
+): ReadonlyArray<BlockPair> {
+  const group = (samples: ReadonlyArray<RunSample>): Map<number, Array<number>> | undefined => {
+    const blocks = new Map<number, Array<number>>();
+    for (const sample of samples) {
+      if (sample.block === undefined) return undefined;
+      blocks.set(sample.block, [...(blocks.get(sample.block) ?? []), sample.wallMs]);
+    }
+    return blocks;
+  };
+  const baseBlocks = group(base);
+  const branchBlocks = group(branch);
+  if (baseBlocks === undefined || branchBlocks === undefined) return [];
+  return [...baseBlocks.entries()].flatMap(([block, values]) => {
+    const other = branchBlocks.get(block);
+    return other === undefined ? [] : [{ base: median(values), branch: median(other) }];
+  });
 }
 
 export interface ApiSummary {
@@ -559,6 +628,7 @@ export function analyzeWorkload(workload: WorkloadReport, traced: boolean): Work
   const verdict = timingVerdict(base.timing, branch.timing, {
     comparable: !differ && runsCleanly(base) && runsCleanly(branch),
     samples: Math.min(base.samples, branch.samples),
+    blocks: pairBlocks(workload.base, workload.branch),
   });
   const spans =
     base.spans === undefined || branch.spans === undefined