Build monitor: show collector runs' uploads, keep dry runs apart, share the collector run list

docs/build-monitor.md

@@ -32,6 +32,32 @@ checks stop builds. The run selected on either tab carries over to the other.
   same run id, except that a collector record that stopped before the run
   ended never hides an end the Hugging Face copy recorded after the record's
   latest update. The Candidate tab applies the same rule to the runs it lists.
+
+  Uploads to Hugging Face never pass through the collector, so its record
+  always says delivery mode `collector` with no counts. A collector record
+  that replaces a copy keeps the copy's upload record (files attempted and
+  uploaded, or a failed write check), and a run's uploads show as `unknown`
+  only when it has no copy that records them.
+
+  Every Staging route reads the collector's run list (the country's 60 most
+  recent runs) through one cache on each server instance. On Vercel the
+  Candidate tab's selected-run route (`staging/run`, `maxDuration` 300) runs
+  in a different function from the Build progress routes and the Candidate
+  run list (120), because Vercel groups routes into one function only when
+  their settings match, so it keeps its own cache. An answer is
+  reused for 15 seconds, and a failed request is tried again on the next
+  call. The Candidate tab waits at most 5 seconds for the list. After that it
+  uses the list's latest earlier answer if that answered less than 60 seconds
+  ago (two of its 30-second polls), and otherwise shows the Hugging Face
+  states with a note; the late answer serves the next poll. A request still
+  in flight is shared only for 5 seconds after it was made, and a later call
+  makes its own request: once the Candidate tab has stopped waiting for a
+  request, nothing keeps it running, and Vercel can suspend the instance
+  with it in flight. Every collector request needs the collector's read
+  credential from Google Secret Manager, and that read follows the same
+  rule: a read still in flight is shared for 5 seconds after it began, and a
+  read that answered is reused for 60 seconds after it began. Each run's own
+  record is read on every request, as before.
 - **Local run folders.** The dashboard can still inspect legacy or diagnostic
   run folders on the same machine. Point it at the directory that holds them:
 
@@ -85,6 +111,13 @@ The forecast compares the run with finished runs of the same pipeline from
 the same source (version 2 and 3 runs by pipeline id; version 1 US runs as one
 release pipeline), up to the 12 most recent.
 
+Smoke runs (run kind `smoke`) and dry runs (`dry_run`) form pipelines of
+their own, labelled "(smoke)" and "(dry run)", with their own history and
+forecasts. A dry run checks its configuration, pins its inputs and stops, so
+it would otherwise count as a passed run that ended after its first two
+stages. The UK spine build registers its own pipeline id, `uk-frs-spine`, so
+it is never compared with the calibration runs that read its output.
+
 Runs of one pipeline can still differ in what they do: UK local candidates kept
 the pipeline id `uk-local-candidate` 0.1.0 when cloning and surface resolution
 moved into target compilation (September against October 2026). So a past run

frontend/app/api/microcosm/staging/runs/route.ts

@@ -13,7 +13,8 @@ import { scrub } from "@policyengine/calibration-tree/latest-artifact";
 export const dynamic = "force-dynamic";
 export const revalidate = 0;
 export const runtime = "nodejs";
-// The collector lookup can add two 30-second request timeouts.
+// The collector lookup waits at most 5 seconds for the shared run list, but a
+// run's own record can still take its 30-second request timeout.
 export const maxDuration = 120;
 
 export interface StagingRunsRouteDependencies {

frontend/lib/microcosm/build-monitor.test.ts

@@ -9,6 +9,7 @@ import {
   type BuildTimeline,
   buildTimeline,
   calibrationRate,
+  comparableHistory,
   stopStatistics,
   failureReason,
   forecastCompletion,
@@ -258,6 +259,84 @@ describe("buildTimeline, version 2 lifecycle", () => {
     expect(smoke.pipeline_label).toBe("UK FRS calibration (smoke)");
   });
 
+  // A calibration dry run (`--dry-run`) registers run kind `dry_run`: it
+  // checks its configuration, pins its inputs and stops, so its two stages
+  // are the start of every full run's and it ends in minutes.
+  function dryRun(runId: string): BuildRunDocuments {
+    return {
+      run_id: runId,
+      source: "staging",
+      country: "uk",
+      progress: {
+        schema_version: 2,
+        run_id: runId,
+        country_code: "GB",
+        pipeline: { id: "uk-frs-calibration", version: "collector-v1" },
+        run_kind: "dry_run",
+        started_at: at(-min(20)),
+        updated_at: at(-min(2)),
+        status: "completed",
+        current_stage: "complete",
+      },
+      run_manifest: null,
+      calibration_progress: null,
+      events: [
+        v2Event(1, -min(20), "created", "started"),
+        v2Event(2, -min(20), "preflight", "started"),
+        v2Event(3, -min(19), "preflight", "completed"),
+        v2Event(4, -min(19), "input_pinning", "started"),
+        v2Event(5, -min(2), "input_pinning", "completed"),
+        v2Event(6, -min(2), "complete", "completed"),
+      ],
+    };
+  }
+
+  const calibrationRun: BuildRunDocuments = {
+    ...documents,
+    progress: { ...documents.progress!, run_kind: "calibration" },
+  };
+
+  test("keeps dry runs apart from full runs of the same pipeline", () => {
+    const dry = buildTimeline(dryRun("uk-dry"), T0 + min(9));
+
+    expect(dry.state).toBe("passed");
+    expect(dry.pipeline).toBe("GB:uk-frs-calibration:dry_run");
+    expect(dry.pipeline_label).toBe("UK FRS calibration (dry run)");
+  });
+
+  test("a dry run is not in a calibration run's history or forecast", () => {
+    const live = buildTimeline(calibrationRun, T0 + min(9));
+    const dry = buildTimeline(dryRun("uk-dry"), T0 + min(9));
+
+    expect(live.pipeline).toBe("GB:uk-frs-calibration");
+    expect(comparableHistory(live, [live, dry])).toEqual([]);
+    expect(forecastCompletion(live, [live, dry], T0 + min(9))?.basis_runs).toEqual([]);
+  });
+
+  test("a spine build is a pipeline of its own", () => {
+    // The spine build registers pipeline `uk-frs-spine` and run kind `spine`,
+    // so it never shares a calibration run's history.
+    const spine = buildTimeline(
+      {
+        ...calibrationRun,
+        run_id: "uk-spine",
+        progress: {
+          ...calibrationRun.progress!,
+          run_id: "uk-spine",
+          pipeline: { id: "uk-frs-spine", version: "collector-v1" },
+          run_kind: "spine",
+          status: "completed",
+        },
+      },
+      T0 + min(9),
+    );
+    const live = buildTimeline(calibrationRun, T0 + min(9));
+
+    expect(spine.pipeline).toBe("GB:uk-frs-spine");
+    expect(spine.pipeline_label).toBe("UK FRS spine");
+    expect(comparableHistory(live, [live, spine])).toEqual([]);
+  });
+
   test("reads the solver rate from calibration progress", () => {
     const run = buildTimeline(documents, T0 + min(9));
     const rate = calibrationRate(run.calibration)!;

frontend/lib/microcosm/build-monitor.ts

@@ -96,6 +96,9 @@ export interface StageWork {
 // Whether a run's telemetry reached the staging repository. `local_only`
 // runs kept it on the build machine (no write token, uploads failed, or a
 // local-only run); `uploaded` says how many files reached the repository.
+// The collector does not know (its record says delivery mode "collector"),
+// so a collector record reports the uploads its run's Hugging Face copy
+// records, and `unknown` when there is no such record.
 export interface RunDelivery {
   uploads: "local_only" | "uploaded" | "disabled" | "unknown";
   repository: string | null;
@@ -257,6 +260,16 @@ export function formatStageName(stage: string): string {
   return text ? text[0].toUpperCase() + text.slice(1) : stage;
 }
 
+// Run kinds that are not comparable with full runs of the same pipeline, with
+// the label suffix for each. A smoke run builds a synthetic fixture; a dry run
+// (`--dry-run`) checks its configuration, pins its inputs and stops. The run
+// kind is part of their pipeline key, so they have their own group, history
+// and forecasts.
+const SEPARATE_RUN_KINDS: ReadonlyMap<string, string> = new Map([
+  ["smoke", "smoke"],
+  ["dry_run", "dry run"],
+]);
+
 function pipelineIdentity(
   documents: BuildRunDocuments,
   schemaVersion: 1 | 2 | 3,
@@ -266,13 +279,11 @@ function pipelineIdentity(
   const pipelineId = str(pipeline?.id);
   const country = str(identity?.country_code) ?? documents.country.toUpperCase();
   if (schemaVersion >= 2 && pipelineId) {
-    // A smoke run on a synthetic fixture is not comparable with a full run of
-    // the same pipeline, so the run kind is part of the key.
     const kind = str(identity?.run_kind);
-    const smoke = kind === "smoke";
+    const kindLabel = kind == null ? undefined : SEPARATE_RUN_KINDS.get(kind);
     return {
-      key: `${country}:${pipelineId}${smoke ? ":smoke" : ""}`,
-      label: `${formatStageName(pipelineId)}${smoke ? " (smoke)" : ""}`,
+      key: `${country}:${pipelineId}${kindLabel ? `:${kind}` : ""}`,
+      label: `${formatStageName(pipelineId)}${kindLabel ? ` (${kindLabel})` : ""}`,
     };
   }
   return {
@@ -396,6 +407,52 @@ function runDelivery(documents: BuildRunDocuments): RunDelivery | null {
   };
 }
 
+function uploadsKnown(delivery: RunDelivery | null): boolean {
+  return delivery != null && delivery.uploads !== "unknown";
+}
+
+// A collector record with its run's upload record taken from the run's Hugging
+// Face copy: the copy's failed write check (version 1) or delivery block
+// (version 2 and later). Uploads to Hugging Face never pass through the
+// collector, so its own delivery block always says mode "collector" with no
+// counts. The record is unchanged when the copy records no uploads.
+export function withHuggingFaceUploads(
+  collector: BuildRunDocuments,
+  huggingFace: BuildRunDocuments,
+): BuildRunDocuments {
+  const check = obj(huggingFace.run_manifest?.delivery_check);
+  const delivery =
+    obj(huggingFace.progress?.delivery) ?? obj(huggingFace.run_manifest?.delivery);
+  if (!check && !delivery) return collector;
+  let progress = collector.progress;
+  if (progress && "delivery" in progress) {
+    const { delivery: _collectorDelivery, ...rest } = progress;
+    progress = rest;
+  }
+  return {
+    ...collector,
+    progress,
+    run_manifest: {
+      ...collector.run_manifest,
+      ...(check ? { delivery_check: check } : {}),
+      ...(delivery ? { delivery } : {}),
+    },
+  };
+}
+
+// A run read again from the collector, keeping the uploads `listed` (the same
+// run in the monitor's list, where they came from the Hugging Face copy) when
+// the collector's record does not know them.
+export function keepListedUploads(
+  fresh: BuildTimeline,
+  listed: BuildTimeline | null,
+): BuildTimeline {
+  if (uploadsKnown(fresh.delivery) || !listed || !uploadsKnown(listed.delivery)) {
+    return fresh;
+  }
+  return { ...fresh, delivery: listed.delivery };
+}
+
 function runIdentity(manifest: JsonObject | null): RunIdentity | null {
   const identity = obj(manifest?.identity);
   if (!identity) return null;