Give boot-time restore a real cancellation path instead of an abandoning timeout

apps/sidecar/src/workflow-deployment-record.test.ts

@@ -184,15 +184,14 @@ describe("recordWorkflowDeploymentRestoreFailure", () => {
     const updated = await recordWorkflowDeploymentRestoreFailure(
       dataDir,
       "dep_1",
-      baseRecord,
       { kind: "permanent", reason: "address derives a different slug" },
     );
 
-    expect(updated.restoreFailure).toEqual({
+    expect(updated?.restoreFailure).toEqual({
       kind: "permanent",
       attempts: 1,
       reason: "address derives a different slug",
-      lastAttemptAt: updated.restoreFailure?.lastAttemptAt ?? "",
+      lastAttemptAt: updated?.restoreFailure?.lastAttemptAt ?? "",
     });
     const onDisk = await readWorkflowDeploymentRecord(dataDir, "dep_1");
     expect(onDisk?.restoreFailure?.attempts).toBe(1);
@@ -202,47 +201,59 @@ describe("recordWorkflowDeploymentRestoreFailure", () => {
     const dataDir = await makeDataDir();
     await writeWorkflowDeploymentRecord(dataDir, "dep_1", baseRecord);
 
-    let record = baseRecord;
+    let record: WorkflowDeploymentRecord | undefined;
     for (let i = 0; i < 3; i++) {
-      record = await recordWorkflowDeploymentRestoreFailure(
-        dataDir,
-        "dep_1",
-        record,
-        { kind: "permanent", reason: "still malformed" },
-      );
+      record = await recordWorkflowDeploymentRestoreFailure(dataDir, "dep_1", {
+        kind: "permanent",
+        reason: "still malformed",
+      });
     }
 
-    expect(record.restoreFailure?.attempts).toBe(3);
-    expect(record.restoreFailure?.kind).toBe("permanent");
+    expect(record?.restoreFailure?.attempts).toBe(3);
+    expect(record?.restoreFailure?.kind).toBe("permanent");
   });
 
   test("a kind change resets the counter rather than adding to the other kind's count", async () => {
     const dataDir = await makeDataDir();
     await writeWorkflowDeploymentRecord(dataDir, "dep_1", baseRecord);
 
+    await recordWorkflowDeploymentRestoreFailure(dataDir, "dep_1", {
+      kind: "permanent",
+      reason: "malformed",
+    });
     let record = await recordWorkflowDeploymentRestoreFailure(
       dataDir,
       "dep_1",
-      baseRecord,
-      { kind: "permanent", reason: "malformed" },
-    );
-    record = await recordWorkflowDeploymentRestoreFailure(
-      dataDir,
-      "dep_1",
-      record,
-      { kind: "permanent", reason: "still malformed" },
+      {
+        kind: "permanent",
+        reason: "still malformed",
+      },
     );
-    expect(record.restoreFailure?.attempts).toBe(2);
+    expect(record?.restoreFailure?.attempts).toBe(2);
+
+    record = await recordWorkflowDeploymentRestoreFailure(dataDir, "dep_1", {
+      kind: "transient",
+      reason: "provider not registered",
+    });
 
-    record = await recordWorkflowDeploymentRestoreFailure(
+    expect(record?.restoreFailure?.kind).toBe("transient");
+    expect(record?.restoreFailure?.attempts).toBe(1);
+  });
+
+  test("is a no-op that returns undefined when the record is already gone (CL-7215: a reclaiming teardown raced ahead of this write)", async () => {
+    const dataDir = await makeDataDir();
+    // Deliberately never written: simulates a teardown deleting the record
+    // before the boot loop's failure-recording catch acquires the lock.
+
+    const updated = await recordWorkflowDeploymentRestoreFailure(
       dataDir,
-      "dep_1",
-      record,
+      "dep_missing",
       { kind: "transient", reason: "provider not registered" },
     );
 
-    expect(record.restoreFailure?.kind).toBe("transient");
-    expect(record.restoreFailure?.attempts).toBe(1);
+    expect(updated).toBeUndefined();
+    const onDisk = await readWorkflowDeploymentRecord(dataDir, "dep_missing");
+    expect(onDisk).toBeUndefined();
   });
 });
 
@@ -253,11 +264,14 @@ describe("clearWorkflowDeploymentRestoreFailure", () => {
     const failed = await recordWorkflowDeploymentRestoreFailure(
       dataDir,
       "dep_1",
-      baseRecord,
       { kind: "transient", reason: "provider not registered" },
     );
 
-    await clearWorkflowDeploymentRestoreFailure(dataDir, "dep_1", failed);
+    await clearWorkflowDeploymentRestoreFailure(
+      dataDir,
+      "dep_1",
+      failed as WorkflowDeploymentRecord,
+    );
 
     const onDisk = await readWorkflowDeploymentRecord(dataDir, "dep_1");
     expect(onDisk?.restoreFailure).toBeUndefined();

apps/sidecar/src/workflow-deployment-record.ts

@@ -227,24 +227,35 @@ export async function markWorkflowDeploymentRecordParked(
  * `attempts` (reset to 1 when `failure.kind` differs from the previously
  * recorded kind, so a transient streak can never inflate the permanent
  * counter or vice versa), stamp `reason`/`lastAttemptAt`, and persist.
- * Returns the updated record so the caller can check `isWorkflowDeploymentRestoreQuarantined`
- * without a re-read. Takes the caller's in-memory `record` rather than
- * re-reading it, matching `markWorkflowDeploymentRecordParked`'s sibling
- * shape but avoiding a redundant read on the hot boot-restore path.
+ * Returns the updated record so the caller can check
+ * `isWorkflowDeploymentRestoreQuarantined` without a re-read.
+ *
+ * Re-reads the record from disk rather than trusting the caller's
+ * in-memory copy (CL-7215): the boot-restore path's per-deployment lock
+ * serializes this against a reclaiming teardown that can delete the
+ * record out from under a dangling (timed-out but still-running) restore
+ * between when the boot loop scanned it and when this runs. Blindly
+ * writing the stale in-memory `record` would resurrect `deployment.json`
+ * with a false failure stamp for a deployment that was fully reclaimed --
+ * the exact "durable record disagrees with reality" failure this ticket
+ * exists to close, just from the opposite direction. Returns `undefined`
+ * (no write) when the record is already gone, matching
+ * `markWorkflowDeploymentRecordParked`'s no-op-on-missing shape.
  */
 export async function recordWorkflowDeploymentRestoreFailure(
   dataDir: string,
   deploymentId: string,
-  record: WorkflowDeploymentRecord,
   failure: { kind: RestoreFailureKind; reason: string },
-): Promise<WorkflowDeploymentRecord> {
-  const previous = record.restoreFailure;
+): Promise<WorkflowDeploymentRecord | undefined> {
+  const existing = await readWorkflowDeploymentRecord(dataDir, deploymentId);
+  if (existing === undefined) return undefined;
+  const previous = existing.restoreFailure;
   const attempts =
     previous !== undefined && previous.kind === failure.kind
       ? previous.attempts + 1
       : 1;
   const updated: WorkflowDeploymentRecord = {
-    ...record,
+    ...existing,
     restoreFailure: {
       kind: failure.kind,
       attempts,