Leave Workers without a proxy token binding on the token they hold

A rotation recorded the new generation on the row of a Worker that has no
__VOID_PROXY_TOKEN binding, although nothing was patched. Senders then
presented a token that Worker does not hold. Such a Worker is now skipped
like one whose author cannot deploy: its row, queue entries and cron
scheduler keep the generation it holds, the step logs that it needs a
redeploy, and REVOKE raises the floor without waiting for it.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>

Author: Brooooooklyn

platform/meta/operations.md

@@ -72,7 +72,7 @@ These commands are for Void Cloud's Workers. Self-hosted installations use the [
 
 A deploy gives a project still on generation 0 a generation 1 token. Its older Workers keep the legacy token, which stays accepted until the project rotates.
 
-Workers deployed before the binding rename hold the legacy token in `__AI_TOKEN`, which rotation cannot replace, so they lose AI, email and ISR revalidation when their project rotates, including after a rollback to such a deployment. Get them redeployed. To list the active ones, set `rename_live_at` (psql `\set`) to the first production deploy that contains commit `54375ae1` (2026-03-03), or a later time if unsure; static deployments in the list need nothing.
+Workers deployed before the binding rename hold the legacy token in `__AI_TOKEN`, which rotation cannot replace. A rotation skips them (`[proxy-token-rotation] skipped <project> worker=<name> has no __VOID_PROXY_TOKEN binding`) and leaves their deployment rows, queue entries and cron scheduler on the legacy token, so queues, crons and proxy calls into them keep working. Their own calls to the proxy (AI, email, ISR revalidation) fail once the floor rises, including after a rollback to such a deployment. Get them redeployed. To list the active ones, set `rename_live_at` (psql `\set`) to the first production deploy that contains commit `54375ae1` (2026-03-03), or a later time if unsure; static deployments in the list need nothing.
 
 ```sql
 SELECT p.id, p.slug, d.id AS deployment_id, d.created_at

platform/packages/api/src/proxy-token-rotation.ts

@@ -324,14 +324,32 @@ async function selectInventory(
  * every retained one (rollback leases excluded), oldest first. A retained
  * Worker whose author can no longer deploy is left out: it keeps its old
  * token, which the raised floor rejects, so rolling back to it needs a new
- * deploy before platform services work again. Rows of a static deployment
- * stay in; their GET finds no script.
+ * deploy before platform services work again. A Worker the running rotation
+ * found without a proxy token binding is left out the same way; one the walk
+ * has not reached yet looks like any other until its GET. Rows of a static
+ * deployment stay in; their GET finds no script.
  */
 export async function listProxyTokenInventory(
   db: DbOrTx,
   project: Pick<Project, 'id' | 'slug' | 'active_deployment_id'>,
 ): Promise<ReadonlyArray<InventoryRow>> {
-  return (await selectInventory(db, project)).filter((row) => !row.skipped);
+  const [job] = await db
+    .select({ payload: projectCleanupJobs.payload })
+    .from(projectCleanupJobs)
+    .where(and(jobTarget(project.id), eq(projectCleanupJobs.kind, PROXY_TOKEN_ROTATION_JOB_KIND)));
+  const unbound = unboundDeployments(job?.payload);
+  return (await selectInventory(db, project)).filter(
+    (row) => !row.skipped && !unbound.has(row.deploymentId),
+  );
+}
+
+/**
+ * Deployments whose Worker this rotation found without a proxy token binding
+ * (`unbound` in the job payload, comma-separated). Their rows keep the
+ * generation they hold.
+ */
+function unboundDeployments(payload: JobPayload | undefined): ReadonlySet<string> {
+  return new Set(payload?.unbound ? payload.unbound.split(',') : []);
 }
 
 function parseCursor(cursor: string | undefined): { createdAt: string; id: string } | undefined {
@@ -547,17 +565,34 @@ async function patchNext(tx: DbOrTx, ctx: StepContext, locked: LockedStep): Prom
   const gen = project.proxy_token_gen;
   const token = await deriveProxyToken(ctx.env.JWT_SECRET, ctx.projectId, gen);
   const bindings = await ctx.cf.getScriptBindings(row.workerName);
+  const current = bindings ? tokenBinding(bindings) : undefined;
+  if (bindings && !current) {
+    // Deployed before the binding rename: there is no binding to PATCH. Its
+    // row and copies keep the generation it holds, so senders still reach it;
+    // its own proxy calls fail once the floor rises, until a redeploy.
+    rotationLog(
+      'skipped',
+      ctx.projectId,
+      `worker=${row.workerName} has no ${PROXY_TOKEN_BINDING} binding; redeploy the project to give it one`,
+    );
+    const unbound = [...new Set(unboundDeployments(payload)).add(row.deploymentId)].join(',');
+    await finishStep(
+      tx,
+      ctx.projectId,
+      sql`jsonb_build_object('cursor', ${cursor}::text, 'unbound', ${unbound}::text)`,
+    );
+    return { kind: 'continue' };
+  }
   let patched = false;
   if (bindings === null) {
     if (row.active && !isStaticProjectType(project.type)) {
       throw new Error(`active Worker ${row.workerName} not found`);
     }
-  } else {
-    const current = tokenBinding(bindings);
-    if (current && current.type !== 'plain_text') {
+  } else if (current) {
+    if (current.type !== 'plain_text') {
       throw new Error(`Worker ${row.workerName} has a ${current.type} proxy token binding`);
     }
-    if (current && current.text !== token) {
+    if (current.text !== token) {
       const result =
         (await ctx.cf.patchScriptBindings(
           row.workerName,
@@ -568,15 +603,15 @@ async function patchNext(tx: DbOrTx, ctx: StepContext, locked: LockedStep): Prom
       }
       patched = true;
       rotationLog('patched', ctx.projectId, `worker=${row.workerName} gen=${gen}`);
-    } else if (current && (locked.attempts > 0 || row.proxyTokenGen !== gen)) {
+    } else if (locked.attempts > 0 || row.proxyTokenGen !== gen) {
       // An earlier attempt patched this Worker and then lost the response,
       // failed or rolled back before its step committed. Its old version can
       // still serve, so it drains as if patched now.
       patched = true;
     }
   }
-  // The Worker now holds tok(gen), or holds no proxy token at all. Only now do
-  // the row and the copies name it, so senders never run ahead of the Worker.
+  // The Worker now holds tok(gen), or has no script. Only now do the row and
+  // the copies name it, so senders never run ahead of the Worker.
   if (patched && row.active) {
     const active = await activeWorker(tx, project);
     if (hasTokenCopies(project, active)) {
@@ -611,8 +646,9 @@ async function revoke(tx: DbOrTx, ctx: StepContext, locked: LockedStep): Promise
     return { kind: 'continue' };
   }
   const gen = project.proxy_token_gen;
+  const unbound = unboundDeployments(payload);
   const stale = (await selectInventory(tx, project)).find(
-    (row) => !row.skipped && row.proxyTokenGen !== gen,
+    (row) => !row.skipped && !unbound.has(row.deploymentId) && row.proxyTokenGen !== gen,
   );
   if (stale) {
     // The floor would reject the token a sender still presents to this Worker.
@@ -633,11 +669,12 @@ async function revoke(tx: DbOrTx, ctx: StepContext, locked: LockedStep): Promise
       await finishStep(tx, ctx.projectId, sql`'{"phase":"patch","cursor":""}'::jsonb`);
       return { kind: 'continue' };
     }
-    if (hasTokenCopies(project, active)) {
+    // An active Worker without the binding keeps the copies of the token it holds.
+    if (current && hasTokenCopies(project, active)) {
       await writeCopies(ctx.env, ctx.projectId, active, token);
     }
   }
-  // Written before COMMIT: KV may only run ahead, and every Worker holds T.
+  // Written before COMMIT: KV may only run ahead, and every patched Worker holds T.
   await ctx.env.ROUTING.put(proxyTokenStateKey(ctx.projectId), serializeProxyTokenState({ gen }));
   await finishStep(
     tx,

platform/packages/api/test/integration/proxy-token-rotation.test.ts

@@ -23,6 +23,7 @@ import { reconcileProjectCleanupJobs } from '../../src/provisioning/cleanup.ts';
 import {
   advanceProxyTokenRotation,
   issueProjectProxyToken,
+  listProxyTokenInventory,
   PROXY_TOKEN_ROTATION_STUCK_ATTEMPTS,
   PROXY_TOKEN_ROTATION_WAIT_MS,
   proxyTokenRotationStuckWarning,
@@ -1028,9 +1029,51 @@ describe('proxy token rotation', () => {
       cf.state.hostScripts.get(fx.worker(fx.retained[0]!))!.bindings = [
         { type: 'd1', name: 'DB', id: 'd1-database-id' },
       ];
+      const log = vi.spyOn(console, 'log').mockImplementation(() => {});
       await request(fx.projectId);
-      expect(await advance(fx.projectId)).toEqual({ status: 'done', gen: 1 });
+      await advance(fx.projectId, { waitMs: 60_000 });
+      expect((await job(fx.projectId))!.payload.phase).toBe('revoke');
+      const [project] = await env.db.select().from(projects).where(eq(projects.id, fx.projectId));
+      expect(
+        (await listProxyTokenInventory(env.db, project!)).map((row) => row.workerName),
+      ).toEqual([fx.retained[1]!, fx.active].map(fx.worker));
+
+      expect(await step(fx.projectId)).toMatchObject({ status: 'pending', phase: 'settle' });
+      expect(await step(fx.projectId)).toEqual({ status: 'done', gen: 1 });
       expect(patchedScripts()).toEqual([fx.worker(fx.retained[1]!), fx.worker(fx.active)]);
+      expect(log).toHaveBeenCalledWith(
+        `[proxy-token-rotation] skipped ${fx.projectId} worker=${fx.worker(fx.retained[0]!)} has no __VOID_PROXY_TOKEN binding; redeploy the project to give it one`,
+      );
+      // Its row still names the token it holds; the floor rises past it.
+      expect(await held(fx.retained[0]!)).toBe(0);
+      expect(await held(fx.retained[1]!)).toBe(1);
+      expect(await held(fx.active)).toBe(1);
+      expect(await floor(fx.projectId)).toEqual({ gen: 1 });
+    });
+
+    it('leaves the row and copies of an active Worker without a proxy token binding', async () => {
+      const cf = cfApi();
+      const fx = await setupProject(cf);
+      cf.state.hostScripts.get(fx.worker(fx.active))!.bindings = [
+        { type: 'd1', name: 'DB', id: 'd1-database-id' },
+      ];
+      vi.spyOn(console, 'log').mockImplementation(() => {});
+      await request(fx.projectId);
+      expect(await advance(fx.projectId)).toEqual({ status: 'done', gen: 1 });
+
+      expect(patchedScripts()).toEqual(fx.retained.map(fx.worker));
+      // One GET in the walk and one at REVOKE: no re-walk for its row.
+      expect(events.filter((event) => event === `GET ${fx.worker(fx.active)}`)).toHaveLength(2);
+      expect(await held(fx.active)).toBe(0);
+      expect(await resolvedGen(fx.projectId)).toBe(0);
+      expect((await routing.get(fx.queueKey, 'json')) as Record<string, unknown>).toMatchObject({
+        proxyToken: fx.legacy,
+      });
+      expect(cron.states.get(fx.projectId)).toMatchObject({ proxyToken: fx.legacy });
+      for (const id of fx.retained) {
+        expect(await held(id)).toBe(1);
+      }
+      expect(await floor(fx.projectId)).toEqual({ gen: 1 });
     });
 
     it('fails when the active Worker of a Worker project has no script', async () => {