fix(platform): preserve public bypasses during Zero Trust disable

Author: Brooooooklyn

platform/meta/design-docs/0081-project-zero-trust.md

@@ -139,8 +139,12 @@ over stale Worker variables and KV reads, including scheduled retries.
 
 Project deletion treats Access application deletion as critical because the
 project row is the durable handle. The database cascade occurs only after those
-deletes succeed. Platform disablement removes route enforcement and every
-Void-managed application before recording the feature disabled.
+deletes succeed. Platform disablement clears route enforcement and protected
+custom-domain applications first, retaining public bypasses until the wildcard
+is deleted. Wildcard deletion checkpoints enforcement as disabled and advances
+to bounded bypass cleanup. The saved disable operation remains pending until
+every retained bypass is removed, so failures can resume without interrupting
+previously public applications or repeating completed routing writes.
 
 ## Independent dispatch verification
 

platform/packages/api/src/zero-trust/configuration.ts

@@ -20,7 +20,7 @@ export type ZeroTrustOperation = Readonly<{
   /** Stable across retries; separates completion even when database clocks coincide. */
   id?: string;
   kind: 'configure' | 'reconcile' | 'disable';
-  phase: 'seed' | 'public-bypasses' | 'wildcard' | 'projects' | 'wildcard-delete';
+  phase: 'seed' | 'public-bypasses' | 'wildcard' | 'projects' | 'wildcard-delete' | 'bypass-delete';
   owner: string | null;
   leaseExpiresAt: string | null;
   startedAt: string;
@@ -132,9 +132,14 @@ function parseOperation(value: unknown): ZeroTrustOperation | null {
     throw new ZeroTrustConfigurationError('Zero Trust operation kind is invalid');
   }
   if (
-    !['seed', 'public-bypasses', 'wildcard', 'projects', 'wildcard-delete'].includes(
-      String(input.phase),
-    )
+    ![
+      'seed',
+      'public-bypasses',
+      'wildcard',
+      'projects',
+      'wildcard-delete',
+      'bypass-delete',
+    ].includes(String(input.phase))
   ) {
     throw new ZeroTrustConfigurationError('Zero Trust operation phase is invalid');
   }
@@ -157,8 +162,9 @@ function parseOperation(value: unknown): ZeroTrustOperation | null {
   const kind = input.kind as ZeroTrustOperation['kind'];
   const phase = input.phase as ZeroTrustOperation['phase'];
   if (
-    (kind === 'disable' && !['seed', 'projects', 'wildcard-delete'].includes(phase)) ||
-    (kind !== 'disable' && phase === 'wildcard-delete')
+    (kind === 'disable' &&
+      !['seed', 'projects', 'wildcard-delete', 'bypass-delete'].includes(phase)) ||
+    (kind !== 'disable' && ['wildcard-delete', 'bypass-delete'].includes(phase))
   ) {
     throw new ZeroTrustConfigurationError('Zero Trust operation phase is inconsistent');
   }
@@ -254,6 +260,9 @@ function parseZeroTrustConfiguration(value: unknown): ZeroTrustConfiguration | n
   const wildcardApplication = parseApplication(input.wildcardApplication);
   const operation = parseOperation(input.operation);
   const status = input.status as ZeroTrustConfiguration['status'];
+  if (operation?.phase === 'bypass-delete' && (input.enabled || wildcardApplication)) {
+    throw new ZeroTrustConfigurationError('Zero Trust bypass cleanup requires wildcard removal');
+  }
   if (
     (status === 'ready' && operation !== null) ||
     (status === 'configuring' && (!operation || operation.kind === 'disable')) ||

platform/packages/api/src/zero-trust/service.ts

@@ -939,8 +939,6 @@ async function disableCandidates(
           sql`${projectZeroTrust.reconciled_operation} IS DISTINCT FROM ${operationCompletionKey(operation)}`,
           eq(projectZeroTrust.effective_protected, true),
           sql`${projectZeroTrust.state} <> 'public'`,
-          sql`${projectZeroTrust.bypass_application_id} IS NOT NULL`,
-          sql`${projectZeroTrust.bypass_application_name} IS NOT NULL`,
           sql`${projectZeroTrust.protected_application_id} IS NOT NULL`,
           sql`${projectZeroTrust.protected_application_name} IS NOT NULL`,
         ),
@@ -950,6 +948,24 @@ async function disableCandidates(
     .limit(limit);
 }
 
+async function bypassCleanupCandidates(
+  db: Db,
+  limit: number,
+): Promise<ReadonlyArray<ProjectCandidate>> {
+  return candidateQuery(db)
+    .where(
+      and(
+        eq(projects.status, 'active'),
+        or(
+          sql`${projectZeroTrust.bypass_application_id} IS NOT NULL`,
+          sql`${projectZeroTrust.bypass_application_name} IS NOT NULL`,
+        ),
+      ),
+    )
+    .orderBy(projects.created_at, projects.id)
+    .limit(limit);
+}
+
 async function setInitialProjectProtection(
   db: Db,
   projectId: string,
@@ -1100,7 +1116,7 @@ function isDeletedProject(error: unknown): boolean {
   return error instanceof ZeroTrustUnavailableError && error.message === 'Project is unavailable.';
 }
 
-async function mutateWildcardApplication(
+async function withZeroTrustOperationLock(
   db: Db,
   configuration: ZeroTrustConfiguration,
   mutate: (lockedDb: Db) => Promise<ZeroTrustConfiguration>,
@@ -1345,7 +1361,7 @@ async function runConfigureOperation(
           throw new Error('The platform Access application checkpoint is missing.');
         }
         const name = configuration.wildcardApplicationName;
-        configuration = await mutateWildcardApplication(db, configuration, async (lockedDb) => {
+        configuration = await withZeroTrustOperationLock(db, configuration, async (lockedDb) => {
           const wildcard = await client.ensureApplication(
             {
               name,
@@ -1459,7 +1475,6 @@ async function disableProjectZeroTrust(
         return;
       }
       if (row) {
-        await client.deleteApplication(row.bypass_application_id, row.bypass_application_name);
         await client.deleteApplication(
           row.protected_application_id,
           row.protected_application_name,
@@ -1470,8 +1485,8 @@ async function disableProjectZeroTrust(
         effective_protected: false,
         state: 'public',
         reconciled_operation: operationCompletionKey(saved.operation),
-        bypass_application_id: null,
-        bypass_application_name: null,
+        // Public exceptions must outlive the wildcard, including when a later
+        // project or wildcard deletion fails. The final phase clears these handles.
         protected_application_id: null,
         protected_application_name: null,
         protected_application_audience: null,
@@ -1487,6 +1502,42 @@ async function disableProjectZeroTrust(
   }
 }
 
+async function deleteProjectZeroTrustBypass(
+  db: Db,
+  projectId: string,
+  client: ZeroTrustCloudflareClient,
+  configuration: ZeroTrustConfiguration,
+): Promise<void> {
+  let operationError: unknown;
+  await db.transaction(async (tx) => {
+    await tx.execute(sql`SELECT id FROM projects WHERE id = ${projectId} FOR UPDATE`);
+    await lockZeroTrustConfiguration(tx);
+    const saved = await readZeroTrustConfigurationStrict(tx);
+    if (!sameConfiguration(configuration, saved) || saved.operation?.phase !== 'bypass-delete') {
+      throw new ZeroTrustUnavailableError('The Zero Trust disable operation was replaced.');
+    }
+    const row = await getProjectZeroTrustState(tx, projectId);
+    if (!row) {
+      return;
+    }
+    try {
+      await client.deleteApplication(row.bypass_application_id, row.bypass_application_name);
+      await updateProjectState(tx, projectId, {
+        bypass_application_id: null,
+        bypass_application_name: null,
+        state: 'public',
+        last_error: null,
+      });
+    } catch (error) {
+      await markProjectError(tx, projectId, error);
+      operationError = error;
+    }
+  });
+  if (operationError !== undefined) {
+    throw operationError;
+  }
+}
+
 async function runDisableOperation(
   env: ApiEnv['Bindings'],
   db: Db,
@@ -1538,7 +1589,7 @@ async function runDisableOperation(
         continue;
       }
       if (operation.phase === 'wildcard-delete') {
-        configuration = await mutateWildcardApplication(db, configuration, async (lockedDb) => {
+        configuration = await withZeroTrustOperationLock(db, configuration, async (lockedDb) => {
           // A resumed cleanup must drain late/legacy project work before losing
           // the wildcard checkpoint, including work left by an errored request.
           if ((await disableCandidates(lockedDb, operation, 1)).length > 0) {
@@ -1548,10 +1599,38 @@ async function runDisableOperation(
             configuration.wildcardApplication?.id,
             configuration.wildcardApplicationName ?? configuration.wildcardApplication?.name,
           );
-          return patchConfiguration(lockedDb, configuration, {
+          return advanceOperation(lockedDb, configuration, 'bypass-delete', {
             enabled: false,
-            status: 'ready',
             wildcardApplication: null,
+            lastError: null,
+          });
+        });
+        continue;
+      }
+      if (operation.phase === 'bypass-delete') {
+        const candidates = await bypassCleanupCandidates(
+          db,
+          MAX_PROJECTS_PER_RUN - projectsProcessed,
+        );
+        for (const candidate of candidates) {
+          await deleteProjectZeroTrustBypass(db, candidate.project.id, client, configuration);
+          projectsProcessed += 1;
+          configuration = await heartbeatConfiguration(db, configuration);
+        }
+        if (projectsProcessed >= MAX_PROJECTS_PER_RUN) {
+          return releaseOperation(db, configuration);
+        }
+        configuration = await withZeroTrustOperationLock(db, configuration, async (lockedDb) => {
+          // Recheck late project work under the lock used by project mutations
+          // before releasing the disable operation and allowing configuration again.
+          if ((await disableCandidates(lockedDb, operation, 1)).length > 0) {
+            return advanceOperation(lockedDb, configuration, 'projects');
+          }
+          if ((await bypassCleanupCandidates(lockedDb, 1)).length > 0) {
+            return configuration;
+          }
+          return patchConfiguration(lockedDb, configuration, {
+            status: 'ready',
             operation: null,
             lastError: null,
           });