🤖 feat: write a cleanup record before each CoderAIProvider create (#353 part 2a)

.cspell.json

@@ -2,6 +2,7 @@
   "version": "0.2",
   "language": "en",
   "words": [
+    "buildinfo",
     "allapp",
     "automount",
     "prebuild",

.github/workflows/ci.yaml

@@ -95,6 +95,7 @@ jobs:
               # Shared Coder access that CoderTemplateTest and later pool controllers call.
               - 'internal/controller/controlplane_access.go'
               - 'internal/controller/controlplane_hold.go'
+              - 'internal/controller/cleanup_*'
               - 'internal/controller/coder_access.go'
               - 'internal/controller/coder_workspace_steps.go'
               # The URL that the aggregated API server and the controllers use for Coder API calls.

config/default/kustomization.yaml

@@ -10,6 +10,7 @@ resources:
   - ../crd/bases/coder.com_coderprovisioners.yaml
   - ../crd/bases/coder.com_codertemplatetests.yaml
   - ../crd/bases/coder.com_coderworkspaceproxies.yaml
+  - ../rbac/cleanup-records-rolebinding.yaml
   - ../rbac/clusterrolebinding.yaml
   - ../rbac/role.yaml
   - ../rbac/serviceaccount.yaml

config/rbac/cleanup-records-rolebinding.yaml

@@ -0,0 +1,16 @@
+# Lets the operator write its cleanup records (#353): ConfigMaps named ck8s-cleanup-<object UID> in its own
+# namespace. The namespaced Role manager-role comes from the kubebuilder RBAC markers in role.yaml. No other
+# subject gets write access to these ConfigMaps.
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: coder-k8s-cleanup-records
+  namespace: coder-system
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: manager-role
+subjects:
+  - kind: ServiceAccount
+    name: coder-k8s
+    namespace: coder-system

config/rbac/role.yaml

@@ -198,3 +198,21 @@ rules:
   - patch
   - update
   - watch
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: manager-role
+  namespace: coder-system
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - configmaps
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - update
+  - watch

dist/install.yaml

@@ -6960,6 +6960,24 @@ metadata:
   namespace: coder-system
 ---
 apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: manager-role
+  namespace: coder-system
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - configmaps
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - update
+  - watch
+---
+apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
 metadata:
   name: manager-role
@@ -7160,6 +7178,20 @@ rules:
   - watch
 ---
 apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: coder-k8s-cleanup-records
+  namespace: coder-system
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: manager-role
+subjects:
+- kind: ServiceAccount
+  name: coder-k8s
+  namespace: coder-system
+---
+apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
 metadata:
   name: coder-k8s

docs/explanation/architecture.md

@@ -76,7 +76,7 @@ graph TD
 | Path | Contents |
 | --- | --- |
 | `config/crd/bases/` | The generated CRDs for `CoderControlPlane`, `CoderProvisioner`, `CoderWorkspaceProxy`, `CoderTemplateTest`, `CoderAIProvider`, `CoderAIGatewayKey`, and `CoderMCPServer` |
-| `config/rbac/` | The `coder-k8s` ServiceAccount, the `manager-role` ClusterRole, and the bindings (including auth-delegator) |
+| `config/rbac/` | The `coder-k8s` ServiceAccount, the `manager-role` ClusterRole, the `manager-role` Role in `coder-system` (write access to the cleanup-record ConfigMaps there), and the bindings (including auth-delegator) |
 | `config/admission/` | Opt-in admission policies that the install bundle does not apply, for example the operator-only status write policy for `CoderMCPServer` |
 | `config/apiserver-standalone/` | For standalone `--app=aggregated-apiserver`: the `coder-k8s-apiserver` ServiceAccount, its RBAC, and the placeholder Secret for the serving CA |
 | `deploy/deployment.yaml` | The `coder-k8s` Deployment (default `--app=all`) |

docs/how-to/manage-ai-gateway-keys.md

@@ -139,7 +139,7 @@ The finalizer stays while Coder cannot confirm the deletion:
 
 When the control plane is gone, nothing can delete the keys. The finalizer goes at once. The operator logs `Releasing the finalizer without deleting the keys in Coder` with reason `ControlPlaneGone` and the key IDs: delete those keys with `coder ai-gateway keys delete`.
 
-When you delete the control plane while `CoderAIGatewayKey` or `CoderAIProvider` objects that use it are being deleted, the control plane waits for them for up to 10 minutes: it keeps its finalizer, so Coder keeps running, and the objects delete their keys and providers as usual, with retries. The operator log shows `Waiting for the objects that use this control plane` with their names. The operator removes the workspace RBAC of the control plane before it waits, so Coder loses its Kubernetes access at once. An object that you do not delete holds the control plane for only 30 seconds, so delete the objects together with the control plane (or within 30 seconds). After 10 minutes, the objects that remain get 30 more seconds plus 10 seconds per object of one kind (at most 5 minutes) for their bounded pass below, and then the control plane goes anyway. A foreground deletion (`kubectl delete --cascade=foreground`) does not wait, because Kubernetes then deletes the Coder Deployment first.
+When you delete the control plane while `CoderAIGatewayKey` or `CoderAIProvider` objects that use it are being deleted, the control plane delete can take up to about 15 minutes. The control plane waits for the objects for up to 10 minutes: it keeps its finalizer, so Coder keeps running, and the objects delete their keys and providers as usual, with retries. The operator log shows `Waiting for the objects that use this control plane` with their names. The operator removes the workspace RBAC of the control plane before it waits, so Coder loses its Kubernetes access at once. An object that you do not delete holds the control plane for only 30 seconds, so delete the objects together with the control plane (or within 30 seconds). After 10 minutes, the objects that remain get at most 5 more minutes (30 seconds plus 10 seconds per object of one kind) for their bounded pass below, and then the control plane goes anyway. A foreground deletion (`kubectl delete --cascade=foreground`) does not wait, because Kubernetes then deletes the Coder Deployment first.
 
 When the control plane does not wait (its namespace is being deleted, it is deleted in the foreground, or the 10 minutes have passed) and its operator access still works, the operator runs one pass of at most 10 seconds without the settle window, then removes the finalizer whatever the answer. A key create that was still in flight at that moment can commit later, and its key stays in Coder. When the pass fails, the log lists the IDs that it did not delete.
 

docs/how-to/manage-ai-providers.md

@@ -122,11 +122,19 @@ Deleting the object deletes the provider in Coder. With only an unconfirmed crea
 
 - When Coder is unreachable, or the control plane has no operator access, the finalizer `coder.com/ai-provider-cleanup` stays and the operator retries every 60 seconds. See [Stuck finalizer](#stuck-finalizer).
 - When the control plane is gone, nothing can delete the provider. The finalizer goes at once, and the operator logs the provider ID. Delete the provider in the Coder dashboard.
-- When you delete the control plane while `CoderAIProvider` or `CoderAIGatewayKey` objects that use it are being deleted, the control plane waits for them for up to 10 minutes, plus 30 seconds and 10 seconds per object of one kind (at most 5 minutes) for a last bounded attempt. It keeps its finalizer, so Coder keeps running, and the delete above runs with its retries. The operator removes the workspace RBAC of the control plane before it waits, so Coder loses its Kubernetes access at once. An object that you do not delete holds the control plane for only 30 seconds, so delete the objects together with the control plane. A foreground deletion (`kubectl delete --cascade=foreground`) does not wait.
+- When you delete the control plane while `CoderAIProvider` or `CoderAIGatewayKey` objects that use it are being deleted, the control plane delete can take up to about 15 minutes. The control plane waits for the objects for up to 10 minutes, then for at most 5 more minutes (30 seconds plus 10 seconds per object of one kind) for a last bounded attempt. It keeps its finalizer, so Coder keeps running, and the delete above runs with its retries. The operator removes the workspace RBAC of the control plane before it waits, so Coder loses its Kubernetes access at once. An object that you do not delete holds the control plane for only 30 seconds, so delete the objects together with the control plane. A foreground deletion (`kubectl delete --cascade=foreground`) does not wait.
 - When the control plane does not wait (its namespace is being deleted, it is deleted in the foreground, or the 10 minutes have passed) and its operator access still works, the operator sends one delete with a 10-second limit, then removes the finalizer. Without working access, the finalizer goes without a delete. Either way a namespace deletion finishes. When the delete did not succeed, the operator logs the provider ID: delete the provider in the Coder dashboard. The metric `coder_k8s_cleanup_released_without_delete_total` (label `kind`) counts these releases.
 
 Delete `CoderAIProvider` objects before their namespace: a namespace deletion removes Coder's pods at once, so the control plane cannot wait for them. Coder deletes a provider even while chat models use it. Those models stop working.
 
+### Cleanup records
+
+Before the operator creates a provider in Coder, it writes a cleanup record: a ConfigMap named `ck8s-cleanup-<object UID>` with the label `coder.com/cleanup-record=CoderAIProvider` in the operator's namespace (`coder-system`). If it cannot write the record, it does not create the provider. The record holds no key: it holds the object's namespace, name and UID, Coder's deployment ID from `/api/v2/buildinfo`, the provider's display-name nonce, and the provider ID once the operator knows it. The deployment ID lives in the Coder database, so it names the database that holds the provider.
+
+When the control plane later reaches another Coder deployment (for example after its database Secret changed), the record still locates the provider in the old one. The operator then stalls the object with `CoderDeploymentChanged`, keeps the record and creates nothing in the new deployment. Delete the provider in the old deployment, then delete the ConfigMap.
+
+The operator removes the record when Coder confirms that the provider is deleted. When the finalizer goes without a delete, for example in a namespace deletion, the record stays. Find these records with `kubectl -n coder-system get configmaps -l coder.com/cleanup-record`. The operator gets write access to ConfigMaps only in that namespace (the Role `manager-role` and the RoleBinding `coder-k8s-cleanup-records`). Do not give other users write access to ConfigMaps there: a record names a Coder provider ID that a cleanup deletes. If you install the operator in another namespace, change the namespace of both.
+
 ### Stuck finalizer
 
 A deleted `CoderAIProvider` stays `Terminating` while the operator cannot prove that Coder no longer holds its provider. Two cases never end by themselves:

internal/app/controllerapp/controllerapp.go

@@ -129,6 +129,9 @@ func SetupControllers(mgr manager.Manager) error {
 	aiProviderReconciler := &controller.CoderAIProviderReconciler{
 		Client: client,
 		Scheme: managerScheme,
+		// The operator's own namespace holds the cleanup records (#353).
+		OperatorNamespace: detectOperatorNamespace(),
+		APIReader:         mgr.GetAPIReader(),
 	}
 	if err := aiProviderReconciler.SetupWithManager(mgr); err != nil {
 		return fmt.Errorf("unable to create AI provider controller: %w", err)
@@ -211,6 +214,26 @@ func Run(ctx context.Context) error {
 	return nil
 }
 
+// defaultOperatorNamespace is the namespace of the installed operator, where
+// the Role for the cleanup records lives (config/rbac).
+const defaultOperatorNamespace = "coder-system"
+
+// detectOperatorNamespace returns the namespace that holds the cleanup
+// records: POD_NAMESPACE, else the in-cluster namespace file, else
+// coder-system, which the shipped Role names. Only that fallback differs
+// from detectLeaderElectionNamespace.
+func detectOperatorNamespace() string {
+	if ns := strings.TrimSpace(os.Getenv("POD_NAMESPACE")); ns != "" {
+		return ns
+	}
+	if data, err := os.ReadFile(inClusterNamespacePath); err == nil {
+		if ns := strings.TrimSpace(string(data)); ns != "" {
+			return ns
+		}
+	}
+	return defaultOperatorNamespace
+}
+
 // detectLeaderElectionNamespace returns the namespace to use for leader-election
 // lease objects. Resolution order:
 //  1. POD_NAMESPACE env var (allows explicit override for any environment).

internal/app/controllerapp/controllerapp_internal_test.go

@@ -0,0 +1,22 @@
+package controllerapp
+
+import (
+	"os"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+// #353: the cleanup records live in the namespace that the shipped Role
+// names when neither POD_NAMESPACE nor the in-cluster namespace file says
+// otherwise, also for a controller that runs outside the cluster.
+func TestDetectOperatorNamespace(t *testing.T) {
+	if _, err := os.Stat(inClusterNamespacePath); err == nil {
+		t.Skip("this host has an in-cluster namespace file")
+	}
+	t.Setenv("POD_NAMESPACE", "")
+	require.Equal(t, "coder-system", detectOperatorNamespace())
+	require.Equal(t, "kube-system", detectLeaderElectionNamespace(), "the leader-election fallback stays")
+	t.Setenv("POD_NAMESPACE", "ops")
+	require.Equal(t, "ops", detectOperatorNamespace())
+}