From 73b39fe42d462e325d0d58f25d8dc20ad0df420b Mon Sep 17 00:00:00 2001 From: Baylem Date: Sat, 7 Nov 2026 12:30:32 -0500 Subject: [PATCH 1/3] fix(backup): invalidate captures after game container restarts Signed-off-by: Baylem --- .../backup_container_restart_test.go | 284 ++++++++++++++++++ .../internal/controller/backup_guard_test.go | 2 +- .../internal/controller/backup_preparation.go | 34 ++- .../controller/helpers_envtest_test.go | 1 + operator/specs.md | 2 +- 5 files changed, 318 insertions(+), 5 deletions(-) create mode 100644 operator/internal/controller/backup_container_restart_test.go diff --git a/operator/internal/controller/backup_container_restart_test.go b/operator/internal/controller/backup_container_restart_test.go new file mode 100644 index 000000000..3743b384f --- /dev/null +++ b/operator/internal/controller/backup_container_restart_test.go @@ -0,0 +1,284 @@ +package controller + +import ( + "encoding/json" + "strings" + "testing" + "time" + + snapshotv1 "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1" + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + + gameplanev1alpha1 "github.com/GameplanePanel/gameplane/operator/api/v1alpha1" + "github.com/GameplanePanel/gameplane/operator/internal/agent" +) + +func backupRunningGameStatus() corev1.ContainerStatus { + return corev1.ContainerStatus{Name: "game", Image: "test/game:fixture", ImageID: "containerd://fixture-image", ContainerID: "containerd://original-game", RestartCount: 0, + State: corev1.ContainerState{Running: &corev1.ContainerStateRunning{StartedAt: metav1.NewTime(time.Unix(1, 0))}}} +} + +func changeBackupGamePod(t *testing.T, r *BackupReconciler, change func(*corev1.Pod)) { + t.Helper() + var pod corev1.Pod + if err := r.Get(t.Context(), client.ObjectKey{Namespace: "ns", Name: "server-0"}, &pod); err != nil { + t.Fatal(err) + } + change(&pod) + if err := r.Status().Update(t.Context(), &pod); err != nil { + t.Fatal(err) + } +} + +// Pod phase/UID remain stable through all of these kubelet observations. +func backupGameChanges() map[string]func(*corev1.Pod) { + return map[string]func(*corev1.Pod){ + "container-id": func(p *corev1.Pod) { p.Status.ContainerStatuses[0].ContainerID = "containerd://replacement-game" }, + "restart-count": func(p *corev1.Pod) { p.Status.ContainerStatuses[0].RestartCount++ }, + "missing-status": func(p *corev1.Pod) { p.Status.ContainerStatuses = nil }, + "missing-id": func(p *corev1.Pod) { p.Status.ContainerStatuses[0].ContainerID = "" }, + "waiting": func(p *corev1.Pod) { + p.Status.ContainerStatuses[0].State = corev1.ContainerState{Waiting: &corev1.ContainerStateWaiting{Reason: "CrashLoopBackOff"}} + }, + "terminated": func(p *corev1.Pod) { + p.Status.ContainerStatuses[0].State = corev1.ContainerState{Terminated: &corev1.ContainerStateTerminated{ExitCode: 1}} + }, + } +} + +func TestBackupGameContainerChangesAbortAndDrain(t *testing.T) { + for _, stage := range []string{"before-quiesce", "during-quiesce", "during-copy", "before-success"} { + for name, change := range backupGameChanges() { + t.Run(stage+"/"+name, func(t *testing.T) { + r, q := guardFixtureWorkers(t, false, guardBackup("first")) + budgetPass(t, r, "first") + worker := budgetReadyWorker(t, r, "first", "copy-worker") + mutate := func() { changeBackupGamePod(t, r, change) } + if stage == "during-quiesce" { + q.afterQuiesce = mutate + } else if stage == "before-quiesce" { + mutate() + } else { + budgetPass(t, r, "first") + if q.quiesced != 1 { + t.Fatalf("copy was not quiesced before source change: %+v", q) + } + mutate() + if stage == "before-success" { + var job batchv1.Job + if err := r.Get(t.Context(), client.ObjectKey{Namespace: "ns", Name: "first"}, &job); err != nil { + t.Fatal(err) + } + job.Status.Succeeded = 1 + if err := r.Status().Update(t.Context(), &job); err != nil { + t.Fatal(err) + } + } + } + budgetPass(t, r, "first") + b := guardGetBackup(t, r, "first") + if (stage == "before-quiesce" && q.quiesced != 0) || (stage != "before-quiesce" && q.quiesced != 1) { + t.Fatalf("unexpected quiesce attempts across source change: %+v", q) + } + if b.Status.Phase != gameplanev1alpha1.BackupPhaseFailed || b.Status.SnapshotID != "" || !strings.Contains(b.Status.Message, "source changed") { + t.Fatalf("same-Pod source change allowed capture: %+v", b.Status) + } + if condition := meta.FindStatusCondition(b.Status.Conditions, "Consistency"); condition == nil || condition.Status != metav1.ConditionFalse { + t.Fatalf("invalid capture retained a consistency guarantee: %+v", condition) + } + var gate corev1.ConfigMap + if err := r.Get(t.Context(), client.ObjectKey{Namespace: "ns", Name: backupStartConfigMapName(b)}, &gate); err != nil || gate.Data[backupStartTokenKey] != "" { + t.Fatalf("source change retained copy authorization: %+v, %v", gate.Data, err) + } + var gs gameplanev1alpha1.GameServer + if err := r.Get(t.Context(), client.ObjectKey{Namespace: "ns", Name: "server"}, &gs); err != nil { + t.Fatal(err) + } + if gs.Annotations[lifecycleOperationAnnotation] == "" || q.unquiesced != 0 { + t.Fatalf("live copy worker lost protection: owner=%q, agent=%+v", gs.Annotations[lifecycleOperationAnnotation], q) + } + if err := r.Delete(t.Context(), worker); err != nil { + t.Fatal(err) + } + budgetPass(t, r, "first") + if err := r.Get(t.Context(), client.ObjectKeyFromObject(&gs), &gs); err != nil { + t.Fatal(err) + } + if gs.Annotations[lifecycleOperationAnnotation] != "" || (stage != "before-quiesce" && q.unquiesced != 1) { + t.Fatalf("drained copy did not finish cleanup: owner=%q, agent=%+v", gs.Annotations[lifecycleOperationAnnotation], q) + } + }) + } + } +} + +func TestBackupInitialGameContainerEvidenceUsesExistingDeadline(t *testing.T) { + for _, name := range []string{"missing-status", "missing-id", "waiting", "terminated"} { + t.Run(name, func(t *testing.T) { + r, q := guardFixture(t, guardBackup("first")) + changeBackupGamePod(t, r, backupGameChanges()[name]) + budgetPass(t, r, "first") + b := guardGetBackup(t, r, "first") + deadline := b.Annotations[backupPreparationDeadlineAnnotation] + if deadline == "" || b.Annotations[backupPreparationAnnotation] != "" || q.quiesced != 0 || b.Status.Phase == gameplanev1alpha1.BackupPhaseFailed { + t.Fatalf("unproven container was admitted or failed before deadline: %+v %+v", b, q) + } + // Reinstantiation cannot refresh the existing preparation budget. + r = &BackupReconciler{Client: r.Client, Scheme: r.Scheme, AgentClient: q, LogReader: r.LogReader} + budgetPass(t, r, "first") + b = guardGetBackup(t, r, "first") + if b.Annotations[backupPreparationDeadlineAnnotation] != deadline { + t.Fatal("missing container evidence reset the deadline") + } + b.Annotations[backupPreparationDeadlineAnnotation] = time.Now().Add(-time.Minute).UTC().Format(time.RFC3339Nano) + if err := r.Update(t.Context(), b); err != nil { + t.Fatal(err) + } + budgetPass(t, r, "first") + if got := guardGetBackup(t, r, "first"); got.Status.Phase != gameplanev1alpha1.BackupPhaseFailed || q.quiesced != 0 { + t.Fatalf("unproven container outlived its budget: %+v %+v", got.Status, q) + } + }) + } +} + +func TestBackupUnchangedGameIgnoresAgentRestart(t *testing.T) { + r, q := guardFixture(t, guardBackup("first")) + guardPass(t, r, "first") + b := guardGetBackup(t, r, "first") + var proof map[string]any + if err := json.Unmarshal([]byte(b.Annotations[backupPreparationAnnotation]), &proof); err != nil { + t.Fatal(err) + } + if proof["gameContainerID"] != "containerd://original-game" || proof["gameRestartCount"] != float64(0) { + t.Fatalf("source incarnation was not recorded: %+v", proof) + } + changeBackupGamePod(t, r, func(p *corev1.Pod) { + p.Status.ContainerStatuses = append([]corev1.ContainerStatus{{Name: "agent", ContainerID: "containerd://replacement-agent", RestartCount: 7}}, p.Status.ContainerStatuses...) + }) + var job batchv1.Job + if err := r.Get(t.Context(), client.ObjectKeyFromObject(b), &job); err != nil { + t.Fatal(err) + } + job.Status.Succeeded = 1 + if err := r.Status().Update(t.Context(), &job); err != nil { + t.Fatal(err) + } + guardPass(t, r, "first") + b = guardGetBackup(t, r, "first") + if b.Status.Phase != gameplanev1alpha1.BackupPhaseSucceeded || b.Status.SnapshotID == "" || q.quiesced != 1 { + t.Fatalf("unchanged game lost its capture: %+v %+v", b.Status, q) + } + if condition := meta.FindStatusCondition(b.Status.Conditions, "Consistency"); condition == nil || condition.Reason != "Quiesced" || condition.Status != metav1.ConditionTrue { + t.Fatalf("unchanged game lost achieved consistency: %+v", condition) + } +} + +func TestBackupCSIRejectsSamePodContainerRestart(t *testing.T) { + for _, duringRPC := range []bool{true, false} { + t.Run(map[bool]string{true: "during-quiesce", false: "before-success"}[duringRPC], func(t *testing.T) { + b := guardBackup("first") + b.Spec.Strategy = "volume-snapshot" + r, q := guardFixture(t, b) + restart := func() { changeBackupGamePod(t, r, backupGameChanges()["container-id"]) } + if duringRPC { + q.afterQuiesce = restart + } else { + guardPass(t, r, "first") + var snapshot snapshotv1.VolumeSnapshot + if err := r.Get(t.Context(), client.ObjectKeyFromObject(b), &snapshot); err != nil { + t.Fatal(err) + } + ready := true + snapshot.Status = &snapshotv1.VolumeSnapshotStatus{ReadyToUse: &ready} + if err := r.Status().Update(t.Context(), &snapshot); err != nil { + t.Fatal(err) + } + restart() + } + guardPass(t, r, "first") + if got := guardGetBackup(t, r, "first"); got.Status.Phase != gameplanev1alpha1.BackupPhaseFailed || got.Status.SnapshotID != "" || !strings.Contains(got.Status.Message, "source changed") { + t.Fatalf("CSI captured a replacement game incarnation: %+v", got.Status) + } + }) + } +} + +func TestBackupLegacyLiveIncarnationCannotBeInvented(t *testing.T) { + for _, proof := range []string{"legacy-preparation", "partial-preparation", "no-preparation", "no-preparation-disabled-agent"} { + t.Run(proof, func(t *testing.T) { + r, q := guardFixtureWorkers(t, false, guardBackup("first")) + budgetPass(t, r, "first") + worker := budgetReadyWorker(t, r, "first", "legacy-worker") + budgetPass(t, r, "first") + b := guardGetBackup(t, r, "first") + var identity map[string]any + if err := json.Unmarshal([]byte(b.Annotations[backupPreparationAnnotation]), &identity); err != nil { + t.Fatal(err) + } + delete(identity, "gameRestartCount") + if proof == "legacy-preparation" { + delete(identity, "gameContainerID") + } + encoded, err := json.Marshal(identity) + if err != nil { + t.Fatal(err) + } + b.Annotations[backupPreparationAnnotation] = string(encoded) + if strings.HasPrefix(proof, "no-preparation") { + delete(b.Annotations, backupPreparationAnnotation) + } + if proof == "no-preparation-disabled-agent" { + r.AgentClient = &agent.Client{Disabled: true} + } + if err := r.Update(t.Context(), b); err != nil { + t.Fatal(err) + } + budgetPass(t, r, "first") + got := guardGetBackup(t, r, "first") + if got.Status.Phase != gameplanev1alpha1.BackupPhaseFailed || got.Status.SnapshotID != "" || got.Annotations[backupPreparationAnnotation] != b.Annotations[backupPreparationAnnotation] || q.quiesced != 1 || q.unquiesced != 0 { + t.Fatalf("legacy proof was rebound or cleanup debt lost: %+v %+v", got, q) + } + if err := r.Delete(t.Context(), worker); err != nil { + t.Fatal(err) + } + budgetPass(t, r, "first") + if !strings.HasPrefix(proof, "no-preparation") && q.unquiesced != 1 { + t.Fatalf("legacy Pod-bound cleanup debt was lost: %+v", q) + } + if strings.HasPrefix(proof, "no-preparation") && q.unquiesced != 0 { + t.Fatal("legacy unbound cleanup adopted the observed Pod") + } + }) + } +} + +func TestBackupRawCopyDoesNotRequireGameContainerEvidence(t *testing.T) { + for _, raw := range []bool{true, false} { + t.Run(map[bool]string{true: "quiesce-not-requested", false: "agent-disabled"}[raw], func(t *testing.T) { + b := guardBackup("first") + b.Spec.Quiesce = !raw + r, q := guardFixture(t, b) + if !raw { + r.AgentClient = &agent.Client{Disabled: true} + } + changeBackupGamePod(t, r, backupGameChanges()["missing-status"]) + guardPass(t, r, b.Name) + b = guardGetBackup(t, r, b.Name) + if b.Annotations[backupAuthorizedWorkerAnnotation] == "" || b.Annotations[backupPreparationAnnotation] != "" || b.Status.CaptureManifest == nil || q.quiesced != 0 { + t.Fatalf("raw source acquired an incarnation requirement: %+v %+v", b, q) + } + want := "agent-unavailable" + if raw { + want = "not-requested" + } + if b.Status.CaptureManifest.QuiesceOutcome != want { + t.Fatalf("raw source lost its recorded semantics: %+v", b.Status.CaptureManifest) + } + }) + } +} diff --git a/operator/internal/controller/backup_guard_test.go b/operator/internal/controller/backup_guard_test.go index 90e6ee14a..5a2839be4 100644 --- a/operator/internal/controller/backup_guard_test.go +++ b/operator/internal/controller/backup_guard_test.go @@ -75,7 +75,7 @@ func guardFixtureWorkers(t *testing.T, automatic bool, backups ...*gameplanev1al objects := []client.Object{&gameplanev1alpha1.GameServer{ObjectMeta: metav1.ObjectMeta{Name: "server", Namespace: "ns", UID: "server-uid"}, Spec: gameplanev1alpha1.GameServerSpec{TemplateRef: gameplanev1alpha1.GameTemplateRef{Name: "template"}}}, &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "repo", Namespace: "ns"}, Data: map[string][]byte{"repo": []byte("repo"), "password": []byte("password")}}} one := int32(1) objects = append(objects, &appsv1.StatefulSet{ObjectMeta: metav1.ObjectMeta{Name: "server", Namespace: "ns", UID: "workload-uid", OwnerReferences: []metav1.OwnerReference{{APIVersion: gameplanev1alpha1.GroupVersion.String(), Kind: "GameServer", Name: "server", UID: "server-uid", Controller: ownerBoolPtr(true)}}}, Spec: appsv1.StatefulSetSpec{Replicas: &one}}) - objects = append(objects, &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "server-0", Namespace: "ns", UID: "game-pod-uid", OwnerReferences: []metav1.OwnerReference{{APIVersion: "apps/v1", Kind: "StatefulSet", Name: "server", UID: "workload-uid", Controller: ownerBoolPtr(true)}}}, Spec: corev1.PodSpec{NodeName: "node-a"}, Status: corev1.PodStatus{Phase: corev1.PodRunning}}) + objects = append(objects, &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "server-0", Namespace: "ns", UID: "game-pod-uid", OwnerReferences: []metav1.OwnerReference{{APIVersion: "apps/v1", Kind: "StatefulSet", Name: "server", UID: "workload-uid", Controller: ownerBoolPtr(true)}}}, Spec: corev1.PodSpec{NodeName: "node-a", Containers: []corev1.Container{{Name: "game"}, {Name: "agent"}}}, Status: corev1.PodStatus{Phase: corev1.PodRunning, ContainerStatuses: []corev1.ContainerStatus{backupRunningGameStatus()}}}) for _, b := range backups { objects = append(objects, b) } diff --git a/operator/internal/controller/backup_preparation.go b/operator/internal/controller/backup_preparation.go index be7cfecc8..fa1d4a08d 100644 --- a/operator/internal/controller/backup_preparation.go +++ b/operator/internal/controller/backup_preparation.go @@ -98,6 +98,8 @@ type backupPreparation struct { WorkloadUID types.UID `json:"workloadUID"` WorkloadGeneration int64 `json:"workloadGeneration"` PodUID types.UID `json:"podUID,omitempty"` + GameContainerID string `json:"gameContainerID,omitempty"` + GameRestartCount *int32 `json:"gameRestartCount,omitempty"` } func parseBackupPreparation(b *gameplanev1alpha1.Backup) (*backupPreparation, error) { @@ -125,12 +127,21 @@ func (r *BackupReconciler) prepareBackupSource(ctx context.Context, b *gameplane if err != nil { return false, err } + // Old live preparations cannot prove which game incarnation was paused. + // Keep them parseable for cleanup, but never backfill identity after a + // possible save-off or copy. A pre-upgrade worker is observed separately. + if p != nil && p.Outcome == "live" && (p.GameContainerID == "" || p.GameRestartCount == nil) { + return false, fmt.Errorf("%w: live preparation lacks game container identity", errBackupSourceChanged) + } + if p == nil && b.Annotations[annoQuiesceAttempted] != "" && b.Annotations[annoQuiesceAttempted] != "unsupported" { + return false, fmt.Errorf("%w: quiesce attempt lacks source preparation", errBackupSourceChanged) + } if p == nil && !agentQuiesceAvailable(r.AgentClient) { return true, nil } - // Pre-upgrade attempts have no source identity. Do not invent a new one - // after their RPC; preserve their existing cleanup contract. - if p == nil && b.Annotations[annoQuiesceAttempted] != "" { + // An unsupported attempt did not pause the game and retains raw-copy + // behavior without inventing a protected source identity. + if p == nil && b.Annotations[annoQuiesceAttempted] == "unsupported" { return true, nil } var ss appsv1.StatefulSet @@ -190,7 +201,24 @@ func (r *BackupReconciler) prepareBackupSource(ctx context.Context, b *gameplane if !pod.DeletionTimestamp.IsZero() || pod.Status.Phase != corev1.PodRunning { return r.waitForBackupSource(ctx, b, "source pod is not Running or is terminating") } + var game *corev1.ContainerStatus + for i := range pod.Status.ContainerStatuses { + if pod.Status.ContainerStatuses[i].Name == "game" { + game = &pod.Status.ContainerStatuses[i] + break + } + } + if game == nil || game.ContainerID == "" || game.State.Running == nil { + if p != nil { + return false, fmt.Errorf("%w: original game container is not observably running", errBackupSourceChanged) + } + return r.waitForBackupSource(ctx, b, "game container identity or running state is unavailable") + } + if p != nil && (p.GameContainerID != game.ContainerID || *p.GameRestartCount != game.RestartCount) { + return false, fmt.Errorf("%w: game container restarted", errBackupSourceChanged) + } candidate.Outcome, candidate.PodUID = "live", pod.UID + candidate.GameContainerID, candidate.GameRestartCount = game.ContainerID, &game.RestartCount } if p != nil { if p.Outcome != candidate.Outcome || p.PodUID != candidate.PodUID { diff --git a/operator/internal/controller/helpers_envtest_test.go b/operator/internal/controller/helpers_envtest_test.go index 0a18108ad..98a5cab1d 100644 --- a/operator/internal/controller/helpers_envtest_test.go +++ b/operator/internal/controller/helpers_envtest_test.go @@ -893,6 +893,7 @@ func seedBackupWorkload(t *testing.T, gs *gameplanev1alpha1.GameServer) { t.Fatalf("seed scheduled source pod: %v", err) } pod.Status.Phase = corev1.PodRunning + pod.Status.ContainerStatuses = []corev1.ContainerStatus{backupRunningGameStatus()} if err := k8sClient.Status().Update(context.Background(), pod); err != nil { t.Fatalf("observe running source pod: %v", err) } diff --git a/operator/specs.md b/operator/specs.md index 06ca0b645..db265d81a 100644 --- a/operator/specs.md +++ b/operator/specs.md @@ -361,7 +361,7 @@ Primary reconcilers register with the manager in `cmd/main.go` and handle CRD li - Initial source/fence waits share an existing preparation deadline. CSI captures and older Jobs without one persist a separate two-minute source-startup deadline after acquiring lifecycle ownership; queue age and restarts do not spend or reset that budget. Expired unavailable sources fail visibly and enter normal cleanup. A foreign same-name worker is awaited before acquiring ownership or quiescing. Existing owned workers continue through their original gate/completion checks without resolving the template again; template resolution failure when constructing a new worker aborts and releases only after drainage and save-on. - With quiesce and an agent client enabled, preparation is recorded once in `backup.gameplane.local/preparation`, bound to the target UID and StatefulSet UID/generation. `already-stopped` requires uncached proof of desired and observed zero replicas, caught-up observed generation, and no live game/PVC user pods or pending writer Jobs. Current copy workers are excluded by Job ownership. This path sends no agent calls and preserves suspend/idle/wake intent under the shared workload fence. It applies equally to restic and CSI copies. - Live preparation also pins the game Pod UID. The operator checks the agent's read-only `/v1/pods/{uid}/identity` endpoint before recording the first cleanup intent, then uses pod-bound quiesce/unquiesce routes; the agent rejects a different UID before RCON. A missing endpoint or identity mismatch fails closed without a legacy-route fallback. Ambiguous quiesce errors retain cleanup intent and the original Pod UID on both the Backup and shared owner. Only confirmed save-on or uncached evidence that the original pod is absent/replaced/terminal permits cleanup; an HTTP timeout, unready pod, or terminating pod is insufficient. Legacy attempts without a UID retire only on absence/terminal proof and never adopt a new UID. `Unquiesced=True, Reason=SourceProcessGone` distinguishes process exit from an actual save-on RPC. - - Source proof is checked before and after quiesce and on each copy reconcile. Observed workload/process replacement fails the Backup before it can report a restorable success. Owner recovery retains the pod UID even if the Backup is force-deleted, drains predecessor workers, and never sends its cleanup to a successor pod. + - Live source proof also records the running `game` container's ContainerID and RestartCount, independently of the agent container. It is checked before and after quiesce and on each copy reconcile, including completion. A changed incarnation or missing/non-running game status after binding invalidates the capture even when the Pod UID and phase remain unchanged. Initial unavailable container evidence waits within the existing persisted startup/preparation deadline. In-progress legacy live preparations without this proof fail conservatively without backfilling an incarnation after quiesce; they remain parseable for cleanup, and completed historical manifests are unchanged. Observed workload/process replacement fails the Backup before it can report a restorable success. Owner recovery retains the pod UID even if the Backup is force-deleted, drains predecessor workers, and never sends its cleanup to a successor pod. - Imported legacy recovery records and backups without a recorded Pod UID cannot call unbound save-on, even if an agent is available. Their cleanup intent remains durable until an uncached read proves the game Pod absent or terminal and not terminating; a live or unready replacement is not exit proof. - Upgrade the operator and agent image together and allow existing GameServer pods to roll out the new agent and `GAMEPLANE_POD_UID` DownwardAPI env. Older agents fail the identity preflight before cleanup debt, so the workload can continue its upgrade after terminal cleanup releases the owner. Retry failed copies after rollout; preflight never bypasses protection. This does not automatically stop a running game. - Every copy, including `quiesce=false` and CSI snapshots, acquires the shared lifecycle owner before starting. The Backup pins its target in `backup.gameplane.local/target`; `backup.gameplane.local/owner` remains a compatibility marker. The shared owner is described below. Quiesce cleanup intent is recorded on both the Backup and the surviving GameServer before calling the agent, including partial failures. Cleanup waits for live copy workers; deletion cancels the copy Job with foreground propagation first. Every Backup carries the cleanup finalizer. Pre-upgrade quiesced workers are observed through completion even when overlapping legacy copies cannot acquire the new exclusive owner. From 99349ca5d32c813d350d51b97d8d56780af337c2 Mon Sep 17 00:00:00 2001 From: Baylem Date: Sat, 7 Nov 2026 12:52:17 -0500 Subject: [PATCH 2/3] test(backup): use tagged switch for restart stages Signed-off-by: Baylem --- .../internal/controller/backup_container_restart_test.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/operator/internal/controller/backup_container_restart_test.go b/operator/internal/controller/backup_container_restart_test.go index 3743b384f..5043340b2 100644 --- a/operator/internal/controller/backup_container_restart_test.go +++ b/operator/internal/controller/backup_container_restart_test.go @@ -58,11 +58,12 @@ func TestBackupGameContainerChangesAbortAndDrain(t *testing.T) { budgetPass(t, r, "first") worker := budgetReadyWorker(t, r, "first", "copy-worker") mutate := func() { changeBackupGamePod(t, r, change) } - if stage == "during-quiesce" { + switch stage { + case "during-quiesce": q.afterQuiesce = mutate - } else if stage == "before-quiesce" { + case "before-quiesce": mutate() - } else { + default: budgetPass(t, r, "first") if q.quiesced != 1 { t.Fatalf("copy was not quiesced before source change: %+v", q) From ba50011d0b395edfaa2195cdd0eb55fca576ad34 Mon Sep 17 00:00:00 2001 From: Baylem Date: Sat, 7 Nov 2026 13:34:44 -0500 Subject: [PATCH 3/3] test(backup): seed pre-existing workers in legacy fixtures Signed-off-by: Baylem --- .../internal/controller/backup_guard_test.go | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/operator/internal/controller/backup_guard_test.go b/operator/internal/controller/backup_guard_test.go index 5a2839be4..c995b8b38 100644 --- a/operator/internal/controller/backup_guard_test.go +++ b/operator/internal/controller/backup_guard_test.go @@ -265,7 +265,21 @@ func TestBackupGuardRolloutWaitsForLegacyWorkersAndCleanup(t *testing.T) { legacy := guardBackup("legacy") legacy.Annotations = map[string]string{annoQuiesceAttempted: "true"} r, q := guardFixture(t, legacy, guardBackup("new")) + // An upgrade can observe an existing pre-v1 copy without source proof; + // that proof must never be invented to start a fresh worker after save-off. + podSpec := r.buildBackupPodSpec(legacy, &gameplanev1alpha1.GameTemplate{}) + podSpec.RestartPolicy = corev1.RestartPolicyNever + backoff, deadline := resolveJobLimits(r.JobBackoffLimit, r.JobActiveDeadlineSeconds) + job := &batchv1.Job{ObjectMeta: metav1.ObjectMeta{Name: legacy.Name, Namespace: legacy.Namespace, + OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(legacy, gameplanev1alpha1.GroupVersion.WithKind("Backup"))}}, + Spec: batchv1.JobSpec{BackoffLimit: &backoff, ActiveDeadlineSeconds: &deadline, Template: corev1.PodTemplateSpec{Spec: podSpec}}} + if err := r.Create(t.Context(), job); err != nil { + t.Fatal(err) + } guardPass(t, r, "legacy") + if got := guardGetBackup(t, r, "legacy"); got.Annotations[backupPreparationAnnotation] != "" || got.Status.CaptureManifest != nil { + t.Fatalf("pre-upgrade Job acquired invented capture history: %+v", got) + } guardPass(t, r, "new") var jobs batchv1.JobList if err := r.List(context.Background(), &jobs); err != nil { @@ -589,7 +603,23 @@ func TestBackupGuardPendingSnapshotCancellationInvalidatesBeforeSaveOn(t *testin a.Annotations = map[string]string{annoQuiesceAttempted: "true"} } r, q := guardFixture(t, a, guardBackup("next")) + if mode == "legacy" { + // Model a pending capture created before the source-proof protocol, + // rather than asking the upgraded operator to create one unsafely. + pvc := a.Spec.ServerRef.Name + "-data" + snapshot := &snapshotv1.VolumeSnapshot{ObjectMeta: metav1.ObjectMeta{Name: a.Name, Namespace: a.Namespace, + OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(a, gameplanev1alpha1.GroupVersion.WithKind("Backup"))}}, + Spec: snapshotv1.VolumeSnapshotSpec{Source: snapshotv1.VolumeSnapshotSource{PersistentVolumeClaimName: &pvc}, VolumeSnapshotClassName: a.Spec.VolumeSnapshotClassName}} + if err := r.Create(t.Context(), snapshot); err != nil { + t.Fatal(err) + } + } guardPass(t, r, "snapshot") + if mode == "legacy" { + if got := guardGetBackup(t, r, "snapshot"); got.Annotations[backupPreparationAnnotation] != "" || got.Status.CaptureManifest != nil { + t.Fatalf("pre-upgrade CSI capture acquired invented history: %+v", got) + } + } switch mode { case "orphan": forceDeleteGuardBackup(t, r, "snapshot")