diff --git a/api/internal/handlers/recovery_policies_test.go b/api/internal/handlers/recovery_policies_test.go new file mode 100644 index 000000000..110218940 --- /dev/null +++ b/api/internal/handlers/recovery_policies_test.go @@ -0,0 +1,30 @@ +package handlers + +import ( + "encoding/json" + "net/http" + "testing" +) + +func TestRecoveryPoliciesRoundTripThroughGenericAPI(t *testing.T) { + for _, resource := range []struct{ path, kind string }{{"backups", "Backup"}, {"schedules", "BackupSchedule"}} { + t.Run(resource.path, func(t *testing.T) { + router := mountResourcesRouter(fakeKubeClient()) + body := map[string]any{"apiVersion": "gameplane.local/v1alpha1", "kind": resource.kind, "metadata": map[string]any{"name": "recovery"}, "spec": map[string]any{"serverRef": map[string]any{"name": "game"}, "repoRef": map[string]any{"name": "repo", "key": "repo"}, "schedule": "* * * * *", "strategy": "restic-snapshot", "powerPolicy": "stop-copy-restart", "volumePolicy": "all-data"}} + rr := do(t, router, http.MethodPost, "/"+resource.path+"/", body) + if rr.Code != http.StatusCreated { + t.Fatalf("create: %d %s", rr.Code, rr.Body) + } + rr = do(t, router, http.MethodGet, "/"+resource.path+"/recovery", nil) + var got struct { + Spec struct{ PowerPolicy, VolumePolicy string } + } + if err := json.Unmarshal(rr.Body.Bytes(), &got); err != nil { + t.Fatal(err) + } + if got.Spec.PowerPolicy != "stop-copy-restart" || got.Spec.VolumePolicy != "all-data" { + t.Fatalf("policies lost: %+v", got.Spec) + } + }) + } +} diff --git a/assets/design-export/json/bHJy6.json b/assets/design-export/json/bHJy6.json index 84d34efaf..1a32b2e8f 100644 --- a/assets/design-export/json/bHJy6.json +++ b/assets/design-export/json/bHJy6.json @@ -50,7 +50,7 @@ "fill": "$muted", "textGrowth": "fixed-width", "width": "fill_container", - "content": "Only the primary data claim is copied. Rebuildable is a declaration, not proof that recovery can reconstruct its contents.", + "content": "Included claims are copied at the recorded archive roots. Primary-only backups exclude extra data volumes and separate mod claims. Rebuildable volumes are excluded. Check recovery prerequisites before restarting.", "lineHeight": 1.333333, "fontFamily": "$typography/font-sans", "fontSize": 12, diff --git a/assets/design-export/screenshots/bHJy6.png b/assets/design-export/screenshots/bHJy6.png index 4f6308960..b62cbd0ec 100644 Binary files a/assets/design-export/screenshots/bHJy6.png and b/assets/design-export/screenshots/bHJy6.png differ diff --git a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_backups.yaml b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_backups.yaml index e64dc3000..e579af4f7 100644 --- a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_backups.yaml +++ b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_backups.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: bf84afb28a947929dcc56cef655364589a4be2e498da9eafb35363eda630676c + gameplane.local/crd-bundle-sha256: e900c220cd57442f03e3264da6ad0a2e746450523b88edf38affff441b150072 controller-gen.kubebuilder.io/version: v0.22.0 name: backups.gameplane.local spec: @@ -58,6 +58,13 @@ spec: spec: description: BackupSpec is the desired state of a one-shot backup. properties: + powerPolicy: + description: PowerPolicy preserves power by default; stop-copy-restart + temporarily stops under lifecycle ownership. + enum: + - preserve + - stop-copy-restart + type: string quiesce: default: true description: |- @@ -113,6 +120,13 @@ spec: items: type: string type: array + volumePolicy: + description: VolumePolicy captures primary-only by default; all-data + includes classified persistent data. + enum: + - primary-only + - all-data + type: string volumeSnapshotClassName: description: |- VolumeSnapshotClassName names the VolumeSnapshotClass to use when @@ -126,6 +140,10 @@ spec: x-kubernetes-validations: - message: repoRef is required for the restic-snapshot strategy rule: self.strategy == 'volume-snapshot' || has(self.repoRef) + - message: stopped and all-data policies require restic-snapshot + rule: self.strategy != 'volume-snapshot' || ((!has(self.powerPolicy) + || self.powerPolicy == 'preserve') && (!has(self.volumePolicy) || + self.volumePolicy == 'primary-only')) status: description: BackupStatus is the observed state of a Backup. properties: @@ -135,6 +153,13 @@ spec: It describes persistent-volume coverage independently of copy success. Missing on pre-upgrade backups; never reconstruct it from current layout. properties: + formatVersion: + description: FormatVersion 2 uses validated disjoint roots. Absent + means legacy /data. + format: int32 + type: integer + gameVersion: + type: string layoutSource: description: LayoutSource identifies the frozen workload used for mount paths. @@ -159,6 +184,12 @@ spec: type: string strategy: type: string + templateName: + type: string + templateRevision: + type: string + volumePolicy: + type: string volumes: items: description: |- @@ -182,12 +213,24 @@ spec: type: string included: type: boolean + logicalName: + description: LogicalName identifies the primary, declared + extra, or active loader volume. + type: string mountPaths: items: type: string type: array reason: type: string + recoveryPrerequisite: + description: RecoveryPrerequisite explains excluded content + that must be reconstructed. + type: string + snapshotRoot: + description: SnapshotRoot is the explicit disjoint archive + root; never a glob. + type: string required: - claimName - classification diff --git a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_backupschedules.yaml b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_backupschedules.yaml index 5c5d95552..1a47d5a0b 100644 --- a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_backupschedules.yaml +++ b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_backupschedules.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: bf84afb28a947929dcc56cef655364589a4be2e498da9eafb35363eda630676c + gameplane.local/crd-bundle-sha256: e900c220cd57442f03e3264da6ad0a2e746450523b88edf38affff441b150072 controller-gen.kubebuilder.io/version: v0.22.0 name: backupschedules.gameplane.local spec: @@ -72,6 +72,13 @@ spec: - Forbid - Replace type: string + powerPolicy: + description: PowerPolicy preserves power by default; stop-copy-restart + temporarily stops under lifecycle ownership. + enum: + - preserve + - stop-copy-restart + type: string quiesce: default: true description: |- @@ -163,6 +170,13 @@ spec: default: false description: Suspend pauses the schedule without deleting it. type: boolean + volumePolicy: + description: VolumePolicy captures primary-only by default; all-data + includes classified persistent data. + enum: + - primary-only + - all-data + type: string required: - schedule - serverRef @@ -170,6 +184,10 @@ spec: x-kubernetes-validations: - message: repoRef is required for the restic-snapshot strategy rule: self.strategy == 'volume-snapshot' || has(self.repoRef) + - message: stopped and all-data policies require restic-snapshot + rule: self.strategy != 'volume-snapshot' || ((!has(self.powerPolicy) + || self.powerPolicy == 'preserve') && (!has(self.volumePolicy) || + self.volumePolicy == 'primary-only')) status: description: BackupScheduleStatus is the observed state of a BackupSchedule. properties: diff --git a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_clusters.yaml b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_clusters.yaml index 8253ec874..4ed0507d0 100644 --- a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_clusters.yaml +++ b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_clusters.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: bf84afb28a947929dcc56cef655364589a4be2e498da9eafb35363eda630676c + gameplane.local/crd-bundle-sha256: e900c220cd57442f03e3264da6ad0a2e746450523b88edf38affff441b150072 controller-gen.kubebuilder.io/version: v0.22.0 name: clusters.gameplane.local spec: diff --git a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_gameservers.yaml b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_gameservers.yaml index d9dab854d..2e12b7d78 100644 --- a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_gameservers.yaml +++ b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_gameservers.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: bf84afb28a947929dcc56cef655364589a4be2e498da9eafb35363eda630676c + gameplane.local/crd-bundle-sha256: e900c220cd57442f03e3264da6ad0a2e746450523b88edf38affff441b150072 controller-gen.kubebuilder.io/version: v0.22.0 name: gameservers.gameplane.local spec: @@ -978,6 +978,11 @@ spec: the operator creates/maintains a BackupSchedule owned by this GameServer. Remove to stop scheduled backups. properties: + powerPolicy: + enum: + - preserve + - stop-copy-restart + type: string repoRef: description: |- RepoRef points at a BackupRepo (cluster-scoped resource, TBD) or @@ -1028,6 +1033,11 @@ spec: type: string suspend: type: boolean + volumePolicy: + enum: + - primary-only + - all-data + type: string required: - repoRef - schedule @@ -2222,10 +2232,10 @@ spec: properties: backupRole: description: |- - BackupRole describes recovery requirements without adding this volume - to a backup. "data" needs a separate copy; "rebuildable" may be recreated. - Unset remains unclassified and not covered. Both backup strategies still - capture only the primary data claim. + BackupRole declares recovery scope. all-data Restic backups include data + extras and report rebuildable extras as excluded reconstruction prerequisites. + Unset is unclassified and rejects all-data admission. Default primary-only + backups and CSI snapshots still capture only the primary data claim. enum: - data - rebuildable diff --git a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_gametemplates.yaml b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_gametemplates.yaml index 851a1f2dc..2e31837da 100644 --- a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_gametemplates.yaml +++ b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_gametemplates.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: bf84afb28a947929dcc56cef655364589a4be2e498da9eafb35363eda630676c + gameplane.local/crd-bundle-sha256: e900c220cd57442f03e3264da6ad0a2e746450523b88edf38affff441b150072 controller-gen.kubebuilder.io/version: v0.22.0 name: gametemplates.gameplane.local spec: @@ -2209,10 +2209,10 @@ spec: properties: backupRole: description: |- - BackupRole describes recovery requirements without adding this volume - to a backup. "data" needs a separate copy; "rebuildable" may be recreated. - Unset remains unclassified and not covered. Both backup strategies still - capture only the primary data claim. + BackupRole declares recovery scope. all-data Restic backups include data + extras and report rebuildable extras as excluded reconstruction prerequisites. + Unset is unclassified and rejects all-data admission. Default primary-only + backups and CSI snapshots still capture only the primary data claim. enum: - data - rebuildable diff --git a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_modules.yaml b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_modules.yaml index b1cd8c6e9..a2054ca99 100644 --- a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_modules.yaml +++ b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_modules.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: bf84afb28a947929dcc56cef655364589a4be2e498da9eafb35363eda630676c + gameplane.local/crd-bundle-sha256: e900c220cd57442f03e3264da6ad0a2e746450523b88edf38affff441b150072 controller-gen.kubebuilder.io/version: v0.22.0 name: modules.gameplane.local spec: diff --git a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_modulesources.yaml b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_modulesources.yaml index be111610f..db10e1c62 100644 --- a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_modulesources.yaml +++ b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_modulesources.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: bf84afb28a947929dcc56cef655364589a4be2e498da9eafb35363eda630676c + gameplane.local/crd-bundle-sha256: e900c220cd57442f03e3264da6ad0a2e746450523b88edf38affff441b150072 controller-gen.kubebuilder.io/version: v0.22.0 name: modulesources.gameplane.local spec: diff --git a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_networkcaptures.yaml b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_networkcaptures.yaml index cce0df632..28adf5343 100644 --- a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_networkcaptures.yaml +++ b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_networkcaptures.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: bf84afb28a947929dcc56cef655364589a4be2e498da9eafb35363eda630676c + gameplane.local/crd-bundle-sha256: e900c220cd57442f03e3264da6ad0a2e746450523b88edf38affff441b150072 controller-gen.kubebuilder.io/version: v0.22.0 name: networkcaptures.gameplane.local spec: diff --git a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_restores.yaml b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_restores.yaml index 44ad7190b..f7ea09a1f 100644 --- a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_restores.yaml +++ b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_restores.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: bf84afb28a947929dcc56cef655364589a4be2e498da9eafb35363eda630676c + gameplane.local/crd-bundle-sha256: e900c220cd57442f03e3264da6ad0a2e746450523b88edf38affff441b150072 controller-gen.kubebuilder.io/version: v0.22.0 name: restores.gameplane.local spec: @@ -171,6 +171,87 @@ spec: - Succeeded - Failed type: string + recoveryPlan: + description: RestoreRecoveryPlan pins a validated target mapping before + any power mutation. + properties: + admittedGeneration: + format: int64 + type: integer + admittedIdle: + type: string + admittedSuspend: + description: Admitted power intent is informational; cleanup never + overwrites newer user intent. + type: boolean + backupUID: + type: string + gameVersion: + type: string + manifestDigest: + type: string + targetUID: + type: string + templateRevision: + type: string + volumes: + items: + description: |- + BackupVolumeCoverage describes one persistent claim. Extra and loader/mod + claims remain uncovered even when mounted beneath the primary data directory. + properties: + claimName: + type: string + claimUID: + description: ClaimUID is empty if the claim did not yet + exist at admission. + type: string + classification: + description: |- + Classification records the module/server declaration at admission; + it does not prove that the volume can actually be reconstructed. + enum: + - data + - rebuildable + - unclassified + type: string + included: + type: boolean + logicalName: + description: LogicalName identifies the primary, declared + extra, or active loader volume. + type: string + mountPaths: + items: + type: string + type: array + reason: + type: string + recoveryPrerequisite: + description: RecoveryPrerequisite explains excluded content + that must be reconstructed. + type: string + snapshotRoot: + description: SnapshotRoot is the explicit disjoint archive + root; never a glob. + type: string + required: + - claimName + - classification + - included + - reason + type: object + type: array + required: + - admittedGeneration + - admittedSuspend + - backupUID + - gameVersion + - manifestDigest + - targetUID + - templateRevision + - volumes + type: object snapshotID: description: |- SnapshotID is the restic snapshot used. Resolved from the diff --git a/charts/gameplane/crd-manifests/gameplane.local_backups.yaml b/charts/gameplane/crd-manifests/gameplane.local_backups.yaml index e64dc3000..e579af4f7 100644 --- a/charts/gameplane/crd-manifests/gameplane.local_backups.yaml +++ b/charts/gameplane/crd-manifests/gameplane.local_backups.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: bf84afb28a947929dcc56cef655364589a4be2e498da9eafb35363eda630676c + gameplane.local/crd-bundle-sha256: e900c220cd57442f03e3264da6ad0a2e746450523b88edf38affff441b150072 controller-gen.kubebuilder.io/version: v0.22.0 name: backups.gameplane.local spec: @@ -58,6 +58,13 @@ spec: spec: description: BackupSpec is the desired state of a one-shot backup. properties: + powerPolicy: + description: PowerPolicy preserves power by default; stop-copy-restart + temporarily stops under lifecycle ownership. + enum: + - preserve + - stop-copy-restart + type: string quiesce: default: true description: |- @@ -113,6 +120,13 @@ spec: items: type: string type: array + volumePolicy: + description: VolumePolicy captures primary-only by default; all-data + includes classified persistent data. + enum: + - primary-only + - all-data + type: string volumeSnapshotClassName: description: |- VolumeSnapshotClassName names the VolumeSnapshotClass to use when @@ -126,6 +140,10 @@ spec: x-kubernetes-validations: - message: repoRef is required for the restic-snapshot strategy rule: self.strategy == 'volume-snapshot' || has(self.repoRef) + - message: stopped and all-data policies require restic-snapshot + rule: self.strategy != 'volume-snapshot' || ((!has(self.powerPolicy) + || self.powerPolicy == 'preserve') && (!has(self.volumePolicy) || + self.volumePolicy == 'primary-only')) status: description: BackupStatus is the observed state of a Backup. properties: @@ -135,6 +153,13 @@ spec: It describes persistent-volume coverage independently of copy success. Missing on pre-upgrade backups; never reconstruct it from current layout. properties: + formatVersion: + description: FormatVersion 2 uses validated disjoint roots. Absent + means legacy /data. + format: int32 + type: integer + gameVersion: + type: string layoutSource: description: LayoutSource identifies the frozen workload used for mount paths. @@ -159,6 +184,12 @@ spec: type: string strategy: type: string + templateName: + type: string + templateRevision: + type: string + volumePolicy: + type: string volumes: items: description: |- @@ -182,12 +213,24 @@ spec: type: string included: type: boolean + logicalName: + description: LogicalName identifies the primary, declared + extra, or active loader volume. + type: string mountPaths: items: type: string type: array reason: type: string + recoveryPrerequisite: + description: RecoveryPrerequisite explains excluded content + that must be reconstructed. + type: string + snapshotRoot: + description: SnapshotRoot is the explicit disjoint archive + root; never a glob. + type: string required: - claimName - classification diff --git a/charts/gameplane/crd-manifests/gameplane.local_backupschedules.yaml b/charts/gameplane/crd-manifests/gameplane.local_backupschedules.yaml index 5c5d95552..1a47d5a0b 100644 --- a/charts/gameplane/crd-manifests/gameplane.local_backupschedules.yaml +++ b/charts/gameplane/crd-manifests/gameplane.local_backupschedules.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: bf84afb28a947929dcc56cef655364589a4be2e498da9eafb35363eda630676c + gameplane.local/crd-bundle-sha256: e900c220cd57442f03e3264da6ad0a2e746450523b88edf38affff441b150072 controller-gen.kubebuilder.io/version: v0.22.0 name: backupschedules.gameplane.local spec: @@ -72,6 +72,13 @@ spec: - Forbid - Replace type: string + powerPolicy: + description: PowerPolicy preserves power by default; stop-copy-restart + temporarily stops under lifecycle ownership. + enum: + - preserve + - stop-copy-restart + type: string quiesce: default: true description: |- @@ -163,6 +170,13 @@ spec: default: false description: Suspend pauses the schedule without deleting it. type: boolean + volumePolicy: + description: VolumePolicy captures primary-only by default; all-data + includes classified persistent data. + enum: + - primary-only + - all-data + type: string required: - schedule - serverRef @@ -170,6 +184,10 @@ spec: x-kubernetes-validations: - message: repoRef is required for the restic-snapshot strategy rule: self.strategy == 'volume-snapshot' || has(self.repoRef) + - message: stopped and all-data policies require restic-snapshot + rule: self.strategy != 'volume-snapshot' || ((!has(self.powerPolicy) + || self.powerPolicy == 'preserve') && (!has(self.volumePolicy) || + self.volumePolicy == 'primary-only')) status: description: BackupScheduleStatus is the observed state of a BackupSchedule. properties: diff --git a/charts/gameplane/crd-manifests/gameplane.local_clusters.yaml b/charts/gameplane/crd-manifests/gameplane.local_clusters.yaml index 8253ec874..4ed0507d0 100644 --- a/charts/gameplane/crd-manifests/gameplane.local_clusters.yaml +++ b/charts/gameplane/crd-manifests/gameplane.local_clusters.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: bf84afb28a947929dcc56cef655364589a4be2e498da9eafb35363eda630676c + gameplane.local/crd-bundle-sha256: e900c220cd57442f03e3264da6ad0a2e746450523b88edf38affff441b150072 controller-gen.kubebuilder.io/version: v0.22.0 name: clusters.gameplane.local spec: diff --git a/charts/gameplane/crd-manifests/gameplane.local_gameservers.yaml b/charts/gameplane/crd-manifests/gameplane.local_gameservers.yaml index d9dab854d..2e12b7d78 100644 --- a/charts/gameplane/crd-manifests/gameplane.local_gameservers.yaml +++ b/charts/gameplane/crd-manifests/gameplane.local_gameservers.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: bf84afb28a947929dcc56cef655364589a4be2e498da9eafb35363eda630676c + gameplane.local/crd-bundle-sha256: e900c220cd57442f03e3264da6ad0a2e746450523b88edf38affff441b150072 controller-gen.kubebuilder.io/version: v0.22.0 name: gameservers.gameplane.local spec: @@ -978,6 +978,11 @@ spec: the operator creates/maintains a BackupSchedule owned by this GameServer. Remove to stop scheduled backups. properties: + powerPolicy: + enum: + - preserve + - stop-copy-restart + type: string repoRef: description: |- RepoRef points at a BackupRepo (cluster-scoped resource, TBD) or @@ -1028,6 +1033,11 @@ spec: type: string suspend: type: boolean + volumePolicy: + enum: + - primary-only + - all-data + type: string required: - repoRef - schedule @@ -2222,10 +2232,10 @@ spec: properties: backupRole: description: |- - BackupRole describes recovery requirements without adding this volume - to a backup. "data" needs a separate copy; "rebuildable" may be recreated. - Unset remains unclassified and not covered. Both backup strategies still - capture only the primary data claim. + BackupRole declares recovery scope. all-data Restic backups include data + extras and report rebuildable extras as excluded reconstruction prerequisites. + Unset is unclassified and rejects all-data admission. Default primary-only + backups and CSI snapshots still capture only the primary data claim. enum: - data - rebuildable diff --git a/charts/gameplane/crd-manifests/gameplane.local_gametemplates.yaml b/charts/gameplane/crd-manifests/gameplane.local_gametemplates.yaml index 851a1f2dc..2e31837da 100644 --- a/charts/gameplane/crd-manifests/gameplane.local_gametemplates.yaml +++ b/charts/gameplane/crd-manifests/gameplane.local_gametemplates.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: bf84afb28a947929dcc56cef655364589a4be2e498da9eafb35363eda630676c + gameplane.local/crd-bundle-sha256: e900c220cd57442f03e3264da6ad0a2e746450523b88edf38affff441b150072 controller-gen.kubebuilder.io/version: v0.22.0 name: gametemplates.gameplane.local spec: @@ -2209,10 +2209,10 @@ spec: properties: backupRole: description: |- - BackupRole describes recovery requirements without adding this volume - to a backup. "data" needs a separate copy; "rebuildable" may be recreated. - Unset remains unclassified and not covered. Both backup strategies still - capture only the primary data claim. + BackupRole declares recovery scope. all-data Restic backups include data + extras and report rebuildable extras as excluded reconstruction prerequisites. + Unset is unclassified and rejects all-data admission. Default primary-only + backups and CSI snapshots still capture only the primary data claim. enum: - data - rebuildable diff --git a/charts/gameplane/crd-manifests/gameplane.local_modules.yaml b/charts/gameplane/crd-manifests/gameplane.local_modules.yaml index b1cd8c6e9..a2054ca99 100644 --- a/charts/gameplane/crd-manifests/gameplane.local_modules.yaml +++ b/charts/gameplane/crd-manifests/gameplane.local_modules.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: bf84afb28a947929dcc56cef655364589a4be2e498da9eafb35363eda630676c + gameplane.local/crd-bundle-sha256: e900c220cd57442f03e3264da6ad0a2e746450523b88edf38affff441b150072 controller-gen.kubebuilder.io/version: v0.22.0 name: modules.gameplane.local spec: diff --git a/charts/gameplane/crd-manifests/gameplane.local_modulesources.yaml b/charts/gameplane/crd-manifests/gameplane.local_modulesources.yaml index be111610f..db10e1c62 100644 --- a/charts/gameplane/crd-manifests/gameplane.local_modulesources.yaml +++ b/charts/gameplane/crd-manifests/gameplane.local_modulesources.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: bf84afb28a947929dcc56cef655364589a4be2e498da9eafb35363eda630676c + gameplane.local/crd-bundle-sha256: e900c220cd57442f03e3264da6ad0a2e746450523b88edf38affff441b150072 controller-gen.kubebuilder.io/version: v0.22.0 name: modulesources.gameplane.local spec: diff --git a/charts/gameplane/crd-manifests/gameplane.local_networkcaptures.yaml b/charts/gameplane/crd-manifests/gameplane.local_networkcaptures.yaml index cce0df632..28adf5343 100644 --- a/charts/gameplane/crd-manifests/gameplane.local_networkcaptures.yaml +++ b/charts/gameplane/crd-manifests/gameplane.local_networkcaptures.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: bf84afb28a947929dcc56cef655364589a4be2e498da9eafb35363eda630676c + gameplane.local/crd-bundle-sha256: e900c220cd57442f03e3264da6ad0a2e746450523b88edf38affff441b150072 controller-gen.kubebuilder.io/version: v0.22.0 name: networkcaptures.gameplane.local spec: diff --git a/charts/gameplane/crd-manifests/gameplane.local_restores.yaml b/charts/gameplane/crd-manifests/gameplane.local_restores.yaml index 44ad7190b..f7ea09a1f 100644 --- a/charts/gameplane/crd-manifests/gameplane.local_restores.yaml +++ b/charts/gameplane/crd-manifests/gameplane.local_restores.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: bf84afb28a947929dcc56cef655364589a4be2e498da9eafb35363eda630676c + gameplane.local/crd-bundle-sha256: e900c220cd57442f03e3264da6ad0a2e746450523b88edf38affff441b150072 controller-gen.kubebuilder.io/version: v0.22.0 name: restores.gameplane.local spec: @@ -171,6 +171,87 @@ spec: - Succeeded - Failed type: string + recoveryPlan: + description: RestoreRecoveryPlan pins a validated target mapping before + any power mutation. + properties: + admittedGeneration: + format: int64 + type: integer + admittedIdle: + type: string + admittedSuspend: + description: Admitted power intent is informational; cleanup never + overwrites newer user intent. + type: boolean + backupUID: + type: string + gameVersion: + type: string + manifestDigest: + type: string + targetUID: + type: string + templateRevision: + type: string + volumes: + items: + description: |- + BackupVolumeCoverage describes one persistent claim. Extra and loader/mod + claims remain uncovered even when mounted beneath the primary data directory. + properties: + claimName: + type: string + claimUID: + description: ClaimUID is empty if the claim did not yet + exist at admission. + type: string + classification: + description: |- + Classification records the module/server declaration at admission; + it does not prove that the volume can actually be reconstructed. + enum: + - data + - rebuildable + - unclassified + type: string + included: + type: boolean + logicalName: + description: LogicalName identifies the primary, declared + extra, or active loader volume. + type: string + mountPaths: + items: + type: string + type: array + reason: + type: string + recoveryPrerequisite: + description: RecoveryPrerequisite explains excluded content + that must be reconstructed. + type: string + snapshotRoot: + description: SnapshotRoot is the explicit disjoint archive + root; never a glob. + type: string + required: + - claimName + - classification + - included + - reason + type: object + type: array + required: + - admittedGeneration + - admittedSuspend + - backupUID + - gameVersion + - manifestDigest + - targetUID + - templateRevision + - volumes + type: object snapshotID: description: |- SnapshotID is the restic snapshot used. Resolved from the diff --git a/design.pen b/design.pen index d43856582..4030ed3d5 100644 --- a/design.pen +++ b/design.pen @@ -127593,7 +127593,7 @@ "fill": "$muted", "textGrowth": "fixed-width", "width": "fill_container", - "content": "Only the primary data claim is copied. Rebuildable is a declaration, not proof that recovery can reconstruct its contents.", + "content": "Included claims are copied at the recorded archive roots. Primary-only backups exclude extra data volumes and separate mod claims. Rebuildable volumes are excluded. Check recovery prerequisites before restarting.", "lineHeight": 1.333333, "fontFamily": "$typography/font-sans", "fontSize": 12, diff --git a/docs/operator-operations.md b/docs/operator-operations.md index f24810524..9a123bdc1 100644 --- a/docs/operator-operations.md +++ b/docs/operator-operations.md @@ -2,38 +2,96 @@ ## Backup consistency and volume coverage -A successful copy is not necessarily a complete server recovery point. -`status.conditions[type=Consistency]` reports the achieved result separately -from `Completed`: Quiesced means the agent confirmed the module's quiesce -commands before the completed copy; it does not prove those commands flush all -application state. UnsupportedRawCopy, RawCopy and AgentUnavailableRawCopy -describe restic copies without confirmed quiesce. CSI without confirmed -quiesce reports CrashConsistentSnapshot. StoppedSource means the source was -already stopped and its writers drained before capture; the backup did not stop -it. Failed or unfinished copies, including completed Jobs still awaiting a -snapshot ID, cannot report achieved quiesce or stopped-source consistency. - -`status.captureManifest` is recorded once before copy authorization or CSI -capture, after lifecycle ownership and quiesce preparation. It lists the primary -`-data` claim as included and every other observed persistent claim as -uncovered, including extra volumes, nested mod/loader mounts and retained -server-owned claims. It records claim UIDs, observed mount paths, source layout -and optional `data`/`rebuildable` declarations. Included describes copy scope; -check the Backup's phase and snapshot ID for copy success. A declaration that -an uncovered claim is rebuildable is not a recovery guarantee. - -The API returns this status and the backup details drawer displays consistency -and coverage. Capture metadata is immutable through ordinary reconciliation: -editing a template later never changes the historical list. This release does -not add multi-volume restic, automatic game stopping or a stricter consistency -request policy. - -Pre-upgrade captures have missing metadata or `state: LegacyUnknown`; their -Consistency condition is Unknown. The operator never reconstructs their history -from the current template. Before recovery, inspect the original restic snapshot -or CSI source claim, recover the original module/server configuration and arrange -separate recovery of extra/mod volumes. Use an isolated recovery server to -verify those prerequisites before replacing a live world. +A successful copy is not necessarily a complete recovery point. Defaults remain +`powerPolicy: preserve` and `volumePolicy: primary-only`: only `-data` +is captured, using the existing quiesce behavior. Explicit opt-in is available on +Backup, BackupSchedule and GameServer's inline backupPolicy through the CRD/API: + +```yaml +spec: + serverRef: {name: my-game} + repoRef: {name: restic-credentials, key: repo} + strategy: restic-snapshot + powerPolicy: stop-copy-restart + volumePolicy: all-data + quiesce: true +``` + +The two policies are independent. CSI (`volume-snapshot`) rejects stopped or +all-data policies; it remains primary-only and offers no cross-volume atomicity. +A preserve/all-data Restic copy captures one snapshot but reports +`Consistency=False/LiveMultiVolumeCopy`, even when primary-game quiesce succeeds. +Only a stopped, fully drained source supports consistency across included roots. +Quiesce still depends on the module's commands and cannot prove application state. +`Completed`/phase and snapshot ID report copy success separately from consistency. + +`stop-copy-restart` acquires shared lifecycle ownership, persists original intent +and process identity, requests the existing graceful stop, and waits for zero +replicas, observed workload generation and all selected-claim users to drain. +It never changes `spec.suspend`. The source's planned stop precedes worker binding; +replacements after that binding invalidate the copy. Existing persisted preparation +and copy deadlines remain in force across controller restart. Failed stop/copy or +cancellation revokes authorization and drains workers before release. A grace +backstop may force Kubernetes shutdown; timeout alone is never proof of drainage. +After cleanup, only current allowed power intent resumes: manual suspension, +idle markers and pending restart/wake requests are preserved. A force-deleted +Backup leaves its stop debt on the GameServer; autonomous reconciliation drains +orphan workers and continues the original stop before releasing ownership. +`RecoveryIntervention=True` retains the fence when provenance is missing or a +workload was replaced. Inspect those identities and resolve the conflicting +workload/worker before recovery; deleting an owner marker does not prove safety. +The condition clears after cleanup releases lifecycle ownership and its guards. + +`all-data` includes primary data, explicitly classified `data` extra volumes and +active separate loader/mod volumes. Every observed or declared required claim +must be classified and have a stable identity. Unknown or retained unmounted +claims reject admission rather than silently reporting a complete backup. +Declare genuinely reconstructible extra volumes `backupRole: rebuildable`; +these are excluded and reported as recovery prerequisites. Optional excluded +claims may be absent on the target; recovery never mounts or overwrites them. +Reconstruction remains an administrator/module responsibility and is not verified +by copy success. Suspend a recovery target until its prerequisites are prepared +if starting the game would otherwise be unsafe. + +A v2 `status.captureManifest` pins template digest/name, selected game version, +claim names/UIDs, logical identities, original paths and exact disjoint +`/volumes/` archive roots. A single Restic invocation receives an +explicit argument per included root, mounted read-only; nested game mounts never +alias one another in the archive. The API and existing coverage rows expose +included/excluded volumes, roots and prerequisites. Captures keep that metadata +across later template changes. + +V2 restore preflights the saved manifest against an existing target before +stopping it or creating a writer. Same-server recovery requires original claim +UIDs; another existing server must match the exact logical layout, template digest +and selected version, and its target claim UIDs are pinned. Missing/replaced +included claims, changed paths, duplicate/traversing roots or missing/downgraded +all-data metadata fail visibly without suspension or destructive work. A prepared +restore Pod mounts only validated included roots; after readiness the operator +revalidates the mapping, claim UIDs and PVC protection before issuing a Pod-bound, +expiring authorization. Retries need their own authorization. Only those explicit +roots receive `--delete`/`--include`; excluded roots are untouched. V2 restore +uses the lifecycle guard to stop without changing user suspend intent. Success +releases the guard and preserves current suspend/idle/restart intent; failed or +cancelled writes leave the target suspended for inspection. + +Restic backup and restore workers run as UID 65532, with the template's +`security.fsGroup` (65532 by default). Source files must be readable by that +identity; restore requires writable existing files and writable/searchable +parent directories on every included claim. A successful backup does not prove +that an in-place restore can overwrite existing files. Configure ownership, +group permissions or ACLs for the worker before recovery. `fsGroup` relies on the +storage driver applying ownership; hostPath-based provisioners may not do so. +For example, a root-owned mode-0644 file remains unwritable to this worker even +when its parent directory is writable. Game containers may use other UIDs when +their storage permissions provide the required access. Permission failures are +reported as failed restores and leave the target suspended; recovery does not +elevate worker privileges or bypass filesystem permissions. + +Legacy primary-only snapshots retain `/data` restore semantics. Missing historical +metadata or `LegacyUnknown` never receives fabricated coverage. All-data archives +cannot downgrade into that legacy path. Inspect original module/server settings +and recover uncovered volumes separately before using an old primary-only copy. ## Shared operation ownership diff --git a/docs/template-rollouts.md b/docs/template-rollouts.md index e7871f0f3..5851a4cf5 100644 --- a/docs/template-rollouts.md +++ b/docs/template-rollouts.md @@ -123,7 +123,11 @@ admitted with the same process. Reverting a runtime does not reverse save-format changes. Restoring data may be necessary, and compatibility is game-specific. Backups remain data backups, not automatic runtime rollback bundles. Record the effective digest, immutable image reference and server configuration with your -backup procedure. Restic restores use the target's current effective layout; CSI +backup procedure. Legacy primary-only Restic restores use the target's current effective layout. +V2 all-data Restic restores pin the capture template digest, selected version and +logical volume paths; an incompatible target fails preflight before any stop or +write. They restore included data/mod roots only and report excluded rebuildable +content as reconstruction prerequisites. CSI restores copy the source server's effective template at restore planning time, not necessarily its template at backup time. Select a compatible Module version and `OnRequest` policy before recovery if automatic adoption would be unsafe. diff --git a/i18n/locales/en/app.json b/i18n/locales/en/app.json index a32980921..10f9a35c1 100644 --- a/i18n/locales/en/app.json +++ b/i18n/locales/en/app.json @@ -211,8 +211,12 @@ "defaultMessage": "Capture-time metadata is unavailable. Do not infer consistency or volume coverage from the current template.", "description": "Backup achieved consistency: LegacyMetadataUnknown." }, + "condition.consistency.livemultivolumecopy": { + "defaultMessage": "Included volumes share one snapshot, but live copies have no atomic cross-volume consistency guarantee, even when the primary game's quiesce succeeds.", + "description": "Backup achieved consistency: LiveMultiVolumeCopy." + }, "condition.consistency.quiesced": { - "defaultMessage": "The primary volume copy completed after the agent confirmed quiesce. This depends on the module's quiesce commands; other volumes remain outside this backup.", + "defaultMessage": "The included volumes were copied after the agent confirmed quiesce. This depends on the module's quiesce commands; consult the manifest for excluded volumes.", "description": "Backup achieved consistency: Quiesced." }, "condition.consistency.rawcopy": { @@ -220,7 +224,7 @@ "description": "Backup achieved consistency: RawCopy." }, "condition.consistency.stoppedsource": { - "defaultMessage": "The primary volume copy completed from a source that was already stopped and drained before capture. Other volumes remain outside this backup.", + "defaultMessage": "The included volumes were copied from a stopped and drained source. Consult the manifest for excluded volumes and recovery prerequisites.", "description": "Backup achieved consistency: StoppedSource." }, "condition.consistency.unsupportedrawcopy": { @@ -619,6 +623,10 @@ "defaultMessage": "Waiting for the module source to index this module.", "description": "Module condition while the source has no catalog entry yet." }, + "condition.recoveryintervention.provenanceunavailable": { + "defaultMessage": "Recovery cannot continue safely because the original operation metadata is unavailable. An administrator must inspect the server and its active workers.", + "description": "Recovery requires intervention when the original operation provenance cannot be established." + }, "condition.retentionstopfailed.sidecar_stop_failed": { "defaultMessage": "Stopping the capture sidecar at the end of retention failed.", "description": "Condition when the retention-expiry stop failed." @@ -1324,8 +1332,8 @@ "description": "Backup details: timestamp of the immutable capture manifest" }, "web.components.backups.backup_detail_drawer.coverage_scope": { - "defaultMessage": "Only the primary data claim is copied. Rebuildable is a declaration, not proof that recovery can reconstruct its contents.", - "description": "Backup details: explains the current single-volume backup scope and rebuildable declarations" + "defaultMessage": "Included claims are copied at the recorded archive roots. Primary-only backups exclude extra data volumes and separate mod claims. Rebuildable volumes are excluded. Check recovery prerequisites before restarting.", + "description": "Backup details: explains recorded volume coverage, primary-only exclusions, and recovery prerequisites for excluded volumes" }, "web.components.backups.backup_detail_drawer.delete": { "defaultMessage": "Delete", diff --git a/i18n/locales/fr/.review.json b/i18n/locales/fr/.review.json index 6c97ce3e0..3fdd422e1 100644 --- a/i18n/locales/fr/.review.json +++ b/i18n/locales/fr/.review.json @@ -43,6 +43,9 @@ "condition.completed.pending": "fcd4ee5f73b3", "condition.completed.running": "6045c2e3f230", "condition.completed.succeeded": "ed54690bd16d", + "condition.consistency.livemultivolumecopy": "8acea8f3b27e", + "condition.consistency.quiesced": "876727991b14", + "condition.consistency.stoppedsource": "f1233d8fcfd7", "condition.datawipe.jobfailed": "f1c100d69216", "condition.failed.capture_already_in_progress": "8c6c3fb2a7bb", "condition.failed.failed": "a2f668070978", @@ -127,6 +130,7 @@ "condition.ready.verifyconfig": "3d84bdf1c513", "condition.ready.versionunavailable": "a4b327ee30da", "condition.ready.waitingforcatalog": "4618394ce7f4", + "condition.recoveryintervention.provenanceunavailable": "8b888cfbc3ae", "condition.retentionstopfailed.sidecar_stop_failed": "1bf2ca9bc80f", "condition.retentiontrimmed.trimfailed": "2fa1a4345931", "condition.retentiontrimmed.trimsucceeded": "c389ba94d579", @@ -283,6 +287,7 @@ "web.components.app_layout.users_rbac": "0c6ed78342e6", "web.components.backups.backup_detail_drawer.backup_details": "85ece27eaabf", "web.components.backups.backup_detail_drawer.completed": "22a970d2e5b1", + "web.components.backups.backup_detail_drawer.coverage_scope": "47601d765ca2", "web.components.backups.backup_detail_drawer.delete": "e2d0a54968ea", "web.components.backups.backup_detail_drawer.deleting": "43b5894c3e4e", "web.components.backups.backup_detail_drawer.phase": "46342ec1eec9", diff --git a/i18n/locales/fr/app.json b/i18n/locales/fr/app.json index ea81b4460..b38361dc5 100644 --- a/i18n/locales/fr/app.json +++ b/i18n/locales/fr/app.json @@ -175,6 +175,18 @@ "defaultMessage": "Le backup ou le restore s'est terminé avec succès.", "description": "Completed condition on a Backup or Restore that finished." }, + "condition.consistency.livemultivolumecopy": { + "defaultMessage": "Les volumes inclus partagent un même instantané, mais leur copie à chaud ne garantit pas une cohérence atomique entre les volumes, même si la mise au repos du jeu principal réussit.", + "description": "Backup achieved consistency: LiveMultiVolumeCopy." + }, + "condition.consistency.quiesced": { + "defaultMessage": "Les volumes inclus ont été copiés après confirmation de la mise au repos par l’agent. Cela dépend des commandes de mise au repos du module ; consultez le manifeste pour connaître les volumes exclus.", + "description": "Backup achieved consistency: Quiesced." + }, + "condition.consistency.stoppedsource": { + "defaultMessage": "Les volumes inclus ont été copiés depuis une source arrêtée, sans processus utilisant ces volumes. Consultez le manifeste pour connaître les volumes exclus et les prérequis de récupération.", + "description": "Backup achieved consistency: StoppedSource." + }, "condition.datawipe.jobfailed": { "defaultMessage": "La tâche d'effacement des données ne s'est pas terminée. Le volume n'est peut-être pas entièrement vidé.", "description": "DataWipe condition when the wipe job fails." @@ -511,6 +523,10 @@ "defaultMessage": "En attente de l'indexation de ce module par la source de modules.", "description": "Module condition while the source has no catalog entry yet." }, + "condition.recoveryintervention.provenanceunavailable": { + "defaultMessage": "La récupération ne peut pas continuer en toute sécurité, car les métadonnées de l’opération d’origine sont indisponibles. Un administrateur doit examiner le serveur et ses tâches actives.", + "description": "Recovery requires intervention when the original operation provenance cannot be established." + }, "condition.retentionstopfailed.sidecar_stop_failed": { "defaultMessage": "L'arrêt du sidecar de capture à la fin de la rétention a échoué.", "description": "Condition when the retention-expiry stop failed." @@ -1135,6 +1151,10 @@ "defaultMessage": "Terminé", "description": "Backup details drawer: field label above the time the backup completed, shown in relative time" }, + "web.components.backups.backup_detail_drawer.coverage_scope": { + "defaultMessage": "Les volumes inclus sont copiés aux racines d’archive enregistrées. Les sauvegardes limitées au volume principal excluent les volumes de données supplémentaires et les volumes de mods distincts. Les volumes reconstructibles sont exclus. Vérifiez les prérequis de récupération avant de redémarrer.", + "description": "Backup details: explains recorded volume coverage, primary-only exclusions, and recovery prerequisites for excluded volumes" + }, "web.components.backups.backup_detail_drawer.delete": { "defaultMessage": "Supprimer", "description": "Backup detail drawer: delete button label" diff --git a/operator/api/v1alpha1/backup_types.go b/operator/api/v1alpha1/backup_types.go index c2ec70c00..bddbb5088 100644 --- a/operator/api/v1alpha1/backup_types.go +++ b/operator/api/v1alpha1/backup_types.go @@ -35,7 +35,17 @@ const BackupSnapshotFinalizer = "gameplane.local/backup-snapshot-finalizer" // BackupSpec is the desired state of a one-shot backup. // +kubebuilder:validation:XValidation:rule="self.strategy == 'volume-snapshot' || has(self.repoRef)",message="repoRef is required for the restic-snapshot strategy" +// +kubebuilder:validation:XValidation:rule="self.strategy != 'volume-snapshot' || ((!has(self.powerPolicy) || self.powerPolicy == 'preserve') && (!has(self.volumePolicy) || self.volumePolicy == 'primary-only'))",message="stopped and all-data policies require restic-snapshot" type BackupSpec struct { + // PowerPolicy preserves power by default; stop-copy-restart temporarily stops under lifecycle ownership. + // +kubebuilder:validation:Enum=preserve;stop-copy-restart + // +optional + PowerPolicy string `json:"powerPolicy,omitempty"` + // VolumePolicy captures primary-only by default; all-data includes classified persistent data. + // +kubebuilder:validation:Enum=primary-only;all-data + // +optional + VolumePolicy string `json:"volumePolicy,omitempty"` + // ServerRef is the GameServer whose data volume should be backed up. ServerRef LocalObjectRef `json:"serverRef"` @@ -137,6 +147,18 @@ type BackupStatus struct { // BackupCaptureManifest pins the layout and quiesce result at capture admission. // Recorded coverage does not imply a successful or complete server backup. type BackupCaptureManifest struct { + // FormatVersion 2 uses validated disjoint roots. Absent means legacy /data. + // +optional + FormatVersion int32 `json:"formatVersion,omitempty"` + // +optional + VolumePolicy string `json:"volumePolicy,omitempty"` + // +optional + TemplateName string `json:"templateName,omitempty"` + // +optional + TemplateRevision string `json:"templateRevision,omitempty"` + // +optional + GameVersion string `json:"gameVersion,omitempty"` + // State is Recorded for new captures, LegacyUnknown for an existing worker // whose original layout was never recorded. // +kubebuilder:validation:Enum=Recorded;LegacyUnknown @@ -161,6 +183,16 @@ type BackupCaptureManifest struct { // BackupVolumeCoverage describes one persistent claim. Extra and loader/mod // claims remain uncovered even when mounted beneath the primary data directory. type BackupVolumeCoverage struct { + // LogicalName identifies the primary, declared extra, or active loader volume. + // +optional + LogicalName string `json:"logicalName,omitempty"` + // SnapshotRoot is the explicit disjoint archive root; never a glob. + // +optional + SnapshotRoot string `json:"snapshotRoot,omitempty"` + // RecoveryPrerequisite explains excluded content that must be reconstructed. + // +optional + RecoveryPrerequisite string `json:"recoveryPrerequisite,omitempty"` + ClaimName string `json:"claimName"` // ClaimUID is empty if the claim did not yet exist at admission. // +optional diff --git a/operator/api/v1alpha1/backupschedule_types.go b/operator/api/v1alpha1/backupschedule_types.go index 617ed14b8..662de66c9 100644 --- a/operator/api/v1alpha1/backupschedule_types.go +++ b/operator/api/v1alpha1/backupschedule_types.go @@ -7,7 +7,17 @@ import ( // BackupScheduleSpec defines a recurring backup policy for a GameServer. // +kubebuilder:validation:XValidation:rule="self.strategy == 'volume-snapshot' || has(self.repoRef)",message="repoRef is required for the restic-snapshot strategy" +// +kubebuilder:validation:XValidation:rule="self.strategy != 'volume-snapshot' || ((!has(self.powerPolicy) || self.powerPolicy == 'preserve') && (!has(self.volumePolicy) || self.volumePolicy == 'primary-only'))",message="stopped and all-data policies require restic-snapshot" type BackupScheduleSpec struct { + // PowerPolicy preserves power by default; stop-copy-restart temporarily stops under lifecycle ownership. + // +kubebuilder:validation:Enum=preserve;stop-copy-restart + // +optional + PowerPolicy string `json:"powerPolicy,omitempty"` + // VolumePolicy captures primary-only by default; all-data includes classified persistent data. + // +kubebuilder:validation:Enum=primary-only;all-data + // +optional + VolumePolicy string `json:"volumePolicy,omitempty"` + // ServerRef is the GameServer to back up on this schedule. ServerRef LocalObjectRef `json:"serverRef"` diff --git a/operator/api/v1alpha1/gameserver_types.go b/operator/api/v1alpha1/gameserver_types.go index e6377f3cb..a8683597b 100644 --- a/operator/api/v1alpha1/gameserver_types.go +++ b/operator/api/v1alpha1/gameserver_types.go @@ -499,6 +499,13 @@ type PlayitTunnelSpec struct { // configure when enabling backups directly from GameServer. The // operator materializes this into a managed BackupSchedule. type InlineBackupPolicy struct { + // +kubebuilder:validation:Enum=preserve;stop-copy-restart + // +optional + PowerPolicy string `json:"powerPolicy,omitempty"` + // +kubebuilder:validation:Enum=primary-only;all-data + // +optional + VolumePolicy string `json:"volumePolicy,omitempty"` + // Schedule is a standard cron expression; same structural guard as // BackupScheduleSpec.Schedule so typos are rejected at admission // instead of failing inside the reconcile loop of the materialized diff --git a/operator/api/v1alpha1/gametemplate_types.go b/operator/api/v1alpha1/gametemplate_types.go index 503130e14..8c87a75c0 100644 --- a/operator/api/v1alpha1/gametemplate_types.go +++ b/operator/api/v1alpha1/gametemplate_types.go @@ -974,10 +974,10 @@ type GameStorageSpec struct { // primary Storage.MountPath, since these exist precisely for directories // that share no safe common parent with it. type ExtraVolumeSpec struct { - // BackupRole describes recovery requirements without adding this volume - // to a backup. "data" needs a separate copy; "rebuildable" may be recreated. - // Unset remains unclassified and not covered. Both backup strategies still - // capture only the primary data claim. + // BackupRole declares recovery scope. all-data Restic backups include data + // extras and report rebuildable extras as excluded reconstruction prerequisites. + // Unset is unclassified and rejects all-data admission. Default primary-only + // backups and CSI snapshots still capture only the primary data claim. // +kubebuilder:validation:Enum=data;rebuildable // +optional BackupRole string `json:"backupRole,omitempty"` diff --git a/operator/api/v1alpha1/restore_types.go b/operator/api/v1alpha1/restore_types.go index a54e56a4a..3a42b1542 100644 --- a/operator/api/v1alpha1/restore_types.go +++ b/operator/api/v1alpha1/restore_types.go @@ -37,8 +37,27 @@ type RestoreSpec struct { ServerRef LocalObjectRef `json:"serverRef"` } +// RestoreRecoveryPlan pins a validated target mapping before any power mutation. +type RestoreRecoveryPlan struct { + // Admitted power intent is informational; cleanup never overwrites newer user intent. + AdmittedSuspend bool `json:"admittedSuspend"` + // +optional + AdmittedIdle string `json:"admittedIdle,omitempty"` + AdmittedGeneration int64 `json:"admittedGeneration"` + + BackupUID string `json:"backupUID"` + ManifestDigest string `json:"manifestDigest"` + TargetUID string `json:"targetUID"` + TemplateRevision string `json:"templateRevision"` + GameVersion string `json:"gameVersion"` + Volumes []BackupVolumeCoverage `json:"volumes"` +} + // RestoreStatus is the observed state of a Restore. type RestoreStatus struct { + // +optional + RecoveryPlan *RestoreRecoveryPlan `json:"recoveryPlan,omitempty"` + // +optional Phase RestorePhase `json:"phase,omitempty"` diff --git a/operator/api/v1alpha1/zz_generated.deepcopy.go b/operator/api/v1alpha1/zz_generated.deepcopy.go index 89c9b1129..9b349cc91 100644 --- a/operator/api/v1alpha1/zz_generated.deepcopy.go +++ b/operator/api/v1alpha1/zz_generated.deepcopy.go @@ -2495,6 +2495,28 @@ func (in *RestoreList) DeepCopyObject() runtime.Object { return nil } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *RestoreRecoveryPlan) DeepCopyInto(out *RestoreRecoveryPlan) { + *out = *in + if in.Volumes != nil { + in, out := &in.Volumes, &out.Volumes + *out = make([]BackupVolumeCoverage, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RestoreRecoveryPlan. +func (in *RestoreRecoveryPlan) DeepCopy() *RestoreRecoveryPlan { + if in == nil { + return nil + } + out := new(RestoreRecoveryPlan) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *RestoreSpec) DeepCopyInto(out *RestoreSpec) { *out = *in @@ -2515,6 +2537,11 @@ func (in *RestoreSpec) DeepCopy() *RestoreSpec { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *RestoreStatus) DeepCopyInto(out *RestoreStatus) { *out = *in + if in.RecoveryPlan != nil { + in, out := &in.RecoveryPlan, &out.RecoveryPlan + *out = new(RestoreRecoveryPlan) + (*in).DeepCopyInto(*out) + } if in.StartTime != nil { in, out := &in.StartTime, &out.StartTime *out = (*in).DeepCopy() diff --git a/operator/config/crd/gameplane.local_backups.yaml b/operator/config/crd/gameplane.local_backups.yaml index 1a46399b8..67fb20227 100644 --- a/operator/config/crd/gameplane.local_backups.yaml +++ b/operator/config/crd/gameplane.local_backups.yaml @@ -57,6 +57,13 @@ spec: spec: description: BackupSpec is the desired state of a one-shot backup. properties: + powerPolicy: + description: PowerPolicy preserves power by default; stop-copy-restart + temporarily stops under lifecycle ownership. + enum: + - preserve + - stop-copy-restart + type: string quiesce: default: true description: |- @@ -112,6 +119,13 @@ spec: items: type: string type: array + volumePolicy: + description: VolumePolicy captures primary-only by default; all-data + includes classified persistent data. + enum: + - primary-only + - all-data + type: string volumeSnapshotClassName: description: |- VolumeSnapshotClassName names the VolumeSnapshotClass to use when @@ -125,6 +139,10 @@ spec: x-kubernetes-validations: - message: repoRef is required for the restic-snapshot strategy rule: self.strategy == 'volume-snapshot' || has(self.repoRef) + - message: stopped and all-data policies require restic-snapshot + rule: self.strategy != 'volume-snapshot' || ((!has(self.powerPolicy) + || self.powerPolicy == 'preserve') && (!has(self.volumePolicy) || + self.volumePolicy == 'primary-only')) status: description: BackupStatus is the observed state of a Backup. properties: @@ -134,6 +152,13 @@ spec: It describes persistent-volume coverage independently of copy success. Missing on pre-upgrade backups; never reconstruct it from current layout. properties: + formatVersion: + description: FormatVersion 2 uses validated disjoint roots. Absent + means legacy /data. + format: int32 + type: integer + gameVersion: + type: string layoutSource: description: LayoutSource identifies the frozen workload used for mount paths. @@ -158,6 +183,12 @@ spec: type: string strategy: type: string + templateName: + type: string + templateRevision: + type: string + volumePolicy: + type: string volumes: items: description: |- @@ -181,12 +212,24 @@ spec: type: string included: type: boolean + logicalName: + description: LogicalName identifies the primary, declared + extra, or active loader volume. + type: string mountPaths: items: type: string type: array reason: type: string + recoveryPrerequisite: + description: RecoveryPrerequisite explains excluded content + that must be reconstructed. + type: string + snapshotRoot: + description: SnapshotRoot is the explicit disjoint archive + root; never a glob. + type: string required: - claimName - classification diff --git a/operator/config/crd/gameplane.local_backupschedules.yaml b/operator/config/crd/gameplane.local_backupschedules.yaml index 2011ca4cd..b0e945b84 100644 --- a/operator/config/crd/gameplane.local_backupschedules.yaml +++ b/operator/config/crd/gameplane.local_backupschedules.yaml @@ -71,6 +71,13 @@ spec: - Forbid - Replace type: string + powerPolicy: + description: PowerPolicy preserves power by default; stop-copy-restart + temporarily stops under lifecycle ownership. + enum: + - preserve + - stop-copy-restart + type: string quiesce: default: true description: |- @@ -162,6 +169,13 @@ spec: default: false description: Suspend pauses the schedule without deleting it. type: boolean + volumePolicy: + description: VolumePolicy captures primary-only by default; all-data + includes classified persistent data. + enum: + - primary-only + - all-data + type: string required: - schedule - serverRef @@ -169,6 +183,10 @@ spec: x-kubernetes-validations: - message: repoRef is required for the restic-snapshot strategy rule: self.strategy == 'volume-snapshot' || has(self.repoRef) + - message: stopped and all-data policies require restic-snapshot + rule: self.strategy != 'volume-snapshot' || ((!has(self.powerPolicy) + || self.powerPolicy == 'preserve') && (!has(self.volumePolicy) || + self.volumePolicy == 'primary-only')) status: description: BackupScheduleStatus is the observed state of a BackupSchedule. properties: diff --git a/operator/config/crd/gameplane.local_gameservers.yaml b/operator/config/crd/gameplane.local_gameservers.yaml index 6598e07ff..7bc7f5eeb 100644 --- a/operator/config/crd/gameplane.local_gameservers.yaml +++ b/operator/config/crd/gameplane.local_gameservers.yaml @@ -977,6 +977,11 @@ spec: the operator creates/maintains a BackupSchedule owned by this GameServer. Remove to stop scheduled backups. properties: + powerPolicy: + enum: + - preserve + - stop-copy-restart + type: string repoRef: description: |- RepoRef points at a BackupRepo (cluster-scoped resource, TBD) or @@ -1027,6 +1032,11 @@ spec: type: string suspend: type: boolean + volumePolicy: + enum: + - primary-only + - all-data + type: string required: - repoRef - schedule @@ -2221,10 +2231,10 @@ spec: properties: backupRole: description: |- - BackupRole describes recovery requirements without adding this volume - to a backup. "data" needs a separate copy; "rebuildable" may be recreated. - Unset remains unclassified and not covered. Both backup strategies still - capture only the primary data claim. + BackupRole declares recovery scope. all-data Restic backups include data + extras and report rebuildable extras as excluded reconstruction prerequisites. + Unset is unclassified and rejects all-data admission. Default primary-only + backups and CSI snapshots still capture only the primary data claim. enum: - data - rebuildable diff --git a/operator/config/crd/gameplane.local_gametemplates.yaml b/operator/config/crd/gameplane.local_gametemplates.yaml index c588d074d..cf08d43d1 100644 --- a/operator/config/crd/gameplane.local_gametemplates.yaml +++ b/operator/config/crd/gameplane.local_gametemplates.yaml @@ -2208,10 +2208,10 @@ spec: properties: backupRole: description: |- - BackupRole describes recovery requirements without adding this volume - to a backup. "data" needs a separate copy; "rebuildable" may be recreated. - Unset remains unclassified and not covered. Both backup strategies still - capture only the primary data claim. + BackupRole declares recovery scope. all-data Restic backups include data + extras and report rebuildable extras as excluded reconstruction prerequisites. + Unset is unclassified and rejects all-data admission. Default primary-only + backups and CSI snapshots still capture only the primary data claim. enum: - data - rebuildable diff --git a/operator/config/crd/gameplane.local_restores.yaml b/operator/config/crd/gameplane.local_restores.yaml index 4984f6817..f7d1af468 100644 --- a/operator/config/crd/gameplane.local_restores.yaml +++ b/operator/config/crd/gameplane.local_restores.yaml @@ -170,6 +170,87 @@ spec: - Succeeded - Failed type: string + recoveryPlan: + description: RestoreRecoveryPlan pins a validated target mapping before + any power mutation. + properties: + admittedGeneration: + format: int64 + type: integer + admittedIdle: + type: string + admittedSuspend: + description: Admitted power intent is informational; cleanup never + overwrites newer user intent. + type: boolean + backupUID: + type: string + gameVersion: + type: string + manifestDigest: + type: string + targetUID: + type: string + templateRevision: + type: string + volumes: + items: + description: |- + BackupVolumeCoverage describes one persistent claim. Extra and loader/mod + claims remain uncovered even when mounted beneath the primary data directory. + properties: + claimName: + type: string + claimUID: + description: ClaimUID is empty if the claim did not yet + exist at admission. + type: string + classification: + description: |- + Classification records the module/server declaration at admission; + it does not prove that the volume can actually be reconstructed. + enum: + - data + - rebuildable + - unclassified + type: string + included: + type: boolean + logicalName: + description: LogicalName identifies the primary, declared + extra, or active loader volume. + type: string + mountPaths: + items: + type: string + type: array + reason: + type: string + recoveryPrerequisite: + description: RecoveryPrerequisite explains excluded content + that must be reconstructed. + type: string + snapshotRoot: + description: SnapshotRoot is the explicit disjoint archive + root; never a glob. + type: string + required: + - claimName + - classification + - included + - reason + type: object + type: array + required: + - admittedGeneration + - admittedSuspend + - backupUID + - gameVersion + - manifestDigest + - targetUID + - templateRevision + - volumes + type: object snapshotID: description: |- SnapshotID is the restic snapshot used. Resolved from the diff --git a/operator/hack/condition-ids/main.go b/operator/hack/condition-ids/main.go index ab8e3e5be..b22008df3 100644 --- a/operator/hack/condition-ids/main.go +++ b/operator/hack/condition-ids/main.go @@ -177,12 +177,15 @@ var rows = []row{ {types: []string{"SnapshotForgotten"}, reason: "Forgotten", english: "The snapshot was removed from the backup repository.", description: "SnapshotForgotten condition after a deleted Backup's snapshot was forgotten."}, {types: []string{"SnapshotForgotten"}, reason: "ForgetAbandoned", english: "The snapshot was not removed from the backup repository when the backup was deleted.", description: "SnapshotForgotten condition when the forget step was abandoned."}, + {types: []string{"RecoveryIntervention"}, reason: "ProvenanceUnavailable", english: "Recovery cannot continue safely because the original operation metadata is unavailable. An administrator must inspect the server and its active workers.", description: "Recovery requires intervention when the original operation provenance cannot be established."}, + {types: []string{"Consistency"}, reason: "LiveMultiVolumeCopy", english: "Included volumes share one snapshot, but live copies have no atomic cross-volume consistency guarantee, even when the primary game's quiesce succeeds.", description: "Backup achieved consistency: LiveMultiVolumeCopy."}, + // Consistency is separate from copy completion and volume coverage. {types: []string{"Consistency"}, reason: "CaptureFailed", english: "The capture failed; no usable consistency guarantee was achieved.", description: "Backup achieved consistency: CaptureFailed."}, {types: []string{"Consistency"}, reason: "LegacyMetadataUnknown", english: "Capture-time metadata is unavailable. Do not infer consistency or volume coverage from the current template.", description: "Backup achieved consistency: LegacyMetadataUnknown."}, {types: []string{"Consistency"}, reason: "CapturePending", english: "The capture has not completed; its achieved consistency is not yet known.", description: "Backup achieved consistency: CapturePending."}, - {types: []string{"Consistency"}, reason: "Quiesced", english: "The primary volume copy completed after the agent confirmed quiesce. This depends on the module's quiesce commands; other volumes remain outside this backup.", description: "Backup achieved consistency: Quiesced."}, - {types: []string{"Consistency"}, reason: "StoppedSource", english: "The primary volume copy completed from a source that was already stopped and drained before capture. Other volumes remain outside this backup.", description: "Backup achieved consistency: StoppedSource."}, + {types: []string{"Consistency"}, reason: "Quiesced", english: "The included volumes were copied after the agent confirmed quiesce. This depends on the module's quiesce commands; consult the manifest for excluded volumes.", description: "Backup achieved consistency: Quiesced."}, + {types: []string{"Consistency"}, reason: "StoppedSource", english: "The included volumes were copied from a stopped and drained source. Consult the manifest for excluded volumes and recovery prerequisites.", description: "Backup achieved consistency: StoppedSource."}, {types: []string{"Consistency"}, reason: "CrashConsistentSnapshot", english: "The primary CSI snapshot is crash-consistent; application quiesce was not confirmed.", description: "Backup achieved consistency: CrashConsistentSnapshot."}, {types: []string{"Consistency"}, reason: "UnsupportedRawCopy", english: "The game does not support quiesce. The primary volume was copied while it could still change.", description: "Backup achieved consistency: UnsupportedRawCopy."}, {types: []string{"Consistency"}, reason: "AgentUnavailableRawCopy", english: "No agent quiesce capability was configured. The primary volume was copied without pausing game writes.", description: "Backup achieved consistency: AgentUnavailableRawCopy."}, diff --git a/operator/internal/controller/backup_controller.go b/operator/internal/controller/backup_controller.go index 0dcb5217a..694429e53 100644 --- a/operator/internal/controller/backup_controller.go +++ b/operator/internal/controller/backup_controller.go @@ -310,6 +310,9 @@ func (r *BackupReconciler) reconcile(ctx context.Context, req ctrl.Request) (ctr } return r.runUnquiesce(ctx, &b) } + if b.Spec.Strategy == "volume-snapshot" && (b.Spec.PowerPolicy == "stop-copy-restart" || b.Spec.VolumePolicy == "all-data") { + return r.fail(ctx, &b, "stopped and all-data policies require restic-snapshot") + } // Observe pre-upgrade workers even when multiple legacy quiesced copies // already overlap. They must finish and unquiesce before new admission; // trying to acquire exclusive ownership first would deadlock their cleanup. @@ -1033,7 +1036,7 @@ func (r *BackupReconciler) buildBackupPodSpec(b *gameplanev1alpha1.Backup, tmpl args = append(args, "--tag", t) } - return corev1.PodSpec{ + spec := corev1.PodSpec{ SecurityContext: buildBackupPodSecurityContext(tmpl), InitContainers: []corev1.Container{{ Name: "restic-init", @@ -1077,6 +1080,17 @@ func (r *BackupReconciler) buildBackupPodSpec(b *gameplanev1alpha1.Backup, tmpl }, }, } + if b.Spec.VolumePolicy == "all-data" && b.Status.CaptureManifest != nil { + recoveryWorkerVolumes(&spec, b.Status.CaptureManifest.Volumes, true) + roots := []string{"backup"} + for _, volume := range b.Status.CaptureManifest.Volumes { + if volume.Included { + roots = append(roots, volume.SnapshotRoot) + } + } + spec.Containers[0].Args = append(roots, args[2:]...) + } + return spec } // resticEnv is the container environment shared by every Job that talks to a diff --git a/operator/internal/controller/backup_guard.go b/operator/internal/controller/backup_guard.go index 77832d77c..b28da3f3a 100644 --- a/operator/internal/controller/backup_guard.go +++ b/operator/internal/controller/backup_guard.go @@ -297,6 +297,18 @@ func (r *BackupReconciler) prepareBackupWorkload(ctx context.Context, b *gamepla return ctrl.Result{RequeueAfter: 5 * time.Second}, err } ready := false + if fenced && b.Spec.PowerPolicy == "stop-copy-restart" { + stopped, stopErr := r.prepareStoppedBackup(ctx, b, gs) + if stopErr != nil { + if errors.Is(stopErr, errBackupSourceChanged) { + return r.abortResticCopy(ctx, b, stopErr.Error()) + } + return ctrl.Result{}, stopErr + } + if !stopped { + return ctrl.Result{RequeueAfter: 2 * time.Second}, nil + } + } if fenced { ready, err = r.prepareBackupSource(ctx, b, gs) } else { @@ -384,6 +396,29 @@ func (r *BackupReconciler) releaseBackupTarget(ctx context.Context, b *gameplane if err != nil || gs == nil { return err } + op, _, err := lifecycleOwner(gs) + if err != nil { + return err + } + if op.StoppedBackup != nil { + ready, err := stoppedBackupCleanup(ctx, r.Client, r.backupAPIReader(), gs, op.StoppedBackup) + if err != nil { + _ = recoveryIntervention(ctx, r.Client, gs, err.Error()) + return err + } + if !ready { + return fmt.Errorf("waiting for stopped source and workers to drain before recovery release") + } + delete(gs.Annotations, stopRequestedAtAnnotation) + reason, message := "UserIntentPreserved", "Cleanup completed; current suspend, idle and restart intent is preserved." + if stoppedBackupMayResume(gs, op.StoppedBackup) { + reason, message = "RestartPermitted", "Workers and original source drained; the admitted running intent permits normal startup." + } + b.Status.Conditions = upsertCondition(b.Status.Conditions, metav1.Condition{Type: "Recovery", Status: metav1.ConditionTrue, Reason: reason, Message: message, ObservedGeneration: b.Generation}) + if err := r.Status().Update(ctx, b); err != nil { + return err + } + } if err := clearLifecycleOperation(ctx, r.Client, r.backupAPIReader(), gs, backupLifecycleIdentity(gs, b, false)); err != nil { return err } diff --git a/operator/internal/controller/backup_manifest.go b/operator/internal/controller/backup_manifest.go index 2ba50f7e6..102faae64 100644 --- a/operator/internal/controller/backup_manifest.go +++ b/operator/internal/controller/backup_manifest.go @@ -39,6 +39,18 @@ func (r *BackupReconciler) recordBackupManifest(ctx context.Context, b *gameplan return nil } reader := r.backupAPIReader() + if b.Spec.VolumePolicy == "all-data" { + manifest, err := recoveryLayout(ctx, reader, gs) + if err != nil { + return err + } + now := metav1.Now() + manifest.RecordedAt = &now + manifest.QuiesceOutcome = backupQuiesceOutcome(b) + b.Status.CaptureManifest = manifest + return r.Status().Update(ctx, b) + } + worker, err := r.captureWorker(ctx, b) if err != nil { return err @@ -184,10 +196,12 @@ func backupConsistencyCondition(b *gameplanev1alpha1.Backup) metav1.Condition { condition.Reason, condition.Message = "LegacyMetadataUnknown", "Capture-time metadata is unavailable. Do not infer consistency or volume coverage from the current template." case b.Status.Phase != gameplanev1alpha1.BackupPhaseSucceeded || b.Status.SnapshotID == "": condition.Reason, condition.Message = "CapturePending", "The capture has not completed; its achieved consistency is not yet known." + case b.Status.CaptureManifest.VolumePolicy == "all-data" && b.Status.CaptureManifest.QuiesceOutcome != "already-stopped": + condition.Status, condition.Reason, condition.Message = metav1.ConditionFalse, "LiveMultiVolumeCopy", "Included volumes share one snapshot, but live copies have no atomic cross-volume consistency guarantee, even when the primary game's quiesce succeeds." case b.Status.CaptureManifest.QuiesceOutcome == "confirmed": - condition.Status, condition.Reason, condition.Message = metav1.ConditionTrue, "Quiesced", "The primary volume copy completed after the agent confirmed quiesce. This depends on the module's quiesce commands; other volumes remain outside this backup." + condition.Status, condition.Reason, condition.Message = metav1.ConditionTrue, "Quiesced", "The included volumes were copied after the agent confirmed quiesce. This depends on the module's quiesce commands; consult the manifest for excluded volumes." case b.Status.CaptureManifest.QuiesceOutcome == "already-stopped": - condition.Status, condition.Reason, condition.Message = metav1.ConditionTrue, "StoppedSource", "The primary volume copy completed from a source that was already stopped and drained before capture. Other volumes remain outside this backup." + condition.Status, condition.Reason, condition.Message = metav1.ConditionTrue, "StoppedSource", "The included volumes were copied from a stopped and drained source. Consult the manifest for excluded volumes and recovery prerequisites." case b.Status.CaptureManifest.Strategy == "volume-snapshot": condition.Status, condition.Reason, condition.Message = metav1.ConditionFalse, "CrashConsistentSnapshot", "The primary CSI snapshot is crash-consistent; application quiesce was not confirmed ("+b.Status.CaptureManifest.QuiesceOutcome+")." case b.Status.CaptureManifest.QuiesceOutcome == "unsupported": @@ -208,3 +222,18 @@ func (r *BackupReconciler) syncBackupConsistency(ctx context.Context, b *gamepla b.Status.Conditions = conditions return r.Status().Update(ctx, b) } + +// The all-data layout must precede worker creation, while the quiesce RPC +// deliberately follows worker readiness. Finalize only its outcome before the +// first authorization; the claim selection and provenance never change. +func (r *BackupReconciler) finalizeBackupCaptureOutcome(ctx context.Context, b *gameplanev1alpha1.Backup) error { + if b.Spec.VolumePolicy != "all-data" || b.Status.CaptureManifest == nil || b.Annotations[backupAuthorizedWorkerAnnotation] != "" { + return nil + } + outcome := backupQuiesceOutcome(b) + if b.Status.CaptureManifest.QuiesceOutcome == outcome { + return nil + } + b.Status.CaptureManifest.QuiesceOutcome = outcome + return r.Status().Update(ctx, b) +} diff --git a/operator/internal/controller/backup_preparation.go b/operator/internal/controller/backup_preparation.go index fa1d4a08d..1993d0500 100644 --- a/operator/internal/controller/backup_preparation.go +++ b/operator/internal/controller/backup_preparation.go @@ -120,7 +120,7 @@ func parseBackupPreparation(b *gameplanev1alpha1.Backup) (*backupPreparation, er // Call only after the shared owner and StatefulSet fence are held. Readiness // and desired suspend/idle state are not evidence that writers have exited. func (r *BackupReconciler) prepareBackupSource(ctx context.Context, b *gameplanev1alpha1.Backup, gs *gameplanev1alpha1.GameServer) (bool, error) { - if !b.Spec.Quiesce { + if !b.Spec.Quiesce && b.Spec.PowerPolicy != "stop-copy-restart" { return true, nil } p, err := parseBackupPreparation(b) @@ -136,7 +136,7 @@ func (r *BackupReconciler) prepareBackupSource(ctx context.Context, b *gameplane if p == nil && b.Annotations[annoQuiesceAttempted] != "" && b.Annotations[annoQuiesceAttempted] != "unsupported" { return false, fmt.Errorf("%w: quiesce attempt lacks source preparation", errBackupSourceChanged) } - if p == nil && !agentQuiesceAvailable(r.AgentClient) { + if p == nil && !agentQuiesceAvailable(r.AgentClient) && b.Spec.PowerPolicy != "stop-copy-restart" { return true, nil } // An unsupported attempt did not pause the game and retains raw-copy @@ -242,6 +242,18 @@ func (r *BackupReconciler) backupSourceDrained(ctx context.Context, b *gameplane if err := r.backupAPIReader().List(ctx, &jobs, client.InNamespace(gs.Namespace)); err != nil { return false, err } + var selected []gameplanev1alpha1.BackupVolumeCoverage + if b.Status.CaptureManifest != nil { + selected = b.Status.CaptureManifest.Volumes + } + if b.Status.CaptureManifest == nil { + if op, _, err := lifecycleOwner(gs); err == nil && op.Kind == "Backup" && op.UID == b.UID && op.StoppedBackup != nil { + selected = op.StoppedBackup.Claims + } + } + mounts := func(pod *corev1.Pod) bool { + return podMountsRestoreTarget(pod, gs.Name) || podMountsRecoveryClaims(pod, selected) + } copyJobs := map[types.UID]bool{} for i := range jobs.Items { job := &jobs.Items[i] @@ -249,7 +261,7 @@ func (r *BackupReconciler) backupSourceDrained(ctx context.Context, b *gameplane copyJobs[job.UID] = true continue } - if ((job.Status.Succeeded == 0 && !jobPermanentlyFailed(job)) || job.Status.Active > 0 || !job.DeletionTimestamp.IsZero()) && podMountsRestoreTarget(&corev1.Pod{Spec: job.Spec.Template.Spec}, gs.Name) { + if ((job.Status.Succeeded == 0 && !jobPermanentlyFailed(job)) || job.Status.Active > 0 || !job.DeletionTimestamp.IsZero()) && mounts(&corev1.Pod{Spec: job.Spec.Template.Spec}) { return false, nil } } @@ -266,7 +278,7 @@ func (r *BackupReconciler) backupSourceDrained(ctx context.Context, b *gameplane if owner != nil && owner.Kind == "Job" && copyJobs[owner.UID] { continue } - if metav1.IsControlledBy(pod, ss) || metav1.IsControlledBy(pod, gs) || pod.Name == gs.Name+"-0" || podMountsRestoreTarget(pod, gs.Name) { + if metav1.IsControlledBy(pod, ss) || metav1.IsControlledBy(pod, gs) || pod.Name == gs.Name+"-0" || mounts(pod) { return false, nil } } diff --git a/operator/internal/controller/backup_stop.go b/operator/internal/controller/backup_stop.go new file mode 100644 index 000000000..7f10f04ac --- /dev/null +++ b/operator/internal/controller/backup_stop.go @@ -0,0 +1,219 @@ +package controller + +import ( + "context" + "fmt" + "time" + + v1 "github.com/GameplanePanel/gameplane/operator/api/v1alpha1" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// This record lives on the arbitration object, including during orphan transfer. +// Admitted is persisted atomically with ownership; Stopping precedes any stop RPC. +type stoppedBackup struct { + Stage string `json:"stage"` + WasRunning bool `json:"wasRunning"` + Generation int64 `json:"generation"` + Suspend bool `json:"suspend"` + Idle string `json:"idle,omitempty"` + Restart string `json:"restart,omitempty"` + Wake string `json:"wake,omitempty"` + WorkloadUID types.UID `json:"workloadUID,omitempty"` + OriginalPodUID types.UID `json:"originalPodUID,omitempty"` + Deadline string `json:"deadline,omitempty"` + Claims []v1.BackupVolumeCoverage `json:"claims,omitempty"` +} + +func newStoppedBackup(gs *v1.GameServer) *stoppedBackup { + return &stoppedBackup{Stage: "Admitted", WasRunning: !gs.Spec.Suspend && gs.Annotations[IdleAsleepSinceAnnotation] == "", Generation: gs.Generation, Suspend: gs.Spec.Suspend, Idle: gs.Annotations[IdleAsleepSinceAnnotation], Restart: gs.Annotations[RestartRequestedAnnotation], Wake: gs.Annotations[IdleWakeRequestedAnnotation]} +} +func stoppedBackupMayResume(gs *v1.GameServer, s *stoppedBackup) bool { + return s != nil && s.WasRunning && !gs.Spec.Suspend && gs.Annotations[IdleAsleepSinceAnnotation] == "" && gs.Generation == s.Generation && gs.Annotations[RestartRequestedAnnotation] == s.Restart && gs.Annotations[IdleWakeRequestedAnnotation] == s.Wake +} + +func (r *BackupReconciler) prepareStoppedBackup(ctx context.Context, b *v1.Backup, gs *v1.GameServer) (bool, error) { + op, held, err := lifecycleOwner(gs) + if err != nil { + return false, err + } + if !held || op.StoppedBackup == nil { + return false, fmt.Errorf("stopped backup lacks durable owner provenance") + } + s := op.StoppedBackup + var ss appsv1.StatefulSet + if err := r.backupAPIReader().Get(ctx, client.ObjectKeyFromObject(gs), &ss); err != nil { + return false, err + } + if !metav1.IsControlledBy(&ss, gs) { + return false, fmt.Errorf("%w: stopped workload owner", errBackupSourceChanged) + } + if s.Stage == "Admitted" { + // Select coverage before the first stop; failures cannot interrupt a running game. + if b.Spec.VolumePolicy == "all-data" { + m, err := recoveryLayout(ctx, r.backupAPIReader(), gs) + if err != nil { + return false, fmt.Errorf("%w: %w", errBackupSourceChanged, err) + } + s.Claims = m.Volumes + } else { + s.Claims = []v1.BackupVolumeCoverage{{ClaimName: gs.Name + "-data", Included: true}} + } + var pod corev1.Pod + err := r.backupAPIReader().Get(ctx, client.ObjectKey{Namespace: gs.Namespace, Name: gs.Name + "-0"}, &pod) + if err != nil && !apierrors.IsNotFound(err) { + return false, err + } + if err == nil { + if !metav1.IsControlledBy(&pod, &ss) { + return false, fmt.Errorf("%w: original pod owner", errBackupSourceChanged) + } + s.OriginalPodUID = pod.UID + } + s.WorkloadUID = ss.UID + s.Deadline = b.Annotations[backupPreparationDeadlineAnnotation] + s.Stage = "Stopping" + gs.Annotations[lifecycleOperationAnnotation] = lifecycleOperationJSON(op) + return false, r.Update(ctx, gs) + } + if s.WorkloadUID != ss.UID { + return false, fmt.Errorf("%w: stopped workload replaced", errBackupSourceChanged) + } + if ss.Spec.Replicas == nil || *ss.Spec.Replicas != 0 || ss.Status.Replicas != 0 || ss.Status.ObservedGeneration < ss.Generation { + return false, nil + } + drained, err := r.backupSourceDrained(ctx, b, gs, &ss) + if err != nil || !drained { + return false, err + } + if s.Stage != "Drained" { + s.Stage = "Drained" + gs.Annotations[lifecycleOperationAnnotation] = lifecycleOperationJSON(op) + if err := r.Update(ctx, gs); err != nil { + return false, err + } + } + return true, nil +} + +// Cleanup never changes spec or acknowledges a user's pending restart/wake. +func stoppedBackupCleanup(ctx context.Context, c client.Client, reader client.Reader, gs *v1.GameServer, s *stoppedBackup) (bool, error) { + if s == nil || s.Stage == "Admitted" { + return true, nil + } + if s.WorkloadUID == "" || s.Deadline == "" || (s.Stage != "Stopping" && s.Stage != "Drained") { + return false, fmt.Errorf("stopped backup provenance is incomplete; recovery intervention required") + } + if _, err := time.Parse(time.RFC3339Nano, s.Deadline); err != nil { + return false, fmt.Errorf("invalid stopped backup deadline: %w", err) + } + var ss appsv1.StatefulSet + if err := reader.Get(ctx, client.ObjectKeyFromObject(gs), &ss); err != nil { + return false, client.IgnoreNotFound(err) + } + if ss.UID != s.WorkloadUID || !metav1.IsControlledBy(&ss, gs) { + return false, fmt.Errorf("stopped backup workload replaced; recovery intervention required") + } + if ss.Spec.Replicas == nil || *ss.Spec.Replicas != 0 || ss.Status.Replicas != 0 || ss.Status.ObservedGeneration < ss.Generation { + return false, nil + } + gone, err := backupQuiesceProcessGone(ctx, reader, gs.Namespace, gs.Name, s.OriginalPodUID) + if err != nil || !gone { + return false, err + } + fake := &v1.Backup{ObjectMeta: metav1.ObjectMeta{Namespace: gs.Namespace}, Status: v1.BackupStatus{CaptureManifest: &v1.BackupCaptureManifest{Volumes: s.Claims}}} + return (&BackupReconciler{Client: c, APIReader: reader}).backupSourceDrained(ctx, fake, gs, &ss) +} + +func recoveryIntervention(ctx context.Context, c client.Client, gs *v1.GameServer, message string) error { + gs.Status.Conditions = upsertCondition(gs.Status.Conditions, metav1.Condition{Type: "RecoveryIntervention", Status: metav1.ConditionTrue, Reason: "ProvenanceUnavailable", Message: message, ObservedGeneration: gs.Generation}) + return c.Status().Update(ctx, gs) +} + +// Autonomous orphan cleanup runs even if nobody requests a successor operation. +func (r *GameServerReconciler) recoverStoppedBackupOwner(ctx context.Context, gs *v1.GameServer) (bool, error) { + op, held, err := lifecycleOwner(gs) + if err != nil { + return true, recoveryIntervention(ctx, r.Client, gs, err.Error()) + } + if !held { + // Wait until cleanup has durably released every owner and compatibility + // guard. A failed status write is retried on the next reconciliation. + if meta.RemoveStatusCondition(&gs.Status.Conditions, "RecoveryIntervention") { + return true, r.Status().Update(ctx, gs) + } + return false, nil + } + if op.Kind != "Backup" || op.StoppedBackup == nil { + return false, nil + } + exists, err := lifecycleOwnerExists(ctx, r.wipeReader(), gs.Namespace, op.lifecycleIdentity) + if err != nil || exists { + return false, err + } + backup := &BackupReconciler{Client: r.Client, APIReader: r.wipeReader(), Scheme: r.Scheme} + busy, err := backup.backupWorkersLive(ctx, gs.Namespace, backupIdentity{Name: op.Name, UID: op.UID}, true) + if err != nil || busy { + return false, err + } + ready, err := recoverLifecycleOperation(ctx, r.Client, r.wipeReader(), gs, op, r.OperationQuiescer) + if err != nil || !ready { + return false, err + } + ready, err = stoppedBackupCleanup(ctx, r.Client, r.wipeReader(), gs, op.StoppedBackup) + if err != nil { + return true, recoveryIntervention(ctx, r.Client, gs, err.Error()) + } + if !ready { + return false, nil + } + if err := clearLifecycleOperation(ctx, r.Client, r.wipeReader(), gs, op.lifecycleIdentity); err != nil { + return false, err + } + delete(gs.Annotations, backupGuardAnnotation) + delete(gs.Annotations, stopRequestedAtAnnotation) + return true, r.Update(ctx, gs) +} + +// Recovery keeps the oldest outstanding graceful stop ahead of a successor. +// Admitted carries no stop side effect; Drained is handled by the zero fence. +func lifecycleStoppingBackup(op lifecycleOperation) *stoppedBackup { + for _, id := range append(append([]lifecycleIdentity{}, op.Recovery...), op.lifecycleIdentity) { + if id.StoppedBackup != nil && id.StoppedBackup.Stage == "Stopping" { + return id.StoppedBackup + } + } + return nil +} +func (r *GameServerReconciler) continueBackupStop(ctx context.Context, gs *v1.GameServer, tmpl *v1.GameTemplate, debt *stoppedBackup) (int32, time.Duration, error) { + deadline, err := time.Parse(time.RFC3339Nano, debt.Deadline) + if err != nil { + return 0, 5 * time.Second, fmt.Errorf("invalid persisted backup stop deadline: %w", err) + } + if !time.Now().Before(deadline) { + return 0, 5 * time.Second, nil + } + // A replacement process must not receive the old process's graceful RPC. + var pod corev1.Pod + err = r.wipeReader().Get(ctx, client.ObjectKey{Namespace: gs.Namespace, Name: gs.Name + "-0"}, &pod) + if apierrors.IsNotFound(err) { + return 0, 5 * time.Second, nil + } + if err != nil { + return 0, 0, err + } + if debt.OriginalPodUID == "" || pod.UID != debt.OriginalPodUID { + return 0, 5 * time.Second, nil + } + replicas, poll, err := r.softStop(ctx, gs, tmpl) + if poll == 0 || poll > time.Until(deadline) { + poll = time.Until(deadline) + } + return replicas, poll, err +} diff --git a/operator/internal/controller/backup_worker.go b/operator/internal/controller/backup_worker.go index 80cf6c98c..14852e727 100644 --- a/operator/internal/controller/backup_worker.go +++ b/operator/internal/controller/backup_worker.go @@ -36,7 +36,7 @@ const ( const resticWorkerScript = `set -eu for utility in timeout date cat touch sleep restic; do command -v "$utility" >/dev/null; done timeout -s TERM -k 5 1 /bin/sh -c ':' -test -d /data && test -r /data && test -x /data +test -d /tmp && test -w /tmp restic snapshots --json --retry-lock 10m >/dev/null touch /tmp/prepared while :; do @@ -251,6 +251,20 @@ func (r *BackupReconciler) reconcilePreparedRestic(ctx context.Context, b *gamep if bound == nil { return ctrl.Result{RequeueAfter: 5 * time.Second}, nil } + if b.Spec.VolumePolicy == "all-data" { + if b.Annotations[captureFormatAnnotation] != "all-data-v2" { + patchBackupAnnotations(b, map[string]string{captureFormatAnnotation: "all-data-v2"}) + if err := r.Update(ctx, b); err != nil { + return ctrl.Result{}, err + } + } + if err := r.recordBackupManifest(ctx, b, bound); err != nil { + return r.abortResticCopy(ctx, b, err.Error()) + } + if err := r.validateBackupRecoveryLayout(ctx, b, bound); err != nil { + return r.abortResticCopy(ctx, b, err.Error()) + } + } source, err := r.prepareBackupWorkerSource(ctx, b) if err != nil { if errors.Is(err, errBackupSourceChanged) { @@ -359,12 +373,23 @@ func (r *BackupReconciler) reconcilePreparedRestic(ctx context.Context, b *gamep if !time.Now().Before(deadline) { return r.abortResticCopy(ctx, b, "maximum quiesce duration exceeded before start authorization") } + if err := r.finalizeBackupCaptureOutcome(ctx, b); err != nil { + return ctrl.Result{}, err + } if b.Annotations[backupAuthorizedWorkerAnnotation] != string(worker.UID) { patchBackupAnnotations(b, map[string]string{backupAuthorizedWorkerAnnotation: string(worker.UID)}) if err := r.Update(ctx, b); err != nil { return ctrl.Result{}, err } } + if err := r.validateBackupRecoveryLayout(ctx, b, bound); err != nil { + return r.abortResticCopy(ctx, b, err.Error()) + } + if b.Spec.VolumePolicy == "all-data" { + if err := validateMountedRecoveryClaims(ctx, r.backupAPIReader(), b.Namespace, b.Status.CaptureManifest.Volumes); err != nil { + return r.abortResticCopy(ctx, b, err.Error()) + } + } token := fmt.Sprintf("v1 %s %s %s %d\n", b.UID, worker.UID, backupWorkerSourceID(b), deadline.Unix()) if gate.Data[backupStartTokenKey] != token { gate.Data[backupStartTokenKey] = token diff --git a/operator/internal/controller/backupschedule_controller.go b/operator/internal/controller/backupschedule_controller.go index 5d53b7318..a584525bf 100644 --- a/operator/internal/controller/backupschedule_controller.go +++ b/operator/internal/controller/backupschedule_controller.go @@ -178,10 +178,12 @@ func (r *BackupScheduleReconciler) fire( Labels: map[string]string{"gameplane.local/backup-schedule": sched.Name}, }, Spec: gameplanev1alpha1.BackupSpec{ - ServerRef: sched.Spec.ServerRef, - RepoRef: sched.Spec.RepoRef, - Strategy: sched.Spec.Strategy, - Quiesce: sched.Spec.Quiesce, + ServerRef: sched.Spec.ServerRef, + RepoRef: sched.Spec.RepoRef, + Strategy: sched.Spec.Strategy, + PowerPolicy: sched.Spec.PowerPolicy, + VolumePolicy: sched.Spec.VolumePolicy, + Quiesce: sched.Spec.Quiesce, }, } return r.Create(ctx, b) diff --git a/operator/internal/controller/gameserver_controller.go b/operator/internal/controller/gameserver_controller.go index 05f548573..c2a5d1998 100644 --- a/operator/internal/controller/gameserver_controller.go +++ b/operator/internal/controller/gameserver_controller.go @@ -268,6 +268,11 @@ func (r *GameServerReconciler) Reconcile(ctx context.Context, req ctrl.Request) } else if changed { return ctrl.Result{RequeueAfter: reconcileRetryInterval}, nil } + if changed, err := r.recoverStoppedBackupOwner(ctx, &gs); err != nil { + return requeueOnConflict(ctrl.Result{}, err) + } else if changed { + return ctrl.Result{RequeueAfter: 5 * time.Second}, nil + } // Exposure compatibility depends only on the server. Reject it before // resolving/adopting a template, which may otherwise initiate a rollout. if r.RestrictedWorkloads && gs.Spec.Networking.Expose == "Hostport" { @@ -1276,6 +1281,10 @@ func (r *GameServerReconciler) desiredReplicas( poll = restartDrainPoll } return replicas, poll, err + } else if held && lifecycleStoppingBackup(op) != nil { + return r.continueBackupStop(ctx, &live, tmpl, lifecycleStoppingBackup(op)) + } else if held && op.Kind == "Backup" && op.StoppedBackup != nil && op.StoppedBackup.Stage == "Drained" { + return 0, 5 * time.Second, nil } else if held && op.InitialStartup && len(op.Recovery) == 0 { if live.Spec.Suspend { return 0, 5 * time.Second, nil @@ -1623,6 +1632,16 @@ func (r *GameServerReconciler) reconcileStatefulSet( // Orphaned restores may leave a running initial workload. // Stop it while preserving its complete pod template. Other // protected operations also preserve the replica count. + if (op.StoppedBackup != nil && op.StoppedBackup.Stage != "Admitted") || lifecycleStoppingBackup(op) != nil { + // A replica decision may predate admission. Never raise a stopped + // workload with that stale decision, including during a copy. + if op.StoppedBackup != nil && op.StoppedBackup.Stage == "Drained" { + zero := int32(0) + ss.Spec.Replicas = &zero + } else if ss.Spec.Replicas != nil && replicas < *ss.Spec.Replicas { + ss.Spec.Replicas = &replicas + } + } if lifecycleRecoveryNeedsStop(op) { zero := int32(0) ss.Spec.Replicas = &zero @@ -2627,6 +2646,8 @@ func (r *GameServerReconciler) reconcileBackupSchedule( _, err := controllerutil.CreateOrUpdate(ctx, r.Client, bs, func() error { bs.Spec.ServerRef = gameplanev1alpha1.LocalObjectRef{Name: gs.Name} + bs.Spec.PowerPolicy = gs.Spec.BackupPolicy.PowerPolicy + bs.Spec.VolumePolicy = gs.Spec.BackupPolicy.VolumePolicy bs.Spec.Schedule = gs.Spec.BackupPolicy.Schedule bs.Spec.RepoRef = &gs.Spec.BackupPolicy.RepoRef bs.Spec.Retention = gs.Spec.BackupPolicy.Retention diff --git a/operator/internal/controller/lifecycle_operation.go b/operator/internal/controller/lifecycle_operation.go index 6445aa927..20aff9bbb 100644 --- a/operator/internal/controller/lifecycle_operation.go +++ b/operator/internal/controller/lifecycle_operation.go @@ -24,14 +24,18 @@ var errLifecycleGenerationChanged = errors.New("lifecycle operation spec changed // request token and GameServer UID identify the operation. Generation is pinned // at admission; subsequent power requests must not change that identity. type lifecycleIdentity struct { - Kind string `json:"kind"` - Name string `json:"name"` - UID types.UID `json:"uid"` - TargetUID types.UID `json:"targetUID"` - Generation int64 `json:"generation"` - Token string `json:"token,omitempty"` - NeedsUnquiesce bool `json:"needsUnquiesce,omitempty"` - QuiescePodUID types.UID `json:"quiescePodUID,omitempty"` + // PreservePower allows v2 restores to stop with the guard, without altering user suspend intent. + PreservePower bool `json:"preservePower,omitempty"` + + StoppedBackup *stoppedBackup `json:"stoppedBackup,omitempty"` + Kind string `json:"kind"` + Name string `json:"name"` + UID types.UID `json:"uid"` + TargetUID types.UID `json:"targetUID"` + Generation int64 `json:"generation"` + Token string `json:"token,omitempty"` + NeedsUnquiesce bool `json:"needsUnquiesce,omitempty"` + QuiescePodUID types.UID `json:"quiescePodUID,omitempty"` // InitialStartup permits only the first workload of a CSI restore's new // target (including WaitForFirstConsumer PVCs), while other operations wait. InitialStartup bool `json:"initialStartup,omitempty"` @@ -69,11 +73,15 @@ func parseLifecycleOperation(value string) (lifecycleOperation, error) { } func backupLifecycleIdentity(gs *gameplanev1alpha1.GameServer, b *gameplanev1alpha1.Backup, quiesce bool) lifecycleIdentity { - return lifecycleIdentity{Kind: "Backup", Name: b.Name, UID: b.UID, TargetUID: gs.UID, Generation: b.Generation, NeedsUnquiesce: quiesce} + id := lifecycleIdentity{Kind: "Backup", Name: b.Name, UID: b.UID, TargetUID: gs.UID, Generation: b.Generation, NeedsUnquiesce: quiesce} + if b.Spec.PowerPolicy == "stop-copy-restart" { + id.StoppedBackup = newStoppedBackup(gs) + } + return id } func restoreLifecycleIdentity(gs *gameplanev1alpha1.GameServer, rs *gameplanev1alpha1.Restore) lifecycleIdentity { - return lifecycleIdentity{Kind: "Restore", Name: rs.Name, UID: rs.UID, TargetUID: gs.UID, Generation: rs.Generation} + return lifecycleIdentity{Kind: "Restore", Name: rs.Name, UID: rs.UID, TargetUID: gs.UID, Generation: rs.Generation, PreservePower: rs.Status.RecoveryPlan != nil} } func wipeLifecycleIdentity(gs *gameplanev1alpha1.GameServer, token string) lifecycleIdentity { @@ -168,6 +176,9 @@ func claimLifecycleOperation(ctx context.Context, c client.Client, reader client return false, errLifecycleGenerationChanged } *gs = live + if want.StoppedBackup != nil { + want.StoppedBackup = newStoppedBackup(gs) + } op, occupied, err := lifecycleOwner(gs) if err != nil { return false, err @@ -225,7 +236,7 @@ func claimLifecycleOperation(ctx context.Context, c client.Client, reader client gs.Annotations[backupGuardAnnotation] = backupIdentityJSON(want.Name, want.UID) case "Restore": gs.Annotations[restoreGuardAnnotation] = restoreIdentityJSON(want.Name, want.UID) - if !op.InitialStartup { + if !op.InitialStartup && !op.PreservePower { gs.Spec.Suspend = true } case "Wipe": @@ -270,7 +281,7 @@ func legacyLifecycleCopiesLive(ctx context.Context, c client.Client, reader clie func lifecycleRecoveryNeedsStop(op lifecycleOperation) bool { for _, predecessor := range op.Recovery { - if predecessor.Kind == "Restore" { + if predecessor.Kind == "Restore" || (predecessor.StoppedBackup != nil && predecessor.StoppedBackup.Stage == "Drained") { return true } } @@ -303,6 +314,12 @@ func recoverLifecycleOperation(ctx context.Context, c client.Client, reader clie } } for _, predecessor := range op.Recovery { + if predecessor.StoppedBackup != nil { + ready, err := stoppedBackupCleanup(ctx, c, reader, gs, predecessor.StoppedBackup) + if err != nil || !ready { + return false, err + } + } if !predecessor.NeedsUnquiesce { continue } diff --git a/operator/internal/controller/recovery_controller_test.go b/operator/internal/controller/recovery_controller_test.go new file mode 100644 index 000000000..f8005a616 --- /dev/null +++ b/operator/internal/controller/recovery_controller_test.go @@ -0,0 +1,1154 @@ +package controller + +import ( + "context" + "errors" + "strings" + "testing" + "time" + + v1 "github.com/GameplanePanel/gameplane/operator/api/v1alpha1" + appsv1 "k8s.io/api/apps/v1" + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" +) + +func recoveryFixture(t *testing.T) (client.Client, *v1.GameServer, *v1.Backup, *v1.Restore) { + t.Helper() + scheme := runtime.NewScheme() + _ = v1.AddToScheme(scheme) + _ = appsv1.AddToScheme(scheme) + _ = batchv1.AddToScheme(scheme) + _ = corev1.AddToScheme(scheme) + gs := &v1.GameServer{ObjectMeta: metav1.ObjectMeta{Name: "game", Namespace: "games", UID: "game-uid", Generation: 3, Annotations: map[string]string{}}, Spec: v1.GameServerSpec{TemplateRef: v1.GameTemplateRef{Name: "template"}, Version: "v1"}} + tmpl := &v1.GameTemplate{ObjectMeta: metav1.ObjectMeta{Name: "template"}, Spec: v1.GameTemplateSpec{Storage: v1.GameStorageSpec{MountPath: "/data", Extra: []v1.ExtraVolumeSpec{{Name: "world", MountPath: "/world", BackupRole: "data"}, {Name: "cache", MountPath: "/cache", BackupRole: "rebuildable"}}}, Versions: []v1.GameVersion{{ID: "v1", Loader: "paper", Image: "busybox"}}, Capabilities: &v1.CapabilitiesSpec{Mods: &v1.ModsSpec{Loaders: map[string]v1.ModLoaderSpec{"paper": {Path: "mods"}}}}}} + one := int32(1) + ss := &appsv1.StatefulSet{ObjectMeta: metav1.ObjectMeta{Name: gs.Name, Namespace: gs.Namespace, UID: "workload", Generation: 1, OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(gs, v1.GroupVersion.WithKind("GameServer"))}}, Spec: appsv1.StatefulSetSpec{Replicas: &one, Template: corev1.PodTemplateSpec{Spec: corev1.PodSpec{Containers: []corev1.Container{{Name: "game"}}}}}, Status: appsv1.StatefulSetStatus{ObservedGeneration: 1}} + objects := []client.Object{gs, tmpl, ss} + for i, pair := range []struct{ name, mount string }{{"data", "/data"}, {"extra-world", "/world"}, {"extra-cache", "/cache"}, {"mods-v1", "/data/mods"}} { + name := gs.Name + "-" + pair.name + ss.Spec.Template.Spec.Volumes = append(ss.Spec.Template.Spec.Volumes, corev1.Volume{Name: pair.name, VolumeSource: corev1.VolumeSource{PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ClaimName: name}}}) + ss.Spec.Template.Spec.Containers[0].VolumeMounts = append(ss.Spec.Template.Spec.Containers[0].VolumeMounts, corev1.VolumeMount{Name: pair.name, MountPath: pair.mount}) + objects = append(objects, &corev1.PersistentVolumeClaim{ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: gs.Namespace, UID: types.UID(name + "-uid"), Finalizers: []string{"kubernetes.io/pvc-protection"}}, Spec: corev1.PersistentVolumeClaimSpec{VolumeName: []string{"pv0", "pv1", "pv2", "pv3"}[i]}, Status: corev1.PersistentVolumeClaimStatus{Phase: corev1.ClaimBound}}) + } + b := &v1.Backup{ObjectMeta: metav1.ObjectMeta{Name: "backup", Namespace: gs.Namespace, UID: "backup-uid", Generation: 1, Annotations: map[string]string{captureFormatAnnotation: "all-data-v2"}}, Spec: v1.BackupSpec{ServerRef: v1.LocalObjectRef{Name: gs.Name}, RepoRef: &v1.SecretKeySelector{Name: "repo"}, VolumePolicy: "all-data", PowerPolicy: "stop-copy-restart"}, Status: v1.BackupStatus{Phase: v1.BackupPhaseSucceeded, SnapshotID: "snapshot"}} + rs := &v1.Restore{ObjectMeta: metav1.ObjectMeta{Name: "restore", Namespace: gs.Namespace, UID: "restore-uid", Annotations: map[string]string{restoreFormatAnnotation: "all-data-v2"}}, Spec: v1.RestoreSpec{ServerRef: v1.LocalObjectRef{Name: gs.Name}, BackupRef: v1.LocalObjectRef{Name: b.Name}}, Status: v1.RestoreStatus{SnapshotID: "snapshot", Phase: v1.RestorePhaseSuspending}} + objects = append(objects, b, rs) + c := fake.NewClientBuilder().WithScheme(scheme).WithStatusSubresource(gs, b, rs).WithObjects(objects...).Build() + m, err := recoveryLayout(t.Context(), c, gs) + if err != nil { + t.Fatal(err) + } + b.Status.CaptureManifest = m + if err := c.Status().Update(t.Context(), b); err != nil { + t.Fatal(err) + } + return c, gs, b, rs +} + +func TestRecoveryLayoutIncludesDataAndSeparateMods(t *testing.T) { + c, gs, b, _ := recoveryFixture(t) + included, excluded := 0, 0 + for _, v := range b.Status.CaptureManifest.Volumes { + if v.Included { + included++ + } else { + excluded++ + if v.RecoveryPrerequisite == "" { + t.Fatal("missing rebuild prerequisite") + } + } + } + if included != 3 || excluded != 1 { + t.Fatalf("coverage: %d/%d", included, excluded) + } + var tmpl v1.GameTemplate + if err := c.Get(t.Context(), client.ObjectKey{Name: "template"}, &tmpl); err != nil { + t.Fatal(err) + } + tmpl.Spec.Storage.Extra[0].BackupRole = "" + if err := c.Update(t.Context(), &tmpl); err != nil { + t.Fatal(err) + } + if _, err := recoveryLayout(t.Context(), c, gs); err == nil { + t.Fatal("unclassified required claim admitted") + } +} + +func TestRecoveryPreflightFailsBeforeSuspension(t *testing.T) { + for _, mutation := range []string{"missing", "replaced", "path", "traversal", "duplicate", "downgrade"} { + t.Run(mutation, func(t *testing.T) { + c, gs, b, rs := recoveryFixture(t) + switch mutation { + case "missing", "replaced": + var pvc corev1.PersistentVolumeClaim + key := client.ObjectKey{Namespace: gs.Namespace, Name: gs.Name + "-extra-world"} + if err := c.Get(t.Context(), key, &pvc); err != nil { + t.Fatal(err) + } + pvc.Finalizers = nil + if err := c.Update(t.Context(), &pvc); err != nil { + t.Fatal(err) + } + if err := c.Delete(t.Context(), &pvc); err != nil { + t.Fatal(err) + } + if mutation == "replaced" { + pvc.ResourceVersion = "" + pvc.UID = "replacement" + if err := c.Create(t.Context(), &pvc); err != nil { + t.Fatal(err) + } + } + case "path": + var ss appsv1.StatefulSet + if err := c.Get(t.Context(), client.ObjectKeyFromObject(gs), &ss); err != nil { + t.Fatal(err) + } + ss.Spec.Template.Spec.Containers[0].VolumeMounts[1].MountPath = "/redirect" + if err := c.Update(t.Context(), &ss); err != nil { + t.Fatal(err) + } + case "traversal": + b.Status.CaptureManifest.Volumes[0].SnapshotRoot = "/volumes/../data" + case "duplicate": + b.Status.CaptureManifest.Volumes = append(b.Status.CaptureManifest.Volumes, b.Status.CaptureManifest.Volumes[0]) + case "downgrade": + b.Status.CaptureManifest = nil + } + if err := c.Status().Update(t.Context(), b); err != nil { + t.Fatal(err) + } + r := &RestoreReconciler{Client: c, APIReader: c, Scheme: c.Scheme()} + if _, err := r.Reconcile(t.Context(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(rs)}); err != nil { + t.Fatal(err) + } + if err := c.Get(t.Context(), client.ObjectKeyFromObject(gs), gs); err != nil { + t.Fatal(err) + } + if gs.Spec.Suspend || gs.Annotations[restoreGuardAnnotation] != "" { + t.Fatal("invalid restore mutated target power/ownership") + } + if err := c.Get(t.Context(), client.ObjectKeyFromObject(rs), rs); err != nil { + t.Fatal(err) + } + if rs.Status.Phase != v1.RestorePhaseFailed { + t.Fatalf("expected visible failure: %+v", rs.Status) + } + var jobs batchv1.JobList + if err := c.List(t.Context(), &jobs); err != nil { + t.Fatal(err) + } + if len(jobs.Items) != 0 { + t.Fatal("invalid restore created destructive job") + } + }) + } +} +func TestRecoveryPlanPinsTargetAgainstReplacement(t *testing.T) { + c, gs, b, rs := recoveryFixture(t) + r := &RestoreReconciler{Client: c, APIReader: c, Scheme: c.Scheme()} + if changed, err := r.preflightRecovery(t.Context(), rs, b, gs); err != nil || !changed { + t.Fatalf("preflight: %v %v", changed, err) + } + before := rs.Status.RecoveryPlan.DeepCopy() + var pvc corev1.PersistentVolumeClaim + key := client.ObjectKey{Namespace: gs.Namespace, Name: gs.Name + "-mods-v1"} + if err := c.Get(t.Context(), key, &pvc); err != nil { + t.Fatal(err) + } + pvc.UID = "replacement" + if err := c.Update(t.Context(), &pvc); err != nil { + t.Fatal(err) + } + if _, err := r.preflightRecovery(t.Context(), rs, b, gs); err == nil { + t.Fatal("replacement admitted") + } + if before.Volumes[0].ClaimUID != rs.Status.RecoveryPlan.Volumes[0].ClaimUID { + t.Fatal("mapping rewritten") + } +} +func TestRecoveryMountedClaimsRequireProtection(t *testing.T) { + c, gs, b, _ := recoveryFixture(t) + if err := validateMountedRecoveryClaims(t.Context(), c, gs.Namespace, b.Status.CaptureManifest.Volumes); err != nil { + t.Fatal(err) + } + var pvc corev1.PersistentVolumeClaim + if err := c.Get(t.Context(), client.ObjectKey{Namespace: gs.Namespace, Name: gs.Name + "-data"}, &pvc); err != nil { + t.Fatal(err) + } + pvc.Finalizers = nil + if err := c.Update(t.Context(), &pvc); err != nil { + t.Fatal(err) + } + if err := validateMountedRecoveryClaims(t.Context(), c, gs.Namespace, b.Status.CaptureManifest.Volumes); err == nil { + t.Fatal("unprotected mounted claim admitted") + } +} +func TestRecoveryAntiDowngradeAndConsistency(t *testing.T) { + _, _, b, _ := recoveryFixture(t) + b.Status.CaptureManifest.QuiesceOutcome = "confirmed" + if backupConsistencyCondition(b).Status != metav1.ConditionFalse { + t.Fatal("live all-data overclaimed atomic consistency") + } + b.Status.CaptureManifest.QuiesceOutcome = "already-stopped" + if backupConsistencyCondition(b).Status != metav1.ConditionTrue { + t.Fatal("drained all-data not recognized") + } + for _, state := range []string{"LegacyUnknown", "Unknown"} { + candidate := b.DeepCopy() + candidate.Status.CaptureManifest.State = state + if _, err := recoveryCaptureMode(candidate); err == nil { + t.Fatal("unknown manifest downgraded") + } + } + legacy := &v1.Backup{} + if all, err := recoveryCaptureMode(legacy); err != nil || all { + t.Fatal("legacy primary restore rejected") + } +} +func TestStoppedOwnerSurvivesRestartAndForceDeletion(t *testing.T) { + for _, stage := range []string{"Admitted", "Stopping", "Drained"} { + t.Run(stage, func(t *testing.T) { + c, gs, b, _ := recoveryFixture(t) + debt := newStoppedBackup(gs) + debt.Stage = stage + debt.WorkloadUID = "workload" + debt.Deadline = time.Now().Add(time.Minute).Format(time.RFC3339Nano) + debt.Claims = b.Status.CaptureManifest.Volumes + op := lifecycleOperation{lifecycleIdentity: backupLifecycleIdentity(gs, b, false)} + op.StoppedBackup = debt + gs.Annotations[lifecycleOperationAnnotation] = lifecycleOperationJSON(op) + gs.Annotations[backupGuardAnnotation] = backupIdentityJSON(b.Name, b.UID) + gs.Spec.Suspend = true // A later user stop must survive recovery. + if err := c.Update(t.Context(), gs); err != nil { + t.Fatal(err) + } + if err := c.Delete(t.Context(), b); err != nil { + t.Fatal(err) + } + var ss appsv1.StatefulSet + if err := c.Get(t.Context(), client.ObjectKeyFromObject(gs), &ss); err != nil { + t.Fatal(err) + } + zero := int32(0) + ss.Spec.Replicas = &zero + if err := c.Update(t.Context(), &ss); err != nil { + t.Fatal(err) + } + r := &GameServerReconciler{Client: c, APIReader: c, Scheme: c.Scheme()} + if changed, err := r.recoverStoppedBackupOwner(t.Context(), gs); err != nil || !changed { + t.Fatalf("orphan recovery: %v %v", changed, err) + } + if err := c.Get(t.Context(), client.ObjectKeyFromObject(gs), gs); err != nil { + t.Fatal(err) + } + if gs.Annotations[lifecycleOperationAnnotation] != "" || !gs.Spec.Suspend { + t.Fatal("orphan recovery lost cleanup or user suspend") + } + }) + } +} +func TestStoppedOwnerRejectsAmbiguousWorkload(t *testing.T) { + c, gs, b, _ := recoveryFixture(t) + debt := newStoppedBackup(gs) + debt.Stage = "Drained" + debt.WorkloadUID = "replacement" + debt.Deadline = time.Now().Format(time.RFC3339Nano) + debt.Claims = b.Status.CaptureManifest.Volumes + if ready, err := stoppedBackupCleanup(t.Context(), c, c, gs, debt); ready || err == nil || !strings.Contains(err.Error(), "intervention") { + t.Fatalf("ambiguous provenance: %v %v", ready, err) + } +} + +func TestRecoveryInterventionClearsAfterStoppedBackupCleanup(t *testing.T) { + for _, orphan := range []bool{false, true} { + t.Run(map[bool]string{false: "normal", true: "orphan"}[orphan], func(t *testing.T) { + c, gs, b, _ := recoveryFixture(t) + b.Annotations[backupTargetAnnotation] = backupIdentityJSON(gs.Name, gs.UID) + if err := c.Update(t.Context(), b); err != nil { + t.Fatal(err) + } + op := lifecycleOperation{lifecycleIdentity: backupLifecycleIdentity(gs, b, false)} + op.StoppedBackup.Stage = "Drained" + op.StoppedBackup.WorkloadUID = "unverified-workload" + op.StoppedBackup.Deadline = time.Now().Format(time.RFC3339Nano) + op.StoppedBackup.Claims = b.Status.CaptureManifest.Volumes + gs.Annotations[lifecycleOperationAnnotation] = lifecycleOperationJSON(op) + gs.Annotations[backupGuardAnnotation] = backupIdentityJSON(b.Name, b.UID) + if err := c.Update(t.Context(), gs); err != nil { + t.Fatal(err) + } + gs.Status.Conditions = []metav1.Condition{{Type: "Ready", Status: metav1.ConditionTrue, Reason: "Ready"}} + if err := c.Status().Update(t.Context(), gs); err != nil { + t.Fatal(err) + } + gr := &GameServerReconciler{Client: c, APIReader: c, Scheme: c.Scheme()} + br := &BackupReconciler{Client: c, APIReader: c, Scheme: c.Scheme()} + cleanup := func(wantError string) { + t.Helper() + var err error + if orphan { + _, err = gr.recoverStoppedBackupOwner(t.Context(), gs) + wantError = "" // Orphan intervention is reported through status. + } else { + err = br.releaseBackupTarget(t.Context(), b) + } + if wantError == "" && err != nil { + t.Fatal(err) + } + if wantError != "" && (err == nil || !strings.Contains(err.Error(), wantError)) { + t.Fatalf("cleanup error = %v, want %q", err, wantError) + } + if err := c.Get(t.Context(), client.ObjectKeyFromObject(gs), gs); err != nil { + t.Fatal(err) + } + } + if orphan { + if err := c.Delete(t.Context(), b); err != nil { + t.Fatal(err) + } + } + cleanup("intervention") + if !meta.IsStatusConditionTrue(gs.Status.Conditions, "RecoveryIntervention") || gs.Annotations[lifecycleOperationAnnotation] == "" { + t.Fatal("unverified workload did not retain intervention and ownership") + } + op.StoppedBackup.WorkloadUID = "workload" + gs.Annotations[lifecycleOperationAnnotation] = lifecycleOperationJSON(op) + if err := c.Update(t.Context(), gs); err != nil { + t.Fatal(err) + } + var ss appsv1.StatefulSet + if err := c.Get(t.Context(), client.ObjectKeyFromObject(gs), &ss); err != nil { + t.Fatal(err) + } + zero := int32(0) + ss.Spec.Replicas = &zero + if err := c.Update(t.Context(), &ss); err != nil { + t.Fatal(err) + } + writer := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "remaining-writer", Namespace: gs.Namespace}, Spec: corev1.PodSpec{Volumes: []corev1.Volume{{Name: "world", VolumeSource: corev1.VolumeSource{PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ClaimName: gs.Name + "-extra-world"}}}}}} + if err := c.Create(t.Context(), writer); err != nil { + t.Fatal(err) + } + cleanup("waiting for stopped source and workers to drain") + if changed, err := gr.recoverStoppedBackupOwner(t.Context(), gs); err != nil || changed { + t.Fatalf("intervention changed before writer drainage: %v %v", changed, err) + } + if !meta.IsStatusConditionTrue(gs.Status.Conditions, "RecoveryIntervention") || gs.Annotations[lifecycleOperationAnnotation] == "" { + t.Fatal("live writer did not retain intervention and ownership") + } + if err := c.Delete(t.Context(), writer); err != nil { + t.Fatal(err) + } + cleanup("") + if gs.Annotations[lifecycleOperationAnnotation] != "" || gs.Annotations[backupGuardAnnotation] != "" { + t.Fatal("completed cleanup retained ownership") + } + if changed, err := gr.recoverStoppedBackupOwner(t.Context(), gs); err != nil || !changed { + t.Fatalf("clear completed intervention: %v %v", changed, err) + } + if err := c.Get(t.Context(), client.ObjectKeyFromObject(gs), gs); err != nil { + t.Fatal(err) + } + if meta.FindStatusCondition(gs.Status.Conditions, "RecoveryIntervention") != nil || !meta.IsStatusConditionTrue(gs.Status.Conditions, "Ready") { + t.Fatal("completed cleanup retained intervention or removed unrelated status") + } + if changed, err := gr.recoverStoppedBackupOwner(t.Context(), gs); err != nil || changed { + t.Fatalf("clean recovery did not settle: %v %v", changed, err) + } + }) + } +} + +func TestStoppedPredecessorClaimsSurviveSuccessorAdmission(t *testing.T) { + c, gs, b, _ := recoveryFixture(t) + debt := newStoppedBackup(gs) + debt.Stage = "Drained" + debt.WorkloadUID = "workload" + debt.Deadline = time.Now().Add(time.Minute).Format(time.RFC3339Nano) + debt.Claims = b.Status.CaptureManifest.Volumes + predecessor := backupLifecycleIdentity(gs, b, false) + predecessor.StoppedBackup = debt + successor := b.DeepCopy() + successor.Name = "successor" + successor.UID = "successor-uid" + op := lifecycleOperation{lifecycleIdentity: backupLifecycleIdentity(gs, successor, false), Recovery: []lifecycleIdentity{predecessor}} + gs.Annotations[lifecycleOperationAnnotation] = lifecycleOperationJSON(op) + if err := c.Update(t.Context(), gs); err != nil { + t.Fatal(err) + } + var ss appsv1.StatefulSet + if err := c.Get(t.Context(), client.ObjectKeyFromObject(gs), &ss); err != nil { + t.Fatal(err) + } + zero := int32(0) + ss.Spec.Replicas = &zero + if err := c.Update(t.Context(), &ss); err != nil { + t.Fatal(err) + } + writer := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "extra-writer", Namespace: gs.Namespace}, Spec: corev1.PodSpec{Volumes: []corev1.Volume{{Name: "extra", VolumeSource: corev1.VolumeSource{PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ClaimName: gs.Name + "-extra-world"}}}}}} + if err := c.Create(t.Context(), writer); err != nil { + t.Fatal(err) + } + for i := 0; i < 2; i++ { + if ready, err := stoppedBackupCleanup(t.Context(), c, c, gs, debt); err != nil || ready { + t.Fatalf("predecessor writer ignored: %v %v", ready, err) + } + op.Recovery = append(op.Recovery, op.lifecycleIdentity) + op.UID = types.UID("another-successor") + gs.Annotations[lifecycleOperationAnnotation] = lifecycleOperationJSON(op) + if err := c.Update(t.Context(), gs); err != nil { + t.Fatal(err) + } + } + if err := c.Delete(t.Context(), writer); err != nil { + t.Fatal(err) + } + if ready, err := stoppedBackupCleanup(t.Context(), c, c, gs, debt); err != nil || !ready { + t.Fatalf("drained predecessor retained: %v %v", ready, err) + } +} +func TestStoppedPredecessorStageControlsGracefulRecovery(t *testing.T) { + for _, stage := range []string{"Admitted", "Stopping", "Drained"} { + op := lifecycleOperation{Recovery: []lifecycleIdentity{{Kind: "Backup", StoppedBackup: &stoppedBackup{Stage: stage}}}} + if (lifecycleStoppingBackup(op) != nil) != (stage == "Stopping") { + t.Fatalf("stage %s lost graceful debt", stage) + } + if lifecycleRecoveryNeedsStop(op) != (stage == "Drained") { + t.Fatalf("stage %s incorrectly forced zero", stage) + } + } +} +func TestStaleRunningReplicaDecisionCannotRestartStoppedCapture(t *testing.T) { + for _, stage := range []string{"Stopping", "Drained"} { + t.Run(stage, func(t *testing.T) { + c, gs, b, _ := recoveryFixture(t) + stale := gs.DeepCopy() + op := lifecycleOperation{lifecycleIdentity: backupLifecycleIdentity(gs, b, false)} + op.StoppedBackup.Stage = stage + gs.Annotations[lifecycleOperationAnnotation] = lifecycleOperationJSON(op) + if err := c.Update(t.Context(), gs); err != nil { + t.Fatal(err) + } + var ss appsv1.StatefulSet + if err := c.Get(t.Context(), client.ObjectKeyFromObject(gs), &ss); err != nil { + t.Fatal(err) + } + zero := int32(0) + ss.Spec.Replicas = &zero + if err := c.Update(t.Context(), &ss); err != nil { + t.Fatal(err) + } + tmpl, err := effectiveGameTemplate(t.Context(), c, gs) + if err != nil { + t.Fatal(err) + } + r := &GameServerReconciler{Client: c, APIReader: c, Scheme: c.Scheme()} + if err := r.reconcileStatefulSet(t.Context(), stale, tmpl, nil, &materializedConfig{}, 1); err != nil { + t.Fatal(err) + } + if err := c.Get(t.Context(), client.ObjectKeyFromObject(gs), &ss); err != nil { + t.Fatal(err) + } + if ss.Spec.Replicas == nil || *ss.Spec.Replicas != 0 { + t.Fatal("stale reconcile restarted captured workload") + } + }) + } +} +func TestRecoveryRestorePreservesPowerDuringAdmissionAndSuccess(t *testing.T) { + for _, mode := range []string{"running", "suspended", "idle"} { + t.Run(mode, func(t *testing.T) { + c, gs, b, rs := recoveryFixture(t) + gs.Spec.Suspend = mode == "suspended" + if mode == "idle" { + gs.Annotations[IdleAsleepSinceAnnotation] = "2026-01-01T00:00:00Z" + } + if err := c.Update(t.Context(), gs); err != nil { + t.Fatal(err) + } + r := &RestoreReconciler{Client: c, APIReader: c, Scheme: c.Scheme()} + if _, err := r.preflightRecovery(t.Context(), rs, b, gs); err != nil { + t.Fatal(err) + } + if _, err := r.bindRestoreTarget(t.Context(), rs, gs); err != nil { + t.Fatal(err) + } + if claimed, err := r.claimRestoreTarget(t.Context(), rs, gs); err != nil || !claimed { + t.Fatalf("claim: %v %v", claimed, err) + } + if gs.Spec.Suspend != (mode == "suspended") { + t.Fatal("admission overwrote user power intent") + } + if mode == "idle" && gs.Annotations[IdleAsleepSinceAnnotation] == "" { + t.Fatal("admission cleared idle intent") + } + rs.Status.Phase = v1.RestorePhaseSucceeded + if err := c.Status().Update(t.Context(), rs); err != nil { + t.Fatal(err) + } + if _, err := r.cleanupRestore(t.Context(), rs); err != nil { + t.Fatal(err) + } + if err := c.Get(t.Context(), client.ObjectKeyFromObject(gs), gs); err != nil { + t.Fatal(err) + } + if gs.Spec.Suspend != (mode == "suspended") { + t.Fatal("successful recovery overwrote power intent") + } + if mode == "idle" && gs.Annotations[IdleAsleepSinceAnnotation] == "" { + t.Fatal("successful recovery woke idle game") + } + }) + } +} +func TestRecoveryMissingRebuildableIsExplicitPrerequisite(t *testing.T) { + c, gs, b, rs := recoveryFixture(t) + var pvc corev1.PersistentVolumeClaim + if err := c.Get(t.Context(), client.ObjectKey{Namespace: gs.Namespace, Name: gs.Name + "-extra-cache"}, &pvc); err != nil { + t.Fatal(err) + } + pvc.Finalizers = nil + if err := c.Update(t.Context(), &pvc); err != nil { + t.Fatal(err) + } + if err := c.Delete(t.Context(), &pvc); err != nil { + t.Fatal(err) + } + r := &RestoreReconciler{Client: c, APIReader: c, Scheme: c.Scheme()} + if _, err := r.preflightRecovery(t.Context(), rs, b, gs); err != nil { + t.Fatal(err) + } + for _, volume := range rs.Status.RecoveryPlan.Volumes { + if volume.LogicalName == "extra-cache" { + if volume.Included || volume.ClaimUID != "" || volume.RecoveryPrerequisite == "" { + t.Fatal("missing rebuildable silently treated as covered") + } + return + } + } + t.Fatal("missing prerequisite from plan") +} +func TestFailedRecoveryWorkerDoesNotWaitForReadinessDeadline(t *testing.T) { + c, gs, b, rs := recoveryFixture(t) + r := &RestoreReconciler{Client: c, APIReader: c, Scheme: c.Scheme()} + if _, err := r.preflightRecovery(t.Context(), rs, b, gs); err != nil { + t.Fatal(err) + } + rs.Annotations[restoreDeadlineAnnotation] = time.Now().Add(time.Hour).Format(time.RFC3339Nano) + job := &batchv1.Job{Status: batchv1.JobStatus{Conditions: []batchv1.JobCondition{{Type: batchv1.JobFailed, Status: corev1.ConditionTrue}}}} + if ready, err := r.authorizeRecoveryRestore(t.Context(), rs, b, gs, job); ready || err == nil || !strings.Contains(err.Error(), "job reported Failed") { + t.Fatalf("failed job hidden by readiness: %v %v", ready, err) + } +} + +func TestStoppedCopyFailureAndCancellationWaitForWorkers(t *testing.T) { + for _, reason := range []string{"copy failed", "backup cancelled", "preparation deadline expired"} { + t.Run(reason, func(t *testing.T) { + c, gs, b, _ := recoveryFixture(t) + b.Status.Phase = v1.BackupPhaseRunning + b.Status.SnapshotID = "" + if err := c.Status().Update(t.Context(), b); err != nil { + t.Fatal(err) + } + b.Annotations[backupTargetAnnotation] = backupIdentityJSON(gs.Name, gs.UID) + b.Annotations[backupPreparationDeadlineAnnotation] = time.Now().Add(-time.Minute).Format(time.RFC3339Nano) + if err := c.Update(t.Context(), b); err != nil { + t.Fatal(err) + } + debt := newStoppedBackup(gs) + debt.Stage = "Drained" + debt.WorkloadUID = "workload" + debt.Deadline = b.Annotations[backupPreparationDeadlineAnnotation] + debt.Claims = b.Status.CaptureManifest.Volumes + op := lifecycleOperation{lifecycleIdentity: backupLifecycleIdentity(gs, b, false)} + op.StoppedBackup = debt + gs.Annotations[lifecycleOperationAnnotation] = lifecycleOperationJSON(op) + gs.Annotations[backupGuardAnnotation] = backupIdentityJSON(b.Name, b.UID) + gs.Spec.Suspend = true + if err := c.Update(t.Context(), gs); err != nil { + t.Fatal(err) + } + var ss appsv1.StatefulSet + if err := c.Get(t.Context(), client.ObjectKeyFromObject(gs), &ss); err != nil { + t.Fatal(err) + } + zero := int32(0) + ss.Spec.Replicas = &zero + if err := c.Update(t.Context(), &ss); err != nil { + t.Fatal(err) + } + pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "copy-worker", Namespace: gs.Namespace, Labels: map[string]string{backupUIDLabel: string(b.UID)}}, Spec: corev1.PodSpec{Volumes: []corev1.Volume{{Name: "data", VolumeSource: corev1.VolumeSource{PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ClaimName: gs.Name + "-data"}}}}}} + if err := c.Create(t.Context(), pod); err != nil { + t.Fatal(err) + } + r := &BackupReconciler{Client: c, APIReader: c, Scheme: c.Scheme()} + if result, err := r.abortResticCopy(t.Context(), b, reason); err != nil || result.RequeueAfter == 0 { + t.Fatalf("worker drain: %v %v", result, err) + } + if err := c.Get(t.Context(), client.ObjectKeyFromObject(gs), gs); err != nil { + t.Fatal(err) + } + if gs.Annotations[lifecycleOperationAnnotation] == "" { + t.Fatal("failure released live reader") + } + if err := c.Delete(t.Context(), pod); err != nil { + t.Fatal(err) + } + // A new reconciler instance proves the persisted abort and stop debt are sufficient. + r = &BackupReconciler{Client: c, APIReader: c, Scheme: c.Scheme()} + if _, err := r.Reconcile(t.Context(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(b)}); err != nil { + t.Fatal(err) + } + if err := c.Get(t.Context(), client.ObjectKeyFromObject(gs), gs); err != nil { + t.Fatal(err) + } + if gs.Annotations[lifecycleOperationAnnotation] != "" || !gs.Spec.Suspend { + t.Fatal("failure cleanup lost release or newer suspend intent") + } + }) + } +} +func TestRecoveryAuthorizationRejectsPostPreparationClaimReplacement(t *testing.T) { + c, gs, b, rs := recoveryFixture(t) + r := &RestoreReconciler{Client: c, APIReader: c, Scheme: c.Scheme()} + if _, err := r.preflightRecovery(t.Context(), rs, b, gs); err != nil { + t.Fatal(err) + } + rs.Annotations[restoreDeadlineAnnotation] = time.Now().Add(time.Hour).Format(time.RFC3339Nano) + if err := c.Update(t.Context(), rs); err != nil { + t.Fatal(err) + } + gate, err := r.restoreRecoveryGate(t.Context(), rs) + if err != nil { + t.Fatal(err) + } + gate.Data[backupStartTokenKey] = "old authorization" + if err := c.Update(t.Context(), gate); err != nil { + t.Fatal(err) + } + var pvc corev1.PersistentVolumeClaim + if err := c.Get(t.Context(), client.ObjectKey{Namespace: gs.Namespace, Name: gs.Name + "-extra-world"}, &pvc); err != nil { + t.Fatal(err) + } + pvc.UID = "replacement" + if err := c.Update(t.Context(), &pvc); err != nil { + t.Fatal(err) + } + if ready, err := r.authorizeRecoveryRestore(t.Context(), rs, b, gs, &batchv1.Job{}); ready || err == nil { + t.Fatal("post-preparation replacement authorized") + } + if err := c.Get(t.Context(), client.ObjectKeyFromObject(gate), gate); err != nil { + t.Fatal(err) + } + if gate.Data[backupStartTokenKey] != "" { + t.Fatal("failed revalidation retained authorization") + } +} + +// The transport records real calls made by desiredReplicas/softStop. Failure +// must retain the same durable grace timestamp across new reconciler instances. +type recoveryStopper struct { + calls int + fail bool +} + +func (s *recoveryStopper) Enabled() bool { return true } +func (s *recoveryStopper) Stop(context.Context, string, string) error { + s.calls++ + if s.fail { + return errors.New("stop transport unavailable") + } + return nil +} + +func TestStoppedBackupGracefulDebtAcrossReconcileAndForceDeletion(t *testing.T) { + for _, deleteAt := range []string{"before-rpc", "after-rpc"} { + t.Run(deleteAt, func(t *testing.T) { + c, gs, b, _ := recoveryFixture(t) + var tmpl v1.GameTemplate + if err := c.Get(t.Context(), client.ObjectKey{Name: "template"}, &tmpl); err != nil { + t.Fatal(err) + } + tmpl.Spec.RCON = &v1.RCONSpec{Protocol: "source", Port: 25575} + tmpl.Spec.Capabilities.Lifecycle = &v1.LifecycleSpec{Stop: []string{"stop"}} + if err := c.Update(t.Context(), &tmpl); err != nil { + t.Fatal(err) + } + b.Status = v1.BackupStatus{} + if err := c.Status().Update(t.Context(), b); err != nil { + t.Fatal(err) + } + b.Annotations = map[string]string{} + if err := c.Update(t.Context(), b); err != nil { + t.Fatal(err) + } + secret := &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "repo", Namespace: gs.Namespace}, Data: map[string][]byte{"repo": []byte("s3:https://example.invalid/repo"), "password": []byte("fixture")}} + if err := c.Create(t.Context(), secret); err != nil { + t.Fatal(err) + } + var ss appsv1.StatefulSet + if err := c.Get(t.Context(), client.ObjectKeyFromObject(gs), &ss); err != nil { + t.Fatal(err) + } + ss.Status.ReadyReplicas = 1 + ss.Status.Replicas = 1 + if err := c.Status().Update(t.Context(), &ss); err != nil { + t.Fatal(err) + } + pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: gs.Name + "-0", Namespace: gs.Namespace, UID: "original-pod", OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(&ss, appsv1.SchemeGroupVersion.WithKind("StatefulSet"))}}, Spec: ss.Spec.Template.Spec, Status: corev1.PodStatus{Phase: corev1.PodRunning}} + pod.Spec.NodeName = "node" + if err := c.Create(t.Context(), pod); err != nil { + t.Fatal(err) + } + br := &BackupReconciler{Client: c, APIReader: c, Scheme: c.Scheme()} + if _, err := br.Reconcile(t.Context(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(b)}); err != nil { + t.Fatal(err) + } + if err := c.Get(t.Context(), client.ObjectKeyFromObject(gs), gs); err != nil { + t.Fatal(err) + } + op, held, err := lifecycleOwner(gs) + if err != nil || !held || op.StoppedBackup == nil || op.StoppedBackup.Stage != "Stopping" { + t.Fatalf("missing admitted stop debt: %+v %v", op, err) + } + deadline := op.StoppedBackup.Deadline + if deadline == "" || op.StoppedBackup.OriginalPodUID != pod.UID || gs.Spec.Suspend { + t.Fatal("admission lost deadline/process or mutated suspend") + } + stopper := &recoveryStopper{fail: true} + gr := &GameServerReconciler{Client: c, APIReader: c, Scheme: c.Scheme(), AgentClient: stopper} + if deleteAt == "after-rpc" { + replicas, _, err := gr.desiredReplicas(t.Context(), gs, &tmpl, idleAwake) + if err != nil || replicas != 1 || stopper.calls != 1 { + t.Fatalf("initial graceful stop: %d %v calls=%d", replicas, err, stopper.calls) + } + } + if err := c.Get(t.Context(), client.ObjectKeyFromObject(b), b); err != nil { + t.Fatal(err) + } + b.Finalizers = nil + if err := c.Update(t.Context(), b); err != nil { + t.Fatal(err) + } + if err := c.Delete(t.Context(), b); err != nil { + t.Fatal(err) + } + if err := c.Get(t.Context(), client.ObjectKeyFromObject(gs), gs); err != nil { + t.Fatal(err) + } + if changed, err := gr.recoverStoppedBackupOwner(t.Context(), gs); err != nil || changed { + t.Fatalf("orphan released original process: %v %v", changed, err) + } + replicas, poll, err := gr.desiredReplicas(t.Context(), gs, &tmpl, idleAwake) + if err != nil || replicas != 1 || poll <= 0 || stopper.calls != 1 { + t.Fatalf("orphan skipped/replayed graceful stop: %d %s %v calls=%d", replicas, poll, err, stopper.calls) + } + if err := c.Get(t.Context(), client.ObjectKeyFromObject(gs), gs); err != nil { + t.Fatal(err) + } + stamp := gs.Annotations[stopRequestedAtAnnotation] + if stamp == "" { + t.Fatal("stop attempt lacks durable timestamp") + } + // Controller restart must neither replay a failed RPC nor renew its deadline. + gr = &GameServerReconciler{Client: c, APIReader: c, Scheme: c.Scheme(), AgentClient: stopper} + if _, _, err := gr.desiredReplicas(t.Context(), gs, &tmpl, idleAwake); err != nil { + t.Fatal(err) + } + if stopper.calls != 1 { + t.Fatal("operator restart replayed stop RPC") + } + op, _, _ = lifecycleOwner(gs) + if op.StoppedBackup.Deadline != deadline || gs.Annotations[stopRequestedAtAnnotation] != stamp { + t.Fatal("operator restart renewed stop budget") + } + gs.Annotations[stopRequestedAtAnnotation] = time.Now().Add(-time.Hour).UTC().Format(time.RFC3339) + gs.Spec.Suspend = true + if err := c.Update(t.Context(), gs); err != nil { + t.Fatal(err) + } + replicas, _, err = gr.desiredReplicas(t.Context(), gs, &tmpl, idleAwake) + if err != nil || replicas != 0 || stopper.calls != 1 { + t.Fatalf("expired graceful backstop: %d %v", replicas, err) + } + if err := gr.reconcileStatefulSet(t.Context(), gs, &tmpl, nil, &materializedConfig{}, replicas); err != nil { + t.Fatal(err) + } + if changed, err := gr.recoverStoppedBackupOwner(t.Context(), gs); err != nil || changed { + t.Fatalf("timeout mistaken for process exit: %v %v", changed, err) + } + if err := c.Get(t.Context(), client.ObjectKeyFromObject(gs), &ss); err != nil { + t.Fatal(err) + } + ss.Status.Replicas = 0 + ss.Status.ReadyReplicas = 0 + ss.Status.ObservedGeneration = ss.Generation + if err := c.Status().Update(t.Context(), &ss); err != nil { + t.Fatal(err) + } + if err := c.Delete(t.Context(), pod); err != nil { + t.Fatal(err) + } + if changed, err := gr.recoverStoppedBackupOwner(t.Context(), gs); err != nil || !changed { + t.Fatalf("drained orphan failed release: %v %v", changed, err) + } + if err := c.Get(t.Context(), client.ObjectKeyFromObject(gs), gs); err != nil { + t.Fatal(err) + } + if !gs.Spec.Suspend || gs.Annotations[lifecycleOperationAnnotation] != "" { + t.Fatal("orphan release lost user stop intent") + } + }) + } +} + +func TestStoppedBackupWorkerRetryCancellationAcrossReconcile(t *testing.T) { + c, gs, b, _ := recoveryFixture(t) + b.Status = v1.BackupStatus{} + if err := c.Status().Update(t.Context(), b); err != nil { + t.Fatal(err) + } + b.Annotations = map[string]string{} + if err := c.Update(t.Context(), b); err != nil { + t.Fatal(err) + } + if err := c.Create(t.Context(), &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "repo", Namespace: gs.Namespace}, Data: map[string][]byte{"repo": []byte("s3:https://example.invalid/repo"), "password": []byte("fixture")}}); err != nil { + t.Fatal(err) + } + var ss appsv1.StatefulSet + if err := c.Get(t.Context(), client.ObjectKeyFromObject(gs), &ss); err != nil { + t.Fatal(err) + } + original := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: gs.Name + "-0", Namespace: gs.Namespace, UID: "original", OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(&ss, appsv1.SchemeGroupVersion.WithKind("StatefulSet"))}}, Spec: ss.Spec.Template.Spec, Status: corev1.PodStatus{Phase: corev1.PodRunning}} + original.Spec.NodeName = "node" + if err := c.Create(t.Context(), original); err != nil { + t.Fatal(err) + } + reconcile := func() { + t.Helper() + r := &BackupReconciler{Client: c, APIReader: c, Scheme: c.Scheme()} + if _, err := r.Reconcile(t.Context(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(b)}); err != nil { + t.Fatal(err) + } + } + reconcile() + if err := c.Get(t.Context(), client.ObjectKeyFromObject(gs), gs); err != nil { + t.Fatal(err) + } + tmpl, err := effectiveGameTemplate(t.Context(), c, gs) + if err != nil { + t.Fatal(err) + } + gr := &GameServerReconciler{Client: c, APIReader: c, Scheme: c.Scheme()} + replicas, _, err := gr.desiredReplicas(t.Context(), gs, tmpl, idleAwake) + if err != nil || replicas != 0 { + t.Fatalf("stop decision: %d %v", replicas, err) + } + if err := gr.reconcileStatefulSet(t.Context(), gs, tmpl, nil, &materializedConfig{}, replicas); err != nil { + t.Fatal(err) + } + if err := c.Delete(t.Context(), original); err != nil { + t.Fatal(err) + } + reconcile() + var job batchv1.Job + if err := c.Get(t.Context(), client.ObjectKeyFromObject(b), &job); err != nil { + t.Fatal(err) + } + job.UID = "copy-job" + if err := c.Update(t.Context(), &job); err != nil { + t.Fatal(err) + } + readyPod := func(name string, ready bool) *corev1.Pod { + status := corev1.ConditionFalse + if ready { + status = corev1.ConditionTrue + } + return &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: gs.Namespace, UID: types.UID(name), Labels: map[string]string{backupUIDLabel: string(b.UID)}, OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(&job, batchv1.SchemeGroupVersion.WithKind("Job"))}}, Spec: corev1.PodSpec{NodeName: "node", Volumes: job.Spec.Template.Spec.Volumes}, Status: corev1.PodStatus{Phase: corev1.PodRunning, Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: status}}}} + } + first := readyPod("copy-one", true) + if err := c.Create(t.Context(), first); err != nil { + t.Fatal(err) + } + reconcile() + if err := c.Get(t.Context(), client.ObjectKeyFromObject(b), b); err != nil { + t.Fatal(err) + } + deadline := b.Annotations[backupQuiesceDeadlineAnnotation] + var gate corev1.ConfigMap + key := client.ObjectKey{Namespace: gs.Namespace, Name: backupStartConfigMapName(b)} + if err := c.Get(t.Context(), key, &gate); err != nil { + t.Fatal(err) + } + oldToken := gate.Data[backupStartTokenKey] + if !strings.Contains(oldToken, "copy-one") { + t.Fatal("first ready worker was not authorized") + } + first.Status.Phase = corev1.PodFailed + if err := c.Status().Update(t.Context(), first); err != nil { + t.Fatal(err) + } + retry := readyPod("copy-two", false) + retry.Finalizers = []string{"fixture/terminating"} + if err := c.Create(t.Context(), retry); err != nil { + t.Fatal(err) + } + reconcile() + if err := c.Get(t.Context(), key, &gate); err != nil { + t.Fatal(err) + } + if gate.Data[backupStartTokenKey] != "" { + t.Fatal("unprepared retry inherited old token") + } + retry.Status.Conditions[0].Status = corev1.ConditionTrue + if err := c.Status().Update(t.Context(), retry); err != nil { + t.Fatal(err) + } + reconcile() + if err := c.Get(t.Context(), key, &gate); err != nil { + t.Fatal(err) + } + if !strings.Contains(gate.Data[backupStartTokenKey], "copy-two") || gate.Data[backupStartTokenKey] == oldToken { + t.Fatal("retry authorization not Pod-bound") + } + if err := c.Get(t.Context(), client.ObjectKeyFromObject(b), b); err != nil { + t.Fatal(err) + } + if b.Annotations[backupQuiesceDeadlineAnnotation] != deadline { + t.Fatal("retry reset copy deadline") + } + if err := c.Get(t.Context(), client.ObjectKeyFromObject(gs), gs); err != nil { + t.Fatal(err) + } + gs.Spec.Suspend = true + gs.Annotations[IdleAsleepSinceAnnotation] = "later-idle" + gs.Annotations[RestartRequestedAnnotation] = "later-restart" + gs.Annotations[IdleWakeRequestedAnnotation] = "later-wake" + if err := c.Update(t.Context(), gs); err != nil { + t.Fatal(err) + } + if err := c.Delete(t.Context(), b); err != nil { + t.Fatal(err) + } + reconcile() + if err := c.Delete(t.Context(), retry); err != nil { + t.Fatal(err) + } + reconcile() + if err := c.Get(t.Context(), client.ObjectKeyFromObject(gs), gs); err != nil { + t.Fatal(err) + } + if gs.Annotations[lifecycleOperationAnnotation] == "" { + t.Fatal("cancel released terminating reader") + } + if err := c.Get(t.Context(), key, &gate); err != nil { + t.Fatal(err) + } + if gate.Data[backupStartTokenKey] != "" { + t.Fatal("cancel retained authorization") + } + if err := c.Get(t.Context(), client.ObjectKeyFromObject(retry), retry); err != nil { + t.Fatal(err) + } + retry.Finalizers = nil + if err := c.Update(t.Context(), retry); err != nil { + t.Fatal(err) + } + reconcile() + if err := c.Get(t.Context(), client.ObjectKeyFromObject(gs), gs); err != nil { + t.Fatal(err) + } + if gs.Annotations[lifecycleOperationAnnotation] != "" || !gs.Spec.Suspend || gs.Annotations[IdleAsleepSinceAnnotation] != "later-idle" || gs.Annotations[RestartRequestedAnnotation] != "later-restart" || gs.Annotations[IdleWakeRequestedAnnotation] != "later-wake" { + t.Fatal("cancel cleanup lost newer power/admin intent") + } +} + +func TestRecoveryRestoreGateRetryAndCancelledWriter(t *testing.T) { + c, gs, b, rs := recoveryFixture(t) + r := &RestoreReconciler{Client: c, APIReader: c, Scheme: c.Scheme()} + if _, err := r.preflightRecovery(t.Context(), rs, b, gs); err != nil { + t.Fatal(err) + } + if _, err := r.bindRestoreTarget(t.Context(), rs, gs); err != nil { + t.Fatal(err) + } + if claimed, err := r.claimRestoreTarget(t.Context(), rs, gs); err != nil || !claimed { + t.Fatalf("claim: %v %v", claimed, err) + } + tmpl, err := effectiveGameTemplate(t.Context(), c, gs) + if err != nil { + t.Fatal(err) + } + spec := r.buildRestorePodSpec(rs, b, tmpl) + if err := r.prepareRecoveryRestorePod(t.Context(), rs, &spec); err != nil { + t.Fatal(err) + } + job := &batchv1.Job{ObjectMeta: metav1.ObjectMeta{Name: "restore-" + rs.Name, Namespace: gs.Namespace, UID: "restore-job", OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(rs, v1.GroupVersion.WithKind("Restore"))}}, Spec: batchv1.JobSpec{Template: corev1.PodTemplateSpec{Spec: spec}}} + if err := c.Create(t.Context(), job); err != nil { + t.Fatal(err) + } + pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "restore-one", Namespace: gs.Namespace, UID: "restore-one", Labels: map[string]string{restoreUIDLabel: string(rs.UID)}, OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(job, batchv1.SchemeGroupVersion.WithKind("Job"))}}, Spec: spec, Status: corev1.PodStatus{Phase: corev1.PodRunning, Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: corev1.ConditionTrue}}}} + pod.Spec.NodeName = "node" + if err := c.Create(t.Context(), pod); err != nil { + t.Fatal(err) + } + if ready, err := r.authorizeRecoveryRestore(t.Context(), rs, b, gs, job); err != nil || !ready { + t.Fatalf("authorization: %v %v", ready, err) + } + key := client.ObjectKey{Namespace: gs.Namespace, Name: restoreStartName(rs)} + var gate corev1.ConfigMap + if err := c.Get(t.Context(), key, &gate); err != nil { + t.Fatal(err) + } + oldToken := gate.Data[backupStartTokenKey] + deadline := rs.Annotations[restoreDeadlineAnnotation] + pod.Status.Phase = corev1.PodFailed + if err := c.Status().Update(t.Context(), pod); err != nil { + t.Fatal(err) + } + second := pod.DeepCopy() + second.ResourceVersion = "" + second.Name = "restore-two" + second.UID = "restore-two" + second.Status.Phase = corev1.PodRunning + second.Status.Conditions[0].Status = corev1.ConditionFalse + if err := c.Create(t.Context(), second); err != nil { + t.Fatal(err) + } + r = &RestoreReconciler{Client: c, APIReader: c, Scheme: c.Scheme()} + if err := c.Get(t.Context(), client.ObjectKeyFromObject(rs), rs); err != nil { + t.Fatal(err) + } + if ready, err := r.authorizeRecoveryRestore(t.Context(), rs, b, gs, job); err != nil || ready { + t.Fatalf("unprepared retry: %v %v", ready, err) + } + if err := c.Get(t.Context(), key, &gate); err != nil { + t.Fatal(err) + } + if gate.Data[backupStartTokenKey] != "" { + t.Fatal("restore retry inherited prior authorization") + } + second.Status.Conditions[0].Status = corev1.ConditionTrue + if err := c.Status().Update(t.Context(), second); err != nil { + t.Fatal(err) + } + if ready, err := r.authorizeRecoveryRestore(t.Context(), rs, b, gs, job); err != nil || !ready { + t.Fatalf("retry ready: %v %v", ready, err) + } + if err := c.Get(t.Context(), key, &gate); err != nil { + t.Fatal(err) + } + if gate.Data[backupStartTokenKey] == oldToken || !strings.Contains(gate.Data[backupStartTokenKey], "restore-two") || rs.Annotations[restoreDeadlineAnnotation] != deadline { + t.Fatal("retry lost Pod binding or reset deadline") + } + if result, err := r.fail(t.Context(), rs, "cancelled during copy"); err != nil || result.RequeueAfter == 0 { + t.Fatalf("cancel failed to drain: %v %v", result, err) + } + if err := c.Get(t.Context(), key, &gate); err != nil { + t.Fatal(err) + } + if gate.Data[backupStartTokenKey] != "" { + t.Fatal("failed restore retained token") + } + if err := c.Get(t.Context(), client.ObjectKeyFromObject(gs), gs); err != nil { + t.Fatal(err) + } + if gs.Annotations[restoreGuardAnnotation] == "" { + t.Fatal("failed restore released live writer") + } + if err := c.Delete(t.Context(), second); err != nil { + t.Fatal(err) + } + var ss appsv1.StatefulSet + if err := c.Get(t.Context(), client.ObjectKeyFromObject(gs), &ss); err != nil { + t.Fatal(err) + } + zero := int32(0) + ss.Spec.Replicas = &zero + if err := c.Update(t.Context(), &ss); err != nil { + t.Fatal(err) + } + for i := 0; i < 3; i++ { + if _, err := r.cleanupRestore(t.Context(), rs); err != nil { + t.Fatal(err) + } + } + if err := c.Get(t.Context(), client.ObjectKeyFromObject(gs), gs); err != nil { + t.Fatal(err) + } + if !gs.Spec.Suspend || gs.Annotations[restoreGuardAnnotation] != "" { + t.Fatal("failed restore did not stay suspended after worker drain") + } +} + +func TestRecoveryQuiesceOutcomeFinalizedOnlyBeforeFirstAuthorization(t *testing.T) { + c, _, b, _ := recoveryFixture(t) + b.Spec.PowerPolicy = "preserve" + b.Spec.Quiesce = true + b.Annotations[annoQuiesceAttempted] = "true" + if err := c.Update(t.Context(), b); err != nil { + t.Fatal(err) + } + before := b.Status.CaptureManifest.DeepCopy() + r := &BackupReconciler{Client: c, APIReader: c} + if err := r.finalizeBackupCaptureOutcome(t.Context(), b); err != nil { + t.Fatal(err) + } + if b.Status.CaptureManifest.QuiesceOutcome != "confirmed" { + t.Fatal("capture retained pre-RPC quiesce outcome") + } + for i, v := range before.Volumes { + if v.ClaimUID != b.Status.CaptureManifest.Volumes[i].ClaimUID || v.SnapshotRoot != b.Status.CaptureManifest.Volumes[i].SnapshotRoot { + t.Fatal("quiesce finalization rewrote coverage") + } + } + b.Annotations[backupAuthorizedWorkerAnnotation] = "worker" + b.Annotations[annoQuiesceAttempted] = "unsupported" + if err := c.Update(t.Context(), b); err != nil { + t.Fatal(err) + } + if err := r.finalizeBackupCaptureOutcome(t.Context(), b); err != nil { + t.Fatal(err) + } + if b.Status.CaptureManifest.QuiesceOutcome != "confirmed" { + t.Fatal("post-authorization outcome rewrote capture history") + } + if backupConsistencyCondition(b).Status != metav1.ConditionFalse { + t.Fatal("live multi-volume copy overclaimed consistency") + } +} + +func TestRestoreFailurePreservesExactWorkerDiagnosticsBeforeCleanup(t *testing.T) { + c, _, _, rs := recoveryFixture(t) + r := &RestoreReconciler{Client: c, APIReader: c, Scheme: c.Scheme()} + job := &batchv1.Job{ObjectMeta: metav1.ObjectMeta{Name: "restore-worker", Namespace: rs.Namespace, UID: "worker-job"}, Status: batchv1.JobStatus{Conditions: []batchv1.JobCondition{{Type: batchv1.JobFailed, Status: corev1.ConditionTrue, Reason: "BackoffLimitExceeded", Message: "Job has reached its retry limit"}}}} + controller := true + for _, tc := range []struct{ name, owner, message string }{{"ours", "worker-job", "Fatal: repository https://user:secret@example.invalid failed"}, {"foreign", "replaced-job", "foreign diagnostic must not leak"}} { + pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: tc.name, Namespace: rs.Namespace, OwnerReferences: []metav1.OwnerReference{{APIVersion: "batch/v1", Kind: "Job", Name: job.Name, UID: types.UID(tc.owner), Controller: &controller}}}, Status: corev1.PodStatus{ContainerStatuses: []corev1.ContainerStatus{{Name: "restic", State: corev1.ContainerState{Terminated: &corev1.ContainerStateTerminated{ExitCode: 1, Reason: "Error", Message: tc.message}}}}}} + if err := c.Create(t.Context(), pod); err != nil { + t.Fatal(err) + } + } + message := r.restoreJobFailure(t.Context(), job).Error() + if !strings.Contains(message, "BackoffLimitExceeded") || !strings.Contains(message, "exited 1 (Error)") || strings.Contains(message, "secret") || strings.Contains(message, "repository") || strings.Contains(message, "foreign diagnostic") { + t.Fatalf("incomplete or foreign failure evidence: %s", message) + } + if _, err := r.fail(t.Context(), rs, message); err != nil { + t.Fatal(err) + } + if err := c.Get(t.Context(), client.ObjectKeyFromObject(rs), rs); err != nil { + t.Fatal(err) + } + if rs.Status.Phase != v1.RestorePhaseFailed || rs.Status.Message != message { + t.Fatalf("cleanup lost persisted worker failure: %+v", rs.Status) + } +} diff --git a/operator/internal/controller/recovery_manifest.go b/operator/internal/controller/recovery_manifest.go new file mode 100644 index 000000000..1cc14194f --- /dev/null +++ b/operator/internal/controller/recovery_manifest.go @@ -0,0 +1,212 @@ +package controller + +import ( + "context" + "crypto/sha256" + "encoding/json" + "fmt" + "path" + "reflect" + "regexp" + "sort" + + v1 "github.com/GameplanePanel/gameplane/operator/api/v1alpha1" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +var recoveryLogicalName = regexp.MustCompile(`^[a-z0-9][a-z0-9-]{0,62}$`) + +func validateRecoveryManifest(m *v1.BackupCaptureManifest) error { + if m == nil || m.State != "Recorded" || m.FormatVersion != 2 || m.VolumePolicy != "all-data" { + return fmt.Errorf("all-data requires a recorded v2 manifest") + } + claims, roots, logical := map[string]bool{}, map[string]bool{}, map[string]bool{} + primary := false + for _, v := range m.Volumes { + if v.ClaimName == "" || !recoveryLogicalName.MatchString(v.LogicalName) || claims[v.ClaimName] || logical[v.LogicalName] { + return fmt.Errorf("ambiguous recovery claim or logical identity %q", v.ClaimName) + } + claims[v.ClaimName], logical[v.LogicalName] = true, true + if len(v.MountPaths) == 0 { + return fmt.Errorf("claim %s has no frozen mount path", v.ClaimName) + } + for _, mount := range v.MountPaths { + if !path.IsAbs(mount) || path.Clean(mount) != mount { + return fmt.Errorf("invalid original mount path") + } + } + if v.Included { + if v.ClaimUID == "" || v.Classification != "data" || v.SnapshotRoot != "/volumes/"+v.LogicalName || roots[v.SnapshotRoot] { + return fmt.Errorf("invalid recovery root for %s", v.ClaimName) + } + roots[v.SnapshotRoot] = true + primary = primary || v.LogicalName == "primary" + } else if v.Classification != "rebuildable" || v.RecoveryPrerequisite == "" || v.SnapshotRoot != "" { + return fmt.Errorf("unclassified or ambiguous excluded claim %s", v.ClaimName) + } + } + if !primary { + return fmt.Errorf("recovery manifest lacks primary data") + } + return nil +} + +// Read the frozen StatefulSet rather than rebuilding a layout from today's catalog. +func recoveryLayout(ctx context.Context, reader client.Reader, gs *v1.GameServer) (*v1.BackupCaptureManifest, error) { + tmpl, err := effectiveGameTemplate(ctx, reader, gs) + if err != nil { + return nil, err + } + _, revision, err := templateSnapshot(tmpl) + if err != nil { + return nil, err + } + ver, err := resolveVersion(gs, tmpl) + if err != nil { + return nil, err + } + var ss appsv1.StatefulSet + if err := reader.Get(ctx, client.ObjectKeyFromObject(gs), &ss); err != nil { + return nil, err + } + if !metav1.IsControlledBy(&ss, gs) { + return nil, fmt.Errorf("recovery workload has foreign owner") + } + m := &v1.BackupCaptureManifest{State: "Recorded", FormatVersion: 2, VolumePolicy: "all-data", ServerUID: string(gs.UID), Strategy: "restic-snapshot", LayoutSource: "StatefulSet", TemplateName: tmpl.Name, TemplateRevision: revision, GameVersion: gs.Spec.Version} + if ver != nil { + m.GameVersion = ver.ID + } + declared := map[string]v1.BackupVolumeCoverage{gs.Name + "-data": {LogicalName: "primary", Classification: "data", MountPaths: []string{effectiveMountPath(tmpl)}}} + for _, extra := range effectiveExtraVolumes(gs, tmpl) { + role := extra.BackupRole + if role == "" { + role = "unclassified" + } + declared[extraPVCName(gs, extra.Name)] = v1.BackupVolumeCoverage{LogicalName: extraVolumePodName(extra.Name), Classification: role, MountPaths: []string{extra.MountPath}} + } + if volume := modVolume(gs, tmpl, ver); volume != nil { + mount := modVolumeMount(tmpl, ver) + declared[volume.PersistentVolumeClaim.ClaimName] = v1.BackupVolumeCoverage{LogicalName: volume.Name, Classification: "data", MountPaths: []string{mount.MountPath}} + } + observed := map[string][]string{} + podNames := map[string]string{} + for _, v := range ss.Spec.Template.Spec.Volumes { + if v.PersistentVolumeClaim != nil { + podNames[v.Name] = v.PersistentVolumeClaim.ClaimName + observed[v.PersistentVolumeClaim.ClaimName] = nil + } + } + for _, c := range ss.Spec.Template.Spec.Containers { + if c.Name != "game" { + continue + } + for _, mount := range c.VolumeMounts { + if claim := podNames[mount.Name]; claim != "" { + if mount.SubPath != "" || mount.SubPathExpr != "" { + return nil, fmt.Errorf("all-data does not support claim subpaths") + } + observed[claim] = append(observed[claim], mount.MountPath) + } + } + } + var claims corev1.PersistentVolumeClaimList + if err := reader.List(ctx, &claims, client.InNamespace(gs.Namespace)); err != nil { + return nil, err + } + uids := map[string]string{} + for _, claim := range claims.Items { + if claim.DeletionTimestamp.IsZero() { + uids[claim.Name] = string(claim.UID) + } + if metav1.IsControlledBy(&claim, gs) { + if _, ok := observed[claim.Name]; !ok { + observed[claim.Name] = nil + } + } + } + for name, mounts := range observed { + v, ok := declared[name] + if !ok { + return nil, fmt.Errorf("unclassified observed claim %s", name) + } + sort.Strings(mounts) + if !reflect.DeepEqual(mounts, v.MountPaths) { + return nil, fmt.Errorf("claim %s frozen mount paths differ from declaration", name) + } + } + for name, v := range declared { + if !reflect.DeepEqual(observed[name], v.MountPaths) { + return nil, fmt.Errorf("required claim %s is absent from frozen workload", name) + } + v.ClaimName, v.ClaimUID = name, uids[name] + switch v.Classification { + case "data": + v.Included = true + v.SnapshotRoot = "/volumes/" + v.LogicalName + v.CoverageReason = "IncludedData" + case "rebuildable": + v.CoverageReason = "RebuildRequired" + v.RecoveryPrerequisite = "If its contents are missing or incompatible, reconstruct this excluded volume from the declared module and installation sources before starting the recovered game." + default: + return nil, fmt.Errorf("claim %s must declare backupRole data or rebuildable", name) + } + m.Volumes = append(m.Volumes, v) + } + sort.Slice(m.Volumes, func(i, j int) bool { return m.Volumes[i].LogicalName < m.Volumes[j].LogicalName }) + return m, validateRecoveryManifest(m) +} + +func recoveryDigest(m *v1.BackupCaptureManifest) string { + data, _ := json.Marshal(m) + return fmt.Sprintf("%x", sha256.Sum256(data)) +} + +func sameRecoveryLayout(a, b *v1.BackupCaptureManifest) bool { + return a.TemplateName == b.TemplateName && a.TemplateRevision == b.TemplateRevision && a.GameVersion == b.GameVersion && a.ServerUID == b.ServerUID && reflect.DeepEqual(a.Volumes, b.Volumes) +} +func (r *BackupReconciler) validateBackupRecoveryLayout(ctx context.Context, b *v1.Backup, gs *v1.GameServer) error { + if b.Spec.VolumePolicy != "all-data" { + return nil + } + if err := validateRecoveryManifest(b.Status.CaptureManifest); err != nil { + return err + } + current, err := recoveryLayout(ctx, r.backupAPIReader(), gs) + if err != nil { + return err + } + if !sameRecoveryLayout(b.Status.CaptureManifest, current) { + return fmt.Errorf("capture layout or claim identity changed after admission") + } + return nil +} +func podMountsRecoveryClaims(pod *corev1.Pod, volumes []v1.BackupVolumeCoverage) bool { + for _, v := range pod.Spec.Volumes { + if v.PersistentVolumeClaim == nil { + continue + } + for _, want := range volumes { + if want.Included && want.ClaimName == v.PersistentVolumeClaim.ClaimName { + return true + } + } + } + return false +} + +// Replace only persistent mounts, leaving repository credentials and scratch intact. +func recoveryWorkerVolumes(spec *corev1.PodSpec, volumes []v1.BackupVolumeCoverage, readOnly bool) { + spec.Volumes = []corev1.Volume{{Name: "cache", VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}}} + spec.Containers[0].VolumeMounts = []corev1.VolumeMount{{Name: "cache", MountPath: "/tmp"}} + for i, v := range volumes { + if !v.Included { + continue + } + name := fmt.Sprintf("root-%d", i) + spec.Volumes = append(spec.Volumes, corev1.Volume{Name: name, VolumeSource: corev1.VolumeSource{PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ClaimName: v.ClaimName, ReadOnly: readOnly}}}) + spec.Containers[0].VolumeMounts = append(spec.Containers[0].VolumeMounts, corev1.VolumeMount{Name: name, MountPath: v.SnapshotRoot, ReadOnly: readOnly}) + } +} diff --git a/operator/internal/controller/recovery_policy_envtest_test.go b/operator/internal/controller/recovery_policy_envtest_test.go new file mode 100644 index 000000000..fcb253980 --- /dev/null +++ b/operator/internal/controller/recovery_policy_envtest_test.go @@ -0,0 +1,31 @@ +//go:build envtest + +package controller + +import ( + "testing" + + v1 "github.com/GameplanePanel/gameplane/operator/api/v1alpha1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func TestCRDValidation_RecoveryPoliciesRejectCSI(t *testing.T) { + ns := newNamespace(t) + for _, tc := range []struct{ name, power, volume string }{{"stopped", "stop-copy-restart", "primary-only"}, {"all-data", "preserve", "all-data"}} { + t.Run(tc.name, func(t *testing.T) { + b := &v1.Backup{ObjectMeta: metav1.ObjectMeta{Name: tc.name, Namespace: ns}, Spec: v1.BackupSpec{ServerRef: v1.LocalObjectRef{Name: "game"}, Strategy: "volume-snapshot", PowerPolicy: tc.power, VolumePolicy: tc.volume}} + if err := k8sClient.Create(t.Context(), b); !apierrors.IsInvalid(err) { + t.Fatalf("CSI policy accepted: %v", err) + } + schedule := &v1.BackupSchedule{ObjectMeta: metav1.ObjectMeta{Name: tc.name, Namespace: ns}, Spec: v1.BackupScheduleSpec{ServerRef: v1.LocalObjectRef{Name: "game"}, Strategy: "volume-snapshot", Schedule: "* * * * *", PowerPolicy: tc.power, VolumePolicy: tc.volume}} + if err := k8sClient.Create(t.Context(), schedule); !apierrors.IsInvalid(err) { + t.Fatalf("CSI schedule accepted: %v", err) + } + }) + } + b := &v1.Backup{ObjectMeta: metav1.ObjectMeta{Name: "restic-all", Namespace: ns}, Spec: v1.BackupSpec{ServerRef: v1.LocalObjectRef{Name: "game"}, RepoRef: &v1.SecretKeySelector{Name: "repo", Key: "repo"}, PowerPolicy: "stop-copy-restart", VolumePolicy: "all-data"}} + if err := k8sClient.Create(t.Context(), b); err != nil { + t.Fatalf("valid recovery policy rejected: %v", err) + } +} diff --git a/operator/internal/controller/recovery_test.go b/operator/internal/controller/recovery_test.go new file mode 100644 index 000000000..b5a811a33 --- /dev/null +++ b/operator/internal/controller/recovery_test.go @@ -0,0 +1,147 @@ +package controller + +import ( + "path" + "strings" + "testing" + + v1 "github.com/GameplanePanel/gameplane/operator/api/v1alpha1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// Runtime execution is CI-only; these regression sources precede implementation. +func TestRecoveryRejectsUnsafeRoots(t *testing.T) { + for _, root := range []string{"/", "/data", "/volumes/../data", "/volumes/a/b", "/volumes/*", "/volumes/extra;echo"} { + t.Run(root, func(t *testing.T) { + m := &v1.BackupCaptureManifest{State: "Recorded", FormatVersion: 2, VolumePolicy: "all-data", Volumes: []v1.BackupVolumeCoverage{{ClaimName: "s-data", ClaimUID: "uid", LogicalName: "primary", SnapshotRoot: root, Included: true, Classification: "data", MountPaths: []string{"/data"}}}} + if err := validateRecoveryManifest(m); err == nil { + t.Fatal("unsafe root admitted") + } + }) + } +} +func TestRecoveryRejectsDuplicateClaims(t *testing.T) { + m := &v1.BackupCaptureManifest{State: "Recorded", FormatVersion: 2, VolumePolicy: "all-data", Volumes: []v1.BackupVolumeCoverage{ + {ClaimName: "s-data", ClaimUID: "uid", LogicalName: "primary", SnapshotRoot: "/volumes/primary", Included: true, Classification: "data", MountPaths: []string{"/data"}}, + {ClaimName: "s-data", ClaimUID: "uid", LogicalName: "extra-world", SnapshotRoot: "/volumes/extra-world", Included: true, Classification: "data", MountPaths: []string{"/world"}}, + }} + if err := validateRecoveryManifest(m); err == nil { + t.Fatal("duplicate claim admitted") + } +} +func TestRecoveryWorkersUseDisjointExplicitRoots(t *testing.T) { + b := &v1.Backup{Spec: v1.BackupSpec{RepoRef: &v1.SecretKeySelector{Name: "repo"}, VolumePolicy: "all-data"}, Status: v1.BackupStatus{CaptureManifest: &v1.BackupCaptureManifest{Volumes: []v1.BackupVolumeCoverage{ + {ClaimName: "world", LogicalName: "primary", SnapshotRoot: "/volumes/primary", Included: true}, + {ClaimName: "mods", LogicalName: "mods-v1", SnapshotRoot: "/volumes/mods-v1", Included: true}, + {ClaimName: "cache", LogicalName: "extra-cache", RecoveryPrerequisite: "rebuild", Included: false}, + }}}} + spec := (&BackupReconciler{}).buildBackupPodSpec(b, &v1.GameTemplate{}) + if strings.Contains(strings.Join(spec.Containers[0].Args, " "), "*") { + t.Fatal("wildcard capture") + } + seen := map[string]bool{} + for _, m := range spec.Containers[0].VolumeMounts { + if m.Name != "cache" { + if !m.ReadOnly { + t.Fatal("writable capture") + } + seen[m.MountPath] = true + } + } + if !seen["/volumes/primary"] || !seen["/volumes/mods-v1"] || seen["/data"] { + t.Fatalf("unexpected mounts: %v", seen) + } + for _, v := range spec.Volumes { + if v.PersistentVolumeClaim != nil && v.PersistentVolumeClaim.ClaimName == "cache" { + t.Fatal("rebuildable mounted") + } + } +} +func TestStoppedBackupPowerBaseline(t *testing.T) { + for _, tc := range []struct { + name string + suspend bool + idle string + running bool + }{{"running", false, "", true}, {"suspended", true, "", false}, {"idle", false, "2026-01-01", false}} { + t.Run(tc.name, func(t *testing.T) { + gs := &v1.GameServer{ObjectMeta: metav1.ObjectMeta{Generation: 4, Annotations: map[string]string{IdleAsleepSinceAnnotation: tc.idle}}, Spec: v1.GameServerSpec{Suspend: tc.suspend}} + debt := newStoppedBackup(gs) + if debt.WasRunning != tc.running { + t.Fatal("incorrect admitted power") + } + gs.Spec.Suspend = true + if stoppedBackupMayResume(gs, debt) { + t.Fatal("overrode user suspension") + } + gs.Spec.Suspend = false + gs.Annotations[IdleAsleepSinceAnnotation] = "later" + if stoppedBackupMayResume(gs, debt) { + t.Fatal("overrode idle intent") + } + }) + } +} +func TestSelectedClaimDrainMatchesForeignClaims(t *testing.T) { + pod := &corev1.Pod{Spec: corev1.PodSpec{Volumes: []corev1.Volume{{Name: "foreign", VolumeSource: corev1.VolumeSource{PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ClaimName: "external-world"}}}}}} + if !podMountsRecoveryClaims(pod, []v1.BackupVolumeCoverage{{ClaimName: "external-world", Included: true}}) { + t.Fatal("foreign claim writer ignored") + } +} + +// Restic restores metadata on every ancestor of selected files, even when the +// ancestor is not itself included. Selecting the /volumes subtree makes that +// ancestor the metadata-free restore target instead of a read-only image path. +func TestRecoveryRestoreScopesArchiveSubtreeToMountedClaims(t *testing.T) { + volumes := []v1.BackupVolumeCoverage{ + {ClaimName: "world", LogicalName: "primary", SnapshotRoot: "/volumes/primary", Included: true}, + {ClaimName: "mods", LogicalName: "mods-v1", SnapshotRoot: "/volumes/mods-v1", Included: true}, + {ClaimName: "cache", LogicalName: "extra-cache", Included: false}, + } + rs := &v1.Restore{Status: v1.RestoreStatus{SnapshotID: "0123456789abcdef", RecoveryPlan: &v1.RestoreRecoveryPlan{Volumes: volumes}}} + src := &v1.Backup{Spec: v1.BackupSpec{RepoRef: &v1.SecretKeySelector{Name: "repo"}}} + spec := (&RestoreReconciler{}).buildRestorePodSpec(rs, src, &v1.GameTemplate{}) + worker := spec.Containers[0] + if worker.TerminationMessagePolicy != corev1.TerminationMessageFallbackToLogsOnError { + t.Fatal("restore failure would lose worker stderr before cleanup") + } + if worker.SecurityContext.ReadOnlyRootFilesystem == nil || !*worker.SecurityContext.ReadOnlyRootFilesystem { + t.Fatal("restore weakened container root filesystem protection") + } + if worker.Args[1] != rs.Status.SnapshotID+":/volumes" { + t.Fatalf("restore would apply archive ancestor metadata: %v", worker.Args) + } + target := "" + includes := []string{} + deleteEnabled := false + for i, arg := range worker.Args { + switch arg { + case "--target": + target = worker.Args[i+1] + case "--include": + includes = append(includes, worker.Args[i+1]) + case "--delete": + deleteEnabled = true + } + } + if target != "/volumes" || len(includes) != 2 || !deleteEnabled { + t.Fatalf("incorrect restore scope: %v", worker.Args) + } + mounted := map[string]bool{} + for _, mount := range worker.VolumeMounts { + if mount.MountPath != "/tmp" { + mounted[mount.MountPath] = !mount.ReadOnly + } + } + for _, include := range includes { + if strings.ContainsAny(include, "*?[") || !mounted[path.Join(target, include)] { + t.Fatalf("include %q does not resolve to an exact writable PVC: %v", include, mounted) + } + } + for _, volume := range spec.Volumes { + if volume.PersistentVolumeClaim != nil && volume.PersistentVolumeClaim.ClaimName == "cache" { + t.Fatal("restore mounted excluded rebuildable claim") + } + } +} diff --git a/operator/internal/controller/restore_controller.go b/operator/internal/controller/restore_controller.go index 41839e2a6..68c0eeb47 100644 --- a/operator/internal/controller/restore_controller.go +++ b/operator/internal/controller/restore_controller.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "strings" "time" batchv1 "k8s.io/api/batch/v1" @@ -104,6 +105,21 @@ func (r *RestoreReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct } return ctrl.Result{RequeueAfter: 15 * time.Second}, nil } + all, modeErr := recoveryCaptureMode(&src) + if modeErr != nil { + return r.fail(ctx, &rs, modeErr.Error()) + } + if rs.Annotations == nil { + rs.Annotations = map[string]string{} + } + mode := "legacy-primary" + if all { + mode = "all-data-v2" + } + rs.Annotations[restoreFormatAnnotation] = mode + if err := r.Update(ctx, &rs); err != nil { + return ctrl.Result{}, err + } rs.Status.SnapshotID = src.Status.SnapshotID if src.Spec.Strategy == "volume-snapshot" { // Volume-snapshot restores stand up a brand-new server rather @@ -134,6 +150,12 @@ func (r *RestoreReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct // Binding the target does not change the restore strategy. CSI's own // preparation path checks that binding before awaiting startup, repairing // references and enforcing the deadline on every subsequent reconcile. + if _, err := recoveryCaptureMode(&src); err != nil { + return r.fail(ctx, &rs, err.Error()) + } + if rs.Annotations[restoreFormatAnnotation] == "all-data-v2" && src.Spec.Strategy == "volume-snapshot" { + return r.fail(ctx, &rs, "recovery mode changed") + } if src.Spec.Strategy == "volume-snapshot" { if src.Annotations[backupSnapshotAbortAnnotation] == "true" || src.Status.Phase != gameplanev1alpha1.BackupPhaseSucceeded || src.Status.SnapshotID != rs.Status.SnapshotID { return r.fail(ctx, &rs, "source CSI capture is no longer valid") @@ -152,6 +174,11 @@ func (r *RestoreReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct if !gs.DeletionTimestamp.IsZero() { return r.fail(ctx, &rs, "target server is being deleted") } + if changed, err := r.preflightRecovery(ctx, &rs, &src, &gs); err != nil { + return r.fail(ctx, &rs, err.Error()) + } else if changed { + return ctrl.Result{RequeueAfter: reconcileRetryInterval}, nil + } changed, err := r.bindRestoreTarget(ctx, &rs, &gs) if err != nil { return ctrl.Result{}, err @@ -192,7 +219,7 @@ func (r *RestoreReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct tmpl := *effective if rs.Status.Phase == gameplanev1alpha1.RestorePhaseSuspending { - if gs.Status.Phase != gameplanev1alpha1.GameServerPhaseSuspended && + if rs.Status.RecoveryPlan == nil && gs.Status.Phase != gameplanev1alpha1.GameServerPhaseSuspended && gs.Status.Phase != gameplanev1alpha1.GameServerPhaseStopped { return ctrl.Result{RequeueAfter: 5 * time.Second}, nil } @@ -234,11 +261,22 @@ func (r *RestoreReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct if src.Spec.RepoRef == nil { return r.fail(ctx, &rs, "source backup has no restic repository") } + if _, err := r.preflightRecovery(ctx, &rs, &src, &gs); err != nil { + return r.fail(ctx, &rs, err.Error()) + } + if rs.Status.RecoveryPlan != nil { + if err := r.recoveryRestoreWritersDrained(ctx, &rs, &gs); err != nil { + return r.fail(ctx, &rs, err.Error()) + } + } job.Spec.BackoffLimit = &backoff job.Spec.ActiveDeadlineSeconds = &deadline job.Annotations = map[string]string{restoreTargetAnnotation: rs.Annotations[restoreTargetAnnotation]} job.Spec.Template.Labels = map[string]string{backupRestoreJobLabel: backupRestoreJobValue, restoreUIDLabel: string(rs.UID)} job.Spec.Template.Spec = r.buildRestorePodSpec(&rs, &src, &tmpl) + if err := r.prepareRecoveryRestorePod(ctx, &rs, &job.Spec.Template.Spec); err != nil { + return r.fail(ctx, &rs, err.Error()) + } job.Spec.Template.Spec.RestartPolicy = corev1.RestartPolicyNever if err := controllerutil.SetControllerReference(&rs, job, r.Scheme); err != nil { return ctrl.Result{}, err @@ -257,6 +295,11 @@ func (r *RestoreReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct return r.fail(ctx, &rs, "restore Job does not belong to this Restore and target") } + if ready, err := r.authorizeRecoveryRestore(ctx, &rs, &src, &gs, job); err != nil { + return r.fail(ctx, &rs, err.Error()) + } else if !ready { + return ctrl.Result{RequeueAfter: 2 * time.Second}, nil + } switch { case job.Status.Succeeded > 0: busy, err := r.restoreWorkersLive(ctx, rs.Namespace, restoreIdentity{Name: rs.Name, UID: rs.UID}, false) @@ -291,7 +334,7 @@ func (r *RestoreReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct case jobPermanentlyFailed(job): // Leave the server suspended; surface the failure. - return r.fail(ctx, &rs, "restore job reported Failed") + return r.fail(ctx, &rs, r.restoreJobFailure(ctx, job).Error()) default: if rs.Status.StartTime == nil && job.Status.StartTime != nil { @@ -304,6 +347,44 @@ func (r *RestoreReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct } } +// Preserve worker failure evidence in Restore status before cleanup deletes +// the Job and its Pods. Only diagnostics from this exact Job are admitted. +// Raw termination messages can contain repository credentials and remain on Pods. +func (r *RestoreReconciler) restoreJobFailure(ctx context.Context, job *batchv1.Job) error { + details := []string{"restore job reported Failed"} + for _, condition := range job.Status.Conditions { + if condition.Type == batchv1.JobFailed && condition.Status == corev1.ConditionTrue { + details = append(details, condition.Reason) + } + } + if job.UID == "" { + return errors.New(strings.Join(details, "; ")) + } + var pods corev1.PodList + if err := r.apiReader().List(ctx, &pods, client.InNamespace(job.Namespace)); err != nil { + details = append(details, "worker diagnostics unavailable") + return errors.New(strings.Join(details, "; ")) + } + for i := range pods.Items { + pod := &pods.Items[i] + if !metav1.IsControlledBy(pod, job) { + continue + } + statuses := append(append([]corev1.ContainerStatus{}, pod.Status.InitContainerStatuses...), pod.Status.ContainerStatuses...) + for _, status := range statuses { + terminated := status.State.Terminated + if terminated == nil || terminated.ExitCode == 0 { + continue + } + details = append(details, fmt.Sprintf("Pod %s container %s exited %d (%s)", pod.Name, status.Name, terminated.ExitCode, terminated.Reason)) + if len(details) >= 5 { + return errors.New(strings.Join(details, "; ")) + } + } + } + return errors.New(strings.Join(details, "; ")) +} + func (r *RestoreReconciler) fail(ctx context.Context, rs *gameplanev1alpha1.Restore, msg string) (ctrl.Result, error) { now := metav1.Now() rs.Status.Phase = gameplanev1alpha1.RestorePhaseFailed @@ -340,12 +421,13 @@ func (r *RestoreReconciler) buildRestorePodSpec( roRootFS := true noPrivEsc := false uid := int64(65532) - return corev1.PodSpec{ + spec := corev1.PodSpec{ SecurityContext: buildBackupPodSecurityContext(tmpl), Containers: []corev1.Container{{ - Name: "restic", - Image: resticImageOrDefault(r.ResticImage), - Resources: auxiliaryResources(r.BackupResources, "100m", "128Mi"), + Name: "restic", + TerminationMessagePolicy: corev1.TerminationMessageFallbackToLogsOnError, + Image: resticImageOrDefault(r.ResticImage), + Resources: auxiliaryResources(r.BackupResources, "100m", "128Mi"), // --target / restores each snapshot entry at its original // absolute path. The companion backup runs `restic backup // /data`, so the snapshot tree is rooted at /data/...; with @@ -406,4 +488,18 @@ func (r *RestoreReconciler) buildRestorePodSpec( }, }, } + if rs.Status.RecoveryPlan != nil { + recoveryWorkerVolumes(&spec, rs.Status.RecoveryPlan.Volumes, false) + // Restore the archive subtree itself so restic never applies the + // archived /volumes ancestor metadata to the read-only container root. + // Includes are relative to that subtree; physical PVC paths stay fixed. + args := []string{"restore", rs.Status.SnapshotID + ":/volumes", "--target", "/volumes", "--delete", "--retry-lock", "10m"} + for _, v := range rs.Status.RecoveryPlan.Volumes { + if v.Included { + args = append(args, "--include", "/"+v.LogicalName) + } + } + spec.Containers[0].Args = args + } + return spec } diff --git a/operator/internal/controller/restore_lifecycle.go b/operator/internal/controller/restore_lifecycle.go index 6e392cbc2..78e4bf462 100644 --- a/operator/internal/controller/restore_lifecycle.go +++ b/operator/internal/controller/restore_lifecycle.go @@ -191,6 +191,12 @@ func (r *RestoreReconciler) restoreTargetStopped(ctx context.Context, gs *gamepl // available for their logs. Cancellation uses foreground deletion and UID // preconditions, retaining the guard until both Jobs and live pods have drained. func (r *RestoreReconciler) restoreWorkersLive(ctx context.Context, namespace string, identity restoreIdentity, cancel bool) (bool, error) { + if cancel { + if err := r.revokeRecoveryRestore(ctx, &gameplanev1alpha1.Restore{ObjectMeta: metav1.ObjectMeta{Namespace: namespace, Name: identity.Name, UID: identity.UID}}); err != nil { + return true, err + } + } + var jobs batchv1.JobList if err := r.apiReader().List(ctx, &jobs, client.InNamespace(namespace)); err != nil { return false, err @@ -338,7 +344,9 @@ func (r *RestoreReconciler) cleanupRestore(ctx context.Context, rs *gameplanev1a } delete(gs.Annotations, restoreGuardAnnotation) // Partial or canceled restores stay suspended for investigation. - gs.Spec.Suspend = !success + if !success || rs.Status.RecoveryPlan == nil { + gs.Spec.Suspend = !success + } if err := r.Update(ctx, &gs); err != nil { return ctrl.Result{}, err } diff --git a/operator/internal/controller/restore_recovery.go b/operator/internal/controller/restore_recovery.go new file mode 100644 index 000000000..173eb0c27 --- /dev/null +++ b/operator/internal/controller/restore_recovery.go @@ -0,0 +1,276 @@ +package controller + +import ( + "context" + "crypto/sha256" + "fmt" + "reflect" + "strconv" + "time" + + v1 "github.com/GameplanePanel/gameplane/operator/api/v1alpha1" + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" +) + +const captureFormatAnnotation = "backup.gameplane.local/capture-format" +const restoreFormatAnnotation = "restore.gameplane.local/capture-format" +const restoreDeadlineAnnotation = "restore.gameplane.local/copy-deadline" +const restoreAuthorizedAnnotation = "restore.gameplane.local/authorized-worker" + +func recoveryCaptureMode(src *v1.Backup) (bool, error) { + m := src.Status.CaptureManifest + all := src.Spec.VolumePolicy == "all-data" || src.Annotations[captureFormatAnnotation] == "all-data-v2" || (m != nil && (m.FormatVersion != 0 || m.VolumePolicy == "all-data")) + if all { + if src.Spec.Strategy == "volume-snapshot" { + return false, fmt.Errorf("CSI cannot restore an all-data archive") + } + if err := validateRecoveryManifest(m); err != nil { + return false, err + } + if m.TemplateName == "" || m.TemplateRevision == "" || m.ServerUID == "" { + return false, fmt.Errorf("recovery manifest lacks provenance") + } + return true, nil + } + if m != nil { + for _, v := range m.Volumes { + if v.SnapshotRoot != "" || v.LogicalName != "" || (v.Included && v.CoverageReason != "PrimaryDataClaim") { + return false, fmt.Errorf("contradictory legacy manifest") + } + } + } + return false, nil +} + +func (r *RestoreReconciler) preflightRecovery(ctx context.Context, rs *v1.Restore, src *v1.Backup, gs *v1.GameServer) (bool, error) { + all, err := recoveryCaptureMode(src) + if err != nil { + return false, err + } + if rs.Annotations[restoreFormatAnnotation] == "all-data-v2" && !all { + return false, fmt.Errorf("source recovery manifest was removed or downgraded") + } + if !all { + if rs.Status.RecoveryPlan != nil { + return false, fmt.Errorf("recovery plan cannot downgrade to legacy") + } + return false, nil + } + if src.Status.SnapshotID != rs.Status.SnapshotID || src.Status.Phase != v1.BackupPhaseSucceeded { + return false, fmt.Errorf("source snapshot changed or is invalid") + } + current, err := recoveryLayout(ctx, r.apiReader(), gs) + if err != nil { + return false, err + } + saved := src.Status.CaptureManifest + if current.TemplateName != saved.TemplateName || current.TemplateRevision != saved.TemplateRevision || current.GameVersion != saved.GameVersion { + return false, fmt.Errorf("target template or version differs from capture provenance") + } + sourceByLogical := map[string]v1.BackupVolumeCoverage{} + for _, v := range saved.Volumes { + sourceByLogical[v.LogicalName] = v + } + if len(current.Volumes) != len(saved.Volumes) { + return false, fmt.Errorf("target recovery volume set differs") + } + for _, target := range current.Volumes { + original, ok := sourceByLogical[target.LogicalName] + if !ok || target.Included != original.Included || target.Classification != original.Classification || !reflect.DeepEqual(target.MountPaths, original.MountPaths) || target.SnapshotRoot != original.SnapshotRoot { + return false, fmt.Errorf("target logical layout differs for %s", target.LogicalName) + } + if target.Included && saved.ServerUID == string(gs.UID) && (target.ClaimName != original.ClaimName || target.ClaimUID != original.ClaimUID) { + return false, fmt.Errorf("original claim %s was replaced", target.LogicalName) + } + } + // Excluded rebuildable claims may be created or reconstructed during recovery; + // they have no writable target mapping and must not invalidate included roots. + for i := range current.Volumes { + if !current.Volumes[i].Included { + current.Volumes[i].ClaimUID = "" + } + } + candidate := &v1.RestoreRecoveryPlan{AdmittedSuspend: gs.Spec.Suspend, AdmittedIdle: gs.Annotations[IdleAsleepSinceAnnotation], AdmittedGeneration: gs.Generation, BackupUID: string(src.UID), ManifestDigest: recoveryDigest(saved), TargetUID: string(gs.UID), TemplateRevision: current.TemplateRevision, GameVersion: current.GameVersion, Volumes: current.Volumes} + if rs.Status.RecoveryPlan != nil { + candidate.AdmittedSuspend = rs.Status.RecoveryPlan.AdmittedSuspend + candidate.AdmittedIdle = rs.Status.RecoveryPlan.AdmittedIdle + candidate.AdmittedGeneration = rs.Status.RecoveryPlan.AdmittedGeneration + if !reflect.DeepEqual(candidate, rs.Status.RecoveryPlan) { + return false, fmt.Errorf("pinned recovery mapping or source manifest changed") + } + return false, nil + } + rs.Status.RecoveryPlan = candidate + return true, r.Status().Update(ctx, rs) +} + +// A ready Pod proves the claims are mounted. PVC protection prevents deletion +// and name reuse until all Pods mounting that claim are gone. UID revalidation +// after readiness closes the pre-scheduling replacement gap. +func validateMountedRecoveryClaims(ctx context.Context, reader client.Reader, namespace string, volumes []v1.BackupVolumeCoverage) error { + for _, v := range volumes { + if !v.Included { + continue + } + var pvc corev1.PersistentVolumeClaim + if err := reader.Get(ctx, client.ObjectKey{Namespace: namespace, Name: v.ClaimName}, &pvc); err != nil { + return err + } + protected := false + for _, finalizer := range pvc.Finalizers { + protected = protected || finalizer == "kubernetes.io/pvc-protection" + } + if string(pvc.UID) != v.ClaimUID || !pvc.DeletionTimestamp.IsZero() || pvc.Status.Phase != corev1.ClaimBound || pvc.Spec.VolumeName == "" || !protected { + return fmt.Errorf("mounted claim %s is replaced, unbound, deleting or lacks PVC protection", v.ClaimName) + } + } + return nil +} +func restoreStartName(rs *v1.Restore) string { + return fmt.Sprintf("restore-start-%x", sha256.Sum256([]byte(rs.Name+"/"+string(rs.UID))))[:61] +} +func (r *RestoreReconciler) restoreRecoveryGate(ctx context.Context, rs *v1.Restore) (*corev1.ConfigMap, error) { + cm := &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: restoreStartName(rs), Namespace: rs.Namespace}} + _, err := controllerutil.CreateOrUpdate(ctx, r.Client, cm, func() error { + if !cm.CreationTimestamp.IsZero() && !metav1.IsControlledBy(cm, rs) { + return fmt.Errorf("foreign restore gate") + } + if cm.Data == nil { + cm.Data = map[string]string{backupStartTokenKey: ""} + } + return controllerutil.SetControllerReference(rs, cm, r.Scheme) + }) + return cm, err +} +func (r *RestoreReconciler) revokeRecoveryRestore(ctx context.Context, rs *v1.Restore) error { + var cm corev1.ConfigMap + if err := r.apiReader().Get(ctx, client.ObjectKey{Namespace: rs.Namespace, Name: restoreStartName(rs)}, &cm); err != nil { + return client.IgnoreNotFound(err) + } + if !metav1.IsControlledBy(&cm, rs) || cm.Data[backupStartTokenKey] == "" { + return nil + } + cm.Data[backupStartTokenKey] = "" + return r.Update(ctx, &cm) +} +func (r *RestoreReconciler) prepareRecoveryRestorePod(ctx context.Context, rs *v1.Restore, spec *corev1.PodSpec) error { + if rs.Status.RecoveryPlan == nil { + return nil + } + if rs.Annotations[restoreDeadlineAnnotation] == "" { + _, seconds := resolveJobLimits(r.JobBackoffLimit, r.JobActiveDeadlineSeconds) + if rs.Annotations == nil { + rs.Annotations = map[string]string{} + } + rs.Annotations[restoreDeadlineAnnotation] = time.Now().Add(time.Duration(seconds) * time.Second).UTC().Format(time.RFC3339Nano) + if err := r.Update(ctx, rs); err != nil { + return err + } + } + deadline, err := time.Parse(time.RFC3339Nano, rs.Annotations[restoreDeadlineAnnotation]) + if err != nil { + return err + } + if _, err := r.restoreRecoveryGate(ctx, rs); err != nil { + return err + } + worker := &spec.Containers[0] + worker.Command = []string{"/bin/sh", "-c"} + worker.Args = append([]string{resticWorkerScript, "restore-worker"}, worker.Args...) + worker.Env = append(worker.Env, corev1.EnvVar{Name: "BACKUP_UID", Value: string(rs.UID)}, corev1.EnvVar{Name: "POD_UID", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{FieldPath: "metadata.uid"}}}, corev1.EnvVar{Name: "SOURCE_ID", Value: rs.Status.RecoveryPlan.ManifestDigest}, corev1.EnvVar{Name: "PREPARATION_EXPIRES", Value: strconv.FormatInt(deadline.Unix(), 10)}) + worker.VolumeMounts = append(worker.VolumeMounts, corev1.VolumeMount{Name: "start", MountPath: "/start", ReadOnly: true}) + worker.ReadinessProbe = &corev1.Probe{ProbeHandler: corev1.ProbeHandler{Exec: &corev1.ExecAction{Command: []string{"/bin/sh", "-c", "test -f /tmp/prepared"}}}, PeriodSeconds: 2, FailureThreshold: 1} + spec.Volumes = append(spec.Volumes, corev1.Volume{Name: "start", VolumeSource: corev1.VolumeSource{ConfigMap: &corev1.ConfigMapVolumeSource{LocalObjectReference: corev1.LocalObjectReference{Name: restoreStartName(rs)}}}}) + noToken := false + spec.AutomountServiceAccountToken = &noToken + return nil +} +func (r *RestoreReconciler) authorizeRecoveryRestore(ctx context.Context, rs *v1.Restore, src *v1.Backup, gs *v1.GameServer, job *batchv1.Job) (bool, error) { + if rs.Status.RecoveryPlan == nil { + return true, nil + } + if err := r.revokeRecoveryRestore(ctx, rs); err != nil { + return false, err + } + if _, err := r.preflightRecovery(ctx, rs, src, gs); err != nil { + return false, err + } + deadline, err := time.Parse(time.RFC3339Nano, rs.Annotations[restoreDeadlineAnnotation]) + if err != nil { + return false, err + } + if job.Status.Succeeded > 0 { + if rs.Annotations[restoreAuthorizedAnnotation] == "" { + return false, fmt.Errorf("restore completed without authorization") + } + return true, nil + } + if jobPermanentlyFailed(job) { + return false, r.restoreJobFailure(ctx, job) + } + if !time.Now().Before(deadline) { + return false, fmt.Errorf("restore copy deadline expired") + } + worker, err := (&BackupReconciler{Client: r.Client, APIReader: r.apiReader()}).preparedBackupWorker(ctx, &v1.Backup{ObjectMeta: metav1.ObjectMeta{Namespace: rs.Namespace}}, job) + if err != nil || worker == nil { + return false, err + } + if err := validateMountedRecoveryClaims(ctx, r.apiReader(), rs.Namespace, rs.Status.RecoveryPlan.Volumes); err != nil { + return false, err + } + // Recheck other writers, excluding only this exact Restore's worker. + if err := r.recoveryRestoreWritersDrained(ctx, rs, gs); err != nil { + return false, err + } + if _, err := r.preflightRecovery(ctx, rs, src, gs); err != nil { + return false, err + } + rs.Annotations[restoreAuthorizedAnnotation] = string(worker.UID) + if err := r.Update(ctx, rs); err != nil { + return false, err + } + cm, err := r.restoreRecoveryGate(ctx, rs) + if err != nil { + return false, err + } + cm.Data[backupStartTokenKey] = fmt.Sprintf("v1 %s %s %s %d\n", rs.UID, worker.UID, rs.Status.RecoveryPlan.ManifestDigest, deadline.Unix()) + return true, r.Update(ctx, cm) +} +func (r *RestoreReconciler) recoveryRestoreWritersDrained(ctx context.Context, rs *v1.Restore, gs *v1.GameServer) error { + var pods corev1.PodList + var jobs batchv1.JobList + if err := r.apiReader().List(ctx, &pods, client.InNamespace(gs.Namespace)); err != nil { + return err + } + if err := r.apiReader().List(ctx, &jobs, client.InNamespace(gs.Namespace)); err != nil { + return err + } + own := map[string]bool{} + for i := range jobs.Items { + job := &jobs.Items[i] + if metav1.IsControlledBy(job, rs) { + own[string(job.UID)] = true + continue + } + if (job.Status.Succeeded == 0 && !jobPermanentlyFailed(job)) || job.Status.Active > 0 || !job.DeletionTimestamp.IsZero() { + if podMountsRecoveryClaims(&corev1.Pod{Spec: job.Spec.Template.Spec}, rs.Status.RecoveryPlan.Volumes) { + return fmt.Errorf("another Job still uses recovery claims") + } + } + } + for i := range pods.Items { + pod := &pods.Items[i] + owner := metav1.GetControllerOf(pod) + if owner != nil && own[string(owner.UID)] { + continue + } + if liveRestorePod(pod) && podMountsRecoveryClaims(pod, rs.Status.RecoveryPlan.Volumes) { + return fmt.Errorf("another Pod still uses recovery claims") + } + } + return nil +} diff --git a/operator/specs.md b/operator/specs.md index 4f0e5c168..30def9620 100644 --- a/operator/specs.md +++ b/operator/specs.md @@ -453,16 +453,16 @@ Primary reconcilers register with the manager in `cmd/main.go` and handle CRD li - Report RetentionTrimmed condition (success) or retention failure (TrimFailed). ### RestoreReconciler -- **Responsibility:** Drive Restore to completion: suspend GameServer, run restic restore Job, resume. +- **Responsibility:** Drive Restore to completion: stop the target workload, run the restore, and release lifecycle ownership according to the restore's power-intent contract. - **Key functions:** - Both restore strategies participate in shared lifecycle exclusion. A CSI restore reserves its new target with a Restore guard in the GameServer Create, then pins the target UID and adds the full owner. Its `initialStartup` mode permits the first workload to run, including PVCs using WaitForFirstConsumer; backup, wipe, restart, idle transitions and template rollout still wait for restore completion. The original server is unaffected. - Pin snapshot ID at observation (immutable during restore). - - Restic restores pin the target GameServer name/UID on the Restore and add a cleanup finalizer before acquiring an annotation guard on that GameServer. The guard records the owning Restore name/UID and is acquired with an optimistic resourceVersion update that also sets spec.suspend=true. Other restores targeting that server wait; restores of independent targets may run concurrently. The game workload remains suspended while the guard exists, even if a power request clears spec.suspend; data-wipe reconciliation also waits for the guard to clear. + - Restic restores pin the target GameServer name/UID on the Restore and add a cleanup finalizer before acquiring an annotation guard on that GameServer. The guard records the owning Restore name/UID and is acquired with an optimistic resourceVersion update. Legacy restores also set spec.suspend=true; v2 restores with a pinned recovery plan preserve the target's current spec.suspend and stop through lifecycle ownership. Other restores targeting that server wait; restores of independent targets may run concurrently. The game workload remains suspended while the guard exists, even if a power request clears spec.suspend; data-wipe reconciliation also waits for the guard to clear. - A nonempty pending data-wipe request blocks guard acquisition, including after wipe failure. Restore status explains this wait. An uncached read of the fixed-name wipe Job identifies a failed current request only when the current GameServer controls it, its token matches, and it reports Failed=True; DataWipe=False alone is insufficient. Confirmed failure reports that restore is blocked and asks the operator to inspect wipe Job logs and resolve the wipe failure. The restore does not cancel or acknowledge the wipe, since an in-flight wipe reconciliation could still create a writer. The wait message clears when restore acquires the target normally. - Suspend: wait for Suspended/Stopped status plus the actual workload: the owned StatefulSet must exist, request zero replicas, report zero replicas, and have observed its current generation. An absent StatefulSet waits for workload reconciliation; a missing template or broken workload must be repaired before restore can proceed. No live or terminating game pod or pod mounting the data PVC may remain. Nonterminal Jobs mounting that PVC must also finish before a new restore Job starts. These checks use the uncached API reader and run again immediately before creating the Job. - The stopped StatefulSet also receives the owning Restore guard. A successful fence write immediately requeues the Restore to repeat the workload and writer checks; only external drainage waits use the polling delay. Its mutation callback honors that guard even when the GameServer cache snapshot predates acquisition, fencing delayed scale-up writes with the StatefulSet resourceVersion. Cleanup removes both guards after workers drain. - Restore Job: mount snapshot (from VolumeSnapshot or restic), run restic restore. Job carries BackoffLimit and ActiveDeadlineSeconds from the reconciler's JobBackoffLimit / JobActiveDeadlineSeconds fields (operator flags --backup-job-backoff-limit / --backup-job-active-deadline-seconds, Helm operator.backupJobBackoffLimit / operator.backupJobActiveDeadlineSeconds; defaults 2 and 86400, applied when unset; spec 018 OD-027); Job failure is recognized only from a Failed=True condition. Legacy in-flight Jobs without the pinned target annotation fail the Restore and are canceled/drained; operators can start a fresh Restore after cleanup. Pod SecurityContext takes FSGroup from the target GameTemplate's spec.security.fsGroup (fallback 65532 if template missing). - - Resume: once the restore Job succeeds and its live pods have drained, persist Succeeded before releasing the matching guard and clearing suspend in the same pass. This ordering survives interrupted status writes or operator restarts; it does not wait for the GameServer to reach Running again. Failure/deletion cancels active owned Jobs with foreground deletion, retires CSI initial-startup permission and suspends the target while retaining the guard/finalizer. Release additionally requires an owned StatefulSet with desired/observed zero replicas at its current generation, no live or terminating game/PVC writer pods and no pending PVC writer Jobs. Orphan recovery applies the same stopped-target proof before admitting a successor, preserving the template while forcing zero replicas. Successful CSI restores keep their initial workload running. Target recreation fails the old restore; cleanup never changes the replacement server. + - Resume: once the restore Job succeeds and its live pods have drained, persist Succeeded before releasing the matching guard in the same pass. Legacy restores clear spec.suspend; v2 restores preserve current suspend, idle and restart intent, so a suspended target stays suspended. This ordering survives interrupted status writes or operator restarts; it does not wait for the GameServer to reach Running again. Failure/deletion cancels active owned Jobs with foreground deletion, retires CSI initial-startup permission and suspends the target while retaining the guard/finalizer. Release additionally requires an owned StatefulSet with desired/observed zero replicas at its current generation, no live or terminating game/PVC writer pods and no pending PVC writer Jobs. Orphan recovery applies the same stopped-target proof before admitting a successor, preserving the template while forcing zero replicas. Successful CSI restores keep their initial workload running. Target recreation fails the old restore; cleanup never changes the replacement server. - Status: report phase (Pending/Suspending/Running/Succeeded/Failed). `RestorePhaseResuming` is declared on the type but never assigned by this reconciler (F-260). - Volume-snapshot restore reference handling: When a volume-snapshot restore stands up a new GameServer from the original server's spec, any references in spec.env (SecretKeyRef, ConfigMapKeyRef) and spec.networking.tunnel.credentialsSecretRef that are owned by the original server must be copied to new objects owned by the restored server, with rewritten names, because the restored server does not own the originals. Ownership of a source object is an OwnerReference to the original server (Kind, Name, UID) and is re-verified against the source object immediately before each copy, not just once during planning. An object that already exists under a copy name is accepted only if the restored server controls it (a controller OwnerReference, `controller: true`); otherwise the restore fails. The restore fails before the new GameServer is created if the original server does not own every referenced object. The references are rewritten in the new GameServer's spec when it is created, and the copies are made right after creation and re-ensured on every reconcile pass. Only an ownership refusal fails the restore; transient errors (stale cache reads, write conflicts, a referenced object not yet visible in the cache) requeue it. A restore's 10-minute deadline starts when the restore first plans the new server (before it is created) and covers both creating the server with its reference copies and waiting for it to reach Running; it is enforced even while a transient error (such as a referenced object that stays missing) keeps recurring, so the restore cannot run past the limit, and once it expires the restore fails with a deadline message. @@ -804,3 +804,20 @@ CI run: `make cover` generates merged profile (unit + envtest), `make cover-ratc - **CRD YAML:** `operator/config/crd/gameplane.local_*.yaml` (generated from types). - **RBAC YAML:** `operator/config/rbac/*.yaml` (generated, defines operator ServiceAccount + permissions). - **Helm integration:** `charts/gameplane/charts/gameplane-crds/crds/` (native CRDs) + `charts/gameplane/crd-manifests/` and `charts/gameplane/templates/crd-apply-hook.yaml` (pre-upgrade CRD sync). + +## Stopped multi-volume recovery + +Backup/BackupSchedule powerPolicy defaults to preserve; stop-copy-restart uses +durable lifecycle ownership and graceful stopping without spec.suspend mutation. +volumePolicy defaults to primary-only; all-data Restic includes classified data +extras and active separate mod claims, excluding rebuildable prerequisites. +CSI rejects the opt-in policies. All-data manifests and restore plans pin logical +roots, paths, claim UIDs and template/version provenance. Prepared restore workers +receive Pod-bound authorization only after mounted claims and mapping revalidate. +V2 restores stop through lifecycle guards and preserve current power intent on +success. Failed/cancelled writes stay suspended. Orphan stopped backups recover +autonomously through the GameServer owner; unknown provenance remains fenced. +RecoveryIntervention is removed only after cleanup has durably released lifecycle +ownership and its compatibility guards; unresolved owners retain the condition. +See [the recovery contract](../specs/033-stopped-multivolume-recovery/contracts/recovery.md) +and [operator operations](../docs/operator-operations.md) for defaults and failures. diff --git a/specs/033-stopped-multivolume-recovery/OPEN-DECISIONS.md b/specs/033-stopped-multivolume-recovery/OPEN-DECISIONS.md new file mode 100644 index 000000000..6ea8c2197 --- /dev/null +++ b/specs/033-stopped-multivolume-recovery/OPEN-DECISIONS.md @@ -0,0 +1,3 @@ +# Open decisions + +None for implementation. Policy names follow the issue proposal and make defaults explicit. New dashboard forms are deferred; CRD/API opt-in and existing coverage presentation are sufficient. CI runtime results remain pending and must not be represented as successful before execution. diff --git a/specs/033-stopped-multivolume-recovery/contracts/recovery.md b/specs/033-stopped-multivolume-recovery/contracts/recovery.md new file mode 100644 index 000000000..e80fd21e7 --- /dev/null +++ b/specs/033-stopped-multivolume-recovery/contracts/recovery.md @@ -0,0 +1,14 @@ +# Recovery contract + +1. Acquire shared lifecycle owner. Persist original intent before graceful stop. No spec.suspend mutation by backup. +2. Freeze workload template; graceful stop uses existing deadline. Confirm zero replicas, observed generation, original Pod gone, all selected PVC writers drained. Never interpret timeout as drainage. +3. Persist exact coverage before creating immutable worker mounts. Reject unavailable, unclassified or ambiguous all-data layouts. Recheck claim UIDs/layout before authorizing worker token. +4. Capture one Restic snapshot with explicit argument roots. No glob or shell expansion from manifest text. Copy readers are read-only. +5. Revoke, drain and resolve original-process debt before release on success, failure or deletion. Original running intent allows normal desired-state reconciliation; newer user power requests remain untouched. Unknown debt retains ownership and intervention condition. +6. Restore preflight precedes suspension. Persist mapping and digest; compare again immediately before Job creation and before authorization of the ready mounted worker. Writable mounts and --delete scopes are exactly validated included roots. Legacy captures explicitly retain historical primary-only restore. + +CI acceptance includes running/suspended/idle backup, stop/copy failure, cancellation and owner loss, claim replacement, changed paths, missing required extras, rebuildable exclusion, malicious roots, legacy and CSI compatibility; Kind round-trip covers primary + extra + separate mod volume. + +Restic v2 restore selects `:/volumes` with target `/volumes`; each validated root `/volumes/` receives the subtree-relative include `/`. This preserves the exact mounted PVC paths and scoped deletion while preventing restoration of archived ancestor metadata onto the container's read-only `/volumes` directory. The snapshot and persisted manifest retain their original absolute roots. + +Recovery retains the existing non-root Restic worker identity (UID 65532 with the template fsGroup). Included data must grant it backup read access and restore write access to existing files and parent directories. Storage-driver ownership behavior is a deployment prerequisite; manifest identity validation does not establish filesystem permissions. Failed permission checks during copying keep the target suspended and never trigger privilege escalation. diff --git a/specs/033-stopped-multivolume-recovery/data-model.md b/specs/033-stopped-multivolume-recovery/data-model.md new file mode 100644 index 000000000..1cf631885 --- /dev/null +++ b/specs/033-stopped-multivolume-recovery/data-model.md @@ -0,0 +1,9 @@ +# Data model + +BackupSpec and BackupScheduleSpec: powerPolicy enum preserve|stop-copy-restart; volumePolicy enum primary-only|all-data. Omitted fields retain existing behavior; CSI only permits preserve/primary-only. + +BackupCaptureManifest: formatVersion, volumePolicy, templateName, templateRevision, gameVersion. Each BackupVolumeCoverage adds logicalName, snapshotRoot, recoveryPrerequisite. all-data logical roots are primary, extra-, mods-; one root per included claim. + +GameServer lifecycle identity: optional stoppedBackup record with initial running intent, GameServer generation, suspend/idle/restart/wake baselines, original Pod UID and durable stage. Record survives Backup force deletion through recovery lineage. Status Recovery condition exposes stop/copy/release/intervention. + +RestoreStatus: recoveryPlan containing immutable source Backup UID/snapshot/manifest digest, target GameServer UID/template/version and mapped volume records with pinned target claim UID and names. Reconciliation compares the live layout and PVCs before destructive worker creation. diff --git a/specs/033-stopped-multivolume-recovery/spec.md b/specs/033-stopped-multivolume-recovery/spec.md new file mode 100644 index 000000000..1039c35f4 --- /dev/null +++ b/specs/033-stopped-multivolume-recovery/spec.md @@ -0,0 +1,13 @@ +# Stopped and multi-volume recovery (#801) + +Status: implementation approved; runtime verification belongs to CI. + +Existing backups default to powerPolicy=preserve and volumePolicy=primary-only. Opt-in stop-copy-restart uses the existing shared lifecycle owner and graceful stop sequence, never writes spec.suspend, and requires observed zero replicas and all source writers drained. all-data captures primary data, declared data extras, and active separate loader/mod claims in one Restic snapshot. Rebuildable extras are reported prerequisites; unclassified claims reject admission. CSI rejects either new policy and remains a single primary volume snapshot with no cross-volume atomicity. + +The owner on GameServer durably records admitted power intent, original pod and stop stage before stopping. Existing persisted preparation/copy budgets bound waiting and copying. Stop failure, copy failure and cancellation revoke and drain workers before releasing the fence. An orphan owner preserves stop debt through recovery; malformed provenance fails closed with visible RecoveryIntervention. Release never overwrites current suspend, idle or restart intent. Automatic resumption requires admitted running intent, unchanged power intent and completed drain; user changes take precedence after cleanup. + +Each new all-data manifest pins template/version provenance, claim names and UIDs, logical identities, original mount paths and disjoint /volumes/ roots. Claim mounts in the worker are read-only. Primary-only keeps /data and older manifests remain legacy, without invented metadata. + +Restore validates a manifest against an existing target before claiming ownership (which suspends), provisioning or creating a writer. It persists target claim UIDs/layout and revalidates both before Job creation and after the worker mounts become ready, immediately before Pod-bound authorization. PVC protection plus the final live UID check prevents name replacement while the mounted worker remains live. Missing/replaced required claims, changed mount paths/provenance, duplicate/traversing roots and ambiguous mappings fail visibly. A same-server restore requires original claim UIDs; a different existing server maps by exact logical identity and compatible template/version, pinning target UIDs. Excluded rebuildable claims may be absent: their logical declarations remain pinned, and the manifest reports reconstruction prerequisites. Restore never mounts or writes them. Only validated included roots receive --include and writable mounts. V2 restore uses lifecycle ownership to stop without changing spec.suspend; successful cleanup preserves current power intent, while failed or cancelled writes retain the established suspended safety behavior. Legacy primary-only restore retains /data behavior. + +The existing coverage rows expose included/excluded claims and prerequisites; API types gain policy and manifest fields. No new dashboard controls or layout are introduced. diff --git a/specs/033-stopped-multivolume-recovery/tasks.md b/specs/033-stopped-multivolume-recovery/tasks.md new file mode 100644 index 000000000..054c0a2d6 --- /dev/null +++ b/specs/033-stopped-multivolume-recovery/tasks.md @@ -0,0 +1,12 @@ +# Tasks + +- [x] Specify defaults, durable lifecycle and restore mapping. +- [x] Add regression sources before implementation (execution deferred to CI). +- [x] Implement policies and generated schema. +- [x] Implement stopped lifecycle and orphan recovery. +- [x] Implement all-data manifest and worker. +- [x] Implement restore preflight and pinned mapping. +- [x] Expose metadata and update operations documentation. +- [x] Add CI Kind multi-volume round-trip fixture. +- [x] Generate CRDs/deepcopy and compile production. +- [ ] Independent review and CI verification (root orchestrator). diff --git a/test/e2e/backup_forget_e2e_test.go b/test/e2e/backup_forget_e2e_test.go index 9a5cb307f..1e0e92232 100644 --- a/test/e2e/backup_forget_e2e_test.go +++ b/test/e2e/backup_forget_e2e_test.go @@ -102,9 +102,14 @@ func TestBackup_DeleteForgetsSnapshot(t *testing.T) { // deleteBackupCR deletes a Backup CR, tolerating one that is already gone. func deleteBackupCR(t *testing.T, ns, name string) { + t.Helper() + deleteBackupCRContext(context.Background(), t, ns, name) +} + +func deleteBackupCRContext(ctx context.Context, t *testing.T, ns, name string) { t.Helper() err := envInstance.Dyn.Resource(backupGVR).Namespace(ns). - Delete(context.Background(), name, metav1.DeleteOptions{}) + Delete(ctx, name, metav1.DeleteOptions{}) if err != nil && !apierrors.IsNotFound(err) { t.Fatalf("delete backup %s/%s: %v", ns, name, err) } diff --git a/test/e2e/buckets.sh b/test/e2e/buckets.sh index fff502f9a..c4a7afdec 100755 --- a/test/e2e/buckets.sh +++ b/test/e2e/buckets.sh @@ -86,6 +86,7 @@ TestModule_ScaffoldAndPackage TestModule_SteamcmdScaffoldKeepsNonRootSecurity TestBackup_OperatorMaterializesJob TestBackup_StoppedCopiesRemainRestorable +TestBackup_MultiVolumeRecoveryRoundTrip TestBackup_FailsOnMissingPVC TestBackup_FailsOnBadCredentials TestBackup_DeleteForgetsSnapshot diff --git a/test/e2e/recovery_multivolume_e2e_test.go b/test/e2e/recovery_multivolume_e2e_test.go new file mode 100644 index 000000000..522b872f1 --- /dev/null +++ b/test/e2e/recovery_multivolume_e2e_test.go @@ -0,0 +1,244 @@ +//go:build e2e + +package e2e + +import ( + "context" + "encoding/json" + "fmt" + "strings" + "testing" + "time" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/types" +) + +// Real Restic/PVC fixture: one world, a declared extra, a separate loader +// claim and a rebuildable cache. No API logins; CI operator bucket owns execution. +func TestBackup_MultiVolumeRecoveryRoundTrip(t *testing.T) { + t.Parallel() + ctx := context.Background() + ns := "gameplane-games" + suffix := time.Now().UnixNano() + gs := fmt.Sprintf("recovery-%d", suffix) + tmpl := gs + "-template" + backup := gs + "-backup" + ensureResticRepo(t) + applyBusyboxTemplate(t, tmpl) + // Kind's hostPath provisioner does not apply fsGroup ownership. Run the + // fixture as the existing Restic worker UID so overwriting existing files + // exercises recovery rather than unsupported root-owned file permissions. + patch := map[string]any{"spec": map[string]any{ + "security": map[string]any{"runAsUser": int64(65532), "runAsGroup": int64(65532), "fsGroup": int64(65532)}, + "storage": map[string]any{"size": "1Gi", "mountPath": "/data", "extra": []any{map[string]any{"name": "world", "mountPath": "/world", "size": "1Gi", "backupRole": "data"}, map[string]any{"name": "cache", "mountPath": "/rebuild", "size": "1Gi", "backupRole": "rebuildable"}}}, + "versions": []any{map[string]any{"id": "v1", "displayName": "Test loader", "image": "busybox:1.36", "loader": "paper", "default": true}}, + "capabilities": map[string]any{"mods": map[string]any{"loaders": map[string]any{"paper": map[string]any{"path": "mods"}}}}, + }} + data, _ := json.Marshal(patch) + if _, err := envInstance.Dyn.Resource(gameTemplateGVR).Patch(ctx, tmpl, types.MergePatchType, data, metav1.PatchOptions{}); err != nil { + t.Fatal(err) + } + applyBusyboxGameServer(t, ns, gs, tmpl) + ready := func(name string) { + t.Helper() + envInstance.Eventually(t, 3*time.Minute, func() (bool, string) { + pod, err := envInstance.K8s.CoreV1().Pods(ns).Get(ctx, name+"-0", metav1.GetOptions{}) + if err != nil { + return false, err.Error() + } + for _, s := range pod.Status.ContainerStatuses { + if s.Name == "game" && s.Ready { + return true, "" + } + } + return false, "game not ready" + }) + } + exec := func(script string) { + t.Helper() + if out, err := envInstance.KubectlExec(t, ns, "pod/"+gs+"-0", "sh", "-c", "set -eu; "+script); err != nil { + t.Fatalf("fixture command: %v %s", err, out) + } + } + ready(gs) + exec("test $(id -u) = 65532; test $(id -g) = 65532; echo primary-before > /data/marker; echo extra-before > /world/marker; echo mods-before > /data/mods/marker; echo cache-before > /rebuild/marker; sync") + capture := func(name string) { + t.Helper() + obj := &unstructured.Unstructured{Object: map[string]any{"apiVersion": "gameplane.local/v1alpha1", "kind": "Backup", "metadata": map[string]any{"name": name, "namespace": ns}, "spec": map[string]any{"serverRef": map[string]any{"name": gs}, "repoRef": map[string]any{"name": "e2e-restic-creds", "key": "repo"}, "quiesce": false, "powerPolicy": "stop-copy-restart", "volumePolicy": "all-data"}}} + if _, err := envInstance.Dyn.Resource(backupGVR).Namespace(ns).Create(ctx, obj, metav1.CreateOptions{}); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second) + defer cancel() + deleteBackupCRContext(cleanupCtx, t, ns, name) + }) + waitBackupSucceededContext(ctx, t, ns, name, 6*time.Minute) + } + capture(backup) + ready(gs) + got, err := envInstance.Dyn.Resource(backupGVR).Namespace(ns).Get(ctx, backup, metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + volumes, _, _ := unstructured.NestedSlice(got.Object, "status", "captureManifest", "volumes") + included, excluded := 0, 0 + for _, raw := range volumes { + v := raw.(map[string]any) + if v["included"] == true { + included++ + if !strings.HasPrefix(v["snapshotRoot"].(string), "/volumes/") { + t.Fatal("non-disjoint capture root") + } + } else { + excluded++ + if prerequisite, ok := v["recoveryPrerequisite"].(string); !ok || prerequisite == "" { + t.Fatal("missing recovery prerequisite") + } + } + } + if included != 3 || excluded != 1 { + t.Fatalf("wrong coverage: %d/%d", included, excluded) + } + exec("echo changed > /data/marker; echo changed > /world/marker; echo changed > /data/mods/marker; echo cache-after > /rebuild/marker; touch /data/stale /world/stale /data/mods/stale; sync") + restore := gs + "-restore" + watchRecoveryRestoreWorkers(ctx, t, ns, restore) + createRestoreContext(ctx, t, ns, restore, gs, backup) + waitRestoreSucceededContext(ctx, t, ns, restore, 6*time.Minute) + ready(gs) + exec("test $(cat /data/marker) = primary-before; test $(cat /world/marker) = extra-before; test $(cat /data/mods/marker) = mods-before; test $(cat /rebuild/marker) = cache-after; test ! -e /data/stale; test ! -e /world/stale; test ! -e /data/mods/stale") + + // Existing explicit suspend and idle intent survive both stopped capture and + // a successful v2 restore. Verify Pod absence after owner cleanup. + for _, mode := range []string{"suspended", "idle"} { + t.Run(mode, func(t *testing.T) { + body := map[string]any{"spec": map[string]any{"suspend": mode == "suspended", "idle": map[string]any{"enabled": mode == "idle"}}} + asleep := time.Now().UTC().Format(time.RFC3339) + if mode == "idle" { + body["metadata"] = map[string]any{"annotations": map[string]any{"gameplane.local/idle-asleep-since": asleep}} + } + data, _ := json.Marshal(body) + if _, err := envInstance.Dyn.Resource(gameServerGVR).Namespace(ns).Patch(ctx, gs, types.MergePatchType, data, metav1.PatchOptions{}); err != nil { + t.Fatal(err) + } + envInstance.Eventually(t, 3*time.Minute, func() (bool, string) { + _, err := envInstance.K8s.CoreV1().Pods(ns).Get(ctx, gs+"-0", metav1.GetOptions{}) + return apierrors.IsNotFound(err), fmt.Sprint(err) + }) + capture(gs + "-" + mode) + name := gs + "-restore-" + mode + watchRecoveryRestoreWorkers(ctx, t, ns, name) + createRestoreContext(ctx, t, ns, name, gs, backup) + waitRestoreSucceededContext(ctx, t, ns, name, 6*time.Minute) + envInstance.Eventually(t, time.Minute, func() (bool, string) { + obj, err := envInstance.Dyn.Resource(gameServerGVR).Namespace(ns).Get(ctx, gs, metav1.GetOptions{}) + if err != nil { + return false, err.Error() + } + if obj.GetAnnotations()["operations.gameplane.local/owner"] != "" { + return false, "cleanup pending" + } + suspended, _, _ := unstructured.NestedBool(obj.Object, "spec", "suspend") + if suspended != (mode == "suspended") { + t.Fatal("restore changed suspend intent") + } + if mode == "idle" && obj.GetAnnotations()["gameplane.local/idle-asleep-since"] != asleep { + t.Fatal("restore changed idle intent") + } + _, err = envInstance.K8s.CoreV1().Pods(ns).Get(ctx, gs+"-0", metav1.GetOptions{}) + if !apierrors.IsNotFound(err) { + t.Fatalf("recovery woke parked game: %v", err) + } + return true, "" + }) + }) + } + + // An incompatible existing target must fail before power mutation or a Job. + for _, mode := range []string{"missing", "changed-path"} { + t.Run(mode, func(t *testing.T) { + target := gs + "-" + mode + extra := []any{map[string]any{"name": "cache", "mountPath": "/rebuild", "size": "1Gi", "backupRole": "rebuildable"}} + if mode == "changed-path" { + extra = append(extra, map[string]any{"name": "world", "mountPath": "/redirected-world", "size": "1Gi", "backupRole": "data"}) + } + obj := &unstructured.Unstructured{Object: map[string]any{"apiVersion": "gameplane.local/v1alpha1", "kind": "GameServer", "metadata": map[string]any{"name": target, "namespace": ns}, "spec": map[string]any{"templateRef": map[string]any{"name": tmpl}, "storage": map[string]any{"extra": extra}}}} + if _, err := envInstance.Dyn.Resource(gameServerGVR).Namespace(ns).Create(ctx, obj, metav1.CreateOptions{}); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + _ = envInstance.Dyn.Resource(gameServerGVR).Namespace(ns).Delete(ctx, target, metav1.DeleteOptions{}) + }) + ready(target) + name := target + "-restore" + watchRecoveryRestoreWorkers(ctx, t, ns, name) + createRestoreContext(ctx, t, ns, name, target, backup) + envInstance.Eventually(t, time.Minute, func() (bool, string) { + rs, err := envInstance.Dyn.Resource(restoreGVR).Namespace(ns).Get(ctx, name, metav1.GetOptions{}) + if err != nil { + return false, err.Error() + } + phase, _, _ := unstructured.NestedString(rs.Object, "status", "phase") + return phase == "Failed", phase + }) + targetObj, err := envInstance.Dyn.Resource(gameServerGVR).Namespace(ns).Get(ctx, target, metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + suspended, _, _ := unstructured.NestedBool(targetObj.Object, "spec", "suspend") + if suspended || targetObj.GetAnnotations()["restore.gameplane.local/owner"] != "" { + t.Fatal("preflight failure mutated power or ownership") + } + if _, err := envInstance.K8s.BatchV1().Jobs(ns).Get(ctx, "restore-"+name, metav1.GetOptions{}); !apierrors.IsNotFound(err) { + t.Fatalf("preflight created writer: %v", err) + } + }) + } +} + +// Start before creating the Restore so foreground cleanup cannot erase the +// failed worker evidence. This fixture uses only its disposable test repository; +// raw termination messages are deliberately confined to CI diagnostics. +func watchRecoveryRestoreWorkers(ctx context.Context, t *testing.T, ns, restoreName string) { + t.Helper() + watchCtx, cancel := context.WithCancel(ctx) + workerWatch, err := envInstance.K8s.CoreV1().Pods(ns).Watch(watchCtx, metav1.ListOptions{LabelSelector: "job-name=restore-" + restoreName}) + if err != nil { + cancel() + t.Fatalf("watch restore workers: %v", err) + } + done := make(chan struct{}) + go func() { + defer close(done) + for event := range workerWatch.ResultChan() { + pod, ok := event.Object.(*corev1.Pod) + if !ok { + continue + } + for _, status := range pod.Status.ContainerStatuses { + if status.State.Terminated == nil && !status.Ready { + continue + } + state, _ := json.Marshal(status) + t.Logf("restore worker %s event=%s uid=%s phase=%s status=%s", pod.Name, event.Type, pod.UID, pod.Status.Phase, state) + // Capture the persisted gate identity and deadline alongside the + // worker exit to distinguish preparation from authorized copying. + readCtx, cancelRead := context.WithTimeout(watchCtx, 5*time.Second) + rs, err := envInstance.Dyn.Resource(restoreGVR).Namespace(ns).Get(readCtx, restoreName, metav1.GetOptions{}) + cancelRead() + if err == nil { + t.Logf("restore %s authorization=%s deadline=%s", restoreName, rs.GetAnnotations()["restore.gameplane.local/authorized-worker"], rs.GetAnnotations()["restore.gameplane.local/copy-deadline"]) + } + } + } + }() + t.Cleanup(func() { + cancel() + workerWatch.Stop() + <-done + }) +} diff --git a/test/e2e/test_helpers_e2e_test.go b/test/e2e/test_helpers_e2e_test.go index 66056840f..2573cf063 100644 --- a/test/e2e/test_helpers_e2e_test.go +++ b/test/e2e/test_helpers_e2e_test.go @@ -164,7 +164,11 @@ func createBackup(t *testing.T, ns, name, gsName, repoSecretName, repoKey string // createRestore applies a Restore CR and registers cleanup on t. func createRestore(t *testing.T, ns, name, gsName, backupName string) *unstructured.Unstructured { t.Helper() - ctx := context.Background() + return createRestoreContext(context.Background(), t, ns, name, gsName, backupName) +} + +func createRestoreContext(ctx context.Context, t *testing.T, ns, name, gsName, backupName string) *unstructured.Unstructured { + t.Helper() rs := &unstructured.Unstructured{Object: map[string]any{ "apiVersion": "gameplane.local/v1alpha1", "kind": "Restore", @@ -180,8 +184,10 @@ func createRestore(t *testing.T, ns, name, gsName, backupName string) *unstructu t.Fatalf("create restore %s/%s: %v", ns, name, err) } t.Cleanup(func() { + cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second) + defer cancel() _ = envInstance.Dyn.Resource(restoreGVR).Namespace(ns). - Delete(context.Background(), name, metav1.DeleteOptions{}) + Delete(cleanupCtx, name, metav1.DeleteOptions{}) }) return created } @@ -191,7 +197,11 @@ func createRestore(t *testing.T, ns, name, gsName, backupName string) *unstructu // terminal Failed phase. func waitBackupSucceeded(t *testing.T, ns, name string, timeout time.Duration) string { t.Helper() - ctx := context.Background() + return waitBackupSucceededContext(context.Background(), t, ns, name, timeout) +} + +func waitBackupSucceededContext(ctx context.Context, t *testing.T, ns, name string, timeout time.Duration) string { + t.Helper() var snapshotID string envInstance.Eventually(t, timeout, func() (bool, string) { got, err := envInstance.Dyn.Resource(backupGVR).Namespace(ns). @@ -222,7 +232,11 @@ func waitBackupSucceeded(t *testing.T, ns, name string, timeout time.Duration) s // Fails the test on timeout or on a terminal Failed phase. func waitRestoreSucceeded(t *testing.T, ns, name string, timeout time.Duration) { t.Helper() - ctx := context.Background() + waitRestoreSucceededContext(context.Background(), t, ns, name, timeout) +} + +func waitRestoreSucceededContext(ctx context.Context, t *testing.T, ns, name string, timeout time.Duration) { + t.Helper() envInstance.Eventually(t, timeout, func() (bool, string) { got, err := envInstance.Dyn.Resource(restoreGVR).Namespace(ns). Get(ctx, name, metav1.GetOptions{}) diff --git a/web/specs.md b/web/specs.md index bf1749378..4f7ad06e1 100644 --- a/web/specs.md +++ b/web/specs.md @@ -620,6 +620,7 @@ Every file in slice 2b imports **only** from `@heroui/react` and `@/components/u **BackupDetailDrawer.tsx:** - Slide-out right panel (modal positioned right-side or Drawer component) - Shows backup metadata: ID, server, size (human-readable), created/expires timestamps +- Shows recorded capture coverage with archive roots and saved recovery prerequisites. Coverage guidance states that primary-only backups exclude extra data volumes and separate mod claims, and that rebuildable volumes are excluded. - Retention expiry badge (green/yellow/red based on days remaining) - Associated backup file listing (if applicable) - Close button or swipe-to-close diff --git a/web/src/components/backups/BackupDetailDrawer.test.tsx b/web/src/components/backups/BackupDetailDrawer.test.tsx index 835470dd7..e3068da8c 100644 --- a/web/src/components/backups/BackupDetailDrawer.test.tsx +++ b/web/src/components/backups/BackupDetailDrawer.test.tsx @@ -8,14 +8,18 @@ import { makeBackup } from "@/test/factories"; import { BackupDetailDrawer } from "./BackupDetailDrawer"; describe("BackupDetailDrawer", () => { - it("shows captured and excluded claims independently of copy success", async () => { + it.each([ + { policy: "legacy/default primary", volumePolicy: undefined }, + { policy: "explicit primary-only", volumePolicy: "primary-only" as const }, + ])("shows captured and excluded claims for $policy independently of copy success", async ({ volumePolicy }) => { server.use(http.get("/backups/coverage", () => HttpResponse.json(makeBackup({ metadata: { name: "coverage" }, status: { phase: "Succeeded", snapshotID: "copy", conditions: [{ type: "Consistency", status: "False", reason: "UnsupportedRawCopy", message: "The game does not support quiesce." }], - captureManifest: { state: "Recorded", recordedAt: "2026-10-21T12:00:00Z", volumes: [ + captureManifest: { state: "Recorded", volumePolicy, recordedAt: "2026-10-21T12:00:00Z", volumes: [ { claimName: "game-data", claimUID: "world-uid", mountPaths: ["/world"], included: true, classification: "data", reason: "PrimaryDataClaim" }, + { claimName: "game-extra-data", mountPaths: ["/extra-data"], included: false, classification: "data", reason: "NotCovered" }, { claimName: "game-extra-install", mountPaths: ["/serverfiles"], included: false, classification: "rebuildable", reason: "NotCovered" }, { claimName: "game-mods", mountPaths: ["/world/mods"], included: false, classification: "unclassified", reason: "NotCovered" }, ] }, @@ -23,12 +27,33 @@ describe("BackupDetailDrawer", () => { })))); renderWithQuery( {}} onRestore={() => {}} />); expect(await screen.findByText("The game does not support quiesce.")).toBeInTheDocument(); + expect(screen.getByText("Included claims are copied at the recorded archive roots. Primary-only backups exclude extra data volumes and separate mod claims. Rebuildable volumes are excluded. Check recovery prerequisites before restarting.")).toBeInTheDocument(); expect(screen.getByText("game-data")).toBeInTheDocument(); + expect(screen.getByText("game-extra-data")).toBeInTheDocument(); expect(screen.getByText("/world/mods")).toBeInTheDocument(); - expect(screen.getAllByText("Not covered")).toHaveLength(2); + expect(screen.getAllByText("Not covered")).toHaveLength(3); expect(screen.getByText("Rebuildable")).toBeInTheDocument(); }); + it("shows named archive roots and the saved prerequisites for missing and intact excluded claims", async () => { + server.use(http.get("/backups/recovery", () => HttpResponse.json(makeBackup({ + metadata: { name: "recovery" }, + status: { phase: "Succeeded", snapshotID: "snapshot", captureManifest: { + state: "Recorded", formatVersion: 2, volumePolicy: "all-data", volumes: [ + { claimName: "world", claimUID: "world-uid", logicalName: "primary", snapshotRoot: "/volumes/primary", included: true, classification: "data", reason: "IncludedData" }, + { claimName: "missing-install", included: false, classification: "rebuildable", reason: "RebuildRequired", recoveryPrerequisite: "Missing install files must be reconstructed from the module." }, + { claimName: "intact-cache", claimUID: "cache-uid", included: false, classification: "rebuildable", reason: "RebuildRequired", recoveryPrerequisite: "Keep compatible existing cache content; reconstruct only if needed." }, + ], + } }, + })))); + renderWithQuery( {}} onRestore={() => {}} />); + expect(await screen.findByText("/volumes/primary")).toBeInTheDocument(); + expect(screen.getByText("Missing install files must be reconstructed from the module.")).toBeInTheDocument(); + expect(screen.getByText("Keep compatible existing cache content; reconstruct only if needed.")).toBeInTheDocument(); + expect(screen.getAllByText("Not covered")).toHaveLength(2); + expect(screen.queryByText(/must be reconstructed before recovery can run/)).not.toBeInTheDocument(); + }); + it("shows legacy coverage as unknown without inventing a current layout", async () => { server.use(http.get("/backups/legacy", () => HttpResponse.json(makeBackup({ metadata: { name: "legacy" }, status: { phase: "Succeeded" } })))); renderWithQuery( {}} onRestore={() => {}} />); diff --git a/web/src/components/backups/BackupDetailDrawer.tsx b/web/src/components/backups/BackupDetailDrawer.tsx index 37acbe02a..d81969f88 100644 --- a/web/src/components/backups/BackupDetailDrawer.tsx +++ b/web/src/components/backups/BackupDetailDrawer.tsx @@ -89,7 +89,7 @@ export function BackupDetailDrawer({ name, ns, onClose, onRestore, target: expli {manifest?.state === "Recorded" ? (
{manifest.recordedAt &&
{intl.formatMessage({ id: "web.components.backups.backup_detail_drawer.coverage_recorded", defaultMessage: "Recorded: {time}", description: "Backup details: timestamp of the immutable capture manifest" }, { time: manifest.recordedAt })}
} -

{intl.formatMessage({ id: "web.components.backups.backup_detail_drawer.coverage_scope", defaultMessage: "Only the primary data claim is copied. Rebuildable is a declaration, not proof that recovery can reconstruct its contents.", description: "Backup details: explains the current single-volume backup scope and rebuildable declarations" })}

+

{intl.formatMessage({ id: "web.components.backups.backup_detail_drawer.coverage_scope", defaultMessage: "Included claims are copied at the recorded archive roots. Primary-only backups exclude extra data volumes and separate mod claims. Rebuildable volumes are excluded. Check recovery prerequisites before restarting.", description: "Backup details: explains recorded volume coverage, primary-only exclusions, and recovery prerequisites for excluded volumes" })}

    {manifest.volumes?.map((volume) => (
  • @@ -98,7 +98,8 @@ export function BackupDetailDrawer({ name, ns, onClose, onRestore, target: expli ? intl.formatMessage({ id: "web.components.backups.backup_detail_drawer.included", defaultMessage: "Included", description: "Backup volume: this claim is selected for copying; success is reported separately" }) : intl.formatMessage({ id: "web.components.backups.backup_detail_drawer.not_covered", defaultMessage: "Not covered", description: "Backup volume: contents of this claim are not included" })}
{intl.formatMessage({ id: "web.components.backups.backup_detail_drawer.classification", defaultMessage: "{role, select, data {Data} rebuildable {Rebuildable} other {Unclassified}}", description: "Backup volume: optional declaration from the module or server at capture time" }, { role: volume.classification })}
- {volume.mountPaths?.map((path) =>
{path}
)} + {[...(volume.mountPaths ?? []), ...(volume.snapshotRoot ? [volume.snapshotRoot] : [])].map((path) =>
{path}
)} + {volume.recoveryPrerequisite &&
{volume.recoveryPrerequisite}
} {volume.reason === "RetainedUnmountedClaim" &&
{intl.formatMessage({ id: "web.components.backups.backup_detail_drawer.retained_unmounted", defaultMessage: "Retained claim; not mounted in the captured workload.", description: "Backup volume: an older server-owned claim was retained but was not part of the observed pod layout" })}
} ))} diff --git a/web/src/types.ts b/web/src/types.ts index 84b0f1f6d..2091c1f42 100644 --- a/web/src/types.ts +++ b/web/src/types.ts @@ -662,6 +662,11 @@ export interface ServerEvent { } export interface BackupCaptureManifest { + formatVersion?: number; + volumePolicy?: "primary-only" | "all-data"; + templateName?: string; + templateRevision?: string; + gameVersion?: string; state: "Recorded" | "LegacyUnknown"; recordedAt?: string; serverUID?: string; @@ -669,6 +674,9 @@ export interface BackupCaptureManifest { layoutSource?: string; quiesceOutcome?: string; volumes?: Array<{ + logicalName?: string; + snapshotRoot?: string; + recoveryPrerequisite?: string; claimName: string; claimUID?: string; mountPaths?: string[]; @@ -682,7 +690,7 @@ export interface Backup { metadata: ObjectMeta; // strategy: restic-snapshot writes to a restic repo; volume-snapshot takes a // CSI snapshot and is restored by provisioning a NEW server from it. - spec: { serverRef: { name: string }; strategy?: "restic-snapshot" | "volume-snapshot" }; + spec: { serverRef: { name: string }; strategy?: "restic-snapshot" | "volume-snapshot"; powerPolicy?: "preserve" | "stop-copy-restart"; volumePolicy?: "primary-only" | "all-data" }; status?: { phase?: "Pending" | "Running" | "Succeeded" | "Failed"; startTime?: string; @@ -699,6 +707,8 @@ export interface BackupSchedule { spec: { serverRef: { name: string }; schedule: string; + powerPolicy?: "preserve" | "stop-copy-restart"; + volumePolicy?: "primary-only" | "all-data"; suspend?: boolean; retention?: { keepLast?: number; @@ -725,6 +735,14 @@ export interface Restore { serverRef: { name: string }; }; status?: { + recoveryPlan?: { + backupUID: string; + manifestDigest: string; + targetUID: string; + templateRevision: string; + gameVersion: string; + volumes: NonNullable; + }; phase?: RestorePhase; snapshotID?: string; startTime?: string;