From 15e21e2df71ca3a934ecb3238dd4130e5f3d48e5 Mon Sep 17 00:00:00 2001 From: David Boreham Date: Tue, 18 Aug 2026 13:08:39 -0600 Subject: [PATCH] Add a test for cross-target backup restore --- .github/workflows/test-deploy-k3s.yml | 29 ++-- CLAUDE.md | 9 + tests/backup/run-cross-target-test.sh | 231 ++++++++++++++++++++++++++ 3 files changed, 258 insertions(+), 11 deletions(-) create mode 100755 tests/backup/run-cross-target-test.sh diff --git a/.github/workflows/test-deploy-k3s.yml b/.github/workflows/test-deploy-k3s.yml index e461f3c..7e522db 100644 --- a/.github/workflows/test-deploy-k3s.yml +++ b/.github/workflows/test-deploy-k3s.yml @@ -31,7 +31,7 @@ on: required: false type: choice default: all - options: [all, app-deploy, database, backup, database-backup, volumes, kata] + options: [all, app-deploy, database, backup, database-backup, cross-backup, volumes, kata] mode: description: "Which leg to run; each one provisions its own VM" required: false @@ -73,11 +73,14 @@ jobs: # # The backup tests only run here: K8up, which runs backups on a # cluster, is installed by the machine provisioning and is not present - # on a kind cluster. There are two of them, covering the two shapes a - # backup can have -- files (tests/backup) and a streamed logical dump - # (tests/database/run-backup-test.sh) -- and this is the only place the - # dump one runs against the engine that actually takes it in - # production. + # on a kind cluster. There are three of them: two covering the two + # shapes a backup can have -- files (tests/backup) and a streamed + # logical dump (tests/database/run-backup-test.sh) -- and this is the + # only place the dump one runs against the engine that actually takes + # it in production. The third (cross-backup) restores each engine's + # backup with the other one, compose deployments on the runner's own + # Docker against K8up on the cluster, and so runs nowhere but here by + # construction. # # The volumes test only runs here against a real cluster: its remote # leg is the one place the node-path volume mechanism (a `local` @@ -87,12 +90,12 @@ jobs: # # These are names rather than script paths: each one selects the test # step of the same name in the job below, which is what runs it. - gateway_tests='["app-deploy","database","backup","database-backup","volumes","kata"]' + gateway_tests='["app-deploy","database","backup","database-backup","cross-backup","volumes","kata"]' ingress_tests='["app-deploy"]' # The `tests` dropdown names one test to run instead of the per-leg set - # above. Only the six that go through select_deploy_target and accept - # the remote target are offered; the rest are single-target by - # construction (k8s-deployment-control owns its own kind cluster). + # above. Only the ones that accept the remote target are offered; the + # rest are single-target by construction (k8s-deployment-control owns + # its own kind cluster). case "${TESTS}" in ''|all) ;; # kata is gateway-only, because only that leg provisions a cluster @@ -103,7 +106,7 @@ jobs: gateway_tests="[\"${TESTS}\"]" ingress_tests='[]' ;; - app-deploy|database|backup|database-backup|volumes) + app-deploy|database|backup|database-backup|cross-backup|volumes) gateway_tests="[\"${TESTS}\"]" ingress_tests="${gateway_tests}" ;; @@ -230,6 +233,10 @@ jobs: if: ${{ !cancelled() && steps.provision.outcome == 'success' && contains(fromJSON(matrix.tests), 'database-backup') }} timeout-minutes: 30 run: ./tests/database/run-backup-test.sh + - name: "Cross-target backup: each engine restores the other's backup" + if: ${{ !cancelled() && steps.provision.outcome == 'success' && contains(fromJSON(matrix.tests), 'cross-backup') }} + timeout-minutes: 30 + run: ./tests/backup/run-cross-target-test.sh - name: "Volumes: data at a chosen node path reaches the container" if: ${{ !cancelled() && steps.provision.outcome == 'success' && contains(fromJSON(matrix.tests), 'volumes') }} timeout-minutes: 30 diff --git a/CLAUDE.md b/CLAUDE.md index 481b3ea..33c1c09 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -125,6 +125,15 @@ the machine provisioning installs and a kind cluster does not have. Its per-target divergence (which object store, and whether the backup stack has to be mixed in) lives in `select_backup_target`, next to `select_deploy_target`. +`tests/backup/run-cross-target-test.sh` is the cross-target restore test (issue +#242): a compose deployment's restic-written backup restored by K8up into a k8s +deployment, and that deployment's K8up-written backup restored by restic into a +new compose deployment. It is remote-only by construction — the k8s side needs +K8up and both sides need an object store both can reach, so the compose side is +the machine the test runs on and the store is the remote target's real one. It +covers the volume path only; the dump path's recovery route is external and +target-independent, and covered by `run-backup-test.sh`. + `tests/kata` is single-target the other way: it needs the `kata` RuntimeClass, which means a runtime installed on the nodes and a host allowing nested virtualization, so it runs on `remote` alone and refuses anything else. The diff --git a/tests/backup/run-cross-target-test.sh b/tests/backup/run-cross-target-test.sh new file mode 100755 index 0000000..415f669 --- /dev/null +++ b/tests/backup/run-cross-target-test.sh @@ -0,0 +1,231 @@ +#!/usr/bin/env bash +# Check that a backup taken on one deployment target restores on the other, in +# both directions: Docker Compose to Kubernetes, and Kubernetes back to Docker +# Compose. +# +# The two targets run different backup engines -- restic in a mixed-in backup +# container on Docker, K8up on a cluster -- against the same restic repository +# format, under the same repository and snapshot names. That sameness is the +# claim that lets an operator move a deployment between targets, and it is not +# covered by the backup test, whose deployments are all on one target: each +# engine there only ever reads what it wrote itself. Here each engine restores +# a repository the *other* one wrote. +# +# Flow, one bucket and three deployments of the same stack: +# +# A (compose) write a payload -> back it up (restic) +# B (k8s) assert the payload absent -> `backup restore --from` A's +# repository (K8up reading restic's) -> assert it present; +# then append a payload of its own -> back it up (K8up) +# C (compose) assert both payloads absent -> `backup restore --from` B's +# repository (restic reading K8up's) -> assert both present: +# B's is the k8s-to-compose direction, and A's has crossed +# targets twice +# +# The volume path only: `backup restore` fills volumes, and the streamed-dump +# path is deliberately recovered outside the tool with the bare restic CLI +# (see tests/database/run-backup-test.sh), which is target-independent by +# construction and already covered there. +# +# Both targets at once is what this test is, so it is remote-only three times +# over: the k8s side needs K8up, which a kind cluster does not have; the object +# store has to be reachable from the cluster and from the local daemon alike, +# which the compose target's usual in-deployment SeaweedFS is not; and the +# compose side is this machine, so "remote" here still needs a local Docker. +# It refuses any other STACK_TEST_TARGET rather than silently testing less, +# and still goes through select_deploy_target/select_backup_target for the +# remote plumbing; the compose side's arrangement is inherent to the test, so +# it lives here rather than in a target helper. +# +# Requires Docker, plus the remote-target settings (STACK_KUBE_CONFIG, +# STACK_IMAGE_REGISTRY, STACK_K8S_HOSTNAME -- tests/k3s-deploy sets them up) +# and the object store settings select_backup_target lists for remote. Run +# from the repo root, either: +# STACK_TEST_TARGET=remote ./tests/backup/run-cross-target-test.sh +# STACK_TEST_TARGET=remote ./tests/backup/run-cross-target-test.sh from-path +# +# NOTE: this fetches the test stacks and the backup stack from GitHub, so the +# `test-backup-stack` additions in bozemanpass/stack-test-stacks and the +# bozemanpass/backup-stack repo must be pushed for this to run. +source "$( dirname -- "${BASH_SOURCE[0]}" )/../lib/common.sh" + +require_commands docker + +if [ -n "$STACK_TEST_TARGET" ] && [ "$STACK_TEST_TARGET" != "remote" ]; then + fail "Error: the cross-target backup test needs a real cluster and runs on remote only, not $STACK_TEST_TARGET" +fi +export STACK_TEST_TARGET=remote + +select_test_target "$@" +select_deploy_target +select_backup_target + +app_stack="test-backup-stack" +backup_stack="backup" + +# The compose deployments start the stack's app and the mixed-in backup +# container; the k8s deployment starts the app alone, K8up being the cluster's. +compose_service_count=2 +k8s_service_count=1 + +# The compose target's startup wait: the images are local, so the remote +# target's TEST_START_CHECK_LIMIT (sized for registry pulls) does not apply. +compose_check_limit=10 + +# The payload file is /data/payload.txt because the stack's backup-command +# annotation is a cat of that path, run at backup time -- a payload anywhere +# else would make every `backup now` here fail on its dump step. +payload_a="cross-backup-compose-$$" +payload_b="cross-backup-k8s-$$" + +setup_test_dir cross-backup-test-dir + +# Fetch and prepare the stacks: the app stack for all three deployments, and +# the backup stack for the compose ones, where it is the engine. No object +# store stack: the store is the real one the remote settings name, which is +# what both targets can reach. +remove_local_images bozemanpass/backup +$TEST_TARGET_STACK fetch repo github.com/bozemanpass/stack-test-stacks +$TEST_TARGET_STACK fetch repo github.com/bozemanpass/backup-stack +$TEST_TARGET_STACK prepare --stack ${app_stack} +$TEST_TARGET_STACK prepare --stack ${backup_stack} + +compose_app_spec=$STACK_TEST_DIR/${app_stack}-compose-spec.yml +compose_backup_spec=$STACK_TEST_DIR/${backup_stack}-spec.yml +k8s_app_spec=$STACK_TEST_DIR/${app_stack}-k8s-spec.yml +deployment_a_dir=$STACK_TEST_DIR/${app_stack}-deployment-compose-a +deployment_b_dir=$STACK_TEST_DIR/${app_stack}-deployment-k8s +deployment_c_dir=$STACK_TEST_DIR/${app_stack}-deployment-compose-c + +# Two specs for one stack: the compose deployments take no target plumbing at +# all, and the k8s one takes the remote target's. `init` defaults --kube-config +# from the ambient STACK_KUBE_CONFIG the cluster harness exports, and refuses it +# on a compose spec, so the variable has to be genuinely absent -- not empty -- +# for the compose inits. +env -u STACK_KUBE_CONFIG $TEST_TARGET_STACK init --stack ${app_stack} --output "$compose_app_spec" +env -u STACK_KUBE_CONFIG $TEST_TARGET_STACK init --stack ${backup_stack} --output "$compose_backup_spec" +$TEST_TARGET_STACK init --stack ${app_stack} $TEST_INIT_ARGS --output "$k8s_app_spec" + +compose_spec_args=(--spec-file "$compose_backup_spec" --spec-file "$compose_app_spec") + +# Three deployments, torn down however the test ends: the helpers act on the +# current one (stop_deployment_on_exit), and each deployment it moves on from +# is added to this list for the extra cleanup to stop. +extra_deployment_dirs=() +stop_extra_deployments () { + local dir + for dir in "${extra_deployment_dirs[@]}"; do + $TEST_TARGET_STACK manage --dir "$dir" stop --delete-volumes + done +} +TEST_EXTRA_CLEANUP=stop_extra_deployments + +# --- A (compose): write the payload and back it up --------------------------- + +stop_deployment_on_exit "$deployment_a_dir" +$TEST_TARGET_STACK deploy "${compose_spec_args[@]}" --deployment-dir "$deployment_a_dir" +$TEST_TARGET_STACK manage --dir "$deployment_a_dir" start +wait_for_running $compose_service_count $compose_check_limit + +deployment_exec app "echo ${payload_a} > /data/payload.txt" +echo "wrote payload to the compose deployment: ${payload_a}" + +# The name of the repository this deployment's backups land in, which is all +# the other deployments are given of it. +deployment_a=$( grep "^cluster-id:" "$deployment_a_dir/deployment.yml" | awk '{print $2}' ) +if [ -z "$deployment_a" ]; then + fail "Compose backup test: FAILED - no cluster-id in the deployment" +fi + +# The engine creates the repository on first use, so no readiness loop; a +# couple of attempts cover a transient hiccup (see tests/backup/run-test.sh). +backed_up= +for i in {1..3}; do + if $TEST_TARGET_STACK manage --dir "$deployment_a_dir" backup now; then backed_up=1; break; fi + echo "backup attempt ${i} failed, retrying" + sleep 5 +done +if [ -z "$backed_up" ]; then + fail "Compose backup test: FAILED" +fi +echo "Compose backup test: passed (restic wrote ${deployment_a})" + +# --- B (k8s): restore compose's backup, then take one of its own ------------- + +$TEST_TARGET_STACK deploy --spec-file "$k8s_app_spec" --deployment-dir "$deployment_b_dir" +extra_deployment_dirs+=("$deployment_a_dir") +stop_deployment_on_exit "$deployment_b_dir" +push_images_if_needed "$deployment_b_dir" +$TEST_TARGET_STACK manage --dir "$deployment_b_dir" start +wait_for_running $k8s_service_count $TEST_START_CHECK_LIMIT + +# Fresh volumes, so the payload cannot be there yet -- only a restore that +# actually read the compose deployment's repository can put it there. +empty=$( deployment_exec app "cat /data/payload.txt" 2>/dev/null || true ) +if [[ "$empty" == *"$payload_a"* ]]; then + fail "Compose-to-k8s restore test: FAILED - the k8s deployment already had the payload before restoring" +fi + +# K8up reading the repository restic wrote. The app's second volume is +# excluded from backups, so its restore is expected to be skipped with a +# warning rather than fail the rest (see backup_restore in deploy_k8s.py). +$TEST_TARGET_STACK manage --dir "$deployment_b_dir" backup restore --from "$deployment_a" + +restored=$( deployment_exec app "cat /data/payload.txt" || true ) +if [[ "$restored" != *"$payload_a"* ]]; then + fail "Compose-to-k8s restore test: FAILED (expected '${payload_a}', got '${restored}')" +fi +echo "Compose-to-k8s restore test: passed" + +# Appended rather than written, so the file now carries both payloads and the +# compose one gets to cross targets a second time below. +deployment_exec app "echo ${payload_b} >> /data/payload.txt" +echo "appended payload to the k8s deployment: ${payload_b}" + +deployment_b=$( grep "^cluster-id:" "$deployment_b_dir/deployment.yml" | awk '{print $2}' ) +if [ -z "$deployment_b" ]; then + fail "K8s backup test: FAILED - no cluster-id in the deployment" +fi + +backed_up= +for i in {1..3}; do + if $TEST_TARGET_STACK manage --dir "$deployment_b_dir" backup now; then backed_up=1; break; fi + echo "backup attempt ${i} failed, retrying" + sleep 5 +done +if [ -z "$backed_up" ]; then + fail "K8s backup test: FAILED" +fi +echo "K8s backup test: passed (K8up wrote ${deployment_b})" + +# --- C (compose): restore the k8s deployment's backup ------------------------ + +$TEST_TARGET_STACK deploy "${compose_spec_args[@]}" --deployment-dir "$deployment_c_dir" +extra_deployment_dirs+=("$deployment_b_dir") +stop_deployment_on_exit "$deployment_c_dir" +$TEST_TARGET_STACK manage --dir "$deployment_c_dir" start +wait_for_running $compose_service_count $compose_check_limit + +empty=$( deployment_exec app "cat /data/payload.txt" 2>/dev/null || true ) +if [[ "$empty" == *"$payload_b"* ]]; then + fail "K8s-to-compose restore test: FAILED - the new compose deployment already had the payload before restoring" +fi + +# restic reading the repository K8up wrote. +$TEST_TARGET_STACK manage --dir "$deployment_c_dir" backup restore --from "$deployment_b" + +restored=$( deployment_exec app "cat /data/payload.txt" || true ) +if [[ "$restored" != *"$payload_b"* ]]; then + fail "K8s-to-compose restore test: FAILED (expected '${payload_b}', got '${restored}')" +fi +echo "K8s-to-compose restore test: passed" + +# The compose payload again, now that it has been through both engines: written +# by restic, restored by K8up, backed up by K8up, restored by restic. +if [[ "$restored" != *"$payload_a"* ]]; then + fail "Round trip test: FAILED (expected '${payload_a}' as well, got '${restored}')" +fi +echo "Round trip test: passed (the compose payload survived both directions)" + +# The registered teardown stops all three deployments and deletes their volumes. +echo "Test passed"