diff --git a/docs/changelog.md b/docs/changelog.md index 0bfea0a3..816ab6ce 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -19,6 +19,36 @@ docstring gives: it is a record, not a claim about now. ## 2026-11-08 +- **A copy run under the wrong `STACK` pruned five observability sets on + `oracle`, and `backup-volumes.sh` now refuses it.** `ScheduledJobStale` + fired for `backup-sensitive` and `verify-backups-sensitive` on `trinity`. + The backup had already recovered: its 04:30 UTC run exited 0. The verify + had failed since 2026-11-02, because that day's and 2026-11-03's sets were + written while `oracle` could not be reached and were missing there. The + 04:30 run's copy step caught them up, and the 06:31 verify passed. + - **The mistake:** while catching up by hand, `make backup + ARGS=--copy-only` was run on `trinity` without `STACK=sensitive`. The + systemd units set it and a shell does not, so it ran as `observability`. + `backups/volumes/` is one directory for every stack on a host, and only + the far side is per stack. So all seven of `trinity`'s sensitive sets + were copied into `oracle`'s `backups/volumes/observability`. Retention + there then kept the newest seven and deleted the five real observability + sets, `20261004T033010Z` to `20261029T213621Z`. Nothing was exposed: + the misplaced sets were the same ciphertext as `oracle`'s sensitive + copies. + - **Repaired the same morning:** the seven misplaced sets were removed + after their MANIFESTs were matched against `oracle`'s sensitive copies. + `make backup ARGS=--copy-only STACK=observability` on `prometheus` put + the five back from its local sets, each hash-checked on arrival. + `oracle` again holds all seven observability sets, 2026-10-04 to + 2026-11-08. + - **The guard:** `copy_offhost` now refuses, before any ssh, every set + whose MANIFEST names a stack other than the one it copies for. That is + `STACK`, or `media` for `backup-nas.sh`, which shares the copy step. The + refusal fails the copy, so the far-side prune never runs. `--self-test` + plants a set from another stack and checks the refusal, and checks that + NAS sets still pass. With either half disabled, the test fails. + - **The WAN's receive errors came back on the XB7's gigabit port, and moving to its other gigabit port cleared them.** Later on the same day as the [#923](https://github.com/Gerrrt/HomeLab/issues/923) move, `em0` flapped diff --git a/scripts/backup-nas.sh b/scripts/backup-nas.sh index cef0c435..678bb390 100755 --- a/scripts/backup-nas.sh +++ b/scripts/backup-nas.sh @@ -162,6 +162,11 @@ REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" # here, and set unconditionally: the environment file the timer units share # carries the volume sets' value, which is the wrong directory for this set. VOL_OFFHOST="${NAS_OFFHOST:-atropos@10.0.99.30:backups/nas}" +# Every set this script writes names `stack media` (its MANIFEST block), while +# STACK stays at backup-volumes.sh's default. copy_offhost refuses a set whose +# MANIFEST names a stack other than SET_STACK, so this is set here, +# unconditionally, for the same reason VOL_OFFHOST is. +SET_STACK=media # shellcheck source=scripts/backup-volumes.sh source "${REPO_ROOT}/scripts/backup-volumes.sh" diff --git a/scripts/backup-volumes.sh b/scripts/backup-volumes.sh index cbe18c3a..5bda8876 100755 --- a/scripts/backup-volumes.sh +++ b/scripts/backup-volumes.sh @@ -173,7 +173,9 @@ REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" # property the two-recipient rule (#835) exists for, which until this was # asserted only about the secrets file and never about an archive; # - a flipped byte, an archive without its sentinel, an archive carrying -# another volume's, and a project whose volume does not exist all refuse. +# another volume's, and a project whose volume does not exist all refuse; +# - --copy-only refuses a set whose MANIFEST names another stack, before +# ssh (2026-11-08). # --------------------------------------------------------------------------- if [[ ${BASH_SOURCE[0]} == "$0" && "${1:-}" == "--self-test" ]]; then # shellcheck source=scripts/volume-selftest-fixture.sh @@ -210,6 +212,31 @@ if [[ ${BASH_SOURCE[0]} == "$0" && "${1:-}" == "--self-test" ]]; then expect "a flipped byte does not verify" 1 "FAILED to decrypt or read" rm -rf "${VF_REPO}/backups/volumes/20000101T000000Z" + # Another stack's set in the shared ${OUT_DIR}, the 2026-11-08 shape. The + # refusal comes before ssh. If it ever stops doing that, the run would try + # the .invalid target and fail with "cannot reach" instead. The real oracle + # is never named. + cp -a "${SET}" "${VF_REPO}/backups/volumes/20000102T000000Z" + sed -i 's/^stack\t.*/stack\tsensitive/' "${VF_REPO}/backups/volumes/20000102T000000Z/MANIFEST" + vf_run "${ENV[@]}" VOL_OFFHOST=selftest@offhost.invalid:backups/volumes/observability -- backup-volumes.sh --copy-only + expect "refuses to copy a set whose MANIFEST names another stack" 1 "names stack 'sensitive', and this run copies observability's sets" + check " and names no other set" "$(grep -c 'names stack' <<<"${OUT}")" 1 + check " and never reaches for the far side" "$(grep -c 'cannot reach' <<<"${OUT}")" 0 + vf_run "${ENV[@]}" SET_STACK=sensitive VOL_OFFHOST=selftest@offhost.invalid:backups/volumes/observability -- backup-volumes.sh --copy-only + expect " and a SET_STACK in the environment does not loosen a direct run" 1 "and this run copies observability's sets" + rm -rf "${VF_REPO}/backups/volumes/20000102T000000Z" + + # The same check from a caller whose sets name another stack: backup-nas.sh + # leaves STACK at observability and writes `stack media`. Its set must get + # PAST the check, to the .invalid far side, and fail there. + mkdir -p "${VF_REPO}/backups/nas" + cp -a "${SET}" "${VF_REPO}/backups/nas/20000103T000000Z" + sed -i 's/^stack\t.*/stack\tmedia/' "${VF_REPO}/backups/nas/20000103T000000Z/MANIFEST" + vf_run NAS_OFFHOST=selftest@offhost.invalid:backups/nas TEXTFILE_DIR="${VF}/no-textfile" -- backup-nas.sh --copy-only + expect "backup-nas.sh's media sets pass the stack check" 1 "cannot reach selftest@offhost.invalid" + check " and none is refused as another stack's" "$(grep -c 'names stack' <<<"${OUT}")" 0 + rm -rf "${VF_REPO}/backups/nas" + vf_volume "${VF_P}u" unmarked vf_new_second vf_run "${ENV[@]}" COMPOSE_PROJECT_NAME="${VF_P}u" SOPS_AGE_KEY_FILE="${VF_K1}" -- backup-volumes.sh --local-only @@ -236,6 +263,16 @@ if [[ ${BASH_SOURCE[0]} == "$0" && "${1:-}" == "--self-test" ]]; then fi STACK="${STACK:-observability}" +# The stack a set's MANIFEST must name before copy_offhost sends it +# (refuse_foreign_sets). Run directly, that is STACK. A caller whose sets name +# another stack sets SET_STACK before its source line: backup-nas.sh's sets +# name `media` while STACK stays at its default. The environment is read only +# when sourced, so a stray SET_STACK cannot loosen a direct run. +if [[ ${BASH_SOURCE[0]} == "$0" ]]; then + SET_STACK="${STACK}" +else + SET_STACK="${SET_STACK:-${STACK}}" +fi STACK_DIR="${REPO_ROOT}/stacks/${STACK}" COMPOSE_FILE="${STACK_DIR}/compose.yaml" OUT_DIR="${REPO_ROOT}/backups/volumes" @@ -1004,6 +1041,28 @@ verify_remote_sets() { green "verified ${n} set(s) on ${OFFHOST_TARGET}:${OFFHOST_DIR} — every archive hashes to its MANIFEST entry, every MANIFEST byte-identical" } +# ${OUT_DIR} is one directory for every stack on this host; only the far side +# is per STACK. So a run under the wrong STACK sees another stack's sets as its +# own. On 2026-11-08 `make backup ARGS=--copy-only` on trinity, without +# STACK=sensitive, copied trinity's seven sensitive sets into oracle's +# observability directory. prune_offhost then kept the newest KEEP of them and +# deleted the five real observability sets there. Every MANIFEST has named its +# stack since the first set this script wrote, so a set whose MANIFEST names +# a stack other than SET_STACK, or none, is refused before anything is sent. One refusal stops +# the whole copy, and the prune after it never runs. +refuse_foreign_sets() { + local d stack bad=0 + for d in "$@"; do + stack="$(manifest_field "${d}" stack)" + [[ ${stack} == "${SET_STACK}" ]] && continue + red "$(basename "${d}"): its MANIFEST names stack '${stack:-}', and this run copies ${SET_STACK}'s sets" + bad=1 + done + ((bad == 0)) && return 0 + red "refusing to copy to ${OFFHOST_TARGET}:${OFFHOST_DIR} — run it with STACK set to the stack these sets belong to" + return 1 +} + # Every complete local set the far side does not have, newest first, not just # the one this run wrote. A Sunday oracle was off would otherwise leave one set # that never left this host, and nothing would ever go back for it. Archives @@ -1017,6 +1076,7 @@ copy_offhost() { info "no complete sets in ${OUT_DIR} — nothing to copy" return 0 fi + refuse_foreign_sets "${local_sets[@]}" || return 1 read_remote || { red "cannot reach ${OFFHOST_TARGET}"; return 1; } remote "mkdir -p '${OFFHOST_DIR}' && chmod 700 '${OFFHOST_DIR}'" \ || { red "cannot create ${OFFHOST_DIR} on ${OFFHOST_TARGET}"; return 1; } diff --git a/scripts/volume-selftest-fixture.sh b/scripts/volume-selftest-fixture.sh index 4b575689..cbecce1a 100644 --- a/scripts/volume-selftest-fixture.sh +++ b/scripts/volume-selftest-fixture.sh @@ -61,6 +61,7 @@ vf_setup() { VF_REPO="${VF}/repo" mkdir -p "${VF_REPO}/scripts" "${VF_REPO}/stacks/observability" "${VF_REPO}/secrets" cp "${real}/scripts/backup-volumes.sh" "${real}/scripts/restore-volumes.sh" \ + "${real}/scripts/backup-nas.sh" \ "${real}/scripts/key-recipients.sh" "${real}/scripts/image-for.sh" \ "${VF_REPO}/scripts/"