From 259df4554fa787c1bf6507ae8ce9fdebf0eec3a8 Mon Sep 17 00:00:00 2001
From: Garrett Allen <98648590+Gerrrt@users.noreply.github.com>
Date: Sun, 8 Nov 2026 22:01:02 +0000
Subject: [PATCH 1/2] fix(backup): refuse to copy another stack's sets off-host

backups/volumes/ is one directory for every stack on a host; only the far
side is per STACK. On 2026-11-08 a hand-run `make backup ARGS=--copy-only`
on trinity without STACK=sensitive copied the sensitive sets into oracle's
observability directory, and the far-side prune deleted five real
observability sets. They were restored the same morning.

copy_offhost now refuses, before any ssh, every set whose MANIFEST names a
stack other than STACK, so the prune after it never runs. --self-test
plants a foreign set and checks the refusal; it fails with the guard off.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 docs/changelog.md         | 29 +++++++++++++++++++++++++++++
 scripts/backup-volumes.sh | 39 ++++++++++++++++++++++++++++++++++++++-
 2 files changed, 67 insertions(+), 1 deletion(-)

diff --git a/docs/changelog.md b/docs/changelog.md
index 0bfea0a3..06a2e3c5 100644
--- a/docs/changelog.md
+++ b/docs/changelog.md
@@ -19,6 +19,35 @@ docstring gives: it is a record, not a claim about now.
 
 ## 2026-11-08
 
+- **A copy run under the wrong `STACK` pruned five observability sets on
+  `oracle`, and `backup-volumes.sh` now refuses it.** `ScheduledJobStale`
+  fired for `backup-sensitive` and `verify-backups-sensitive` on `trinity`.
+  The backup had already recovered: its 04:30 UTC run exited 0. The verify
+  had failed since 2026-11-02, because that day's and 2026-11-03's sets were
+  written while `oracle` could not be reached and were missing there. The
+  04:30 run's copy step caught them up, and the 06:31 verify passed.
+  - **The mistake:** while catching up by hand, `make backup
+    ARGS=--copy-only` was run on `trinity` without `STACK=sensitive`. The
+    systemd units set it and a shell does not, so it ran as `observability`.
+    `backups/volumes/` is one directory for every stack on a host, and only
+    the far side is per stack. So all seven of `trinity`'s sensitive sets
+    were copied into `oracle`'s `backups/volumes/observability`. Retention
+    there then kept the newest seven and deleted the five real observability
+    sets, `20261004T033010Z` to `20261029T213621Z`. Nothing was exposed:
+    the misplaced sets were the same ciphertext as `oracle`'s sensitive
+    copies.
+  - **Repaired the same morning:** the seven misplaced sets were removed
+    after their MANIFESTs were matched against `oracle`'s sensitive copies.
+    `make backup ARGS=--copy-only STACK=observability` on `prometheus` put
+    the five back from its local sets, each hash-checked on arrival.
+    `oracle` again holds all seven observability sets, 2026-10-04 to
+    2026-11-08.
+  - **The guard:** `copy_offhost` now refuses, before any ssh, every set
+    whose MANIFEST names a stack other than `STACK`. The refusal fails the
+    copy, so the far-side prune never runs. `--self-test` plants a set
+    from another stack and checks the refusal. With the guard disabled, the
+    test fails.
+
 - **The WAN's receive errors came back on the XB7's gigabit port, and moving
   to its other gigabit port cleared them.** Later on the same day as the
   [#923](https://github.com/Gerrrt/HomeLab/issues/923) move, `em0` flapped
diff --git a/scripts/backup-volumes.sh b/scripts/backup-volumes.sh
index cbe18c3a..b4f60fce 100755
--- a/scripts/backup-volumes.sh
+++ b/scripts/backup-volumes.sh
@@ -173,7 +173,9 @@ REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
 #     property the two-recipient rule (#835) exists for, which until this was
 #     asserted only about the secrets file and never about an archive;
 #   - a flipped byte, an archive without its sentinel, an archive carrying
-#     another volume's, and a project whose volume does not exist all refuse.
+#     another volume's, and a project whose volume does not exist all refuse;
+#   - --copy-only refuses a set whose MANIFEST names another stack, before
+#     ssh (2026-11-08).
 # ---------------------------------------------------------------------------
 if [[ ${BASH_SOURCE[0]} == "$0" && "${1:-}" == "--self-test" ]]; then
   # shellcheck source=scripts/volume-selftest-fixture.sh
@@ -210,6 +212,18 @@ if [[ ${BASH_SOURCE[0]} == "$0" && "${1:-}" == "--self-test" ]]; then
   expect "a flipped byte does not verify" 1 "FAILED to decrypt or read"
   rm -rf "${VF_REPO}/backups/volumes/20000101T000000Z"
 
+  # Another stack's set in the shared ${OUT_DIR}, the 2026-11-08 shape. The
+  # refusal comes before ssh. If it ever stops doing that, the run would try
+  # the .invalid target and fail with "cannot reach" instead. The real oracle
+  # is never named.
+  cp -a "${SET}" "${VF_REPO}/backups/volumes/20000102T000000Z"
+  sed -i 's/^stack\t.*/stack\tsensitive/' "${VF_REPO}/backups/volumes/20000102T000000Z/MANIFEST"
+  vf_run "${ENV[@]}" VOL_OFFHOST=selftest@offhost.invalid:backups/volumes/observability -- backup-volumes.sh --copy-only
+  expect "refuses to copy a set whose MANIFEST names another stack" 1 "names stack 'sensitive', and this run is STACK=observability"
+  check "  and names no other set" "$(grep -c 'names stack' <<<"${OUT}")" 1
+  check "  and never reaches for the far side" "$(grep -c 'cannot reach' <<<"${OUT}")" 0
+  rm -rf "${VF_REPO}/backups/volumes/20000102T000000Z"
+
   vf_volume "${VF_P}u" unmarked
   vf_new_second
   vf_run "${ENV[@]}" COMPOSE_PROJECT_NAME="${VF_P}u" SOPS_AGE_KEY_FILE="${VF_K1}" -- backup-volumes.sh --local-only
@@ -1004,6 +1018,28 @@ verify_remote_sets() {
   green "verified ${n} set(s) on ${OFFHOST_TARGET}:${OFFHOST_DIR} — every archive hashes to its MANIFEST entry, every MANIFEST byte-identical"
 }
 
+# ${OUT_DIR} is one directory for every stack on this host; only the far side
+# is per STACK. So a run under the wrong STACK sees another stack's sets as its
+# own. On 2026-11-08 `make backup ARGS=--copy-only` on trinity, without
+# STACK=sensitive, copied trinity's seven sensitive sets into oracle's
+# observability directory. prune_offhost then kept the newest KEEP of them and
+# deleted the five real observability sets there. Every MANIFEST has named its
+# stack since the first set this script wrote, so a set whose MANIFEST names
+# another stack, or none, is refused before anything is sent. One refusal stops
+# the whole copy, and the prune after it never runs.
+refuse_foreign_sets() {
+  local d stack bad=0
+  for d in "$@"; do
+    stack="$(manifest_field "${d}" stack)"
+    [[ ${stack} == "${STACK}" ]] && continue
+    red "$(basename "${d}"): its MANIFEST names stack '${stack:-<none>}', and this run is STACK=${STACK}"
+    bad=1
+  done
+  ((bad == 0)) && return 0
+  red "refusing to copy to ${OFFHOST_TARGET}:${OFFHOST_DIR} — set STACK to the stack these sets belong to"
+  return 1
+}
+
 # Every complete local set the far side does not have, newest first, not just
 # the one this run wrote. A Sunday oracle was off would otherwise leave one set
 # that never left this host, and nothing would ever go back for it. Archives
@@ -1017,6 +1053,7 @@ copy_offhost() {
     info "no complete sets in ${OUT_DIR} — nothing to copy"
     return 0
   fi
+  refuse_foreign_sets "${local_sets[@]}" || return 1
   read_remote || { red "cannot reach ${OFFHOST_TARGET}"; return 1; }
   remote "mkdir -p '${OFFHOST_DIR}' && chmod 700 '${OFFHOST_DIR}'" \
     || { red "cannot create ${OFFHOST_DIR} on ${OFFHOST_TARGET}"; return 1; }

From 1ff26054e3d75a2077dcb94cb76be19792b23018 Mon Sep 17 00:00:00 2001
From: Garrett Allen <98648590+Gerrrt@users.noreply.github.com>
Date: Sun, 8 Nov 2026 22:23:06 +0000
Subject: [PATCH 2/2] fix(backup): let backup-nas.sh's media sets through the
 stack check

backup-nas.sh sources backup-volumes.sh with STACK at its observability
default, but every NAS MANIFEST names `stack media`, so the new check in
copy_offhost refused every NAS set. The check now compares against
SET_STACK: STACK on a direct run (the environment is not read), and what
a sourcing caller sets otherwise. backup-nas.sh sets SET_STACK=media.

The self-test now copies backup-nas.sh into its fixture repo and checks
that a media set passes the check to the far side, and that SET_STACK
in the environment does not loosen a direct run. With SET_STACK=media
removed, the NAS fixture fails.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 docs/changelog.md                  |  9 ++++----
 scripts/backup-nas.sh              |  5 +++++
 scripts/backup-volumes.sh          | 33 +++++++++++++++++++++++++-----
 scripts/volume-selftest-fixture.sh |  1 +
 4 files changed, 39 insertions(+), 9 deletions(-)

diff --git a/docs/changelog.md b/docs/changelog.md
index 06a2e3c5..816ab6ce 100644
--- a/docs/changelog.md
+++ b/docs/changelog.md
@@ -43,10 +43,11 @@ docstring gives: it is a record, not a claim about now.
     `oracle` again holds all seven observability sets, 2026-10-04 to
     2026-11-08.
   - **The guard:** `copy_offhost` now refuses, before any ssh, every set
-    whose MANIFEST names a stack other than `STACK`. The refusal fails the
-    copy, so the far-side prune never runs. `--self-test` plants a set
-    from another stack and checks the refusal. With the guard disabled, the
-    test fails.
+    whose MANIFEST names a stack other than the one it copies for. That is
+    `STACK`, or `media` for `backup-nas.sh`, which shares the copy step. The
+    refusal fails the copy, so the far-side prune never runs. `--self-test`
+    plants a set from another stack and checks the refusal, and checks that
+    NAS sets still pass. With either half disabled, the test fails.
 
 - **The WAN's receive errors came back on the XB7's gigabit port, and moving
   to its other gigabit port cleared them.** Later on the same day as the
diff --git a/scripts/backup-nas.sh b/scripts/backup-nas.sh
index cef0c435..678bb390 100755
--- a/scripts/backup-nas.sh
+++ b/scripts/backup-nas.sh
@@ -162,6 +162,11 @@ REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
 # here, and set unconditionally: the environment file the timer units share
 # carries the volume sets' value, which is the wrong directory for this set.
 VOL_OFFHOST="${NAS_OFFHOST:-atropos@10.0.99.30:backups/nas}"
+# Every set this script writes names `stack media` (its MANIFEST block), while
+# STACK stays at backup-volumes.sh's default. copy_offhost refuses a set whose
+# MANIFEST names a stack other than SET_STACK, so this is set here,
+# unconditionally, for the same reason VOL_OFFHOST is.
+SET_STACK=media
 # shellcheck source=scripts/backup-volumes.sh
 source "${REPO_ROOT}/scripts/backup-volumes.sh"
 
diff --git a/scripts/backup-volumes.sh b/scripts/backup-volumes.sh
index b4f60fce..5bda8876 100755
--- a/scripts/backup-volumes.sh
+++ b/scripts/backup-volumes.sh
@@ -219,11 +219,24 @@ if [[ ${BASH_SOURCE[0]} == "$0" && "${1:-}" == "--self-test" ]]; then
   cp -a "${SET}" "${VF_REPO}/backups/volumes/20000102T000000Z"
   sed -i 's/^stack\t.*/stack\tsensitive/' "${VF_REPO}/backups/volumes/20000102T000000Z/MANIFEST"
   vf_run "${ENV[@]}" VOL_OFFHOST=selftest@offhost.invalid:backups/volumes/observability -- backup-volumes.sh --copy-only
-  expect "refuses to copy a set whose MANIFEST names another stack" 1 "names stack 'sensitive', and this run is STACK=observability"
+  expect "refuses to copy a set whose MANIFEST names another stack" 1 "names stack 'sensitive', and this run copies observability's sets"
   check "  and names no other set" "$(grep -c 'names stack' <<<"${OUT}")" 1
   check "  and never reaches for the far side" "$(grep -c 'cannot reach' <<<"${OUT}")" 0
+  vf_run "${ENV[@]}" SET_STACK=sensitive VOL_OFFHOST=selftest@offhost.invalid:backups/volumes/observability -- backup-volumes.sh --copy-only
+  expect "  and a SET_STACK in the environment does not loosen a direct run" 1 "and this run copies observability's sets"
   rm -rf "${VF_REPO}/backups/volumes/20000102T000000Z"
 
+  # The same check from a caller whose sets name another stack: backup-nas.sh
+  # leaves STACK at observability and writes `stack media`. Its set must get
+  # PAST the check, to the .invalid far side, and fail there.
+  mkdir -p "${VF_REPO}/backups/nas"
+  cp -a "${SET}" "${VF_REPO}/backups/nas/20000103T000000Z"
+  sed -i 's/^stack\t.*/stack\tmedia/' "${VF_REPO}/backups/nas/20000103T000000Z/MANIFEST"
+  vf_run NAS_OFFHOST=selftest@offhost.invalid:backups/nas TEXTFILE_DIR="${VF}/no-textfile" -- backup-nas.sh --copy-only
+  expect "backup-nas.sh's media sets pass the stack check" 1 "cannot reach selftest@offhost.invalid"
+  check "  and none is refused as another stack's" "$(grep -c 'names stack' <<<"${OUT}")" 0
+  rm -rf "${VF_REPO}/backups/nas"
+
   vf_volume "${VF_P}u" unmarked
   vf_new_second
   vf_run "${ENV[@]}" COMPOSE_PROJECT_NAME="${VF_P}u" SOPS_AGE_KEY_FILE="${VF_K1}" -- backup-volumes.sh --local-only
@@ -250,6 +263,16 @@ if [[ ${BASH_SOURCE[0]} == "$0" && "${1:-}" == "--self-test" ]]; then
 fi
 
 STACK="${STACK:-observability}"
+# The stack a set's MANIFEST must name before copy_offhost sends it
+# (refuse_foreign_sets). Run directly, that is STACK. A caller whose sets name
+# another stack sets SET_STACK before its source line: backup-nas.sh's sets
+# name `media` while STACK stays at its default. The environment is read only
+# when sourced, so a stray SET_STACK cannot loosen a direct run.
+if [[ ${BASH_SOURCE[0]} == "$0" ]]; then
+  SET_STACK="${STACK}"
+else
+  SET_STACK="${SET_STACK:-${STACK}}"
+fi
 STACK_DIR="${REPO_ROOT}/stacks/${STACK}"
 COMPOSE_FILE="${STACK_DIR}/compose.yaml"
 OUT_DIR="${REPO_ROOT}/backups/volumes"
@@ -1025,18 +1048,18 @@ verify_remote_sets() {
 # observability directory. prune_offhost then kept the newest KEEP of them and
 # deleted the five real observability sets there. Every MANIFEST has named its
 # stack since the first set this script wrote, so a set whose MANIFEST names
-# another stack, or none, is refused before anything is sent. One refusal stops
+# a stack other than SET_STACK, or none, is refused before anything is sent. One refusal stops
 # the whole copy, and the prune after it never runs.
 refuse_foreign_sets() {
   local d stack bad=0
   for d in "$@"; do
     stack="$(manifest_field "${d}" stack)"
-    [[ ${stack} == "${STACK}" ]] && continue
-    red "$(basename "${d}"): its MANIFEST names stack '${stack:-<none>}', and this run is STACK=${STACK}"
+    [[ ${stack} == "${SET_STACK}" ]] && continue
+    red "$(basename "${d}"): its MANIFEST names stack '${stack:-<none>}', and this run copies ${SET_STACK}'s sets"
     bad=1
   done
   ((bad == 0)) && return 0
-  red "refusing to copy to ${OFFHOST_TARGET}:${OFFHOST_DIR} — set STACK to the stack these sets belong to"
+  red "refusing to copy to ${OFFHOST_TARGET}:${OFFHOST_DIR} — run it with STACK set to the stack these sets belong to"
   return 1
 }
 
diff --git a/scripts/volume-selftest-fixture.sh b/scripts/volume-selftest-fixture.sh
index 4b575689..cbecce1a 100644
--- a/scripts/volume-selftest-fixture.sh
+++ b/scripts/volume-selftest-fixture.sh
@@ -61,6 +61,7 @@ vf_setup() {
   VF_REPO="${VF}/repo"
   mkdir -p "${VF_REPO}/scripts" "${VF_REPO}/stacks/observability" "${VF_REPO}/secrets"
   cp "${real}/scripts/backup-volumes.sh" "${real}/scripts/restore-volumes.sh" \
+     "${real}/scripts/backup-nas.sh" \
      "${real}/scripts/key-recipients.sh" "${real}/scripts/image-for.sh" \
      "${VF_REPO}/scripts/"