fix: allow safe Boat destroy after suspension

bin/fm-boat.py

@@ -236,6 +236,7 @@ def wake(id):
                     auth.acquire(id, desc['alias'], desc['config'], rows['model'])
                 reply(id, 'arm')
                 return
+            rows.pop('sleep_quiesced', None)
             rows['lifecycle'] = 'waking'; write(id, rows)
             deadline = time.monotonic() + TIMEOUT
             while True:
@@ -280,6 +281,10 @@ def sleep(id, destroy=False):
         before = rows['lifecycle']
         if before not in ('ready', 'provisioned', 'suspended'):
             raise Failure('unresolved compute must be reconciled before sleep or deletion')
+        if destroy and os.environ.get('FM_BOAT_DESTROY_DORMANT') == '1' \
+                and (before not in ('provisioned', 'suspended')
+                     or rows.get('sleep_quiesced') != '1'):
+            raise Failure('placement changed after dormant destroy checks; retry destroy')
         try:
             rows['lifecycle'] = 'suspending'; write(id, rows)
             if rows['omp_auth'] == '1':
@@ -290,6 +295,10 @@ def sleep(id, destroy=False):
                 record_path(id).unlink()
                 print('deleted: ' + id)
                 return
+            if os.environ.get('FM_BOAT_SLEEP_QUIESCED') == '1':
+                rows['sleep_quiesced'] = '1'
+            else:
+                rows.pop('sleep_quiesced', None)
             rows['lifecycle'] = 'suspended' if rows['ever_ready'] == '1' else 'provisioned'
             write(id, rows)
             print('suspended: ' + id)

bin/fm-boat.sh

@@ -2,7 +2,9 @@
 # Usage: fm-boat.sh provision|wake|sleep|destroy|status|cost|ssh <id> [options]
 # provision requires --identity <private-key> --model <provider/model>; --omp-auth
 # enables the scoped workstation credential lease. No ephemeral crew path exists.
-# Sleep/destroy share existing delivery, reply, work and decision guards.
+# Sleep/destroy share existing delivery, reply, work and decision guards;
+# destroy of a dormant placement skips remote checks only with sleep-time
+# quiescence proof, since failed wakes can also leave a suspended record.
 # Python owns provider compensation; systemd cgroups own local credential custody.
 set -euo pipefail
 SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
@@ -36,10 +38,29 @@ case "${1:-}" in
       fi
     done
     [ -z "$(status_open_decisions "$STATE/$id.status")" ] || { printf 'error: unresolved decisions\n' >&2; exit 1; }
-    if [ "$(secondmate_registry_field "$DATA/secondmates.md" "$id" remote 2>/dev/null || true)" = 1 ]; then
+    dormant_destroy=0
+    checked=0
+    remote_route=0
+    [ "$(secondmate_registry_field "$DATA/secondmates.md" "$id" remote 2>/dev/null || true)" = 1 ] && remote_route=1
+    if [ "$1" = destroy ] && [ "$remote_route" = 1 ]; then
+      lifecycle=$(grep -m1 '^lifecycle=' "$DATA/boat/$id.meta" 2>/dev/null | cut -d= -f2- || true)
+      quiesced=$(grep -m1 '^sleep_quiesced=' "$DATA/boat/$id.meta" 2>/dev/null | cut -d= -f2- || true)
+      case "$lifecycle" in
+        suspended|provisioned)
+          if [ "$quiesced" = 1 ]; then
+            dormant_destroy=1
+          else
+            printf 'error: dormant placement has no sleep-time quiescence proof; wake it so destroy can run remote checks\n' >&2
+            exit 1
+          fi
+          ;;
+      esac
+    fi
+    if [ "$remote_route" = 1 ] && [ "$dormant_destroy" = 0 ]; then
       "$SCRIPT_DIR/fm-on.sh" "$id" fm-remote-secondmate-control.sh sleep-reconcile "$id"
       children=$("$SCRIPT_DIR/fm-on.sh" "$id" fm-remote-secondmate-control.sh children "$id")
       [ "$children" = children=0 ] || { printf 'error: remote child work is active or unknown\n' >&2; exit 1; }
+      checked=1
     fi
     remote=0
     if [ -f "$STATE/$id.meta" ]; then
@@ -50,7 +71,8 @@ case "${1:-}" in
         exit 1
       fi
     fi
-    if uv run --no-project "$SCRIPT_DIR/fm-boat.py" "$@"; then
+    if FM_BOAT_SLEEP_QUIESCED=$checked FM_BOAT_DESTROY_DORMANT=$dormant_destroy \
+       uv run --no-project "$SCRIPT_DIR/fm-boat.py" "$@"; then
       [ "$remote" = 0 ] || "$SCRIPT_DIR/fm-procevent-remote-reply.sh" retire-finalize-locked "$id"
     else
       [ "$remote" = 0 ] || "$SCRIPT_DIR/fm-procevent-remote-reply.sh" arm-locked "$id" \

docs/boat-secondmates.md

@@ -62,7 +62,13 @@ A failed wake compensates by retiring credentials and requesting a confirmed sto
 Failed compensation remains explicitly unresolved and never reports a clean stop.
 A failed sleep restores credential and reply availability when the sandbox is still running; incomplete restoration remains unresolved.
 A failed bearer shred prevents deletion.
-Sleep and destroy accept an already stopped or archived sandbox without issuing another provider stop; destroy still requires checked credential cleanup before deletion.
+Sleep and destroy do not issue another provider stop for an already stopped or archived sandbox.
+For a registered remote route, destroy skips remote reconciliation and child-work checks only when the placement is suspended or provisioned and has durable proof of a successful sleep whose remote checks passed.
+Wake clears that proof before transitioning, so a suspended record left by failed-wake compensation does not qualify.
+Without proof, dormant destroy refuses before remote checks; wake the placement so destroy can run those checks.
+A running placement still requires remote reconciliation and no active or unknown child work.
+Dormant destroy rechecks the lifecycle and proof under the lifecycle lock and refuses if either no longer qualifies.
+All destroy paths retain the pending-reply, handoff, decision, credential-cleanup, and explicit confirmation guards.
 Reconcile an unresolved placement or credential record before retrying sleep or destroy; do not delete its records to bypass cleanup.
 
 ## Subscription credential boundary

tests/boat-lifecycle-cases.py

@@ -198,6 +198,40 @@ def test_sleep_guards_refuse_unresolved_reply_and_decision(self):
         self.call('sleep', ok=False); self.assertEqual(self.provider()['state'], 'ready')
         (pending / 'fixture').unlink(); (state / 'mate.status').write_text('needs-decision: [key=fixture] unresolved\n')
         self.call('sleep', ok=False); self.assertEqual(self.provider()['state'], 'ready')
+    def boat_py(self, verb, *args, ok=True, **env_extra):
+        result = subprocess.run(['uv', 'run', '--no-project', str(ROOT / 'bin/fm-boat.py'),
+                                 verb, 'mate', *args],
+                                env=dict(self.lab.env, **env_extra), capture_output=True, text=True, timeout=20)
+        if ok and result.returncode: raise AssertionError(result.stderr)
+        if not ok and not result.returncode: raise AssertionError('operation unexpectedly succeeded')
+        return result
+    def test_dormant_destroy_requires_sleep_time_quiescence_proof(self):
+        self.provision(); self.call('wake')
+        refused = self.boat_py('destroy', '--yes', ok=False, FM_BOAT_DESTROY_DORMANT='1')
+        self.assertIn('retry destroy', refused.stderr)
+        self.assertNotEqual(self.provider()['state'], 'deleted')
+        self.call('sleep')
+        self.assertNotIn('sleep_quiesced', self.state())
+        refused = self.boat_py('destroy', '--yes', ok=False, FM_BOAT_DESTROY_DORMANT='1')
+        self.assertIn('retry destroy', refused.stderr)
+        self.assertNotEqual(self.provider()['state'], 'deleted')
+        self.boat_py('destroy', '--yes', FM_BOAT_DESTROY_DORMANT='0')
+        self.assertEqual(self.provider()['state'], 'deleted')
+        self.assertFalse(self.path.exists())
+    def test_wake_invalidates_sleep_time_quiescence_proof(self):
+        self.provision(); self.call('wake')
+        self.boat_py('sleep', FM_BOAT_SLEEP_QUIESCED='1')
+        self.assertEqual(self.state()['sleep_quiesced'], '1')
+        self.call('wake')
+        self.assertNotIn('sleep_quiesced', self.state())
+        self.boat_py('sleep', FM_BOAT_SLEEP_QUIESCED='1')
+        self.lab.update(fail=['resume']); self.call('wake', ok=False)
+        self.assertEqual(self.state()['lifecycle'], 'suspended')
+        self.assertNotIn('sleep_quiesced', self.state())
+        refused = self.boat_py('destroy', '--yes', ok=False, FM_BOAT_DESTROY_DORMANT='1')
+        self.assertIn('retry destroy', refused.stderr)
+        self.assertNotEqual(self.provider()['state'], 'deleted')
+        self.lab.update(fail=[]); self.call('destroy', '--yes')
     def test_shred_failure_prevents_deletion_and_failed_stop_restores_credentials(self):
         check = subprocess.run(['systemctl', '--user', 'show', '--property=ControlGroup'], capture_output=True)
         if check.returncode: self.skipTest('Linux systemd user manager required for auth transition')

tests/fm-boat-routing.test.sh

@@ -201,4 +201,76 @@ if out=$(FM_FAKE_DOCTOR_MODE=unready world_env "$ROOT/bin/fm-spawn.sh" ios --sec
 grep -qx 'lifecycle=ready' "$w/home/data/boat/ios.meta" || fail 'spawn did not wake before readiness'
 grep -qx 'boat resume' "$w/calls" || fail 'spawn bypassed Boat wake'
 pass 'Boat wake precedes public remote spawn readiness'
+
+# Destroy of a running placement keeps the remote reachability checks: an
+# unreachable host or active remote child work refuses before the provider.
+for request in "$w/home/state/pending-replies/"*; do
+  [ -f "$request" ] || continue
+  printf 'done [corr=%s]: fixture reply\n' "$(basename "$request")" >> "$w/home/state/ios.status"
+done
+: > "$w/calls"
+: > "$w/calls.log"
+if out=$(FM_FAKE_SSH_MODE=unreachable world_env "$ROOT/bin/fm-boat.sh" destroy ios --yes 2>&1); then
+  fail 'destroy on a running placement ignored an unreachable remote'
+fi
+assert_no_grep 'boat delete' "$w/calls" 'unreachable remote still deleted the sandbox'
+grep -qx 'lifecycle=ready' "$w/home/data/boat/ios.meta" || fail 'refused destroy changed the lifecycle'
+out=$(FM_FAKE_REMOTE_CHILDREN=1 world_env "$ROOT/bin/fm-boat.sh" destroy ios --yes 2>&1) \
+  && fail 'destroy on a running placement ignored active remote child work'
+assert_contains "$out" 'remote child work is active or unknown' 'remote child refusal lost its reason'
+assert_no_grep 'boat delete' "$w/calls" 'remote child work still deleted the sandbox'
+assert_grep 'fm-remote-secondmate-control.sh children' "$w/calls.log" \
+  'running placement skipped the remote checks'
+pass 'Boat destroy on a running placement keeps remote checks and refuses when they fail'
+
+# A suspended record alone is not quiescence proof: a failed wake also ends
+# suspended through compensation, so destroy must refuse before any SSH.
+world_env "$ROOT/bin/fm-boat.sh" sleep ios >/dev/null \
+  || fail 'could not suspend the route before dormant destroy'
+grep -qx 'lifecycle=suspended' "$w/home/data/boat/ios.meta" || fail 'route did not suspend'
+grep -qx 'sleep_quiesced=1' "$w/home/data/boat/ios.meta" \
+  || fail 'proven sleep did not record quiescence proof'
+jq '.fail = ["resume"]' "$w/provider.json" > "$w/provider.tmp" && mv "$w/provider.tmp" "$w/provider.json"
+if out=$(world_env "$ROOT/bin/fm-boat.sh" wake ios 2>&1); then
+  fail 'wake unexpectedly ignored the injected resume failure'
+fi
+jq '.fail = []' "$w/provider.json" > "$w/provider.tmp" && mv "$w/provider.tmp" "$w/provider.json"
+grep -qx 'lifecycle=suspended' "$w/home/data/boat/ios.meta" \
+  || fail "compensated wake did not leave a suspended record: $(cat "$w/home/data/boat/ios.meta")"
+if grep -q '^sleep_quiesced=' "$w/home/data/boat/ios.meta"; then
+  fail 'compensated wake kept the sleep-time quiescence proof'
+fi
+: > "$w/calls"
+: > "$w/calls.log"
+out=$(FM_FAKE_SSH_MODE=unreachable world_env "$ROOT/bin/fm-boat.sh" destroy ios --yes 2>&1) \
+  && fail 'destroy on an unproven dormant placement unexpectedly succeeded'
+assert_contains "$out" 'no sleep-time quiescence proof' 'unproven dormant refusal lost its reason'
+assert_no_grep 'boat delete' "$w/calls" 'unproven dormant destroy still deleted the sandbox'
+grep -qx 'lifecycle=suspended' "$w/home/data/boat/ios.meta" || fail 'refused destroy removed the record'
+assert_no_grep 'fm-remote-secondmate-control.sh' "$w/calls.log" \
+  'unproven dormant destroy probed the remote'
+pass 'Boat destroy on an unproven dormant placement refuses before any remote check'
+
+# With durable sleep-time proof the dormant route is already known idle, so
+# destroy skips the unreachable remote checks and deletes through the provider.
+world_env "$ROOT/bin/fm-boat.sh" wake ios >/dev/null \
+  || fail 'could not wake the route after clearing the injected failure'
+world_env "$ROOT/bin/fm-boat.sh" sleep ios >/dev/null \
+  || fail 'could not suspend the route before dormant destroy'
+grep -qx 'sleep_quiesced=1' "$w/home/data/boat/ios.meta" \
+  || fail 'proven sleep did not record quiescence proof'
+: > "$w/calls"
+: > "$w/calls.log"
+if out=$(FM_FAKE_SSH_MODE=unreachable world_env "$ROOT/bin/fm-boat.sh" destroy ios 2>&1); then
+  fail 'destroy without --yes unexpectedly succeeded'
+fi
+assert_no_grep 'boat delete' "$w/calls" 'unconfirmed destroy still deleted the sandbox'
+grep -qx 'lifecycle=suspended' "$w/home/data/boat/ios.meta" || fail 'unconfirmed destroy removed the record'
+out=$(FM_FAKE_SSH_MODE=unreachable world_env "$ROOT/bin/fm-boat.sh" destroy ios --yes 2>&1) \
+  || fail "destroy on a proven suspended placement failed: $out"
+assert_grep 'boat delete' "$w/calls" 'suspended destroy did not delete the sandbox'
+[ ! -e "$w/home/data/boat/ios.meta" ] || fail 'suspended destroy left the record behind'
+assert_no_grep 'fm-remote-secondmate-control.sh' "$w/calls.log" \
+  'suspended destroy probed the unreachable remote'
+pass 'Boat destroy on a suspended placement skips unreachable remote checks and deletes'
 fm_test_cleanup