From ade1a1aebf9d9c2a1c14e556b3d38c38bc9f78a2 Mon Sep 17 00:00:00 2001 From: dnth Date: Mon, 5 Oct 2026 19:25:19 +0800 Subject: [PATCH 1/2] fix(bin): let Boat destroy a proven-dormant placement without SSH Destroy on a suspended remote placement ran sleep-reconcile and children through fm-on.sh, which needs a reachable host; a stopped sandbox returns SSH 255, so sleep then destroy required a wake first. Sleep now records sleep_quiesced=1 when its remote checks passed, and wake clears it, so a failed-wake compensation that ends suspended carries no proof. Destroy skips the remote checks only for a suspended or provisioned record with that proof, refuses an unproven dormant record before any remote call, and fm-boat.py re-checks the proof under its lock. Running placements keep the remote checks; children, unlanded work, decisions and --yes guards are unchanged. --- bin/fm-boat.py | 9 +++++ bin/fm-boat.sh | 28 ++++++++++++-- tests/boat-lifecycle-cases.py | 34 +++++++++++++++++ tests/fm-boat-routing.test.sh | 72 +++++++++++++++++++++++++++++++++++ 4 files changed, 140 insertions(+), 3 deletions(-) diff --git a/bin/fm-boat.py b/bin/fm-boat.py index 66241065acd..9f32982d2ff 100755 --- a/bin/fm-boat.py +++ b/bin/fm-boat.py @@ -236,6 +236,7 @@ def wake(id): auth.acquire(id, desc['alias'], desc['config'], rows['model']) reply(id, 'arm') return + rows.pop('sleep_quiesced', None) rows['lifecycle'] = 'waking'; write(id, rows) deadline = time.monotonic() + TIMEOUT while True: @@ -280,6 +281,10 @@ def sleep(id, destroy=False): before = rows['lifecycle'] if before not in ('ready', 'provisioned', 'suspended'): raise Failure('unresolved compute must be reconciled before sleep or deletion') + if destroy and os.environ.get('FM_BOAT_DESTROY_DORMANT') == '1' \ + and (before not in ('provisioned', 'suspended') + or rows.get('sleep_quiesced') != '1'): + raise Failure('placement changed after dormant destroy checks; retry destroy') try: rows['lifecycle'] = 'suspending'; write(id, rows) if rows['omp_auth'] == '1': @@ -290,6 +295,10 @@ def sleep(id, destroy=False): record_path(id).unlink() print('deleted: ' + id) return + if os.environ.get('FM_BOAT_SLEEP_QUIESCED') == '1': + rows['sleep_quiesced'] = '1' + else: + rows.pop('sleep_quiesced', None) rows['lifecycle'] = 'suspended' if rows['ever_ready'] == '1' else 'provisioned' write(id, rows) print('suspended: ' + id) diff --git a/bin/fm-boat.sh b/bin/fm-boat.sh index 023f99cab64..896c0916d90 100755 --- a/bin/fm-boat.sh +++ b/bin/fm-boat.sh @@ -2,7 +2,9 @@ # Usage: fm-boat.sh provision|wake|sleep|destroy|status|cost|ssh [options] # provision requires --identity --model ; --omp-auth # enables the scoped workstation credential lease. No ephemeral crew path exists. -# Sleep/destroy share existing delivery, reply, work and decision guards. +# Sleep/destroy share existing delivery, reply, work and decision guards; +# destroy of a dormant placement skips remote checks only with sleep-time +# quiescence proof, since failed wakes can also leave a suspended record. # Python owns provider compensation; systemd cgroups own local credential custody. set -euo pipefail SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) @@ -36,10 +38,29 @@ case "${1:-}" in fi done [ -z "$(status_open_decisions "$STATE/$id.status")" ] || { printf 'error: unresolved decisions\n' >&2; exit 1; } - if [ "$(secondmate_registry_field "$DATA/secondmates.md" "$id" remote 2>/dev/null || true)" = 1 ]; then + dormant_destroy=0 + checked=0 + remote_route=0 + [ "$(secondmate_registry_field "$DATA/secondmates.md" "$id" remote 2>/dev/null || true)" = 1 ] && remote_route=1 + if [ "$1" = destroy ] && [ "$remote_route" = 1 ]; then + lifecycle=$(grep -m1 '^lifecycle=' "$DATA/boat/$id.meta" 2>/dev/null | cut -d= -f2- || true) + quiesced=$(grep -m1 '^sleep_quiesced=' "$DATA/boat/$id.meta" 2>/dev/null | cut -d= -f2- || true) + case "$lifecycle" in + suspended|provisioned) + if [ "$quiesced" = 1 ]; then + dormant_destroy=1 + else + printf 'error: dormant placement has no sleep-time quiescence proof; wake it so destroy can run remote checks\n' >&2 + exit 1 + fi + ;; + esac + fi + if [ "$remote_route" = 1 ] && [ "$dormant_destroy" = 0 ]; then "$SCRIPT_DIR/fm-on.sh" "$id" fm-remote-secondmate-control.sh sleep-reconcile "$id" children=$("$SCRIPT_DIR/fm-on.sh" "$id" fm-remote-secondmate-control.sh children "$id") [ "$children" = children=0 ] || { printf 'error: remote child work is active or unknown\n' >&2; exit 1; } + checked=1 fi remote=0 if [ -f "$STATE/$id.meta" ]; then @@ -50,7 +71,8 @@ case "${1:-}" in exit 1 fi fi - if uv run --no-project "$SCRIPT_DIR/fm-boat.py" "$@"; then + if FM_BOAT_SLEEP_QUIESCED=$checked FM_BOAT_DESTROY_DORMANT=$dormant_destroy \ + uv run --no-project "$SCRIPT_DIR/fm-boat.py" "$@"; then [ "$remote" = 0 ] || "$SCRIPT_DIR/fm-procevent-remote-reply.sh" retire-finalize-locked "$id" else [ "$remote" = 0 ] || "$SCRIPT_DIR/fm-procevent-remote-reply.sh" arm-locked "$id" \ diff --git a/tests/boat-lifecycle-cases.py b/tests/boat-lifecycle-cases.py index a3d63a053b8..52839acfb7f 100755 --- a/tests/boat-lifecycle-cases.py +++ b/tests/boat-lifecycle-cases.py @@ -198,6 +198,40 @@ def test_sleep_guards_refuse_unresolved_reply_and_decision(self): self.call('sleep', ok=False); self.assertEqual(self.provider()['state'], 'ready') (pending / 'fixture').unlink(); (state / 'mate.status').write_text('needs-decision: [key=fixture] unresolved\n') self.call('sleep', ok=False); self.assertEqual(self.provider()['state'], 'ready') + def boat_py(self, verb, *args, ok=True, **env_extra): + result = subprocess.run(['uv', 'run', '--no-project', str(ROOT / 'bin/fm-boat.py'), + verb, 'mate', *args], + env=dict(self.lab.env, **env_extra), capture_output=True, text=True, timeout=20) + if ok and result.returncode: raise AssertionError(result.stderr) + if not ok and not result.returncode: raise AssertionError('operation unexpectedly succeeded') + return result + def test_dormant_destroy_requires_sleep_time_quiescence_proof(self): + self.provision(); self.call('wake') + refused = self.boat_py('destroy', '--yes', ok=False, FM_BOAT_DESTROY_DORMANT='1') + self.assertIn('retry destroy', refused.stderr) + self.assertNotEqual(self.provider()['state'], 'deleted') + self.call('sleep') + self.assertNotIn('sleep_quiesced', self.state()) + refused = self.boat_py('destroy', '--yes', ok=False, FM_BOAT_DESTROY_DORMANT='1') + self.assertIn('retry destroy', refused.stderr) + self.assertNotEqual(self.provider()['state'], 'deleted') + self.boat_py('destroy', '--yes', FM_BOAT_DESTROY_DORMANT='0') + self.assertEqual(self.provider()['state'], 'deleted') + self.assertFalse(self.path.exists()) + def test_wake_invalidates_sleep_time_quiescence_proof(self): + self.provision(); self.call('wake') + self.boat_py('sleep', FM_BOAT_SLEEP_QUIESCED='1') + self.assertEqual(self.state()['sleep_quiesced'], '1') + self.call('wake') + self.assertNotIn('sleep_quiesced', self.state()) + self.boat_py('sleep', FM_BOAT_SLEEP_QUIESCED='1') + self.lab.update(fail=['resume']); self.call('wake', ok=False) + self.assertEqual(self.state()['lifecycle'], 'suspended') + self.assertNotIn('sleep_quiesced', self.state()) + refused = self.boat_py('destroy', '--yes', ok=False, FM_BOAT_DESTROY_DORMANT='1') + self.assertIn('retry destroy', refused.stderr) + self.assertNotEqual(self.provider()['state'], 'deleted') + self.lab.update(fail=[]); self.call('destroy', '--yes') def test_shred_failure_prevents_deletion_and_failed_stop_restores_credentials(self): check = subprocess.run(['systemctl', '--user', 'show', '--property=ControlGroup'], capture_output=True) if check.returncode: self.skipTest('Linux systemd user manager required for auth transition') diff --git a/tests/fm-boat-routing.test.sh b/tests/fm-boat-routing.test.sh index c3aa2a10330..254757f1784 100755 --- a/tests/fm-boat-routing.test.sh +++ b/tests/fm-boat-routing.test.sh @@ -201,4 +201,76 @@ if out=$(FM_FAKE_DOCTOR_MODE=unready world_env "$ROOT/bin/fm-spawn.sh" ios --sec grep -qx 'lifecycle=ready' "$w/home/data/boat/ios.meta" || fail 'spawn did not wake before readiness' grep -qx 'boat resume' "$w/calls" || fail 'spawn bypassed Boat wake' pass 'Boat wake precedes public remote spawn readiness' + +# Destroy of a running placement keeps the remote reachability checks: an +# unreachable host or active remote child work refuses before the provider. +for request in "$w/home/state/pending-replies/"*; do + [ -f "$request" ] || continue + printf 'done [corr=%s]: fixture reply\n' "$(basename "$request")" >> "$w/home/state/ios.status" +done +: > "$w/calls" +: > "$w/calls.log" +if out=$(FM_FAKE_SSH_MODE=unreachable world_env "$ROOT/bin/fm-boat.sh" destroy ios --yes 2>&1); then + fail 'destroy on a running placement ignored an unreachable remote' +fi +assert_no_grep 'boat delete' "$w/calls" 'unreachable remote still deleted the sandbox' +grep -qx 'lifecycle=ready' "$w/home/data/boat/ios.meta" || fail 'refused destroy changed the lifecycle' +out=$(FM_FAKE_REMOTE_CHILDREN=1 world_env "$ROOT/bin/fm-boat.sh" destroy ios --yes 2>&1) \ + && fail 'destroy on a running placement ignored active remote child work' +assert_contains "$out" 'remote child work is active or unknown' 'remote child refusal lost its reason' +assert_no_grep 'boat delete' "$w/calls" 'remote child work still deleted the sandbox' +assert_grep 'fm-remote-secondmate-control.sh children' "$w/calls.log" \ + 'running placement skipped the remote checks' +pass 'Boat destroy on a running placement keeps remote checks and refuses when they fail' + +# A suspended record alone is not quiescence proof: a failed wake also ends +# suspended through compensation, so destroy must refuse before any SSH. +world_env "$ROOT/bin/fm-boat.sh" sleep ios >/dev/null \ + || fail 'could not suspend the route before dormant destroy' +grep -qx 'lifecycle=suspended' "$w/home/data/boat/ios.meta" || fail 'route did not suspend' +grep -qx 'sleep_quiesced=1' "$w/home/data/boat/ios.meta" \ + || fail 'proven sleep did not record quiescence proof' +jq '.fail = ["resume"]' "$w/provider.json" > "$w/provider.tmp" && mv "$w/provider.tmp" "$w/provider.json" +if out=$(world_env "$ROOT/bin/fm-boat.sh" wake ios 2>&1); then + fail 'wake unexpectedly ignored the injected resume failure' +fi +jq '.fail = []' "$w/provider.json" > "$w/provider.tmp" && mv "$w/provider.tmp" "$w/provider.json" +grep -qx 'lifecycle=suspended' "$w/home/data/boat/ios.meta" \ + || fail "compensated wake did not leave a suspended record: $(cat "$w/home/data/boat/ios.meta")" +if grep -q '^sleep_quiesced=' "$w/home/data/boat/ios.meta"; then + fail 'compensated wake kept the sleep-time quiescence proof' +fi +: > "$w/calls" +: > "$w/calls.log" +out=$(FM_FAKE_SSH_MODE=unreachable world_env "$ROOT/bin/fm-boat.sh" destroy ios --yes 2>&1) \ + && fail 'destroy on an unproven dormant placement unexpectedly succeeded' +assert_contains "$out" 'no sleep-time quiescence proof' 'unproven dormant refusal lost its reason' +assert_no_grep 'boat delete' "$w/calls" 'unproven dormant destroy still deleted the sandbox' +grep -qx 'lifecycle=suspended' "$w/home/data/boat/ios.meta" || fail 'refused destroy removed the record' +assert_no_grep 'fm-remote-secondmate-control.sh' "$w/calls.log" \ + 'unproven dormant destroy probed the remote' +pass 'Boat destroy on an unproven dormant placement refuses before any remote check' + +# With durable sleep-time proof the dormant route is already known idle, so +# destroy skips the unreachable remote checks and deletes through the provider. +world_env "$ROOT/bin/fm-boat.sh" wake ios >/dev/null \ + || fail 'could not wake the route after clearing the injected failure' +world_env "$ROOT/bin/fm-boat.sh" sleep ios >/dev/null \ + || fail 'could not suspend the route before dormant destroy' +grep -qx 'sleep_quiesced=1' "$w/home/data/boat/ios.meta" \ + || fail 'proven sleep did not record quiescence proof' +: > "$w/calls" +: > "$w/calls.log" +if out=$(FM_FAKE_SSH_MODE=unreachable world_env "$ROOT/bin/fm-boat.sh" destroy ios 2>&1); then + fail 'destroy without --yes unexpectedly succeeded' +fi +assert_no_grep 'boat delete' "$w/calls" 'unconfirmed destroy still deleted the sandbox' +grep -qx 'lifecycle=suspended' "$w/home/data/boat/ios.meta" || fail 'unconfirmed destroy removed the record' +out=$(FM_FAKE_SSH_MODE=unreachable world_env "$ROOT/bin/fm-boat.sh" destroy ios --yes 2>&1) \ + || fail "destroy on a proven suspended placement failed: $out" +assert_grep 'boat delete' "$w/calls" 'suspended destroy did not delete the sandbox' +[ ! -e "$w/home/data/boat/ios.meta" ] || fail 'suspended destroy left the record behind' +assert_no_grep 'fm-remote-secondmate-control.sh' "$w/calls.log" \ + 'suspended destroy probed the unreachable remote' +pass 'Boat destroy on a suspended placement skips unreachable remote checks and deletes' fm_test_cleanup From de8dcd46777c2b57e4e9fc7d212e0089fe0694a4 Mon Sep 17 00:00:00 2001 From: dnth Date: Mon, 5 Oct 2026 19:36:50 +0800 Subject: [PATCH 2/2] no-mistakes(document): Clarify Boat dormant destroy proof requirements --- docs/boat-secondmates.md | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/docs/boat-secondmates.md b/docs/boat-secondmates.md index ec0a17668ce..82aa616b5ec 100644 --- a/docs/boat-secondmates.md +++ b/docs/boat-secondmates.md @@ -62,7 +62,13 @@ A failed wake compensates by retiring credentials and requesting a confirmed sto Failed compensation remains explicitly unresolved and never reports a clean stop. A failed sleep restores credential and reply availability when the sandbox is still running; incomplete restoration remains unresolved. A failed bearer shred prevents deletion. -Sleep and destroy accept an already stopped or archived sandbox without issuing another provider stop; destroy still requires checked credential cleanup before deletion. +Sleep and destroy do not issue another provider stop for an already stopped or archived sandbox. +For a registered remote route, destroy skips remote reconciliation and child-work checks only when the placement is suspended or provisioned and has durable proof of a successful sleep whose remote checks passed. +Wake clears that proof before transitioning, so a suspended record left by failed-wake compensation does not qualify. +Without proof, dormant destroy refuses before remote checks; wake the placement so destroy can run those checks. +A running placement still requires remote reconciliation and no active or unknown child work. +Dormant destroy rechecks the lifecycle and proof under the lifecycle lock and refuses if either no longer qualifies. +All destroy paths retain the pending-reply, handoff, decision, credential-cleanup, and explicit confirmation guards. Reconcile an unresolved placement or credential record before retrying sleep or destroy; do not delete its records to bypass cleanup. ## Subscription credential boundary