From d23499da39b352eeabdb38eadf66d2f4a5914472 Mon Sep 17 00:00:00 2001 From: Tejas Shah Date: Thu, 17 Sep 2026 18:41:44 +0700 Subject: [PATCH 1/5] fix(spawn): claim a task's pool worktree with a durable lease Treehouse counts a pool slot as in use only while a process runs inside it, so a task whose agent has exited - paused, crashed, or simply between turns - leaves its own working copy looking free. The next spawn was then handed that copy and reset it to a detached HEAD, silently, with no refusal (observed 2026-08-19; nothing was lost only because that task had already pushed its branch). fm-spawn now takes the pool's own durable claim itself - `treehouse get --lease` - instead of relying on the pane's `treehouse get` subshell to hold the slot, and moves the pane into the leased copy. Treehouse never hands a leased slot to a later get and never prunes it, with or without a process inside, so the claim is enforced by the layer that does the allocating rather than re-derived from task records after the copy has already been reset. The lease holder label names the task record that owns it ("fm:@"), which is what keeps a claim from outliving its task unnoticed: teardown's `treehouse return` releases it in the ordinary case, an aborted spawn returns it holder-matched before any record can name it, and anything left over is self-identifying and releasable with one command. A stale lease costs a pool slot until then; it can never wedge a spawn, because treehouse allocates another slot rather than refusing. The leased path is now screened for isolation before the pane is touched at all, so a non-isolated copy is refused earlier and the refusal carries the reason it was rejected. --- bin/fm-spawn.sh | 159 ++++++++++-------- docs/architecture.md | 3 +- docs/cmux-backend.md | 2 +- docs/verification/runtime-backends.md | 22 +++ docs/zellij-backend.md | 2 +- tests/fixtures.sh | 7 +- .../fm-backend-herdr-presentation-e2e.test.sh | 4 +- tests/fm-backend.test.sh | 2 + tests/fm-backlog-atomicity.test.sh | 1 + tests/fm-bootstrap-network-parallel.test.sh | 1 + tests/fm-captain-hold-lifecycle.test.sh | 7 + tests/fm-kimi-harness.test.sh | 1 + tests/fm-muse-harness.test.sh | 1 + tests/fm-public-followup.test.sh | 9 + tests/fm-rovo-harness.test.sh | 1 + tests/fm-secondmate-sync.test.sh | 3 + tests/fm-shared-captain-inheritance.test.sh | 1 + tests/fm-spawn-base-branch.test.sh | 1 + tests/fm-spawn-pool-base-freshen.test.sh | 15 +- tests/fm-spawn-worktree-lease.test.sh | 120 +++++++++++++ tests/fm-spawn-worktree-settle.test.sh | 67 ++++---- tests/fm-tangle-guard.test.sh | 12 +- tests/fm-trace-context-spawn.test.sh | 1 + tests/lib.sh | 36 ++++ 24 files changed, 349 insertions(+), 129 deletions(-) create mode 100644 tests/fm-spawn-worktree-lease.test.sh diff --git a/bin/fm-spawn.sh b/bin/fm-spawn.sh index 36422519902..03d46c0cbe1 100755 --- a/bin/fm-spawn.sh +++ b/bin/fm-spawn.sh @@ -63,7 +63,7 @@ # then tmux. # Spawn-capable backends are the reference tmux adapter and experimental # herdr, zellij, orca, and cmux. Orca owns both the task worktree and -# terminal, so ship/scout Orca spawns do not run treehouse get; cmux is a +# terminal, so ship/scout Orca spawns take no pool worktree; cmux is a # session provider only, exactly like herdr/zellij, so it does. An # auto-detected herdr or cmux spawn prints a loud stderr notice; # auto-detected tmux stays silent; zellij and orca are never auto-detected. @@ -188,13 +188,17 @@ # Ship/scout spawns refuse to launch unless the resolved task path is a real # git worktree root distinct from both the spawning project and its repository's # primary checkout, including when the spawning project is a linked worktree. -# On the backends that discover that path by reading the task pane's own cwd, -# the same isolation test screens every read: a pane still showing the project -# or the repository primary while `treehouse get` prepares the slot is waited -# out as a transient rather than adopted and then refused, so a home that is -# itself a linked worktree of the project repository still launches. A pane -# that never reaches an isolated worktree refuses at the end of that wait, -# naming the last path seen and why it was rejected. +# The task worktree is a durable treehouse LEASE fm-spawn takes itself, held +# under a label naming the task record that owns it, so a copy whose agent is +# not currently running is never handed to another spawn and reset under it. +# The isolation test screens the leased path before the pane is touched at all. +# The pane is then moved into that copy and must be observed there: a pane +# still showing the project or the repository primary in its first reads is +# waited out as a transient, so a home that is itself a linked worktree of the +# project repository still launches, and a pane that never arrives refuses at +# the end of that wait, naming the leased copy and the last path seen. +# An aborted spawn returns the lease before any record can name it; otherwise +# the published record owns it and bin/fm-teardown.sh's return releases it. # That placement is proven only at launch. Every ship or scout pane therefore # also receives `export FM_TASK_ID=` before the launch command, on # the same channel as GOTMPDIR, and bin/fm-test-run.sh refuses to execute the @@ -1055,6 +1059,9 @@ SPAWN_TASK_SET_LOCK_HELD=0 SPAWN_TREEHOUSE_PROJECT_LOCK= SPAWN_TREEHOUSE_PROJECT_LOCK_HELD=0 SPAWN_SLOT_CLAIMED=0 +SPAWN_TREEHOUSE_LEASE_HOLDER= +SPAWN_TREEHOUSE_LEASE_PATH= +SPAWN_TREEHOUSE_LEASE_PENDING=0 RELAUNCH_REPLACEMENT_PENDING=0 RELAUNCH_REPLACEMENT_BUSY_GEN= RELAUNCH_REPLACEMENT_HARNESS= @@ -1203,6 +1210,17 @@ spawn_abort_cleanup() { echo "warning: leaving task $ID's slot claim on $WT in place; the Treehouse project lock is no longer held, so the next spawn's claim replaces it" >&2 fi fi + if [ "$SPAWN_TREEHOUSE_LEASE_PENDING" = 1 ]; then + # The spawn is aborting before any task record can name this lease, so the + # slot would otherwise stay claimed by a task that never existed. Return it + # while the project lock is still held, and match on the holder label so a + # slot some later allocation has already taken over is never returned here. + SPAWN_TREEHOUSE_LEASE_PENDING=0 + ( cd "$PROJ_ABS" && treehouse return --force \ + --if-lease-holder "$SPAWN_TREEHOUSE_LEASE_HOLDER" \ + "$SPAWN_TREEHOUSE_LEASE_PATH" >/dev/null 2>&1 ) \ + || echo "warning: could not release the treehouse lease on $SPAWN_TREEHOUSE_LEASE_PATH held as $SPAWN_TREEHOUSE_LEASE_HOLDER; release it with: treehouse return --force --if-lease-holder '$SPAWN_TREEHOUSE_LEASE_HOLDER' '$SPAWN_TREEHOUSE_LEASE_PATH'" >&2 + fi if [ "$SPAWN_TREEHOUSE_PROJECT_LOCK_HELD" = 1 ]; then SPAWN_TREEHOUSE_PROJECT_LOCK_HELD=0 fm_lock_release "$SPAWN_TREEHOUSE_PROJECT_LOCK" || true @@ -2744,7 +2762,7 @@ spawn_worktree_isolated() { # validate_spawn_worktree() { # local source=$1 inspect_target=$2 if ! spawn_worktree_isolated "$WT"; then - echo "error: $source did not yield an isolated worktree (resolved '$WT'; worktree root '${SPAWN_WT_TOP:-none}'; spawning project '$PROJ_ABS'); refusing to launch to avoid tangling the primary checkout. Inspect target $inspect_target" >&2 + echo "error: $source did not yield an isolated worktree (resolved '$WT': ${SPAWN_WT_REASON:-no reason recorded}; worktree root '${SPAWN_WT_TOP:-none}'; spawning project '$PROJ_ABS'); refusing to launch to avoid tangling the primary checkout. Inspect target $inspect_target" >&2 exit 1 fi } @@ -3613,77 +3631,71 @@ if [ "$RELAUNCH" -eq 1 ]; then fi [ "$KIND" = secondmate ] || validate_spawn_worktree "relaunch" "$T" elif [ "$KIND" != secondmate ] && [ "$BACKEND" != orca ]; then - spawn_send_text_line "$WT_TARGET" 'treehouse get' + # Claim the pool slot from HERE, durably, instead of letting the pane's + # `treehouse get` subshell hold it. Treehouse counts a slot as in use only + # while a process is running inside it, so a task whose agent has exited - + # paused, crashed, or between turns - leaves its own working copy looking + # free. The next `treehouse get` then hands that copy to a new spawn and + # resets it (observed 2026-08-19: a live task's copy was reassigned and reset + # to a detached HEAD, silently, with no refusal). A lease is the pool's own + # durable claim: treehouse never hands a leased slot to a later get and never + # prunes it, with or without a process inside, so the claim is enforced by + # the layer that does the allocating rather than re-derived from task records + # after the copy has already been reset. + # + # How the claim is released, since a claim nothing can release is the trap + # this fix could otherwise create: + # - Normally bin/fm-teardown.sh returns the worktree, and `treehouse return` + # releases the lease with it. That is the whole ordinary lifecycle. + # - An aborted spawn releases it below in spawn_abort_cleanup, before any + # task record exists to point at it. + # - Otherwise the holder label names the exact record that owns the lease, + # so a lease outliving its task is self-identifying and releasable with + # `treehouse return --force --if-lease-holder