From 5780f8db3252166d2c24c5ae35907bf64ac336c5 Mon Sep 17 00:00:00 2001 From: Tejas Shah Date: Sun, 20 Sep 2026 14:31:57 +0700 Subject: [PATCH 01/10] fix(spawn): claim a task's pool worktree so it is never reassigned under a live task Supersedes #4, which cannot be revalidated: its branch was rewritten (the pipeline rebased it onto a moved main, and custody was then returned at that rebased head), and no-mistakes refuses to reconcile its private mirror record afterwards. Same content, clean branch from main. On 2026-08-19 a spawn was handed a pooled working copy that another task's own record still named as its live copy. The pool reset it to a detached HEAD, evicting a paused task from the copy holding its work; nothing was lost only because that task had already pushed. Treehouse counts a slot as in use only while a process runs inside it, so a task whose agent has exited - paused, crashed, or between turns - leaves its copy looking free. bin/fm-spawn.sh now takes the pool's own durable claim itself (treehouse get --lease) instead of relying on the pane's subshell to hold the slot, then moves the pane into the leased copy in a subshell so the pane's own shell does not sit inside the worktree. Release paths: ordinary teardown returns the worktree and the lease with it; an aborted spawn returns it holder-matched in cleanup; otherwise the holder label names the owning record and the task records worktree_lease= so a stale claim is self-identifying and releasable with one command. A named refusal stays as a second line of defence for the one copy a claim cannot protect - a task spawned before claims existed, holding its slot with nothing but its processes. The claim is deliberately kept on that refusal, so the slot is quarantined rather than offered to the next spawn in a refuse-loop. Upstream's slot-owner claim marker is kept as-is and this claim sits above it: the lease stops the pool reassigning a live copy, the marker answers at teardown whether the slot beside a record is still that record's own. --- bin/backends/cmux.sh | 3 +- bin/backends/herdr.sh | 16 +- bin/backends/tmux.sh | 8 +- bin/backends/zellij.sh | 10 +- bin/fm-spawn.sh | 200 +++++++++++------- bin/fm-teardown.sh | 3 + docs/architecture.md | 3 +- docs/cmux-backend.md | 2 +- docs/verification/runtime-backends.md | 22 ++ docs/zellij-backend.md | 2 +- tests/fixtures.sh | 7 +- .../fm-backend-herdr-presentation-e2e.test.sh | 49 +++-- tests/fm-backend.test.sh | 20 +- tests/fm-backlog-atomicity.test.sh | 3 +- tests/fm-bootstrap-network-parallel.test.sh | 3 +- tests/fm-captain-hold-lifecycle.test.sh | 18 +- tests/fm-kimi-harness.test.sh | 3 +- tests/fm-muse-harness.test.sh | 3 +- tests/fm-public-followup.test.sh | 27 ++- tests/fm-rovo-harness.test.sh | 3 +- tests/fm-secondmate-harness.test.sh | 10 +- tests/fm-secondmate-sync.test.sh | 9 +- tests/fm-shared-captain-inheritance.test.sh | 3 +- tests/fm-spawn-base-branch.test.sh | 2 +- tests/fm-spawn-pool-base-freshen.test.sh | 15 +- tests/fm-spawn-worktree-lease.test.sh | 166 +++++++++++++++ tests/fm-spawn-worktree-settle.test.sh | 68 +++--- tests/fm-tangle-guard.test.sh | 13 +- tests/fm-trace-context-spawn.test.sh | 2 +- tests/lib.sh | 32 +++ 30 files changed, 531 insertions(+), 194 deletions(-) create mode 100644 tests/fm-spawn-worktree-lease.test.sh diff --git a/bin/backends/cmux.sh b/bin/backends/cmux.sh index 747d3fc2ccd..348186ee709 100644 --- a/bin/backends/cmux.sh +++ b/bin/backends/cmux.sh @@ -36,7 +36,8 @@ # (herdr-shape): `workspace list`'s `current_directory` field reflects a # `cd` run directly in the surface's own top-level shell, but stays # frozen at wherever that shell was when it launched a foreground -# subshell (exactly what `treehouse get` does) - verified live: a nested +# subshell (exactly what the spawn-time `(cd -- '' && exec $SHELL)` +# does) - verified live: a nested # `bash -c 'cd /Users && exec bash'` left `current_directory` reporting # the PARENT shell's last cwd, never following into the subshell. Fixed # with zellij's own pwd-marker-probe workaround, reused verbatim in diff --git a/bin/backends/herdr.sh b/bin/backends/herdr.sh index 88a8cb61492..33b0301f626 100644 --- a/bin/backends/herdr.sh +++ b/bin/backends/herdr.sh @@ -2081,8 +2081,10 @@ fm_backend_herdr_explicit_close_pane_confirmed() { # # shell - every foreground process is a recognized shell AND no # descendant of the pane shell is a verified harness: positive # proof the pane is shell-only. The descendant walk is what makes -# this safe for the crew shape, where a nested `treehouse get` -# shell sits under the pane's top shell. +# this safe for the crew shape, where the nested worktree +# shell fm-spawn.sh starts with +# `(cd -- '' && exec $SHELL)` sits under the pane's top +# shell. # other - the foreground group holds something that is neither: a tool # the agent is running in its own process group, a pager, a # stranger's process. Not a shell-only pane. An idle shell @@ -2933,15 +2935,16 @@ fm_backend_herdr_target_ready() { # # fm_backend_herdr_current_path: the live FOREGROUND process's cwd, or empty on # any error. Mirrors tmux's pane_current_path poll used for worktree-path -# discovery after `treehouse get`. +# discovery after the pane enters its worktree. # # Verified pitfall: `pane get`'s `.result.pane.cwd` is the pane's cwd AT # CREATION TIME - the top-level shell's cwd - and does NOT update when that -# shell `cd`s or enters a subshell (as `treehouse get` does). Reading it here +# shell `cd`s or enters a subshell (as fm-spawn.sh's +# `(cd -- '' && exec $SHELL)` does). Reading it here # would make fm-spawn.sh's worktree-discovery poll never see the pane "leave" # the project directory, since `cwd` stays frozen at the original path forever. # `.result.pane.foreground_cwd` tracks the ACTUALLY RUNNING foreground -# process's cwd instead, which is what changes when `treehouse get` enters its +# process's cwd instead, which is what changes when that command enters its # worktree subshell - confirmed live against a real treehouse acquisition. fm_backend_herdr_current_path() { # fm_backend_herdr_target_ready "$1" || return 0 @@ -2951,7 +2954,8 @@ fm_backend_herdr_current_path() { # # fm_backend_herdr_send_text_line: send one line of TEXT then submit, # ATOMICALLY - mirrors tmux's `send-keys -t T text Enter`. Used for the fixed -# spawn-time commands (treehouse get, the GOTMPDIR export). `pane run` types +# spawn-time commands (the worktree-entry subshell, the GOTMPDIR export). +# `pane run` types # the command and submits it in one call (verified). fm_backend_herdr_send_text_line() { # fm_backend_herdr_target_ready "$1" || return 1 diff --git a/bin/backends/tmux.sh b/bin/backends/tmux.sh index 2bc1c8aa0d7..e9c70aeeba8 100644 --- a/bin/backends/tmux.sh +++ b/bin/backends/tmux.sh @@ -8,8 +8,9 @@ # default (tmux, `backend=` absent) path stays byte-identical. Sourced only # through bin/fm-backend.sh's fm_backend_source, never directly. # -# Worktree acquisition (running `treehouse get` inside the pane, and polling -# its cwd) is unchanged by this extraction: P1 scopes only the session +# Worktree acquisition (fm-spawn.sh takes the pool's lease itself, sends +# `(cd -- '' && exec $SHELL)` into the pane, and polls its cwd) is +# unchanged by this extraction: P1 scopes only the session # provider, not the worktree provider, so fm-spawn.sh still drives that part # inline with these same send/current-path primitives. # @@ -115,7 +116,8 @@ fm_backend_tmux_current_path() { # # fm_backend_tmux_send_text_line: send one line of TEXT then Enter, with no # composer verification - used for the fixed spawn-time commands -# (`treehouse get`, the GOTMPDIR export) that already ran this exact sequence +# (the `(cd -- '' && exec $SHELL)` worktree entry, the GOTMPDIR export) +# that already ran this exact sequence # inline in fm-spawn.sh. Mirrors `tmux send-keys -t "$T" "" Enter`. fm_backend_tmux_send_text_line() { # tmux send-keys -t "$1" "$2" Enter diff --git a/bin/backends/zellij.sh b/bin/backends/zellij.sh index a90247899f1..bfac5a2ad83 100644 --- a/bin/backends/zellij.sh +++ b/bin/backends/zellij.sh @@ -53,8 +53,8 @@ # 4. `list-panes --json`'s `pane_cwd` reflects a `cd` run DIRECTLY in the # pane's own top-level shell within one poll (<0.3s) - but does NOT # reflect a `cd` performed by a NESTED SUBSHELL the pane's shell -# launched as a foreground command (verified: `treehouse get` opens -# exactly such a subshell). `pane_cwd` stays frozen at wherever the +# launched as a foreground command (verified: the spawn-time +# `(cd -- '' && exec $SHELL)` opens exactly such a subshell). `pane_cwd` stays frozen at wherever the # pane's shell was when it invoked that foreground command - worse than # herdr's frozen-cwd trap (herdr at least exposes a `foreground_cwd` # that tracks this; zellij's CLI exposes no live-process cwd field and @@ -388,13 +388,13 @@ fm_backend_zellij_target_ready() { # [expected-label] # fm_backend_zellij_current_path: the live pane's cwd, or empty on any error. # Mirrors tmux's pane_current_path poll used for worktree-path discovery after -# `treehouse get`. +# the pane enters its worktree. # # Verified pitfall (docs/zellij-backend.md "Worktree-path discovery: pane_cwd # does not track a subshell"): `list-panes --json`'s `pane_cwd` DOES reflect a # `cd` run directly in the pane's own top-level shell, but stays FROZEN at -# whatever directory the pane's shell was in when it launched `treehouse get` -# as a foreground command - it never follows that command's own internal `cd` +# whatever directory the pane's shell was in when it launched the +# worktree-entry command as a foreground command - it never follows that command's own internal `cd` # into the acquired worktree, even after the subshell is fully interactive and # a `pwd` typed into it prints the correct live path on screen. Zellij's CLI # exposes no per-pane pid and no live-process cwd field to read instead diff --git a/bin/fm-spawn.sh b/bin/fm-spawn.sh index 9c35c7b014b..7721825547d 100755 --- a/bin/fm-spawn.sh +++ b/bin/fm-spawn.sh @@ -63,7 +63,7 @@ # then tmux. # Spawn-capable backends are the reference tmux adapter and experimental # herdr, zellij, orca, and cmux. Orca owns both the task worktree and -# terminal, so ship/scout Orca spawns do not run treehouse get; cmux is a +# terminal, so ship/scout Orca spawns take no pool worktree; cmux is a # session provider only, exactly like herdr/zellij, so it does. An # auto-detected herdr or cmux spawn prints a loud stderr notice; # auto-detected tmux stays silent; zellij and orca are never auto-detected. @@ -202,13 +202,17 @@ # Ship/scout spawns refuse to launch unless the resolved task path is a real # git worktree root distinct from both the spawning project and its repository's # primary checkout, including when the spawning project is a linked worktree. -# On the backends that discover that path by reading the task pane's own cwd, -# the same isolation test screens every read: a pane still showing the project -# or the repository primary while `treehouse get` prepares the slot is waited -# out as a transient rather than adopted and then refused, so a home that is -# itself a linked worktree of the project repository still launches. A pane -# that never reaches an isolated worktree refuses at the end of that wait, -# naming the last path seen and why it was rejected. +# The task worktree is a durable treehouse LEASE fm-spawn takes itself, held +# under a label naming the task record that owns it, so a copy whose agent is +# not currently running is never handed to another spawn and reset under it. +# The isolation test screens the leased path before the pane is touched at all. +# The pane is then moved into that copy and must be observed there: a pane +# still showing the project or the repository primary in its first reads is +# waited out as a transient, so a home that is itself a linked worktree of the +# project repository still launches, and a pane that never arrives refuses at +# the end of that wait, naming the leased copy and the last path seen. +# An aborted spawn returns the lease before any record can name it; otherwise +# the published record owns it and bin/fm-teardown.sh's return releases it. # That placement is proven only at launch. Every ship or scout pane therefore # also receives `export FM_TASK_ID=` before the launch command, on # the same channel as GOTMPDIR, and bin/fm-test-run.sh refuses to execute the @@ -1094,6 +1098,9 @@ SPAWN_TASK_SET_LOCK_HELD=0 SPAWN_TREEHOUSE_PROJECT_LOCK= SPAWN_TREEHOUSE_PROJECT_LOCK_HELD=0 SPAWN_SLOT_CLAIMED=0 +SPAWN_TREEHOUSE_LEASE_HOLDER= +SPAWN_TREEHOUSE_LEASE_PATH= +SPAWN_TREEHOUSE_LEASE_PENDING=0 RELAUNCH_REPLACEMENT_PENDING=0 RELAUNCH_REPLACEMENT_BUSY_GEN= RELAUNCH_REPLACEMENT_HARNESS= @@ -1242,6 +1249,17 @@ spawn_abort_cleanup() { echo "warning: leaving task $ID's slot claim on $WT in place; the Treehouse project lock is no longer held, so the next spawn's claim replaces it" >&2 fi fi + if [ "$SPAWN_TREEHOUSE_LEASE_PENDING" = 1 ]; then + # The spawn is aborting before any task record can name this lease, so the + # slot would otherwise stay claimed by a task that never existed. Return it + # while the project lock is still held, and match on the holder label so a + # slot some later allocation has already taken over is never returned here. + SPAWN_TREEHOUSE_LEASE_PENDING=0 + ( cd "$PROJ_ABS" && treehouse return --force \ + --if-lease-holder "$SPAWN_TREEHOUSE_LEASE_HOLDER" \ + "$SPAWN_TREEHOUSE_LEASE_PATH" >/dev/null 2>&1 ) \ + || echo "warning: could not release the treehouse lease on $SPAWN_TREEHOUSE_LEASE_PATH held as $SPAWN_TREEHOUSE_LEASE_HOLDER; release it with: treehouse return --force --if-lease-holder '$SPAWN_TREEHOUSE_LEASE_HOLDER' '$SPAWN_TREEHOUSE_LEASE_PATH'" >&2 + fi if [ "$SPAWN_TREEHOUSE_PROJECT_LOCK_HELD" = 1 ]; then SPAWN_TREEHOUSE_PROJECT_LOCK_HELD=0 fm_lock_release "$SPAWN_TREEHOUSE_PROJECT_LOCK" || true @@ -2784,7 +2802,7 @@ spawn_worktree_isolated() { # validate_spawn_worktree() { # local source=$1 inspect_target=$2 if ! spawn_worktree_isolated "$WT"; then - echo "error: $source did not yield an isolated worktree (resolved '$WT'; worktree root '${SPAWN_WT_TOP:-none}'; spawning project '$PROJ_ABS'); refusing to launch to avoid tangling the primary checkout. Inspect target $inspect_target" >&2 + echo "error: $source did not yield an isolated worktree (resolved '$WT': ${SPAWN_WT_REASON:-no reason recorded}; worktree root '${SPAWN_WT_TOP:-none}'; spawning project '$PROJ_ABS'); refusing to launch to avoid tangling the primary checkout. Inspect target $inspect_target" >&2 exit 1 fi } @@ -3687,77 +3705,107 @@ if [ "$RELAUNCH" -eq 1 ]; then fi [ "$KIND" = secondmate ] || validate_spawn_worktree "relaunch" "$T" elif [ "$KIND" != secondmate ] && [ "$BACKEND" != orca ]; then - spawn_send_text_line "$WT_TARGET" 'treehouse get' + # Claim the pool slot from HERE, durably, instead of letting the pane's + # `treehouse get` subshell hold it. Treehouse counts a slot as in use only + # while a process is running inside it, so a task whose agent has exited - + # paused, crashed, or between turns - leaves its own working copy looking + # free. The next `treehouse get` then hands that copy to a new spawn and + # resets it (observed 2026-08-19: a live task's copy was reassigned and reset + # to a detached HEAD, silently, with no refusal). A lease is the pool's own + # durable claim: treehouse never hands a leased slot to a later get and never + # prunes it, with or without a process inside, so the claim is enforced by + # the layer that does the allocating rather than re-derived from task records + # after the copy has already been reset. + # + # How the claim is released, since a claim nothing can release is the trap + # this fix could otherwise create: + # - Normally bin/fm-teardown.sh returns the worktree, and `treehouse return` + # releases the lease with it. That is the whole ordinary lifecycle. + # - An aborted spawn releases it below in spawn_abort_cleanup, before any + # task record exists to point at it. + # - Otherwise the holder label names the exact record that owns the lease, + # so a lease outliving its task is self-identifying and releasable with + # `treehouse return --force --if-lease-holder