check: 3725 option 2 (bounded queue-lock wait as separate function)

bin/backends/cmux.sh

@@ -398,8 +398,15 @@ fm_backend_cmux_parse_target() {  # <target>
 # (fm_backend_zellij_pane_exists) rather than the design sketch's original
 # read-screen-based suggestion.
 fm_backend_cmux_surface_exists() {  # <workspace_id> <surface_id>
-  local wsid=$1 sfid=$2
-  fm_backend_cmux_cli list-panes --workspace "$wsid" --json --id-format uuids 2>/dev/null \
+  local wsid=$1 sfid=$2 out
+  # Capture and check the CLI call's own exit status before piping to jq: a
+  # bare `cli ... | jq -e ...` pipe (without pipefail) takes its exit status
+  # from jq alone, and jq 1.6 (still the apt-installed default on several
+  # supported Linux distros) reports success (exit 0) for `-e` on completely
+  # empty stdin instead of jq 1.7+'s failure (exit 4) - so a failed,
+  # output-less list-panes call would misreport the surface as existing.
+  out=$(fm_backend_cmux_cli list-panes --workspace "$wsid" --json --id-format uuids 2>/dev/null) || return 1
+  printf '%s' "$out" \
     | jq -e --arg s "$sfid" '[.panes[]? | select(.surface_ids // [] | index($s))] | length > 0' >/dev/null 2>&1
 }
 

bin/fm-afk-return.sh

@@ -231,11 +231,12 @@ write_gate() {  # <evidence-file> <blockers-file>
 }
 
 print_evidence() {  # <file>
-  local file=$1 kind text
+  local file=$1 kind text status=0
   while IFS="$(printf '\t')" read -r tag kind text; do
     [ "$tag" = evidence ] || continue
-    printf 'catch-up %s: %s\n' "$kind" "$text"
+    printf 'catch-up %s: %s\n' "$kind" "$text" || status=1
   done < "$file"
+  return "$status"
 }
 
 print_blockers() {  # <file>

bin/fm-install-herdr.sh

@@ -76,13 +76,20 @@ fi
 mkdir -p "$DESTINATION"
 install -m 0755 "$TMP/$ASSET" "$DESTINATION/herdr"
 
-# Post-install version and protocol gates (no floating latest).
-installed_version=$("$DESTINATION/herdr" --version 2>/dev/null | awk '{print $2; exit}')
+# Post-install version and protocol gates (no floating latest). stderr is
+# captured to its own file rather than merged with 2>&1: merging would pollute
+# a successful call's parsed stdout (version_output/status) with any incidental
+# stderr text (a deprecation notice, update check, telemetry banner, etc.),
+# breaking the awk/jq parse below even on a healthy install.
+version_output=$("$DESTINATION/herdr" --version 2>"$TMP/herdr-version.err") || {
+  die "'$DESTINATION/herdr --version' exited $? with output: $(cat "$TMP/herdr-version.err" 2>/dev/null)"
+}
+installed_version=$(printf '%s' "$version_output" | awk '{print $2; exit}')
 [ "$installed_version" = "$FM_HERDR_CI_VERSION" ] \
   || die "installed herdr version is '${installed_version:-<empty>}', expected exact pin $FM_HERDR_CI_VERSION"
 
-status=$("$DESTINATION/herdr" status --json 2>/dev/null) \
-  || die "could not run 'herdr status --json' after install"
+status=$("$DESTINATION/herdr" status --json 2>"$TMP/herdr-status.err") \
+  || die "could not run 'herdr status --json' after install: $(cat "$TMP/herdr-status.err" 2>/dev/null)"
 protocol=$(printf '%s' "$status" | jq -r '.client.protocol // empty' 2>/dev/null) \
   || die "jq is required to parse herdr status after install"
 case "$protocol" in

bin/fm-install-treehouse.sh

@@ -81,12 +81,20 @@ fi
 mkdir -p "$DESTINATION"
 install -m 0755 "$BIN" "$DESTINATION/treehouse"
 
-installed_version=$("$DESTINATION/treehouse" --version 2>/dev/null | tr -d '[:space:]')
+# stderr is captured to its own file rather than merged with 2>&1: merging
+# would pollute a successful call's parsed stdout (version_output) with any
+# incidental stderr text (a deprecation notice, update check, etc.), which
+# tr -d '[:space:]' below would then splice directly onto the version string
+# with no separator, breaking the pin comparison even on a healthy install.
+version_output=$("$DESTINATION/treehouse" --version 2>"$TMP/treehouse-version.err") || {
+  die "'$DESTINATION/treehouse --version' exited $? with output: $(cat "$TMP/treehouse-version.err" 2>/dev/null)"
+}
+installed_version=$(printf '%s' "$version_output" | tr -d '[:space:]')
 # treehouse prints "v2.0.1" (leading v) on --version.
 case "$installed_version" in
   "v${FM_TREEHOUSE_CI_VERSION}"|"${FM_TREEHOUSE_CI_VERSION}") ;;
   *)
-    die "installed treehouse version is '${installed_version:-<empty>}', expected exact pin v${FM_TREEHOUSE_CI_VERSION}"
+    die "installed treehouse version is '${installed_version:-<empty>}', expected exact pin v${FM_TREEHOUSE_CI_VERSION}. treehouse --version said: ${version_output:-<no output>}"
     ;;
 esac
 

bin/fm-test-run.sh

@@ -1165,7 +1165,16 @@ skipped = 0
 total = 0
 wall_ms = 0
 for path in inputs:
-    doc = json.loads(path.read_text(encoding="utf-8"))
+    # A lane artifact can be truncated when its own job is killed mid-write
+    # (host memory pressure, a hang timeout). This aggregate step runs with
+    # if: always() precisely so per-lane failures still get reported here;
+    # skip an unreadable lane instead of taking the whole aggregate down
+    # with it.
+    try:
+        doc = json.loads(path.read_text(encoding="utf-8"))
+    except (OSError, ValueError) as exc:
+        print(f"fm-test-run.sh: skipping unreadable timing artifact {path}: {exc}", file=sys.stderr)
+        continue
     summary = doc.get("summary") or {}
     lane = {
         "path": str(path),

bin/fm-wake-drain.sh

@@ -9,7 +9,8 @@
 # Keep sequence-bound row consumption independent from generation-bound episode
 # retirement; docs/watcher-continuity.md owns the recovery contract.
 # FM_STATUS_PRESENTATION_LOCK_TIMEOUT sets the positive whole-second wait for
-# presentation-path locks (default 10); queue mutation locks remain blocking.
+# presentation-path locks (default 10); the acknowledgement queue-lock acquires
+# wait FM_WAKE_QUEUE_LOCK_WAIT seconds (default 30) and refuse on timeout.
 set -u
 
 SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
@@ -625,7 +626,10 @@ trap 'exit 130' INT
 trap 'exit 143' TERM
 
 if [ -n "$ACK_THROUGH" ]; then
-  fm_lock_acquire_wait "$FM_WAKE_QUEUE_LOCK"
+  fm_lock_acquire_wait_bounded "$FM_WAKE_QUEUE_LOCK" "$FM_WAKE_QUEUE_LOCK_WAIT" || {
+    printf 'wake drain: queue lock could not be acquired safely\n' >&2
+    exit 1
+  }
 elif fm_lock_acquire_wait_bounded "$FM_WAKE_QUEUE_LOCK" "$PRESENTATION_LOCK_TIMEOUT"; then
   :
 else
@@ -681,7 +685,10 @@ if [ -n "$ACK_THROUGH" ]; then
     echo "wake drain: inactive outcome receipt could not be recorded safely" >&2
     exit 1
   fi
-  fm_lock_acquire_wait "$FM_WAKE_QUEUE_LOCK"
+  fm_lock_acquire_wait_bounded "$FM_WAKE_QUEUE_LOCK" "$FM_WAKE_QUEUE_LOCK_WAIT" || {
+    printf 'wake drain: queue lock could not be acquired safely\n' >&2
+    exit 1
+  }
   DRAIN_LOCK_HELD=true
   DRAIN_TMP=$(mktemp "$STATE/.wake-queue.ack.XXXXXX") || exit 1
   chmod 0600 "$DRAIN_TMP" || exit 1

bin/fm-wake-grant.sh

@@ -41,7 +41,7 @@ case "${1:-}" in
     TMP=$(mktemp "$STATE/.branch-eligible-owner.tmp.XXXXXX") || exit 1
     printf '%s\n%s\n%s\n%s\n' fm-branch-eligible-owner-v1 "$pid" "$identity" "$generation" > "$TMP" || exit 1
     chmod 0600 "$TMP" || exit 1
-    fm_lock_acquire_wait "$FM_WAKE_QUEUE_LOCK"
+    fm_lock_acquire_wait_bounded "$FM_WAKE_QUEUE_LOCK" "$FM_WAKE_QUEUE_LOCK_WAIT" || exit 1
     LOCK_HELD=true
     [ "$(fm_pid_identity "$pid" 2>/dev/null || true)" = "$identity" ] || exit 1
     rm -f -- "$BRANCH_ROWS" || exit 1
@@ -57,7 +57,7 @@ case "${1:-}" in
     printf '%s\n' "$@" > "$TMP" || exit 1
     chmod 0600 "$TMP" || exit 1
     rows_valid "$TMP" || exit 2
-    fm_lock_acquire_wait "$FM_WAKE_QUEUE_LOCK"
+    fm_lock_acquire_wait_bounded "$FM_WAKE_QUEUE_LOCK" "$FM_WAKE_QUEUE_LOCK_WAIT" || exit 1
     LOCK_HELD=true
     owner_matches '' "$generation" || exit 1
     replace=1
@@ -86,7 +86,7 @@ case "${1:-}" in
   release)
     generation=${2:-}
     [ "$#" -eq 2 ] || exit 2
-    fm_lock_acquire_wait "$FM_WAKE_QUEUE_LOCK"
+    fm_lock_acquire_wait_bounded "$FM_WAKE_QUEUE_LOCK" "$FM_WAKE_QUEUE_LOCK_WAIT" || exit 1
     LOCK_HELD=true
     owner_matches '' "$generation" || exit 1
     rm -f -- "$BRANCH_ROWS" || exit 1
@@ -95,7 +95,7 @@ case "${1:-}" in
     pid=${2:-}
     generation=${3:-}
     [ "$#" -eq 3 ] || exit 2
-    fm_lock_acquire_wait "$FM_WAKE_QUEUE_LOCK"
+    fm_lock_acquire_wait_bounded "$FM_WAKE_QUEUE_LOCK" "$FM_WAKE_QUEUE_LOCK_WAIT" || exit 1
     LOCK_HELD=true
     owner_matches "$pid" "$generation" || exit 1
     rm -f -- "$BRANCH_ROWS" "$BRANCH_OWNER" || exit 1

bin/fm-wake-lib.sh

@@ -9,6 +9,9 @@ STATE="${FM_STATE_OVERRIDE:-${STATE:-$FM_HOME/state}}"
 FM_WAKE_QUEUE="${FM_WAKE_QUEUE:-$STATE/.wake-queue}"
 FM_WAKE_QUEUE_LOCK="${FM_WAKE_QUEUE_LOCK:-$STATE/.wake-queue.lock}"
 FM_LOCK_STALE_AFTER="${FM_LOCK_STALE_AFTER:-2}"
+# Positive whole seconds the queue-lock callers that check the result of
+# fm_lock_acquire_wait_bounded wait before refusing; any other value makes them refuse.
+FM_WAKE_QUEUE_LOCK_WAIT="${FM_WAKE_QUEUE_LOCK_WAIT:-30}"
 # Resolved once at source time: fm_pid_identity and fm_path_mtime run inside 0.2s
 # confirm and 0.5s attach polls, and forking uname per call is a measurable cost on
 # the platform (Git Bash/MSYS) that already pays the highest fork price.
@@ -1056,7 +1059,8 @@ _fm_lock_acquire_wait_handoff() {  # <lockdir> <caller-pid>
 # Use it where a caller must refuse rather than block: wake presentation, and
 # the guarded remote link clear, whose whole contract is to return a
 # reconciliation refusal instead of wedging an unattended close.
-# Mutation-critical callers that can safely block keep fm_lock_acquire_wait.
+# Callers that cannot check a result keep fm_lock_acquire_wait, which waits
+# indefinitely and cannot fail.
 fm_lock_acquire_wait_bounded() {
   local lockdir=$1 seconds=$2 caller_pid rc owner_pid
   case "$seconds" in ''|*[!0-9]*|0) return 2 ;; esac
@@ -1858,7 +1862,7 @@ fm_wake_queued_keys() {
     signal|stale|check|heartbeat) ;;
     *) printf 'fm_wake_queued_keys: invalid wake kind: %s\n' "$kind" >&2; return 2 ;;
   esac
-  fm_lock_acquire_wait "$FM_WAKE_QUEUE_LOCK"
+  fm_lock_acquire_wait_bounded "$FM_WAKE_QUEUE_LOCK" "$FM_WAKE_QUEUE_LOCK_WAIT" || return 1
   fm_wake_queued_keys_locked "$kind"
   fm_lock_release "$FM_WAKE_QUEUE_LOCK"
 }

docs/configuration.md

@@ -1118,6 +1118,7 @@ FMX_FOLLOWUP_MAX_AGE_SECS=604800   # local window for posting Relay completion f
 FMX_FOLLOWUP_MAX_COUNT=3   # local cap on Relay completion follow-ups per linked mention
 FM_PF_RETRY_BACKOFF_SECS=900   # seconds before the next attempt after a retryable promised-public-reply delivery error
 FM_LOCK_STALE_AFTER=2   # grace seconds for missing or nonnumeric lock-owner PIDs (minimum 2s); dead numeric PIDs have no age grace
+FM_WAKE_QUEUE_LOCK_WAIT=30   # positive whole seconds the wake acknowledgement, branch-grant, and queued-key callers wait for the queue lock before refusing
 FM_GUARD_GRACE=300      # beacon freshness threshold for guard verdicts, arm health checks, and the primary turn-end guard; see docs/turnend-guard.md for model-aware exceptions
 FM_CLAUDE_AUTOARM_ATTEMPTS=2   # bounded Stop-owned arm attempts per Claude auto-arm cycle; accepted values are 1, 2, or 3
 FM_CLAUDE_AUTOARM_SYNC_WAIT_MS=800   # milliseconds the --claude turn-end guard waits for watcher health, an open Stop auto-arm generation claim, or a fresh epoch before deciding recovery ownership or failure progression

docs/watcher-continuity.md

@@ -69,7 +69,7 @@ An acknowledged episode does not freeze the generation, because the next downtim
 Every presented row is claimed to exactly one actor under the durable queue lock.
 An ordinary presentation drain bounds both its initial queue-lock acquire and its later status-presentation-lock acquire at the deadline owned by the script header.
 A live initial queue-lock holder produces one PID-naming advisory and skips the whole drain before any claim or mutation, while a live status-presentation-lock holder produces one such advisory after raw wake presentation and leaves status annotations, sections, and cursors retriable on the next drain.
-Acknowledgement invocations and every other mutation-critical queue-lock acquire retain blocking semantics, so acknowledgement atomicity is unchanged.
+Acknowledgement invocations, the branch-grant helper, and the queued-key read wait for the queue lock through `fm_lock_acquire_wait_bounded` at the 30-second default of `FM_WAKE_QUEUE_LOCK_WAIT` and refuse rather than proceed without it, so acknowledgement atomicity is unchanged; every other queue-lock acquire keeps the indefinite wait of `fm_lock_acquire_wait`.
 Main records its presented set in `state/.main-eligible-rows`.
 A branch grant is published through `bin/fm-wake-grant.sh` under that same lock in `state/.branch-eligible-rows`, bound to the live branch process and extension generation recorded in `state/.branch-eligible-owner`, and publication is refused if main already claimed any requested row.
 A main drain validates that owner evidence under the queue lock and reclaims the grant when its process is gone or its identity no longer matches.

tests/fm-backend-cmux.test.sh

@@ -511,6 +511,40 @@ test_target_ready_fails_when_target_absent() {
   pass "fm_backend_cmux_target_ready: fails when the workspace/surface is not found (list-panes structural check)"
 }
 
+test_target_ready_fails_when_list_panes_errors_empty() {
+  local dir fb status real_jq
+  dir="$TMP_ROOT/ready-list-panes-error"; mkdir -p "$dir/responses"
+  fb=$(make_cmux_fakebin "$dir")
+  real_jq=$(command -v jq)
+  # 1: list-panes --json --id-format uuids -> exits nonzero with no stdout
+  # (a failed call, not "no matching pane"). Regression coverage for the
+  # incident where a bare `cli ... | jq -e ...` pipe took its exit status
+  # from jq alone, and jq 1.6 treated empty stdin as success under -e, so a
+  # failed list-panes call misreported the surface as existing. A real jq
+  # 1.7+ already refuses empty stdin under -e, which would mask the bug on
+  # this machine, so this stub jq mimics 1.6's `-e`-on-empty-stdin=success
+  # quirk to make the regression check version-independent: it fails unless
+  # the code checks the CLI's own exit status before ever handing off to jq.
+  cat > "$fb/jq" <<SH
+#!/usr/bin/env bash
+set -u
+input=\$(cat)
+for a in "\$@"; do
+  if [ "\$a" = "-e" ] && [ -z "\$input" ]; then
+    exit 0
+  fi
+done
+printf '%s' "\$input" | exec "$real_jq" "\$@"
+SH
+  chmod +x "$fb/jq"
+  printf '1' > "$dir/responses/1.exit"
+  PATH="$fb:$PATH" FM_CMUX_LOG="$dir/log" FM_CMUX_RESPONSES="$dir/responses" \
+    bash -c '. "$0/bin/backends/cmux.sh"; fm_backend_cmux_target_ready "aaaaaaaa-0000-0000-0000-000000000000:bbbbbbbb-1111-1111-1111-111111111111"' "$ROOT"
+  status=$?
+  [ "$status" -ne 0 ] || fail "target_ready should fail when list-panes itself errors out, not just when it reports no match"
+  pass "fm_backend_cmux_target_ready: fails when list-panes itself errors (not just when it reports no match)"
+}
+
 test_target_ready_checks_expected_label() {
   local dir fb title
   dir="$TMP_ROOT/ready-label-ok"; mkdir -p "$dir/responses"
@@ -1131,6 +1165,7 @@ test_ensure_running_fails_fast_on_unauth_without_launching
 test_create_task_refuses_duplicate_label
 test_create_task_creates_and_parses_ids
 test_target_ready_fails_when_target_absent
+test_target_ready_fails_when_list_panes_errors_empty
 test_target_ready_checks_expected_label
 test_target_ready_rejects_label_mismatch
 test_capture_trims_locally

tests/fm-test-run.test.sh

@@ -1736,6 +1736,39 @@ assert len(doc["scripts"])==3
   pass "aggregate-json merges lane timing artifacts"
 }
 
+test_aggregate_json_skips_unreadable_lane() {
+  local tmp rc out
+  tmp=$(mktemp -d "${TMPDIR:-/tmp}/fm-test-run-aggjson-bad.XXXXXX")
+  cat >"$tmp/good.json" <<'JSON'
+{
+  "run_id": "good",
+  "selection": "lane=portable-serial-1of4",
+  "started_at": "2026-07-22T00:00:00Z",
+  "finished_at": "2026-07-22T00:01:00Z",
+  "summary": {"total": 1, "failed": 0, "skipped_gate": 0, "duration_ms": 1000},
+  "scripts": [{"path": "tests/a.test.sh", "family": "pure-contract-unit", "duration_ms": 1000, "exit": 0, "gate_skip": false}]
+}
+JSON
+  # Simulates a lane artifact truncated when its own job was killed
+  # (OOM, hang timeout) mid-write, before the closing brace landed.
+  printf '{"run_id": "truncated", "summary": {"total": 1' >"$tmp/truncated.json"
+
+  rc=0
+  out=$("$RUNNER" --aggregate-json "$tmp/out.json" "$tmp/good.json" "$tmp/truncated.json" 2>"$tmp/err") || rc=$?
+  [ "$rc" -eq 0 ] || { rm -rf "$tmp"; fail "aggregate-json must not crash on an unreadable lane, got rc=$rc: $(cat "$tmp/err")"; }
+  assert_contains "$out" "FM_TEST_AGGREGATE lanes=1 total=1 failed=0" "aggregate summary line must count only the readable lane"
+  assert_grep "skipping unreadable timing artifact" "$tmp/err" "no warning was printed for the unreadable lane"
+  python3 -c '
+import json,sys
+doc=json.load(open(sys.argv[1]))
+assert doc["summary"]["lanes"]==1
+assert len(doc["lanes"])==1
+assert doc["lanes"][0]["run_id"]=="good"
+' "$tmp/out.json" || { rm -rf "$tmp"; fail "aggregate JSON must contain only the readable lane"; }
+  rm -rf "$tmp"
+  pass "aggregate-json skips an unreadable lane instead of crashing the whole aggregate"
+}
+
 test_list_all_exact_suite_coverage
 test_family_selection
 test_single_script_selection
@@ -1775,3 +1808,4 @@ test_max_wall_ms_is_a_result_not_advice
 test_jobs_parallel_scheduler_and_failure_propagation
 test_herdr_ci_family_run_has_a_step_timeout
 test_aggregate_json
+test_aggregate_json_skips_unreadable_lane