diff --git a/.agents/skills/stuck-crewmate-recovery/SKILL.md b/.agents/skills/stuck-crewmate-recovery/SKILL.md index cf741b9d95f..b9b94b27d43 100644 --- a/.agents/skills/stuck-crewmate-recovery/SKILL.md +++ b/.agents/skills/stuck-crewmate-recovery/SKILL.md @@ -23,7 +23,7 @@ The target window's harness is recorded as `harness=` in `state/.meta`. This procedure covers ordinary `kind=ship` and `kind=scout` direct reports. Load `secondmate-provisioning` instead for `kind=secondmate` recovery. -For a REMOTE secondmate, `fm-crew-state`'s `unknown`/`worktree gone` and `fm-send`'s `remote send failed`/`delivery unconfirmed` verdicts are unreliable and routinely false-negative; do not conclude the mate is dead or the send failed from those alone, confirm against the actual remote pane first. +For a REMOTE secondmate, `fm-crew-state` and `fm-peek` read the actual remote endpoint over `fm-on.sh`, and `fm-send` reports a delivered-with-pending-confirmation steer as delivered (their headers own the contracts); an `unknown-remote` read or unreachable-host failure means the remote state could not be read, never that the mate is dead or the send failed. Recover a genuinely stuck remote mate only through `bin/fm-spawn.sh --secondmate`, never raw herdr pane close/kill surgery, which strands the endpoint binding. Treat the digest's endpoint result as a presence signal, not proof that the task's work or validation run is gone. diff --git a/bin/fm-brief.sh b/bin/fm-brief.sh index 206e5a947ae..6dacf0f67a1 100755 --- a/bin/fm-brief.sh +++ b/bin/fm-brief.sh @@ -54,6 +54,7 @@ # it carries the AGENTS.md authoring bar (widely useful knowledge only, pointers # over copied detail) and has the crewmate add the fm-ensure-agents-md.sh # self-governance section when a touched project AGENTS.md lacks it. +# Refuses claiming a task id whose data// directory already exists. # Refuses to overwrite an existing brief. set -eu @@ -154,6 +155,7 @@ elif [ "$MODE_SET" -eq 1 ]; then echo "error: --mode applies only to ship briefs; a scout delivers a report and a secondmate charter is not a delivery contract" >&2 exit 1 fi +[ "${#POS[@]}" -ge 1 ] || { echo "error: task id is required" >&2; exit 1; } ID=${POS[0]} if [ "$KIND" = secondmate ] && [ "$HERDR_LAB" -eq 1 ]; then @@ -166,6 +168,34 @@ if [ "$NO_PROJECTS" -eq 1 ] && [ "$KIND" != secondmate ]; then exit 1 fi +if [ -e "$DATA/$ID" ]; then + contents="" + if [ -d "$DATA/$ID" ]; then + entries=() + shopt -s nullglob dotglob + for entry in "$DATA/$ID"/*; do + entries+=("$(basename "$entry")") + done + shopt -u nullglob dotglob + for entry in "${entries[@]}"; do + if [ -n "$contents" ]; then + contents="$contents, $entry" + else + contents="$entry" + fi + done + fi + if [ -n "$contents" ]; then + contents_desc="contains: $contents" + elif [ -d "$DATA/$ID" ]; then + contents_desc="empty directory" + else + contents_desc="existing file" + fi + echo "error: task id '$ID' already exists at $DATA/$ID ($contents_desc); choose a distinct task id (e.g. '$ID-2' or mint a new one) to preserve retained history and avoid collisions" >&2 + exit 1 +fi + BRIEF="$DATA/$ID/brief.md" [ -e "$BRIEF" ] && { echo "error: $BRIEF already exists" >&2; exit 1; } mkdir -p "$DATA/$ID" diff --git a/bin/fm-crew-state.sh b/bin/fm-crew-state.sh index 2cb290373cb..df627b487f2 100755 --- a/bin/fm-crew-state.sh +++ b/bin/fm-crew-state.sh @@ -16,10 +16,17 @@ # fixed mapping logic, no heuristics and no LLM. Output is one stable, parseable, # token-tight line firstmate can read every heartbeat: # -# state: · source: · +# state: · source: · # # Logic, in order: -# 1. Resolve worktree + backend target + kind from state/.meta. +# 1. Resolve worktree + backend target + kind from state/.meta. A meta +# recording remote_host= is a remote secondmate: its worktree and endpoint +# live on that host, so the local worktree and pane reads are skipped and +# the remote host is asked for the endpoint's recovery-grade state +# (fm-on.sh + fm-remote-secondmate-control.sh state). alive falls through +# to the routed status log; dead/missing report the remote verdict; an +# unreachable or unreadable remote reports unknown-remote, never a false +# gone/dead. # 2. Matching no-mistakes run for this crew's branch AND current code identity, # active or terminal (from `axi status`, or the coarse `no-mistakes runs` # fallback)? Branch name alone is not enough: a historical run on a reused @@ -101,10 +108,13 @@ meta_value() { # WT=$(meta_value worktree) KIND=$(meta_value kind) HARNESS=$(meta_value harness) +REMOTE_HOST=$(meta_value remote_host) [ -n "$KIND" ] || KIND=ship -# A torn-down (or never-created) worktree has no current state to read. -if [ -z "$WT" ] || [ ! -d "$WT" ]; then +# A torn-down (or never-created) worktree has no current state to read. A +# remote secondmate's recorded worktree is a path on ITS host, so the local +# probe proves nothing for it - the remote arm below reads the true source. +if [ -z "$REMOTE_HOST" ] && { [ -z "$WT" ] || [ ! -d "$WT" ]; }; then emit unknown none "worktree gone (torn down?)" fi @@ -138,6 +148,45 @@ map_log_state() { # LOG_LINE=$(log_last_line || true) LOG_VERB=$(status_line_verb "$LOG_LINE") +# --- remote secondmate: the true source is the remote endpoint --------------- +# A remote mate's recorded worktree and backend target live on its own host, so +# the local worktree probe above and the local pane reads below would misreport +# a healthy remote mate as gone or dead. Ask the remote host for the endpoint's +# recovery-grade state over the same fm-on.sh transport fm-send uses, then read +# current activity from the routed status log exactly as for a local +# secondmate (an idle endpoint is healthy for a secondmate either way). An +# unreachable host or unreadable endpoint is reported as unknown-remote - +# explicitly NOT proof of death - so a transport blip never reads as a torn +# down or dead mate; only the remote host's own dead/missing verdict may say +# the endpoint is actually gone. +if [ -n "$REMOTE_HOST" ]; then + if ! REMOTE_STATE=$(FM_HOME="$FM_HOME" "$SCRIPT_DIR/fm-on.sh" "$ID" \ + fm-remote-secondmate-control.sh state "$ID" < /dev/null 2>/dev/null); then + REMOTE_STATE= + fi + REMOTE_STATE=$(printf '%s\n' "$REMOTE_STATE" | tail -1) + case "$REMOTE_STATE" in + alive) + if [ -n "$LOG_VERB" ]; then + LOG_STATE=$(map_log_state "$LOG_LINE") + if [ "$LOG_STATE" != unknown ]; then + emit "$LOG_STATE" status-log "$(status_line_note "$LOG_LINE")${SEP}remote endpoint alive on $REMOTE_HOST" + fi + fi + emit unknown remote-endpoint "alive on $REMOTE_HOST (an idle secondmate is healthy)" + ;; + dead|missing) + emit unknown remote-endpoint "remote endpoint $REMOTE_STATE on $REMOTE_HOST" + ;; + '') + emit unknown remote-endpoint "unknown-remote: $REMOTE_HOST unreachable or endpoint unreadable (not proof of death)" + ;; + *) + emit unknown remote-endpoint "unknown-remote: endpoint state '$REMOTE_STATE' on $REMOTE_HOST (not proof of death)" + ;; + esac +fi + # pane_readable is consulted ONLY in the no-run fallback below. The run-step path # stays authoritative regardless of pane liveness - judge by the run-step, not the # shell - so a finished crew whose endpoint has closed still reports its run-step diff --git a/bin/fm-peek.sh b/bin/fm-peek.sh index 97d2ffe2d25..e3156f66ed4 100755 --- a/bin/fm-peek.sh +++ b/bin/fm-peek.sh @@ -3,6 +3,11 @@ # Usage: fm-peek.sh [lines=40] # may be an exact task id, a legacy fm- task label resolved # through this home's state/.meta, or an explicit backend target. +# A selector whose meta records remote_host= is a remote secondmate: its pane +# lives on that host, so the capture routes over fm-on.sh to the host-local +# capture (fm-remote-secondmate-control.sh), clamped to that command's +# 100-line cap. An unreachable host or unreadable endpoint fails loudly naming +# the host; the local backend adapters are never asked to read a remote target. set -eu SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" @@ -16,9 +21,25 @@ STATE="${FM_STATE_OVERRIDE:-$FM_HOME/state}" "$SCRIPT_DIR/fm-guard.sh" || true RAW_TARGET=$1 -T=$(fm_backend_resolve_selector "$RAW_TARGET" "$STATE") N=${2:-40} +REMOTE_META=$(fm_backend_meta_for_selector "$RAW_TARGET" "$STATE" 2>/dev/null || true) +if [ -n "$REMOTE_META" ] && [ -n "$(fm_meta_get "$REMOTE_META" remote_host)" ]; then + REMOTE_ID=${REMOTE_META##*/} + REMOTE_ID=${REMOTE_ID%.meta} + REMOTE_HOST=$(fm_meta_get "$REMOTE_META" remote_host) + case "$N" in ''|*[!0-9]*|0) N=40 ;; esac + [ "$N" -le 100 ] || N=100 + if ! FM_HOME="$FM_HOME" "$SCRIPT_DIR/fm-on.sh" "$REMOTE_ID" \ + fm-remote-secondmate-control.sh capture "$REMOTE_ID" "$N" < /dev/null; then + echo "error: could not read the remote pane of $REMOTE_ID on $REMOTE_HOST (host unreachable or endpoint unreadable; the mate is not thereby dead)" >&2 + exit 1 + fi + exit 0 +fi + +T=$(fm_backend_resolve_selector "$RAW_TARGET" "$STATE") + BACKEND=$(fm_backend_of_selector "$RAW_TARGET" "$T" "$STATE") EXPECTED_LABEL=$(fm_backend_expected_label_of_selector "$RAW_TARGET" "$STATE") diff --git a/bin/fm-remote-secondmate-control.sh b/bin/fm-remote-secondmate-control.sh index f2edb32a7bb..aa17c952860 100755 --- a/bin/fm-remote-secondmate-control.sh +++ b/bin/fm-remote-secondmate-control.sh @@ -188,6 +188,12 @@ cmd_send() { validate_id "$id" validate_home "$id" remote_endpoint_require "$id" + # fm-send's exit status is the delivery verdict the parent home acts on + # (0 = confirmed, 3 = delivered with the submit read-back unconfirmed, other + # nonzero = failed; see bin/fm-send.sh's header). The job worker, entrypoint, + # and ssh all preserve it, so no mapping may happen here: flattening exit 3 + # into a generic failure is exactly the false-negative the parent's remote + # send path exists to avoid. FM_HOME="$TARGET_HOME" FM_ROOT_OVERRIDE="$FM_ROOT" FM_STATE_OVERRIDE="$TARGET_HOME/state" \ "$SCRIPT_DIR/fm-send.sh" "$REMOTE_ENDPOINT_TARGET" "$message" } diff --git a/bin/fm-send.sh b/bin/fm-send.sh index c46c55a340f..1da45d86f46 100755 --- a/bin/fm-send.sh +++ b/bin/fm-send.sh @@ -15,6 +15,12 @@ # submit or reports an inconclusive send. If a swallowed Enter is positively # confirmed, fm-send exits NON-ZERO so the caller knows the steer did not land # instead of silently leaving an unsubmitted instruction. +# Exit status contract: 0 = submit confirmed (or, for a remote secondmate +# target, delivered with confirmation pending - see the remote paragraph); +# 3 = the text was typed into the live endpoint and Enter was sent, but the +# submit read-back stayed unconfirmed (verify the pane before any resend, and +# never re-type blindly); any other nonzero = the send failed and nothing may +# be assumed delivered. # Submission dispatches through the target's recorded backend; the tmux adapter # shares its composer/submit core with the away-mode daemon via bin/fm-tmux-lib.sh. # Tune with FM_SEND_RETRIES (default 3) / FM_SEND_SLEEP (0.4). @@ -37,6 +43,20 @@ # re-sending a recovery request for an already-open expectation so a second # record is not created. Direct unmarked captain input never creates one. # +# Remote secondmate delivery: the send crosses fm-on.sh to a host-local leg +# (bin/fm-remote-secondmate-control.sh cmd_send) that runs this same verified +# submit against the recorded remote Herdr pane and relays its exit status +# unchanged. A leg that delivered the text into the live verified pane but +# could not synchronously confirm the submit (exit 3 - typically a busy mate +# whose harness queues the steer and keeps rendering it) is reported here as +# DELIVERED with confirmation pending: fm-send prints a non-error notice, +# exits 0, marks the pending-reply expectation delivered, and closes any +# --resolve-key decisions. Empirically that pattern is a delivered steer, a +# resend duplicates the instruction, and the parent's pending-reply +# recovery/escalation still surfaces the rare genuinely lost request. Transport +# loss (ssh exit 255, completion unknown) and every real remote failure keep +# failing loudly with the remote leg's own stderr attached. +# # Decision closure (answerer-closes): pass --resolve-key (repeatable, # before the message) when this send answers an open keyed needs-decision: or # blocked: record in the target task's state/.status. After the submit is @@ -63,7 +83,9 @@ # in this home's status log per status_open_decisions (bin/fm-classify-lib.sh), or # an active captain hold for the target task. A key in neither is refused before # sending, so a mistyped key cannot deliver an answer while silently orphaning the -# decision. A failed or unconfirmed send never closes a key; a +# decision. A failed or unconfirmed send never closes a key (a remote +# delivered-with-pending-confirmation outcome counts as delivered - see the +# remote paragraph above); a # delivered answer whose closing append fails exits nonzero with the exact # manual close command, leaving the decision open to re-surface (the safe # direction). A send without the flag never closes anything: a routine steer, @@ -537,12 +559,27 @@ else # Type once, submit, verify. Only exact empty confirms delivery; every other # verdict preserves the loud refusal boundary. send_rc=0 + REMOTE_DELIVERY_NOTICE=0 if [ "$TARGET_BACKEND" = remote ]; then - if "$SCRIPT_DIR/fm-on.sh" "$TARGET_REMOTE_ID" fm-remote-secondmate-control.sh send "$TARGET_REMOTE_ID" "$MESSAGE" < /dev/null >/dev/null; then + # The remote leg is this same script running host-locally against the + # recorded Herdr pane (cmd_send in fm-remote-secondmate-control.sh), so its + # submit verification IS the local one, and fm-on/the remote worker relay + # its exit status unchanged. Exit 3 is the delivered-unconfirmed contract + # (see this script's header) crossing the ssh boundary: the text reached + # the live verified pane and Enter was sent; only the synchronous read-back + # stayed unconfirmed. The remote stderr is held back and replayed only for + # a real failure, so a delivered outcome does not surface the inner leg's + # diagnostics as alarm. + remote_err=$("$SCRIPT_DIR/fm-on.sh" "$TARGET_REMOTE_ID" fm-remote-secondmate-control.sh send "$TARGET_REMOTE_ID" "$MESSAGE" < /dev/null 2>&1 >/dev/null) || send_rc=$? + if [ "$send_rc" -eq 0 ]; then + verdict=empty + elif [ "$send_rc" -eq 3 ]; then verdict=empty + send_rc=0 + REMOTE_DELIVERY_NOTICE=1 else - send_rc=$? verdict=send-failed + [ -z "$remote_err" ] || printf '%s\n' "$remote_err" >&2 fi elif verdict=$(fm_backend_send_text_submit "$TARGET_BACKEND" "$T" "$MESSAGE" "$retries" "$sleep_s" "$settle" "$EXPECTED_LABEL"); then : @@ -571,6 +608,19 @@ else echo "error: text not sent to $T ($TARGET_BACKEND send failed; tried $RESOLUTION_TRIED)" >&2 exit 1 ;; + pending) + # The text was typed into the live target and Enter was sent; only the + # submit read-back stayed unconfirmed (e.g. a busy harness queues the + # steer and keeps rendering it). That is not a proven failure, so never + # re-type the message: verify the pane instead. Exit 3 is the documented + # delivered-unconfirmed status, and the remote send leg above depends on + # it crossing the ssh boundary intact. + if [ "$PENDING_REPLY_CREATED" = 1 ] && [ -n "$PENDING_REPLY_CORR" ]; then + fm_pending_reply_discard_undelivered "$STATE" "$PENDING_REPLY_CORR" || true + fi + echo "fm-send: text delivered to $T but submission is unconfirmed (verdict=pending; tried $RESOLUTION_TRIED); do not retype or blindly resend - verify with fm-peek.sh, then re-send '--key Enter' only if the composer still holds the text" >&2 + exit 3 + ;; *) if [ "$PENDING_REPLY_CREATED" = 1 ] && [ -n "$PENDING_REPLY_CORR" ]; then fm_pending_reply_discard_undelivered "$STATE" "$PENDING_REPLY_CORR" || true @@ -600,6 +650,12 @@ else fm_send_close_resolved_keys "$RESOLVE_ANSWER_TEXT" || exit 1 fm_send_feed_resolved_holds "$RESOLVE_ANSWER_TEXT" || exit 1 fi + # Remote delivered-with-pending-confirmation: the outcome above is treated as + # delivered (expectation marked, keys closed), and this one non-error notice + # carries the remaining nuance so nobody re-sends the steer. + if [ "$REMOTE_DELIVERY_NOTICE" = 1 ]; then + echo "fm-send: delivered to remote secondmate $TARGET_REMOTE_ID; the remote pane accepted the text and Enter, and only the synchronous submit confirmation is still pending. This is not a failure - do not resend; the pending-reply expectation stays armed." >&2 + fi # Submit landed with exact empty. Confirmation only proves the text was # accepted; the harness still needs a beat to spin up the # turn before its busy footer shows. Pause so an immediate peek catches the diff --git a/docs/remote-secondmates.md b/docs/remote-secondmates.md index 5a38d48e52b..a1560e20b5d 100644 --- a/docs/remote-secondmates.md +++ b/docs/remote-secondmates.md @@ -168,6 +168,11 @@ Send routed requests normally: FM_HOME= bin/fm-send.sh fm- '' ``` +The [`fm-send.sh` header](../bin/fm-send.sh) owns the exact delivery-status contract. +When the verified remote endpoint accepts the text and Enter but synchronous submit confirmation remains pending, the primary reports the request as delivered rather than failed; do not resend it, because its pending-reply expectation remains armed. +`fm-peek.sh` and `fm-crew-state.sh` route remote-secondmate reads to the endpoint's host instead of consulting local worktree or backend state. +An unreachable or unreadable remote read is unknown, not evidence that the endpoint is dead. + Marked requests keep the existing correlation contract. The remote charter appends replies to `state/parent-replies.status` in the remote home. A process-event source performs a non-destructive, cursor-anchored delta read, fetches only referenced `data/*.md` documents through the confined reader, mirrors every content-bearing line at most once into the primary status channel, and does not carry blank separators. @@ -231,6 +236,9 @@ The lifecycle test covers seeding a registered project that this machine has nev ```sh bin/fm-test-run.sh tests/fm-on.test.sh +bin/fm-test-run.sh tests/fm-send-remote-delivery.test.sh +bin/fm-test-run.sh tests/fm-peek-remote.test.sh +bin/fm-test-run.sh tests/fm-crew-state.test.sh bin/fm-test-run.sh tests/fm-remote-job.test.sh bin/fm-test-run.sh tests/fm-remote-doctor.test.sh bin/fm-test-run.sh tests/fm-project-origin.test.sh diff --git a/docs/tmux-backend.md b/docs/tmux-backend.md index 4d8c3e75feb..c2acead0c2f 100644 --- a/docs/tmux-backend.md +++ b/docs/tmux-backend.md @@ -85,7 +85,7 @@ The supervisor guard selects only the detected primary harness's signature rathe It types a message once and retries Enter only until the composer clears. Only a proven empty composer is a positive delivery acknowledgement. Text left in established structure remains `pending`, text in ambiguous structure remains unproven, and unreadable or unsafe state remains unknown. -`fm-send.sh` reports every unconfirmed verdict as a failure instead of retyping or assuming delivery. +`fm-send.sh` never retypes or assumes a confirmed submit for an unconfirmed verdict; its header owns the distinct delivered-unconfirmed exit status and operator response. OpenCode 1.18.4 has one busy-queue exception. While OpenCode is mid-turn, Enter queues the message but leaves its text visible until the turn completes. diff --git a/tests/fm-brief.test.sh b/tests/fm-brief.test.sh index c5ee3d00f05..18bd25314cd 100755 --- a/tests/fm-brief.test.sh +++ b/tests/fm-brief.test.sh @@ -543,7 +543,7 @@ test_secondmate_directory_paths_are_absolute_and_output_is_stable() { "$ROOT/bin/fm-brief.sh" relative-home --secondmate --no-projects >/dev/null 2>&1 baseline="$root/absolute-home-charter" cp "$brief" "$baseline" - rm -f "$brief" + rm -rf "$home/data/relative-home" ( cd "$root" || exit 1 CDPATH="$root/cdpath" FM_HOME=home FM_SECONDMATE_CHARTER=x \ @@ -559,7 +559,7 @@ test_secondmate_directory_paths_are_absolute_and_output_is_stable() { "$ROOT/bin/fm-brief.sh" relative-state --secondmate --no-projects >/dev/null 2>&1 baseline="$root/absolute-state-charter" cp "$brief" "$baseline" - rm -f "$brief" + rm -rf "$home/data/relative-state" ( cd "$root" || exit 1 CDPATH="$root/cdpath" FM_HOME="$home" FM_STATE_OVERRIDE=state-override FM_SECONDMATE_CHARTER=x \ @@ -575,7 +575,7 @@ test_secondmate_directory_paths_are_absolute_and_output_is_stable() { "$ROOT/bin/fm-brief.sh" relative-data --secondmate --no-projects >/dev/null 2>&1 baseline="$root/absolute-data-charter" cp "$brief" "$baseline" - rm -f "$brief" + rm -rf "$data_override/relative-data" ( cd "$root" || exit 1 CDPATH="$root/cdpath" FM_HOME="$home" FM_DATA_OVERRIDE=data-override FM_SECONDMATE_CHARTER=x \ @@ -712,6 +712,72 @@ test_scout_and_secondmate_scaffold() { pass "fm-brief: scout and secondmate code paths still scaffold well-formed briefs" } +# Claiming a task id whose data// already exists is refused and preserves retained reports. +test_task_id_reuse_refused_and_preserves_retained_report() { + local home id report_path report_content err status + home="$TMP_ROOT/id-reuse-home" + mkdir -p "$home/data" + + # Case 1: Existing data// containing a retained report + id="vm-verify-12" + mkdir -p "$home/data/$id" + report_path="$home/data/$id/report.md" + report_content="Detailed verification report for earlier work. Preserved intact." + printf '%s\n' "$report_content" > "$report_path" + + err="$home/retained-report.err" + FM_HOME="$home" "$ROOT/bin/fm-brief.sh" "$id" some-proj --mode no-mistakes 2>"$err"; status=$? + expect_code 1 "$status" "claiming an id with an existing data// directory must exit 1" + assert_grep "error: task id '$id' already exists at $home/data/$id (contains: report.md)" "$err" \ + "refusal did not name the directory and list its retained contents" + assert_grep "choose a distinct task id (e.g. '$id-2' or mint a new one)" "$err" \ + "refusal did not provide a way forward for the operator" + + # Verify the retained report survives intact + [ -f "$report_path" ] || fail "retained report.md was removed" + [ "$(cat "$report_path")" = "$report_content" ] \ + || fail "retained report.md content was modified during attempted id reuse" + assert_absent "$home/data/$id/brief.md" "brief.md was written despite id reuse refusal" + + # Case 2: Existing data// containing multiple artifacts (brief and report) + id="multi-artifact-task" + mkdir -p "$home/data/$id" + printf 'prior brief\n' > "$home/data/$id/brief.md" + printf 'prior report\n' > "$home/data/$id/report.md" + err="$home/multi-artifact.err" + FM_HOME="$home" "$ROOT/bin/fm-brief.sh" "$id" some-proj --scout 2>"$err"; status=$? + expect_code 1 "$status" "scout scaffold with existing data// must exit 1" + assert_grep "error: task id '$id' already exists at $home/data/$id (contains: brief.md, report.md)" "$err" \ + "refusal did not list all entries in data/" + + # Case 3: Existing empty data// directory + id="empty-dir-task" + mkdir -p "$home/data/$id" + err="$home/empty-dir.err" + FM_HOME="$home" "$ROOT/bin/fm-brief.sh" "$id" some-proj --mode direct-PR 2>"$err"; status=$? + expect_code 1 "$status" "scaffold with empty existing data/ must exit 1" + assert_grep "error: task id '$id' already exists at $home/data/$id (empty directory)" "$err" \ + "refusal did not identify empty directory" + + # Case 4: Existing file at data/ + id="file-task" + touch "$home/data/$id" + err="$home/file-task.err" + FM_HOME="$home" "$ROOT/bin/fm-brief.sh" "$id" some-proj --mode local-only 2>"$err"; status=$? + expect_code 1 "$status" "scaffold with existing file at data/ must exit 1" + assert_grep "error: task id '$id' already exists at $home/data/$id (existing file)" "$err" \ + "refusal did not identify existing file" + + # Case 5: Unused task id succeeds without tripping the refusal + id="fresh-task-unused" + err="$home/fresh-task.err" + FM_HOME="$home" "$ROOT/bin/fm-brief.sh" "$id" some-proj --mode no-mistakes >/dev/null 2>"$err"; status=$? + expect_code 0 "$status" "scaffold with unused task id must exit 0" + assert_present "$home/data/$id/brief.md" "brief was not scaffolded for unused task id" + + pass "fm-brief: task id reuse is refused, reports directory contents, gives a way forward, and preserves retained artifacts" +} + test_script_parses test_no_heredoc_in_command_substitution test_help_includes_entire_header @@ -732,3 +798,4 @@ test_secondmate_directory_paths_are_absolute_and_output_is_stable test_pause_verb_override_renders_all_brief_scaffolds test_scout_and_secondmate_load_decision_hold_policy test_scout_and_secondmate_scaffold +test_task_id_reuse_refused_and_preserves_retained_report diff --git a/tests/fm-calm-pi-extension.test.sh b/tests/fm-calm-pi-extension.test.sh index 5284491ec96..d730f2d2abe 100755 --- a/tests/fm-calm-pi-extension.test.sh +++ b/tests/fm-calm-pi-extension.test.sh @@ -1817,6 +1817,15 @@ TS fi pane=$(tmux -L "$TMUX_SOCKET" capture-pane -p -t "$TMUX_SESSION" -S - 2>/dev/null || true) + i=0 + while [ "$i" -lt 120 ]; do + pane=$(tmux -L "$TMUX_SOCKET" capture-pane -p -t "$TMUX_SESSION" -S - 2>/dev/null || true) + if printf '%s\n' "$pane" | grep -Fq "CAPTAIN_ANSWER_$label" && printf '%s\n' "$pane" | grep -Fq "MONITOR_HANDLED_${label}_ONE"; then + break + fi + sleep 0.05 + i=$((i + 1)) + done [ "$(printf '%s\n' "$pane" | grep -Fc "CAPTAIN_ANSWER_$label" || true)" -eq 1 ] \ || fail "Pi follow-up $label case rendered a duplicate captain answer" assert_contains "$pane" "CAPTAIN_PROMPT_$label" "Pi follow-up $label case hid the genuine captain prompt" diff --git a/tests/fm-crew-state.test.sh b/tests/fm-crew-state.test.sh index 8f986b6139e..602b3e5cfc3 100755 --- a/tests/fm-crew-state.test.sh +++ b/tests/fm-crew-state.test.sh @@ -1150,6 +1150,105 @@ test_torn_down_worktree() { pass "torn-down worktree is handled gracefully" } +# --- remote secondmate arm --------------------------------------------------- +# A meta recording remote_host= must never be read through the local worktree +# probe or a local backend adapter: the recorded worktree and pane live on the +# remote host, and the old local reads misreported a healthy remote mate as +# "worktree gone". These cases drive the real helper over the real fm-on.sh +# route with a stubbed ssh transport (FM_SSH_BIN seam): the stub prints +# FM_FAKE_REMOTE_STATE_OUT as the remote endpoint's recovery-grade state and +# exits FM_FAKE_SSH_RC. + +setup_remote_case() { # -> echoes case dir with remote meta + registry + local d + d=$(new_case "$1") + mkdir -p "$d/data" "$d/fakebin" + fm_write_meta "$d/state/rsm.meta" \ + "window=remote:rsm" \ + "endpoint_task_id=rsm" \ + "worktree=/remote/home/never-locally-present" \ + "harness=claude" \ + "kind=secondmate" \ + "mode=secondmate" \ + "remote_host=remote-mac" \ + "remote_root=/remote/root" \ + "remote_backend=herdr" \ + "remote_herdr_session=fm-remote" \ + "remote_target=fm-remote:w1:p1" + cat > "$d/data/secondmates.md" < "$d/fakebin/fake-ssh" <<'SH' +#!/usr/bin/env bash +cat > /dev/null +[ -z "${FM_FAKE_REMOTE_STATE_OUT:-}" ] || printf '%s\n' "$FM_FAKE_REMOTE_STATE_OUT" +exit "${FM_FAKE_SSH_RC:-0}" +SH + chmod +x "$d/fakebin/fake-ssh" + printf '%s\n' "$d" +} + +run_remote_crew_state() { # + PATH="$1/fakebin:$PATH" FM_HOME="$1" FM_STATE_OVERRIDE="$1/state" \ + FM_SSH_BIN="$1/fakebin/fake-ssh" "$CREW_STATE" "$2" +} + +test_remote_alive_with_log_uses_status_log() { + reset_fakes + local d out rc + d=$(setup_remote_case remote-alive-log) + make_fakebin "$d" >/dev/null + printf 'working: refactoring the quota adapter\n' > "$d/state/rsm.status" + out=$(FM_FAKE_REMOTE_STATE_OUT=alive FM_FAKE_SSH_RC=0 run_remote_crew_state "$d" rsm); rc=$? + expect_code 0 "$rc" "remote alive exits 0" + assert_contains "$out" "state: working" "alive remote mate with a working log reads working" + assert_contains "$out" "source: status-log" "alive remote mate reads current activity from the routed log" + assert_contains "$out" "remote endpoint alive on remote-mac" "the remote liveness read should be visible" + assert_not_contains "$out" "worktree gone" "a healthy remote mate must never read as torn down" + pass "fm-crew-state remote: alive endpoint falls through to the routed status log" +} + +test_remote_alive_idle_is_healthy_not_gone() { + reset_fakes + local d out rc + d=$(setup_remote_case remote-alive-idle) + make_fakebin "$d" >/dev/null + out=$(FM_FAKE_REMOTE_STATE_OUT=alive FM_FAKE_SSH_RC=0 run_remote_crew_state "$d" rsm); rc=$? + expect_code 0 "$rc" "remote alive-idle exits 0" + assert_contains "$out" "source: remote-endpoint" "the remote endpoint is the reported source" + assert_contains "$out" "alive on remote-mac" "an idle remote mate reads alive" + assert_not_contains "$out" "worktree gone" "a healthy remote mate must never read as torn down" + assert_not_contains "$out" "backend target gone" "a healthy remote mate must never read as a dead target" + pass "fm-crew-state remote: an idle alive endpoint reads alive, never gone or dead" +} + +test_remote_unreachable_is_unknown_remote_not_dead() { + reset_fakes + local d out rc + d=$(setup_remote_case remote-unreachable) + make_fakebin "$d" >/dev/null + printf 'working: refactoring the quota adapter\n' > "$d/state/rsm.status" + out=$(FM_FAKE_SSH_RC=255 run_remote_crew_state "$d" rsm); rc=$? + expect_code 0 "$rc" "unreachable remote exits 0" + assert_contains "$out" "unknown-remote" "an unreachable remote must be labeled unknown-remote" + assert_contains "$out" "not proof of death" "an unreachable remote must not read as dead" + assert_not_contains "$out" "worktree gone" "an unreachable remote must never read as torn down" + assert_not_contains "$out" "backend target gone" "an unreachable remote must never read as a dead target" + pass "fm-crew-state remote: an unreachable host reads unknown-remote, never gone or dead" +} + +test_remote_dead_reports_remote_verdict() { + reset_fakes + local d out rc + d=$(setup_remote_case remote-dead) + make_fakebin "$d" >/dev/null + out=$(FM_FAKE_REMOTE_STATE_OUT=dead FM_FAKE_SSH_RC=0 run_remote_crew_state "$d" rsm); rc=$? + expect_code 0 "$rc" "remote dead exits 0" + assert_contains "$out" "remote endpoint dead on remote-mac" \ + "a genuinely dead remote endpoint reports the remote host's own verdict" + pass "fm-crew-state remote: the remote host's own dead verdict is reported truthfully" +} + test_missing_meta() { reset_fakes local d; d=$(new_case nometa) @@ -1350,6 +1449,10 @@ test_dead_window_still_reports_active_run_step test_no_timeout_uses_perl_bound test_scout_skips_run_lookup test_torn_down_worktree +test_remote_alive_with_log_uses_status_log +test_remote_alive_idle_is_healthy_not_gone +test_remote_unreachable_is_unknown_remote_not_dead +test_remote_dead_reports_remote_verdict test_missing_meta test_provably_working_via_runs_list_fallback test_not_provably_working_when_stopped diff --git a/tests/fm-daemon.test.sh b/tests/fm-daemon.test.sh index 2fe02fb4318..ad0925be4d4 100755 --- a/tests/fm-daemon.test.sh +++ b/tests/fm-daemon.test.sh @@ -1569,27 +1569,37 @@ test_inject_wedge_alarm_throttles_when_marker_cannot_be_written() { pass "in-process wedge throttle prevents alert spam when the marker cannot persist" } -test_fm_send_exits_nonzero_on_confirmed_swallow() { - # fm-send.sh must exit NON-ZERO when a steer's Enter is positively swallowed - # (text left in the composer), so firstmate learns the instruction did not land - # — and exit ZERO on a clean submit. - local dir fakebin err +test_fm_send_reports_delivered_unconfirmed_submit() { + # When text was typed and Enter sent but the submit read-back remains pending, + # fm-send must return its documented delivered-unconfirmed status and prevent + # a duplicate resend reflex. A synchronously confirmed submit remains zero. + local dir fakebin err rc dir=$(make_bordered_case send-swallow) fakebin="$dir/fakebin"; err="$dir/send.err" # Clean submit -> exit 0. PATH="$fakebin:$PATH" FM_HOME="$dir" FM_STATE_OVERRIDE="$dir/state" FM_FAKE_COMPOSER="$dir/composer" \ FM_SEND_SLEEP=0.05 "$ROOT/bin/fm-send.sh" sess:win 'route this work' >/dev/null 2>"$err" \ || fail "fm-send exited non-zero on a clean submit: $(cat "$err")" - # Persistent swallow -> exit non-zero with a clear message. + # Persistent composer text after Enter -> delivered-unconfirmed exit 3 with + # a non-error warning that explicitly tells the operator not to resend. printf '╭─────╮\n│ > │\n╰─────╯\n' > "$dir/composer" touch "$dir/.swallow" if PATH="$fakebin:$PATH" FM_HOME="$dir" FM_STATE_OVERRIDE="$dir/state" FM_FAKE_COMPOSER="$dir/composer" \ FM_FAKE_SWALLOW="$dir/.swallow" FM_FAKE_PERSIST_SWALLOW=1 FM_SEND_SLEEP=0.05 \ "$ROOT/bin/fm-send.sh" sess:win 'fix findings 1 and 3, skip 2' >/dev/null 2>"$err"; then - fail "fm-send exited zero despite a swallowed Enter (silent unsubmitted instruction)" + rc=0 + else + rc=$? + fi + [ "$rc" -eq 3 ] || fail "fm-send returned $rc instead of delivered-unconfirmed exit 3: $(cat "$err")" + grep -F 'submission is unconfirmed' "$err" >/dev/null \ + || fail "fm-send did not explain the pending confirmation: $(cat "$err")" + grep -F 'do not retype or blindly resend' "$err" >/dev/null \ + || fail "fm-send did not prevent a duplicate resend: $(cat "$err")" + if grep -F 'error:' "$err" >/dev/null; then + fail "fm-send mislabeled delivered-unconfirmed as an error: $(cat "$err")" fi - grep -F 'not submitted' "$err" >/dev/null || fail "fm-send did not explain the swallowed submit: $(cat "$err")" - pass "fm-send exits non-zero on a confirmed swallow, zero on a clean submit" + pass "fm-send returns 3 with a non-error no-resend warning when confirmation stays pending" } test_fm_send_exits_nonzero_on_initial_send_failure() { @@ -1916,7 +1926,7 @@ test_wedge_alarm_hung_override_times_out_and_falls_through test_wedge_alarm_shutdown_stops_active_notifier_group test_inject_wedge_alarm_fires_active_alert_on_non_tmux_backend test_inject_wedge_alarm_throttles_when_marker_cannot_be_written -test_fm_send_exits_nonzero_on_confirmed_swallow +test_fm_send_reports_delivered_unconfirmed_submit test_fm_send_exits_nonzero_on_initial_send_failure test_fm_send_exits_nonzero_on_unproven_submit test_discover_supervisor_backend_precedence diff --git a/tests/fm-peek-remote.test.sh b/tests/fm-peek-remote.test.sh new file mode 100755 index 00000000000..7ef7286fb23 --- /dev/null +++ b/tests/fm-peek-remote.test.sh @@ -0,0 +1,110 @@ +#!/usr/bin/env bash +# fm-peek remote-secondmate capture routing. +# +# A remote secondmate's pane lives on its own host. The old path resolved the +# meta's "remote:" window through the local backend adapters and handed it +# to tmux, which failed with "can't find session: remote" - a healthy remote +# mate misreported as an unreadable endpoint. These tests drive the real +# fm-peek + fm-on executables with a stubbed ssh transport (FM_SSH_BIN seam) +# and a poisoned local tmux, pinning: +# 1. A remote selector routes the capture over the remote transport and +# prints the remote pane tail; the local adapters are never consulted. +# 2. An unreachable host fails loudly naming the host, without claiming the +# mate is dead. +set -u + +# shellcheck source=tests/lib.sh +. "$(dirname "${BASH_SOURCE[0]}")/lib.sh" + +PEEK="$ROOT/bin/fm-peek.sh" + +TMP_ROOT=$(fm_test_tmproot fm-peek-remote) + +# fake-ssh prints the canned remote capture; the poisoned tmux records any +# local read attempt so the "never consulted" property is a real assertion. +make_stubs() { # -> echoes fakebin dir + local dir=$1 fb="$1/fakebin" + mkdir -p "$fb" + cat > "$fb/fake-ssh" <<'SH' +#!/usr/bin/env bash +cat > /dev/null +[ -z "${FM_FAKE_REMOTE_CAPTURE:-}" ] || printf '%s\n' "$FM_FAKE_REMOTE_CAPTURE" +exit "${FM_FAKE_SSH_RC:-0}" +SH + chmod +x "$fb/fake-ssh" + cat > "$fb/tmux" <<'SH' +#!/usr/bin/env bash +printf 'tmux\n' >> "${FM_FAKE_TMUX_TOUCHED:?}" +exit 1 +SH + chmod +x "$fb/tmux" + printf '%s\n' "$fb" +} + +setup_remote_home() { # -> echoes home dir with remote meta + registry + local home="$TMP_ROOT/$1-$RANDOM" + mkdir -p "$home/state" "$home/data" + fm_write_meta "$home/state/rsm.meta" \ + "window=remote:rsm" \ + "endpoint_task_id=rsm" \ + "harness=claude" \ + "kind=secondmate" \ + "mode=secondmate" \ + "remote_host=remote-mac" \ + "remote_root=/remote/root" \ + "remote_backend=herdr" \ + "remote_herdr_session=fm-remote" \ + "remote_target=fm-remote:w1:p1" + cat > "$home/data/secondmates.md" < "$touched" + + out=$(env PATH="$fb:$PATH" \ + FM_HOME="$home" FM_STATE_OVERRIDE="$home/state" \ + FM_SSH_BIN="$fb/fake-ssh" FM_FAKE_SSH_RC=0 \ + FM_FAKE_REMOTE_CAPTURE='● the remote mate is mid-refactor' \ + FM_FAKE_TMUX_TOUCHED="$touched" \ + "$PEEK" rsm 20 2>"$dir/err"); rc=$? + expect_code 0 "$rc" "a healthy remote peek should succeed" + assert_contains "$out" "the remote mate is mid-refactor" \ + "the remote pane tail should be printed" + assert_not_contains "$out" "can't find session" \ + "a remote peek must not fall into a local session lookup" + [ ! -s "$touched" ] || fail "the local tmux adapter was consulted for a remote target" + pass "fm-peek remote: the capture routes over the remote transport, local adapters untouched" +} + +test_remote_peek_unreachable_fails_loudly_without_death_claim() { + local dir fb home touched rc err + dir="$TMP_ROOT/peek-down"; mkdir -p "$dir" + fb=$(make_stubs "$dir") + home=$(setup_remote_home peek-down) + touched="$dir/tmux-touched"; : > "$touched" + + env PATH="$fb:$PATH" \ + FM_HOME="$home" FM_STATE_OVERRIDE="$home/state" \ + FM_SSH_BIN="$fb/fake-ssh" FM_FAKE_SSH_RC=255 \ + FM_FAKE_TMUX_TOUCHED="$touched" \ + "$PEEK" rsm >"$dir/out" 2>"$dir/err"; rc=$? + err=$(cat "$dir/err") + [ "$rc" -ne 0 ] || fail "an unreachable remote peek must exit nonzero" + assert_contains "$err" "remote pane of rsm on remote-mac" \ + "the failure must name the remote mate and host" + assert_contains "$err" "not thereby dead" \ + "an unreadable remote pane must not be presented as a dead mate" + pass "fm-peek remote: an unreachable host fails loudly without a false death claim" +} + +test_remote_peek_reads_remote_pane +test_remote_peek_unreachable_fails_loudly_without_death_claim + +echo "all fm-peek-remote tests passed" diff --git a/tests/fm-send-remote-delivery.test.sh b/tests/fm-send-remote-delivery.test.sh new file mode 100755 index 00000000000..af546fbb4a4 --- /dev/null +++ b/tests/fm-send-remote-delivery.test.sh @@ -0,0 +1,285 @@ +#!/usr/bin/env bash +# fm-send remote-secondmate delivery reporting. +# +# The remote send leg (fm-on.sh -> fm-remote-secondmate-control.sh cmd_send) +# runs fm-send's own verified submit host-locally on the remote machine and +# relays its exit status unchanged. A leg that delivered the text into the +# live verified pane but could not synchronously confirm the submit exits 3 +# (the delivered-unconfirmed contract in bin/fm-send.sh's header); flattening +# that into a generic failure produced the false "error: text not sent" +# report that tempted duplicate resends of steers that had actually landed. +# These tests pin the delivery-reporting contract over the real fm-send + +# fm-on executables with a stubbed ssh transport (FM_SSH_BIN seam - the same +# process boundary tests/fm-on.test.sh proves preserves exit status): +# 1. Remote delivered-unconfirmed (ssh exit 3) is NOT a failure: exit 0, a +# non-error delivered notice, the inner leg's stderr held back, and the +# pending-reply expectation marked delivered (awaiting_report). +# 2. A real remote failure (nonzero, not 3/255) still fails loudly with the +# remote stderr replayed and the undelivered expectation discarded. +# 3. Transport-unknown (ssh exit 255) still refuses loudly and preserves the +# expectation as delivery_unknown. +# 4. A delivered-unconfirmed remote answer still closes its --resolve-key +# decision (delivered-with-pending-confirmation counts as delivered). +# 5. A LOCAL send whose submit read-back stays pending exits 3 with an +# honest non-error message (text delivered, submission unconfirmed). +# 6. That local unconfirmed send still never closes a --resolve-key +# decision (the local ledger boundary is unchanged). +set -u + +# shellcheck source=tests/lib.sh +. "$(dirname "${BASH_SOURCE[0]}")/lib.sh" + +SEND="$ROOT/bin/fm-send.sh" +DRAIN="$ROOT/bin/fm-wake-drain.sh" + +TMP_ROOT=$(fm_test_tmproot fm-send-remote-delivery) + +# Stub tmux for the local legs: logs literal typed text to FM_SEND_LOG. The +# default composer reads empty (clean submit); FM_FAKE_TMUX_PENDING=1 keeps a +# proven pending composer with no busy footer, so the real submit core +# exhausts its Enter budget and reports the pending verdict. The ssh stub +# records the invocation, emits FM_FAKE_SSH_STDERR as the remote leg's stderr, +# and exits FM_FAKE_SSH_RC - the exact relay contract the real transport +# preserves. +make_stubs() { # -> echoes fakebin dir + local dir=$1 fb="$1/fakebin" + mkdir -p "$fb" + cat > "$fb/tmux" <<'SH' +#!/usr/bin/env bash +set -u +case "${1:-}" in + send-keys) + shift + literal=0 + while [ $# -gt 0 ]; do + case "$1" in + -t) shift 2 ;; + -l) literal=1; shift ;; + *) break ;; + esac + done + if [ "$literal" = 1 ]; then + printf '%s' "${1:-}" >> "$FM_SEND_LOG" + fi + exit 0 ;; + display-message) + for a in "$@"; do case "$a" in *cursor_y*) printf '1\n'; exit 0 ;; esac; done + printf 'fakepane\n'; exit 0 ;; + capture-pane) + if [ "${FM_FAKE_TMUX_PENDING:-0}" = 1 ]; then + printf '╭────────────╮\n│ > steer │\n╰────────────╯\n' + else + printf '╭────╮\n│ │\n╰────╯\n' + fi + exit 0 ;; + list-windows) exit 0 ;; +esac +exit 0 +SH + chmod +x "$fb/tmux" + cat > "$fb/sleep" <<'SH' +#!/usr/bin/env bash +exit 0 +SH + chmod +x "$fb/sleep" + cat > "$fb/fake-ssh" <<'SH' +#!/usr/bin/env bash +cat > /dev/null +printf '%s\n' "$*" >> "$FM_SSH_LOG" +[ -z "${FM_FAKE_SSH_STDERR:-}" ] || printf '%s\n' "$FM_FAKE_SSH_STDERR" >&2 +exit "${FM_FAKE_SSH_RC:-0}" +SH + chmod +x "$fb/fake-ssh" + printf '%s\n' "$fb" +} + +setup_home() { # -> echoes a fresh home dir with an empty state/ + local home="$TMP_ROOT/$1-$RANDOM" + mkdir -p "$home/state" + printf '%s\n' "$home" +} + +# A home with a remote-secondmate task meta plus the registry row fm-on.sh +# resolves the ssh route from - the same shape a live remote mate records. +setup_remote_home() { # -> echoes home dir + local home + home=$(setup_home "$1") + mkdir -p "$home/data" + fm_write_meta "$home/state/rsm.meta" \ + "window=fm-remote:w1:p1" \ + "endpoint_task_id=rsm" \ + "harness=claude" \ + "kind=secondmate" \ + "mode=secondmate" \ + "yolo=off" \ + "remote_host=remote-mac" \ + "remote_root=/remote/root" \ + "remote_backend=herdr" \ + "remote_herdr_session=fm-remote" \ + "remote_target=fm-remote:w1:p1" + cat > "$home/data/secondmates.md" <, or empty. +pending_record() { # + find "$1/state/pending-replies" -maxdepth 1 -type f ! -name '.*' 2>/dev/null | head -1 +} + +drain_out() { # + FM_STATE_OVERRIDE="$1/state" "$DRAIN" 2>/dev/null +} + +test_remote_delivered_unconfirmed_is_not_failure() { + local dir fb log ssh_log home rc err rec + dir="$TMP_ROOT/remote-du"; mkdir -p "$dir" + fb=$(make_stubs "$dir"); log="$dir/send.log"; ssh_log="$dir/ssh.log"; : > "$ssh_log" + home=$(setup_remote_home remote-du) + + : > "$log" + env PATH="$fb:$PATH" \ + FM_ROOT_OVERRIDE="$ROOT" FM_HOME="$home" FM_SEND_LOG="$log" FM_SEND_SETTLE=0 \ + FM_SSH_BIN="$fb/fake-ssh" FM_SSH_LOG="$ssh_log" FM_FAKE_SSH_RC=3 \ + FM_FAKE_SSH_STDERR='fm-send: text delivered to fm-remote:w1:p1 but submission is unconfirmed (verdict=pending; tried meta=/remote/home/state/fm-remote:w1:p1.meta; metadata window/terminal lookup; backend=herdr; endpoint=verified)' \ + "$SEND" rsm "please rename the metric" >"$dir/out" 2>"$dir/err"; rc=$? + err=$(cat "$dir/err") + expect_code 0 "$rc" "a delivered-unconfirmed remote send must not exit as a failure" + assert_grep 'fm-remote-entrypoint.sh' "$ssh_log" "the steer should cross the remote transport" + assert_contains "$err" "delivered to remote secondmate rsm" \ + "the outcome must be reported as delivered" + assert_not_contains "$err" "text not sent" "a delivered steer must not read as not sent" + assert_not_contains "$err" "not submitted" "a delivered steer must not read as not submitted" + assert_not_contains "$err" "error: text" "a delivered steer must not carry an error-styled report" + assert_not_contains "$err" "verdict=pending" \ + "the inner leg's unconfirmed diagnostics must be held back on a delivered outcome" + + rec=$(pending_record "$home") + [ -n "$rec" ] || fail "the pending-reply expectation must survive a delivered-unconfirmed send" + [ -n "$(grep '^delivered_epoch=' "$rec" | cut -d= -f2-)" ] \ + || fail "a delivered-unconfirmed send must mark the expectation delivered: $(cat "$rec")" + [ "$(grep '^phase=' "$rec" | tail -1 | cut -d= -f2-)" = awaiting_report ] \ + || fail "a delivered-unconfirmed send must leave the expectation awaiting its report: $(cat "$rec")" + pass "fm-send remote: delivered-unconfirmed reports delivered, exits 0, keeps the expectation armed" +} + +test_remote_real_failure_still_fails() { + local dir fb log ssh_log home rc err + dir="$TMP_ROOT/remote-fail"; mkdir -p "$dir" + fb=$(make_stubs "$dir"); log="$dir/send.log"; ssh_log="$dir/ssh.log"; : > "$ssh_log" + home=$(setup_remote_home remote-fail) + + : > "$log" + env PATH="$fb:$PATH" \ + FM_ROOT_OVERRIDE="$ROOT" FM_HOME="$home" FM_SEND_LOG="$log" FM_SEND_SETTLE=0 \ + FM_SSH_BIN="$fb/fake-ssh" FM_SSH_LOG="$ssh_log" FM_FAKE_SSH_RC=1 \ + FM_FAKE_SSH_STDERR='error: remote secondmate rsm endpoint metadata is invalid; refusing access until it is explicitly migrated' \ + "$SEND" rsm "please rename the metric" >"$dir/out" 2>"$dir/err"; rc=$? + err=$(cat "$dir/err") + [ "$rc" -ne 0 ] || fail "a genuinely failed remote send must exit nonzero" + assert_contains "$err" "error: text not sent to remote:rsm" \ + "a real remote failure must still report a real error" + assert_contains "$err" "endpoint metadata is invalid" \ + "a real remote failure must replay the remote leg's own stderr" + [ -z "$(pending_record "$home")" ] \ + || fail "a failed send must discard its undelivered expectation" + pass "fm-send remote: a real remote failure still fails loudly with the remote diagnostics" +} + +test_remote_transport_unknown_preserves_expectation() { + local dir fb log ssh_log home rc err rec + dir="$TMP_ROOT/remote-255"; mkdir -p "$dir" + fb=$(make_stubs "$dir"); log="$dir/send.log"; ssh_log="$dir/ssh.log"; : > "$ssh_log" + home=$(setup_remote_home remote-255) + + : > "$log" + env PATH="$fb:$PATH" \ + FM_ROOT_OVERRIDE="$ROOT" FM_HOME="$home" FM_SEND_LOG="$log" FM_SEND_SETTLE=0 \ + FM_SSH_BIN="$fb/fake-ssh" FM_SSH_LOG="$ssh_log" FM_FAKE_SSH_RC=255 \ + "$SEND" rsm "please rename the metric" >"$dir/out" 2>"$dir/err"; rc=$? + err=$(cat "$dir/err") + [ "$rc" -ne 0 ] || fail "an unknown-completion transport loss must exit nonzero" + assert_contains "$err" "delivery to remote secondmate rsm is unknown" \ + "transport loss must be reported as unknown delivery, not silently dropped" + rec=$(pending_record "$home") + [ -n "$rec" ] || fail "transport loss must preserve the expectation for reconciliation" + [ "$(grep '^phase=' "$rec" | tail -1 | cut -d= -f2-)" = delivery_unknown ] \ + || fail "transport loss must move the expectation to delivery_unknown: $(cat "$rec")" + pass "fm-send remote: ssh 255 still refuses loudly and preserves the expectation as delivery_unknown" +} + +test_remote_delivered_unconfirmed_closes_resolve_key() { + local dir fb log ssh_log home rc out + dir="$TMP_ROOT/remote-key"; mkdir -p "$dir" + fb=$(make_stubs "$dir"); log="$dir/send.log"; ssh_log="$dir/ssh.log"; : > "$ssh_log" + home=$(setup_remote_home remote-key) + printf 'needs-decision [key=upgrade-window]: tonight or the weekend\n' > "$home/state/rsm.status" + + : > "$log" + env PATH="$fb:$PATH" \ + FM_ROOT_OVERRIDE="$ROOT" FM_HOME="$home" FM_SEND_LOG="$log" FM_SEND_SETTLE=0 \ + FM_SSH_BIN="$fb/fake-ssh" FM_SSH_LOG="$ssh_log" FM_FAKE_SSH_RC=3 \ + "$SEND" rsm --resolve-key upgrade-window "the weekend, freeze Friday" >/dev/null 2>&1; rc=$? + expect_code 0 "$rc" "a delivered-unconfirmed remote answer must not exit as a failure" + grep -F 'resolved [key=upgrade-window]: answered: the weekend, freeze Friday' "$home/state/rsm.status" >/dev/null \ + || fail "a delivered-unconfirmed remote answer must close the decision: $(cat "$home/state/rsm.status")" + out=$(drain_out "$home") + if printf '%s' "$out" | grep -F 'OPEN DECISIONS' >/dev/null; then + fail "the answered decision still lists as open after a delivered-unconfirmed answer: $out" + fi + pass "fm-send remote: a delivered-unconfirmed answer closes its --resolve-key decision" +} + +test_local_pending_reports_delivered_unconfirmed() { + local dir fb log home rc err + dir="$TMP_ROOT/local-pending"; mkdir -p "$dir" + fb=$(make_stubs "$dir"); log="$dir/send.log" + home=$(setup_home local-pending) + fm_write_meta "$home/state/t1.meta" "window=sess:fm-t1" "kind=ship" + + : > "$log" + env PATH="$fb:$PATH" FM_FAKE_TMUX_PENDING=1 \ + FM_ROOT_OVERRIDE="$home" FM_HOME="$home" FM_SEND_LOG="$log" FM_SEND_SETTLE=0 \ + "$SEND" t1 "steer text" >"$dir/out" 2>"$dir/err"; rc=$? + err=$(cat "$dir/err") + expect_code 3 "$rc" "an unconfirmed local submit must exit with the delivered-unconfirmed status" + assert_contains "$err" "submission is unconfirmed" \ + "the unconfirmed local submit must be described honestly" + assert_not_contains "$err" "not submitted" \ + "an unconfirmed local submit must not claim the text was not submitted" + assert_not_contains "$err" "error:" \ + "an unconfirmed local submit must not carry an error-styled report" + pass "fm-send local: an unconfirmed submit exits 3 with an honest non-error report" +} + +test_local_pending_does_not_close_resolve_key() { + local dir fb log home rc out + dir="$TMP_ROOT/local-pending-key"; mkdir -p "$dir" + fb=$(make_stubs "$dir"); log="$dir/send.log" + home=$(setup_home local-pending-key) + fm_write_meta "$home/state/t2.meta" "window=sess:fm-t2" "kind=ship" + printf 'blocked [key=creds]: need the deploy token\n' > "$home/state/t2.status" + + : > "$log" + env PATH="$fb:$PATH" FM_FAKE_TMUX_PENDING=1 \ + FM_ROOT_OVERRIDE="$home" FM_HOME="$home" FM_SEND_LOG="$log" FM_SEND_SETTLE=0 \ + "$SEND" t2 --resolve-key creds "token is in the vault now" >/dev/null 2>&1; rc=$? + expect_code 3 "$rc" "an unconfirmed local answer must exit with the delivered-unconfirmed status" + if grep -F 'resolved' "$home/state/t2.status" >/dev/null; then + fail "an unconfirmed local answer must not close the decision: $(cat "$home/state/t2.status")" + fi + out=$(drain_out "$home") + printf '%s' "$out" | grep -F '[key=creds]' >/dev/null \ + || fail "the blocker must stay open after an unconfirmed local answer: $out" + pass "fm-send local: an unconfirmed submit still never closes a --resolve-key decision" +} + +test_remote_delivered_unconfirmed_is_not_failure +test_remote_real_failure_still_fails +test_remote_transport_unknown_preserves_expectation +test_remote_delivered_unconfirmed_closes_resolve_key +test_local_pending_reports_delivered_unconfirmed +test_local_pending_does_not_close_resolve_key + +echo "all fm-send-remote-delivery tests passed"