Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
130 changes: 115 additions & 15 deletions bin/fm-backlog-handoff.sh
Original file line number Diff line number Diff line change
Expand Up @@ -61,8 +61,10 @@
# original batch is retried, so it cannot discard wake intent for work that
# already moved. No two-phase journal exists.
# Every newly durable backlog delivery attempts one marked wake to the receiving
# endpoint. A local route moves directly into the destination backlog, and a
# missing or rejected local wake makes that command fail with the move intact so
# endpoint, naming the routed item keys; a remote batch that reuses a
# still-pending wake adds its keys to that wake's list. A local route moves
# directly into the destination backlog, and a missing or rejected local wake
# makes that command fail with the move intact so
# rerunning the same handoff retries its prepared wake intent. After a durable
# remote receipt, the outbox is released and the handoff succeeds regardless of
# the best-effort wake outcome; an undelivered remote wake remains separately
Expand Down Expand Up @@ -97,6 +99,80 @@ MAIN_BACKLOG="$DATA/backlog.md"

RECEIVER_WAKE_MESSAGE='New routed work is in your backlog. Run bin/fm-session-start.sh now, then act on the routed task.'

# Names the routed item keys so a receiver can tell a second genuine handoff
# apart from a duplicate of the first; an unlabeled fixed line once made a
# receiver block on a real second handoff as an unproven repeat (regression:
# test_two_consecutive_handoffs_name_their_own_items in
# tests/fm-backlog-handoff.test.sh). Appended, never inserted, so the fixed
# sentence itself stays a stable substring for every existing caller and test
# that greps for it.
receiver_wake_message() { # <item-key>...
local ids
[ "$#" -gt 0 ] || { printf '%s' "$RECEIVER_WAKE_MESSAGE"; return; }
ids=$(printf '%s, ' "$@")
ids=${ids%, }
printf '%s Routed: %s.' "$RECEIVER_WAKE_MESSAGE" "$ids"
}

# Extracts every item key from a "- [ ] <key> - <title>" backlog/outbox line,
# in file order, one per line - the same line shape outbox_item_count counts.
receiver_wake_item_keys_from_file() { # <path>
grep -E '^- \[[ x]\] ' "$1" 2>/dev/null | sed -E 's/^- \[[ x]\] ([^ ]+) .*/\1/'
}

receiver_wake_message_path() { # <secondmate-id>
printf '%s\n' "$STATE/.backlog-handoff-$1.wake-message"
}

receiver_wake_message_write() { # <secondmate-id> <message>
local id=$1 message=$2 path tmp
path=$(receiver_wake_message_path "$id")
tmp=$(umask 077; mktemp "$STATE/.backlog-handoff-wake-message.XXXXXX") || return 1
if ! printf '%s' "$message" > "$tmp" || ! chmod 600 "$tmp" || ! mv -f -- "$tmp" "$path"; then
rm -f -- "$tmp"
return 1
fi
}

# Falls back to the fixed generic line when no per-batch message was ever
# recorded (a legacy marker predating this, or the bare-`pending` transition
# in wake_pending_secondmate_receiver, which has no fresh key list to work
# from) so a receiver is never left without an actionable instruction.
receiver_wake_message_read() { # <secondmate-id>
local path
path=$(receiver_wake_message_path "$1")
if [ -f "$path" ] && [ ! -L "$path" ]; then
cat "$path"
else
printf '%s' "$RECEIVER_WAKE_MESSAGE"
fi
}

receiver_wake_message_clear() { # <secondmate-id>
rm -f -- "$(receiver_wake_message_path "$1")"
}

# A still-pending wake that a later batch reuses must name every item routed
# since it was recorded, so the new keys are merged into the stored list. A
# stored generic line names no items and is left as it is.
receiver_wake_message_add_keys() { # <secondmate-id> <item-key>...
local id=$1 prefix="$RECEIVER_WAKE_MESSAGE Routed: " message key
local -a keys=()
shift
message=$(receiver_wake_message_read "$id")
case "$message" in "$prefix"*.) ;; *) return 0 ;; esac
message=${message#"$prefix"}
message=${message%.}
while [ -n "$message" ]; do
keys+=("${message%%, *}")
case "$message" in *", "*) message=${message#*, } ;; *) message= ;; esac
done
for key in "$@"; do
case " ${keys[*]} " in *" $key "*) ;; *) keys+=("$key") ;; esac
done
receiver_wake_message_write "$id" "$(receiver_wake_message "${keys[@]}")"
}

ACTIVE_HANDOFF_LOCK=
ACTIVE_REGISTRY_LOCK=
RECEIVER_WAKE_IGNORE_ID=
Expand Down Expand Up @@ -384,8 +460,9 @@ receiver_wake_state_write() { # <secondmate-id> <state>
fi
}

receiver_wake_mark() { # <secondmate-id> <prepared|pending> [batch-id]
local id=$1 wake_phase=$2 batch=${3:-} marker="$STATE/.backlog-handoff-$1.wake-pending" value corr rec
receiver_wake_mark() { # <secondmate-id> <prepared|pending> [batch-id] [message]
local id=$1 wake_phase=$2 batch=${3:-} message=${4:-$RECEIVER_WAKE_MESSAGE}
local marker="$STATE/.backlog-handoff-$1.wake-pending" value corr rec
local wake_state
case "$wake_phase" in prepared | pending) ;; *) return 1 ;; esac
if [ -e "$marker" ] || [ -L "$marker" ]; then
Expand All @@ -408,24 +485,29 @@ receiver_wake_mark() { # <secondmate-id> <prepared|pending> [batch-id]
*) return 1 ;;
esac
fi
corr=$(fm_pending_reply_create "$FM_HOME" "$STATE" "$id" "$RECEIVER_WAKE_MESSAGE") || return 1
corr=$(fm_pending_reply_create "$FM_HOME" "$STATE" "$id" "$message") || return 1
if ! receiver_wake_message_write "$id" "$message"; then
fm_pending_reply_discard_undelivered "$STATE" "$corr" || true
return 1
fi
wake_state="$wake_phase:$corr"
if [ "$wake_phase" = prepared ]; then
printf '%s' "$batch" | grep -Eq '^[a-f0-9]{16}$' || return 1
wake_state="$wake_state:$batch"
fi
if ! receiver_wake_state_write "$id" "$wake_state"; then
fm_pending_reply_discard_undelivered "$STATE" "$corr" || true
receiver_wake_message_clear "$id"
return 1
fi
}

receiver_wake_mark_pending() { # <secondmate-id>
receiver_wake_mark "$1" pending
receiver_wake_mark_pending() { # <secondmate-id> [message]
receiver_wake_mark "$1" pending "" "${2:-$RECEIVER_WAKE_MESSAGE}"
}

receiver_wake_mark_prepared() { # <secondmate-id> <batch-id>
receiver_wake_mark "$1" prepared "$2"
receiver_wake_mark_prepared() { # <secondmate-id> <batch-id> [message]
receiver_wake_mark "$1" prepared "$2" "${3:-$RECEIVER_WAKE_MESSAGE}"
}

receiver_wake_discard_prepared() { # <secondmate-id>
Expand All @@ -440,6 +522,7 @@ receiver_wake_discard_prepared() { # <secondmate-id>
*) return 1 ;;
esac
fm_pending_reply_discard_undelivered "$STATE" "$corr" || return 1
receiver_wake_message_clear "$id"
rm -f -- "$marker"
}

Expand Down Expand Up @@ -470,6 +553,7 @@ receiver_wake_discard_pending() { # <secondmate-id>
pending) ;;
*) return 1 ;;
esac
receiver_wake_message_clear "$id"
rm -f -- "$marker"
}

Expand Down Expand Up @@ -535,6 +619,7 @@ receiver_wake_clear_confirmed() { # <secondmate-id>
return 0
fi
if receiver_wake_pending_delivered_valid "$id" || receiver_wake_confirmed_valid "$id"; then
receiver_wake_message_clear "$id"
if ! rm -f -- "$marker"; then
RECEIVER_WAKE_IGNORE_ID=$id
printf 'warning: confirmed receiver wake left a stale marker at %s; later handoffs will ignore it\n' "$marker" >&2
Expand All @@ -548,7 +633,7 @@ receiver_wake_clear_confirmed() { # <secondmate-id>
}

wake_secondmate_receiver() { # <secondmate-id> <correlation-id>
local id=$1 corr=$2 meta="$STATE/$1.meta" out rc=0
local id=$1 corr=$2 meta="$STATE/$1.meta" out rc=0 message
if [ ! -f "$meta" ] || [ -L "$meta" ]; then
printf 'error: handed off work to secondmate %s, but no live receiver endpoint is recorded; the destination backlog is durable and the receiver was not woken\n' "$id" >&2
return 1
Expand All @@ -557,9 +642,10 @@ wake_secondmate_receiver() { # <secondmate-id> <correlation-id>
printf 'error: secondmate %s has non-secondmate endpoint metadata; backlog is durable but the receiver was not woken\n' "$id" >&2
return 1
}
message=$(receiver_wake_message_read "$id")
out=$(FM_HOME="$FM_HOME" FM_STATE_OVERRIDE="$STATE" FM_ROOT_OVERRIDE="$FM_ROOT" \
FM_PENDING_REPLY_EXISTING_CORR="$corr" \
"$SCRIPT_DIR/fm-send.sh" "$id" "$RECEIVER_WAKE_MESSAGE" 2>&1) || rc=$?
"$SCRIPT_DIR/fm-send.sh" "$id" "$message" 2>&1) || rc=$?
if [ "$rc" -ne 0 ]; then
[ -z "$out" ] || printf '%s\n' "$out" >&2
printf 'error: backlog delivery to secondmate %s succeeded, but its receiver wake failed; retry a tracked remote wake with --resume-pending or a later new handoff, and retry a local wake by rerunning its handoff\n' "$id" >&2
Expand Down Expand Up @@ -614,6 +700,7 @@ wake_pending_secondmate_receiver() { # <secondmate-id> [retain-confirmed]
printf 'error: receiver wake for secondmate %s was confirmed, but pending state could not be cleared\n' "$id" >&2
return 1
}
receiver_wake_message_clear "$id"
fi
}

Expand All @@ -622,7 +709,8 @@ outbox_item_count() { # <path>
}

remote_deliver_outbox() { # <secondmate-id> <outbox-path>
local id=$1 outbox=$2 remote_rel receive_out snapshot bytes hash generation counter counter_tmp current marker wake_rc=0 wake_state=pending
local id=$1 outbox=$2 remote_rel receive_out snapshot bytes hash generation counter counter_tmp current marker wake_rc=0 wake_state=pending key
local -a wake_keys=()
[ -f "$outbox" ] && [ ! -L "$outbox" ] || {
echo "error: pending outbox is unavailable or unsafe: $outbox" >&2
return 1
Expand Down Expand Up @@ -700,11 +788,22 @@ remote_deliver_outbox() { # <secondmate-id> <outbox-path>
return 1
fi
marker="$STATE/.backlog-handoff-$id.wake-pending"
# The outbox's own item lines are the batch's ground truth here, valid
# for the fresh-stage call and every later resume alike since resuming
# only ever re-reads this same durable file.
while IFS= read -r key; do
[ -n "$key" ] && wake_keys+=("$key")
done < <(receiver_wake_item_keys_from_file "$outbox")
if [ "$RECEIVER_WAKE_IGNORE_ID" = "$id" ]; then
wake_state=dropped
wake_rc=1
elif ! receiver_wake_pending_valid "$id" && ! receiver_wake_confirmed_valid "$id"; then
receiver_wake_mark_pending "$id" || {
elif receiver_wake_pending_valid "$id"; then
receiver_wake_message_add_keys "$id" ${wake_keys[@]+"${wake_keys[@]}"} || {
wake_state=dropped
wake_rc=1
}
elif ! receiver_wake_confirmed_valid "$id"; then
receiver_wake_mark_pending "$id" "$(receiver_wake_message ${wake_keys[@]+"${wake_keys[@]}"})" || {
wake_state=dropped
wake_rc=1
}
Expand All @@ -717,6 +816,7 @@ remote_deliver_outbox() { # <secondmate-id> <outbox-path>
return 1
}
if [ "$wake_rc" -eq 0 ]; then
receiver_wake_message_clear "$id"
if ! rm -f -- "$marker"; then
RECEIVER_WAKE_IGNORE_ID=$id
echo "warning: remote outbox and receiver wake completed, but a stale confirmed wake marker remains at $marker; later handoffs will ignore it" >&2
Expand Down Expand Up @@ -1048,7 +1148,7 @@ if [ -e "$WAKE_PENDING_MARKER" ] || [ -L "$WAKE_PENDING_MARKER" ]; then
;;
esac
fi
receiver_wake_mark_prepared "$ID" "$REQUESTED_BATCH" || {
receiver_wake_mark_prepared "$ID" "$REQUESTED_BATCH" "$(receiver_wake_message "${TO_MOVE[@]}")" || {
echo "error: receiver wake state for secondmate $ID could not be recorded; nothing was moved" >&2
exit 1
}
Expand Down
1 change: 1 addition & 0 deletions bin/fm-config-inherit-lib.sh
Original file line number Diff line number Diff line change
Expand Up @@ -221,6 +221,7 @@ warn_inheritable_config_error() {
shared_captain_header_valid() {
local src=$1 head
head=$(sed -n '1,12p' "$src" 2>/dev/null) || return 1
head=$(printf '%s' "$head" | tr '\n' ' ' | tr -s '[:space:]' ' ')
case "$head" in *main-authoritative*) ;; *) return 1 ;; esac
case "$head" in *"read-only in secondmate homes"*) ;; *) return 1 ;; esac
case "$head" in *"must not be edited there"*) ;; *) return 1 ;; esac
Expand Down
9 changes: 0 additions & 9 deletions bin/fm-remote-inherit-push.sh
Original file line number Diff line number Diff line change
Expand Up @@ -29,15 +29,6 @@ sha256_file() {
file_link_count() {
if [ "$(uname)" = Darwin ]; then /usr/bin/stat -f %l "$1" 2>/dev/null; else stat -c %h "$1" 2>/dev/null; fi
}
shared_captain_header_valid() {
local head
head=$(sed -n '1,12p' "$1" 2>/dev/null) || return 1
case "$head" in *main-authoritative*) ;; *) return 1 ;; esac
case "$head" in *"read-only in secondmate homes"*) ;; *) return 1 ;; esac
case "$head" in *"must not be edited there"*) ;; *) return 1 ;; esac
case "$head" in *"main firstmate"*) ;; *) return 1 ;; esac
case "$head" in *"marked status"*|*"document pointer"*) ;; *) return 1 ;; esac
}
[ "$#" -eq 2 ] || { echo "usage: fm-remote-inherit-push.sh <secondmate-id> <generation>" >&2; exit 2; }
ID=$1
GENERATION=$2
Expand Down
73 changes: 73 additions & 0 deletions tests/fm-backlog-handoff.test.sh
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,19 @@ inbox_body_stream() { # <state-dir> <task-id>
done
}

# One line per durable inbox record's own body (fm_task_inbox_body prints no
# trailing newline of its own, so inbox_body_stream's concatenation is not
# usable when a caller needs to tell one record's body apart from another's).
inbox_bodies_by_record() { # <state-dir> <task-id>
local rec
for rec in "$1/$2.inbox"/*.msg; do
[ -f "$rec" ] || continue
bash -c '. "$1"; fm_task_inbox_body "$2"' _ \
"$ROOT/bin/fm-task-inbox-lib.sh" "$rec"
printf '\n'
done
}

inbox_record_count() { # <state-dir> <task-id>
find "$1/$2.inbox" -maxdepth 1 -type f -name '*.msg' 2>/dev/null | wc -l | tr -d '[:space:]'
}
Expand Down Expand Up @@ -1344,7 +1357,67 @@ EOF
pass "registry entry without (home: ...) fails cleanly with has no home"
}

# Regression for a real 2026-08-25 incident (backlog item
# handoff-nachricht-nennt-auftrag-nicht): the receiver's wake instruction used
# to be one fixed line with no indication which task it named, so two
# handoffs delivered close together were textually identical - the receiver
# read the second as an unproven repeat of the first and blocked on it.
test_two_consecutive_handoffs_name_their_own_items() {
local home="$TMP_ROOT/two-handoffs-main" sub="$TMP_ROOT/two-handoffs-sub" fakebin
local out1 out2 bodies first_line second_line
setup_homes "$home" "$sub"
mkdir -p "$sub/state" "$sub/data"
cat > "$home/data/backlog.md" <<'EOF'
## Queued
- [ ] postfach-uebergang-abschliessen - first routed item (repo: alpha)
- [ ] vault-regelfragen-umsetzung - second routed item (repo: alpha)

## Done
EOF
printf '## Queued\n\n## Done\n' > "$sub/data/backlog.md"
fakebin=$(make_fake_tmux "$TMP_ROOT/two-handoffs-fake")
out1="$TMP_ROOT/two-handoffs-1.out"
out2="$TMP_ROOT/two-handoffs-2.out"
FM_HOME="$home" FM_ROOT_OVERRIDE="$ROOT" PATH="$fakebin:$PATH" \
FM_FAKE_TMUX_WINDOW='firstmate:fm-design' \
FM_FAKE_TMUX_LOG="$TMP_ROOT/two-handoffs-tmux.log" \
FM_FAKE_TMUX_CAPTURE="$TMP_ROOT/two-handoffs-fake/pane.txt" \
FM_SEND_SETTLE=0 FM_SEND_SLEEP=0 FM_SEND_RETRIES=1 \
"$ROOT/bin/fm-backlog-handoff.sh" design postfach-uebergang-abschliessen \
> "$out1" 2>&1 \
|| fail "first handoff failed: $(cat "$out1")"
FM_HOME="$home" FM_ROOT_OVERRIDE="$ROOT" PATH="$fakebin:$PATH" \
FM_FAKE_TMUX_WINDOW='firstmate:fm-design' \
FM_FAKE_TMUX_LOG="$TMP_ROOT/two-handoffs-tmux.log" \
FM_FAKE_TMUX_CAPTURE="$TMP_ROOT/two-handoffs-fake/pane.txt" \
FM_SEND_SETTLE=0 FM_SEND_SLEEP=0 FM_SEND_RETRIES=1 \
"$ROOT/bin/fm-backlog-handoff.sh" design vault-regelfragen-umsetzung \
> "$out2" 2>&1 \
|| fail "second handoff failed: $(cat "$out2")"

bodies=$(inbox_body_stream "$home/state" design)
assert_contains "$bodies" 'New routed work is in your backlog.' \
"receiver inbox lost the fixed routed-work instruction"
assert_contains "$bodies" 'postfach-uebergang-abschliessen' \
"first handoff's message did not name its own item"
assert_contains "$bodies" 'vault-regelfragen-umsetzung' \
"second handoff's message did not name its own item"

bodies=$(inbox_bodies_by_record "$home/state" design)
first_line=$(printf '%s\n' "$bodies" | grep -F 'postfach-uebergang-abschliessen')
second_line=$(printf '%s\n' "$bodies" | grep -F 'vault-regelfragen-umsetzung')
[ "$first_line" != "$second_line" ] \
|| fail "two handoffs for different items produced the same message text: $first_line"
printf '%s\n' "$first_line" | grep -qF 'vault-regelfragen-umsetzung' \
&& fail "the first handoff's message also named the second item: $first_line"
printf '%s\n' "$second_line" | grep -qF 'postfach-uebergang-abschliessen' \
&& fail "the second handoff's message also named the first item: $second_line"

pass "two consecutive handoffs each name their own item, so they are never textually identical"
}

test_handoff_wakes_live_local_receiver
test_two_consecutive_handoffs_name_their_own_items
test_failed_wake_retries_when_the_item_is_already_present
test_known_receiver_failure_remains_retryable_after_grace
test_known_failure_restores_retry_after_reconciliation_race
Expand Down
7 changes: 7 additions & 0 deletions tests/fm-remote-backlog-handoff.test.sh
Original file line number Diff line number Diff line change
Expand Up @@ -102,6 +102,8 @@ command_name=$(perl -MMIME::Base64=decode_base64 -e '$d=decode_base64($ARGV[0]);
case "${FM_FAKE_SSH_MODE:-normal}:$command_name" in
*:fm-remote-secondmate-control.sh)
printf '%s\n' "$command_name" >> "$FM_FAKE_REMOTE_WAKE_LOG"
perl -MMIME::Base64=decode_base64 -e '$d=decode_base64($ARGV[0]); $d=~tr/\0\n/ /; print "$d\n"' "$argv_b64" \
>> "$FM_FAKE_REMOTE_WAKE_LOG.argv"
[ "${FM_FAKE_REMOTE_WAKE_RC:-0}" -eq 0 ] || printf 'remote receiver wake failed\n' >&2
exit "${FM_FAKE_REMOTE_WAKE_RC:-0}"
;;
Expand Down Expand Up @@ -446,6 +448,11 @@ assert_absent "$PARENT/data/handoff/ios.outbox.md" "permanently lost wake retain
|| fail "later handoff did not retain the same pending wake correlation"
[ "$(grep -cF fm-remote-secondmate-control.sh "$WAKE_LOG")" -gt "$wakes_after_resume" ] \
|| fail "later handoff did not retry the separately pending wake"
reused_wake=$(tail -n 1 "$WAKE_LOG.argv")
assert_contains "$reused_wake" 'wake-permanent-a' \
"reused pending wake dropped the earlier batch's item"
assert_contains "$reused_wake" 'wake-permanent-b' \
"reused pending wake did not name the later batch's item"
pass "a permanently unconfirmable wake never jams later durable handoffs"

RM_FAKEBIN="$TMP_ROOT/rm-fakebin"
Expand Down
Loading
Loading