fix(bin): preserve process identity across host clock steps

bin/fm-herdr-lab-viewer.py

@@ -79,6 +79,21 @@ def _child(slave, master, session):
 
 
 def _process_start(pid):
+    # Must match bin/fm-herdr-lab.sh's fm_herdr_lab_process_start: /proc start
+    # ticks count from boot, so a host clock step cannot change them the way it
+    # re-renders ps lstart.
+    try:
+        with open("/proc/%d/stat" % pid, encoding="utf-8") as handle:
+            stat = handle.read()
+    except OSError:
+        return _process_lstart(pid)
+    fields = stat.rpartition(")")[2].split()
+    if len(fields) < 20 or not fields[19].isdigit():
+        raise RuntimeError("process start ticks unavailable")
+    return "proc-starttime=%s" % fields[19]
+
+
+def _process_lstart(pid):
     result = subprocess.run(
         ["ps", "-p", str(pid), "-o", "lstart="],
         check=True,

bin/fm-herdr-lab.sh

@@ -32,6 +32,8 @@
 # bin/fm-herdr-lab-viewer.py owns the pty mechanics.
 # Start succeeds only when that session reports a foreground client and the
 # recorded viewer process still matches its launch identity.
+# When /proc stat is readable, the viewer records start ticks for both processes;
+# existing ps lstart records remain readable for a running viewer.
 # Stop signals only identity-matched recorded processes and retains its
 # ownership record until detach is confirmed or the session is stopped or
 # absent; teardown refuses when that stop cannot be confirmed.
@@ -207,10 +209,41 @@ fm_herdr_lab_viewer_reason() { # <session>
   printf '%s' "$out" | jq -r '.result.reason // empty' 2>/dev/null
 }
 
+# Prints the process start identity the viewer launcher records. /proc stat
+# field 22 counts clock ticks since boot, so a host clock step cannot change it;
+# ps lstart re-renders those ticks against the wall-clock boot time (WSL2 steps
+# it about every 30 seconds) and would disown a running viewer.
 fm_herdr_lab_process_start() { # <pid>
+  local stat_line starttime
+  local -a stat_fields
+  if [ -r "/proc/$1/stat" ]; then
+    stat_line=$(cat "/proc/$1/stat" 2>/dev/null) || return 1
+    # After the final comm delimiter, array index 19 is proc stat field 22.
+    read -r -a stat_fields <<< "${stat_line##*)}"
+    [ "${#stat_fields[@]}" -ge 20 ] || return 1
+    starttime=${stat_fields[19]}
+    case "$starttime" in ''|*[!0-9]*) return 1 ;; esac
+    printf 'proc-starttime=%s' "$starttime"
+    return 0
+  fi
+  fm_herdr_lab_process_lstart "$1"
+}
+
+fm_herdr_lab_process_lstart() { # <pid>
   LC_ALL=C ps -p "$1" -o lstart= 2>/dev/null | sed 's/^[[:space:]]*//;s/[[:space:]]*$//'
 }
 
+fm_herdr_lab_process_start_matches() { # <pid> <recorded-start>
+  local current
+  case "$2" in
+    proc-starttime=*) current=$(fm_herdr_lab_process_start "$1") || return 1 ;;
+    # A record written before start-tick identity holds ps lstart text; keep
+    # honoring it so an upgrade does not strand a running viewer.
+    *) current=$(fm_herdr_lab_process_lstart "$1") || return 1 ;;
+  esac
+  [ -n "$current" ] && [ "$current" = "$2" ]
+}
+
 fm_herdr_lab_process_parent() { # <pid>
   LC_ALL=C ps -p "$1" -o ppid= 2>/dev/null | sed 's/^[[:space:]]*//;s/[[:space:]]*$//'
 }
@@ -225,18 +258,16 @@ fm_herdr_lab_viewer_recorded_value() { # <session> <key>
 }
 
 fm_herdr_lab_viewer_owned_pair() { # <session>
-  local launcher_pid viewer_pid launcher_start viewer_start current_start parent_pid
+  local launcher_pid viewer_pid launcher_start viewer_start parent_pid
   launcher_pid=$(fm_herdr_lab_viewer_recorded_value "$1" launcher_pid) || return 1
   viewer_pid=$(fm_herdr_lab_viewer_recorded_value "$1" viewer_pid) || return 1
   case "$launcher_pid:$viewer_pid" in
     *[!0-9:]*) return 1 ;;
   esac
   launcher_start=$(fm_herdr_lab_viewer_recorded_value "$1" launcher_start) || return 1
   viewer_start=$(fm_herdr_lab_viewer_recorded_value "$1" viewer_start) || return 1
-  current_start=$(fm_herdr_lab_process_start "$launcher_pid") || return 1
-  [ -n "$current_start" ] && [ "$current_start" = "$launcher_start" ] || return 1
-  current_start=$(fm_herdr_lab_process_start "$viewer_pid") || return 1
-  [ -n "$current_start" ] && [ "$current_start" = "$viewer_start" ] || return 1
+  fm_herdr_lab_process_start_matches "$launcher_pid" "$launcher_start" || return 1
+  fm_herdr_lab_process_start_matches "$viewer_pid" "$viewer_start" || return 1
   parent_pid=$(fm_herdr_lab_process_parent "$viewer_pid") || return 1
   [ "$parent_pid" = "$launcher_pid" ] || return 1
   printf '%s %s' "$launcher_pid" "$viewer_pid"

bin/fm-pending-reply-lib.sh

@@ -48,7 +48,9 @@
 #   request_turn_completed_epoch=
 #   recovery_attempted_epoch=
 #   recovery_sender_pid=
-#   recovery_sender_identity=
+#   recovery_sender_identity= Linux /proc start ticks plus full cmdline hex;
+#                             ps lstart plus command where /proc is unavailable.
+#                             Existing ps-form records remain readable.
 #   recovery_sent_epoch=
 #   recovery_delivery_outcome=
 #   recovery_turn_seen_busy=
@@ -1014,6 +1016,31 @@ fm_pending_reply_send_recovery() {  # <state-dir> <corr_id>
 }
 
 fm_pending_reply_pid_identity() {  # <pid>
+  local pid=$1 proc_root stat_line starttime cmdline_hex
+  local -a stat_fields
+  case "$pid" in ''|*[!0-9]*) return 1 ;; esac
+  proc_root=${FM_PROC_ROOT_OVERRIDE:-/proc}
+  # /proc stat field 22 counts clock ticks since boot, so a host clock step
+  # cannot change it; ps lstart re-renders those ticks against the wall-clock
+  # boot time (WSL2 steps it about every 30 seconds) and would read a live
+  # sender as dead. Start ticks distinguish reused PIDs; the full cmdline
+  # preserves the sender command identity.
+  if [ -r "$proc_root/$pid/stat" ] && [ -r "$proc_root/$pid/cmdline" ]; then
+    stat_line=$(cat "$proc_root/$pid/stat" 2>/dev/null) || return 1
+    # After the final comm delimiter, array index 19 is proc stat field 22.
+    read -r -a stat_fields <<< "${stat_line##*)}"
+    [ "${#stat_fields[@]}" -ge 20 ] || return 1
+    starttime=${stat_fields[19]}
+    case "$starttime" in ''|*[!0-9]*) return 1 ;; esac
+    cmdline_hex=$(od -An -v -tx1 "$proc_root/$pid/cmdline" 2>/dev/null | tr -d '[:space:]') || return 1
+    [ -n "$cmdline_hex" ] || return 1
+    printf 'proc-starttime=%s cmdline-hex=%s' "$starttime" "$cmdline_hex"
+    return 0
+  fi
+  fm_pending_reply_ps_identity "$pid"
+}
+
+fm_pending_reply_ps_identity() {  # <pid>
   local pid=$1 identity
   case "$pid" in ''|*[!0-9]*) return 1 ;; esac
   identity=$(COLUMNS=10000 LC_ALL=C ps -p "$pid" -o lstart= -o command= 2>/dev/null) || return 1
@@ -1026,7 +1053,12 @@ fm_pending_reply_sender_alive() {  # <record-path>
   pid=$(fm_pending_reply_get "$rec" recovery_sender_pid)
   expected=$(fm_pending_reply_get "$rec" recovery_sender_identity)
   [ -n "$expected" ] || return 1
-  actual=$(fm_pending_reply_pid_identity "$pid") || return 1
+  case "$expected" in
+    proc-starttime=*) actual=$(fm_pending_reply_pid_identity "$pid") || return 1 ;;
+    # A record written before start-tick identity holds the ps form; keep
+    # honoring it so an upgrade does not strand an in-flight recovery.
+    *) actual=$(fm_pending_reply_ps_identity "$pid") || return 1 ;;
+  esac
   [ "$actual" = "$expected" ]
 }
 

bin/fm-remote-job-lib.sh

@@ -82,6 +82,17 @@
 # it to stop itself once its root is pruned, and
 # bin/fm-remote-job-reap-orphans.sh uses it to reap workers that were already
 # orphaned that way.
+#
+# fm_remote_job_process_start is the one process-identity reader behind the
+# worker lock, staging, and claim start records. Where /proc/<pid>/stat is
+# readable (Linux) it records starttime=<clock ticks since boot, stat field
+# 22>, which no wall-clock step moves; ps lstart is rendered from the current
+# boot time there, so every NTP, VM or WSL2 time-sync, or resume step would
+# make a live worker stop matching its own records. Elsewhere (Darwin) it
+# records ps lstart, which a clock step does not move. A Linux record still in
+# lstart form was written before this contract: fm_remote_job_process_start_for_record
+# compares it as lstart, and a lock owner in that form is identified by pid and
+# exact command so ensure replaces it in place.
 
 FM_REMOTE_JOB_LABEL=dev.firstmate.remote-job
 FM_REMOTE_JOB_MAX_BYTES=${FM_REMOTE_JOB_MAX_BYTES:-1048576}
@@ -767,7 +778,7 @@ fm_remote_job_stage_owner_alive() { # <stage-dir>
   case "$pid" in ''|*[!0-9]*) return 1 ;; esac
   [ "$pid" -gt 1 ] || return 1
   recorded_start=$(fm_remote_job_read_single_line "$stage/.owner-start" 256 2>/dev/null) || return 1
-  actual_start=$(fm_remote_job_process_start "$pid" 2>/dev/null) || return 1
+  actual_start=$(fm_remote_job_process_start_for_record "$pid" "$recorded_start" 2>/dev/null) || return 1
   [ "$recorded_start" = "$actual_start" ]
 }
 
@@ -902,7 +913,24 @@ fm_remote_job_worker_ready_path() { printf '%s\n' "$FM_REMOTE_JOB_STATE/worker.r
 fm_remote_job_worker_identity_path() { printf '%s\n' "$FM_REMOTE_JOB_STATE/worker.identity"; }
 fm_remote_job_worker_lock_path() { printf '%s\n' "$FM_REMOTE_JOB_STATE/worker.lock"; }
 
-fm_remote_job_process_start() {
+fm_remote_job_process_start() { # <pid>
+  local pid=$1 proc_root stat_line
+  local -a stat_fields
+  case "$pid" in ''|*[!0-9]*) return 1 ;; esac
+  proc_root=${FM_PROC_ROOT_OVERRIDE:-/proc}
+  if [ -r "$proc_root/$pid/stat" ]; then
+    stat_line=$(cat "$proc_root/$pid/stat" 2>/dev/null) || return 1
+    # After the final comm delimiter, array index 19 is proc stat field 22.
+    read -r -a stat_fields <<< "${stat_line##*)}"
+    [ "${#stat_fields[@]}" -ge 20 ] || return 1
+    case "${stat_fields[19]}" in ''|*[!0-9]*) return 1 ;; esac
+    printf 'starttime=%s\n' "${stat_fields[19]}"
+    return 0
+  fi
+  fm_remote_job_process_lstart "$pid"
+}
+
+fm_remote_job_process_lstart() { # <pid>
   local pid=$1 ps_bin value
   if [ -x /bin/ps ]; then ps_bin=/bin/ps; elif [ -x /usr/bin/ps ]; then ps_bin=/usr/bin/ps; else return 1; fi
   value=$("$ps_bin" -p "$pid" -o lstart= 2>/dev/null) || return 1
@@ -911,6 +939,19 @@ fm_remote_job_process_start() {
   printf '%s\n' "$value"
 }
 
+# The current start of <pid> in the form <recorded> was written in, so a record
+# a pre-upgrade worker wrote as ps lstart text is still compared as lstart
+# rather than never matching the starttime= form.
+fm_remote_job_process_start_for_record() { # <pid> <recorded-start>
+  local current
+  current=$(fm_remote_job_process_start "$1") || return 1
+  case "$current:$2" in
+    starttime=*:starttime=*) ;;
+    starttime=*:*) current=$(fm_remote_job_process_lstart "$1") || return 1 ;;
+  esac
+  printf '%s\n' "$current"
+}
+
 fm_remote_job_process_command() {
   local pid=$1 ps_bin value
   if [ -x /bin/ps ]; then ps_bin=/bin/ps; elif [ -x /usr/bin/ps ]; then ps_bin=/usr/bin/ps; else return 1; fi
@@ -1012,7 +1053,15 @@ fm_remote_job_lock_owner_matches_process() {
   [ "$pid" -gt 1 ] || return 1
   recorded_start=$(fm_remote_job_read_single_line "$lock/start" 256) || return 1
   actual_start=$(fm_remote_job_process_start "$pid") || return 1
-  [ "$recorded_start" = "$actual_start" ] || return 1
+  # A Linux owner recorded as ps lstart text is a worker from before start ticks
+  # were recorded. Any clock step since has re-rendered its lstart, so its pid
+  # and exact command identify it, which lets ensure replace it in place
+  # instead of starting a second supervisor beside it.
+  case "$actual_start:$recorded_start" in
+    starttime=*:starttime=*) [ "$recorded_start" = "$actual_start" ] || return 1 ;;
+    starttime=*:*) ;;
+    *) [ "$recorded_start" = "$actual_start" ] || return 1 ;;
+  esac
   recorded_command=$(fm_remote_job_read_single_line "$lock/command" 8192) || return 1
   actual_command=$(fm_remote_job_process_command "$pid") || return 1
   [ "$recorded_command" = "$actual_command" ] || return 1

bin/fm-remote-job-worker.sh

@@ -333,7 +333,7 @@ worker_signal_process_or_group() { # process|group <signal> <pid>
 worker_supervisor_identity_status() { # <job-dir> <pid>
   local job=$1 pid=$2 recorded_start actual_start
   recorded_start=$(fm_remote_job_read_single_line "$job/.claim/supervisor_start" 256 2>/dev/null) || return 2
-  actual_start=$(fm_remote_job_process_start "$pid" 2>/dev/null) || {
+  actual_start=$(fm_remote_job_process_start_for_record "$pid" "$recorded_start" 2>/dev/null) || {
     worker_process_or_group_alive process "$pid" && return 2
     return 1
   }
@@ -350,7 +350,7 @@ worker_group_identity_status() { # <job-dir> <pid>
   local job=$1 pid=$2 recorded_start actual_start file="$1/.claim/group_start"
   [ -e "$file" ] || [ -L "$file" ] || return 3
   recorded_start=$(fm_remote_job_read_single_line "$file" 256 2>/dev/null) || return 2
-  actual_start=$(fm_remote_job_process_start "$pid" 2>/dev/null) || {
+  actual_start=$(fm_remote_job_process_start_for_record "$pid" "$recorded_start" 2>/dev/null) || {
     kill -0 "$pid" 2>/dev/null && return 2
     worker_process_or_group_alive group "$pid" && return 0
     return 1
@@ -446,7 +446,7 @@ worker_stop_recorded_execution() { # <job-dir>
 worker_lane_identity_matches() { # <pid> <start>
   local pid=$1 start=$2 actual_start
   [ -n "$start" ] || return 1
-  actual_start=$(fm_remote_job_process_start "$pid" 2>/dev/null) || return 1
+  actual_start=$(fm_remote_job_process_start_for_record "$pid" "$start" 2>/dev/null) || return 1
   [ "$actual_start" = "$start" ]
 }
 
@@ -571,7 +571,7 @@ worker_claim_owner_alive() { # <job-dir>
   case "$pid" in ''|*[!0-9]*) return 1 ;; esac
   if [ -e "$claim/owner_start" ] || [ -L "$claim/owner_start" ]; then
     recorded_start=$(fm_remote_job_read_single_line "$claim/owner_start" 256 2>/dev/null) || return 1
-    actual_start=$(fm_remote_job_process_start "$pid" 2>/dev/null) || return 1
+    actual_start=$(fm_remote_job_process_start_for_record "$pid" "$recorded_start" 2>/dev/null) || return 1
     [ "$recorded_start" = "$actual_start" ]
     return
   fi

docs/remote-secondmates.md

@@ -82,6 +82,8 @@ On macOS the worker is `dev.firstmate.remote-job`, an Aqua-scoped LaunchAgent at
 After that bootstrap, every non-doctor `fm-on.sh` target runs through that worker in the remote account's GUI session.
 It never runs in the SSH process or a Herdr pane.
 Linux uses the same queue and worker protocol without the Aqua-session requirement.
+On Linux, where `/proc/<pid>/stat` is readable, the worker uses kernel start ticks for process identity so host clock steps do not make a healthy worker appear stale.
+During an upgrade, a worker with an older `ps lstart` lock record is recognized by its PID and exact command and replaced when its code changes; [`bin/fm-remote-job-lib.sh`](../bin/fm-remote-job-lib.sh) owns that identity contract.
 When idle, the worker checks for newly staged work about once per second; after a lane starts or finishes it checks more frequently for a short period.
 
 ### Job lanes and preemption

tests/fm-herdr-lab.test.sh

@@ -451,6 +451,60 @@ test_viewer_stop_requires_the_recorded_parent() {
   pass "fm-herdr-lab: viewer ownership requires the recorded parent"
 }
 
+# Drives the real launcher against a sleeping viewer, then emulates a host
+# clock step with a ps whose lstart text changes afterwards, as procps output
+# does when the wall-clock boot time moves.
+test_viewer_ownership_survives_host_clock_step() {
+  local name="fm-lab-viewer-clock-$$" record viewer_bin="$TMP_ROOT/clock-viewer-bin"
+  local step="$TMP_ROOT/clock-stepped" launcher_pid pair viewer_pid launcher_lstart viewer_lstart
+  if [ ! -r "/proc/$$/stat" ]; then
+    echo "skip: fm-herdr-lab: no /proc start ticks on this host; ps lstart is its identity"
+    return 0
+  fi
+  mkdir -p "$viewer_bin" "$TRIPWIRES"
+  cat > "$viewer_bin/herdr" <<SH
+#!/usr/bin/env bash
+exec "$REAL_SLEEP" 30
+SH
+  cat > "$viewer_bin/ps" <<SH
+#!/usr/bin/env bash
+out=\$("$(command -v ps)" "\$@") || exit
+case " \$* " in
+  *" lstart= "*) [ ! -e "\$FM_FAKE_CLOCK_STEP" ] || out="stepped \$out" ;;
+esac
+printf '%s\n' "\$out"
+SH
+  chmod +x "$viewer_bin/herdr" "$viewer_bin/ps"
+  record=$(run_with_fake fm_herdr_lab_viewer_record_path "$name")
+  FM_FAKE_CLOCK_STEP="$step" PATH="$viewer_bin:$PATH" \
+    python3 "$ROOT/bin/fm-herdr-lab-viewer.py" "$name" "$record" >/dev/null 2>&1 &
+  launcher_pid=$!
+  while [ ! -f "$record" ]; do
+    kill -0 "$launcher_pid" 2>/dev/null || fail "the viewer launcher exited before recording its pair"
+    "$REAL_SLEEP" 0.01
+  done
+  viewer_pid=$(sed -n 's/^viewer_pid=//p' "$record")
+
+  : > "$step"
+  pair=$(FM_FAKE_CLOCK_STEP="$step" PATH="$viewer_bin:$PATH" run_with_fake fm_herdr_lab_viewer_owned_pair "$name") \
+    || fail "a host clock step disowned the running lab viewer"
+  assert_equals "$launcher_pid $viewer_pid" "$pair" "the stepped ownership check named the wrong pair"
+
+  rm -f "$step"
+  launcher_lstart=$(fm_herdr_lab_process_lstart "$launcher_pid")
+  viewer_lstart=$(fm_herdr_lab_process_lstart "$viewer_pid")
+  printf 'launcher_pid=%s\nlauncher_start=%s\nviewer_pid=%s\nviewer_start=%s\n' \
+    "$launcher_pid" "$launcher_lstart" "$viewer_pid" "$viewer_lstart" > "$record"
+  run_with_fake fm_herdr_lab_viewer_owned_alive "$name" \
+    || fail "a viewer recorded in the legacy lstart form was stranded by the upgrade"
+
+  kill -TERM "$launcher_pid" 2>/dev/null || true
+  wait "$launcher_pid" 2>/dev/null || true
+  kill -0 "$viewer_pid" 2>/dev/null && fail "the launcher left its viewer running"
+  rm -f "$record"
+  pass "fm-herdr-lab: viewer ownership survives a host clock step and keeps legacy records"
+}
+
 test_interrupted_viewer_start_cancels_launcher() {
   local name="fm-lab-viewer-interrupt-$$" command_pid launcher_pid status=0
   local started="$TMP_ROOT/viewer-interrupt-started" attached="$TMP_ROOT/viewer-interrupt-attached"
@@ -548,6 +602,7 @@ test_viewer_timeout_allows_launcher_escalation
 test_viewer_start_requires_its_owned_process
 test_viewer_stop_only_signals_owned_processes
 test_viewer_stop_requires_the_recorded_parent
+test_viewer_ownership_survives_host_clock_step
 test_interrupted_viewer_start_cancels_launcher
 test_teardown_refuses_while_viewer_attached
 test_viewer_stop_retains_record_when_detach_is_unreadable