From 680d1bfae22916b00627d8c45341aa4af7888706 Mon Sep 17 00:00:00 2001 From: Rajeev Jain Date: Thu, 29 Oct 2026 10:49:01 -0500 Subject: [PATCH] fix: clear stale PMI metadata before YAC remaps --- docs/ucar.md | 44 +++-- scripts/chrysalis_endpoint.sh | 8 + scripts/ucar_endpoint.sh | 107 ++++++----- src/uxarray_mcp/remote/agent.py | 38 ++-- src/uxarray_mcp/remote/compute_functions.py | 28 +++ tests/test_yac_pmi_singleton.py | 190 ++++++++++++++++++++ 6 files changed, 326 insertions(+), 89 deletions(-) create mode 100644 tests/test_yac_pmi_singleton.py diff --git a/docs/ucar.md b/docs/ucar.md index 3c9439c..2c9694a 100644 --- a/docs/ucar.md +++ b/docs/ucar.md @@ -190,22 +190,34 @@ synthesized from the prefix, so there is nothing to edit in the script. **`doctor` passes but YAC tools fail** — run `yac_smoke_test.py`, which checks importability on the compute worker specifically rather than on the login node. -**A conservative remap kills the worker (`WorkerLost`)** — almost certainly -MPI, not memory, and the two look identical from the outside. Importing YAC -calls `MPI_Init`: Open MPI starts fine as a singleton, MPICH aborts the -process, and an abort inside a Globus worker arrives as a lost worker with -nothing mentioning MPI. - -Tell them apart with `check_remote_yac(endpoint=...)`. It runs the same -conservative remap in a *subprocess*, where an abort is survivable: - -- `conservative_ok: true` — YAC is fine. The worker needs to start inside an - MPI context (`GlobusMPIEngine`, or a launcher prefix on the engine). An - export in `worker_init` cannot do this; that runs before the worker exists. -- `conservative_ok: false` — the build is the problem, not the launcher. - -`./scripts/ucar_endpoint.sh check-mpi` runs the bare and launched imports side -by side and tells you which case you are in. +**A conservative remap kills the worker (`WorkerLost`)** — MPI, not memory, +and the two look identical from the outside. Importing YAC calls `MPI_Init`, +which works out whether this process is one rank of many by reading the +environment. A worker started inside a batch job inherits `PMI_FD` from that +job, pointing at a descriptor whose owner has already exited. MPI writes to +it, gets `Bad file descriptor`, and aborts — which reaches you as a lost +worker with nothing mentioning MPI. + +The server unsets `PMI_FD`, `PMI_RANK` and `PMI_SIZE` before every YAC-backed +remap, so a current endpoint should not hit this. If you still see it, the +endpoint is probably running an older `uxarray-mcp`; check the version in +`check_remote_yac(endpoint=...)` and redeploy. To fix it at the endpoint +instead, add to `worker_init`: + +```bash +unset PMI_FD PMI_RANK PMI_SIZE +``` + +`conservative_ok: false` from `check_remote_yac` means something else — the +YAC build itself. `./scripts/ucar_endpoint.sh check-mpi` distinguishes the two. + +**Do not fix this with an MPI launcher.** `mpiexec -n 1` also makes the import +work, and earlier versions of this page recommended it. At more than one rank +it silently corrupts results: YAC splits the work across ranks, nothing here +gathers it back, and each rank returns its own partition as if it were the +whole field. Measured against a known-correct serial answer of 73344.0, four +ranks gave 56635, 58331, 53862 and 60373 on four consecutive runs — full-length +arrays, no NaNs, no error. Meanwhile `inverse_distance_weighted` and `bilinear` work — they use uxarray's own engine. Neither conserves the field integral, which is usually the reason diff --git a/scripts/chrysalis_endpoint.sh b/scripts/chrysalis_endpoint.sh index 920a678..0a1ebf6 100755 --- a/scripts/chrysalis_endpoint.sh +++ b/scripts/chrysalis_endpoint.sh @@ -170,6 +170,10 @@ engine: export PATH="$YAC_WORKER_PATH" export PYTHONPATH="$YAC_PYTHONPATH" export LD_LIBRARY_PATH="$YAC_LD_LIBRARY_PATH:\${LD_LIBRARY_PATH:-}" + # Stale PMI vars from the batch job make MPI_Init abort when YAC is + # imported (the fd PMI_FD names is already closed). The server also + # clears these per-remap; doing it here covers older clients too. + unset PMI_FD PMI_RANK PMI_SIZE idle_heartbeats_soft: 10 idle_heartbeats_hard: 5760 @@ -199,6 +203,10 @@ engine: export PATH="$YAC_WORKER_PATH" export PYTHONPATH="$YAC_PYTHONPATH" export LD_LIBRARY_PATH="$YAC_LD_LIBRARY_PATH:\${LD_LIBRARY_PATH:-}" + # Stale PMI vars from the batch job make MPI_Init abort when YAC is + # imported (the fd PMI_FD names is already closed). The server also + # clears these per-remap; doing it here covers older clients too. + unset PMI_FD PMI_RANK PMI_SIZE launcher: type: SrunLauncher diff --git a/scripts/ucar_endpoint.sh b/scripts/ucar_endpoint.sh index 3a1e208..62b2606 100755 --- a/scripts/ucar_endpoint.sh +++ b/scripts/ucar_endpoint.sh @@ -23,18 +23,17 @@ WITH_YAC="${WITH_YAC:-0}" # HEALPix remap ran. worker_init sources activate-yac.sh from this prefix, so # `restart` is what picks the change up. YAC_VERSION="${YAC_VERSION:-3.20.2}" -# Whether this site needs YAC imported under an MPI launcher. +# Importing YAC calls MPI_Init, which reads the environment to decide whether +# it is one rank of many. A worker inside a batch allocation inherits PMI_FD +# from the job; the descriptor it names is already closed, so MPI writes to +# it, gets EBADF and aborts the worker. `check-mpi` detects this; worker_init +# below unsets the variables so it cannot happen. # -# `check-mpi` answers that question; acting on the answer is still manual. -# Open MPI initialises as a singleton and needs nothing. MPICH aborts, and -# the endpoint then needs its *workers* started inside an MPI context -- -# which means `GlobusMPIEngine`, or a `launcher_prefix` on the engine, not an -# export in worker_init. Casper was observed failing this way and has not yet -# been reconfigured, so this is deliberately not automated: a wrong engine -# block takes the endpoint down for every tool, not just the remap. -# -# Tracked rather than guessed at. See the WorkerLost message in -# remote/agent.py for what a caller is told in the meantime. +# Not fixed with GlobusMPIEngine or a launcher prefix, which earlier versions +# of this script recommended. `mpiexec -n 1` works, but at more than one rank +# YAC splits the work and nothing gathers it back, so each rank returns its +# own partition as the whole field -- silently, with no NaNs. Unsetting one +# variable has no such failure mode. YAC_PREFIX="$HOME/opt/yac-$YAC_VERSION" YAC_ACTIVATE="$YAC_PREFIX/activate-yac.sh" TMUX_SESSION="uxarray-endpoint" @@ -51,7 +50,7 @@ Usage (run on a Casper login node): ucar_endpoint.sh start Load env + start endpoint in tmux ucar_endpoint.sh restart Stop running endpoint, then start fresh ucar_endpoint.sh status Show endpoint list - ucar_endpoint.sh check-mpi Does importing YAC need an MPI launcher? + ucar_endpoint.sh check-mpi Can YAC import without a stale PMI_FD? Environment overrides: ENDPOINT_NAME Globus Compute endpoint profile name (default: ucar-uxarray-yac) @@ -122,60 +121,58 @@ EOF echo " wrote $YAC_ACTIVATE (YAC $YAC_VERSION, python $pyver)" } -# Does importing YAC survive outside an MPI launcher? +# Can YAC import here, and is a stale PMI_FD the reason it cannot? # -# Importing YAC calls MPI_Init. Open MPI initialises as a singleton and is -# fine; MPICH aborts the process instead. Inside a Globus Compute worker that -# abort takes the worker down, and the caller sees `WorkerLost` with nothing -# mentioning MPI -- which reads as an out-of-memory kill and sends people to -# measure their mesh. Observed on Casper: `method="conservative"` killed the -# worker while the identical remap finished in 0.09s under -# `check_remote_yac`, which runs it in a subprocess. +# Importing YAC calls MPI_Init, which reads the environment to decide whether +# it is one rank of many. A worker inside a batch allocation inherits PMI_FD +# from the job, but that descriptor's owner has exited, so MPI writes to a +# closed fd, gets EBADF and aborts. Inside a Globus worker the abort takes +# the worker down and the caller sees `WorkerLost` with nothing mentioning +# MPI -- which reads as an out-of-memory kill and sends people to measure +# their mesh. # -# This answers the question directly, before anyone submits a remap. If the -# bare import dies and the launched one lives, the endpoint's worker_init -# needs a launcher. +# Confirmed on Chrysalis by dropping one variable at a time: PMI_FD alone is +# the trigger. PMI_RANK, PMI_SIZE, PMI_JOBID, SLURM_* and HYDRA_* are all +# harmless on their own. _check_mpi() { _load_modules _activate_env local probe='import yac.core, sys; print("import ok:", sys.executable)' - echo "==> Bare import (what a Globus worker does today)" - if python -c "$probe" 2>&1 | sed 's/^/ /'; then - local bare_ok=1 - else - local bare_ok=0 - echo " FAILED -- this is what takes the worker down." - fi + echo "==> Inherited PMI variables in this shell" + local pmi_seen + pmi_seen="$(env | grep -E '^PMI' | cut -d= -f1 | tr '\n' ' ')" + echo " ${pmi_seen:-(none)}" echo - echo "==> Under a launcher" - local launcher="" - for cand in "mpiexec -n 1" "srun --ntasks 1"; do - if command -v "${cand%% *}" >/dev/null 2>&1; then launcher="$cand"; break; fi - done - if [[ -z "$launcher" ]]; then - echo " No mpiexec or srun on PATH; cannot compare." - return 1 - fi - echo " using: $launcher" - local launched_ok=1 - $launcher python -c "$probe" 2>&1 | sed 's/^/ /' || launched_ok=0 + echo "==> Bare import (what a worker does without the fix)" + local bare_ok=1 + python -c "$probe" 2>&1 | sed 's/^/ /' || bare_ok=0 + [[ "$bare_ok" == "0" ]] && echo " FAILED -- this is what takes the worker down." + + echo + echo "==> With PMI_FD/PMI_RANK/PMI_SIZE unset (what the server now does)" + local unset_ok=1 + env -u PMI_FD -u PMI_RANK -u PMI_SIZE python -c "$probe" 2>&1 \ + | sed 's/^/ /' || unset_ok=0 echo if [[ "$bare_ok" == "1" ]]; then echo "VERDICT: bare import works. MPI is not what is killing the worker;" echo " if conservative remaps still fail, look at memory next." - elif [[ "$launched_ok" == "1" ]]; then - echo "VERDICT: bare import dies, launched import works -- this is the MPI" - echo " singleton problem." - echo " The workers must start inside an MPI context. In the endpoint's" - echo " user_config_template.yaml.j2, either switch the engine to" - echo " GlobusMPIEngine, or give the current engine a launcher prefix so" - echo " each worker runs under: $launcher" + elif [[ "$unset_ok" == "1" ]]; then + echo "VERDICT: a stale PMI_FD is the problem, and unsetting it fixes it." + echo " uxarray-mcp >= 2026.10.1 does this itself before every YAC remap," + echo " so first check the endpoint is not running an older server." + echo " To fix it at the endpoint, add to worker_init:" + echo " unset PMI_FD PMI_RANK PMI_SIZE" echo " Then: $(basename "$0") restart" + echo + echo " Do NOT switch the engine to GlobusMPIEngine or add a launcher" + echo " prefix. That also makes the import work, but above one rank each" + echo " rank returns only its own partition of the field, with no error." else - echo "VERDICT: YAC does not import either way. This is not MPI --" + echo "VERDICT: YAC does not import either way. This is not PMI --" echo " check the build at $YAC_PREFIX." fi } @@ -205,12 +202,6 @@ _configure() { if [[ "$WITH_YAC" == "1" ]]; then _ensure_yac_activate yac_line=$'\n source '"$YAC_ACTIVATE" - # NOTE: if `check-mpi` says this site needs a launcher, the change goes - # in the engine block below rather than here -- worker_init runs *before* - # the worker process starts, so exporting something here cannot put that - # process inside an MPI context. Left unwired on purpose rather than - # shipped as a setting that looks applied and does nothing. See the - # YAC_LAUNCHER note at the top of this file. elif [[ "$ENDPOINT_NAME" == *yac* ]]; then echo "ERROR: '$ENDPOINT_NAME' is named for YAC but WITH_YAC is 0." >&2 echo " Rebuild it with YAC: WITH_YAC=1 $(basename "$0") configure" >&2 @@ -236,6 +227,10 @@ engine: init_blocks: 1 worker_init: | unset PYTHONPATH + # Stale PMI vars from the batch job make MPI_Init abort when YAC is + # imported (the fd PMI_FD names is already closed). The server also + # clears these per-remap; doing it here covers older clients too. + unset PMI_FD PMI_RANK PMI_SIZE source "\$(conda info --base)/etc/profile.d/conda.sh" conda activate $CONDA_ENV$yac_line diff --git a/src/uxarray_mcp/remote/agent.py b/src/uxarray_mcp/remote/agent.py index f21553a..04f4b5a 100644 --- a/src/uxarray_mcp/remote/agent.py +++ b/src/uxarray_mcp/remote/agent.py @@ -136,27 +136,31 @@ def _normalize_remote_error( ) if yac_backed: # Memory is the usual cause of a lost worker, but not for this - # call. Importing YAC runs ``MPI_Init``; Open MPI initialises - # fine as a singleton, MPICH aborts the process instead, and an - # abort inside a Globus worker arrives here as WorkerLost with - # nothing mentioning MPI. Observed on Casper, where the same - # conservative remap that kills the worker completes in 0.09s - # under ``check_remote_yac`` -- which runs it in a worker-side - # *subprocess*. Same node, same YAC, same method: the difference - # is only whether an abort can take the worker with it. + # call. Importing YAC runs ``MPI_Init``, which decides whether + # this process is one rank of many by reading the environment. + # A worker inside a batch allocation inherits ``PMI_FD`` from + # the job, pointing at a descriptor that is already closed, so + # MPI writes to it, gets EBADF and *aborts* -- which arrives + # here as WorkerLost with nothing mentioning MPI. # - # Sending the caller to look at mesh size here costs them the - # afternoon, because subsetting cannot fix it. + # The server now unsets PMI_FD/PMI_RANK/PMI_SIZE before every + # YAC-backed remap, so reaching this message means something + # else: an older server on the worker, or a genuinely broken + # YAC build. Sending the caller to look at mesh size would cost + # them the afternoon, because subsetting cannot fix either. return RuntimeError( f"The worker on {endpoint} died running a YAC-backed remap" f"{where}. This is usually MPI, not memory: importing YAC " - "calls MPI_Init, and on an MPICH site that aborts the " - "process rather than initialising as a singleton, which " - "arrives here as a lost worker. Confirm with " - "`check_remote_yac(endpoint=...)`: if its `conservative_ok` " - "is true, YAC itself is fine and the endpoint needs its " - "workers launched under an MPI launcher (`mpiexec -n 1` or " - "`srun --ntasks 1`) in worker_init. Until then, " + "calls MPI_Init, and a worker that inherited a stale PMI_FD " + "from its batch job aborts instead of initialising as a " + "singleton. This server unsets those variables before " + "remapping, so the likely cause is an older uxarray-mcp on " + "the endpoint -- compare versions with " + "`check_remote_yac(endpoint=...)` and redeploy, or unset " + "PMI_FD in the endpoint's worker_init. Do not reach for an " + "MPI launcher: at more than one rank each rank remaps only " + "its own partition and returns it as the whole field, with " + "no error and no NaNs. Meanwhile " "method='inverse_distance_weighted' or 'bilinear' run on " "uxarray's own engine and do not need YAC -- but note that " "neither conserves the field integral, which is the reason " diff --git a/src/uxarray_mcp/remote/compute_functions.py b/src/uxarray_mcp/remote/compute_functions.py index 274b60c..f93b614 100644 --- a/src/uxarray_mcp/remote/compute_functions.py +++ b/src/uxarray_mcp/remote/compute_functions.py @@ -3342,6 +3342,10 @@ def _surface(mod): """ env = os.environ.copy() + # The probe must match the real in-process path; otherwise it can report + # conservative_ok while the worker itself still aborts in MPI_Init. + for _pmi_var in ("PMI_FD", "PMI_RANK", "PMI_SIZE"): + env.pop(_pmi_var, None) prefix_applied = None if yac_prefix: prefix = os.path.abspath(os.path.expanduser(yac_prefix)) @@ -4619,6 +4623,14 @@ def remote_remap_variable( """ import os + # Clear stale batch-job PMI metadata before YAC calls MPI_Init. The + # worker may inherit PMI_FD/PMI_RANK/PMI_SIZE even though PMI_FD is a + # closed descriptor; MPICH then aborts instead of choosing singleton mode. + # A launcher is not a safe substitute: multiple ranks need explicit + # gather logic, which these worker functions do not have. + for _pmi_var in ("PMI_FD", "PMI_RANK", "PMI_SIZE"): + os.environ.pop(_pmi_var, None) + import numpy as np import uxarray as ux @@ -4885,6 +4897,14 @@ def remote_regrid_dataset( """ import os + # Clear stale batch-job PMI metadata before YAC calls MPI_Init. The + # worker may inherit PMI_FD/PMI_RANK/PMI_SIZE even though PMI_FD is a + # closed descriptor; MPICH then aborts instead of choosing singleton mode. + # A launcher is not a safe substitute: multiple ranks need explicit + # gather logic, which these worker functions do not have. + for _pmi_var in ("PMI_FD", "PMI_RANK", "PMI_SIZE"): + os.environ.pop(_pmi_var, None) + import numpy as np import uxarray as ux @@ -5149,6 +5169,14 @@ def remote_remap_to_rectilinear( """ import os + # Clear stale batch-job PMI metadata before YAC calls MPI_Init. The + # worker may inherit PMI_FD/PMI_RANK/PMI_SIZE even though PMI_FD is a + # closed descriptor; MPICH then aborts instead of choosing singleton mode. + # A launcher is not a safe substitute: multiple ranks need explicit + # gather logic, which these worker functions do not have. + for _pmi_var in ("PMI_FD", "PMI_RANK", "PMI_SIZE"): + os.environ.pop(_pmi_var, None) + import numpy as np import uxarray as ux import xarray as xr diff --git a/tests/test_yac_pmi_singleton.py b/tests/test_yac_pmi_singleton.py new file mode 100644 index 0000000..9ac1305 --- /dev/null +++ b/tests/test_yac_pmi_singleton.py @@ -0,0 +1,190 @@ +"""YAC-backed remaps must clear the inherited PMI variables before importing. + +Importing YAC calls ``MPI_Init``, which decides whether the process is one +rank of many by reading the environment. A Globus Compute worker running +inside a batch allocation inherits ``PMI_FD``/``PMI_RANK``/``PMI_SIZE`` from +the job that started the endpoint, but the descriptor ``PMI_FD`` names +belongs to a process that has already exited -- ``os.fstat`` on it raises +``EBADF``. MPI writes to it anyway, gets "Bad file descriptor", and *aborts* +the process. An abort is not an exception: it takes the worker down, and the +caller sees ``WorkerLost`` with nothing mentioning MPI, which reads as an +out-of-memory kill and sends people to measure their mesh. + +Measured on Chrysalis (uxarray 2026.10.0, YAC 3.20.2, MPICH) by removing one +variable at a time from an otherwise untouched worker environment:: + + baseline rc=15 MPI_Init abort + strip SLURM_* only rc=15 + strip HYDRA* only rc=15 + strip PMI* rc=0 768 finite + drop PMI_JOBID / RANK / SIZE rc=15 (each individually) + drop PMI_FD rc=0 <- the trigger + +With ``PMI_FD`` unset the remap runs *in the live worker process*, not a +subprocess, and returns the same answer as a laptop with no MPI environment +at all: ``sum=73344.0, zeros=4``, five runs in a row. + +Why not an MPI launcher, which also works +----------------------------------------- +``mpiexec -n 1`` fixes the import too, and was the previous recommendation. +It is the more dangerous option. YAC decomposes its work across whatever +ranks it is given, and nothing in this server gathers the pieces back, so at +``-n > 1`` each rank remaps its own partition and returns it as the whole +field. Measured locally against a known-correct serial answer of 73344.0:: + + 1 rank : 73344.0 73344.0 73344.0 73344.0 (exact, every run) + 4 ranks : 56635 58331 53862 60373 (different every run) + +The corrupted output has the full 768 values, zero NaNs, and differs from +truth in 141 cells -- it fails silently. Unsetting one environment variable +has no equivalent failure mode, which is why that is what the code does. +""" + +from __future__ import annotations + +import inspect + +import pytest + +from uxarray_mcp.remote import agent as agent_mod +from uxarray_mcp.remote import compute_functions as cf + +#: Every remote function that can route a remap through YAC. +YAC_REMAP_FUNCTIONS = ( + "remote_remap_variable", + "remote_regrid_dataset", + "remote_remap_to_rectilinear", +) + +#: Unsetting PMI_RANK/PMI_SIZE is not load-bearing on its own -- PMI_FD is the +#: measured trigger -- but they describe the same phantom job, and leaving a +#: half-consistent set behind invites a future MPI build to believe it. +EXPECTED_PMI_VARS = {"PMI_FD", "PMI_RANK", "PMI_SIZE"} + + +def _function_source(name: str) -> str: + return inspect.getsource(getattr(cf, name)) + + +class TestWorkerFunctionsClearPmi: + """The guard must exist, and must run before YAC can be imported.""" + + @pytest.mark.parametrize("func_name", YAC_REMAP_FUNCTIONS) + def test_clears_the_measured_variables(self, func_name): + source = _function_source(func_name) + for var in EXPECTED_PMI_VARS: + assert var in source, ( + f"{func_name} does not clear {var}. A YAC-backed remap on a " + "batch-allocated worker will abort in MPI_Init and arrive as " + "WorkerLost." + ) + + @pytest.mark.parametrize("func_name", YAC_REMAP_FUNCTIONS) + def test_guard_precedes_every_remap_call(self, func_name): + """Ordering is the whole point: MPI_Init runs at import, once. + + Popping the variables after ``.remap(...)`` has already pulled YAC in + would leave the abort exactly where it was while looking fixed. + """ + source = inspect.getsource(getattr(cf, func_name)) + lines = source.splitlines() + + guard_lines = [i for i, line in enumerate(lines) if "_pmi_var" in line] + assert guard_lines, f"{func_name} has no PMI guard at all" + first_guard = min(guard_lines) + + remap_lines = [ + i for i, line in enumerate(lines) if ".remap(" in line or ".remap." in line + ] + assert remap_lines, f"{func_name} has no remap call to guard" + assert first_guard < min(remap_lines), ( + f"{func_name} clears PMI_FD at line {first_guard} but remaps at " + f"{min(remap_lines)}. MPI_Init runs on import, so the guard has " + "to come first or it does nothing." + ) + + def test_guard_is_identical_across_copies(self): + """These are hand-copied into each worker function; keep them in sync. + + ``remote/compute_functions.py`` cannot use a shared helper -- Globus + Compute ships each function's source and closures do not survive + serialization -- so the only protection against one copy drifting is + a test that compares them. + """ + extracted = [] + for name in YAC_REMAP_FUNCTIONS: + lines = _function_source(name).splitlines() + start = next(i for i, line in enumerate(lines) if "_pmi_var" in line) + extracted.append(tuple(line.strip() for line in lines[start : start + 2])) + assert len(set(extracted)) == 1, ( + f"The PMI guard has drifted between worker copies: {set(extracted)}" + ) + + +class TestProbeMatchesTheRealPath: + """``check_remote_yac`` must measure the configuration callers get.""" + + def test_probe_clears_pmi_before_running(self): + """Otherwise it reports success for a path that cannot work. + + This is the concrete regression: the probe ran its remap in a + subprocess and passed the inherited PMI_FD straight through, so it + reported ``conservative_ok: true`` on an endpoint where every + in-process conservative remap killed the worker. A doomed child is + a non-zero exit code; the same abort in the worker is fatal. + """ + source = inspect.getsource(cf.remote_yac_remap_smoke) + assert "PMI_FD" in source, ( + "remote_yac_remap_smoke does not clear PMI_FD, so it is not " + "testing the same thing the remap path does -- which is exactly " + "how it came to report conservative_ok on a broken endpoint." + ) + assert "env.pop" in source or "env.pop(" in source, ( + "The probe must strip the variables from the subprocess env it " + "builds, not from its own os.environ." + ) + + +class TestWorkerLostGuidance: + """The message must not send people to the dangerous fix.""" + + def _yac_message(self) -> str: + exc = RuntimeError("Task failure due to loss of worker 0 on host chr-0136") + config = type("C", (), {"endpoint_name": "chrysalis"})() + return str(agent_mod._normalize_remote_error(exc, config, yac_backed=True)) + + def test_names_the_actual_cause(self): + message = self._yac_message() + assert "PMI_FD" in message, ( + "The YAC WorkerLost message should name the variable, since that " + "is what someone has to unset." + ) + + def test_does_not_recommend_an_mpi_launcher(self): + """Measured: at -n > 1 this returns wrong numbers with no error. + + The 2026.10.0 message told people to switch the endpoint to an MPI + launcher. At one rank that works and is merely redundant; above one + rank each rank returns its own partition as the whole field. Advice + whose failure mode is silent corruption does not belong in an error + string. + """ + message = self._yac_message() + assert "mpiexec -n 1" not in message + assert "srun --ntasks 1" not in message + assert "GlobusMPIEngine" not in message + + def test_warns_against_reaching_for_one(self): + message = self._yac_message() + assert "launcher" in message.lower(), ( + "Someone who already knows the MPI-launcher trick will try it; " + "the message should say why not." + ) + + def test_still_distinguishes_itself_from_the_memory_message(self): + """The non-YAC path keeps the memory advice, which is right there.""" + exc = RuntimeError("Task failure due to loss of worker 0 on host chr-0136") + config = type("C", (), {"endpoint_name": "chrysalis"})() + plain = str(agent_mod._normalize_remote_error(exc, config, yac_backed=False)) + assert "memory" in plain.lower() + assert "PMI_FD" not in plain