Skip to content

tests: accept staged Cray MPICH in Comm_Backends provenance #188

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
bobrobey merged 1 commit into from
Sep 7, 2026
+23 −1
Merged

tests: accept staged Cray MPICH in Comm_Backends provenance #188

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion tests/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1480,7 +1480,7 @@ set_tests_properties(Pytorch_ROCmSDK_Dlopen_Duplicate_Regression PROPERTIES TIME
# native RCCL / MPI / UCC / UCX libraries (soname + realpath + version) the
# build is actually wired to. Needs no GPU or launcher (pure build introspection).
add_test(NAME Pytorch_Comm_Backends_Check COMMAND ../pytorch_comm_backends_check.sh )
set_property(TEST Pytorch_Comm_Backends_Check PROPERTY PASS_REGULAR_EXPRESSION "GPU-AWARE MPI: OK")
set_property(TEST Pytorch_Comm_Backends_Check PROPERTY PASS_REGULAR_EXPRESSION "MPI PROVENANCE: OK")
set_property(TEST Pytorch_Comm_Backends_Check PROPERTY SKIP_REGULAR_EXPRESSION "module spider" "Unable to locate a modulefile for 'pytorch'" "COMM BACKENDS CHECK: SKIPPED")
set_property(TEST Pytorch_Comm_Backends_Check PROPERTY SKIP_RETURN_CODE 77)
set_tests_properties(Pytorch_Comm_Backends_Check PROPERTIES TIMEOUT 300)
Expand Down
22 changes: 22 additions & 0 deletions tests/pytorch_comm_backends_check.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -356,6 +356,28 @@ print(f" backends: nccl(RCCL)={avail['nccl']} gloo={avail['gloo']} "
f"mpi={avail['mpi']} ucc(native)={avail['ucc']}")
print(f" default comm path: GPU={defaults.get('cuda')}(RCCL) CPU={defaults.get('cpu')}")
print(f"GPU-AWARE MPI: {gpu_aware_mpi}")
# MPI provenance verdict: accept GPU-aware OpenMPI OR Cray/MPICH-family MPI;
# reject OS/system MPI. Evidence only: resolved libmpi realpath, soname, and
# MPI_Get_library_version banner. CTest gates on "MPI PROVENANCE: OK".
mpi_banner_text = mpi_banner(mpi_real) if mpi_real else ""
mpi_banner_lower = mpi_banner_text.lower()
mpi_is_os = bool(mpi_real and (mpi_real.startswith(("/usr/", "/lib/", "/lib64/"))
or "site-packages" in mpi_real or "/.libs/" in mpi_real))
mpi_is_staged_tree = bool(mpi_real and mpi_real.startswith(("/shareddata/", "/opt/cray/", "/opt/rocm")))
mpi_is_mpich_family = bool(("mpich" in mpi_banner_lower) or re.search(r"libmpi\w*\.so\.12(\.|$)", mpi_name or ""))
if not avail["mpi"]:
mpi_provenance = "NO - MPI backend not compiled in (normal for pip/venv wheels)"
elif not mpi_real:
mpi_provenance = "NO - libtorch does not link libmpi"
elif under_module and ompi_rocm:
mpi_provenance = f"OK - GPU-aware OpenMPI (module '{ompi_mod}', {mpi_name} under {ompi_prefix_real})"
elif mpi_is_os:
mpi_provenance = f"NO - OS/system MPI ({mpi_real}); expected GPU-aware OpenMPI or Cray MPICH"
elif mpi_is_mpich_family and mpi_is_staged_tree:
mpi_provenance = f"OK - Cray MPICH ({mpi_name}, '{mpi_banner_text}', {mpi_real})"
else:
mpi_provenance = f"NO - unrecognised MPI ({mpi_name}, '{mpi_banner_text}', {mpi_real}); not the GPU-aware OpenMPI module and not Cray MPICH"
print(f"MPI PROVENANCE: {mpi_provenance}")
sys.exit(0)
EOF

Expand Down