From 39c1ca6e4cb1e69c24fcfc85bcc6a0f5ea83a2e7 Mon Sep 17 00:00:00 2001 From: Yao Xu Date: Fri, 6 Nov 2026 11:34:39 -0800 Subject: [PATCH 1/4] Link libmpistub_audit.so with the plain C compiler src/mpi-wrappers/Makefile.in links libmpistub_audit.so with ${CC}, which it does not define, so make uses its default, cc. On a Cray system, such as Perlmutter, cc is the compiler wrapper, and it links libxpmem.so even with -nostdlib. Then ld.so loads libxpmem and libc into the audit module's namespace: a second libc in the upper half, which the module was written to avoid (PR #506). With it, the lower half patches the mmap of only one of the two libcs (see the next commit), and the upper half's own libc maps memory, among it the stack of DMTCP's checkpoint thread, outside the arena. The checkpoint skips that memory as the lower half's, and every restart segfaults in ThreadList::postRestartWork(), which recreates the thread on its stack. So restart fails on Perlmutter since PR #506 (MANA 1.5.0). CI does not see it: there, cc is gcc. Define CC as configure's C compiler, as the top-level Makefile does, here and in test/Makefile.in, where cc linked libxpmem into the test dlopen_mpi. The audit module's rule now also fails if the module needs any library, or if readelf cannot check it, instead of building a MANA whose restart crashes. It links to a temporary file named after the shell's PID, as the rules that generate sources here do. Tested on Perlmutter: the rule builds the module with configure's CC, and fails, leaving no temporary file, with CC=cc and with a readelf that fails. dlopen_mpi passes natively and under MANA in both P2P modes. --- src/mpi-wrappers/Makefile.in | 20 +++++++++++++++++--- test/Makefile.in | 4 +++- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/src/mpi-wrappers/Makefile.in b/src/mpi-wrappers/Makefile.in index 2df71093c..23d9ce68f 100644 --- a/src/mpi-wrappers/Makefile.in +++ b/src/mpi-wrappers/Makefile.in @@ -1,6 +1,8 @@ # @configure_input@ -# The compilers and flags that configure found. +# The compilers and flags that configure found. CC is the plain C +# compiler: make's default, cc, is the compiler wrapper on a Cray system. +CC = @CC@ CFLAGS = @CFLAGS@ CXXFLAGS = @CXXFLAGS@ MPICC = @MPICC@ @@ -100,10 +102,22 @@ mpi_stub_objects.c: generate-mpi-stub-objects.py libmpistub.so: mpi_stub_wrappers.o mpi_stub_data.o mpi_stub_objects.o ${CC} -fPIC $^ -shared -o $@ -# An LD_AUDIT module for the upper half; it must not need libc. +# An LD_AUDIT module for the upper half. It must need no library: ld.so +# would load it, and the libc that it needs, into the module's namespace, +# a second libc in the upper half. A Cray compiler wrapper adds -lxpmem +# even with -nostdlib, so check the module before installing it, and fail +# if readelf cannot. libmpistub_audit.so: mpi_stub_audit.c + tmp=$@.tmp.$$$$ ; \ ${CC} -O2 -fPIC -shared -nostdlib -fno-stack-protector \ - -Wl,--no-undefined -o $@ $< + -Wl,--no-undefined -o $$tmp $< || { rm -f $$tmp; exit 1; } ; \ + dynamic=$$(readelf -d $$tmp) || { \ + echo "cannot check $@ with readelf"; rm -f $$tmp; exit 1; } ; \ + if echo "$$dynamic" | grep NEEDED; then \ + echo "$@ must need no library; link it with a plain C compiler"; \ + rm -f $$tmp; exit 1; \ + fi ; \ + mv -f $$tmp $@ tidy: rm -f *~ .*.swp dmtcp_restart_script*.sh ckpt_*.dmtcp diff --git a/test/Makefile.in b/test/Makefile.in index f5a417005..7f1f0407f 100644 --- a/test/Makefile.in +++ b/test/Makefile.in @@ -1,6 +1,8 @@ # @configure_input@ -# The compilers and flags that configure found. +# The compilers and flags that configure found. CC is the plain C +# compiler: make's default, cc, is the compiler wrapper on a Cray system. +CC = @CC@ CXXFLAGS = @CXXFLAGS@ FFLAGS = ${CXXFLAGS} @FFLAGS@ MPICC = @MPICC@ From be9eddb9db6ed1e8f1eac3a68204692f0016a681 Mon Sep 17 00:00:00 2001 From: Yao Xu Date: Fri, 6 Nov 2026 11:34:39 -0800 Subject: [PATCH 2/4] Compute each libc's base from the mapping of its code __mmap_wrapper() patches the upper half's libc (mmap, munmap, mremap, _exit and __sbrk) when ld.so maps its code. It kept the libc's base from an earlier mapping: the first mapping of a file named libc.so or libc-* after the previous patch. But after the code, ld.so maps the libc's data segments, and the first of them became the base for the next libc. So when the upper half loaded a second libc, as an LD_AUDIT module's namespace does, the second one was patched at the first one's data plus the offsets of those functions: its own functions stayed as they were, and the trampolines were written into other memory (on Perlmutter, pages of the arena's reserved region). There, the second libc was the upper half's main one, so its mmap placed memory outside the arena, and restart failed. Any other mapping of a libc file between a libc's first mapping and its code, such as one that only reads the file, could change the base the same way. Compute the base at the mapping of the code, from that mapping alone: its address minus the page-aligned address of the executable PT_LOAD at its file offset, as ld.so maps it. No state is kept between calls. This also handles a libc whose code is not at address 0, as with binutils' -z separate-code, the default on Ubuntu and Fedora. Tested: on a Perlmutter login node, the computation gives ld.so's load bias (dl_iterate_phdr) for libc, libm, libgcc_s and ld.so, whose code is at address 0, and for libraries linked with -z separate-code (code at 0x1000) and with -z max-page-size=0x200000 (code at 0x200000). On a Perlmutter CPU node, 9 checkpoint-restart tests pass in both P2P modes, with one libc in the upper half and with two (the audit module linked with the Cray compiler wrapper). --- src/lower-half/mem-wrapper.cpp | 81 ++++++++++++++++++++++++---------- 1 file changed, 57 insertions(+), 24 deletions(-) diff --git a/src/lower-half/mem-wrapper.cpp b/src/lower-half/mem-wrapper.cpp index 36dee9299..ce6744978 100644 --- a/src/lower-half/mem-wrapper.cpp +++ b/src/lower-half/mem-wrapper.cpp @@ -22,6 +22,7 @@ #ifndef _GNU_SOURCE #define _GNU_SOURCE // For MAP_ANONYMOUS #endif +#include #include #include #include @@ -498,9 +499,37 @@ void* mmap_wrapper(void *addr, size_t length, int prot, return ret; } +// The load base of the library open on 'fd', from its executable segment at +// file offset 'offset', which is mapped at 'addr': ld.so maps a segment at +// the base plus the segment's address, both rounded down to a page, from the +// segment's file offset rounded down to a page. NULL if the library has no +// such segment. +static char* library_base(int fd, off_t offset, char *addr) +{ + const Elf64_Off page_mask = ~((Elf64_Off)PAGE_SIZE - 1); + Elf64_Ehdr ehdr; + if (pread(fd, &ehdr, sizeof(ehdr), 0) != (ssize_t)sizeof(ehdr) || + memcmp(ehdr.e_ident, ELFMAG, SELFMAG) != 0 || + ehdr.e_ident[EI_CLASS] != ELFCLASS64 || + ehdr.e_phentsize != sizeof(Elf64_Phdr)) { + return NULL; + } + for (int i = 0; i < ehdr.e_phnum; i++) { + Elf64_Phdr phdr; + if (pread(fd, &phdr, sizeof(phdr), ehdr.e_phoff + i * sizeof(phdr)) != + (ssize_t)sizeof(phdr)) { + return NULL; + } + if (phdr.p_type == PT_LOAD && (phdr.p_flags & PF_X) != 0 && + (phdr.p_offset & page_mask) == (Elf64_Off)offset) { + return addr - (phdr.p_vaddr & page_mask); + } + } + return NULL; +} + static void* __mmap_wrapper(void *addr, size_t length, int prot, int flags, int fd, off_t offset) { - static char *libc_base_addr = NULL; void *ret = MAP_FAILED; length = ROUND_UP(length, PAGE_SIZE); bool chosen = false; // The address is the arena's, not the caller's. @@ -528,34 +557,38 @@ static void* __mmap_wrapper(void *addr, size_t length, int prot, (size_t)(new_block_end_addr - (char*)ret)}; merge_overlap_blocks(new_block, allocated_blocks); remove_overlap_blocks(new_block, free_blocks); - if (fd > 0) { + // Patch each libc when ld.so maps its code, at a base that comes from + // this mapping alone. The upper half can load more than one libc (an + // LD_AUDIT module's namespace has its own), and other mappings of a libc + // file, such as its data or one that only reads the file, can come + // between a libc's first mapping and its code. + if (fd > 0 && (prot & PROT_EXEC)) { char glibcFullPath[PATH_MAX] = {0}; int found = checkLibrary(fd, "libc.so", glibcFullPath, PATH_MAX) || checkLibrary(fd, "libc-", glibcFullPath, PATH_MAX); if (found) { - if (libc_base_addr == NULL) { - libc_base_addr = static_cast(ret); + char *libc_base = library_base(fd, offset, (char*)ret); + if (libc_base == NULL) { + DLOG(ERROR, "No executable segment of %s at file offset 0x%lx\n", + glibcFullPath, (long)offset); } - if (prot & PROT_EXEC) { - int rc = mprotect(ret, length, prot | PROT_WRITE); - if (rc < 0) { - DLOG(ERROR, - "Failed to add PROT_WRITE perms for memory region at: %p " - "of: %zu bytes. Error: %s\n", addr, length, strerror(errno)); - return NULL; - } - patchLibc(fd, libc_base_addr, glibcFullPath); - off_t sbrk_offset = get_symbol_offset(glibcFullPath, "__sbrk"); - if (sbrk_offset) { - patchSbrk(libc_base_addr + sbrk_offset, (char*)ret, length); - } - rc = mprotect(ret, length, prot); - if (rc < 0) { - DLOG(ERROR, "Failed to restore perms for memory region at: %p " - "of: %zu bytes. Error: %s\n", addr, length, strerror(errno)); - return NULL; - } - libc_base_addr = NULL; + int rc = mprotect(ret, length, prot | PROT_WRITE); + if (rc < 0) { + DLOG(ERROR, + "Failed to add PROT_WRITE perms for memory region at: %p " + "of: %zu bytes. Error: %s\n", addr, length, strerror(errno)); + return NULL; + } + patchLibc(fd, libc_base, glibcFullPath); + off_t sbrk_offset = get_symbol_offset(glibcFullPath, "__sbrk"); + if (sbrk_offset) { + patchSbrk(libc_base + sbrk_offset, (char*)ret, length); + } + rc = mprotect(ret, length, prot); + if (rc < 0) { + DLOG(ERROR, "Failed to restore perms for memory region at: %p " + "of: %zu bytes. Error: %s\n", addr, length, strerror(errno)); + return NULL; } } } From 966a5aa4e74bef8807b39468fce4825b971136d5 Mon Sep 17 00:00:00 2001 From: Yao Xu Date: Sat, 7 Nov 2026 11:39:48 -0800 Subject: [PATCH 3/4] Make each communicator's gid without communication Since "Give each communicator its own clock and gid" (PR #552), rank 0 of each new communicator makes its gid, and new_comm_desc() sends it to the other members with MPI_Bcast. That MPI_Bcast is the communicator's first collective call, at its creation, and Cray MPICH gives its shared-memory collectives to communicators in the order of their first MPI_Bcast, MPI_Barrier or MPI_Allreduce. A node has 8 cells for them (MPICH_SHARED_MEM_COLL_NCELLS), shared by all its communicators. A communicator that finds no free cell never gets one; MPI_Comm_free frees its cell. On a Perlmutter CPU node with 128 ranks, MPI_Allreduce takes 4.4 us with a cell and 7.3 us without. So the cells went to communicators in the order the program made them, not used them, and at MPI_Init two went to MANA's own duplicates of MPI_COMM_WORLD, g_world_comm and the Ssend-ack communicator, which make no collective calls. Now each member computes the gid by itself, as a hash of: - the parent communicator's gid; - the number of calls on the parent that made communicators before this one (MPI_Comm_dup, MPI_Comm_split, MPI_Comm_split_type, MPI_Comm_create, MPI_Cart_create, MPI_Cart_sub). These calls are collective, and MPI fixes the order of collective calls on a communicator, so all members count the same; - the world rank of the new communicator's rank 0, which tells apart the disjoint communicators of one call, such as MPI_Comm_split's. Only the members of the group call MPI_Comm_create_group, so it counts the calls on the parent by group and tag. MPI_COMM_WORLD, MPI_COMM_SELF and the Ssend-ack communicator have gids from their names, and MPI_COMM_SELF also from the world rank. No two communicators have the same inputs, so two gids are equal only if the 64-bit hash collides: with 10,000 communicators in the job, the chance is below 3e-12. add_comm_clock() still aborts if a process has two communicators with the same gid. Making a communicator now needs no communication, and MANA makes no collective call that the program does not make. At restart, MANA now calls MPI_Barrier on MPI_COMM_WORLD before reconstruct_descriptors(). A restart starts a new MPI library, which gives cells in the order of the first calls after the restart, so the communicators that the program uses first where it resumes could take world's cell. The MPI_Barrier gives world a cell first, as most programs do natively right after MPI_Init; the other communicators get cells in the order in which the program uses them where it resumes. MANA does not repeat the order from before the checkpoint. How many cells a node has, that the first call takes one, and that a communicator never tries again are Cray MPICH's own policy, which its man page does not describe. Repeating it would take a record of each communicator's first such call, and at each restart a replay of those calls in their order on every node. Tested on a Perlmutter CPU node, 128 ranks, cray-mpich 9.1.0, with a probe that runs MPI_Barrier on MPI_COMM_WORLD, then one MPI_Bcast on each of 6 duplicates, then on the even half of world (MPI_Comm_split by rank parity), then on the odd half, and then loops over them in the reverse order: - At launch, natively all but the odd half have cells. On main, the 6th duplicate and the even half had none either; now the same communicators have cells as natively, in 5 of 5 launches. - After 5 restarts, world kept its cell (MPI_Allreduce 4.3-4.9 us; on main 7.6-8.2). The first duplicate lost its own (7.6-8.1 us), and the odd half, which the loop uses first, got one (3.5-4.1 us instead of 6.4-7.3). - autotest: 53/53 polling and 53/53 blocking. - virtid_bench, 3 runs, us per call: MPI_Allreduce on world 4.24-4.45 (natively 4.34-4.50, on main 7.4), on a duplicate 4.45-4.63 (4.32-4.50), MPI_Barrier 4.54-4.73 (4.57-4.65). --- src/mpi-wrappers/mpi_cart_wrappers.cpp | 6 +- src/mpi-wrappers/mpi_collective_wrappers.cpp | 6 +- src/mpi-wrappers/mpi_comm_wrappers.cpp | 9 +- src/p2p_drain.cpp | 2 +- src/virtual_id.cpp | 126 ++++++++++++++----- src/virtual_id.h | 23 +++- 6 files changed, 135 insertions(+), 37 deletions(-) diff --git a/src/mpi-wrappers/mpi_cart_wrappers.cpp b/src/mpi-wrappers/mpi_cart_wrappers.cpp index d6949f71d..5b7d31262 100644 --- a/src/mpi-wrappers/mpi_cart_wrappers.cpp +++ b/src/mpi-wrappers/mpi_cart_wrappers.cpp @@ -112,11 +112,12 @@ int PMPI_Cart_sub(MPI_Comm comm, const int *remain_dims, MPI_Comm *new_comm) commit_begin(comm); LOWER_HALF_DISABLE_CKPT(); MPI_Comm realComm = get_real_id((mana_mpi_handle){.comm = comm}).comm; + uint64_t seed = child_gid_seed(comm); JUMP_TO_LOWER_HALF(lh_info->fsaddr); retval = NEXT_FUNC(Cart_sub)(realComm, remain_dims, new_comm); RETURN_TO_UPPER_HALF(); if (retval == MPI_SUCCESS) { - *new_comm = new_virt_comm(*new_comm); + *new_comm = new_virt_comm(*new_comm, seed); } LOWER_HALF_ENABLE_CKPT(); commit_finish(comm); @@ -177,12 +178,13 @@ int PMPI_Cart_create(MPI_Comm old_comm, int ndims, commit_begin(old_comm); LOWER_HALF_DISABLE_CKPT(); MPI_Comm realComm = get_real_id((mana_mpi_handle){.comm = old_comm}).comm; + uint64_t seed = child_gid_seed(old_comm); JUMP_TO_LOWER_HALF(lh_info->fsaddr); retval = NEXT_FUNC(Cart_create)(realComm, ndims, dims, periods, reorder, comm_cart); RETURN_TO_UPPER_HALF(); if (retval == MPI_SUCCESS) { - *comm_cart = new_virt_comm(*comm_cart); + *comm_cart = new_virt_comm(*comm_cart, seed); } LOWER_HALF_ENABLE_CKPT(); commit_finish(old_comm); diff --git a/src/mpi-wrappers/mpi_collective_wrappers.cpp b/src/mpi-wrappers/mpi_collective_wrappers.cpp index 19d3eee28..14bef73ff 100644 --- a/src/mpi-wrappers/mpi_collective_wrappers.cpp +++ b/src/mpi-wrappers/mpi_collective_wrappers.cpp @@ -803,6 +803,7 @@ int PMPI_Comm_split(MPI_Comm comm, int color, int key, MPI_Comm *newcomm) int retval; LOWER_HALF_DISABLE_CKPT(); MPI_Comm real_comm = get_real_id((mana_mpi_handle){.comm = comm}).comm; + uint64_t seed = child_gid_seed(comm); JUMP_TO_LOWER_HALF(lh_info->fsaddr); retval = NEXT_FUNC(Comm_split)(real_comm, color, key, newcomm); RETURN_TO_UPPER_HALF(); @@ -810,7 +811,7 @@ int PMPI_Comm_split(MPI_Comm comm, int color, int key, MPI_Comm *newcomm) if (*newcomm == lh_info->MANA_COMM_NULL) { *newcomm = MPI_COMM_NULL; } else { - *newcomm = new_virt_comm(*newcomm); + *newcomm = new_virt_comm(*newcomm, seed); } } LOWER_HALF_ENABLE_CKPT(); @@ -825,6 +826,7 @@ int PMPI_Comm_dup(MPI_Comm comm, MPI_Comm *newcomm) int retval; LOWER_HALF_DISABLE_CKPT(); MPI_Comm real_comm = get_real_id((mana_mpi_handle){.comm = comm}).comm; + uint64_t seed = child_gid_seed(comm); JUMP_TO_LOWER_HALF(lh_info->fsaddr); retval = NEXT_FUNC(Comm_dup)(real_comm, newcomm); RETURN_TO_UPPER_HALF(); @@ -832,7 +834,7 @@ int PMPI_Comm_dup(MPI_Comm comm, MPI_Comm *newcomm) if (*newcomm == lh_info->MANA_COMM_NULL) { *newcomm = MPI_COMM_NULL; } else { - *newcomm = new_virt_comm(*newcomm); + *newcomm = new_virt_comm(*newcomm, seed); } } LOWER_HALF_ENABLE_CKPT(); diff --git a/src/mpi-wrappers/mpi_comm_wrappers.cpp b/src/mpi-wrappers/mpi_comm_wrappers.cpp index d4cfd2cbc..426ce93d3 100644 --- a/src/mpi-wrappers/mpi_comm_wrappers.cpp +++ b/src/mpi-wrappers/mpi_comm_wrappers.cpp @@ -275,6 +275,7 @@ int PMPI_Comm_create(MPI_Comm comm, MPI_Group group, MPI_Comm *newcomm) LOWER_HALF_DISABLE_CKPT(); MPI_Comm real_comm = get_real_id((mana_mpi_handle){.comm = comm}).comm; MPI_Group real_group = get_real_id((mana_mpi_handle){.group = group}).group; + uint64_t seed = child_gid_seed(comm); JUMP_TO_LOWER_HALF(lh_info->fsaddr); retval = NEXT_FUNC(Comm_create)(real_comm, real_group, newcomm); RETURN_TO_UPPER_HALF(); @@ -282,7 +283,7 @@ int PMPI_Comm_create(MPI_Comm comm, MPI_Group group, MPI_Comm *newcomm) if (*newcomm == lh_info->MANA_COMM_NULL) { *newcomm = MPI_COMM_NULL; } else { - *newcomm = new_virt_comm(*newcomm); + *newcomm = new_virt_comm(*newcomm, seed); } } LOWER_HALF_ENABLE_CKPT(); @@ -547,6 +548,7 @@ int PMPI_Comm_split_type(MPI_Comm comm, int split_type, LOWER_HALF_DISABLE_CKPT(); MPI_Comm real_comm = get_real_id((mana_mpi_handle){.comm = comm}).comm; MPI_Info real_info = lower_half_handle(inf); + uint64_t seed = child_gid_seed(comm); JUMP_TO_LOWER_HALF(lh_info->fsaddr); retval = NEXT_FUNC(Comm_split_type)(real_comm, split_type, key, real_info, newcomm); @@ -555,7 +557,7 @@ int PMPI_Comm_split_type(MPI_Comm comm, int split_type, if (*newcomm == lh_info->MANA_COMM_NULL) { *newcomm = MPI_COMM_NULL; } else { - *newcomm = new_virt_comm(*newcomm); + *newcomm = new_virt_comm(*newcomm, seed); } } LOWER_HALF_ENABLE_CKPT(); @@ -690,12 +692,13 @@ int PMPI_Comm_create_group(MPI_Comm comm, // The member enters the call in the 1 s pause before the next attempt. begin_critical_section(comm, false); LOWER_HALF_DISABLE_CKPT(); + uint64_t seed = group_child_gid_seed(comm, group, tag); int retval = MPI_Comm_create_group_internal(comm, group, tag, newcomm); if (retval == MPI_SUCCESS) { if (*newcomm == lh_info->MANA_COMM_NULL) { *newcomm = MPI_COMM_NULL; } else { - *newcomm = new_virt_comm(*newcomm); + *newcomm = new_virt_comm(*newcomm, seed); } } LOWER_HALF_ENABLE_CKPT(); diff --git a/src/p2p_drain.cpp b/src/p2p_drain.cpp index 650712610..16c3c9a63 100644 --- a/src/p2p_drain.cpp +++ b/src/p2p_drain.cpp @@ -174,7 +174,7 @@ initialize_drain_send_recv() retval = NEXT_FUNC(Comm_dup)(lh_info->MANA_COMM_WORLD, &ack_comm); RETURN_TO_UPPER_HALF(); ASSERT(retval == MPI_SUCCESS, "retval={}", retval); - g_ssend_ack_comm = new_virt_comm(ack_comm); + g_ssend_ack_comm = new_virt_comm(ack_comm, mana_gid_seed("ssend-ack")); } // Sums all ranks' sent and received counts in this round's keys. The next diff --git a/src/virtual_id.cpp b/src/virtual_id.cpp index 37101a0a4..809841570 100644 --- a/src/virtual_id.cpp +++ b/src/virtual_id.cpp @@ -182,16 +182,9 @@ slot_entry(int slot) [slot & (MANA_VIRT_ID_CHUNK_SIZE - 1)]; } -// The number of gids that this process made. -static uint64_t gids_made = 0; // The descriptor of MPI_COMM_SELF. MPI_COMM_SELF has no virtual-ID entry. static mana_comm_desc *self_desc; -// Returns a new gid: the number of gids that this process made before, times -// the world size, plus the world rank of this process. Gids from two -// processes differ modulo the world size. A process never uses the same -// number twice. So no two gids in the job are equal. -// // The gid is the key of the communicator's Collective Clock (struct // comm_clock). The clock counts the collective calls that this process made // on the communicator. Before a checkpoint, the drain lets each member make @@ -211,20 +204,92 @@ static mana_comm_desc *self_desc; // ranks the barrier on Y. Neither barrier could complete, and the checkpoint // would wait for them forever. So each new communicator gets a new gid, even // if another communicator has the same group. +// +// Each member computes the gid by itself, from the call that made the +// communicator. A collective call to agree on it would be the new +// communicator's first, while the application may make its own first call +// much later, or never. That changes which communicators get the MPI +// library's resources for collectives: Cray MPICH gives its shared-memory +// collectives to the first communicators that call MPI_Bcast, MPI_Barrier or +// MPI_Allreduce. +// +// A communicator made by a call on a parent P (MPI_Comm_dup, MPI_Comm_split, +// ...) has the gid hash(P's gid, n, r). The call is P's n-th call that made +// communicators: all members of P make these calls in the same order, as MPI +// requires of collective calls. r is the world rank of the communicator's +// rank 0, which tells apart the disjoint communicators that one call makes, +// as MPI_Comm_split does. MPI_Comm_create_group(P, group, tag) is called +// only by the members of the group, so for it n counts the calls on P with +// the same group and tag. MPI_COMM_WORLD, MPI_COMM_SELF and MANA's own +// communicators have gids from their names, and from the world rank for +// MPI_COMM_SELF. So no two communicators have the same inputs, and two gids +// in use are equal only if the 64-bit hash collides: with 10,000 +// communicators in the job, the chance is below 3e-12. add_comm_clock() +// aborts if one process has two communicators with the same gid. +enum { GID_CHILD = 1, GID_GROUP_CHILD, GID_NAMED }; + +// Mixes 'value' into the hash 'h'. For a given 'h', different values give +// different results: boost's hash_combine, then splitmix64's finalizer, are +// both one-to-one. static uint64_t -make_gid() +gid_mix(uint64_t h, uint64_t value) +{ + uint64_t x = h ^ (value + 0x9e3779b97f4a7c15ULL + (h << 6) + (h >> 2)); + x = (x ^ (x >> 30)) * 0xbf58476d1ce4e5b9ULL; + x = (x ^ (x >> 27)) * 0x94d049bb133111ebULL; + return x ^ (x >> 31); +} + +uint64_t +child_gid_seed(MPI_Comm parent) +{ + mana_comm_desc *desc = get_comm_desc(parent); + if (desc == NULL) { + return 0; // An invalid handle: the MPI call fails. + } + return gid_mix(gid_mix(GID_CHILD, desc->gid), desc->children++); +} + +uint64_t +group_child_gid_seed(MPI_Comm parent, MPI_Group group, int tag) +{ + mana_comm_desc *desc = get_comm_desc(parent); + virt_id_entry *entry = + lookup_virt_id_entry((mana_mpi_handle){.group = group}); + if (desc == NULL || entry == NULL || + entry->virt >> MANA_VIRT_ID_KIND_SHIFT != MANA_GROUP_KIND) { + return 0; // An invalid handle or MPI_GROUP_EMPTY: the MPI call fails. + } + mana_group_desc *group_desc = (mana_group_desc*)entry->desc; + uint64_t key = gid_mix(GID_GROUP_CHILD, (uint32_t)tag); + for (int i = 0; i < group_desc->size; i++) { + key = gid_mix(key, (uint32_t)group_desc->global_ranks[i]); + } + if (desc->group_children == NULL) { + desc->group_children = new std::unordered_map(); + } + uint64_t n = (*desc->group_children)[key]++; + return gid_mix(gid_mix(key, desc->gid), n); +} + +uint64_t +mana_gid_seed(const char *name) { - return gids_made++ * g_world_size + g_world_rank; + uint64_t h = GID_NAMED; + for (const char *c = name; *c != '\0'; c++) { + h = gid_mix(h, (unsigned char)*c); + } + return h; } int is_predefined_id(mana_mpi_handle id) { return find_constant(constant_key(id)) != NULL; } -// Returns a new descriptor of the communicator 'real_comm', with a new gid -// and a new clock. +// Returns a new descriptor of the communicator 'real_comm', with the gid +// hash('seed', world rank of its rank 0) and a new clock. static mana_comm_desc* -new_comm_desc(MPI_Comm real_comm) +new_comm_desc(MPI_Comm real_comm, uint64_t seed) { MPI_Group local_group; int *local_ranks; @@ -272,26 +337,19 @@ new_comm_desc(MPI_Comm real_comm) RETURN_TO_UPPER_HALF(); } - // The communicator's rank 0 makes the gid and sends it to the other - // members. - desc->gid = 0; - if (desc->rank == 0) { - desc->gid = make_gid(); - } - if (desc->size > 1) { - JUMP_TO_LOWER_HALF(lh_info->fsaddr); - NEXT_FUNC(Bcast)(&desc->gid, 1, lh_info->MANA_UINT64_T, 0, real_comm); - RETURN_TO_UPPER_HALF(); - } + // No communication: see the comment on the gid above. + desc->gid = gid_mix(seed, (uint32_t)desc->global_ranks[0]); + desc->children = 0; + desc->group_children = NULL; desc->clock = add_comm_clock(desc->gid); return desc; } -MPI_Comm new_virt_comm(MPI_Comm real_comm) { +MPI_Comm new_virt_comm(MPI_Comm real_comm, uint64_t seed) { if (real_comm == lh_info->MANA_COMM_NULL) { return MPI_COMM_NULL; } - mana_comm_desc *desc = new_comm_desc(real_comm); + mana_comm_desc *desc = new_comm_desc(real_comm, seed); mana_mpi_handle virt_id; virt_id = add_virt_id((mana_mpi_handle){.comm = real_comm}, desc, MANA_COMM_KIND); return virt_id.comm; @@ -556,6 +614,16 @@ void reconstruct_datatype_desc(virt_id_entry *entry) { } void reconstruct_descriptors() { + // MPI_COMM_WORLD runs the new MPI library's first collective, as most + // programs do at MPI_Init. Otherwise the communicators that the program + // uses first where it resumes would take Cray MPICH's shared-memory + // collectives from MPI_COMM_WORLD. The other communicators get them in + // the order in which the program uses them after the restart: MANA does + // not repeat the order from before the checkpoint, which depends on how + // one MPI library hands out such resources. + JUMP_TO_LOWER_HALF(lh_info->fsaddr); + NEXT_FUNC(Barrier)(lh_info->MANA_COMM_WORLD); + RETURN_TO_UPPER_HALF(); // Reconstruct by kind, then in creation order (not slot order: slots are // reused). MPI_Comm_create_group is collective, so all ranks must // reconstruct their communicators in the same order. @@ -719,12 +787,13 @@ void init_predefined_virt_ids() { // of MPI_COMM_SELF. Call it at MPI_Init only. At restart, g_world_comm is // in the virtual-ID table, so reconstruct_descriptors() makes it again. void init_world_comm() { - getLocalRankInfo(); // make_gid() needs g_world_rank and g_world_size. + getLocalRankInfo(); // g_world_rank and g_world_size JUMP_TO_LOWER_HALF(lh_info->fsaddr); NEXT_FUNC(Comm_dup)(lh_info->MANA_COMM_WORLD, &g_world_comm); RETURN_TO_UPPER_HALF(); - g_world_comm = new_virt_comm(g_world_comm); - self_desc = new_comm_desc(lh_info->MANA_COMM_SELF); + g_world_comm = new_virt_comm(g_world_comm, mana_gid_seed("MPI_COMM_WORLD")); + self_desc = new_comm_desc(lh_info->MANA_COMM_SELF, + mana_gid_seed("MPI_COMM_SELF")); } // Takes a slot from the free list, or else the next unused, unreserved slot. @@ -843,6 +912,7 @@ void free_desc(void *desc, int kind) { free(comm_desc->global_ranks); free(comm_desc->cart_dims); free(comm_desc->cart_periods); + delete comm_desc->group_children; } else if (kind == MANA_GROUP_KIND) { mana_group_desc *group_desc = (mana_group_desc*)desc; free(group_desc->global_ranks); diff --git a/src/virtual_id.h b/src/virtual_id.h index ba42e8aec..f3a424ec2 100644 --- a/src/virtual_id.h +++ b/src/virtual_id.h @@ -7,6 +7,7 @@ #include #include #include +#include #include #include "lower-half-api.h" // MANA_*_KIND @@ -65,6 +66,14 @@ typedef struct { // the KVDB keys and values of the Collective Clock and the P2P drain hold // the gid of a communicator, not its handle. uint64_t gid; + // The number of calls on this communicator that made communicators from + // it, and that all its members make: MPI_Comm_dup, MPI_Comm_split, ... + // (child_gid_seed()). + uint64_t children; + // For MPI_Comm_create_group() on this communicator, which only the + // members of the group call: the number of calls by group and tag + // (group_child_gid_seed()), or NULL before the first. + std::unordered_map *group_children; // The communicator's Collective Clock (see seq_num.cpp). struct comm_clock *clock; } mana_comm_desc; @@ -169,7 +178,19 @@ extern virt_id_entry *virt_id_chunks[MANA_VIRT_ID_NUM_CHUNKS]; extern int g_world_rank; void init_world_comm(); -MPI_Comm new_virt_comm(MPI_Comm real_comm); +// The seed of the gid of a communicator that a call on 'parent' makes: +// MPI_Comm_dup, MPI_Comm_split, MPI_Comm_split_type, MPI_Comm_create, +// MPI_Cart_create or MPI_Cart_sub. Each member of 'parent' calls it once +// in each such call, before the call makes the communicator, even if the +// call gives it MPI_COMM_NULL. +uint64_t child_gid_seed(MPI_Comm parent); +// The same for MPI_Comm_create_group(parent, group, tag). +uint64_t group_child_gid_seed(MPI_Comm parent, MPI_Group group, int tag); +// The seed of the gid of MANA's own communicator 'name'. +uint64_t mana_gid_seed(const char *name); +// Returns the virtual handle of a new communicator, whose gid comes from +// 'seed' (see new_comm_desc()). +MPI_Comm new_virt_comm(MPI_Comm real_comm, uint64_t seed); MPI_Group new_virt_group(MPI_Group real_group); MPI_Op new_virt_op(MPI_Op real_op); // Returns a zeroed descriptor; the caller fills it in and passes it to From eefb9755b1f6a1912642fa2ca8a39f372ae02fe0 Mon Sep 17 00:00:00 2001 From: Yao Xu Date: Fri, 6 Nov 2026 12:31:31 -0800 Subject: [PATCH 4/4] Warn when the upper half loads a second libc A second libc in the upper half is in another namespace: that of an LD_AUDIT module that needs libc, such as libmpistub_audit.so linked with the Cray compiler wrapper, or one that dlmopen() makes. The lower half patches its mmap like the first one's, but DMTCP's wrappers are loaded only into the program's namespace. So code in the other namespace creates threads, files and signal handlers that DMTCP does not see, and a restart may not restore them. Say so on stderr when the lower half patches a second libc, from rank 0 only: the ranks normally load the same libraries. Tested on a Perlmutter CPU node with 4 ranks: no warning with one libc in the upper half, one warning (rank 0's) with two. 9 checkpoint-restart tests pass with one libc (polling) and with two (polling and blocking). --- src/lower-half/lower-half.cpp | 1 + src/lower-half/mem-wrapper.cpp | 16 ++++++++++++++++ src/lower-half/mem-wrapper.h | 2 ++ 3 files changed, 19 insertions(+) diff --git a/src/lower-half/lower-half.cpp b/src/lower-half/lower-half.cpp index 991f3e1c2..9bd07b7e4 100644 --- a/src/lower-half/lower-half.cpp +++ b/src/lower-half/lower-half.cpp @@ -213,6 +213,7 @@ int main(int argc, char *argv[], char *envp[]) { remove_dangling_env_entries(initial_argv); record_lower_half_pid(); MPI_Comm_rank(MPI_COMM_WORLD, &rank); + lh_world_rank = rank; // Initialize MPI Functions and Constants mapping table in lower-half initialize_lh_info(); diff --git a/src/lower-half/mem-wrapper.cpp b/src/lower-half/mem-wrapper.cpp index ce6744978..0452d2e86 100644 --- a/src/lower-half/mem-wrapper.cpp +++ b/src/lower-half/mem-wrapper.cpp @@ -499,6 +499,8 @@ void* mmap_wrapper(void *addr, size_t length, int prot, return ret; } +int lh_world_rank = -1; + // The load base of the library open on 'fd', from its executable segment at // file offset 'offset', which is mapped at 'addr': ld.so maps a segment at // the base plus the segment's address, both rounded down to a page, from the @@ -590,6 +592,20 @@ static void* __mmap_wrapper(void *addr, size_t length, int prot, "of: %zu bytes. Error: %s\n", addr, length, strerror(errno)); return NULL; } + // A second libc is in another namespace: that of an LD_AUDIT module + // that needs libc, or one that dlmopen() makes. Its mmap is patched + // like the first one's, but DMTCP's wrappers are only in the + // program's namespace, so code in the other one creates threads, + // files and signal handlers that DMTCP does not see. Only rank 0 + // warns: the ranks normally load the same libraries. + static int libcs_patched = 0; + if (++libcs_patched == 2 && lh_world_rank <= 0) { + fprintf(stderr, + "MANA: warning: the upper half loaded a second libc (%s), " + "for an LD_AUDIT module or dlmopen(). DMTCP does not see " + "the threads, files or signal handlers of code that uses " + "it, so a restart may not restore them.\n", glibcFullPath); + } } } } diff --git a/src/lower-half/mem-wrapper.h b/src/lower-half/mem-wrapper.h index 82a06ba27..e64ce7453 100644 --- a/src/lower-half/mem-wrapper.h +++ b/src/lower-half/mem-wrapper.h @@ -34,4 +34,6 @@ void *mmap_fixed_noreplace(void *addr, size_t length, int prot, int flags, // Records the pid whose exit runs the lower half's exit handlers (uhExit()). void record_lower_half_pid(); std::vector &get_mmapped_list(int *num); +// This process's rank in MPI_COMM_WORLD, or -1 before main() knows it. +extern int lh_world_rank; #endif // MMAP_WRAPPER_H