Fix restart on Cray systems, and world collectives on one node

src/lower-half/lower-half.cpp

@@ -213,6 +213,7 @@ int main(int argc, char *argv[], char *envp[]) {
   remove_dangling_env_entries(initial_argv);
   record_lower_half_pid();
   MPI_Comm_rank(MPI_COMM_WORLD, &rank);
+  lh_world_rank = rank;
 
   // Initialize MPI Functions and Constants mapping table in lower-half
   initialize_lh_info();

src/lower-half/mem-wrapper.cpp

@@ -22,6 +22,7 @@
 #ifndef _GNU_SOURCE
 #define _GNU_SOURCE // For MAP_ANONYMOUS
 #endif
+#include <elf.h>
 #include <errno.h>
 #include <stddef.h>
 #include <assert.h>
@@ -498,9 +499,39 @@ void* mmap_wrapper(void *addr, size_t length, int prot,
   return ret;
 }
 
+int lh_world_rank = -1;
+
+// The load base of the library open on 'fd', from its executable segment at
+// file offset 'offset', which is mapped at 'addr': ld.so maps a segment at
+// the base plus the segment's address, both rounded down to a page, from the
+// segment's file offset rounded down to a page.  NULL if the library has no
+// such segment.
+static char* library_base(int fd, off_t offset, char *addr)
+{
+  const Elf64_Off page_mask = ~((Elf64_Off)PAGE_SIZE - 1);
+  Elf64_Ehdr ehdr;
+  if (pread(fd, &ehdr, sizeof(ehdr), 0) != (ssize_t)sizeof(ehdr) ||
+      memcmp(ehdr.e_ident, ELFMAG, SELFMAG) != 0 ||
+      ehdr.e_ident[EI_CLASS] != ELFCLASS64 ||
+      ehdr.e_phentsize != sizeof(Elf64_Phdr)) {
+    return NULL;
+  }
+  for (int i = 0; i < ehdr.e_phnum; i++) {
+    Elf64_Phdr phdr;
+    if (pread(fd, &phdr, sizeof(phdr), ehdr.e_phoff + i * sizeof(phdr)) !=
+        (ssize_t)sizeof(phdr)) {
+      return NULL;
+    }
+    if (phdr.p_type == PT_LOAD && (phdr.p_flags & PF_X) != 0 &&
+        (phdr.p_offset & page_mask) == (Elf64_Off)offset) {
+      return addr - (phdr.p_vaddr & page_mask);
+    }
+  }
+  return NULL;
+}
+
 static void* __mmap_wrapper(void *addr, size_t length, int prot,
                             int flags, int fd, off_t offset) {
-  static char *libc_base_addr = NULL;
   void *ret = MAP_FAILED;
   length = ROUND_UP(length, PAGE_SIZE);
   bool chosen = false;  // The address is the arena's, not the caller's.
@@ -528,34 +559,52 @@ static void* __mmap_wrapper(void *addr, size_t length, int prot,
                             (size_t)(new_block_end_addr - (char*)ret)};
     merge_overlap_blocks(new_block, allocated_blocks);
     remove_overlap_blocks(new_block, free_blocks);
-    if (fd > 0) {
+    // Patch each libc when ld.so maps its code, at a base that comes from
+    // this mapping alone.  The upper half can load more than one libc (an
+    // LD_AUDIT module's namespace has its own), and other mappings of a libc
+    // file, such as its data or one that only reads the file, can come
+    // between a libc's first mapping and its code.
+    if (fd > 0 && (prot & PROT_EXEC)) {
       char glibcFullPath[PATH_MAX] = {0};
       int found = checkLibrary(fd, "libc.so", glibcFullPath, PATH_MAX) ||
                   checkLibrary(fd, "libc-", glibcFullPath, PATH_MAX);
       if (found) {
-        if (libc_base_addr == NULL) {
-          libc_base_addr = static_cast<char*>(ret);
+        char *libc_base = library_base(fd, offset, (char*)ret);
+        if (libc_base == NULL) {
+          DLOG(ERROR, "No executable segment of %s at file offset 0x%lx\n",
+               glibcFullPath, (long)offset);
         }
-        if (prot & PROT_EXEC) {
-          int rc = mprotect(ret, length, prot | PROT_WRITE);
-          if (rc < 0) {
-            DLOG(ERROR,
-                 "Failed to add PROT_WRITE perms for memory region at: %p "
-                 "of: %zu bytes. Error: %s\n", addr, length, strerror(errno));
-            return NULL;
-          }
-          patchLibc(fd, libc_base_addr, glibcFullPath);
-          off_t sbrk_offset = get_symbol_offset(glibcFullPath, "__sbrk");
-          if (sbrk_offset) {
-            patchSbrk(libc_base_addr + sbrk_offset, (char*)ret, length);
-          }
-          rc = mprotect(ret, length, prot);
-          if (rc < 0) {
-            DLOG(ERROR, "Failed to restore perms for memory region at: %p "
-                "of: %zu bytes. Error: %s\n", addr, length, strerror(errno));
-            return NULL;
-          }
-          libc_base_addr = NULL;
+        int rc = mprotect(ret, length, prot | PROT_WRITE);
+        if (rc < 0) {
+          DLOG(ERROR,
+               "Failed to add PROT_WRITE perms for memory region at: %p "
+               "of: %zu bytes. Error: %s\n", addr, length, strerror(errno));
+          return NULL;
+        }
+        patchLibc(fd, libc_base, glibcFullPath);
+        off_t sbrk_offset = get_symbol_offset(glibcFullPath, "__sbrk");
+        if (sbrk_offset) {
+          patchSbrk(libc_base + sbrk_offset, (char*)ret, length);
+        }
+        rc = mprotect(ret, length, prot);
+        if (rc < 0) {
+          DLOG(ERROR, "Failed to restore perms for memory region at: %p "
+              "of: %zu bytes. Error: %s\n", addr, length, strerror(errno));
+          return NULL;
+        }
+        // A second libc is in another namespace: that of an LD_AUDIT module
+        // that needs libc, or one that dlmopen() makes.  Its mmap is patched
+        // like the first one's, but DMTCP's wrappers are only in the
+        // program's namespace, so code in the other one creates threads,
+        // files and signal handlers that DMTCP does not see.  Only rank 0
+        // warns: the ranks normally load the same libraries.
+        static int libcs_patched = 0;
+        if (++libcs_patched == 2 && lh_world_rank <= 0) {
+          fprintf(stderr,
+                  "MANA: warning: the upper half loaded a second libc (%s), "
+                  "for an LD_AUDIT module or dlmopen().  DMTCP does not see "
+                  "the threads, files or signal handlers of code that uses "
+                  "it, so a restart may not restore them.\n", glibcFullPath);
         }
       }
     }

src/lower-half/mem-wrapper.h

@@ -34,4 +34,6 @@ void *mmap_fixed_noreplace(void *addr, size_t length, int prot, int flags,
 // Records the pid whose exit runs the lower half's exit handlers (uhExit()).
 void record_lower_half_pid();
 std::vector<MmapInfo_t> &get_mmapped_list(int *num);
+// This process's rank in MPI_COMM_WORLD, or -1 before main() knows it.
+extern int lh_world_rank;
 #endif // MMAP_WRAPPER_H

src/mpi-wrappers/Makefile.in

@@ -1,6 +1,8 @@
 # @configure_input@
 
-# The compilers and flags that configure found.
+# The compilers and flags that configure found.  CC is the plain C
+# compiler: make's default, cc, is the compiler wrapper on a Cray system.
+CC = @CC@
 CFLAGS = @CFLAGS@
 CXXFLAGS = @CXXFLAGS@
 MPICC = @MPICC@
@@ -100,10 +102,22 @@ mpi_stub_objects.c: generate-mpi-stub-objects.py
 libmpistub.so: mpi_stub_wrappers.o mpi_stub_data.o mpi_stub_objects.o
 	${CC} -fPIC $^ -shared -o $@
 
-# An LD_AUDIT module for the upper half; it must not need libc.
+# An LD_AUDIT module for the upper half.  It must need no library: ld.so
+# would load it, and the libc that it needs, into the module's namespace,
+# a second libc in the upper half.  A Cray compiler wrapper adds -lxpmem
+# even with -nostdlib, so check the module before installing it, and fail
+# if readelf cannot.
 libmpistub_audit.so: mpi_stub_audit.c
+	tmp=$@.tmp.$$$$ ; \
 	${CC} -O2 -fPIC -shared -nostdlib -fno-stack-protector \
-	  -Wl,--no-undefined -o $@ $<
+	  -Wl,--no-undefined -o $$tmp $< || { rm -f $$tmp; exit 1; } ; \
+	dynamic=$$(readelf -d $$tmp) || { \
+	  echo "cannot check $@ with readelf"; rm -f $$tmp; exit 1; } ; \
+	if echo "$$dynamic" | grep NEEDED; then \
+	  echo "$@ must need no library; link it with a plain C compiler"; \
+	  rm -f $$tmp; exit 1; \
+	fi ; \
+	mv -f $$tmp $@
 
 tidy:
 	rm -f *~ .*.swp dmtcp_restart_script*.sh ckpt_*.dmtcp

src/mpi-wrappers/mpi_cart_wrappers.cpp

@@ -112,11 +112,12 @@ int PMPI_Cart_sub(MPI_Comm comm, const int *remain_dims, MPI_Comm *new_comm)
   commit_begin(comm);
   LOWER_HALF_DISABLE_CKPT();
   MPI_Comm realComm = get_real_id((mana_mpi_handle){.comm = comm}).comm;
+  uint64_t seed = child_gid_seed(comm);
   JUMP_TO_LOWER_HALF(lh_info->fsaddr);
   retval = NEXT_FUNC(Cart_sub)(realComm, remain_dims, new_comm);
   RETURN_TO_UPPER_HALF();
   if (retval == MPI_SUCCESS) {
-    *new_comm = new_virt_comm(*new_comm);
+    *new_comm = new_virt_comm(*new_comm, seed);
   }
   LOWER_HALF_ENABLE_CKPT();
   commit_finish(comm);
@@ -177,12 +178,13 @@ int PMPI_Cart_create(MPI_Comm old_comm, int ndims,
   commit_begin(old_comm);
   LOWER_HALF_DISABLE_CKPT();
   MPI_Comm realComm = get_real_id((mana_mpi_handle){.comm = old_comm}).comm;
+  uint64_t seed = child_gid_seed(old_comm);
   JUMP_TO_LOWER_HALF(lh_info->fsaddr);
   retval = NEXT_FUNC(Cart_create)(realComm, ndims, dims,
                                   periods, reorder, comm_cart);
   RETURN_TO_UPPER_HALF();
   if (retval == MPI_SUCCESS) {
-    *comm_cart = new_virt_comm(*comm_cart);
+    *comm_cart = new_virt_comm(*comm_cart, seed);
   }
   LOWER_HALF_ENABLE_CKPT();
   commit_finish(old_comm);

src/mpi-wrappers/mpi_collective_wrappers.cpp

@@ -803,14 +803,15 @@ int PMPI_Comm_split(MPI_Comm comm, int color, int key, MPI_Comm *newcomm)
   int retval;
   LOWER_HALF_DISABLE_CKPT();
   MPI_Comm real_comm = get_real_id((mana_mpi_handle){.comm = comm}).comm;
+  uint64_t seed = child_gid_seed(comm);
   JUMP_TO_LOWER_HALF(lh_info->fsaddr);
   retval = NEXT_FUNC(Comm_split)(real_comm, color, key, newcomm);
   RETURN_TO_UPPER_HALF();
   if (retval == MPI_SUCCESS) {
     if (*newcomm == lh_info->MANA_COMM_NULL) {
       *newcomm = MPI_COMM_NULL;
     } else {
-      *newcomm = new_virt_comm(*newcomm);
+      *newcomm = new_virt_comm(*newcomm, seed);
     }
   }
   LOWER_HALF_ENABLE_CKPT();
@@ -825,14 +826,15 @@ int PMPI_Comm_dup(MPI_Comm comm, MPI_Comm *newcomm)
   int retval;
   LOWER_HALF_DISABLE_CKPT();
   MPI_Comm real_comm = get_real_id((mana_mpi_handle){.comm = comm}).comm;
+  uint64_t seed = child_gid_seed(comm);
   JUMP_TO_LOWER_HALF(lh_info->fsaddr);
   retval = NEXT_FUNC(Comm_dup)(real_comm, newcomm);
   RETURN_TO_UPPER_HALF();
   if (retval == MPI_SUCCESS) {
     if (*newcomm == lh_info->MANA_COMM_NULL) {
       *newcomm = MPI_COMM_NULL;
     } else {
-      *newcomm = new_virt_comm(*newcomm);
+      *newcomm = new_virt_comm(*newcomm, seed);
     }
   }
   LOWER_HALF_ENABLE_CKPT();

src/mpi-wrappers/mpi_comm_wrappers.cpp

@@ -275,14 +275,15 @@ int PMPI_Comm_create(MPI_Comm comm, MPI_Group group, MPI_Comm *newcomm)
   LOWER_HALF_DISABLE_CKPT();
   MPI_Comm real_comm = get_real_id((mana_mpi_handle){.comm = comm}).comm;
   MPI_Group real_group = get_real_id((mana_mpi_handle){.group = group}).group;
+  uint64_t seed = child_gid_seed(comm);
   JUMP_TO_LOWER_HALF(lh_info->fsaddr);
   retval = NEXT_FUNC(Comm_create)(real_comm, real_group, newcomm);
   RETURN_TO_UPPER_HALF();
   if (retval == MPI_SUCCESS) {
     if (*newcomm == lh_info->MANA_COMM_NULL) {
       *newcomm = MPI_COMM_NULL;
     } else {
-      *newcomm = new_virt_comm(*newcomm);
+      *newcomm = new_virt_comm(*newcomm, seed);
     }
   }
   LOWER_HALF_ENABLE_CKPT();
@@ -547,6 +548,7 @@ int PMPI_Comm_split_type(MPI_Comm comm, int split_type,
   LOWER_HALF_DISABLE_CKPT();
   MPI_Comm real_comm = get_real_id((mana_mpi_handle){.comm = comm}).comm;
   MPI_Info real_info = lower_half_handle(inf);
+  uint64_t seed = child_gid_seed(comm);
   JUMP_TO_LOWER_HALF(lh_info->fsaddr);
   retval = NEXT_FUNC(Comm_split_type)(real_comm, split_type, key, real_info,
                                       newcomm);
@@ -555,7 +557,7 @@ int PMPI_Comm_split_type(MPI_Comm comm, int split_type,
     if (*newcomm == lh_info->MANA_COMM_NULL) {
       *newcomm = MPI_COMM_NULL;
     } else {
-      *newcomm = new_virt_comm(*newcomm);
+      *newcomm = new_virt_comm(*newcomm, seed);
     }
   }
   LOWER_HALF_ENABLE_CKPT();
@@ -690,12 +692,13 @@ int PMPI_Comm_create_group(MPI_Comm comm,
   // The member enters the call in the 1 s pause before the next attempt.
   begin_critical_section(comm, false);
   LOWER_HALF_DISABLE_CKPT();
+  uint64_t seed = group_child_gid_seed(comm, group, tag);
   int retval = MPI_Comm_create_group_internal(comm, group, tag, newcomm);
   if (retval == MPI_SUCCESS) {
     if (*newcomm == lh_info->MANA_COMM_NULL) {
       *newcomm = MPI_COMM_NULL;
     } else {
-      *newcomm = new_virt_comm(*newcomm);
+      *newcomm = new_virt_comm(*newcomm, seed);
     }
   }
   LOWER_HALF_ENABLE_CKPT();

src/p2p_drain.cpp

@@ -174,7 +174,7 @@ initialize_drain_send_recv()
   retval = NEXT_FUNC(Comm_dup)(lh_info->MANA_COMM_WORLD, &ack_comm);
   RETURN_TO_UPPER_HALF();
   ASSERT(retval == MPI_SUCCESS, "retval={}", retval);
-  g_ssend_ack_comm = new_virt_comm(ack_comm);
+  g_ssend_ack_comm = new_virt_comm(ack_comm, mana_gid_seed("ssend-ack"));
 }
 
 // Sums all ranks' sent and received counts in this round's keys.  The next