diff --git a/.github/workflows/mana-tests.yml b/.github/workflows/mana-tests.yml index 098517e9c..c890441d9 100644 --- a/.github/workflows/mana-tests.yml +++ b/.github/workflows/mana-tests.yml @@ -53,14 +53,14 @@ jobs: make -j"$(nproc)" mana tests - name: Test mana_restart wrapper - run: python3 mpi-proxy-split/test/test-mana-restart.py -v + run: python3 test/test-mana-restart.py -v - name: Test loader debug lookup - run: python3 mpi-proxy-split/test/test-loader-debug.py -v + run: python3 test/test-loader-debug.py -v - name: Test MANA run: > - mpi-proxy-split/test/autotest.py --retry-once + test/autotest.py --retry-once --workdir /tmp/autotest --launcher '${{ matrix.launcher }}' - name: Keep the logs of failed tests diff --git a/.gitignore b/.gitignore index 5098bb194..f5ccf104b 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,12 @@ # Made by ./configure and autoreconf. /Makefile /manpages/Makefile +/src/Makefile +/src/lower-half/Makefile +/src/mpi-wrappers/Makefile +/test/Makefile +/examples/Makefile +/bench/Makefile /config.log /config.status /aclocal.m4 @@ -12,6 +18,10 @@ /bin/ /lib/ /manpages/mana.1.gz +/src/lower-half/lower-half +/examples/ring +/bench/virtid_bench +/bench/synthetic_app *.[oa] *.d *.so @@ -21,6 +31,24 @@ *.pyc __pycache__/ +# Made by src/mpi-wrappers/Makefile from the generate-*.py scripts. +/src/mpi-wrappers/mpi_fortran_wrappers.cpp +/src/mpi-wrappers/mpi_unimplemented_wrappers.cpp +/src/mpi-wrappers/mpi_stub_wrappers.c +/src/mpi-wrappers/mpi_stub_objects.c +/src/mpi-wrappers/*.tmp.* + +# The programs of the end-to-end tests have no suffix: +# ignore all but the sources. +/test/* +!/test/*.c +!/test/*.cpp +!/test/*.f90 +!/test/*.h +!/test/*.py +!/test/*.md +!/test/Makefile.in + # Left by runs in the tree. ckpt_*.dmtcp ckpt_rank_*/ diff --git a/Makefile.in b/Makefile.in index 00780747b..49ec5abcc 100644 --- a/Makefile.in +++ b/Makefile.in @@ -37,7 +37,7 @@ all: default MANA_SCRIPTS = mana_coordinator mana_launch mana_restart mana_status mana: mana_prereqs dmtcp $(MANA_SCRIPTS:%=bin/%) - cd mpi-proxy-split && $(MAKE) stage + cd src && $(MAKE) stage # bin/ holds the scripts next to DMTCP's programs and the lower half, which # the scripts run. @@ -47,11 +47,13 @@ $(MANA_SCRIPTS:%=bin/%): bin/%: scripts/% # The end-to-end tests, the example and the benchmarks. tests: - cd mpi-proxy-split && $(MAKE) tests + cd test && $(MAKE) + cd examples && $(MAKE) + cd bench && $(MAKE) # Builds the end-to-end tests and runs them (test/autotest.py). check: mana - cd mpi-proxy-split && $(MAKE) check + cd test && $(MAKE) check dmtcp: dmtcp/configure @ # A 'git pull' can change the DMTCP commit that MANA uses, and it @@ -89,12 +91,11 @@ uninstall: distclean: clean - cd ${DMTCP_ROOT} && $(MAKE) distclean - - cd mpi-proxy-split && $(MAKE) distclean + - cd src && $(MAKE) distclean - cd manpages && $(MAKE) distclean rm -f Makefile config.log config.status config.status-* config.cache - rm -f mpi-proxy-split/Makefile mpi-proxy-split/lower-half/Makefile \ - mpi-proxy-split/mpi-wrappers/Makefile mpi-proxy-split/test/Makefile \ - mpi-proxy-split/examples/Makefile + rm -f src/Makefile src/lower-half/Makefile src/mpi-wrappers/Makefile \ + test/Makefile examples/Makefile bench/Makefile rm -rf autom4te.cache rm -rf $(top_builddir)/lib $(top_builddir)/bin @@ -104,7 +105,7 @@ tidy: rm -f dmtcp_coordinator_db-*.json clean: tidy - cd mpi-proxy-split && $(MAKE) clean + cd src && $(MAKE) clean mana_prereqs: @ if test -n "$$HUGETLB_DEFAULT_PAGE_SIZE"; then \ diff --git a/README.md b/README.md index 6e0890c18..a7d05ce0f 100644 --- a/README.md +++ b/README.md @@ -63,8 +63,8 @@ CI does. To run MANA's tests, which checkpoint and restart each test program: make tests # builds them; 'make' does not - mpi-proxy-split/test/autotest.py # or: make check - mpi-proxy-split/test/autotest.py --help # e.g. --launcher 'srun -n {n}' + test/autotest.py # or: make check + test/autotest.py --help # e.g. --launcher 'srun -n {n}' MANA is free software: you can redistribute it and/or modify it under the terms of the GNU Lesser General Public License, version 3 or (at your option) diff --git a/bench/Makefile.in b/bench/Makefile.in new file mode 100644 index 000000000..011af5204 --- /dev/null +++ b/bench/Makefile.in @@ -0,0 +1,23 @@ +# @configure_input@ + +# The compilers and flags that configure found. +MPICC = @MPICC@ +MPI_CFLAGS?= @CFLAGS@ -g3 -fPIC +MPI_LDFLAGS?= + +# Benchmarks of MANA's overhead; built with -O2 and not run by 'make check'. +BENCH_FILES = virtid_bench synthetic_app + +default: ${BENCH_FILES} + +${BENCH_FILES}: %: %.c + $(MPICC) -g -O2 -o $@ $< $(MPI_CFLAGS) ${MPI_LDFLAGS} + +clean: + rm -f ${BENCH_FILES} + +.PHONY: default clean + +# Regenerate this Makefile when Makefile.in or config.status is newer. +Makefile: Makefile.in @top_builddir@/config.status + cd @top_builddir@ && ./config.status bench/Makefile diff --git a/mpi-proxy-split/test/README-synthetic_app.md b/bench/README-synthetic_app.md similarity index 85% rename from mpi-proxy-split/test/README-synthetic_app.md rename to bench/README-synthetic_app.md index 79ffdb992..10f811c8f 100644 --- a/mpi-proxy-split/test/README-synthetic_app.md +++ b/bench/README-synthetic_app.md @@ -23,10 +23,10 @@ same options. ## Usage ``` -make -C mpi-proxy-split/test synthetic_app -mpirun -np 8 mpi-proxy-split/test/synthetic_app --profile=vasp +make -C bench synthetic_app +mpirun -np 8 bench/synthetic_app --profile=vasp bin/mana_coordinator -mpirun -np 8 bin/mana_launch mpi-proxy-split/test/synthetic_app --profile=vasp +mpirun -np 8 bin/mana_launch bench/synthetic_app --profile=vasp ``` The options are listed at the top of `synthetic_app.c`. diff --git a/mpi-proxy-split/test/synthetic_app.c b/bench/synthetic_app.c similarity index 100% rename from mpi-proxy-split/test/synthetic_app.c rename to bench/synthetic_app.c diff --git a/mpi-proxy-split/test/virtid_bench.c b/bench/virtid_bench.c similarity index 100% rename from mpi-proxy-split/test/virtid_bench.c rename to bench/virtid_bench.c diff --git a/configure b/configure index 4ce5b24b5..721326b6d 100755 --- a/configure +++ b/configure @@ -5500,7 +5500,7 @@ ac_configure_args="$ac_configure_args --disable-dlsym-wrapper" subdirs="$subdirs dmtcp" -ac_config_files="$ac_config_files Makefile manpages/Makefile mpi-proxy-split/Makefile mpi-proxy-split/lower-half/Makefile mpi-proxy-split/mpi-wrappers/Makefile mpi-proxy-split/test/Makefile mpi-proxy-split/examples/Makefile" +ac_config_files="$ac_config_files Makefile manpages/Makefile src/Makefile src/lower-half/Makefile src/mpi-wrappers/Makefile test/Makefile examples/Makefile bench/Makefile" enable_option_checking=fatal @@ -6298,11 +6298,12 @@ do case $ac_config_target in "Makefile") CONFIG_FILES="$CONFIG_FILES Makefile" ;; "manpages/Makefile") CONFIG_FILES="$CONFIG_FILES manpages/Makefile" ;; - "mpi-proxy-split/Makefile") CONFIG_FILES="$CONFIG_FILES mpi-proxy-split/Makefile" ;; - "mpi-proxy-split/lower-half/Makefile") CONFIG_FILES="$CONFIG_FILES mpi-proxy-split/lower-half/Makefile" ;; - "mpi-proxy-split/mpi-wrappers/Makefile") CONFIG_FILES="$CONFIG_FILES mpi-proxy-split/mpi-wrappers/Makefile" ;; - "mpi-proxy-split/test/Makefile") CONFIG_FILES="$CONFIG_FILES mpi-proxy-split/test/Makefile" ;; - "mpi-proxy-split/examples/Makefile") CONFIG_FILES="$CONFIG_FILES mpi-proxy-split/examples/Makefile" ;; + "src/Makefile") CONFIG_FILES="$CONFIG_FILES src/Makefile" ;; + "src/lower-half/Makefile") CONFIG_FILES="$CONFIG_FILES src/lower-half/Makefile" ;; + "src/mpi-wrappers/Makefile") CONFIG_FILES="$CONFIG_FILES src/mpi-wrappers/Makefile" ;; + "test/Makefile") CONFIG_FILES="$CONFIG_FILES test/Makefile" ;; + "examples/Makefile") CONFIG_FILES="$CONFIG_FILES examples/Makefile" ;; + "bench/Makefile") CONFIG_FILES="$CONFIG_FILES bench/Makefile" ;; *) as_fn_error $? "invalid argument: \`$ac_config_target'" "$LINENO" 5;; esac diff --git a/configure.ac b/configure.ac index 583a264d2..399c4b027 100644 --- a/configure.ac +++ b/configure.ac @@ -80,11 +80,12 @@ AC_CONFIG_SUBDIRS([dmtcp]) AC_CONFIG_FILES([Makefile \ manpages/Makefile \ - mpi-proxy-split/Makefile \ - mpi-proxy-split/lower-half/Makefile \ - mpi-proxy-split/mpi-wrappers/Makefile \ - mpi-proxy-split/test/Makefile \ - mpi-proxy-split/examples/Makefile]) + src/Makefile \ + src/lower-half/Makefile \ + src/mpi-wrappers/Makefile \ + test/Makefile \ + examples/Makefile \ + bench/Makefile]) dnl Autoconf manual says option checking is set to warn ("yes") by dnl by default. But it's actually set to "no". diff --git a/mpi-proxy-split/examples/Makefile.in b/examples/Makefile.in similarity index 85% rename from mpi-proxy-split/examples/Makefile.in rename to examples/Makefile.in index 6cfc71430..440cf8dbb 100644 --- a/mpi-proxy-split/examples/Makefile.in +++ b/examples/Makefile.in @@ -18,4 +18,4 @@ clean: # Regenerate this Makefile when Makefile.in or config.status is newer. Makefile: Makefile.in @top_builddir@/config.status - cd @top_builddir@ && ./config.status mpi-proxy-split/examples/Makefile + cd @top_builddir@ && ./config.status examples/Makefile diff --git a/mpi-proxy-split/examples/ring.c b/examples/ring.c similarity index 98% rename from mpi-proxy-split/examples/ring.c rename to examples/ring.c index 122f5838e..dec1607dc 100644 --- a/mpi-proxy-split/examples/ring.c +++ b/examples/ring.c @@ -34,7 +34,7 @@ // run it under MANA, from MANA's directory: // // bin/mana_coordinator -// mpirun -np 4 bin/mana_launch mpi-proxy-split/examples/ring +// mpirun -np 4 bin/mana_launch examples/ring // bin/mana_status --checkpoint (in another shell, while it runs) // // Then stop the job (Ctrl-C), and restart it from the checkpoint: diff --git a/manpages/mana.1 b/manpages/mana.1 index 4d89855ee..8e5820908 100644 --- a/manpages/mana.1 +++ b/manpages/mana.1 @@ -67,10 +67,10 @@ cd dmtcp-mana \&./configure --enable-debug make -j mana tests # Build the program with mpicc as usual; no relinking for MANA is needed. -# 'make tests' also builds an example: mpi-proxy-split/examples/ring +# 'make tests' also builds an example: examples/ring salloc -N 2 -q interactive -C haswell -t 01:00:00 bin/mana_coordinator -i10 -srun -N 2 bin/mana_launch mpi-proxy-split/examples/ring +srun -N 2 bin/mana_launch examples/ring bin/mana_coordinator -i10 srun -N 2 bin/mana_restart \f[R] diff --git a/manpages/mana.1.md b/manpages/mana.1.md index 5e6b129f4..b405946ce 100644 --- a/manpages/mana.1.md +++ b/manpages/mana.1.md @@ -58,10 +58,10 @@ cd dmtcp-mana ./configure --enable-debug make -j mana tests # Build the program with mpicc as usual; no relinking for MANA is needed. -# 'make tests' also builds an example: mpi-proxy-split/examples/ring +# 'make tests' also builds an example: examples/ring salloc -N 2 -q interactive -C haswell -t 01:00:00 bin/mana_coordinator -i10 -srun -N 2 bin/mana_launch mpi-proxy-split/examples/ring +srun -N 2 bin/mana_launch examples/ring bin/mana_coordinator -i10 srun -N 2 bin/mana_restart ``` diff --git a/mpi-proxy-split/.gitignore b/mpi-proxy-split/.gitignore deleted file mode 100644 index ca60bb9e4..000000000 --- a/mpi-proxy-split/.gitignore +++ /dev/null @@ -1,25 +0,0 @@ -# The programs of the end-to-end tests and the benchmarks have no suffix: -# ignore all but the sources. -test/* -!test/*.c -!test/*.cpp -!test/*.f90 -!test/*.h -!test/*.py -!test/*.md -!test/Makefile.in -examples/ring -lower-half/lower-half - -# Made by configure from the Makefile.in files. -/Makefile -/lower-half/Makefile -/mpi-wrappers/Makefile -/examples/Makefile - -# Made by mpi-wrappers/Makefile from the generate-*.py scripts. -mpi-wrappers/mpi_fortran_wrappers.cpp -mpi-wrappers/mpi_unimplemented_wrappers.cpp -mpi-wrappers/mpi_stub_wrappers.c -mpi-wrappers/mpi_stub_objects.c -mpi-wrappers/*.tmp.* diff --git a/mpi-proxy-split/p2p_log_replay.cpp b/mpi-proxy-split/p2p_log_replay.cpp deleted file mode 100644 index ab323efd0..000000000 --- a/mpi-proxy-split/p2p_log_replay.cpp +++ /dev/null @@ -1,295 +0,0 @@ -/**************************************************************************** - * Copyright (C) 2019-2021 by Gene Cooperman, Rohan Garg, Yao Xu * - * gene@ccs.neu.edu, rohgarg@ccs.neu.edu, xu.yao1@northeastern.edu * - * * - * This file is part of DMTCP. * - * * - * DMTCP is free software: you can redistribute it and/or * - * modify it under the terms of the GNU Lesser General Public License as * - * published by the Free Software Foundation, either version 3 of the * - * License, or (at your option) any later version. * - * * - * DMTCP is distributed in the hope that it will be useful, * - * but WITHOUT ANY WARRANTY; without even the implied warranty of * - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * - * GNU Lesser General Public License for more details. * - * * - * You should have received a copy of the GNU Lesser General Public * - * License in the files COPYING and COPYING.LESSER. If not, see * - * . * - ****************************************************************************/ - -#include -#include -#include -#include -#include -#include -#include -#include - -#include "dmtcp.h" -#include "util.h" -#include "dmtcp_assert.h" -#include "jfilesystem.h" - -#include "mpi_plugin.h" -#include "mpi_nextfunc.h" -#include "p2p_log_replay.h" -#include "p2p_drain_send_recv.h" -#include "virtual_id.h" - -using namespace dmtcp; - -int g_world_rank = -1; // Global rank of the current process -int g_world_size = -1; // Total number of ranks in the current computation - -// The pending MPI_Isend/MPI_Irecv calls, stored in their requests' -// virtual-ID entries and linked in posting order. Application threads and -// the checkpoint thread (P2P drain, restart) both change the list. The -// spinlock pendingLock guards the links and call types; never hold it across -// an MPI call. -static virt_id_entry *pendingHead = NULL; -static virt_id_entry *pendingTail = NULL; -static int pendingLock = 0; -// The pending MPI_Issends: changed under pendingLock, read without it. -static int pendingIssends = 0; - -static inline void -lockPending() -{ - while (__atomic_exchange_n(&pendingLock, 1, __ATOMIC_ACQUIRE)) { - while (__atomic_load_n(&pendingLock, __ATOMIC_RELAXED)) { - } - } -} - -static inline void -unlockPending() -{ - __atomic_store_n(&pendingLock, 0, __ATOMIC_RELEASE); -} - -void -getLocalRankInfo() -{ - if (g_world_rank == -1) { - ASSERT(MPI_Comm_rank(MPI_COMM_WORLD, &g_world_rank) == MPI_SUCCESS && - g_world_rank != -1, "MPI_Comm_rank failed"); - } - if (g_world_size == -1) { - ASSERT(MPI_Comm_size(MPI_COMM_WORLD, &g_world_size) == MPI_SUCCESS && - g_world_size != -1, "MPI_Comm_size failed"); - } -} - -void -updateCkptDirByRank() -{ - const char *ckptDir = dmtcp_get_ckpt_dir(); - dmtcp::string baseDir; - - if (strstr(ckptDir, "ckpt_rank_") != NULL) { - baseDir = jalib::Filesystem::DirName(ckptDir); - } else { - baseDir = ckptDir; - } - TRACE("Updating checkpoint directory: ckptDir={} baseDir={}", ckptDir, - baseDir); - dmtcp::ostringstream o; - o << baseDir << "/ckpt_rank_" << g_world_rank; - dmtcp_set_ckpt_dir(o.str().c_str()); -} - -void -addPendingRequestToLog(mpi_req_t req, const void* sbuf, void* rbuf, int cnt, - MPI_Datatype type, int remote, int tag, - MPI_Comm comm, MPI_Request rq) -{ - virt_id_entry *entry = get_virt_id_entry((mana_mpi_handle){.request = rq}); - mpi_nonblocking_call_t *call = &entry->call; - call->sendbuf = sbuf; - call->recvbuf = rbuf; - call->count = cnt; - call->datatype = type; - call->remote_node = remote; - call->tag = tag; - call->comm = comm; - call->later_sends = 0; - lockPending(); - call->type = req; - if (req == ISSEND_REQUEST) { - __atomic_fetch_add(&pendingIssends, 1, __ATOMIC_RELAXED); - } - entry->pending_prev = pendingTail; - entry->pending_next = NULL; - if (pendingTail != NULL) { - pendingTail->pending_next = entry; - } else { - pendingHead = entry; - } - pendingTail = entry; - unlockPending(); -} - -void -clearPendingRequestFromLog(MPI_Request req) -{ - // Look the request up under the lock: while the P2P drain clears it, the - // application may free it and a new request may take its slot. - lockPending(); - virt_id_entry *entry = - lookup_virt_id_entry((mana_mpi_handle){.request = req}); - if (entry == NULL) { - unlockPending(); - return; - } - if (entry->call.type != UNKNOW_REQUEST) { - if (entry->call.type == ISSEND_REQUEST) { - __atomic_fetch_sub(&pendingIssends, 1, __ATOMIC_RELAXED); - } - if (entry->pending_prev != NULL) { - entry->pending_prev->pending_next = entry->pending_next; - } else { - pendingHead = entry->pending_next; - } - if (entry->pending_next != NULL) { - entry->pending_next->pending_prev = entry->pending_prev; - } else { - pendingTail = entry->pending_prev; - } - entry->pending_prev = NULL; - entry->pending_next = NULL; - entry->call.type = UNKNOW_REQUEST; - } - unlockPending(); -} - -void -replacePendingCall(MPI_Request req, const mpi_nonblocking_call_t *call) -{ - lockPending(); - virt_id_entry *entry = - lookup_virt_id_entry((mana_mpi_handle){.request = req}); - ASSERT(entry != NULL && entry->call.type != UNKNOW_REQUEST, "req={}", req); - if (entry->call.type == ISSEND_REQUEST && call->type != ISSEND_REQUEST) { - __atomic_fetch_sub(&pendingIssends, 1, __ATOMIC_RELAXED); - } - entry->call = *call; - unlockPending(); -} - -void -countSendAfterIssends(MPI_Comm comm, int dest, int tag) -{ - if (__atomic_load_n(&pendingIssends, __ATOMIC_RELAXED) == 0) { - return; - } - lockPending(); - for (virt_id_entry *entry = pendingHead; entry != NULL; - entry = entry->pending_next) { - mpi_nonblocking_call_t *call = &entry->call; - if (call->type == ISSEND_REQUEST && call->comm == comm && - call->remote_node == dest && call->tag == tag) { - call->later_sends++; - } - } - unlockPending(); -} - -std::vector -pendingRequestsInPostingOrder() -{ - std::vector requests; - lockPending(); - for (virt_id_entry *entry = pendingHead; entry != NULL; - entry = entry->pending_next) { - requests.push_back((MPI_Request)entry->virt); - } - unlockPending(); - return requests; -} - -mpi_req_t -pendingRequestType(MPI_Request req) -{ - virt_id_entry *entry = - lookup_virt_id_entry((mana_mpi_handle){.request = req}); - if (entry == NULL) { - return UNKNOW_REQUEST; - } - return __atomic_load_n(&entry->call.type, __ATOMIC_RELAXED); -} - -bool -getPendingCall(MPI_Request req, mpi_nonblocking_call_t *call) -{ - virt_id_entry *entry = - lookup_virt_id_entry((mana_mpi_handle){.request = req}); - if (entry == NULL) { - return false; - } - lockPending(); - *call = entry->call; - unlockPending(); - return call->type != UNKNOW_REQUEST; -} - -bool -pendingCallUsesDatatype(MPI_Datatype type) -{ - bool found = false; - lockPending(); - for (virt_id_entry *entry = pendingHead; entry != NULL; - entry = entry->pending_next) { - if (entry->call.datatype == type) { - found = true; - break; - } - } - unlockPending(); - return found; -} - -void -replayMpiP2pOnRestart() -{ - MPI_Request request; - mpi_nonblocking_call_t pendingCall; - mpi_nonblocking_call_t *call = &pendingCall; - TRACE("Replaying unserviced isend/irecv calls"); - - // No other thread runs at restart; a lock saved in the image is stale. - pendingLock = 0; - // Re-post the receives in the order they were posted. - for (MPI_Request pending : pendingRequestsInPostingOrder()) { - int retval = 0; - request = pending; - if (!getPendingCall(request, call)) { - continue; - } - MPI_Comm realComm = get_real_id((mana_mpi_handle){.comm = call->comm}).comm; - MPI_Datatype realType = get_real_id((mana_mpi_handle){.datatype = call->datatype}).datatype; - MPI_Request realRequest; - switch (call->type) { - case IRECV_REQUEST: - TRACE("Replaying Irecv call: remote_node={}", call->remote_node); - JUMP_TO_LOWER_HALF(lh_info->fsaddr); - NEXT_FUNC(Irecv)(call->recvbuf, call->count, - realType, call->remote_node, - call->tag, realComm, &realRequest); - RETURN_TO_UPPER_HALF(); - update_virt_id((mana_mpi_handle){.request = request}, (mana_mpi_handle){.request = realRequest}); - ASSERT(retval == MPI_SUCCESS, "Error while replaying recv"); - break; - case ISEND_REQUEST: - case ISSEND_REQUEST: - ASSERT(false, "There should be no pending MPI_Isend after restart: " - "type={}", call->type); - break; - default: - WARN(false, "Unhandled replay call: type={}", call->type); - break; - } - } -} diff --git a/mpi-proxy-split/p2p_log_replay.h b/mpi-proxy-split/p2p_log_replay.h deleted file mode 100644 index c3a76cc97..000000000 --- a/mpi-proxy-split/p2p_log_replay.h +++ /dev/null @@ -1,95 +0,0 @@ -/**************************************************************************** - * Copyright (C) 2019-2021 by Gene Cooperman, Rohan Garg, Yao Xu * - * gene@ccs.neu.edu, rohgarg@ccs.neu.edu, xu.yao1@northeastern.edu * - * * - * This file is part of DMTCP. * - * * - * DMTCP is free software: you can redistribute it and/or * - * modify it under the terms of the GNU Lesser General Public License as * - * published by the Free Software Foundation, either version 3 of the * - * License, or (at your option) any later version. * - * * - * DMTCP is distributed in the hope that it will be useful, * - * but WITHOUT ANY WARRANTY; without even the implied warranty of * - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * - * GNU Lesser General Public License for more details. * - * * - * You should have received a copy of the GNU Lesser General Public * - * License in the files COPYING and COPYING.LESSER. If not, see * - * . * - ****************************************************************************/ - -#ifndef _P2P_COMM_H -#define _P2P_COMM_H - -#include -#include -#include -#include "dmtcp.h" -#include "dmtcpalloc.h" -#include "virtual_id.h" - -// Struct to store and return the MPI message (data) during draining and -// resuming, also used by p2p_drain_send_recv.h -typedef struct __mpi_message -{ - void *buf; - int count; - MPI_Datatype datatype; - int size; - MPI_Comm comm; - MPI_Status status; - // An MPI_Issend's: its sender's world rank, to ack when it is received, - // and the ack's tag (see publishSsendNotices()). Otherwise ack_to is -1. - int ack_to; - int ack_tag; -} mpi_message_t; - - -extern int g_world_rank; // Global rank of the current process -extern int g_world_size; // Total number of ranks in the current computation - -// Fetches the MPI rank and world size; also, verifies that MPI rank and -// world size match the globally stored values in the plugin -extern void getLocalRankInfo(); - -// Sets the name of the checkpoint directory of the current process to -// "ckpt_rank_", where RANK is the MPI rank of the process. -extern void updateCkptDirByRank(); - -// Restores the state of MPI P2P communication by replaying any pending -// MPI_Isend and MPI_Irecv requests post restart -extern void replayMpiP2pOnRestart(); - -// Saves a nonblocking send/recv call in the virtual-ID entry of request 'rq'. -extern void addPendingRequestToLog(mpi_req_t , const void* , void* , int , - MPI_Datatype , int , int , - MPI_Comm, MPI_Request); - -// remove finished send/recv call from the pending calls -extern void clearPendingRequestFromLog(MPI_Request req); - -// Replaces the call of a pending request; it keeps its place in the log. -extern void replacePendingCall(MPI_Request req, - const mpi_nonblocking_call_t *call); - -// Counts a send in the later_sends of each pending MPI_Issend with its -// envelope. Call it before the send enters the lower half. -extern void countSendAfterIssends(MPI_Comm comm, int dest, int tag); - -// Returns the pending requests in posting order (not handle order). MPI -// matches receives in that order, so the drain and restart replay use it. -extern std::vector pendingRequestsInPostingOrder(); - -// Returns the type of a pending request, or UNKNOW_REQUEST if the request is -// not (or no longer) pending. -extern mpi_req_t pendingRequestType(MPI_Request req); - -// Copies the call of a pending request to *call. Returns false if the -// request is not (or no longer) pending. -extern bool getPendingCall(MPI_Request req, mpi_nonblocking_call_t *call); - -// Returns true if a pending MPI_Isend/MPI_Irecv uses the datatype. -extern bool pendingCallUsesDatatype(MPI_Datatype type); - -#endif // ifndef _P2P_LOG_REPLAY_H diff --git a/mpi-proxy-split/Makefile.in b/src/Makefile.in similarity index 92% rename from mpi-proxy-split/Makefile.in rename to src/Makefile.in index 05703b1da..91cd18588 100644 --- a/mpi-proxy-split/Makefile.in +++ b/src/Makefile.in @@ -22,7 +22,7 @@ WRAPPERS_SRCDIR=mpi-wrappers # As you add new files to your plugin library, add the object file names here. -LIBOBJS = uh_wrappers.o mpi_plugin.o p2p_drain_send_recv.o p2p_log_replay.o \ +LIBOBJS = uh_wrappers.o mpi_plugin.o p2p_drain.o \ seq_num.o virtual_id.o lower_half_ckpt.o \ lower-half/switch-context.o \ @@ -75,14 +75,6 @@ lower-half/switch-context.o: FORCE -include $(wildcard *.d) -tests: - @$(MAKE) -C test/ default - @$(MAKE) -C examples/ - -check: libmana.so - @$(MAKE) -C test/ e2e - @test/autotest.py - ${MANA_ROOT}/lib/dmtcp/libmana.so: libmana.so cp -f $< $@ ${MANA_ROOT}/lib/dmtcp/libmpistub.so: ${WRAPPERS_SRCDIR}/mpi_stub_wrappers.txt \ @@ -117,8 +109,8 @@ distclean: clean FORCE: -.PHONY: default clean distclean tidy check tests stage +.PHONY: default clean distclean tidy stage # Regenerate this Makefile when Makefile.in or config.status is newer. Makefile: Makefile.in @top_builddir@/config.status - cd @top_builddir@ && ./config.status mpi-proxy-split/Makefile + cd @top_builddir@ && ./config.status src/Makefile diff --git a/mpi-proxy-split/lower-half/Makefile.in b/src/lower-half/Makefile.in similarity index 95% rename from mpi-proxy-split/lower-half/Makefile.in rename to src/lower-half/Makefile.in index acbb94f5c..2a8cfe6d1 100644 --- a/mpi-proxy-split/lower-half/Makefile.in +++ b/src/lower-half/Makefile.in @@ -15,7 +15,7 @@ endif DMTCP_INCLUDE=${DMTCP_ROOT}/include JALIB_INCLUDE=${DMTCP_ROOT}/jalib -MANA_INCLUDE=${MANA_ROOT}/mpi-proxy-split +MANA_INCLUDE=${MANA_ROOT}/src DMTCP_DEPS = ${DMTCP_ROOT}/src/libdmtcprestart.a \ ${DMTCP_ROOT}/src/libdmtcpinternal.a \ @@ -74,4 +74,4 @@ distclean: clean # Regenerate this Makefile when Makefile.in or config.status is newer. Makefile: Makefile.in @top_builddir@/config.status - cd @top_builddir@ && ./config.status mpi-proxy-split/lower-half/Makefile + cd @top_builddir@ && ./config.status src/lower-half/Makefile diff --git a/mpi-proxy-split/lower-half/README b/src/lower-half/README similarity index 100% rename from mpi-proxy-split/lower-half/README rename to src/lower-half/README diff --git a/mpi-proxy-split/lower-half/build-id-debug.cpp b/src/lower-half/build-id-debug.cpp similarity index 100% rename from mpi-proxy-split/lower-half/build-id-debug.cpp rename to src/lower-half/build-id-debug.cpp diff --git a/mpi-proxy-split/lower-half/build-id-debug.h b/src/lower-half/build-id-debug.h similarity index 100% rename from mpi-proxy-split/lower-half/build-id-debug.h rename to src/lower-half/build-id-debug.h diff --git a/mpi-proxy-split/lower-half/copy-stack.c b/src/lower-half/copy-stack.c similarity index 100% rename from mpi-proxy-split/lower-half/copy-stack.c rename to src/lower-half/copy-stack.c diff --git a/mpi-proxy-split/lower-half/get-symbol-offset.c b/src/lower-half/get-symbol-offset.c similarity index 100% rename from mpi-proxy-split/lower-half/get-symbol-offset.c rename to src/lower-half/get-symbol-offset.c diff --git a/mpi-proxy-split/lower-half/lh-func-ptr.c b/src/lower-half/lh-func-ptr.c similarity index 100% rename from mpi-proxy-split/lower-half/lh-func-ptr.c rename to src/lower-half/lh-func-ptr.c diff --git a/mpi-proxy-split/lower-half/logging.h b/src/lower-half/logging.h similarity index 100% rename from mpi-proxy-split/lower-half/logging.h rename to src/lower-half/logging.h diff --git a/mpi-proxy-split/lower-half/lower-half-api.h b/src/lower-half/lower-half-api.h similarity index 100% rename from mpi-proxy-split/lower-half/lower-half-api.h rename to src/lower-half/lower-half-api.h diff --git a/mpi-proxy-split/lower-half/lower-half.cpp b/src/lower-half/lower-half.cpp similarity index 100% rename from mpi-proxy-split/lower-half/lower-half.cpp rename to src/lower-half/lower-half.cpp diff --git a/mpi-proxy-split/lower-half/mem-wrapper.cpp b/src/lower-half/mem-wrapper.cpp similarity index 100% rename from mpi-proxy-split/lower-half/mem-wrapper.cpp rename to src/lower-half/mem-wrapper.cpp diff --git a/mpi-proxy-split/lower-half/mem-wrapper.h b/src/lower-half/mem-wrapper.h similarity index 100% rename from mpi-proxy-split/lower-half/mem-wrapper.h rename to src/lower-half/mem-wrapper.h diff --git a/mpi-proxy-split/lower-half/mmap-fixed-noreplace.c b/src/lower-half/mmap-fixed-noreplace.c similarity index 100% rename from mpi-proxy-split/lower-half/mmap-fixed-noreplace.c rename to src/lower-half/mmap-fixed-noreplace.c diff --git a/mpi-proxy-split/lower-half/patch-trampoline.c b/src/lower-half/patch-trampoline.c similarity index 100% rename from mpi-proxy-split/lower-half/patch-trampoline.c rename to src/lower-half/patch-trampoline.c diff --git a/mpi-proxy-split/lower-half/patch-trampoline.h b/src/lower-half/patch-trampoline.h similarity index 100% rename from mpi-proxy-split/lower-half/patch-trampoline.h rename to src/lower-half/patch-trampoline.h diff --git a/mpi-proxy-split/lower-half/switch-context.cpp b/src/lower-half/switch-context.cpp similarity index 100% rename from mpi-proxy-split/lower-half/switch-context.cpp rename to src/lower-half/switch-context.cpp diff --git a/mpi-proxy-split/lower-half/switch-context.h b/src/lower-half/switch-context.h similarity index 100% rename from mpi-proxy-split/lower-half/switch-context.h rename to src/lower-half/switch-context.h diff --git a/mpi-proxy-split/lower_half_ckpt.cpp b/src/lower_half_ckpt.cpp similarity index 100% rename from mpi-proxy-split/lower_half_ckpt.cpp rename to src/lower_half_ckpt.cpp diff --git a/mpi-proxy-split/lower_half_ckpt.h b/src/lower_half_ckpt.h similarity index 100% rename from mpi-proxy-split/lower_half_ckpt.h rename to src/lower_half_ckpt.h diff --git a/mpi-proxy-split/mpi-wrappers/Makefile.in b/src/mpi-wrappers/Makefile.in similarity index 98% rename from mpi-proxy-split/mpi-wrappers/Makefile.in rename to src/mpi-wrappers/Makefile.in index 4725dadf3..2df71093c 100644 --- a/mpi-proxy-split/mpi-wrappers/Makefile.in +++ b/src/mpi-wrappers/Makefile.in @@ -130,4 +130,4 @@ distclean: clean # Regenerate this Makefile when Makefile.in or config.status is newer. Makefile: Makefile.in @top_builddir@/config.status - cd @top_builddir@ && ./config.status mpi-proxy-split/mpi-wrappers/Makefile + cd @top_builddir@ && ./config.status src/mpi-wrappers/Makefile diff --git a/mpi-proxy-split/mpi-wrappers/fortran_constants.f90 b/src/mpi-wrappers/fortran_constants.f90 similarity index 100% rename from mpi-proxy-split/mpi-wrappers/fortran_constants.f90 rename to src/mpi-wrappers/fortran_constants.f90 diff --git a/mpi-proxy-split/mpi-wrappers/fortran_handles.h b/src/mpi-wrappers/fortran_handles.h similarity index 100% rename from mpi-proxy-split/mpi-wrappers/fortran_handles.h rename to src/mpi-wrappers/fortran_handles.h diff --git a/mpi-proxy-split/mpi-wrappers/generate-mpi-fortran-wrappers.py b/src/mpi-wrappers/generate-mpi-fortran-wrappers.py similarity index 100% rename from mpi-proxy-split/mpi-wrappers/generate-mpi-fortran-wrappers.py rename to src/mpi-wrappers/generate-mpi-fortran-wrappers.py diff --git a/mpi-proxy-split/mpi-wrappers/generate-mpi-stub-objects.py b/src/mpi-wrappers/generate-mpi-stub-objects.py similarity index 100% rename from mpi-proxy-split/mpi-wrappers/generate-mpi-stub-objects.py rename to src/mpi-wrappers/generate-mpi-stub-objects.py diff --git a/mpi-proxy-split/mpi-wrappers/generate-mpi-stub-wrappers.py b/src/mpi-wrappers/generate-mpi-stub-wrappers.py similarity index 100% rename from mpi-proxy-split/mpi-wrappers/generate-mpi-stub-wrappers.py rename to src/mpi-wrappers/generate-mpi-stub-wrappers.py diff --git a/mpi-proxy-split/mpi-wrappers/generate-mpi-unimplemented-wrappers.py b/src/mpi-wrappers/generate-mpi-unimplemented-wrappers.py similarity index 100% rename from mpi-proxy-split/mpi-wrappers/generate-mpi-unimplemented-wrappers.py rename to src/mpi-wrappers/generate-mpi-unimplemented-wrappers.py diff --git a/mpi-proxy-split/mpi-wrappers/get_fortran_constants.c b/src/mpi-wrappers/get_fortran_constants.c similarity index 100% rename from mpi-proxy-split/mpi-wrappers/get_fortran_constants.c rename to src/mpi-wrappers/get_fortran_constants.c diff --git a/mpi-proxy-split/mpi-wrappers/mpi_cart_wrappers.cpp b/src/mpi-wrappers/mpi_cart_wrappers.cpp similarity index 99% rename from mpi-proxy-split/mpi-wrappers/mpi_cart_wrappers.cpp rename to src/mpi-wrappers/mpi_cart_wrappers.cpp index 394f04112..d6949f71d 100644 --- a/mpi-proxy-split/mpi-wrappers/mpi_cart_wrappers.cpp +++ b/src/mpi-wrappers/mpi_cart_wrappers.cpp @@ -31,7 +31,7 @@ #include "mpi_nextfunc.h" #include "virtual_id.h" #include "seq_num.h" -#include "p2p_drain_send_recv.h" +#include "p2p_drain.h" extern "C" { diff --git a/mpi-proxy-split/mpi-wrappers/mpi_collective_wrappers.cpp b/src/mpi-wrappers/mpi_collective_wrappers.cpp similarity index 99% rename from mpi-proxy-split/mpi-wrappers/mpi_collective_wrappers.cpp rename to src/mpi-wrappers/mpi_collective_wrappers.cpp index 0ef70138f..19d3eee28 100644 --- a/mpi-proxy-split/mpi-wrappers/mpi_collective_wrappers.cpp +++ b/src/mpi-wrappers/mpi_collective_wrappers.cpp @@ -31,8 +31,7 @@ #include "mpi_nextfunc.h" #include "seq_num.h" #include "virtual_id.h" -#include "p2p_log_replay.h" -#include "p2p_drain_send_recv.h" +#include "p2p_drain.h" extern "C" { diff --git a/mpi-proxy-split/mpi-wrappers/mpi_comm_wrappers.cpp b/src/mpi-wrappers/mpi_comm_wrappers.cpp similarity index 99% rename from mpi-proxy-split/mpi-wrappers/mpi_comm_wrappers.cpp rename to src/mpi-wrappers/mpi_comm_wrappers.cpp index 9a2ecd93c..d4cfd2cbc 100644 --- a/mpi-proxy-split/mpi-wrappers/mpi_comm_wrappers.cpp +++ b/src/mpi-wrappers/mpi_comm_wrappers.cpp @@ -38,7 +38,7 @@ #include "mpi_nextfunc.h" #include "virtual_id.h" #include "seq_num.h" -#include "p2p_drain_send_recv.h" +#include "p2p_drain.h" // TODO // - validate operation status (right now we assume them to be successful by diff --git a/mpi-proxy-split/mpi-wrappers/mpi_error_wrappers.cpp b/src/mpi-wrappers/mpi_error_wrappers.cpp similarity index 100% rename from mpi-proxy-split/mpi-wrappers/mpi_error_wrappers.cpp rename to src/mpi-wrappers/mpi_error_wrappers.cpp diff --git a/mpi-proxy-split/mpi-wrappers/mpi_file_wrappers.cpp b/src/mpi-wrappers/mpi_file_wrappers.cpp similarity index 100% rename from mpi-proxy-split/mpi-wrappers/mpi_file_wrappers.cpp rename to src/mpi-wrappers/mpi_file_wrappers.cpp diff --git a/mpi-proxy-split/mpi-wrappers/mpi_fortran_attr_wrappers.cpp b/src/mpi-wrappers/mpi_fortran_attr_wrappers.cpp similarity index 100% rename from mpi-proxy-split/mpi-wrappers/mpi_fortran_attr_wrappers.cpp rename to src/mpi-wrappers/mpi_fortran_attr_wrappers.cpp diff --git a/mpi-proxy-split/mpi-wrappers/mpi_fortran_wrappers.txt b/src/mpi-wrappers/mpi_fortran_wrappers.txt similarity index 100% rename from mpi-proxy-split/mpi-wrappers/mpi_fortran_wrappers.txt rename to src/mpi-wrappers/mpi_fortran_wrappers.txt diff --git a/mpi-proxy-split/mpi-wrappers/mpi_group_wrappers.cpp b/src/mpi-wrappers/mpi_group_wrappers.cpp similarity index 100% rename from mpi-proxy-split/mpi-wrappers/mpi_group_wrappers.cpp rename to src/mpi-wrappers/mpi_group_wrappers.cpp diff --git a/mpi-proxy-split/mpi-wrappers/mpi_nextfunc.h b/src/mpi-wrappers/mpi_nextfunc.h similarity index 100% rename from mpi-proxy-split/mpi-wrappers/mpi_nextfunc.h rename to src/mpi-wrappers/mpi_nextfunc.h diff --git a/mpi-proxy-split/mpi-wrappers/mpi_op_wrappers.cpp b/src/mpi-wrappers/mpi_op_wrappers.cpp similarity index 100% rename from mpi-proxy-split/mpi-wrappers/mpi_op_wrappers.cpp rename to src/mpi-wrappers/mpi_op_wrappers.cpp diff --git a/mpi-proxy-split/mpi-wrappers/mpi_p2p_wrappers.cpp b/src/mpi-wrappers/mpi_p2p_wrappers.cpp similarity index 99% rename from mpi-proxy-split/mpi-wrappers/mpi_p2p_wrappers.cpp rename to src/mpi-wrappers/mpi_p2p_wrappers.cpp index cb31cbbbf..c1e03ae00 100644 --- a/mpi-proxy-split/mpi-wrappers/mpi_p2p_wrappers.cpp +++ b/src/mpi-wrappers/mpi_p2p_wrappers.cpp @@ -27,8 +27,7 @@ #include "lower_half_ckpt.h" #include "mpi_plugin.h" -#include "p2p_log_replay.h" -#include "p2p_drain_send_recv.h" +#include "p2p_drain.h" #include "jfilesystem.h" #include "protectedfds.h" #include "mpi_nextfunc.h" diff --git a/mpi-proxy-split/mpi-wrappers/mpi_request_wrappers.cpp b/src/mpi-wrappers/mpi_request_wrappers.cpp similarity index 99% rename from mpi-proxy-split/mpi-wrappers/mpi_request_wrappers.cpp rename to src/mpi-wrappers/mpi_request_wrappers.cpp index af17e1076..96a9aaa08 100644 --- a/mpi-proxy-split/mpi-wrappers/mpi_request_wrappers.cpp +++ b/src/mpi-wrappers/mpi_request_wrappers.cpp @@ -30,8 +30,7 @@ #include "jfilesystem.h" #include "protectedfds.h" -#include "p2p_log_replay.h" -#include "p2p_drain_send_recv.h" +#include "p2p_drain.h" #include "mpi_plugin.h" #include "mpi_nextfunc.h" #include "virtual_id.h" diff --git a/mpi-proxy-split/mpi-wrappers/mpi_stub_audit.c b/src/mpi-wrappers/mpi_stub_audit.c similarity index 100% rename from mpi-proxy-split/mpi-wrappers/mpi_stub_audit.c rename to src/mpi-wrappers/mpi_stub_audit.c diff --git a/mpi-proxy-split/mpi-wrappers/mpi_stub_data.c b/src/mpi-wrappers/mpi_stub_data.c similarity index 100% rename from mpi-proxy-split/mpi-wrappers/mpi_stub_data.c rename to src/mpi-wrappers/mpi_stub_data.c diff --git a/mpi-proxy-split/mpi-wrappers/mpi_stub_wrappers.txt b/src/mpi-wrappers/mpi_stub_wrappers.txt similarity index 100% rename from mpi-proxy-split/mpi-wrappers/mpi_stub_wrappers.txt rename to src/mpi-wrappers/mpi_stub_wrappers.txt diff --git a/mpi-proxy-split/mpi-wrappers/mpi_t_wrappers.cpp b/src/mpi-wrappers/mpi_t_wrappers.cpp similarity index 100% rename from mpi-proxy-split/mpi-wrappers/mpi_t_wrappers.cpp rename to src/mpi-wrappers/mpi_t_wrappers.cpp diff --git a/mpi-proxy-split/mpi-wrappers/mpi_type_wrappers.cpp b/src/mpi-wrappers/mpi_type_wrappers.cpp similarity index 100% rename from mpi-proxy-split/mpi-wrappers/mpi_type_wrappers.cpp rename to src/mpi-wrappers/mpi_type_wrappers.cpp diff --git a/mpi-proxy-split/mpi-wrappers/mpi_unimplemented_wrappers.txt b/src/mpi-wrappers/mpi_unimplemented_wrappers.txt similarity index 100% rename from mpi-proxy-split/mpi-wrappers/mpi_unimplemented_wrappers.txt rename to src/mpi-wrappers/mpi_unimplemented_wrappers.txt diff --git a/mpi-proxy-split/mpi-wrappers/mpi_win_wrappers.cpp b/src/mpi-wrappers/mpi_win_wrappers.cpp similarity index 100% rename from mpi-proxy-split/mpi-wrappers/mpi_win_wrappers.cpp rename to src/mpi-wrappers/mpi_win_wrappers.cpp diff --git a/mpi-proxy-split/mpi-wrappers/mpi_wrappers.cpp b/src/mpi-wrappers/mpi_wrappers.cpp similarity index 98% rename from mpi-proxy-split/mpi-wrappers/mpi_wrappers.cpp rename to src/mpi-wrappers/mpi_wrappers.cpp index 536104832..a821e9e25 100644 --- a/mpi-proxy-split/mpi-wrappers/mpi_wrappers.cpp +++ b/src/mpi-wrappers/mpi_wrappers.cpp @@ -31,7 +31,7 @@ #include "protectedfds.h" #include "mpi_nextfunc.h" #include "virtual_id.h" -#include "p2p_drain_send_recv.h" +#include "p2p_drain.h" #include "seq_num.h" #include "uh_wrappers.h" @@ -244,9 +244,9 @@ int PMPI_Get_address(const void *location, MPI_Aint *address) // FOR DEBUGGING ONLY: // This defines a call to MPI_MANA_Internal in the lower half, which // is especially useful in debugging restart. It is called -// from mpi-proxy-split/mpi_plugin.cpp, just before replaying the pending +// from src/mpi_plugin.cpp, just before replaying the pending // receives. -// In mpi-proxy-split/lower-half, redefine MPI_MANA_Internal() +// In src/lower-half, redefine MPI_MANA_Internal() // to do whatever is desired. Then do: // rm bin/lh_proxy // module list # Verify the correct 'module' settings diff --git a/mpi-proxy-split/mpi_files.h b/src/mpi_files.h similarity index 100% rename from mpi-proxy-split/mpi_files.h rename to src/mpi_files.h diff --git a/mpi-proxy-split/mpi_plugin.cpp b/src/mpi_plugin.cpp similarity index 96% rename from mpi-proxy-split/mpi_plugin.cpp rename to src/mpi_plugin.cpp index eacd951ec..b89f4576f 100644 --- a/mpi-proxy-split/mpi_plugin.cpp +++ b/src/mpi_plugin.cpp @@ -42,8 +42,7 @@ #include "mpi_plugin.h" #include "lower-half-api.h" #include "lower_half_ckpt.h" -#include "p2p_log_replay.h" -#include "p2p_drain_send_recv.h" +#include "p2p_drain.h" #include "seq_num.h" #include "mpi_nextfunc.h" #include "virtual_id.h" @@ -83,6 +82,9 @@ EXTERNC pid_t dmtcp_get_real_tid() __attribute((weak)); mana_state_t mana_state = UNKNOWN_STATE; +int g_world_rank = -1; // Global rank of the current process +int g_world_size = -1; // Total number of ranks in the current computation + ProcSelfMaps *preMpiInitMaps = nullptr; ProcSelfMaps *postMpiInitMaps = nullptr; map *mpiInitLhAreas = nullptr; @@ -744,6 +746,37 @@ close_lower_half_except_blocked() g_drain_stats.t_wait_lower_half += drainStatsNow() - t; } +void +getLocalRankInfo() +{ + if (g_world_rank == -1) { + ASSERT(MPI_Comm_rank(MPI_COMM_WORLD, &g_world_rank) == MPI_SUCCESS && + g_world_rank != -1, "MPI_Comm_rank failed"); + } + if (g_world_size == -1) { + ASSERT(MPI_Comm_size(MPI_COMM_WORLD, &g_world_size) == MPI_SUCCESS && + g_world_size != -1, "MPI_Comm_size failed"); + } +} + +void +updateCkptDirByRank() +{ + const char *ckptDir = dmtcp_get_ckpt_dir(); + dmtcp::string baseDir; + + if (strstr(ckptDir, "ckpt_rank_") != NULL) { + baseDir = jalib::Filesystem::DirName(ckptDir); + } else { + baseDir = ckptDir; + } + TRACE("Updating checkpoint directory: ckptDir={} baseDir={}", ckptDir, + baseDir); + dmtcp::ostringstream o; + o << baseDir << "/ckpt_rank_" << g_world_rank; + dmtcp_set_ckpt_dir(o.str().c_str()); +} + static void mpi_plugin_event_hook(DmtcpEvent_t event, DmtcpEventData_t *data) { @@ -843,7 +876,7 @@ mpi_plugin_event_hook(DmtcpEvent_t event, DmtcpEventData_t *data) while (!__atomic_load_n(&g_libmpi_is_initialized, __ATOMIC_ACQUIRE)) { usleep(1000); } - resetDrainStats(); // p2p_drain_send_recv.cpp + resetDrainStats(); // p2p_drain.cpp uint64_t t0 = drainStatsNow(); mana_state = CKPT_COLLECTIVE; // preSuspendBarrier() will send coord response and get worker state. @@ -871,7 +904,7 @@ mpi_plugin_event_hook(DmtcpEvent_t event, DmtcpEventData_t *data) if (own_tls) { lh_info->ckpt_uh_fs = (void *)getFS(); } - drainP2p(); // p2p_drain_send_recv.cpp + drainP2p(); // p2p_drain.cpp lh_info->ckpt_uh_fs = NULL; openCkptFileFds(); if (g_p2p_wait == P2P_WAIT_BLOCKING) { @@ -898,7 +931,7 @@ mpi_plugin_event_hook(DmtcpEvent_t event, DmtcpEventData_t *data) allow_threads_to_enter_lower_half(); recordOpenFds(); dmtcp_local_barrier("MPI:GetLocalRankInfo"); - getLocalRankInfo(); // p2p_log_replay.cpp + getLocalRankInfo(); dmtcp_global_barrier("MPI:update-ckpt-dir-by-rank"); updateCkptDirByRank(); // mpi_plugin.cpp // computeUnionOfCkptImageAddresses(); @@ -917,10 +950,10 @@ mpi_plugin_event_hook(DmtcpEvent_t event, DmtcpEventData_t *data) printEventToStderr("EVENT_RESUME"); processingOpenCkpFileFds = false; dmtcp_local_barrier("MPI:Reset-Drain-Send-Recv-Counters"); - resetDrainCounters(); // p2p_drain_send_recv.cpp + resetDrainCounters(); // p2p_drain.cpp seq_num_reset(); dmtcp_local_barrier("MPI:seq_num_reset"); - sendOwedSsendAcks(); // p2p_drain_send_recv.cpp + sendOwedSsendAcks(); // p2p_drain.cpp mana_state = RUNNING; printEventToStderr("EVENT_RESUME (done)"); break; @@ -947,14 +980,14 @@ mpi_plugin_event_hook(DmtcpEvent_t event, DmtcpEventData_t *data) check_tag_ub(); reconstruct_descriptors(); dmtcp_local_barrier("MPI:Reset-Drain-Send-Recv-Counters"); - resetDrainCounters(); // p2p_drain_send_recv.cpp + resetDrainCounters(); // p2p_drain.cpp char str[1]; MPI_MANA_Internal(str); // This does nothing. Modify in lower-half // for easy debugging of lower half during restart. // See definition in mpi-wrappers/mpi_wrappers.cpp dmtcp_global_barrier("MPI:replayMpiP2pOnRestart"); - replayMpiP2pOnRestart(); // p2p_log_replay.cpp - sendOwedSsendAcks(); // p2p_drain_send_recv.cpp + replayMpiP2pOnRestart(); // p2p_drain.cpp + sendOwedSsendAcks(); // p2p_drain.cpp dmtcp_local_barrier("MPI:p2p_log_replay.cpp-void"); const char *file = get_mpi_file_filename(); restore_mpi_files(file); diff --git a/mpi-proxy-split/mpi_plugin.h b/src/mpi_plugin.h similarity index 82% rename from mpi-proxy-split/mpi_plugin.h rename to src/mpi_plugin.h index c6387ed85..ab46dc1aa 100644 --- a/mpi-proxy-split/mpi_plugin.h +++ b/src/mpi_plugin.h @@ -48,4 +48,15 @@ extern mana_state_t mana_state; extern bool g_libmana_is_initialized; extern bool g_libmpi_is_initialized; // MANA's MPI_Init has returned +extern int g_world_rank; // Global rank of the current process +extern int g_world_size; // Total number of ranks in the current computation + +// Fetches the MPI rank and world size; also, verifies that MPI rank and +// world size match the globally stored values in the plugin +extern void getLocalRankInfo(); + +// Sets the name of the checkpoint directory of the current process to +// "ckpt_rank_", where RANK is the MPI rank of the process. +extern void updateCkptDirByRank(); + #endif // ifndef _MPI_PLUGIN_H diff --git a/mpi-proxy-split/p2p_drain_send_recv.cpp b/src/p2p_drain.cpp similarity index 87% rename from mpi-proxy-split/p2p_drain_send_recv.cpp rename to src/p2p_drain.cpp index a29535ee5..650712610 100644 --- a/mpi-proxy-split/p2p_drain_send_recv.cpp +++ b/src/p2p_drain.cpp @@ -30,8 +30,8 @@ #include #include "kvdb.h" #include "dmtcp_assert.h" -#include "p2p_drain_send_recv.h" -#include "p2p_log_replay.h" +#include "p2p_drain.h" +#include "mpi_plugin.h" #include "mpi_nextfunc.h" #include "seq_num.h" #include "virtual_id.h" @@ -69,7 +69,7 @@ unlockMessageQueue() __atomic_clear(&g_message_queue_lock, __ATOMIC_RELEASE); } -// See p2p_drain_send_recv.h for documentation of these globals. +// See p2p_drain.h for documentation of these globals. pending_recv_t g_pending_recv = { /*.state=*/ PENDING_RECV_IDLE }; MPI_Comm g_ssend_ack_comm = MPI_COMM_NULL; volatile bool p2p_dummy_phase = false; @@ -1187,3 +1187,222 @@ reportDrainStats() v["isends_completed"], v["blocked"], v["dummies"], v["issends"], v["issends_buffered"], v["kvdb_requests"], v["done_polls"]); } + +// The pending MPI_Isend/MPI_Irecv calls, stored in their requests' +// virtual-ID entries and linked in posting order. Application threads and +// the checkpoint thread (P2P drain, restart) both change the list. The +// spinlock pendingLock guards the links and call types; never hold it across +// an MPI call. +static virt_id_entry *pendingHead = NULL; +static virt_id_entry *pendingTail = NULL; +static int pendingLock = 0; +// The pending MPI_Issends: changed under pendingLock, read without it. +static int pendingIssends = 0; + +static inline void +lockPending() +{ + while (__atomic_exchange_n(&pendingLock, 1, __ATOMIC_ACQUIRE)) { + while (__atomic_load_n(&pendingLock, __ATOMIC_RELAXED)) { + } + } +} + +static inline void +unlockPending() +{ + __atomic_store_n(&pendingLock, 0, __ATOMIC_RELEASE); +} + +void +addPendingRequestToLog(mpi_req_t req, const void* sbuf, void* rbuf, int cnt, + MPI_Datatype type, int remote, int tag, + MPI_Comm comm, MPI_Request rq) +{ + virt_id_entry *entry = get_virt_id_entry((mana_mpi_handle){.request = rq}); + mpi_nonblocking_call_t *call = &entry->call; + call->sendbuf = sbuf; + call->recvbuf = rbuf; + call->count = cnt; + call->datatype = type; + call->remote_node = remote; + call->tag = tag; + call->comm = comm; + call->later_sends = 0; + lockPending(); + call->type = req; + if (req == ISSEND_REQUEST) { + __atomic_fetch_add(&pendingIssends, 1, __ATOMIC_RELAXED); + } + entry->pending_prev = pendingTail; + entry->pending_next = NULL; + if (pendingTail != NULL) { + pendingTail->pending_next = entry; + } else { + pendingHead = entry; + } + pendingTail = entry; + unlockPending(); +} + +void +clearPendingRequestFromLog(MPI_Request req) +{ + // Look the request up under the lock: while the P2P drain clears it, the + // application may free it and a new request may take its slot. + lockPending(); + virt_id_entry *entry = + lookup_virt_id_entry((mana_mpi_handle){.request = req}); + if (entry == NULL) { + unlockPending(); + return; + } + if (entry->call.type != UNKNOW_REQUEST) { + if (entry->call.type == ISSEND_REQUEST) { + __atomic_fetch_sub(&pendingIssends, 1, __ATOMIC_RELAXED); + } + if (entry->pending_prev != NULL) { + entry->pending_prev->pending_next = entry->pending_next; + } else { + pendingHead = entry->pending_next; + } + if (entry->pending_next != NULL) { + entry->pending_next->pending_prev = entry->pending_prev; + } else { + pendingTail = entry->pending_prev; + } + entry->pending_prev = NULL; + entry->pending_next = NULL; + entry->call.type = UNKNOW_REQUEST; + } + unlockPending(); +} + +void +replacePendingCall(MPI_Request req, const mpi_nonblocking_call_t *call) +{ + lockPending(); + virt_id_entry *entry = + lookup_virt_id_entry((mana_mpi_handle){.request = req}); + ASSERT(entry != NULL && entry->call.type != UNKNOW_REQUEST, "req={}", req); + if (entry->call.type == ISSEND_REQUEST && call->type != ISSEND_REQUEST) { + __atomic_fetch_sub(&pendingIssends, 1, __ATOMIC_RELAXED); + } + entry->call = *call; + unlockPending(); +} + +void +countSendAfterIssends(MPI_Comm comm, int dest, int tag) +{ + if (__atomic_load_n(&pendingIssends, __ATOMIC_RELAXED) == 0) { + return; + } + lockPending(); + for (virt_id_entry *entry = pendingHead; entry != NULL; + entry = entry->pending_next) { + mpi_nonblocking_call_t *call = &entry->call; + if (call->type == ISSEND_REQUEST && call->comm == comm && + call->remote_node == dest && call->tag == tag) { + call->later_sends++; + } + } + unlockPending(); +} + +std::vector +pendingRequestsInPostingOrder() +{ + std::vector requests; + lockPending(); + for (virt_id_entry *entry = pendingHead; entry != NULL; + entry = entry->pending_next) { + requests.push_back((MPI_Request)entry->virt); + } + unlockPending(); + return requests; +} + +mpi_req_t +pendingRequestType(MPI_Request req) +{ + virt_id_entry *entry = + lookup_virt_id_entry((mana_mpi_handle){.request = req}); + if (entry == NULL) { + return UNKNOW_REQUEST; + } + return __atomic_load_n(&entry->call.type, __ATOMIC_RELAXED); +} + +bool +getPendingCall(MPI_Request req, mpi_nonblocking_call_t *call) +{ + virt_id_entry *entry = + lookup_virt_id_entry((mana_mpi_handle){.request = req}); + if (entry == NULL) { + return false; + } + lockPending(); + *call = entry->call; + unlockPending(); + return call->type != UNKNOW_REQUEST; +} + +bool +pendingCallUsesDatatype(MPI_Datatype type) +{ + bool found = false; + lockPending(); + for (virt_id_entry *entry = pendingHead; entry != NULL; + entry = entry->pending_next) { + if (entry->call.datatype == type) { + found = true; + break; + } + } + unlockPending(); + return found; +} + +void +replayMpiP2pOnRestart() +{ + MPI_Request request; + mpi_nonblocking_call_t pendingCall; + mpi_nonblocking_call_t *call = &pendingCall; + TRACE("Replaying unserviced isend/irecv calls"); + + // No other thread runs at restart; a lock saved in the image is stale. + pendingLock = 0; + // Re-post the receives in the order they were posted. + for (MPI_Request pending : pendingRequestsInPostingOrder()) { + int retval = 0; + request = pending; + if (!getPendingCall(request, call)) { + continue; + } + MPI_Comm realComm = get_real_id((mana_mpi_handle){.comm = call->comm}).comm; + MPI_Datatype realType = get_real_id((mana_mpi_handle){.datatype = call->datatype}).datatype; + MPI_Request realRequest; + switch (call->type) { + case IRECV_REQUEST: + TRACE("Replaying Irecv call: remote_node={}", call->remote_node); + JUMP_TO_LOWER_HALF(lh_info->fsaddr); + NEXT_FUNC(Irecv)(call->recvbuf, call->count, + realType, call->remote_node, + call->tag, realComm, &realRequest); + RETURN_TO_UPPER_HALF(); + update_virt_id((mana_mpi_handle){.request = request}, (mana_mpi_handle){.request = realRequest}); + ASSERT(retval == MPI_SUCCESS, "Error while replaying recv"); + break; + case ISEND_REQUEST: + case ISSEND_REQUEST: + ASSERT(false, "There should be no pending MPI_Isend after restart: " + "type={}", call->type); + break; + default: + WARN(false, "Unhandled replay call: type={}", call->type); + break; + } + } +} diff --git a/mpi-proxy-split/p2p_drain_send_recv.h b/src/p2p_drain.h similarity index 81% rename from mpi-proxy-split/p2p_drain_send_recv.h rename to src/p2p_drain.h index afc952bd4..9bb35e72d 100644 --- a/mpi-proxy-split/p2p_drain_send_recv.h +++ b/src/p2p_drain.h @@ -22,10 +22,29 @@ #ifndef _P2P_SEND_RECV_H #define _P2P_SEND_RECV_H +#include +#include #include +#include #include "dmtcp.h" #include "dmtcpalloc.h" -#include "p2p_log_replay.h" +#include "virtual_id.h" + +// Struct to store and return the MPI message (data) during draining and +// resuming +typedef struct __mpi_message +{ + void *buf; + int count; + MPI_Datatype datatype; + int size; + MPI_Comm comm; + MPI_Status status; + // An MPI_Issend's: its sender's world rank, to ack when it is received, + // and the ack's tag (see publishSsendNotices()). Otherwise ack_to is -1. + int ack_to; + int ack_tag; +} mpi_message_t; extern int64_t global_sent_messages, global_recv_messages; extern int64_t local_sent_messages, local_recv_messages; @@ -180,4 +199,39 @@ int consumeMatchingMsgBuffer(void *buf, int count, MPI_Datatype datatype, int source, int tag, MPI_Comm comm, MPI_Status *mpi_status, int size); void resetDrainCounters(); + +// Restores the state of MPI P2P communication by replaying any pending +// MPI_Isend and MPI_Irecv requests post restart +extern void replayMpiP2pOnRestart(); + +// Saves a nonblocking send/recv call in the virtual-ID entry of request 'rq'. +extern void addPendingRequestToLog(mpi_req_t , const void* , void* , int , + MPI_Datatype , int , int , + MPI_Comm, MPI_Request); + +// remove finished send/recv call from the pending calls +extern void clearPendingRequestFromLog(MPI_Request req); + +// Replaces the call of a pending request; it keeps its place in the log. +extern void replacePendingCall(MPI_Request req, + const mpi_nonblocking_call_t *call); + +// Counts a send in the later_sends of each pending MPI_Issend with its +// envelope. Call it before the send enters the lower half. +extern void countSendAfterIssends(MPI_Comm comm, int dest, int tag); + +// Returns the pending requests in posting order (not handle order). MPI +// matches receives in that order, so the drain and restart replay use it. +extern std::vector pendingRequestsInPostingOrder(); + +// Returns the type of a pending request, or UNKNOW_REQUEST if the request is +// not (or no longer) pending. +extern mpi_req_t pendingRequestType(MPI_Request req); + +// Copies the call of a pending request to *call. Returns false if the +// request is not (or no longer) pending. +extern bool getPendingCall(MPI_Request req, mpi_nonblocking_call_t *call); + +// Returns true if a pending MPI_Isend/MPI_Irecv uses the datatype. +extern bool pendingCallUsesDatatype(MPI_Datatype type); #endif diff --git a/mpi-proxy-split/seq_num.cpp b/src/seq_num.cpp similarity index 100% rename from mpi-proxy-split/seq_num.cpp rename to src/seq_num.cpp diff --git a/mpi-proxy-split/seq_num.h b/src/seq_num.h similarity index 100% rename from mpi-proxy-split/seq_num.h rename to src/seq_num.h diff --git a/mpi-proxy-split/uh_wrappers.cpp b/src/uh_wrappers.cpp similarity index 100% rename from mpi-proxy-split/uh_wrappers.cpp rename to src/uh_wrappers.cpp diff --git a/mpi-proxy-split/uh_wrappers.h b/src/uh_wrappers.h similarity index 100% rename from mpi-proxy-split/uh_wrappers.h rename to src/uh_wrappers.h diff --git a/mpi-proxy-split/virtual_id.cpp b/src/virtual_id.cpp similarity index 99% rename from mpi-proxy-split/virtual_id.cpp rename to src/virtual_id.cpp index 14facf063..37101a0a4 100644 --- a/mpi-proxy-split/virtual_id.cpp +++ b/src/virtual_id.cpp @@ -10,7 +10,8 @@ #include "switch-context.h" #include "mpi_nextfunc.h" #include "seq_num.h" -#include "p2p_log_replay.h" +#include "mpi_plugin.h" +#include "p2p_drain.h" #include MPI_Group g_world_group; diff --git a/mpi-proxy-split/virtual_id.h b/src/virtual_id.h similarity index 99% rename from mpi-proxy-split/virtual_id.h rename to src/virtual_id.h index ded74c0c1..ba42e8aec 100644 --- a/mpi-proxy-split/virtual_id.h +++ b/src/virtual_id.h @@ -147,7 +147,7 @@ typedef struct virt_id_entry { int next_free; // Next slot in the free list, if the slot is free unsigned int gen; // Generation of the next handle that uses this slot // For a pending MPI_Isend/MPI_Irecv: the call, linked in posting order - // (see p2p_log_replay.cpp). Otherwise, call.type is UNKNOW_REQUEST. + // (see p2p_drain.cpp). Otherwise, call.type is UNKNOW_REQUEST. mpi_nonblocking_call_t call; struct virt_id_entry *pending_prev; struct virt_id_entry *pending_next; diff --git a/mpi-proxy-split/test/Makefile.in b/test/Makefile.in similarity index 89% rename from mpi-proxy-split/test/Makefile.in rename to test/Makefile.in index 0cd48d519..f5a417005 100644 --- a/mpi-proxy-split/test/Makefile.in +++ b/test/Makefile.in @@ -13,11 +13,8 @@ MPI_LDFLAGS?= # https://people.math.sc.edu/Burkardt/f_src/mpi/mpi.html ############################################################### -# Benchmarks of MANA's overhead; built with -O2 and not run by 'make check'. -BENCH_FILES = virtid_bench synthetic_app - default: default_prereqs - + $(MAKE) ${BENCH_FILES} e2e + + $(MAKE) e2e # End-to-end tests, run by autotest.py natively and under MANA. Each test is # one program, named after the test and linked as any MPI program. @@ -51,9 +48,6 @@ dlopen_mpi: dlopen_mpi.c ${E2E_FORTRAN}: %: %.f90 $(MPIFC) ${FFLAGS} -g3 -O0 -o $@ $< -${BENCH_FILES}: %: %.c - $(MPICC) -g -O2 -o $@ $< $(MPI_CFLAGS) ${MPI_LDFLAGS} - check: e2e ./autotest.py @@ -81,8 +75,8 @@ tidy: clean: tidy rm -f *.pyc *.so - rm -f ${E2E_TESTS} ${E2E_FORTRAN} ${BENCH_FILES} + rm -f ${E2E_TESTS} ${E2E_FORTRAN} # Regenerate this Makefile when Makefile.in or config.status is newer. Makefile: Makefile.in @top_builddir@/config.status - cd @top_builddir@ && ./config.status mpi-proxy-split/test/Makefile + cd @top_builddir@ && ./config.status test/Makefile diff --git a/mpi-proxy-split/test/abort.c b/test/abort.c similarity index 100% rename from mpi-proxy-split/test/abort.c rename to test/abort.c diff --git a/mpi-proxy-split/test/attributes.c b/test/attributes.c similarity index 100% rename from mpi-proxy-split/test/attributes.c rename to test/attributes.c diff --git a/mpi-proxy-split/test/autotest.py b/test/autotest.py similarity index 99% rename from mpi-proxy-split/test/autotest.py rename to test/autotest.py index 4a0e88577..47dea0b16 100755 --- a/mpi-proxy-split/test/autotest.py +++ b/test/autotest.py @@ -25,7 +25,7 @@ import time TEST_DIR = os.path.dirname(os.path.abspath(__file__)) -MANA_ROOT = os.path.abspath(os.path.join(TEST_DIR, "..", "..")) +MANA_ROOT = os.path.abspath(os.path.join(TEST_DIR, "..")) # In the arguments of a test, stands for the MPI library by its full path. diff --git a/mpi-proxy-split/test/cartesian.c b/test/cartesian.c similarity index 100% rename from mpi-proxy-split/test/cartesian.c rename to test/cartesian.c diff --git a/mpi-proxy-split/test/collectives.c b/test/collectives.c similarity index 100% rename from mpi-proxy-split/test/collectives.c rename to test/collectives.c diff --git a/mpi-proxy-split/test/comm_create_group.c b/test/comm_create_group.c similarity index 100% rename from mpi-proxy-split/test/comm_create_group.c rename to test/comm_create_group.c diff --git a/mpi-proxy-split/test/comm_free.c b/test/comm_free.c similarity index 100% rename from mpi-proxy-split/test/comm_free.c rename to test/comm_free.c diff --git a/mpi-proxy-split/test/comm_free_order.c b/test/comm_free_order.c similarity index 100% rename from mpi-proxy-split/test/comm_free_order.c rename to test/comm_free_order.c diff --git a/mpi-proxy-split/test/comm_self.c b/test/comm_self.c similarity index 100% rename from mpi-proxy-split/test/comm_self.c rename to test/comm_self.c diff --git a/mpi-proxy-split/test/communicators.c b/test/communicators.c similarity index 100% rename from mpi-proxy-split/test/communicators.c rename to test/communicators.c diff --git a/mpi-proxy-split/test/datatypes.c b/test/datatypes.c similarity index 100% rename from mpi-proxy-split/test/datatypes.c rename to test/datatypes.c diff --git a/mpi-proxy-split/test/dlopen_mpi.c b/test/dlopen_mpi.c similarity index 100% rename from mpi-proxy-split/test/dlopen_mpi.c rename to test/dlopen_mpi.c diff --git a/mpi-proxy-split/test/exit_after_finalize.c b/test/exit_after_finalize.c similarity index 100% rename from mpi-proxy-split/test/exit_after_finalize.c rename to test/exit_after_finalize.c diff --git a/mpi-proxy-split/test/finalize_unsync.c b/test/finalize_unsync.c similarity index 100% rename from mpi-proxy-split/test/finalize_unsync.c rename to test/finalize_unsync.c diff --git a/mpi-proxy-split/test/fortran_attributes.f90 b/test/fortran_attributes.f90 similarity index 100% rename from mpi-proxy-split/test/fortran_attributes.f90 rename to test/fortran_attributes.f90 diff --git a/mpi-proxy-split/test/fortran_handles.f90 b/test/fortran_handles.f90 similarity index 100% rename from mpi-proxy-split/test/fortran_handles.f90 rename to test/fortran_handles.f90 diff --git a/mpi-proxy-split/test/fortran_nonblocking.f90 b/test/fortran_nonblocking.f90 similarity index 100% rename from mpi-proxy-split/test/fortran_nonblocking.f90 rename to test/fortran_nonblocking.f90 diff --git a/mpi-proxy-split/test/fortran_p2p.f90 b/test/fortran_p2p.f90 similarity index 100% rename from mpi-proxy-split/test/fortran_p2p.f90 rename to test/fortran_p2p.f90 diff --git a/mpi-proxy-split/test/init_thread.c b/test/init_thread.c similarity index 100% rename from mpi-proxy-split/test/init_thread.c rename to test/init_thread.c diff --git a/mpi-proxy-split/test/keyvals.c b/test/keyvals.c similarity index 100% rename from mpi-proxy-split/test/keyvals.c rename to test/keyvals.c diff --git a/mpi-proxy-split/test/loader-debug-driver.cpp b/test/loader-debug-driver.cpp similarity index 100% rename from mpi-proxy-split/test/loader-debug-driver.cpp rename to test/loader-debug-driver.cpp diff --git a/mpi-proxy-split/test/mana_test.h b/test/mana_test.h similarity index 100% rename from mpi-proxy-split/test/mana_test.h rename to test/mana_test.h diff --git a/mpi-proxy-split/test/memory.c b/test/memory.c similarity index 100% rename from mpi-proxy-split/test/memory.c rename to test/memory.c diff --git a/mpi-proxy-split/test/misc_calls.c b/test/misc_calls.c similarity index 100% rename from mpi-proxy-split/test/misc_calls.c rename to test/misc_calls.c diff --git a/mpi-proxy-split/test/mmap_threads.c b/test/mmap_threads.c similarity index 100% rename from mpi-proxy-split/test/mmap_threads.c rename to test/mmap_threads.c diff --git a/mpi-proxy-split/test/nbc_order.c b/test/nbc_order.c similarity index 100% rename from mpi-proxy-split/test/nbc_order.c rename to test/nbc_order.c diff --git a/mpi-proxy-split/test/nonblocking_collectives.c b/test/nonblocking_collectives.c similarity index 100% rename from mpi-proxy-split/test/nonblocking_collectives.c rename to test/nonblocking_collectives.c diff --git a/mpi-proxy-split/test/p2p_any_source.c b/test/p2p_any_source.c similarity index 100% rename from mpi-proxy-split/test/p2p_any_source.c rename to test/p2p_any_source.c diff --git a/mpi-proxy-split/test/p2p_blocking.c b/test/p2p_blocking.c similarity index 100% rename from mpi-proxy-split/test/p2p_blocking.c rename to test/p2p_blocking.c diff --git a/mpi-proxy-split/test/p2p_issend_comm_free.c b/test/p2p_issend_comm_free.c similarity index 100% rename from mpi-proxy-split/test/p2p_issend_comm_free.c rename to test/p2p_issend_comm_free.c diff --git a/mpi-proxy-split/test/p2p_issend_envelope.c b/test/p2p_issend_envelope.c similarity index 100% rename from mpi-proxy-split/test/p2p_issend_envelope.c rename to test/p2p_issend_envelope.c diff --git a/mpi-proxy-split/test/p2p_issend_nbx.c b/test/p2p_issend_nbx.c similarity index 100% rename from mpi-proxy-split/test/p2p_issend_nbx.c rename to test/p2p_issend_nbx.c diff --git a/mpi-proxy-split/test/p2p_issend_twice.c b/test/p2p_issend_twice.c similarity index 100% rename from mpi-proxy-split/test/p2p_issend_twice.c rename to test/p2p_issend_twice.c diff --git a/mpi-proxy-split/test/p2p_large.c b/test/p2p_large.c similarity index 100% rename from mpi-proxy-split/test/p2p_large.c rename to test/p2p_large.c diff --git a/mpi-proxy-split/test/p2p_nonblocking.c b/test/p2p_nonblocking.c similarity index 100% rename from mpi-proxy-split/test/p2p_nonblocking.c rename to test/p2p_nonblocking.c diff --git a/mpi-proxy-split/test/p2p_probe.c b/test/p2p_probe.c similarity index 100% rename from mpi-proxy-split/test/p2p_probe.c rename to test/p2p_probe.c diff --git a/mpi-proxy-split/test/p2p_proc_null.c b/test/p2p_proc_null.c similarity index 100% rename from mpi-proxy-split/test/p2p_proc_null.c rename to test/p2p_proc_null.c diff --git a/mpi-proxy-split/test/p2p_ring.c b/test/p2p_ring.c similarity index 100% rename from mpi-proxy-split/test/p2p_ring.c rename to test/p2p_ring.c diff --git a/mpi-proxy-split/test/p2p_ssend.c b/test/p2p_ssend.c similarity index 100% rename from mpi-proxy-split/test/p2p_ssend.c rename to test/p2p_ssend.c diff --git a/mpi-proxy-split/test/p2p_ssend_blocked.c b/test/p2p_ssend_blocked.c similarity index 100% rename from mpi-proxy-split/test/p2p_ssend_blocked.c rename to test/p2p_ssend_blocked.c diff --git a/mpi-proxy-split/test/p2p_ssend_collective.c b/test/p2p_ssend_collective.c similarity index 100% rename from mpi-proxy-split/test/p2p_ssend_collective.c rename to test/p2p_ssend_collective.c diff --git a/mpi-proxy-split/test/p2p_ssend_comm.c b/test/p2p_ssend_comm.c similarity index 100% rename from mpi-proxy-split/test/p2p_ssend_comm.c rename to test/p2p_ssend_comm.c diff --git a/mpi-proxy-split/test/p2p_ssend_order.c b/test/p2p_ssend_order.c similarity index 100% rename from mpi-proxy-split/test/p2p_ssend_order.c rename to test/p2p_ssend_order.c diff --git a/mpi-proxy-split/test/p2p_ssend_probe.c b/test/p2p_ssend_probe.c similarity index 100% rename from mpi-proxy-split/test/p2p_ssend_probe.c rename to test/p2p_ssend_probe.c diff --git a/mpi-proxy-split/test/p2p_ssend_sync.c b/test/p2p_ssend_sync.c similarity index 100% rename from mpi-proxy-split/test/p2p_ssend_sync.c rename to test/p2p_ssend_sync.c diff --git a/mpi-proxy-split/test/p2p_ssend_wildcard.c b/test/p2p_ssend_wildcard.c similarity index 100% rename from mpi-proxy-split/test/p2p_ssend_wildcard.c rename to test/p2p_ssend_wildcard.c diff --git a/mpi-proxy-split/test/realloc_grow.c b/test/realloc_grow.c similarity index 100% rename from mpi-proxy-split/test/realloc_grow.c rename to test/realloc_grow.c diff --git a/mpi-proxy-split/test/shared_memory.c b/test/shared_memory.c similarity index 100% rename from mpi-proxy-split/test/shared_memory.c rename to test/shared_memory.c diff --git a/mpi-proxy-split/test/test-loader-debug.py b/test/test-loader-debug.py similarity index 99% rename from mpi-proxy-split/test/test-loader-debug.py rename to test/test-loader-debug.py index 7e33c389c..cb2029710 100644 --- a/mpi-proxy-split/test/test-loader-debug.py +++ b/test/test-loader-debug.py @@ -14,7 +14,7 @@ import unittest -SOURCE = Path(__file__).resolve().parents[1] / "lower-half" +SOURCE = Path(__file__).resolve().parents[1] / "src" / "lower-half" DRIVER = Path(__file__).with_name("loader-debug-driver.cpp") diff --git a/mpi-proxy-split/test/test-mana-restart.py b/test/test-mana-restart.py similarity index 99% rename from mpi-proxy-split/test/test-mana-restart.py rename to test/test-mana-restart.py index 98f2b9252..b50ee21be 100644 --- a/mpi-proxy-split/test/test-mana-restart.py +++ b/test/test-mana-restart.py @@ -7,7 +7,7 @@ import unittest from pathlib import Path -ROOT = Path(__file__).resolve().parents[2] +ROOT = Path(__file__).resolve().parents[1] WRAPPER = Path(os.environ.get( "MANA_RESTART_WRAPPER", ROOT / "scripts" / "mana_restart")) diff --git a/util/hooks/pre-commit b/util/hooks/pre-commit index 6b0439f13..cc3d38c3b 100755 --- a/util/hooks/pre-commit +++ b/util/hooks/pre-commit @@ -37,8 +37,8 @@ if [ -n "$ADDED_OR_MODIFIED" ]; then # # DMTCP's style checker, dmtcp/util/dmtcp-style.py, lints only the files # in DMTCP's source directories, and it runs util/cpplint.py from the - # current directory. So run it in dmtcp/, with mpi-proxy-split/ as its - # source directory. + # current directory. So run it in dmtcp/, with MANA's directories as + # its source directories. if [ ! -f dmtcp/util/dmtcp-style.py ]; then echo "dmtcp/util/dmtcp-style.py is missing." \ "Do: git submodule update --init" @@ -55,7 +55,7 @@ spec.loader.exec_module(dmtcp_style) class ManaCppLinter(dmtcp_style.CppLinter): - source_dirs = ['../mpi-proxy-split'] + source_dirs = ['../src', '../test', '../examples', '../bench'] sys.exit(ManaCppLinter().main(['../' + path for path in sys.argv[1:]]))