From 638dc7009b82caf71410e28da3c0b54060c2b5e2 Mon Sep 17 00:00:00 2001 From: Yao Xu Date: Wed, 4 Nov 2026 21:16:51 -0500 Subject: [PATCH 1/4] Move mpi-proxy-split to src, test and examples mpi-proxy-split/ holds MANA's sources, its tests and its example. Its name comes from DMTCP's contrib/mpi-proxy-split. Move its parts to top-level directories: mpi-proxy-split/ -> src/ (with lower-half/ and mpi-wrappers/) mpi-proxy-split/test/ -> test/ mpi-proxy-split/examples/ -> examples/ The paths in the Makefiles, configure, CI, the style hook, the test scripts, the README and the man page follow. The top-level Makefile now runs 'make tests' and 'make check' in test/ and examples/ itself. The rules of mpi-proxy-split/.gitignore move to the top-level .gitignore. After pulling this change, run ./configure again: it makes the Makefiles in the new directories. --- .github/workflows/mana-tests.yml | 6 ++--- .gitignore | 25 +++++++++++++++++++ Makefile.in | 16 ++++++------ README.md | 4 +-- configure | 12 ++++----- configure.ac | 10 ++++---- .../examples => examples}/Makefile.in | 2 +- {mpi-proxy-split/examples => examples}/ring.c | 2 +- manpages/mana.1 | 4 +-- manpages/mana.1.md | 4 +-- mpi-proxy-split/.gitignore | 25 ------------------- {mpi-proxy-split => src}/Makefile.in | 12 ++------- .../lower-half/Makefile.in | 4 +-- {mpi-proxy-split => src}/lower-half/README | 0 .../lower-half/build-id-debug.cpp | 0 .../lower-half/build-id-debug.h | 0 .../lower-half/copy-stack.c | 0 .../lower-half/get-symbol-offset.c | 0 .../lower-half/lh-func-ptr.c | 0 {mpi-proxy-split => src}/lower-half/logging.h | 0 .../lower-half/lower-half-api.h | 0 .../lower-half/lower-half.cpp | 0 .../lower-half/mem-wrapper.cpp | 0 .../lower-half/mem-wrapper.h | 0 .../lower-half/mmap-fixed-noreplace.c | 0 .../lower-half/patch-trampoline.c | 0 .../lower-half/patch-trampoline.h | 0 .../lower-half/switch-context.cpp | 0 .../lower-half/switch-context.h | 0 {mpi-proxy-split => src}/lower_half_ckpt.cpp | 0 {mpi-proxy-split => src}/lower_half_ckpt.h | 0 .../mpi-wrappers/Makefile.in | 2 +- .../mpi-wrappers/fortran_constants.f90 | 0 .../mpi-wrappers/fortran_handles.h | 0 .../generate-mpi-fortran-wrappers.py | 0 .../mpi-wrappers/generate-mpi-stub-objects.py | 0 .../generate-mpi-stub-wrappers.py | 0 .../generate-mpi-unimplemented-wrappers.py | 0 .../mpi-wrappers/get_fortran_constants.c | 0 .../mpi-wrappers/mpi_cart_wrappers.cpp | 0 .../mpi-wrappers/mpi_collective_wrappers.cpp | 0 .../mpi-wrappers/mpi_comm_wrappers.cpp | 0 .../mpi-wrappers/mpi_error_wrappers.cpp | 0 .../mpi-wrappers/mpi_file_wrappers.cpp | 0 .../mpi_fortran_attr_wrappers.cpp | 0 .../mpi-wrappers/mpi_fortran_wrappers.txt | 0 .../mpi-wrappers/mpi_group_wrappers.cpp | 0 .../mpi-wrappers/mpi_nextfunc.h | 0 .../mpi-wrappers/mpi_op_wrappers.cpp | 0 .../mpi-wrappers/mpi_p2p_wrappers.cpp | 0 .../mpi-wrappers/mpi_request_wrappers.cpp | 0 .../mpi-wrappers/mpi_stub_audit.c | 0 .../mpi-wrappers/mpi_stub_data.c | 0 .../mpi-wrappers/mpi_stub_wrappers.txt | 0 .../mpi-wrappers/mpi_t_wrappers.cpp | 0 .../mpi-wrappers/mpi_type_wrappers.cpp | 0 .../mpi_unimplemented_wrappers.txt | 0 .../mpi-wrappers/mpi_win_wrappers.cpp | 0 .../mpi-wrappers/mpi_wrappers.cpp | 4 +-- {mpi-proxy-split => src}/mpi_files.h | 0 {mpi-proxy-split => src}/mpi_plugin.cpp | 0 {mpi-proxy-split => src}/mpi_plugin.h | 0 .../p2p_drain_send_recv.cpp | 0 .../p2p_drain_send_recv.h | 0 {mpi-proxy-split => src}/p2p_log_replay.cpp | 0 {mpi-proxy-split => src}/p2p_log_replay.h | 0 {mpi-proxy-split => src}/seq_num.cpp | 0 {mpi-proxy-split => src}/seq_num.h | 0 {mpi-proxy-split => src}/uh_wrappers.cpp | 0 {mpi-proxy-split => src}/uh_wrappers.h | 0 {mpi-proxy-split => src}/virtual_id.cpp | 0 {mpi-proxy-split => src}/virtual_id.h | 0 {mpi-proxy-split/test => test}/Makefile.in | 2 +- .../test => test}/README-synthetic_app.md | 6 ++--- {mpi-proxy-split/test => test}/abort.c | 0 {mpi-proxy-split/test => test}/attributes.c | 0 {mpi-proxy-split/test => test}/autotest.py | 2 +- {mpi-proxy-split/test => test}/cartesian.c | 0 {mpi-proxy-split/test => test}/collectives.c | 0 .../test => test}/comm_create_group.c | 0 {mpi-proxy-split/test => test}/comm_free.c | 0 .../test => test}/comm_free_order.c | 0 {mpi-proxy-split/test => test}/comm_self.c | 0 .../test => test}/communicators.c | 0 {mpi-proxy-split/test => test}/datatypes.c | 0 {mpi-proxy-split/test => test}/dlopen_mpi.c | 0 .../test => test}/exit_after_finalize.c | 0 .../test => test}/finalize_unsync.c | 0 .../test => test}/fortran_attributes.f90 | 0 .../test => test}/fortran_handles.f90 | 0 .../test => test}/fortran_nonblocking.f90 | 0 .../test => test}/fortran_p2p.f90 | 0 {mpi-proxy-split/test => test}/init_thread.c | 0 {mpi-proxy-split/test => test}/keyvals.c | 0 .../test => test}/loader-debug-driver.cpp | 0 {mpi-proxy-split/test => test}/mana_test.h | 0 {mpi-proxy-split/test => test}/memory.c | 0 {mpi-proxy-split/test => test}/misc_calls.c | 0 {mpi-proxy-split/test => test}/mmap_threads.c | 0 {mpi-proxy-split/test => test}/nbc_order.c | 0 .../test => test}/nonblocking_collectives.c | 0 .../test => test}/p2p_any_source.c | 0 {mpi-proxy-split/test => test}/p2p_blocking.c | 0 .../test => test}/p2p_issend_comm_free.c | 0 .../test => test}/p2p_issend_envelope.c | 0 .../test => test}/p2p_issend_nbx.c | 0 .../test => test}/p2p_issend_twice.c | 0 {mpi-proxy-split/test => test}/p2p_large.c | 0 .../test => test}/p2p_nonblocking.c | 0 {mpi-proxy-split/test => test}/p2p_probe.c | 0 .../test => test}/p2p_proc_null.c | 0 {mpi-proxy-split/test => test}/p2p_ring.c | 0 {mpi-proxy-split/test => test}/p2p_ssend.c | 0 .../test => test}/p2p_ssend_blocked.c | 0 .../test => test}/p2p_ssend_collective.c | 0 .../test => test}/p2p_ssend_comm.c | 0 .../test => test}/p2p_ssend_order.c | 0 .../test => test}/p2p_ssend_probe.c | 0 .../test => test}/p2p_ssend_sync.c | 0 .../test => test}/p2p_ssend_wildcard.c | 0 {mpi-proxy-split/test => test}/realloc_grow.c | 0 .../test => test}/shared_memory.c | 0 .../test => test}/synthetic_app.c | 0 .../test => test}/test-loader-debug.py | 2 +- .../test => test}/test-mana-restart.py | 2 +- {mpi-proxy-split/test => test}/virtid_bench.c | 0 util/hooks/pre-commit | 6 ++--- 127 files changed, 72 insertions(+), 80 deletions(-) rename {mpi-proxy-split/examples => examples}/Makefile.in (85%) rename {mpi-proxy-split/examples => examples}/ring.c (98%) delete mode 100644 mpi-proxy-split/.gitignore rename {mpi-proxy-split => src}/Makefile.in (94%) rename {mpi-proxy-split => src}/lower-half/Makefile.in (95%) rename {mpi-proxy-split => src}/lower-half/README (100%) rename {mpi-proxy-split => src}/lower-half/build-id-debug.cpp (100%) rename {mpi-proxy-split => src}/lower-half/build-id-debug.h (100%) rename {mpi-proxy-split => src}/lower-half/copy-stack.c (100%) rename {mpi-proxy-split => src}/lower-half/get-symbol-offset.c (100%) rename {mpi-proxy-split => src}/lower-half/lh-func-ptr.c (100%) rename {mpi-proxy-split => src}/lower-half/logging.h (100%) rename {mpi-proxy-split => src}/lower-half/lower-half-api.h (100%) rename {mpi-proxy-split => src}/lower-half/lower-half.cpp (100%) rename {mpi-proxy-split => src}/lower-half/mem-wrapper.cpp (100%) rename {mpi-proxy-split => src}/lower-half/mem-wrapper.h (100%) rename {mpi-proxy-split => src}/lower-half/mmap-fixed-noreplace.c (100%) rename {mpi-proxy-split => src}/lower-half/patch-trampoline.c (100%) rename {mpi-proxy-split => src}/lower-half/patch-trampoline.h (100%) rename {mpi-proxy-split => src}/lower-half/switch-context.cpp (100%) rename {mpi-proxy-split => src}/lower-half/switch-context.h (100%) rename {mpi-proxy-split => src}/lower_half_ckpt.cpp (100%) rename {mpi-proxy-split => src}/lower_half_ckpt.h (100%) rename {mpi-proxy-split => src}/mpi-wrappers/Makefile.in (98%) rename {mpi-proxy-split => src}/mpi-wrappers/fortran_constants.f90 (100%) rename {mpi-proxy-split => src}/mpi-wrappers/fortran_handles.h (100%) rename {mpi-proxy-split => src}/mpi-wrappers/generate-mpi-fortran-wrappers.py (100%) rename {mpi-proxy-split => src}/mpi-wrappers/generate-mpi-stub-objects.py (100%) rename {mpi-proxy-split => src}/mpi-wrappers/generate-mpi-stub-wrappers.py (100%) rename {mpi-proxy-split => src}/mpi-wrappers/generate-mpi-unimplemented-wrappers.py (100%) rename {mpi-proxy-split => src}/mpi-wrappers/get_fortran_constants.c (100%) rename {mpi-proxy-split => src}/mpi-wrappers/mpi_cart_wrappers.cpp (100%) rename {mpi-proxy-split => src}/mpi-wrappers/mpi_collective_wrappers.cpp (100%) rename {mpi-proxy-split => src}/mpi-wrappers/mpi_comm_wrappers.cpp (100%) rename {mpi-proxy-split => src}/mpi-wrappers/mpi_error_wrappers.cpp (100%) rename {mpi-proxy-split => src}/mpi-wrappers/mpi_file_wrappers.cpp (100%) rename {mpi-proxy-split => src}/mpi-wrappers/mpi_fortran_attr_wrappers.cpp (100%) rename {mpi-proxy-split => src}/mpi-wrappers/mpi_fortran_wrappers.txt (100%) rename {mpi-proxy-split => src}/mpi-wrappers/mpi_group_wrappers.cpp (100%) rename {mpi-proxy-split => src}/mpi-wrappers/mpi_nextfunc.h (100%) rename {mpi-proxy-split => src}/mpi-wrappers/mpi_op_wrappers.cpp (100%) rename {mpi-proxy-split => src}/mpi-wrappers/mpi_p2p_wrappers.cpp (100%) rename {mpi-proxy-split => src}/mpi-wrappers/mpi_request_wrappers.cpp (100%) rename {mpi-proxy-split => src}/mpi-wrappers/mpi_stub_audit.c (100%) rename {mpi-proxy-split => src}/mpi-wrappers/mpi_stub_data.c (100%) rename {mpi-proxy-split => src}/mpi-wrappers/mpi_stub_wrappers.txt (100%) rename {mpi-proxy-split => src}/mpi-wrappers/mpi_t_wrappers.cpp (100%) rename {mpi-proxy-split => src}/mpi-wrappers/mpi_type_wrappers.cpp (100%) rename {mpi-proxy-split => src}/mpi-wrappers/mpi_unimplemented_wrappers.txt (100%) rename {mpi-proxy-split => src}/mpi-wrappers/mpi_win_wrappers.cpp (100%) rename {mpi-proxy-split => src}/mpi-wrappers/mpi_wrappers.cpp (98%) rename {mpi-proxy-split => src}/mpi_files.h (100%) rename {mpi-proxy-split => src}/mpi_plugin.cpp (100%) rename {mpi-proxy-split => src}/mpi_plugin.h (100%) rename {mpi-proxy-split => src}/p2p_drain_send_recv.cpp (100%) rename {mpi-proxy-split => src}/p2p_drain_send_recv.h (100%) rename {mpi-proxy-split => src}/p2p_log_replay.cpp (100%) rename {mpi-proxy-split => src}/p2p_log_replay.h (100%) rename {mpi-proxy-split => src}/seq_num.cpp (100%) rename {mpi-proxy-split => src}/seq_num.h (100%) rename {mpi-proxy-split => src}/uh_wrappers.cpp (100%) rename {mpi-proxy-split => src}/uh_wrappers.h (100%) rename {mpi-proxy-split => src}/virtual_id.cpp (100%) rename {mpi-proxy-split => src}/virtual_id.h (100%) rename {mpi-proxy-split/test => test}/Makefile.in (97%) rename {mpi-proxy-split/test => test}/README-synthetic_app.md (85%) rename {mpi-proxy-split/test => test}/abort.c (100%) rename {mpi-proxy-split/test => test}/attributes.c (100%) rename {mpi-proxy-split/test => test}/autotest.py (99%) rename {mpi-proxy-split/test => test}/cartesian.c (100%) rename {mpi-proxy-split/test => test}/collectives.c (100%) rename {mpi-proxy-split/test => test}/comm_create_group.c (100%) rename {mpi-proxy-split/test => test}/comm_free.c (100%) rename {mpi-proxy-split/test => test}/comm_free_order.c (100%) rename {mpi-proxy-split/test => test}/comm_self.c (100%) rename {mpi-proxy-split/test => test}/communicators.c (100%) rename {mpi-proxy-split/test => test}/datatypes.c (100%) rename {mpi-proxy-split/test => test}/dlopen_mpi.c (100%) rename {mpi-proxy-split/test => test}/exit_after_finalize.c (100%) rename {mpi-proxy-split/test => test}/finalize_unsync.c (100%) rename {mpi-proxy-split/test => test}/fortran_attributes.f90 (100%) rename {mpi-proxy-split/test => test}/fortran_handles.f90 (100%) rename {mpi-proxy-split/test => test}/fortran_nonblocking.f90 (100%) rename {mpi-proxy-split/test => test}/fortran_p2p.f90 (100%) rename {mpi-proxy-split/test => test}/init_thread.c (100%) rename {mpi-proxy-split/test => test}/keyvals.c (100%) rename {mpi-proxy-split/test => test}/loader-debug-driver.cpp (100%) rename {mpi-proxy-split/test => test}/mana_test.h (100%) rename {mpi-proxy-split/test => test}/memory.c (100%) rename {mpi-proxy-split/test => test}/misc_calls.c (100%) rename {mpi-proxy-split/test => test}/mmap_threads.c (100%) rename {mpi-proxy-split/test => test}/nbc_order.c (100%) rename {mpi-proxy-split/test => test}/nonblocking_collectives.c (100%) rename {mpi-proxy-split/test => test}/p2p_any_source.c (100%) rename {mpi-proxy-split/test => test}/p2p_blocking.c (100%) rename {mpi-proxy-split/test => test}/p2p_issend_comm_free.c (100%) rename {mpi-proxy-split/test => test}/p2p_issend_envelope.c (100%) rename {mpi-proxy-split/test => test}/p2p_issend_nbx.c (100%) rename {mpi-proxy-split/test => test}/p2p_issend_twice.c (100%) rename {mpi-proxy-split/test => test}/p2p_large.c (100%) rename {mpi-proxy-split/test => test}/p2p_nonblocking.c (100%) rename {mpi-proxy-split/test => test}/p2p_probe.c (100%) rename {mpi-proxy-split/test => test}/p2p_proc_null.c (100%) rename {mpi-proxy-split/test => test}/p2p_ring.c (100%) rename {mpi-proxy-split/test => test}/p2p_ssend.c (100%) rename {mpi-proxy-split/test => test}/p2p_ssend_blocked.c (100%) rename {mpi-proxy-split/test => test}/p2p_ssend_collective.c (100%) rename {mpi-proxy-split/test => test}/p2p_ssend_comm.c (100%) rename {mpi-proxy-split/test => test}/p2p_ssend_order.c (100%) rename {mpi-proxy-split/test => test}/p2p_ssend_probe.c (100%) rename {mpi-proxy-split/test => test}/p2p_ssend_sync.c (100%) rename {mpi-proxy-split/test => test}/p2p_ssend_wildcard.c (100%) rename {mpi-proxy-split/test => test}/realloc_grow.c (100%) rename {mpi-proxy-split/test => test}/shared_memory.c (100%) rename {mpi-proxy-split/test => test}/synthetic_app.c (100%) rename {mpi-proxy-split/test => test}/test-loader-debug.py (99%) rename {mpi-proxy-split/test => test}/test-mana-restart.py (99%) rename {mpi-proxy-split/test => test}/virtid_bench.c (100%) diff --git a/.github/workflows/mana-tests.yml b/.github/workflows/mana-tests.yml index 098517e9c..c890441d9 100644 --- a/.github/workflows/mana-tests.yml +++ b/.github/workflows/mana-tests.yml @@ -53,14 +53,14 @@ jobs: make -j"$(nproc)" mana tests - name: Test mana_restart wrapper - run: python3 mpi-proxy-split/test/test-mana-restart.py -v + run: python3 test/test-mana-restart.py -v - name: Test loader debug lookup - run: python3 mpi-proxy-split/test/test-loader-debug.py -v + run: python3 test/test-loader-debug.py -v - name: Test MANA run: > - mpi-proxy-split/test/autotest.py --retry-once + test/autotest.py --retry-once --workdir /tmp/autotest --launcher '${{ matrix.launcher }}' - name: Keep the logs of failed tests diff --git a/.gitignore b/.gitignore index 5098bb194..11372b2a7 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,11 @@ # Made by ./configure and autoreconf. /Makefile /manpages/Makefile +/src/Makefile +/src/lower-half/Makefile +/src/mpi-wrappers/Makefile +/test/Makefile +/examples/Makefile /config.log /config.status /aclocal.m4 @@ -12,6 +17,8 @@ /bin/ /lib/ /manpages/mana.1.gz +/src/lower-half/lower-half +/examples/ring *.[oa] *.d *.so @@ -21,6 +28,24 @@ *.pyc __pycache__/ +# Made by src/mpi-wrappers/Makefile from the generate-*.py scripts. +/src/mpi-wrappers/mpi_fortran_wrappers.cpp +/src/mpi-wrappers/mpi_unimplemented_wrappers.cpp +/src/mpi-wrappers/mpi_stub_wrappers.c +/src/mpi-wrappers/mpi_stub_objects.c +/src/mpi-wrappers/*.tmp.* + +# The programs of the end-to-end tests and the benchmarks have no suffix: +# ignore all but the sources. +/test/* +!/test/*.c +!/test/*.cpp +!/test/*.f90 +!/test/*.h +!/test/*.py +!/test/*.md +!/test/Makefile.in + # Left by runs in the tree. ckpt_*.dmtcp ckpt_rank_*/ diff --git a/Makefile.in b/Makefile.in index 00780747b..b87ea642d 100644 --- a/Makefile.in +++ b/Makefile.in @@ -37,7 +37,7 @@ all: default MANA_SCRIPTS = mana_coordinator mana_launch mana_restart mana_status mana: mana_prereqs dmtcp $(MANA_SCRIPTS:%=bin/%) - cd mpi-proxy-split && $(MAKE) stage + cd src && $(MAKE) stage # bin/ holds the scripts next to DMTCP's programs and the lower half, which # the scripts run. @@ -47,11 +47,12 @@ $(MANA_SCRIPTS:%=bin/%): bin/%: scripts/% # The end-to-end tests, the example and the benchmarks. tests: - cd mpi-proxy-split && $(MAKE) tests + cd test && $(MAKE) + cd examples && $(MAKE) # Builds the end-to-end tests and runs them (test/autotest.py). check: mana - cd mpi-proxy-split && $(MAKE) check + cd test && $(MAKE) check dmtcp: dmtcp/configure @ # A 'git pull' can change the DMTCP commit that MANA uses, and it @@ -89,12 +90,11 @@ uninstall: distclean: clean - cd ${DMTCP_ROOT} && $(MAKE) distclean - - cd mpi-proxy-split && $(MAKE) distclean + - cd src && $(MAKE) distclean - cd manpages && $(MAKE) distclean rm -f Makefile config.log config.status config.status-* config.cache - rm -f mpi-proxy-split/Makefile mpi-proxy-split/lower-half/Makefile \ - mpi-proxy-split/mpi-wrappers/Makefile mpi-proxy-split/test/Makefile \ - mpi-proxy-split/examples/Makefile + rm -f src/Makefile src/lower-half/Makefile src/mpi-wrappers/Makefile \ + test/Makefile examples/Makefile rm -rf autom4te.cache rm -rf $(top_builddir)/lib $(top_builddir)/bin @@ -104,7 +104,7 @@ tidy: rm -f dmtcp_coordinator_db-*.json clean: tidy - cd mpi-proxy-split && $(MAKE) clean + cd src && $(MAKE) clean mana_prereqs: @ if test -n "$$HUGETLB_DEFAULT_PAGE_SIZE"; then \ diff --git a/README.md b/README.md index 6e0890c18..a7d05ce0f 100644 --- a/README.md +++ b/README.md @@ -63,8 +63,8 @@ CI does. To run MANA's tests, which checkpoint and restart each test program: make tests # builds them; 'make' does not - mpi-proxy-split/test/autotest.py # or: make check - mpi-proxy-split/test/autotest.py --help # e.g. --launcher 'srun -n {n}' + test/autotest.py # or: make check + test/autotest.py --help # e.g. --launcher 'srun -n {n}' MANA is free software: you can redistribute it and/or modify it under the terms of the GNU Lesser General Public License, version 3 or (at your option) diff --git a/configure b/configure index 4ce5b24b5..165f5722a 100755 --- a/configure +++ b/configure @@ -5500,7 +5500,7 @@ ac_configure_args="$ac_configure_args --disable-dlsym-wrapper" subdirs="$subdirs dmtcp" -ac_config_files="$ac_config_files Makefile manpages/Makefile mpi-proxy-split/Makefile mpi-proxy-split/lower-half/Makefile mpi-proxy-split/mpi-wrappers/Makefile mpi-proxy-split/test/Makefile mpi-proxy-split/examples/Makefile" +ac_config_files="$ac_config_files Makefile manpages/Makefile src/Makefile src/lower-half/Makefile src/mpi-wrappers/Makefile test/Makefile examples/Makefile" enable_option_checking=fatal @@ -6298,11 +6298,11 @@ do case $ac_config_target in "Makefile") CONFIG_FILES="$CONFIG_FILES Makefile" ;; "manpages/Makefile") CONFIG_FILES="$CONFIG_FILES manpages/Makefile" ;; - "mpi-proxy-split/Makefile") CONFIG_FILES="$CONFIG_FILES mpi-proxy-split/Makefile" ;; - "mpi-proxy-split/lower-half/Makefile") CONFIG_FILES="$CONFIG_FILES mpi-proxy-split/lower-half/Makefile" ;; - "mpi-proxy-split/mpi-wrappers/Makefile") CONFIG_FILES="$CONFIG_FILES mpi-proxy-split/mpi-wrappers/Makefile" ;; - "mpi-proxy-split/test/Makefile") CONFIG_FILES="$CONFIG_FILES mpi-proxy-split/test/Makefile" ;; - "mpi-proxy-split/examples/Makefile") CONFIG_FILES="$CONFIG_FILES mpi-proxy-split/examples/Makefile" ;; + "src/Makefile") CONFIG_FILES="$CONFIG_FILES src/Makefile" ;; + "src/lower-half/Makefile") CONFIG_FILES="$CONFIG_FILES src/lower-half/Makefile" ;; + "src/mpi-wrappers/Makefile") CONFIG_FILES="$CONFIG_FILES src/mpi-wrappers/Makefile" ;; + "test/Makefile") CONFIG_FILES="$CONFIG_FILES test/Makefile" ;; + "examples/Makefile") CONFIG_FILES="$CONFIG_FILES examples/Makefile" ;; *) as_fn_error $? "invalid argument: \`$ac_config_target'" "$LINENO" 5;; esac diff --git a/configure.ac b/configure.ac index 583a264d2..9f5aa9113 100644 --- a/configure.ac +++ b/configure.ac @@ -80,11 +80,11 @@ AC_CONFIG_SUBDIRS([dmtcp]) AC_CONFIG_FILES([Makefile \ manpages/Makefile \ - mpi-proxy-split/Makefile \ - mpi-proxy-split/lower-half/Makefile \ - mpi-proxy-split/mpi-wrappers/Makefile \ - mpi-proxy-split/test/Makefile \ - mpi-proxy-split/examples/Makefile]) + src/Makefile \ + src/lower-half/Makefile \ + src/mpi-wrappers/Makefile \ + test/Makefile \ + examples/Makefile]) dnl Autoconf manual says option checking is set to warn ("yes") by dnl by default. But it's actually set to "no". diff --git a/mpi-proxy-split/examples/Makefile.in b/examples/Makefile.in similarity index 85% rename from mpi-proxy-split/examples/Makefile.in rename to examples/Makefile.in index 6cfc71430..440cf8dbb 100644 --- a/mpi-proxy-split/examples/Makefile.in +++ b/examples/Makefile.in @@ -18,4 +18,4 @@ clean: # Regenerate this Makefile when Makefile.in or config.status is newer. Makefile: Makefile.in @top_builddir@/config.status - cd @top_builddir@ && ./config.status mpi-proxy-split/examples/Makefile + cd @top_builddir@ && ./config.status examples/Makefile diff --git a/mpi-proxy-split/examples/ring.c b/examples/ring.c similarity index 98% rename from mpi-proxy-split/examples/ring.c rename to examples/ring.c index 122f5838e..dec1607dc 100644 --- a/mpi-proxy-split/examples/ring.c +++ b/examples/ring.c @@ -34,7 +34,7 @@ // run it under MANA, from MANA's directory: // // bin/mana_coordinator -// mpirun -np 4 bin/mana_launch mpi-proxy-split/examples/ring +// mpirun -np 4 bin/mana_launch examples/ring // bin/mana_status --checkpoint (in another shell, while it runs) // // Then stop the job (Ctrl-C), and restart it from the checkpoint: diff --git a/manpages/mana.1 b/manpages/mana.1 index 4d89855ee..8e5820908 100644 --- a/manpages/mana.1 +++ b/manpages/mana.1 @@ -67,10 +67,10 @@ cd dmtcp-mana \&./configure --enable-debug make -j mana tests # Build the program with mpicc as usual; no relinking for MANA is needed. -# 'make tests' also builds an example: mpi-proxy-split/examples/ring +# 'make tests' also builds an example: examples/ring salloc -N 2 -q interactive -C haswell -t 01:00:00 bin/mana_coordinator -i10 -srun -N 2 bin/mana_launch mpi-proxy-split/examples/ring +srun -N 2 bin/mana_launch examples/ring bin/mana_coordinator -i10 srun -N 2 bin/mana_restart \f[R] diff --git a/manpages/mana.1.md b/manpages/mana.1.md index 5e6b129f4..b405946ce 100644 --- a/manpages/mana.1.md +++ b/manpages/mana.1.md @@ -58,10 +58,10 @@ cd dmtcp-mana ./configure --enable-debug make -j mana tests # Build the program with mpicc as usual; no relinking for MANA is needed. -# 'make tests' also builds an example: mpi-proxy-split/examples/ring +# 'make tests' also builds an example: examples/ring salloc -N 2 -q interactive -C haswell -t 01:00:00 bin/mana_coordinator -i10 -srun -N 2 bin/mana_launch mpi-proxy-split/examples/ring +srun -N 2 bin/mana_launch examples/ring bin/mana_coordinator -i10 srun -N 2 bin/mana_restart ``` diff --git a/mpi-proxy-split/.gitignore b/mpi-proxy-split/.gitignore deleted file mode 100644 index ca60bb9e4..000000000 --- a/mpi-proxy-split/.gitignore +++ /dev/null @@ -1,25 +0,0 @@ -# The programs of the end-to-end tests and the benchmarks have no suffix: -# ignore all but the sources. -test/* -!test/*.c -!test/*.cpp -!test/*.f90 -!test/*.h -!test/*.py -!test/*.md -!test/Makefile.in -examples/ring -lower-half/lower-half - -# Made by configure from the Makefile.in files. -/Makefile -/lower-half/Makefile -/mpi-wrappers/Makefile -/examples/Makefile - -# Made by mpi-wrappers/Makefile from the generate-*.py scripts. -mpi-wrappers/mpi_fortran_wrappers.cpp -mpi-wrappers/mpi_unimplemented_wrappers.cpp -mpi-wrappers/mpi_stub_wrappers.c -mpi-wrappers/mpi_stub_objects.c -mpi-wrappers/*.tmp.* diff --git a/mpi-proxy-split/Makefile.in b/src/Makefile.in similarity index 94% rename from mpi-proxy-split/Makefile.in rename to src/Makefile.in index 05703b1da..a3f49cc53 100644 --- a/mpi-proxy-split/Makefile.in +++ b/src/Makefile.in @@ -75,14 +75,6 @@ lower-half/switch-context.o: FORCE -include $(wildcard *.d) -tests: - @$(MAKE) -C test/ default - @$(MAKE) -C examples/ - -check: libmana.so - @$(MAKE) -C test/ e2e - @test/autotest.py - ${MANA_ROOT}/lib/dmtcp/libmana.so: libmana.so cp -f $< $@ ${MANA_ROOT}/lib/dmtcp/libmpistub.so: ${WRAPPERS_SRCDIR}/mpi_stub_wrappers.txt \ @@ -117,8 +109,8 @@ distclean: clean FORCE: -.PHONY: default clean distclean tidy check tests stage +.PHONY: default clean distclean tidy stage # Regenerate this Makefile when Makefile.in or config.status is newer. Makefile: Makefile.in @top_builddir@/config.status - cd @top_builddir@ && ./config.status mpi-proxy-split/Makefile + cd @top_builddir@ && ./config.status src/Makefile diff --git a/mpi-proxy-split/lower-half/Makefile.in b/src/lower-half/Makefile.in similarity index 95% rename from mpi-proxy-split/lower-half/Makefile.in rename to src/lower-half/Makefile.in index acbb94f5c..2a8cfe6d1 100644 --- a/mpi-proxy-split/lower-half/Makefile.in +++ b/src/lower-half/Makefile.in @@ -15,7 +15,7 @@ endif DMTCP_INCLUDE=${DMTCP_ROOT}/include JALIB_INCLUDE=${DMTCP_ROOT}/jalib -MANA_INCLUDE=${MANA_ROOT}/mpi-proxy-split +MANA_INCLUDE=${MANA_ROOT}/src DMTCP_DEPS = ${DMTCP_ROOT}/src/libdmtcprestart.a \ ${DMTCP_ROOT}/src/libdmtcpinternal.a \ @@ -74,4 +74,4 @@ distclean: clean # Regenerate this Makefile when Makefile.in or config.status is newer. Makefile: Makefile.in @top_builddir@/config.status - cd @top_builddir@ && ./config.status mpi-proxy-split/lower-half/Makefile + cd @top_builddir@ && ./config.status src/lower-half/Makefile diff --git a/mpi-proxy-split/lower-half/README b/src/lower-half/README similarity index 100% rename from mpi-proxy-split/lower-half/README rename to src/lower-half/README diff --git a/mpi-proxy-split/lower-half/build-id-debug.cpp b/src/lower-half/build-id-debug.cpp similarity index 100% rename from mpi-proxy-split/lower-half/build-id-debug.cpp rename to src/lower-half/build-id-debug.cpp diff --git a/mpi-proxy-split/lower-half/build-id-debug.h b/src/lower-half/build-id-debug.h similarity index 100% rename from mpi-proxy-split/lower-half/build-id-debug.h rename to src/lower-half/build-id-debug.h diff --git a/mpi-proxy-split/lower-half/copy-stack.c b/src/lower-half/copy-stack.c similarity index 100% rename from mpi-proxy-split/lower-half/copy-stack.c rename to src/lower-half/copy-stack.c diff --git a/mpi-proxy-split/lower-half/get-symbol-offset.c b/src/lower-half/get-symbol-offset.c similarity index 100% rename from mpi-proxy-split/lower-half/get-symbol-offset.c rename to src/lower-half/get-symbol-offset.c diff --git a/mpi-proxy-split/lower-half/lh-func-ptr.c b/src/lower-half/lh-func-ptr.c similarity index 100% rename from mpi-proxy-split/lower-half/lh-func-ptr.c rename to src/lower-half/lh-func-ptr.c diff --git a/mpi-proxy-split/lower-half/logging.h b/src/lower-half/logging.h similarity index 100% rename from mpi-proxy-split/lower-half/logging.h rename to src/lower-half/logging.h diff --git a/mpi-proxy-split/lower-half/lower-half-api.h b/src/lower-half/lower-half-api.h similarity index 100% rename from mpi-proxy-split/lower-half/lower-half-api.h rename to src/lower-half/lower-half-api.h diff --git a/mpi-proxy-split/lower-half/lower-half.cpp b/src/lower-half/lower-half.cpp similarity index 100% rename from mpi-proxy-split/lower-half/lower-half.cpp rename to src/lower-half/lower-half.cpp diff --git a/mpi-proxy-split/lower-half/mem-wrapper.cpp b/src/lower-half/mem-wrapper.cpp similarity index 100% rename from mpi-proxy-split/lower-half/mem-wrapper.cpp rename to src/lower-half/mem-wrapper.cpp diff --git a/mpi-proxy-split/lower-half/mem-wrapper.h b/src/lower-half/mem-wrapper.h similarity index 100% rename from mpi-proxy-split/lower-half/mem-wrapper.h rename to src/lower-half/mem-wrapper.h diff --git a/mpi-proxy-split/lower-half/mmap-fixed-noreplace.c b/src/lower-half/mmap-fixed-noreplace.c similarity index 100% rename from mpi-proxy-split/lower-half/mmap-fixed-noreplace.c rename to src/lower-half/mmap-fixed-noreplace.c diff --git a/mpi-proxy-split/lower-half/patch-trampoline.c b/src/lower-half/patch-trampoline.c similarity index 100% rename from mpi-proxy-split/lower-half/patch-trampoline.c rename to src/lower-half/patch-trampoline.c diff --git a/mpi-proxy-split/lower-half/patch-trampoline.h b/src/lower-half/patch-trampoline.h similarity index 100% rename from mpi-proxy-split/lower-half/patch-trampoline.h rename to src/lower-half/patch-trampoline.h diff --git a/mpi-proxy-split/lower-half/switch-context.cpp b/src/lower-half/switch-context.cpp similarity index 100% rename from mpi-proxy-split/lower-half/switch-context.cpp rename to src/lower-half/switch-context.cpp diff --git a/mpi-proxy-split/lower-half/switch-context.h b/src/lower-half/switch-context.h similarity index 100% rename from mpi-proxy-split/lower-half/switch-context.h rename to src/lower-half/switch-context.h diff --git a/mpi-proxy-split/lower_half_ckpt.cpp b/src/lower_half_ckpt.cpp similarity index 100% rename from mpi-proxy-split/lower_half_ckpt.cpp rename to src/lower_half_ckpt.cpp diff --git a/mpi-proxy-split/lower_half_ckpt.h b/src/lower_half_ckpt.h similarity index 100% rename from mpi-proxy-split/lower_half_ckpt.h rename to src/lower_half_ckpt.h diff --git a/mpi-proxy-split/mpi-wrappers/Makefile.in b/src/mpi-wrappers/Makefile.in similarity index 98% rename from mpi-proxy-split/mpi-wrappers/Makefile.in rename to src/mpi-wrappers/Makefile.in index 4725dadf3..2df71093c 100644 --- a/mpi-proxy-split/mpi-wrappers/Makefile.in +++ b/src/mpi-wrappers/Makefile.in @@ -130,4 +130,4 @@ distclean: clean # Regenerate this Makefile when Makefile.in or config.status is newer. Makefile: Makefile.in @top_builddir@/config.status - cd @top_builddir@ && ./config.status mpi-proxy-split/mpi-wrappers/Makefile + cd @top_builddir@ && ./config.status src/mpi-wrappers/Makefile diff --git a/mpi-proxy-split/mpi-wrappers/fortran_constants.f90 b/src/mpi-wrappers/fortran_constants.f90 similarity index 100% rename from mpi-proxy-split/mpi-wrappers/fortran_constants.f90 rename to src/mpi-wrappers/fortran_constants.f90 diff --git a/mpi-proxy-split/mpi-wrappers/fortran_handles.h b/src/mpi-wrappers/fortran_handles.h similarity index 100% rename from mpi-proxy-split/mpi-wrappers/fortran_handles.h rename to src/mpi-wrappers/fortran_handles.h diff --git a/mpi-proxy-split/mpi-wrappers/generate-mpi-fortran-wrappers.py b/src/mpi-wrappers/generate-mpi-fortran-wrappers.py similarity index 100% rename from mpi-proxy-split/mpi-wrappers/generate-mpi-fortran-wrappers.py rename to src/mpi-wrappers/generate-mpi-fortran-wrappers.py diff --git a/mpi-proxy-split/mpi-wrappers/generate-mpi-stub-objects.py b/src/mpi-wrappers/generate-mpi-stub-objects.py similarity index 100% rename from mpi-proxy-split/mpi-wrappers/generate-mpi-stub-objects.py rename to src/mpi-wrappers/generate-mpi-stub-objects.py diff --git a/mpi-proxy-split/mpi-wrappers/generate-mpi-stub-wrappers.py b/src/mpi-wrappers/generate-mpi-stub-wrappers.py similarity index 100% rename from mpi-proxy-split/mpi-wrappers/generate-mpi-stub-wrappers.py rename to src/mpi-wrappers/generate-mpi-stub-wrappers.py diff --git a/mpi-proxy-split/mpi-wrappers/generate-mpi-unimplemented-wrappers.py b/src/mpi-wrappers/generate-mpi-unimplemented-wrappers.py similarity index 100% rename from mpi-proxy-split/mpi-wrappers/generate-mpi-unimplemented-wrappers.py rename to src/mpi-wrappers/generate-mpi-unimplemented-wrappers.py diff --git a/mpi-proxy-split/mpi-wrappers/get_fortran_constants.c b/src/mpi-wrappers/get_fortran_constants.c similarity index 100% rename from mpi-proxy-split/mpi-wrappers/get_fortran_constants.c rename to src/mpi-wrappers/get_fortran_constants.c diff --git a/mpi-proxy-split/mpi-wrappers/mpi_cart_wrappers.cpp b/src/mpi-wrappers/mpi_cart_wrappers.cpp similarity index 100% rename from mpi-proxy-split/mpi-wrappers/mpi_cart_wrappers.cpp rename to src/mpi-wrappers/mpi_cart_wrappers.cpp diff --git a/mpi-proxy-split/mpi-wrappers/mpi_collective_wrappers.cpp b/src/mpi-wrappers/mpi_collective_wrappers.cpp similarity index 100% rename from mpi-proxy-split/mpi-wrappers/mpi_collective_wrappers.cpp rename to src/mpi-wrappers/mpi_collective_wrappers.cpp diff --git a/mpi-proxy-split/mpi-wrappers/mpi_comm_wrappers.cpp b/src/mpi-wrappers/mpi_comm_wrappers.cpp similarity index 100% rename from mpi-proxy-split/mpi-wrappers/mpi_comm_wrappers.cpp rename to src/mpi-wrappers/mpi_comm_wrappers.cpp diff --git a/mpi-proxy-split/mpi-wrappers/mpi_error_wrappers.cpp b/src/mpi-wrappers/mpi_error_wrappers.cpp similarity index 100% rename from mpi-proxy-split/mpi-wrappers/mpi_error_wrappers.cpp rename to src/mpi-wrappers/mpi_error_wrappers.cpp diff --git a/mpi-proxy-split/mpi-wrappers/mpi_file_wrappers.cpp b/src/mpi-wrappers/mpi_file_wrappers.cpp similarity index 100% rename from mpi-proxy-split/mpi-wrappers/mpi_file_wrappers.cpp rename to src/mpi-wrappers/mpi_file_wrappers.cpp diff --git a/mpi-proxy-split/mpi-wrappers/mpi_fortran_attr_wrappers.cpp b/src/mpi-wrappers/mpi_fortran_attr_wrappers.cpp similarity index 100% rename from mpi-proxy-split/mpi-wrappers/mpi_fortran_attr_wrappers.cpp rename to src/mpi-wrappers/mpi_fortran_attr_wrappers.cpp diff --git a/mpi-proxy-split/mpi-wrappers/mpi_fortran_wrappers.txt b/src/mpi-wrappers/mpi_fortran_wrappers.txt similarity index 100% rename from mpi-proxy-split/mpi-wrappers/mpi_fortran_wrappers.txt rename to src/mpi-wrappers/mpi_fortran_wrappers.txt diff --git a/mpi-proxy-split/mpi-wrappers/mpi_group_wrappers.cpp b/src/mpi-wrappers/mpi_group_wrappers.cpp similarity index 100% rename from mpi-proxy-split/mpi-wrappers/mpi_group_wrappers.cpp rename to src/mpi-wrappers/mpi_group_wrappers.cpp diff --git a/mpi-proxy-split/mpi-wrappers/mpi_nextfunc.h b/src/mpi-wrappers/mpi_nextfunc.h similarity index 100% rename from mpi-proxy-split/mpi-wrappers/mpi_nextfunc.h rename to src/mpi-wrappers/mpi_nextfunc.h diff --git a/mpi-proxy-split/mpi-wrappers/mpi_op_wrappers.cpp b/src/mpi-wrappers/mpi_op_wrappers.cpp similarity index 100% rename from mpi-proxy-split/mpi-wrappers/mpi_op_wrappers.cpp rename to src/mpi-wrappers/mpi_op_wrappers.cpp diff --git a/mpi-proxy-split/mpi-wrappers/mpi_p2p_wrappers.cpp b/src/mpi-wrappers/mpi_p2p_wrappers.cpp similarity index 100% rename from mpi-proxy-split/mpi-wrappers/mpi_p2p_wrappers.cpp rename to src/mpi-wrappers/mpi_p2p_wrappers.cpp diff --git a/mpi-proxy-split/mpi-wrappers/mpi_request_wrappers.cpp b/src/mpi-wrappers/mpi_request_wrappers.cpp similarity index 100% rename from mpi-proxy-split/mpi-wrappers/mpi_request_wrappers.cpp rename to src/mpi-wrappers/mpi_request_wrappers.cpp diff --git a/mpi-proxy-split/mpi-wrappers/mpi_stub_audit.c b/src/mpi-wrappers/mpi_stub_audit.c similarity index 100% rename from mpi-proxy-split/mpi-wrappers/mpi_stub_audit.c rename to src/mpi-wrappers/mpi_stub_audit.c diff --git a/mpi-proxy-split/mpi-wrappers/mpi_stub_data.c b/src/mpi-wrappers/mpi_stub_data.c similarity index 100% rename from mpi-proxy-split/mpi-wrappers/mpi_stub_data.c rename to src/mpi-wrappers/mpi_stub_data.c diff --git a/mpi-proxy-split/mpi-wrappers/mpi_stub_wrappers.txt b/src/mpi-wrappers/mpi_stub_wrappers.txt similarity index 100% rename from mpi-proxy-split/mpi-wrappers/mpi_stub_wrappers.txt rename to src/mpi-wrappers/mpi_stub_wrappers.txt diff --git a/mpi-proxy-split/mpi-wrappers/mpi_t_wrappers.cpp b/src/mpi-wrappers/mpi_t_wrappers.cpp similarity index 100% rename from mpi-proxy-split/mpi-wrappers/mpi_t_wrappers.cpp rename to src/mpi-wrappers/mpi_t_wrappers.cpp diff --git a/mpi-proxy-split/mpi-wrappers/mpi_type_wrappers.cpp b/src/mpi-wrappers/mpi_type_wrappers.cpp similarity index 100% rename from mpi-proxy-split/mpi-wrappers/mpi_type_wrappers.cpp rename to src/mpi-wrappers/mpi_type_wrappers.cpp diff --git a/mpi-proxy-split/mpi-wrappers/mpi_unimplemented_wrappers.txt b/src/mpi-wrappers/mpi_unimplemented_wrappers.txt similarity index 100% rename from mpi-proxy-split/mpi-wrappers/mpi_unimplemented_wrappers.txt rename to src/mpi-wrappers/mpi_unimplemented_wrappers.txt diff --git a/mpi-proxy-split/mpi-wrappers/mpi_win_wrappers.cpp b/src/mpi-wrappers/mpi_win_wrappers.cpp similarity index 100% rename from mpi-proxy-split/mpi-wrappers/mpi_win_wrappers.cpp rename to src/mpi-wrappers/mpi_win_wrappers.cpp diff --git a/mpi-proxy-split/mpi-wrappers/mpi_wrappers.cpp b/src/mpi-wrappers/mpi_wrappers.cpp similarity index 98% rename from mpi-proxy-split/mpi-wrappers/mpi_wrappers.cpp rename to src/mpi-wrappers/mpi_wrappers.cpp index 536104832..b648c3927 100644 --- a/mpi-proxy-split/mpi-wrappers/mpi_wrappers.cpp +++ b/src/mpi-wrappers/mpi_wrappers.cpp @@ -244,9 +244,9 @@ int PMPI_Get_address(const void *location, MPI_Aint *address) // FOR DEBUGGING ONLY: // This defines a call to MPI_MANA_Internal in the lower half, which // is especially useful in debugging restart. It is called -// from mpi-proxy-split/mpi_plugin.cpp, just before replaying the pending +// from src/mpi_plugin.cpp, just before replaying the pending // receives. -// In mpi-proxy-split/lower-half, redefine MPI_MANA_Internal() +// In src/lower-half, redefine MPI_MANA_Internal() // to do whatever is desired. Then do: // rm bin/lh_proxy // module list # Verify the correct 'module' settings diff --git a/mpi-proxy-split/mpi_files.h b/src/mpi_files.h similarity index 100% rename from mpi-proxy-split/mpi_files.h rename to src/mpi_files.h diff --git a/mpi-proxy-split/mpi_plugin.cpp b/src/mpi_plugin.cpp similarity index 100% rename from mpi-proxy-split/mpi_plugin.cpp rename to src/mpi_plugin.cpp diff --git a/mpi-proxy-split/mpi_plugin.h b/src/mpi_plugin.h similarity index 100% rename from mpi-proxy-split/mpi_plugin.h rename to src/mpi_plugin.h diff --git a/mpi-proxy-split/p2p_drain_send_recv.cpp b/src/p2p_drain_send_recv.cpp similarity index 100% rename from mpi-proxy-split/p2p_drain_send_recv.cpp rename to src/p2p_drain_send_recv.cpp diff --git a/mpi-proxy-split/p2p_drain_send_recv.h b/src/p2p_drain_send_recv.h similarity index 100% rename from mpi-proxy-split/p2p_drain_send_recv.h rename to src/p2p_drain_send_recv.h diff --git a/mpi-proxy-split/p2p_log_replay.cpp b/src/p2p_log_replay.cpp similarity index 100% rename from mpi-proxy-split/p2p_log_replay.cpp rename to src/p2p_log_replay.cpp diff --git a/mpi-proxy-split/p2p_log_replay.h b/src/p2p_log_replay.h similarity index 100% rename from mpi-proxy-split/p2p_log_replay.h rename to src/p2p_log_replay.h diff --git a/mpi-proxy-split/seq_num.cpp b/src/seq_num.cpp similarity index 100% rename from mpi-proxy-split/seq_num.cpp rename to src/seq_num.cpp diff --git a/mpi-proxy-split/seq_num.h b/src/seq_num.h similarity index 100% rename from mpi-proxy-split/seq_num.h rename to src/seq_num.h diff --git a/mpi-proxy-split/uh_wrappers.cpp b/src/uh_wrappers.cpp similarity index 100% rename from mpi-proxy-split/uh_wrappers.cpp rename to src/uh_wrappers.cpp diff --git a/mpi-proxy-split/uh_wrappers.h b/src/uh_wrappers.h similarity index 100% rename from mpi-proxy-split/uh_wrappers.h rename to src/uh_wrappers.h diff --git a/mpi-proxy-split/virtual_id.cpp b/src/virtual_id.cpp similarity index 100% rename from mpi-proxy-split/virtual_id.cpp rename to src/virtual_id.cpp diff --git a/mpi-proxy-split/virtual_id.h b/src/virtual_id.h similarity index 100% rename from mpi-proxy-split/virtual_id.h rename to src/virtual_id.h diff --git a/mpi-proxy-split/test/Makefile.in b/test/Makefile.in similarity index 97% rename from mpi-proxy-split/test/Makefile.in rename to test/Makefile.in index 0cd48d519..cd22c5d75 100644 --- a/mpi-proxy-split/test/Makefile.in +++ b/test/Makefile.in @@ -85,4 +85,4 @@ clean: tidy # Regenerate this Makefile when Makefile.in or config.status is newer. Makefile: Makefile.in @top_builddir@/config.status - cd @top_builddir@ && ./config.status mpi-proxy-split/test/Makefile + cd @top_builddir@ && ./config.status test/Makefile diff --git a/mpi-proxy-split/test/README-synthetic_app.md b/test/README-synthetic_app.md similarity index 85% rename from mpi-proxy-split/test/README-synthetic_app.md rename to test/README-synthetic_app.md index 79ffdb992..b80c04a5d 100644 --- a/mpi-proxy-split/test/README-synthetic_app.md +++ b/test/README-synthetic_app.md @@ -23,10 +23,10 @@ same options. ## Usage ``` -make -C mpi-proxy-split/test synthetic_app -mpirun -np 8 mpi-proxy-split/test/synthetic_app --profile=vasp +make -C test synthetic_app +mpirun -np 8 test/synthetic_app --profile=vasp bin/mana_coordinator -mpirun -np 8 bin/mana_launch mpi-proxy-split/test/synthetic_app --profile=vasp +mpirun -np 8 bin/mana_launch test/synthetic_app --profile=vasp ``` The options are listed at the top of `synthetic_app.c`. diff --git a/mpi-proxy-split/test/abort.c b/test/abort.c similarity index 100% rename from mpi-proxy-split/test/abort.c rename to test/abort.c diff --git a/mpi-proxy-split/test/attributes.c b/test/attributes.c similarity index 100% rename from mpi-proxy-split/test/attributes.c rename to test/attributes.c diff --git a/mpi-proxy-split/test/autotest.py b/test/autotest.py similarity index 99% rename from mpi-proxy-split/test/autotest.py rename to test/autotest.py index 4a0e88577..47dea0b16 100755 --- a/mpi-proxy-split/test/autotest.py +++ b/test/autotest.py @@ -25,7 +25,7 @@ import time TEST_DIR = os.path.dirname(os.path.abspath(__file__)) -MANA_ROOT = os.path.abspath(os.path.join(TEST_DIR, "..", "..")) +MANA_ROOT = os.path.abspath(os.path.join(TEST_DIR, "..")) # In the arguments of a test, stands for the MPI library by its full path. diff --git a/mpi-proxy-split/test/cartesian.c b/test/cartesian.c similarity index 100% rename from mpi-proxy-split/test/cartesian.c rename to test/cartesian.c diff --git a/mpi-proxy-split/test/collectives.c b/test/collectives.c similarity index 100% rename from mpi-proxy-split/test/collectives.c rename to test/collectives.c diff --git a/mpi-proxy-split/test/comm_create_group.c b/test/comm_create_group.c similarity index 100% rename from mpi-proxy-split/test/comm_create_group.c rename to test/comm_create_group.c diff --git a/mpi-proxy-split/test/comm_free.c b/test/comm_free.c similarity index 100% rename from mpi-proxy-split/test/comm_free.c rename to test/comm_free.c diff --git a/mpi-proxy-split/test/comm_free_order.c b/test/comm_free_order.c similarity index 100% rename from mpi-proxy-split/test/comm_free_order.c rename to test/comm_free_order.c diff --git a/mpi-proxy-split/test/comm_self.c b/test/comm_self.c similarity index 100% rename from mpi-proxy-split/test/comm_self.c rename to test/comm_self.c diff --git a/mpi-proxy-split/test/communicators.c b/test/communicators.c similarity index 100% rename from mpi-proxy-split/test/communicators.c rename to test/communicators.c diff --git a/mpi-proxy-split/test/datatypes.c b/test/datatypes.c similarity index 100% rename from mpi-proxy-split/test/datatypes.c rename to test/datatypes.c diff --git a/mpi-proxy-split/test/dlopen_mpi.c b/test/dlopen_mpi.c similarity index 100% rename from mpi-proxy-split/test/dlopen_mpi.c rename to test/dlopen_mpi.c diff --git a/mpi-proxy-split/test/exit_after_finalize.c b/test/exit_after_finalize.c similarity index 100% rename from mpi-proxy-split/test/exit_after_finalize.c rename to test/exit_after_finalize.c diff --git a/mpi-proxy-split/test/finalize_unsync.c b/test/finalize_unsync.c similarity index 100% rename from mpi-proxy-split/test/finalize_unsync.c rename to test/finalize_unsync.c diff --git a/mpi-proxy-split/test/fortran_attributes.f90 b/test/fortran_attributes.f90 similarity index 100% rename from mpi-proxy-split/test/fortran_attributes.f90 rename to test/fortran_attributes.f90 diff --git a/mpi-proxy-split/test/fortran_handles.f90 b/test/fortran_handles.f90 similarity index 100% rename from mpi-proxy-split/test/fortran_handles.f90 rename to test/fortran_handles.f90 diff --git a/mpi-proxy-split/test/fortran_nonblocking.f90 b/test/fortran_nonblocking.f90 similarity index 100% rename from mpi-proxy-split/test/fortran_nonblocking.f90 rename to test/fortran_nonblocking.f90 diff --git a/mpi-proxy-split/test/fortran_p2p.f90 b/test/fortran_p2p.f90 similarity index 100% rename from mpi-proxy-split/test/fortran_p2p.f90 rename to test/fortran_p2p.f90 diff --git a/mpi-proxy-split/test/init_thread.c b/test/init_thread.c similarity index 100% rename from mpi-proxy-split/test/init_thread.c rename to test/init_thread.c diff --git a/mpi-proxy-split/test/keyvals.c b/test/keyvals.c similarity index 100% rename from mpi-proxy-split/test/keyvals.c rename to test/keyvals.c diff --git a/mpi-proxy-split/test/loader-debug-driver.cpp b/test/loader-debug-driver.cpp similarity index 100% rename from mpi-proxy-split/test/loader-debug-driver.cpp rename to test/loader-debug-driver.cpp diff --git a/mpi-proxy-split/test/mana_test.h b/test/mana_test.h similarity index 100% rename from mpi-proxy-split/test/mana_test.h rename to test/mana_test.h diff --git a/mpi-proxy-split/test/memory.c b/test/memory.c similarity index 100% rename from mpi-proxy-split/test/memory.c rename to test/memory.c diff --git a/mpi-proxy-split/test/misc_calls.c b/test/misc_calls.c similarity index 100% rename from mpi-proxy-split/test/misc_calls.c rename to test/misc_calls.c diff --git a/mpi-proxy-split/test/mmap_threads.c b/test/mmap_threads.c similarity index 100% rename from mpi-proxy-split/test/mmap_threads.c rename to test/mmap_threads.c diff --git a/mpi-proxy-split/test/nbc_order.c b/test/nbc_order.c similarity index 100% rename from mpi-proxy-split/test/nbc_order.c rename to test/nbc_order.c diff --git a/mpi-proxy-split/test/nonblocking_collectives.c b/test/nonblocking_collectives.c similarity index 100% rename from mpi-proxy-split/test/nonblocking_collectives.c rename to test/nonblocking_collectives.c diff --git a/mpi-proxy-split/test/p2p_any_source.c b/test/p2p_any_source.c similarity index 100% rename from mpi-proxy-split/test/p2p_any_source.c rename to test/p2p_any_source.c diff --git a/mpi-proxy-split/test/p2p_blocking.c b/test/p2p_blocking.c similarity index 100% rename from mpi-proxy-split/test/p2p_blocking.c rename to test/p2p_blocking.c diff --git a/mpi-proxy-split/test/p2p_issend_comm_free.c b/test/p2p_issend_comm_free.c similarity index 100% rename from mpi-proxy-split/test/p2p_issend_comm_free.c rename to test/p2p_issend_comm_free.c diff --git a/mpi-proxy-split/test/p2p_issend_envelope.c b/test/p2p_issend_envelope.c similarity index 100% rename from mpi-proxy-split/test/p2p_issend_envelope.c rename to test/p2p_issend_envelope.c diff --git a/mpi-proxy-split/test/p2p_issend_nbx.c b/test/p2p_issend_nbx.c similarity index 100% rename from mpi-proxy-split/test/p2p_issend_nbx.c rename to test/p2p_issend_nbx.c diff --git a/mpi-proxy-split/test/p2p_issend_twice.c b/test/p2p_issend_twice.c similarity index 100% rename from mpi-proxy-split/test/p2p_issend_twice.c rename to test/p2p_issend_twice.c diff --git a/mpi-proxy-split/test/p2p_large.c b/test/p2p_large.c similarity index 100% rename from mpi-proxy-split/test/p2p_large.c rename to test/p2p_large.c diff --git a/mpi-proxy-split/test/p2p_nonblocking.c b/test/p2p_nonblocking.c similarity index 100% rename from mpi-proxy-split/test/p2p_nonblocking.c rename to test/p2p_nonblocking.c diff --git a/mpi-proxy-split/test/p2p_probe.c b/test/p2p_probe.c similarity index 100% rename from mpi-proxy-split/test/p2p_probe.c rename to test/p2p_probe.c diff --git a/mpi-proxy-split/test/p2p_proc_null.c b/test/p2p_proc_null.c similarity index 100% rename from mpi-proxy-split/test/p2p_proc_null.c rename to test/p2p_proc_null.c diff --git a/mpi-proxy-split/test/p2p_ring.c b/test/p2p_ring.c similarity index 100% rename from mpi-proxy-split/test/p2p_ring.c rename to test/p2p_ring.c diff --git a/mpi-proxy-split/test/p2p_ssend.c b/test/p2p_ssend.c similarity index 100% rename from mpi-proxy-split/test/p2p_ssend.c rename to test/p2p_ssend.c diff --git a/mpi-proxy-split/test/p2p_ssend_blocked.c b/test/p2p_ssend_blocked.c similarity index 100% rename from mpi-proxy-split/test/p2p_ssend_blocked.c rename to test/p2p_ssend_blocked.c diff --git a/mpi-proxy-split/test/p2p_ssend_collective.c b/test/p2p_ssend_collective.c similarity index 100% rename from mpi-proxy-split/test/p2p_ssend_collective.c rename to test/p2p_ssend_collective.c diff --git a/mpi-proxy-split/test/p2p_ssend_comm.c b/test/p2p_ssend_comm.c similarity index 100% rename from mpi-proxy-split/test/p2p_ssend_comm.c rename to test/p2p_ssend_comm.c diff --git a/mpi-proxy-split/test/p2p_ssend_order.c b/test/p2p_ssend_order.c similarity index 100% rename from mpi-proxy-split/test/p2p_ssend_order.c rename to test/p2p_ssend_order.c diff --git a/mpi-proxy-split/test/p2p_ssend_probe.c b/test/p2p_ssend_probe.c similarity index 100% rename from mpi-proxy-split/test/p2p_ssend_probe.c rename to test/p2p_ssend_probe.c diff --git a/mpi-proxy-split/test/p2p_ssend_sync.c b/test/p2p_ssend_sync.c similarity index 100% rename from mpi-proxy-split/test/p2p_ssend_sync.c rename to test/p2p_ssend_sync.c diff --git a/mpi-proxy-split/test/p2p_ssend_wildcard.c b/test/p2p_ssend_wildcard.c similarity index 100% rename from mpi-proxy-split/test/p2p_ssend_wildcard.c rename to test/p2p_ssend_wildcard.c diff --git a/mpi-proxy-split/test/realloc_grow.c b/test/realloc_grow.c similarity index 100% rename from mpi-proxy-split/test/realloc_grow.c rename to test/realloc_grow.c diff --git a/mpi-proxy-split/test/shared_memory.c b/test/shared_memory.c similarity index 100% rename from mpi-proxy-split/test/shared_memory.c rename to test/shared_memory.c diff --git a/mpi-proxy-split/test/synthetic_app.c b/test/synthetic_app.c similarity index 100% rename from mpi-proxy-split/test/synthetic_app.c rename to test/synthetic_app.c diff --git a/mpi-proxy-split/test/test-loader-debug.py b/test/test-loader-debug.py similarity index 99% rename from mpi-proxy-split/test/test-loader-debug.py rename to test/test-loader-debug.py index 7e33c389c..cb2029710 100644 --- a/mpi-proxy-split/test/test-loader-debug.py +++ b/test/test-loader-debug.py @@ -14,7 +14,7 @@ import unittest -SOURCE = Path(__file__).resolve().parents[1] / "lower-half" +SOURCE = Path(__file__).resolve().parents[1] / "src" / "lower-half" DRIVER = Path(__file__).with_name("loader-debug-driver.cpp") diff --git a/mpi-proxy-split/test/test-mana-restart.py b/test/test-mana-restart.py similarity index 99% rename from mpi-proxy-split/test/test-mana-restart.py rename to test/test-mana-restart.py index 98f2b9252..b50ee21be 100644 --- a/mpi-proxy-split/test/test-mana-restart.py +++ b/test/test-mana-restart.py @@ -7,7 +7,7 @@ import unittest from pathlib import Path -ROOT = Path(__file__).resolve().parents[2] +ROOT = Path(__file__).resolve().parents[1] WRAPPER = Path(os.environ.get( "MANA_RESTART_WRAPPER", ROOT / "scripts" / "mana_restart")) diff --git a/mpi-proxy-split/test/virtid_bench.c b/test/virtid_bench.c similarity index 100% rename from mpi-proxy-split/test/virtid_bench.c rename to test/virtid_bench.c diff --git a/util/hooks/pre-commit b/util/hooks/pre-commit index 6b0439f13..645930a09 100755 --- a/util/hooks/pre-commit +++ b/util/hooks/pre-commit @@ -37,8 +37,8 @@ if [ -n "$ADDED_OR_MODIFIED" ]; then # # DMTCP's style checker, dmtcp/util/dmtcp-style.py, lints only the files # in DMTCP's source directories, and it runs util/cpplint.py from the - # current directory. So run it in dmtcp/, with mpi-proxy-split/ as its - # source directory. + # current directory. So run it in dmtcp/, with MANA's directories as + # its source directories. if [ ! -f dmtcp/util/dmtcp-style.py ]; then echo "dmtcp/util/dmtcp-style.py is missing." \ "Do: git submodule update --init" @@ -55,7 +55,7 @@ spec.loader.exec_module(dmtcp_style) class ManaCppLinter(dmtcp_style.CppLinter): - source_dirs = ['../mpi-proxy-split'] + source_dirs = ['../src', '../test', '../examples'] sys.exit(ManaCppLinter().main(['../' + path for path in sys.argv[1:]])) From e9353ea98a17de091e68fd486f767d350ece4c94 Mon Sep 17 00:00:00 2001 From: Yao Xu Date: Wed, 4 Nov 2026 21:19:15 -0500 Subject: [PATCH 2/4] Move the benchmarks to bench test/ holds two benchmarks of MANA's overhead, virtid_bench and synthetic_app, next to the end-to-end tests. test/Makefile.in builds them with other flags (-O2), and 'make check' does not run them. Move them and synthetic_app's README to bench/, with a Makefile.in that has the rule from test/Makefile.in. 'make tests' still builds them. --- .gitignore | 5 ++++- Makefile.in | 3 ++- bench/Makefile.in | 23 +++++++++++++++++++++++ {test => bench}/README-synthetic_app.md | 6 +++--- {test => bench}/synthetic_app.c | 0 {test => bench}/virtid_bench.c | 0 configure | 3 ++- configure.ac | 3 ++- test/Makefile.in | 10 ++-------- util/hooks/pre-commit | 2 +- 10 files changed, 39 insertions(+), 16 deletions(-) create mode 100644 bench/Makefile.in rename {test => bench}/README-synthetic_app.md (88%) rename {test => bench}/synthetic_app.c (100%) rename {test => bench}/virtid_bench.c (100%) diff --git a/.gitignore b/.gitignore index 11372b2a7..f5ccf104b 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,7 @@ /src/mpi-wrappers/Makefile /test/Makefile /examples/Makefile +/bench/Makefile /config.log /config.status /aclocal.m4 @@ -19,6 +20,8 @@ /manpages/mana.1.gz /src/lower-half/lower-half /examples/ring +/bench/virtid_bench +/bench/synthetic_app *.[oa] *.d *.so @@ -35,7 +38,7 @@ __pycache__/ /src/mpi-wrappers/mpi_stub_objects.c /src/mpi-wrappers/*.tmp.* -# The programs of the end-to-end tests and the benchmarks have no suffix: +# The programs of the end-to-end tests have no suffix: # ignore all but the sources. /test/* !/test/*.c diff --git a/Makefile.in b/Makefile.in index b87ea642d..49ec5abcc 100644 --- a/Makefile.in +++ b/Makefile.in @@ -49,6 +49,7 @@ $(MANA_SCRIPTS:%=bin/%): bin/%: scripts/% tests: cd test && $(MAKE) cd examples && $(MAKE) + cd bench && $(MAKE) # Builds the end-to-end tests and runs them (test/autotest.py). check: mana @@ -94,7 +95,7 @@ distclean: clean - cd manpages && $(MAKE) distclean rm -f Makefile config.log config.status config.status-* config.cache rm -f src/Makefile src/lower-half/Makefile src/mpi-wrappers/Makefile \ - test/Makefile examples/Makefile + test/Makefile examples/Makefile bench/Makefile rm -rf autom4te.cache rm -rf $(top_builddir)/lib $(top_builddir)/bin diff --git a/bench/Makefile.in b/bench/Makefile.in new file mode 100644 index 000000000..011af5204 --- /dev/null +++ b/bench/Makefile.in @@ -0,0 +1,23 @@ +# @configure_input@ + +# The compilers and flags that configure found. +MPICC = @MPICC@ +MPI_CFLAGS?= @CFLAGS@ -g3 -fPIC +MPI_LDFLAGS?= + +# Benchmarks of MANA's overhead; built with -O2 and not run by 'make check'. +BENCH_FILES = virtid_bench synthetic_app + +default: ${BENCH_FILES} + +${BENCH_FILES}: %: %.c + $(MPICC) -g -O2 -o $@ $< $(MPI_CFLAGS) ${MPI_LDFLAGS} + +clean: + rm -f ${BENCH_FILES} + +.PHONY: default clean + +# Regenerate this Makefile when Makefile.in or config.status is newer. +Makefile: Makefile.in @top_builddir@/config.status + cd @top_builddir@ && ./config.status bench/Makefile diff --git a/test/README-synthetic_app.md b/bench/README-synthetic_app.md similarity index 88% rename from test/README-synthetic_app.md rename to bench/README-synthetic_app.md index b80c04a5d..10f811c8f 100644 --- a/test/README-synthetic_app.md +++ b/bench/README-synthetic_app.md @@ -23,10 +23,10 @@ same options. ## Usage ``` -make -C test synthetic_app -mpirun -np 8 test/synthetic_app --profile=vasp +make -C bench synthetic_app +mpirun -np 8 bench/synthetic_app --profile=vasp bin/mana_coordinator -mpirun -np 8 bin/mana_launch test/synthetic_app --profile=vasp +mpirun -np 8 bin/mana_launch bench/synthetic_app --profile=vasp ``` The options are listed at the top of `synthetic_app.c`. diff --git a/test/synthetic_app.c b/bench/synthetic_app.c similarity index 100% rename from test/synthetic_app.c rename to bench/synthetic_app.c diff --git a/test/virtid_bench.c b/bench/virtid_bench.c similarity index 100% rename from test/virtid_bench.c rename to bench/virtid_bench.c diff --git a/configure b/configure index 165f5722a..721326b6d 100755 --- a/configure +++ b/configure @@ -5500,7 +5500,7 @@ ac_configure_args="$ac_configure_args --disable-dlsym-wrapper" subdirs="$subdirs dmtcp" -ac_config_files="$ac_config_files Makefile manpages/Makefile src/Makefile src/lower-half/Makefile src/mpi-wrappers/Makefile test/Makefile examples/Makefile" +ac_config_files="$ac_config_files Makefile manpages/Makefile src/Makefile src/lower-half/Makefile src/mpi-wrappers/Makefile test/Makefile examples/Makefile bench/Makefile" enable_option_checking=fatal @@ -6303,6 +6303,7 @@ do "src/mpi-wrappers/Makefile") CONFIG_FILES="$CONFIG_FILES src/mpi-wrappers/Makefile" ;; "test/Makefile") CONFIG_FILES="$CONFIG_FILES test/Makefile" ;; "examples/Makefile") CONFIG_FILES="$CONFIG_FILES examples/Makefile" ;; + "bench/Makefile") CONFIG_FILES="$CONFIG_FILES bench/Makefile" ;; *) as_fn_error $? "invalid argument: \`$ac_config_target'" "$LINENO" 5;; esac diff --git a/configure.ac b/configure.ac index 9f5aa9113..399c4b027 100644 --- a/configure.ac +++ b/configure.ac @@ -84,7 +84,8 @@ AC_CONFIG_FILES([Makefile \ src/lower-half/Makefile \ src/mpi-wrappers/Makefile \ test/Makefile \ - examples/Makefile]) + examples/Makefile \ + bench/Makefile]) dnl Autoconf manual says option checking is set to warn ("yes") by dnl by default. But it's actually set to "no". diff --git a/test/Makefile.in b/test/Makefile.in index cd22c5d75..f5a417005 100644 --- a/test/Makefile.in +++ b/test/Makefile.in @@ -13,11 +13,8 @@ MPI_LDFLAGS?= # https://people.math.sc.edu/Burkardt/f_src/mpi/mpi.html ############################################################### -# Benchmarks of MANA's overhead; built with -O2 and not run by 'make check'. -BENCH_FILES = virtid_bench synthetic_app - default: default_prereqs - + $(MAKE) ${BENCH_FILES} e2e + + $(MAKE) e2e # End-to-end tests, run by autotest.py natively and under MANA. Each test is # one program, named after the test and linked as any MPI program. @@ -51,9 +48,6 @@ dlopen_mpi: dlopen_mpi.c ${E2E_FORTRAN}: %: %.f90 $(MPIFC) ${FFLAGS} -g3 -O0 -o $@ $< -${BENCH_FILES}: %: %.c - $(MPICC) -g -O2 -o $@ $< $(MPI_CFLAGS) ${MPI_LDFLAGS} - check: e2e ./autotest.py @@ -81,7 +75,7 @@ tidy: clean: tidy rm -f *.pyc *.so - rm -f ${E2E_TESTS} ${E2E_FORTRAN} ${BENCH_FILES} + rm -f ${E2E_TESTS} ${E2E_FORTRAN} # Regenerate this Makefile when Makefile.in or config.status is newer. Makefile: Makefile.in @top_builddir@/config.status diff --git a/util/hooks/pre-commit b/util/hooks/pre-commit index 645930a09..cc3d38c3b 100755 --- a/util/hooks/pre-commit +++ b/util/hooks/pre-commit @@ -55,7 +55,7 @@ spec.loader.exec_module(dmtcp_style) class ManaCppLinter(dmtcp_style.CppLinter): - source_dirs = ['../src', '../test', '../examples'] + source_dirs = ['../src', '../test', '../examples', '../bench'] sys.exit(ManaCppLinter().main(['../' + path for path in sys.argv[1:]])) From 89ec720f35d461eaeb6f4a7aa52732e683b2c2aa Mon Sep 17 00:00:00 2001 From: Yao Xu Date: Wed, 4 Nov 2026 21:20:44 -0500 Subject: [PATCH 3/4] Rename p2p_drain_send_recv to p2p_drain The files hold the drain of the point-to-point messages at a checkpoint. Name them after it: p2p_drain.{cpp,h}. --- src/Makefile.in | 2 +- src/mpi-wrappers/mpi_cart_wrappers.cpp | 2 +- src/mpi-wrappers/mpi_collective_wrappers.cpp | 2 +- src/mpi-wrappers/mpi_comm_wrappers.cpp | 2 +- src/mpi-wrappers/mpi_p2p_wrappers.cpp | 2 +- src/mpi-wrappers/mpi_request_wrappers.cpp | 2 +- src/mpi-wrappers/mpi_wrappers.cpp | 2 +- src/mpi_plugin.cpp | 14 +++++++------- src/{p2p_drain_send_recv.cpp => p2p_drain.cpp} | 4 ++-- src/{p2p_drain_send_recv.h => p2p_drain.h} | 0 src/p2p_log_replay.cpp | 2 +- src/p2p_log_replay.h | 2 +- 12 files changed, 18 insertions(+), 18 deletions(-) rename src/{p2p_drain_send_recv.cpp => p2p_drain.cpp} (99%) rename src/{p2p_drain_send_recv.h => p2p_drain.h} (100%) diff --git a/src/Makefile.in b/src/Makefile.in index a3f49cc53..46efcb9e9 100644 --- a/src/Makefile.in +++ b/src/Makefile.in @@ -22,7 +22,7 @@ WRAPPERS_SRCDIR=mpi-wrappers # As you add new files to your plugin library, add the object file names here. -LIBOBJS = uh_wrappers.o mpi_plugin.o p2p_drain_send_recv.o p2p_log_replay.o \ +LIBOBJS = uh_wrappers.o mpi_plugin.o p2p_drain.o p2p_log_replay.o \ seq_num.o virtual_id.o lower_half_ckpt.o \ lower-half/switch-context.o \ diff --git a/src/mpi-wrappers/mpi_cart_wrappers.cpp b/src/mpi-wrappers/mpi_cart_wrappers.cpp index 394f04112..d6949f71d 100644 --- a/src/mpi-wrappers/mpi_cart_wrappers.cpp +++ b/src/mpi-wrappers/mpi_cart_wrappers.cpp @@ -31,7 +31,7 @@ #include "mpi_nextfunc.h" #include "virtual_id.h" #include "seq_num.h" -#include "p2p_drain_send_recv.h" +#include "p2p_drain.h" extern "C" { diff --git a/src/mpi-wrappers/mpi_collective_wrappers.cpp b/src/mpi-wrappers/mpi_collective_wrappers.cpp index 0ef70138f..b53fd2a6d 100644 --- a/src/mpi-wrappers/mpi_collective_wrappers.cpp +++ b/src/mpi-wrappers/mpi_collective_wrappers.cpp @@ -32,7 +32,7 @@ #include "seq_num.h" #include "virtual_id.h" #include "p2p_log_replay.h" -#include "p2p_drain_send_recv.h" +#include "p2p_drain.h" extern "C" { diff --git a/src/mpi-wrappers/mpi_comm_wrappers.cpp b/src/mpi-wrappers/mpi_comm_wrappers.cpp index 9a2ecd93c..d4cfd2cbc 100644 --- a/src/mpi-wrappers/mpi_comm_wrappers.cpp +++ b/src/mpi-wrappers/mpi_comm_wrappers.cpp @@ -38,7 +38,7 @@ #include "mpi_nextfunc.h" #include "virtual_id.h" #include "seq_num.h" -#include "p2p_drain_send_recv.h" +#include "p2p_drain.h" // TODO // - validate operation status (right now we assume them to be successful by diff --git a/src/mpi-wrappers/mpi_p2p_wrappers.cpp b/src/mpi-wrappers/mpi_p2p_wrappers.cpp index cb31cbbbf..aa569d05c 100644 --- a/src/mpi-wrappers/mpi_p2p_wrappers.cpp +++ b/src/mpi-wrappers/mpi_p2p_wrappers.cpp @@ -28,7 +28,7 @@ #include "mpi_plugin.h" #include "p2p_log_replay.h" -#include "p2p_drain_send_recv.h" +#include "p2p_drain.h" #include "jfilesystem.h" #include "protectedfds.h" #include "mpi_nextfunc.h" diff --git a/src/mpi-wrappers/mpi_request_wrappers.cpp b/src/mpi-wrappers/mpi_request_wrappers.cpp index af17e1076..b64607a92 100644 --- a/src/mpi-wrappers/mpi_request_wrappers.cpp +++ b/src/mpi-wrappers/mpi_request_wrappers.cpp @@ -31,7 +31,7 @@ #include "protectedfds.h" #include "p2p_log_replay.h" -#include "p2p_drain_send_recv.h" +#include "p2p_drain.h" #include "mpi_plugin.h" #include "mpi_nextfunc.h" #include "virtual_id.h" diff --git a/src/mpi-wrappers/mpi_wrappers.cpp b/src/mpi-wrappers/mpi_wrappers.cpp index b648c3927..a821e9e25 100644 --- a/src/mpi-wrappers/mpi_wrappers.cpp +++ b/src/mpi-wrappers/mpi_wrappers.cpp @@ -31,7 +31,7 @@ #include "protectedfds.h" #include "mpi_nextfunc.h" #include "virtual_id.h" -#include "p2p_drain_send_recv.h" +#include "p2p_drain.h" #include "seq_num.h" #include "uh_wrappers.h" diff --git a/src/mpi_plugin.cpp b/src/mpi_plugin.cpp index eacd951ec..53e4eaae8 100644 --- a/src/mpi_plugin.cpp +++ b/src/mpi_plugin.cpp @@ -43,7 +43,7 @@ #include "lower-half-api.h" #include "lower_half_ckpt.h" #include "p2p_log_replay.h" -#include "p2p_drain_send_recv.h" +#include "p2p_drain.h" #include "seq_num.h" #include "mpi_nextfunc.h" #include "virtual_id.h" @@ -843,7 +843,7 @@ mpi_plugin_event_hook(DmtcpEvent_t event, DmtcpEventData_t *data) while (!__atomic_load_n(&g_libmpi_is_initialized, __ATOMIC_ACQUIRE)) { usleep(1000); } - resetDrainStats(); // p2p_drain_send_recv.cpp + resetDrainStats(); // p2p_drain.cpp uint64_t t0 = drainStatsNow(); mana_state = CKPT_COLLECTIVE; // preSuspendBarrier() will send coord response and get worker state. @@ -871,7 +871,7 @@ mpi_plugin_event_hook(DmtcpEvent_t event, DmtcpEventData_t *data) if (own_tls) { lh_info->ckpt_uh_fs = (void *)getFS(); } - drainP2p(); // p2p_drain_send_recv.cpp + drainP2p(); // p2p_drain.cpp lh_info->ckpt_uh_fs = NULL; openCkptFileFds(); if (g_p2p_wait == P2P_WAIT_BLOCKING) { @@ -917,10 +917,10 @@ mpi_plugin_event_hook(DmtcpEvent_t event, DmtcpEventData_t *data) printEventToStderr("EVENT_RESUME"); processingOpenCkpFileFds = false; dmtcp_local_barrier("MPI:Reset-Drain-Send-Recv-Counters"); - resetDrainCounters(); // p2p_drain_send_recv.cpp + resetDrainCounters(); // p2p_drain.cpp seq_num_reset(); dmtcp_local_barrier("MPI:seq_num_reset"); - sendOwedSsendAcks(); // p2p_drain_send_recv.cpp + sendOwedSsendAcks(); // p2p_drain.cpp mana_state = RUNNING; printEventToStderr("EVENT_RESUME (done)"); break; @@ -947,14 +947,14 @@ mpi_plugin_event_hook(DmtcpEvent_t event, DmtcpEventData_t *data) check_tag_ub(); reconstruct_descriptors(); dmtcp_local_barrier("MPI:Reset-Drain-Send-Recv-Counters"); - resetDrainCounters(); // p2p_drain_send_recv.cpp + resetDrainCounters(); // p2p_drain.cpp char str[1]; MPI_MANA_Internal(str); // This does nothing. Modify in lower-half // for easy debugging of lower half during restart. // See definition in mpi-wrappers/mpi_wrappers.cpp dmtcp_global_barrier("MPI:replayMpiP2pOnRestart"); replayMpiP2pOnRestart(); // p2p_log_replay.cpp - sendOwedSsendAcks(); // p2p_drain_send_recv.cpp + sendOwedSsendAcks(); // p2p_drain.cpp dmtcp_local_barrier("MPI:p2p_log_replay.cpp-void"); const char *file = get_mpi_file_filename(); restore_mpi_files(file); diff --git a/src/p2p_drain_send_recv.cpp b/src/p2p_drain.cpp similarity index 99% rename from src/p2p_drain_send_recv.cpp rename to src/p2p_drain.cpp index a29535ee5..5a18aee79 100644 --- a/src/p2p_drain_send_recv.cpp +++ b/src/p2p_drain.cpp @@ -30,7 +30,7 @@ #include #include "kvdb.h" #include "dmtcp_assert.h" -#include "p2p_drain_send_recv.h" +#include "p2p_drain.h" #include "p2p_log_replay.h" #include "mpi_nextfunc.h" #include "seq_num.h" @@ -69,7 +69,7 @@ unlockMessageQueue() __atomic_clear(&g_message_queue_lock, __ATOMIC_RELEASE); } -// See p2p_drain_send_recv.h for documentation of these globals. +// See p2p_drain.h for documentation of these globals. pending_recv_t g_pending_recv = { /*.state=*/ PENDING_RECV_IDLE }; MPI_Comm g_ssend_ack_comm = MPI_COMM_NULL; volatile bool p2p_dummy_phase = false; diff --git a/src/p2p_drain_send_recv.h b/src/p2p_drain.h similarity index 100% rename from src/p2p_drain_send_recv.h rename to src/p2p_drain.h diff --git a/src/p2p_log_replay.cpp b/src/p2p_log_replay.cpp index ab323efd0..3a20ec2e0 100644 --- a/src/p2p_log_replay.cpp +++ b/src/p2p_log_replay.cpp @@ -36,7 +36,7 @@ #include "mpi_plugin.h" #include "mpi_nextfunc.h" #include "p2p_log_replay.h" -#include "p2p_drain_send_recv.h" +#include "p2p_drain.h" #include "virtual_id.h" using namespace dmtcp; diff --git a/src/p2p_log_replay.h b/src/p2p_log_replay.h index c3a76cc97..2bd85c75b 100644 --- a/src/p2p_log_replay.h +++ b/src/p2p_log_replay.h @@ -30,7 +30,7 @@ #include "virtual_id.h" // Struct to store and return the MPI message (data) during draining and -// resuming, also used by p2p_drain_send_recv.h +// resuming, also used by p2p_drain.h typedef struct __mpi_message { void *buf; From 480c248f246029548d6a385850c2bfa528e6db1d Mon Sep 17 00:00:00 2001 From: Yao Xu Date: Wed, 4 Nov 2026 21:26:51 -0500 Subject: [PATCH 4/4] Fold p2p_log_replay into p2p_drain and mpi_plugin p2p_log_replay.{cpp,h} holds the list of pending MPI_Isend and MPI_Irecv calls, which the P2P drain uses and from which the restart posts the receives again. It also holds two helpers of mpi_plugin.cpp: the world rank and size, and the checkpoint directory of each rank. Move the pending-call list, mpi_message_t and replayMpiP2pOnRestart() to p2p_drain.{cpp,h}. Move g_world_rank, g_world_size, getLocalRankInfo() and updateCkptDirByRank() to mpi_plugin.{cpp,h}. The code itself does not change. --- src/Makefile.in | 2 +- src/mpi-wrappers/mpi_collective_wrappers.cpp | 1 - src/mpi-wrappers/mpi_p2p_wrappers.cpp | 1 - src/mpi-wrappers/mpi_request_wrappers.cpp | 1 - src/mpi_plugin.cpp | 39 ++- src/mpi_plugin.h | 11 + src/p2p_drain.cpp | 221 +++++++++++++- src/p2p_drain.h | 56 +++- src/p2p_log_replay.cpp | 295 ------------------- src/p2p_log_replay.h | 95 ------ src/virtual_id.cpp | 3 +- src/virtual_id.h | 2 +- 12 files changed, 326 insertions(+), 401 deletions(-) delete mode 100644 src/p2p_log_replay.cpp delete mode 100644 src/p2p_log_replay.h diff --git a/src/Makefile.in b/src/Makefile.in index 46efcb9e9..91cd18588 100644 --- a/src/Makefile.in +++ b/src/Makefile.in @@ -22,7 +22,7 @@ WRAPPERS_SRCDIR=mpi-wrappers # As you add new files to your plugin library, add the object file names here. -LIBOBJS = uh_wrappers.o mpi_plugin.o p2p_drain.o p2p_log_replay.o \ +LIBOBJS = uh_wrappers.o mpi_plugin.o p2p_drain.o \ seq_num.o virtual_id.o lower_half_ckpt.o \ lower-half/switch-context.o \ diff --git a/src/mpi-wrappers/mpi_collective_wrappers.cpp b/src/mpi-wrappers/mpi_collective_wrappers.cpp index b53fd2a6d..19d3eee28 100644 --- a/src/mpi-wrappers/mpi_collective_wrappers.cpp +++ b/src/mpi-wrappers/mpi_collective_wrappers.cpp @@ -31,7 +31,6 @@ #include "mpi_nextfunc.h" #include "seq_num.h" #include "virtual_id.h" -#include "p2p_log_replay.h" #include "p2p_drain.h" extern "C" { diff --git a/src/mpi-wrappers/mpi_p2p_wrappers.cpp b/src/mpi-wrappers/mpi_p2p_wrappers.cpp index aa569d05c..c1e03ae00 100644 --- a/src/mpi-wrappers/mpi_p2p_wrappers.cpp +++ b/src/mpi-wrappers/mpi_p2p_wrappers.cpp @@ -27,7 +27,6 @@ #include "lower_half_ckpt.h" #include "mpi_plugin.h" -#include "p2p_log_replay.h" #include "p2p_drain.h" #include "jfilesystem.h" #include "protectedfds.h" diff --git a/src/mpi-wrappers/mpi_request_wrappers.cpp b/src/mpi-wrappers/mpi_request_wrappers.cpp index b64607a92..96a9aaa08 100644 --- a/src/mpi-wrappers/mpi_request_wrappers.cpp +++ b/src/mpi-wrappers/mpi_request_wrappers.cpp @@ -30,7 +30,6 @@ #include "jfilesystem.h" #include "protectedfds.h" -#include "p2p_log_replay.h" #include "p2p_drain.h" #include "mpi_plugin.h" #include "mpi_nextfunc.h" diff --git a/src/mpi_plugin.cpp b/src/mpi_plugin.cpp index 53e4eaae8..b89f4576f 100644 --- a/src/mpi_plugin.cpp +++ b/src/mpi_plugin.cpp @@ -42,7 +42,6 @@ #include "mpi_plugin.h" #include "lower-half-api.h" #include "lower_half_ckpt.h" -#include "p2p_log_replay.h" #include "p2p_drain.h" #include "seq_num.h" #include "mpi_nextfunc.h" @@ -83,6 +82,9 @@ EXTERNC pid_t dmtcp_get_real_tid() __attribute((weak)); mana_state_t mana_state = UNKNOWN_STATE; +int g_world_rank = -1; // Global rank of the current process +int g_world_size = -1; // Total number of ranks in the current computation + ProcSelfMaps *preMpiInitMaps = nullptr; ProcSelfMaps *postMpiInitMaps = nullptr; map *mpiInitLhAreas = nullptr; @@ -744,6 +746,37 @@ close_lower_half_except_blocked() g_drain_stats.t_wait_lower_half += drainStatsNow() - t; } +void +getLocalRankInfo() +{ + if (g_world_rank == -1) { + ASSERT(MPI_Comm_rank(MPI_COMM_WORLD, &g_world_rank) == MPI_SUCCESS && + g_world_rank != -1, "MPI_Comm_rank failed"); + } + if (g_world_size == -1) { + ASSERT(MPI_Comm_size(MPI_COMM_WORLD, &g_world_size) == MPI_SUCCESS && + g_world_size != -1, "MPI_Comm_size failed"); + } +} + +void +updateCkptDirByRank() +{ + const char *ckptDir = dmtcp_get_ckpt_dir(); + dmtcp::string baseDir; + + if (strstr(ckptDir, "ckpt_rank_") != NULL) { + baseDir = jalib::Filesystem::DirName(ckptDir); + } else { + baseDir = ckptDir; + } + TRACE("Updating checkpoint directory: ckptDir={} baseDir={}", ckptDir, + baseDir); + dmtcp::ostringstream o; + o << baseDir << "/ckpt_rank_" << g_world_rank; + dmtcp_set_ckpt_dir(o.str().c_str()); +} + static void mpi_plugin_event_hook(DmtcpEvent_t event, DmtcpEventData_t *data) { @@ -898,7 +931,7 @@ mpi_plugin_event_hook(DmtcpEvent_t event, DmtcpEventData_t *data) allow_threads_to_enter_lower_half(); recordOpenFds(); dmtcp_local_barrier("MPI:GetLocalRankInfo"); - getLocalRankInfo(); // p2p_log_replay.cpp + getLocalRankInfo(); dmtcp_global_barrier("MPI:update-ckpt-dir-by-rank"); updateCkptDirByRank(); // mpi_plugin.cpp // computeUnionOfCkptImageAddresses(); @@ -953,7 +986,7 @@ mpi_plugin_event_hook(DmtcpEvent_t event, DmtcpEventData_t *data) // for easy debugging of lower half during restart. // See definition in mpi-wrappers/mpi_wrappers.cpp dmtcp_global_barrier("MPI:replayMpiP2pOnRestart"); - replayMpiP2pOnRestart(); // p2p_log_replay.cpp + replayMpiP2pOnRestart(); // p2p_drain.cpp sendOwedSsendAcks(); // p2p_drain.cpp dmtcp_local_barrier("MPI:p2p_log_replay.cpp-void"); const char *file = get_mpi_file_filename(); diff --git a/src/mpi_plugin.h b/src/mpi_plugin.h index c6387ed85..ab46dc1aa 100644 --- a/src/mpi_plugin.h +++ b/src/mpi_plugin.h @@ -48,4 +48,15 @@ extern mana_state_t mana_state; extern bool g_libmana_is_initialized; extern bool g_libmpi_is_initialized; // MANA's MPI_Init has returned +extern int g_world_rank; // Global rank of the current process +extern int g_world_size; // Total number of ranks in the current computation + +// Fetches the MPI rank and world size; also, verifies that MPI rank and +// world size match the globally stored values in the plugin +extern void getLocalRankInfo(); + +// Sets the name of the checkpoint directory of the current process to +// "ckpt_rank_", where RANK is the MPI rank of the process. +extern void updateCkptDirByRank(); + #endif // ifndef _MPI_PLUGIN_H diff --git a/src/p2p_drain.cpp b/src/p2p_drain.cpp index 5a18aee79..650712610 100644 --- a/src/p2p_drain.cpp +++ b/src/p2p_drain.cpp @@ -31,7 +31,7 @@ #include "kvdb.h" #include "dmtcp_assert.h" #include "p2p_drain.h" -#include "p2p_log_replay.h" +#include "mpi_plugin.h" #include "mpi_nextfunc.h" #include "seq_num.h" #include "virtual_id.h" @@ -1187,3 +1187,222 @@ reportDrainStats() v["isends_completed"], v["blocked"], v["dummies"], v["issends"], v["issends_buffered"], v["kvdb_requests"], v["done_polls"]); } + +// The pending MPI_Isend/MPI_Irecv calls, stored in their requests' +// virtual-ID entries and linked in posting order. Application threads and +// the checkpoint thread (P2P drain, restart) both change the list. The +// spinlock pendingLock guards the links and call types; never hold it across +// an MPI call. +static virt_id_entry *pendingHead = NULL; +static virt_id_entry *pendingTail = NULL; +static int pendingLock = 0; +// The pending MPI_Issends: changed under pendingLock, read without it. +static int pendingIssends = 0; + +static inline void +lockPending() +{ + while (__atomic_exchange_n(&pendingLock, 1, __ATOMIC_ACQUIRE)) { + while (__atomic_load_n(&pendingLock, __ATOMIC_RELAXED)) { + } + } +} + +static inline void +unlockPending() +{ + __atomic_store_n(&pendingLock, 0, __ATOMIC_RELEASE); +} + +void +addPendingRequestToLog(mpi_req_t req, const void* sbuf, void* rbuf, int cnt, + MPI_Datatype type, int remote, int tag, + MPI_Comm comm, MPI_Request rq) +{ + virt_id_entry *entry = get_virt_id_entry((mana_mpi_handle){.request = rq}); + mpi_nonblocking_call_t *call = &entry->call; + call->sendbuf = sbuf; + call->recvbuf = rbuf; + call->count = cnt; + call->datatype = type; + call->remote_node = remote; + call->tag = tag; + call->comm = comm; + call->later_sends = 0; + lockPending(); + call->type = req; + if (req == ISSEND_REQUEST) { + __atomic_fetch_add(&pendingIssends, 1, __ATOMIC_RELAXED); + } + entry->pending_prev = pendingTail; + entry->pending_next = NULL; + if (pendingTail != NULL) { + pendingTail->pending_next = entry; + } else { + pendingHead = entry; + } + pendingTail = entry; + unlockPending(); +} + +void +clearPendingRequestFromLog(MPI_Request req) +{ + // Look the request up under the lock: while the P2P drain clears it, the + // application may free it and a new request may take its slot. + lockPending(); + virt_id_entry *entry = + lookup_virt_id_entry((mana_mpi_handle){.request = req}); + if (entry == NULL) { + unlockPending(); + return; + } + if (entry->call.type != UNKNOW_REQUEST) { + if (entry->call.type == ISSEND_REQUEST) { + __atomic_fetch_sub(&pendingIssends, 1, __ATOMIC_RELAXED); + } + if (entry->pending_prev != NULL) { + entry->pending_prev->pending_next = entry->pending_next; + } else { + pendingHead = entry->pending_next; + } + if (entry->pending_next != NULL) { + entry->pending_next->pending_prev = entry->pending_prev; + } else { + pendingTail = entry->pending_prev; + } + entry->pending_prev = NULL; + entry->pending_next = NULL; + entry->call.type = UNKNOW_REQUEST; + } + unlockPending(); +} + +void +replacePendingCall(MPI_Request req, const mpi_nonblocking_call_t *call) +{ + lockPending(); + virt_id_entry *entry = + lookup_virt_id_entry((mana_mpi_handle){.request = req}); + ASSERT(entry != NULL && entry->call.type != UNKNOW_REQUEST, "req={}", req); + if (entry->call.type == ISSEND_REQUEST && call->type != ISSEND_REQUEST) { + __atomic_fetch_sub(&pendingIssends, 1, __ATOMIC_RELAXED); + } + entry->call = *call; + unlockPending(); +} + +void +countSendAfterIssends(MPI_Comm comm, int dest, int tag) +{ + if (__atomic_load_n(&pendingIssends, __ATOMIC_RELAXED) == 0) { + return; + } + lockPending(); + for (virt_id_entry *entry = pendingHead; entry != NULL; + entry = entry->pending_next) { + mpi_nonblocking_call_t *call = &entry->call; + if (call->type == ISSEND_REQUEST && call->comm == comm && + call->remote_node == dest && call->tag == tag) { + call->later_sends++; + } + } + unlockPending(); +} + +std::vector +pendingRequestsInPostingOrder() +{ + std::vector requests; + lockPending(); + for (virt_id_entry *entry = pendingHead; entry != NULL; + entry = entry->pending_next) { + requests.push_back((MPI_Request)entry->virt); + } + unlockPending(); + return requests; +} + +mpi_req_t +pendingRequestType(MPI_Request req) +{ + virt_id_entry *entry = + lookup_virt_id_entry((mana_mpi_handle){.request = req}); + if (entry == NULL) { + return UNKNOW_REQUEST; + } + return __atomic_load_n(&entry->call.type, __ATOMIC_RELAXED); +} + +bool +getPendingCall(MPI_Request req, mpi_nonblocking_call_t *call) +{ + virt_id_entry *entry = + lookup_virt_id_entry((mana_mpi_handle){.request = req}); + if (entry == NULL) { + return false; + } + lockPending(); + *call = entry->call; + unlockPending(); + return call->type != UNKNOW_REQUEST; +} + +bool +pendingCallUsesDatatype(MPI_Datatype type) +{ + bool found = false; + lockPending(); + for (virt_id_entry *entry = pendingHead; entry != NULL; + entry = entry->pending_next) { + if (entry->call.datatype == type) { + found = true; + break; + } + } + unlockPending(); + return found; +} + +void +replayMpiP2pOnRestart() +{ + MPI_Request request; + mpi_nonblocking_call_t pendingCall; + mpi_nonblocking_call_t *call = &pendingCall; + TRACE("Replaying unserviced isend/irecv calls"); + + // No other thread runs at restart; a lock saved in the image is stale. + pendingLock = 0; + // Re-post the receives in the order they were posted. + for (MPI_Request pending : pendingRequestsInPostingOrder()) { + int retval = 0; + request = pending; + if (!getPendingCall(request, call)) { + continue; + } + MPI_Comm realComm = get_real_id((mana_mpi_handle){.comm = call->comm}).comm; + MPI_Datatype realType = get_real_id((mana_mpi_handle){.datatype = call->datatype}).datatype; + MPI_Request realRequest; + switch (call->type) { + case IRECV_REQUEST: + TRACE("Replaying Irecv call: remote_node={}", call->remote_node); + JUMP_TO_LOWER_HALF(lh_info->fsaddr); + NEXT_FUNC(Irecv)(call->recvbuf, call->count, + realType, call->remote_node, + call->tag, realComm, &realRequest); + RETURN_TO_UPPER_HALF(); + update_virt_id((mana_mpi_handle){.request = request}, (mana_mpi_handle){.request = realRequest}); + ASSERT(retval == MPI_SUCCESS, "Error while replaying recv"); + break; + case ISEND_REQUEST: + case ISSEND_REQUEST: + ASSERT(false, "There should be no pending MPI_Isend after restart: " + "type={}", call->type); + break; + default: + WARN(false, "Unhandled replay call: type={}", call->type); + break; + } + } +} diff --git a/src/p2p_drain.h b/src/p2p_drain.h index afc952bd4..9bb35e72d 100644 --- a/src/p2p_drain.h +++ b/src/p2p_drain.h @@ -22,10 +22,29 @@ #ifndef _P2P_SEND_RECV_H #define _P2P_SEND_RECV_H +#include +#include #include +#include #include "dmtcp.h" #include "dmtcpalloc.h" -#include "p2p_log_replay.h" +#include "virtual_id.h" + +// Struct to store and return the MPI message (data) during draining and +// resuming +typedef struct __mpi_message +{ + void *buf; + int count; + MPI_Datatype datatype; + int size; + MPI_Comm comm; + MPI_Status status; + // An MPI_Issend's: its sender's world rank, to ack when it is received, + // and the ack's tag (see publishSsendNotices()). Otherwise ack_to is -1. + int ack_to; + int ack_tag; +} mpi_message_t; extern int64_t global_sent_messages, global_recv_messages; extern int64_t local_sent_messages, local_recv_messages; @@ -180,4 +199,39 @@ int consumeMatchingMsgBuffer(void *buf, int count, MPI_Datatype datatype, int source, int tag, MPI_Comm comm, MPI_Status *mpi_status, int size); void resetDrainCounters(); + +// Restores the state of MPI P2P communication by replaying any pending +// MPI_Isend and MPI_Irecv requests post restart +extern void replayMpiP2pOnRestart(); + +// Saves a nonblocking send/recv call in the virtual-ID entry of request 'rq'. +extern void addPendingRequestToLog(mpi_req_t , const void* , void* , int , + MPI_Datatype , int , int , + MPI_Comm, MPI_Request); + +// remove finished send/recv call from the pending calls +extern void clearPendingRequestFromLog(MPI_Request req); + +// Replaces the call of a pending request; it keeps its place in the log. +extern void replacePendingCall(MPI_Request req, + const mpi_nonblocking_call_t *call); + +// Counts a send in the later_sends of each pending MPI_Issend with its +// envelope. Call it before the send enters the lower half. +extern void countSendAfterIssends(MPI_Comm comm, int dest, int tag); + +// Returns the pending requests in posting order (not handle order). MPI +// matches receives in that order, so the drain and restart replay use it. +extern std::vector pendingRequestsInPostingOrder(); + +// Returns the type of a pending request, or UNKNOW_REQUEST if the request is +// not (or no longer) pending. +extern mpi_req_t pendingRequestType(MPI_Request req); + +// Copies the call of a pending request to *call. Returns false if the +// request is not (or no longer) pending. +extern bool getPendingCall(MPI_Request req, mpi_nonblocking_call_t *call); + +// Returns true if a pending MPI_Isend/MPI_Irecv uses the datatype. +extern bool pendingCallUsesDatatype(MPI_Datatype type); #endif diff --git a/src/p2p_log_replay.cpp b/src/p2p_log_replay.cpp deleted file mode 100644 index 3a20ec2e0..000000000 --- a/src/p2p_log_replay.cpp +++ /dev/null @@ -1,295 +0,0 @@ -/**************************************************************************** - * Copyright (C) 2019-2021 by Gene Cooperman, Rohan Garg, Yao Xu * - * gene@ccs.neu.edu, rohgarg@ccs.neu.edu, xu.yao1@northeastern.edu * - * * - * This file is part of DMTCP. * - * * - * DMTCP is free software: you can redistribute it and/or * - * modify it under the terms of the GNU Lesser General Public License as * - * published by the Free Software Foundation, either version 3 of the * - * License, or (at your option) any later version. * - * * - * DMTCP is distributed in the hope that it will be useful, * - * but WITHOUT ANY WARRANTY; without even the implied warranty of * - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * - * GNU Lesser General Public License for more details. * - * * - * You should have received a copy of the GNU Lesser General Public * - * License in the files COPYING and COPYING.LESSER. If not, see * - * . * - ****************************************************************************/ - -#include -#include -#include -#include -#include -#include -#include -#include - -#include "dmtcp.h" -#include "util.h" -#include "dmtcp_assert.h" -#include "jfilesystem.h" - -#include "mpi_plugin.h" -#include "mpi_nextfunc.h" -#include "p2p_log_replay.h" -#include "p2p_drain.h" -#include "virtual_id.h" - -using namespace dmtcp; - -int g_world_rank = -1; // Global rank of the current process -int g_world_size = -1; // Total number of ranks in the current computation - -// The pending MPI_Isend/MPI_Irecv calls, stored in their requests' -// virtual-ID entries and linked in posting order. Application threads and -// the checkpoint thread (P2P drain, restart) both change the list. The -// spinlock pendingLock guards the links and call types; never hold it across -// an MPI call. -static virt_id_entry *pendingHead = NULL; -static virt_id_entry *pendingTail = NULL; -static int pendingLock = 0; -// The pending MPI_Issends: changed under pendingLock, read without it. -static int pendingIssends = 0; - -static inline void -lockPending() -{ - while (__atomic_exchange_n(&pendingLock, 1, __ATOMIC_ACQUIRE)) { - while (__atomic_load_n(&pendingLock, __ATOMIC_RELAXED)) { - } - } -} - -static inline void -unlockPending() -{ - __atomic_store_n(&pendingLock, 0, __ATOMIC_RELEASE); -} - -void -getLocalRankInfo() -{ - if (g_world_rank == -1) { - ASSERT(MPI_Comm_rank(MPI_COMM_WORLD, &g_world_rank) == MPI_SUCCESS && - g_world_rank != -1, "MPI_Comm_rank failed"); - } - if (g_world_size == -1) { - ASSERT(MPI_Comm_size(MPI_COMM_WORLD, &g_world_size) == MPI_SUCCESS && - g_world_size != -1, "MPI_Comm_size failed"); - } -} - -void -updateCkptDirByRank() -{ - const char *ckptDir = dmtcp_get_ckpt_dir(); - dmtcp::string baseDir; - - if (strstr(ckptDir, "ckpt_rank_") != NULL) { - baseDir = jalib::Filesystem::DirName(ckptDir); - } else { - baseDir = ckptDir; - } - TRACE("Updating checkpoint directory: ckptDir={} baseDir={}", ckptDir, - baseDir); - dmtcp::ostringstream o; - o << baseDir << "/ckpt_rank_" << g_world_rank; - dmtcp_set_ckpt_dir(o.str().c_str()); -} - -void -addPendingRequestToLog(mpi_req_t req, const void* sbuf, void* rbuf, int cnt, - MPI_Datatype type, int remote, int tag, - MPI_Comm comm, MPI_Request rq) -{ - virt_id_entry *entry = get_virt_id_entry((mana_mpi_handle){.request = rq}); - mpi_nonblocking_call_t *call = &entry->call; - call->sendbuf = sbuf; - call->recvbuf = rbuf; - call->count = cnt; - call->datatype = type; - call->remote_node = remote; - call->tag = tag; - call->comm = comm; - call->later_sends = 0; - lockPending(); - call->type = req; - if (req == ISSEND_REQUEST) { - __atomic_fetch_add(&pendingIssends, 1, __ATOMIC_RELAXED); - } - entry->pending_prev = pendingTail; - entry->pending_next = NULL; - if (pendingTail != NULL) { - pendingTail->pending_next = entry; - } else { - pendingHead = entry; - } - pendingTail = entry; - unlockPending(); -} - -void -clearPendingRequestFromLog(MPI_Request req) -{ - // Look the request up under the lock: while the P2P drain clears it, the - // application may free it and a new request may take its slot. - lockPending(); - virt_id_entry *entry = - lookup_virt_id_entry((mana_mpi_handle){.request = req}); - if (entry == NULL) { - unlockPending(); - return; - } - if (entry->call.type != UNKNOW_REQUEST) { - if (entry->call.type == ISSEND_REQUEST) { - __atomic_fetch_sub(&pendingIssends, 1, __ATOMIC_RELAXED); - } - if (entry->pending_prev != NULL) { - entry->pending_prev->pending_next = entry->pending_next; - } else { - pendingHead = entry->pending_next; - } - if (entry->pending_next != NULL) { - entry->pending_next->pending_prev = entry->pending_prev; - } else { - pendingTail = entry->pending_prev; - } - entry->pending_prev = NULL; - entry->pending_next = NULL; - entry->call.type = UNKNOW_REQUEST; - } - unlockPending(); -} - -void -replacePendingCall(MPI_Request req, const mpi_nonblocking_call_t *call) -{ - lockPending(); - virt_id_entry *entry = - lookup_virt_id_entry((mana_mpi_handle){.request = req}); - ASSERT(entry != NULL && entry->call.type != UNKNOW_REQUEST, "req={}", req); - if (entry->call.type == ISSEND_REQUEST && call->type != ISSEND_REQUEST) { - __atomic_fetch_sub(&pendingIssends, 1, __ATOMIC_RELAXED); - } - entry->call = *call; - unlockPending(); -} - -void -countSendAfterIssends(MPI_Comm comm, int dest, int tag) -{ - if (__atomic_load_n(&pendingIssends, __ATOMIC_RELAXED) == 0) { - return; - } - lockPending(); - for (virt_id_entry *entry = pendingHead; entry != NULL; - entry = entry->pending_next) { - mpi_nonblocking_call_t *call = &entry->call; - if (call->type == ISSEND_REQUEST && call->comm == comm && - call->remote_node == dest && call->tag == tag) { - call->later_sends++; - } - } - unlockPending(); -} - -std::vector -pendingRequestsInPostingOrder() -{ - std::vector requests; - lockPending(); - for (virt_id_entry *entry = pendingHead; entry != NULL; - entry = entry->pending_next) { - requests.push_back((MPI_Request)entry->virt); - } - unlockPending(); - return requests; -} - -mpi_req_t -pendingRequestType(MPI_Request req) -{ - virt_id_entry *entry = - lookup_virt_id_entry((mana_mpi_handle){.request = req}); - if (entry == NULL) { - return UNKNOW_REQUEST; - } - return __atomic_load_n(&entry->call.type, __ATOMIC_RELAXED); -} - -bool -getPendingCall(MPI_Request req, mpi_nonblocking_call_t *call) -{ - virt_id_entry *entry = - lookup_virt_id_entry((mana_mpi_handle){.request = req}); - if (entry == NULL) { - return false; - } - lockPending(); - *call = entry->call; - unlockPending(); - return call->type != UNKNOW_REQUEST; -} - -bool -pendingCallUsesDatatype(MPI_Datatype type) -{ - bool found = false; - lockPending(); - for (virt_id_entry *entry = pendingHead; entry != NULL; - entry = entry->pending_next) { - if (entry->call.datatype == type) { - found = true; - break; - } - } - unlockPending(); - return found; -} - -void -replayMpiP2pOnRestart() -{ - MPI_Request request; - mpi_nonblocking_call_t pendingCall; - mpi_nonblocking_call_t *call = &pendingCall; - TRACE("Replaying unserviced isend/irecv calls"); - - // No other thread runs at restart; a lock saved in the image is stale. - pendingLock = 0; - // Re-post the receives in the order they were posted. - for (MPI_Request pending : pendingRequestsInPostingOrder()) { - int retval = 0; - request = pending; - if (!getPendingCall(request, call)) { - continue; - } - MPI_Comm realComm = get_real_id((mana_mpi_handle){.comm = call->comm}).comm; - MPI_Datatype realType = get_real_id((mana_mpi_handle){.datatype = call->datatype}).datatype; - MPI_Request realRequest; - switch (call->type) { - case IRECV_REQUEST: - TRACE("Replaying Irecv call: remote_node={}", call->remote_node); - JUMP_TO_LOWER_HALF(lh_info->fsaddr); - NEXT_FUNC(Irecv)(call->recvbuf, call->count, - realType, call->remote_node, - call->tag, realComm, &realRequest); - RETURN_TO_UPPER_HALF(); - update_virt_id((mana_mpi_handle){.request = request}, (mana_mpi_handle){.request = realRequest}); - ASSERT(retval == MPI_SUCCESS, "Error while replaying recv"); - break; - case ISEND_REQUEST: - case ISSEND_REQUEST: - ASSERT(false, "There should be no pending MPI_Isend after restart: " - "type={}", call->type); - break; - default: - WARN(false, "Unhandled replay call: type={}", call->type); - break; - } - } -} diff --git a/src/p2p_log_replay.h b/src/p2p_log_replay.h deleted file mode 100644 index 2bd85c75b..000000000 --- a/src/p2p_log_replay.h +++ /dev/null @@ -1,95 +0,0 @@ -/**************************************************************************** - * Copyright (C) 2019-2021 by Gene Cooperman, Rohan Garg, Yao Xu * - * gene@ccs.neu.edu, rohgarg@ccs.neu.edu, xu.yao1@northeastern.edu * - * * - * This file is part of DMTCP. * - * * - * DMTCP is free software: you can redistribute it and/or * - * modify it under the terms of the GNU Lesser General Public License as * - * published by the Free Software Foundation, either version 3 of the * - * License, or (at your option) any later version. * - * * - * DMTCP is distributed in the hope that it will be useful, * - * but WITHOUT ANY WARRANTY; without even the implied warranty of * - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * - * GNU Lesser General Public License for more details. * - * * - * You should have received a copy of the GNU Lesser General Public * - * License in the files COPYING and COPYING.LESSER. If not, see * - * . * - ****************************************************************************/ - -#ifndef _P2P_COMM_H -#define _P2P_COMM_H - -#include -#include -#include -#include "dmtcp.h" -#include "dmtcpalloc.h" -#include "virtual_id.h" - -// Struct to store and return the MPI message (data) during draining and -// resuming, also used by p2p_drain.h -typedef struct __mpi_message -{ - void *buf; - int count; - MPI_Datatype datatype; - int size; - MPI_Comm comm; - MPI_Status status; - // An MPI_Issend's: its sender's world rank, to ack when it is received, - // and the ack's tag (see publishSsendNotices()). Otherwise ack_to is -1. - int ack_to; - int ack_tag; -} mpi_message_t; - - -extern int g_world_rank; // Global rank of the current process -extern int g_world_size; // Total number of ranks in the current computation - -// Fetches the MPI rank and world size; also, verifies that MPI rank and -// world size match the globally stored values in the plugin -extern void getLocalRankInfo(); - -// Sets the name of the checkpoint directory of the current process to -// "ckpt_rank_", where RANK is the MPI rank of the process. -extern void updateCkptDirByRank(); - -// Restores the state of MPI P2P communication by replaying any pending -// MPI_Isend and MPI_Irecv requests post restart -extern void replayMpiP2pOnRestart(); - -// Saves a nonblocking send/recv call in the virtual-ID entry of request 'rq'. -extern void addPendingRequestToLog(mpi_req_t , const void* , void* , int , - MPI_Datatype , int , int , - MPI_Comm, MPI_Request); - -// remove finished send/recv call from the pending calls -extern void clearPendingRequestFromLog(MPI_Request req); - -// Replaces the call of a pending request; it keeps its place in the log. -extern void replacePendingCall(MPI_Request req, - const mpi_nonblocking_call_t *call); - -// Counts a send in the later_sends of each pending MPI_Issend with its -// envelope. Call it before the send enters the lower half. -extern void countSendAfterIssends(MPI_Comm comm, int dest, int tag); - -// Returns the pending requests in posting order (not handle order). MPI -// matches receives in that order, so the drain and restart replay use it. -extern std::vector pendingRequestsInPostingOrder(); - -// Returns the type of a pending request, or UNKNOW_REQUEST if the request is -// not (or no longer) pending. -extern mpi_req_t pendingRequestType(MPI_Request req); - -// Copies the call of a pending request to *call. Returns false if the -// request is not (or no longer) pending. -extern bool getPendingCall(MPI_Request req, mpi_nonblocking_call_t *call); - -// Returns true if a pending MPI_Isend/MPI_Irecv uses the datatype. -extern bool pendingCallUsesDatatype(MPI_Datatype type); - -#endif // ifndef _P2P_LOG_REPLAY_H diff --git a/src/virtual_id.cpp b/src/virtual_id.cpp index 14facf063..37101a0a4 100644 --- a/src/virtual_id.cpp +++ b/src/virtual_id.cpp @@ -10,7 +10,8 @@ #include "switch-context.h" #include "mpi_nextfunc.h" #include "seq_num.h" -#include "p2p_log_replay.h" +#include "mpi_plugin.h" +#include "p2p_drain.h" #include MPI_Group g_world_group; diff --git a/src/virtual_id.h b/src/virtual_id.h index ded74c0c1..ba42e8aec 100644 --- a/src/virtual_id.h +++ b/src/virtual_id.h @@ -147,7 +147,7 @@ typedef struct virt_id_entry { int next_free; // Next slot in the free list, if the slot is free unsigned int gen; // Generation of the next handle that uses this slot // For a pending MPI_Isend/MPI_Irecv: the call, linked in posting order - // (see p2p_log_replay.cpp). Otherwise, call.type is UNKNOW_REQUEST. + // (see p2p_drain.cpp). Otherwise, call.type is UNKNOW_REQUEST. mpi_nonblocking_call_t call; struct virt_id_entry *pending_prev; struct virt_id_entry *pending_next;