diff --git a/README.md b/README.md index f71158af..1bea74c1 100644 --- a/README.md +++ b/README.md @@ -9,17 +9,16 @@ HPX-FFT depends on [HPX](https://hpx-docs.stellar-group.org/latest/html/index.ht ### Install dependencies All dependencies can be installed using [Spack](https://github.com/spack/spack). -For HPX we recommend: `spack install hpx@1.10.0 networking={tcp/mpi/lci} max_cpu_count=256` +For HPX we recommend: `spack install hpx@1.11.0 networking={tcp/mpi/lci/all} max_cpu_count=256` For FFTW we recommend `spack install fftw@3.3.10 +mpi +openmp` - -Note that the LCI variant of HPX currently requires to add the [Octo-Tiger repo](https://github.com/G-071/octotiger-spack) to spack. -Alternatively, both librarires can be built from source. +The requirements can be also installed using the provided spack environments. +The spack-repo directory add support for installing HPX with the LCI or all parcelports at once. ## How to Compile HPX-FFT We provide a simple compilation script for HPX-FFT. The script assumes an gcc compiler version 14.2.0. However, it can be easily tailored towards specific needs. -## How to Compile Examples +## How to Compile Examples: HPX-FFT comes with several examples for HPX-FFT itself as well as for a FFTW reference. The examples can be compiled with the respective compilation scripts. @@ -27,6 +26,7 @@ HPX-FFT comes with several examples for HPX-FFT itself as well as for a FFTW ref ### HPX-FFT +Shared examples: ``` 1=Base size of the problem 2=FFTW Planning flag (estimate/measure) @@ -34,25 +34,33 @@ HPX-FFT comes with several examples for HPX-FFT itself as well as for a FFTW ref 4=Number of HPX threads ./hpxfft_shared_loop --nx=$1 --ny=$1 --plan=$2 --header=$3 --hpx:threads=$4 ``` + +Distributed examples with slurm: ``` 1=Base size of the problem 2=FFTW Planning flag (estimate/measure) 3=Print info into runtime file (true/false) 4=Collective operation (scatter/all_to_all) 5=Number of HPX threads -./hpxfft_distributed_loop --nx=$1 --ny=$1 --plan=$2 --header=$3 --run=$4 --hpx:threads=$5 +6=Number of HPX localities +7=Partition + +srun --mpi=pmix -p $6 -N 2 -n $6 -c $5 ./examples/hpxfft/build/hpxfft_distributed_loop --nx=$1 --ny=$1 --plan=$2 --header=$3 --run=$4 --hpx:ini=hpx.parcel.mpi.enable=1 --hpx:ini=hpx.parcel.tcp.enable=0 --hpx:ini=hpx.parcel.lci.enable=0 ``` ### FFTW +Shared examples: ``` // threads N_X N_Y plan header ./fftw_omp 1 8 14 estimate 0 ``` + +Distributed examples with srun or mpirun: ``` -// nodes ranks prog threads N_X N_Y plan header -srun -N 2 -n 4 fftw_mpi_omp 4 8 14 estimate 0 - mpirun -n 4 fftw_mpi_omp 4 8 14 estimate 0 +// nodes ranks prog threads N_X N_Y plan header +srun --mpi=pmix -N 2 -n 4 fftw_mpi_omp 4 8 14 estimate 0 + mpirun -n 4 fftw_mpi_omp 4 8 14 estimate 0 ``` ## How to Run Benchmarks (wip) @@ -73,17 +81,8 @@ Executables only: `sbatch -p $PARTITION -N 1 -n 1 -c $THREADS run_fftw_shared.sh executable_name estimate/measure $THREAD_POW` ### Distributed -All: - -- `./distributed_benchmark.sh estimate/measure partition_name` - -Executables only: - -- fftw_hpx_loop/fftw_hpx_tasg_agas: -`sbatch -p $PARTITION -N $NODES -n $NODES -c $THREADS run_hpx_dist.sh executable_name estimate/measure $NODES` -- fftw_mpi_threads/fftw_mpi_omp: -`sbatch -p $PARTITION -N $NODES -n $NODES -c $THREADS run_fftw_dist.sh executable_name estimate/measure $NODES $THREADS` +- `./distributed_benchmark.sh estimate/measure scatter/all_to_all tcp/mpi/lci` ## The Team @@ -95,6 +94,7 @@ We specifically thank the follow contributors: - [Alexander Strack](https://www.ipvs.uni-stuttgart.de/de/institut/team/Strack-00001/): Maintainer and [initial framework](https://doi.org/10.1007/978-3-031-32316-4_5). +- Heiko Häfner: Testing and CI. ## How To Cite diff --git a/benchmark/distributed_benchmark.sh b/benchmark/distributed_benchmark.sh index dc3df15b..2745de4b 100755 --- a/benchmark/distributed_benchmark.sh +++ b/benchmark/distributed_benchmark.sh @@ -2,6 +2,7 @@ # Benchmark script for shared memory # $1: FFTW planning flag (estimate/measure) # $2: Collective (scatter/all_to_all) +# $3: HPX Parcelport (tcp/mpi/lci) ################################################################################ # Config ################################################################################ @@ -21,16 +22,11 @@ FFTW_EXECUTABLES=( # Get current hostname HOSTNAME=$(hostname -s) -if [[ "$HOSTNAME" == "ipvsmisc" ]]; then - echo "tbd." -elif [[ "$HOSTNAME" == "rostam" ]]; then - echo "tbd." -elif [[ "$HOSTNAME" == "login1" ]]; then - PARTITION=short - NODES_POW=1 +if [[ "$HOSTNAME" == "rostam1" ]]; then + PARTITION=buran + NODES_POW=4 THREADS=48 -elif [[ "$HOSTNAME" == "simcl1" ]]; then - echo "tbd." + module load gcc/14.2.0 else echo "Hostname is $HOSTNAME — no action taken." exit 1 @@ -42,7 +38,7 @@ FFTW_PLAN=$1 # Check if FFTW_PLAN was provided if [ -z "$FFTW_PLAN" ]; then echo "Error: FFTW_PLAN parameter not set." - echo "Usage: distributed_benchmark.sh estimate/measure scatter/all_to_all" + echo "Usage: distributed_benchmark.sh estimate/measure scatter/all_to_all tcp/mpi/lci" exit 1 fi # Set Collective @@ -50,14 +46,22 @@ COLLECTIVE=$2 # Check if COLLECITVE was provided if [ -z "$COLLECTIVE" ]; then echo "Error: COLLECTIVE parameter not set." - echo "Usage: distributed_benchmark.sh estimate/measure scatter/all_to_all" + echo "Usage: distributed_benchmark.sh estimate/measure scatter/all_to_all tcp/mpi/lci" + exit 1 +fi +# Set HPX Parcelport +PARCELPORT=$3 +# Check if COLLECITVE was provided +if [ -z "$PARCELPORT" ]; then + echo "Error: Parcelport parameter not set." + echo "Usage: distributed_benchmark.sh estimate/measure scatter/all_to_all tcp/mpi/lci" exit 1 fi ################################################################################ # Run benchmarks ################################################################################ -RESULT_DIR=$TOP_DIR/distributed_benchmark_on_$HOSTNAME +RESULT_DIR=$TOP_DIR/distributed_benchmark_on_$PARTITION SCRIPT_DIR=$TOP_DIR/benchmark/sbatch_scripts ################################################################################ @@ -68,7 +72,7 @@ mkdir -p $RESULT_DIR/strong_scaling/plans cd $RESULT_DIR/strong_scaling # Config -LOOP=2 +LOOP=10 BASE_SIZE=16384 START_NODES_POW=0 @@ -77,9 +81,8 @@ RUN_SCRIPT="$SCRIPT_DIR/run_hpxfft_strong_distributed.sh" for EXE in "${HPXFFT_EXECUTABLES[@]}"; do echo "Submitting job for executable: $EXE" - sbatch -p $PARTITION -N $NODES -n $NODES -c $THREADS \ - $RUN_SCRIPT \ - $EXE $FFTW_PLAN $START_NODES_POW $NODES_POW $BASE_SIZE $COLLECTIVE $THREADS $LOOP + $RUN_SCRIPT \ + $EXE $FFTW_PLAN $START_NODES_POW $NODES_POW $BASE_SIZE $COLLECTIVE $THREADS $LOOP $PARTITION $PARCELPORT done # Loop over FFTW executables @@ -88,9 +91,8 @@ RUN_SCRIPT="$SCRIPT_DIR/run_fftw_strong_distributed.sh" for EXE in "${FFTW_EXECUTABLES[@]}"; do echo "Submitting job for executable: $EXE" - sbatch -p $PARTITION -N $NODES -n $NODES -c $THREADS \ - $RUN_SCRIPT \ - $EXE $FFTW_PLAN $START_NODES_POW $NODES_POW $BASE_SIZE $THREADS $LOOP + $RUN_SCRIPT \ + $EXE $FFTW_PLAN $START_NODES_POW $NODES_POW $BASE_SIZE $THREADS $LOOP $PARTITION done ################################################################################ @@ -101,7 +103,7 @@ mkdir -p $RESULT_DIR/weak_scaling/plans cd $RESULT_DIR/weak_scaling # Config -LOOP=2 +LOOP=10 BASE_SIZE=512 START_THREAD_POW=0 @@ -110,9 +112,8 @@ RUN_SCRIPT="$SCRIPT_DIR/run_hpxfft_weak_distributed.sh" for EXE in "${HPXFFT_EXECUTABLES[@]}"; do echo "Submitting job for executable: $EXE" - sbatch -p $PARTITION -N $NODES -n $NODES -c $THREADS \ - $RUN_SCRIPT \ - $EXE $FFTW_PLAN $START_NODES_POW $NODES_POW $BASE_SIZE $COLLECTIVE $THREADS $LOOP + $RUN_SCRIPT \ + $EXE $FFTW_PLAN $START_NODES_POW $NODES_POW $BASE_SIZE $COLLECTIVE $THREADS $LOOP $PARTITION $PARCELPORT done # Loop over FFTW executables @@ -121,7 +122,6 @@ RUN_SCRIPT="$SCRIPT_DIR/run_fftw_weak_distributed.sh" for EXE in "${FFTW_EXECUTABLES[@]}"; do echo "Submitting job for executable: $EXE" - sbatch -p $PARTITION -N $NODES -n $NODES -c $THREADS \ - $RUN_SCRIPT \ - $EXE $FFTW_PLAN $START_NODES_POW $NODES_POW $BASE_SIZE $THREADS $LOOP + $RUN_SCRIPT \ + $EXE $FFTW_PLAN $START_NODES_POW $NODES_POW $BASE_SIZE $THREADS $LOOP $PARTITION done diff --git a/benchmark/message_benchmark.sh b/benchmark/message_benchmark.sh index 2b889e4f..9bfc16fc 100755 --- a/benchmark/message_benchmark.sh +++ b/benchmark/message_benchmark.sh @@ -1,6 +1,7 @@ #!/usr/bin/bash # Benchmark script for message size scaling between two nodes # $1: Collective (scatter/all_to_all) +# $2: HPX Parcelport (tcp/mpi/lci) ################################################################################ # Config ################################################################################ @@ -12,23 +13,13 @@ HPXFFT_EXECUTABLES=( "$EXAMPLES_DIR/hpxfft/build/hpxfft_distributed_loop" "$EXAMPLES_DIR/hpxfft/build/hpxfft_distributed_agas" ) -# FFTW implementations -FFTW_EXECUTABLES=( - "$EXAMPLES_DIR/fftw/build/fftw_mpi_omp" - "$EXAMPLES_DIR/fftw/build/fftw_mpi_threads" -) # Get current hostname HOSTNAME=$(hostname -s) -if [[ "$HOSTNAME" == "ipvsmisc" ]]; then - echo "tbd." -elif [[ "$HOSTNAME" == "rostam" ]]; then - echo "tbd." -elif [[ "$HOSTNAME" == "login1" ]]; then - PARTITION=short +if [[ "$HOSTNAME" == "rostam1" ]]; then + PARTITION=buran THREADS=48 -elif [[ "$HOSTNAME" == "simcl1" ]]; then - echo "tbd." + module load gcc/14.2.0 else echo "Hostname is $HOSTNAME — no action taken." exit 1 @@ -42,11 +33,18 @@ if [ -z "$COLLECTIVE" ]; then echo "Usage: message_benchmark.sh scatter/all_to_all" exit 1 fi - +# Set HPX Parcelport +PARCELPORT=$2 +# Check if COLLECITVE was provided +if [ -z "$PARCELPORT" ]; then + echo "Error: Parcelport parameter not set." + echo "Usage: message_benchmark.sh scatter/all_to_all tcp/mpi/lci" + exit 1 +fi ################################################################################ # Run benchmarks ################################################################################ -RESULT_DIR=$TOP_DIR/message_benchmark_on_$HOSTNAME +RESULT_DIR=$TOP_DIR/message_benchmark_on_$PARTITION SCRIPT_DIR=$TOP_DIR/benchmark/sbatch_scripts ################################################################################ @@ -65,18 +63,6 @@ RUN_SCRIPT="$SCRIPT_DIR/run_hpxfft_message_size.sh" for EXE in "${HPXFFT_EXECUTABLES[@]}"; do echo "Submitting job for executable: $EXE" - sbatch -p $PARTITION -N 2 -n 2 -c $THREADS \ - $RUN_SCRIPT \ - $EXE $BASE_SIZE $STOP_POW $COLLECTIVE $THREADS $LOOP -done - -# Loop over FFTW executables -RUN_SCRIPT="$SCRIPT_DIR/run_fftw_message_size.sh" -# Loop over executables -for EXE in "${FFTW_EXECUTABLES[@]}"; do - echo "Submitting job for executable: $EXE" - - sbatch -p $PARTITION -N 2 -n 2 -c $THREADS \ $RUN_SCRIPT \ - $EXE $BASE_SIZE $STOP_POW $THREADS $LOOP + $EXE $BASE_SIZE $STOP_POW $COLLECTIVE $THREADS $LOOP $PARTITION $PARCELPORT done diff --git a/benchmark/sbatch_scripts/run_fftw_message_size.sh b/benchmark/sbatch_scripts/run_fftw_message_size.sh deleted file mode 100755 index be358032..00000000 --- a/benchmark/sbatch_scripts/run_fftw_message_size.sh +++ /dev/null @@ -1,43 +0,0 @@ -#!/bin/bash -#SBATCH --job-name=fftw_job # Job name -#SBATCH --output=fftw_job.log # Standard output and error log -#SBATCH --mail-type=NONE # Mail events (NONE, BEGIN, END, FAIL, ALL) -#SBATCH --mail-user=alexander.strack@ipvs.uni-stuttgart.de # Where to send mail -#SBATCH --time=2:00:00 # Time limit hrs:min:sec -#SBATCH --exclusive # Exclusive ressource access - -# Benchmark script for message size scaling between two nodes -# $1: Executable name -# $2: Base size -# $3: Stopping power of two -# $4: Number of threads per node -# $5: Number of runs - -# Log Info -pwd; hostname; date -# Parameters -BASE_SIZE=$2 -POW_STOP=$3 -THREADS=$4 -LOOP=$5 -# Get run command -COMMAND="srun --mpi=pmix -N 2 -n 2 -c $THREADS" -EXECUTABLE=$1 -ARGUMENTS="$BASE_SIZE $BASE_SIZE $2" -# Message size scaling loop on 2 nodes -$COMMAND $EXECUTABLE $THREADS $ARGUMENTS 1 -for (( j=1; j<$LOOP; j=j+1 )) -do - $COMMAND $EXECUTABLE $THREADS $ARGUMENTS 0 -done -for (( i=2; i<=2**$POW_STOP; i=i*2 )) -do - SIZE=$((i*BASE_SIZE)) - ARGUMENTS="$SIZE $SIZE $2" - for (( j=0; j<$LOOP; j=j+1 )) - do - $COMMAND $EXECUTABLE $THREADS $ARGUMENTS 0 - done -done -# Log Info -date diff --git a/benchmark/sbatch_scripts/run_fftw_size_shared.sh b/benchmark/sbatch_scripts/run_fftw_size_shared.sh index e3e28457..c606af6b 100755 --- a/benchmark/sbatch_scripts/run_fftw_size_shared.sh +++ b/benchmark/sbatch_scripts/run_fftw_size_shared.sh @@ -2,7 +2,6 @@ #SBATCH --job-name=fftw_job # Job name #SBATCH --output=fftw_job.log # Standard output and error log #SBATCH --mail-type=NONE # Mail events (NONE, BEGIN, END, FAIL, ALL) -#SBATCH --mail-user=alexander.strack@ipvs.uni-stuttgart.de # Where to send mail #SBATCH --time=2:00:00 # Time limit hrs:min:sec #SBATCH --exclusive # Exclusive ressource access diff --git a/benchmark/sbatch_scripts/run_fftw_strong_distributed.sh b/benchmark/sbatch_scripts/run_fftw_strong_distributed.sh index c6dae362..fb3e345e 100755 --- a/benchmark/sbatch_scripts/run_fftw_strong_distributed.sh +++ b/benchmark/sbatch_scripts/run_fftw_strong_distributed.sh @@ -1,10 +1,4 @@ #!/bin/bash -#SBATCH --job-name=fftw_job # Job name -#SBATCH --output=fftw_job.log # Standard output and error log -#SBATCH --mail-type=NONE # Mail events (NONE, BEGIN, END, FAIL, ALL) -#SBATCH --mail-user=alexander.strack@ipvs.uni-stuttgart.de # Where to send mail -#SBATCH --time=2:00:00 # Time limit hrs:min:sec -#SBATCH --exclusive # Exclusive ressource access # Benchmark script for distributed memory strong scaling # $1: Executable name @@ -14,6 +8,7 @@ # $5: Base size # $6: Number of threads per node # $7: Number of runs +# $8: Partition # Log Info pwd; hostname; date @@ -23,21 +18,25 @@ POW_STOP=$4 BASE_SIZE=$5 THREADS=$6 LOOP=$7 +PARTITION=$8 # Get run command -COMMAND="srun -N $((2**POW_START)) -n $((2**POW_START)) -c $THREADS" +COMMAND="srun --mpi=pmix -p $PARTITION --nodelist=$PARTITION[00-0$((2**POW_START - 1))] -N $((2**POW_START)) -n $((2**POW_START)) -c $THREADS" EXECUTABLE=$1 ARGUMENTS="$BASE_SIZE $BASE_SIZE $2" # Strong scaling loop from 2^pow_start to 2^pow_stop nodes +echo 'Submiting:' $COMMAND $EXECUTABLE $THREADS $ARGUMENTS $COMMAND $EXECUTABLE $THREADS $ARGUMENTS 1 for (( j=1; j<$LOOP; j=j+1 )) do + echo 'Submiting:' $COMMAND $EXECUTABLE $THREADS $ARGUMENTS $COMMAND $EXECUTABLE $THREADS $ARGUMENTS 0 done for (( i=2**($POW_START+1); i<=2**$POW_STOP; i=i*2 )) do - COMMAND="srun -N $i -n $i -c $THREADS" + COMMAND="srun --mpi=pmix -p $PARTITION --nodelist=$PARTITION[00-0$(($i - 1))] -N $i -n $i -c $THREADS" for (( j=0; j<$LOOP; j=j+1 )) do + echo 'Submiting:' $COMMAND $EXECUTABLE $THREADS $ARGUMENTS $COMMAND $EXECUTABLE $THREADS $ARGUMENTS 0 done done diff --git a/benchmark/sbatch_scripts/run_fftw_strong_shared.sh b/benchmark/sbatch_scripts/run_fftw_strong_shared.sh index 15a7c721..562a2c03 100755 --- a/benchmark/sbatch_scripts/run_fftw_strong_shared.sh +++ b/benchmark/sbatch_scripts/run_fftw_strong_shared.sh @@ -2,7 +2,6 @@ #SBATCH --job-name=fftw_job # Job name #SBATCH --output=fftw_job.log # Standard output and error log #SBATCH --mail-type=NONE # Mail events (NONE, BEGIN, END, FAIL, ALL) -#SBATCH --mail-user=alexander.strack@ipvs.uni-stuttgart.de # Where to send mail #SBATCH --time=2:00:00 # Time limit hrs:min:sec #SBATCH --exclusive # Exclusive ressource access diff --git a/benchmark/sbatch_scripts/run_fftw_weak_distributed.sh b/benchmark/sbatch_scripts/run_fftw_weak_distributed.sh index 06d9970d..6c4e4b1b 100755 --- a/benchmark/sbatch_scripts/run_fftw_weak_distributed.sh +++ b/benchmark/sbatch_scripts/run_fftw_weak_distributed.sh @@ -1,10 +1,4 @@ #!/bin/bash -#SBATCH --job-name=fftw_job # Job name -#SBATCH --output=fftw_job.log # Standard output and error log -#SBATCH --mail-type=NONE # Mail events (NONE, BEGIN, END, FAIL, ALL) -#SBATCH --mail-user=alexander.strack@ipvs.uni-stuttgart.de # Where to send mail -#SBATCH --time=2:00:00 # Time limit hrs:min:sec -#SBATCH --exclusive # Exclusive ressource access # Benchmark script for distributed memory weak scaling # $1: Executable name @@ -14,6 +8,7 @@ # $5: Base size # $6: Number of threads per node # $7: Number of runs +# $8: Partition # Log Info pwd; hostname; date @@ -23,22 +18,26 @@ POW_STOP=$4 BASE_SIZE=$5 THREADS=$6 LOOP=$7 +PARTITION=$8 # Get run command -COMMAND="srun -N $((2**POW_START)) -n $((2**POW_START)) -c $THREADS" +COMMAND="srun --mpi=pmix -p $PARTITION --nodelist=$PARTITION[00-0$((2**POW_START - 1))] -N $((2**POW_START)) -n $((2**POW_START)) -c $THREADS" EXECUTABLE=$1 ARGUMENTS="$BASE_SIZE $BASE_SIZE $2" # Weak scaling loop from 2^pow_start to 2^pow_stop nodes +echo 'Submiting:' $COMMAND $THREADS $EXECUTABLE $ARGUMENTS $COMMAND $EXECUTABLE $THREADS $ARGUMENTS 1 for (( j=1; j<$LOOP; j=j+1 )) do + echo 'Submiting:' $COMMAND $EXECUTABLE $THREADS $ARGUMENTS $COMMAND $EXECUTABLE $THREADS $ARGUMENTS 0 done for (( i=2**($POW_START+1); i<=2**$POW_STOP; i=i*2 )) do - COMMAND="srun -N $i -n $i -c $THREADS" + COMMAND="srun --mpi=pmix -p $PARTITION --nodelist=$PARTITION[00-0$(($i - 1))] -N $i -n $i -c $THREADS" ARGUMENTS="$((BASE_SIZE*i)) $((BASE_SIZE*i)) $2" for (( j=0; j<$LOOP; j=j+1 )) do + echo 'Submiting:' $COMMAND $EXECUTABLE $THREADS $ARGUMENTS $COMMAND $EXECUTABLE $THREADS $ARGUMENTS 0 done done diff --git a/benchmark/sbatch_scripts/run_fftw_weak_shared.sh b/benchmark/sbatch_scripts/run_fftw_weak_shared.sh index 7f82d7e6..c9b31bdb 100755 --- a/benchmark/sbatch_scripts/run_fftw_weak_shared.sh +++ b/benchmark/sbatch_scripts/run_fftw_weak_shared.sh @@ -2,7 +2,6 @@ #SBATCH --job-name=fftw_job # Job name #SBATCH --output=fftw_job.log # Standard output and error log #SBATCH --mail-type=NONE # Mail events (NONE, BEGIN, END, FAIL, ALL) -#SBATCH --mail-user=alexander.strack@ipvs.uni-stuttgart.de # Where to send mail #SBATCH --time=2:00:00 # Time limit hrs:min:sec #SBATCH --exclusive # Exclusive ressource access diff --git a/benchmark/sbatch_scripts/run_hpxfft_message_size.sh b/benchmark/sbatch_scripts/run_hpxfft_message_size.sh index 80d5970c..cdb23b55 100755 --- a/benchmark/sbatch_scripts/run_hpxfft_message_size.sh +++ b/benchmark/sbatch_scripts/run_hpxfft_message_size.sh @@ -1,11 +1,4 @@ #!/bin/bash -#SBATCH --job-name=hpxfft_job # Job name -#SBATCH --output=hpxfft_job.log # Standard output and error log -#SBATCH --mail-type=NONE # Mail events (NONE, BEGIN, END, FAIL, ALL) -#SBATCH --mail-user=alexander.strack@ipvs.uni-stuttgart.de # Where to send mail -#SBATCH --time=2:00:00 # Time limit hrs:min:sec -#SBATCH --exclusive # Exclusive ressource access - # Benchmark script for message size scaling between two nodes # $1: Executable name # $2: Base size @@ -13,6 +6,8 @@ # $4: HPX collective (scatter/all_to_all) # $5: Number of threads per node # $6: Number of runs +# $7: Partition +# $8: HPX Parcelport # Log Info pwd; hostname; date @@ -21,16 +16,21 @@ BASE_SIZE=$2 POW_STOP=$3 THREADS=$5 LOOP=$6 +PARTITION=$7 +PARCELPORT=$8 # Get run command OPTIONS="" -COMMAND="srun -N 2 -n 2 -c $THREADS" +COMMAND="srun --mpi=pmix -p $PARTITION --nodelist=$PARTITION[00-01] -N 2 -n 2 -c $THREADS" EXECUTABLE=$1 ARGUMENTS="--nx=$BASE_SIZE --ny=$BASE_SIZE --plan="estimate" --run=$4" +PARCELPORTS="--hpx:ini=hpx.parcel.mpi.enable=0 --hpx:ini=hpx.parcel.tcp.enable=0 --hpx:ini=hpx.parcel.lci.enable=0 --hpx:ini=hpx.parcel.$PARCELPORT.enable=1" # Message size scaling loop on 2 nodes -$COMMAND $EXECUTABLE $ARGUMENTS --header=true -for (( j=1; j<$LOOP; j=j+1 )) +echo 'Submiting warmup:' $COMMAND $EXECUTABLE $ARGUMENTS $PARCELPORTS +$COMMAND $EXECUTABLE $ARGUMENTS $PARCELPORTS --header=true +for (( j=0; j<$LOOP; j=j+1 )) do - $COMMAND $EXECUTABLE $ARGUMENTS + echo 'Submiting:' $COMMAND $EXECUTABLE $ARGUMENTS $PARCELPORTS + $COMMAND $EXECUTABLE $ARGUMENTS $PARCELPORTS done for (( i=2; i<=2**$POW_STOP; i=i*2 )) do @@ -38,7 +38,8 @@ do ARGUMENTS="--nx=$SIZE --ny=$SIZE --plan="estimate" --run=$4" for (( j=0; j<$LOOP; j=j+1 )) do - $COMMAND $EXECUTABLE $ARGUMENTS + echo 'Submiting:' $COMMAND $EXECUTABLE $ARGUMENTS $PARCELPORTS + $COMMAND $EXECUTABLE $ARGUMENTS $PARCELPORTS done done # Log Info diff --git a/benchmark/sbatch_scripts/run_hpxfft_size_shared.sh b/benchmark/sbatch_scripts/run_hpxfft_size_shared.sh index ba6edb24..79c65fb8 100755 --- a/benchmark/sbatch_scripts/run_hpxfft_size_shared.sh +++ b/benchmark/sbatch_scripts/run_hpxfft_size_shared.sh @@ -2,7 +2,6 @@ #SBATCH --job-name=hpxfft_job # Job name #SBATCH --output=hpxfft_job.log # Standard output and error log #SBATCH --mail-type=NONE # Mail events (NONE, BEGIN, END, FAIL, ALL) -#SBATCH --mail-user=alexander.strack@ipvs.uni-stuttgart.de # Where to send mail #SBATCH --time=2:00:00 # Time limit hrs:min:sec #SBATCH --exclusive # Exclusive ressource access diff --git a/benchmark/sbatch_scripts/run_hpxfft_strong_distributed.sh b/benchmark/sbatch_scripts/run_hpxfft_strong_distributed.sh index dc5add19..543d7d73 100755 --- a/benchmark/sbatch_scripts/run_hpxfft_strong_distributed.sh +++ b/benchmark/sbatch_scripts/run_hpxfft_strong_distributed.sh @@ -1,10 +1,4 @@ #!/bin/bash -#SBATCH --job-name=hpxfft_job # Job name -#SBATCH --output=hpxfft_job.log # Standard output and error log -#SBATCH --mail-type=NONE # Mail events (NONE, BEGIN, END, FAIL, ALL) -#SBATCH --mail-user=alexander.strack@ipvs.uni-stuttgart.de # Where to send mail -#SBATCH --time=2:00:00 # Time limit hrs:min:sec -#SBATCH --exclusive # Exclusive ressource access # Benchmark script for distributed memory strong scaling # $1: Executable name @@ -15,6 +9,8 @@ # $6: HPX collective (scatter/all_to_all) # $7: Number of threads per node # $8: Number of runs +# $9: Partition +# $10: HPX Parcelport # Log Info pwd; hostname; date @@ -24,24 +20,29 @@ POW_STOP=$4 BASE_SIZE=$5 THREADS=$7 LOOP=$8 +PARTITION=$9 +PARCELPORT=${10} # Get run command OPTIONS="" -COMMAND="srun -N $((2**POW_START)) -n $((2**POW_START)) -c $THREADS" +COMMAND="srun --mpi=pmix -p $PARTITION --nodelist=$PARTITION[00-0$((2**POW_START - 1))] -N $((2**POW_START)) -n $((2**POW_START)) -c $THREADS" EXECUTABLE=$1 ARGUMENTS="--nx=$BASE_SIZE --ny=$BASE_SIZE --plan=$2 --run=$6" - +PARCELPORTS="--hpx:ini=hpx.parcel.mpi.enable=0 --hpx:ini=hpx.parcel.tcp.enable=0 --hpx:ini=hpx.parcel.lci.enable=0 --hpx:ini=hpx.parcel.$PARCELPORT.enable=1" # Strong scaling loop from 2^pow_start to 2^pow_stop nodes -HPX_COMMANDLINE_OPTIONS=$OPTIONS $COMMAND $EXECUTABLE $ARGUMENTS --header=true +echo 'Submiting:' $COMMAND $EXECUTABLE $ARGUMENTS $PARCELPORTS +HPX_COMMANDLINE_OPTIONS=$OPTIONS $COMMAND $EXECUTABLE $ARGUMENTS $PARCELPORTS --header=true for (( j=1; j<$LOOP; j=j+1 )) do - HPX_COMMANDLINE_OPTIONS=$OPTIONS $COMMAND $EXECUTABLE $ARGUMENTS + echo 'Submiting:' $COMMAND $EXECUTABLE $ARGUMENTS $PARCELPORTS + HPX_COMMANDLINE_OPTIONS=$OPTIONS $COMMAND $EXECUTABLE $ARGUMENTS $PARCELPORTS done for (( i=2**($POW_START+1); i<=2**$POW_STOP; i=i*2 )) do - COMMAND="srun -N $i -n $i -c $THREADS" + COMMAND="srun --mpi=pmix -p $PARTITION --nodelist=$PARTITION[00-0$(($i - 1))] -N $i -n $i -c $THREADS" for (( j=0; j<$LOOP; j=j+1 )) do - HPX_COMMANDLINE_OPTIONS=$OPTIONS $COMMAND $EXECUTABLE $ARGUMENTS + echo 'Submiting:' $COMMAND $EXECUTABLE $ARGUMENTS $PARCELPORTS + HPX_COMMANDLINE_OPTIONS=$OPTIONS $COMMAND $EXECUTABLE $ARGUMENTS $PARCELPORTS done done # Log Info diff --git a/benchmark/sbatch_scripts/run_hpxfft_strong_shared.sh b/benchmark/sbatch_scripts/run_hpxfft_strong_shared.sh index bafaec25..59741832 100755 --- a/benchmark/sbatch_scripts/run_hpxfft_strong_shared.sh +++ b/benchmark/sbatch_scripts/run_hpxfft_strong_shared.sh @@ -2,7 +2,6 @@ #SBATCH --job-name=hpxfft_job # Job name #SBATCH --output=hpxfft_job.log # Standard output and error log #SBATCH --mail-type=NONE # Mail events (NONE, BEGIN, END, FAIL, ALL) -#SBATCH --mail-user=alexander.strack@ipvs.uni-stuttgart.de # Where to send mail #SBATCH --time=2:00:00 # Time limit hrs:min:sec #SBATCH --exclusive # Exclusive ressource access diff --git a/benchmark/sbatch_scripts/run_hpxfft_weak_distributed.sh b/benchmark/sbatch_scripts/run_hpxfft_weak_distributed.sh index b6490c1b..7258a04e 100755 --- a/benchmark/sbatch_scripts/run_hpxfft_weak_distributed.sh +++ b/benchmark/sbatch_scripts/run_hpxfft_weak_distributed.sh @@ -1,12 +1,6 @@ #!/bin/bash -#SBATCH --job-name=hpxfft_job # Job name -#SBATCH --output=hpxfft_job.log # Standard output and error log -#SBATCH --mail-type=NONE # Mail events (NONE, BEGIN, END, FAIL, ALL) -#SBATCH --mail-user=alexander.strack@ipvs.uni-stuttgart.de # Where to send mail -#SBATCH --time=2:00:00 # Time limit hrs:min:sec -#SBATCH --exclusive # Exclusive ressource access -# Benchmark script for distributed memory weak scaling +# Benchmark script for distributed memory strong scaling # $1: Executable name # $2: FFTW planning flag (estimate/measure) # $3: Starting number of nodes @@ -15,6 +9,8 @@ # $6: HPX collective (scatter/all_to_all) # $7: Number of threads per node # $8: Number of runs +# $9: Partition +# $10: HPX Parcelport # Log Info pwd; hostname; date @@ -24,25 +20,31 @@ POW_STOP=$4 BASE_SIZE=$5 THREADS=$7 LOOP=$8 +PARTITION=$9 +PARCELPORT=${10} # Get run command OPTIONS="" -COMMAND="srun -N $((2**POW_START)) -n $((2**POW_START)) -c $THREADS" +COMMAND="srun --mpi=pmix -p $PARTITION --nodelist=$PARTITION[00-0$((2**POW_START - 1))] -N $((2**POW_START)) -n $((2**POW_START)) -c $THREADS" EXECUTABLE=$1 ARGUMENTS="--nx=$BASE_SIZE --ny=$BASE_SIZE --plan=$2 --run=$6" +PARCELPORTS="--hpx:ini=hpx.parcel.mpi.enable=0 --hpx:ini=hpx.parcel.tcp.enable=0 --hpx:ini=hpx.parcel.lci.enable=0 --hpx:ini=hpx.parcel.$PARCELPORT.enable=1" -# Strong scaling loop from 2^pow_start to 2^pow_stop nodes -HPX_COMMANDLINE_OPTIONS=$OPTIONS $COMMAND $EXECUTABLE $ARGUMENTS --header=true +# Weak scaling loop from 2^pow_start to 2^pow_stop nodes +echo 'Submiting:' $COMMAND $EXECUTABLE $ARGUMENTS $PARCELPORTS +HPX_COMMANDLINE_OPTIONS=$OPTIONS $COMMAND $EXECUTABLE $ARGUMENTS $PARCELPORTS --header=true for (( j=1; j<$LOOP; j=j+1 )) do - HPX_COMMANDLINE_OPTIONS=$OPTIONS $COMMAND $EXECUTABLE $ARGUMENTS + echo 'Submiting:' $COMMAND $EXECUTABLE $ARGUMENTS $PARCELPORTS + HPX_COMMANDLINE_OPTIONS=$OPTIONS $COMMAND $EXECUTABLE $ARGUMENTS $PARCELPORTS done for (( i=2**($POW_START+1); i<=2**$POW_STOP; i=i*2 )) do - COMMAND="srun -N $i -n $i -c $THREADS" + COMMAND="srun --mpi=pmix -p $PARTITION --nodelist=$PARTITION[00-0$(($i - 1))] -N $i -n $i -c $THREADS" ARGUMENTS="--nx=$((BASE_SIZE*i)) --ny=$((BASE_SIZE*i)) --plan=$2" for (( j=0; j<$LOOP; j=j+1 )) do - HPX_COMMANDLINE_OPTIONS=$OPTIONS $COMMAND $EXECUTABLE $ARGUMENTS + echo 'Submiting:' $COMMAND $EXECUTABLE $ARGUMENTS $PARCELPORTS + HPX_COMMANDLINE_OPTIONS=$OPTIONS $COMMAND $EXECUTABLE $ARGUMENTS $PARCELPORTS done done # Log Info diff --git a/benchmark/sbatch_scripts/run_hpxfft_weak_shared.sh b/benchmark/sbatch_scripts/run_hpxfft_weak_shared.sh index 21f4c54e..c6af37d1 100755 --- a/benchmark/sbatch_scripts/run_hpxfft_weak_shared.sh +++ b/benchmark/sbatch_scripts/run_hpxfft_weak_shared.sh @@ -2,7 +2,6 @@ #SBATCH --job-name=hpxfft_job # Job name #SBATCH --output=hpxfft_job.log # Standard output and error log #SBATCH --mail-type=NONE # Mail events (NONE, BEGIN, END, FAIL, ALL) -#SBATCH --mail-user=alexander.strack@ipvs.uni-stuttgart.de # Where to send mail #SBATCH --time=2:00:00 # Time limit hrs:min:sec #SBATCH --exclusive # Exclusive ressource access diff --git a/benchmark/shared_benchmark.sh b/benchmark/shared_benchmark.sh index 1bec8bf9..609f364d 100755 --- a/benchmark/shared_benchmark.sh +++ b/benchmark/shared_benchmark.sh @@ -27,15 +27,19 @@ FFTW_EXECUTABLES=( # Get current hostname HOSTNAME=$(hostname -s) -if [[ "$HOSTNAME" == "ipvsmisc" ]]; then - echo "tbd." -elif [[ "$HOSTNAME" == "rostam" ]]; then - echo "tbd." +if [[ "$HOSTNAME" == "rostam1" ]]; then + PARTITION=buran + THREAD_POW=5 + NODE=buran01 + module load gcc/14.2.0 elif [[ "$HOSTNAME" == "login1" ]]; then PARTITION=short THREAD_POW=5 -elif [[ "$HOSTNAME" == "simcl1" ]]; then - echo "tbd." + NODE=fj001 +elif [[ "$HOSTNAME" == "ipvsmisc" ]]; then + PARTITION=epyc + THREAD_POW=7 + NODE=ipvs-epyc1 else echo "Hostname is $HOSTNAME — no action taken." exit 1 @@ -53,7 +57,7 @@ fi ################################################################################ # Run benchmarks ################################################################################ -RESULT_DIR=$TOP_DIR/shared_benchmark_on_$HOSTNAME +RESULT_DIR=$TOP_DIR/shared_benchmark_on_$PARTITION SCRIPT_DIR=$TOP_DIR/benchmark/sbatch_scripts ################################################################################ @@ -64,16 +68,16 @@ mkdir -p $RESULT_DIR/strong_scaling/plans cd $RESULT_DIR/strong_scaling # Config -LOOP=2 +LOOP=10 BASE_SIZE=16384 -START_THREAD_POW=3 +START_THREAD_POW=0 # Loop over HPX-FFT executables RUN_SCRIPT="$SCRIPT_DIR/run_hpxfft_strong_shared.sh" for EXE in "${HPXFFT_EXECUTABLES[@]}"; do echo "Submitting job for executable: $EXE" - sbatch -p $PARTITION -N 1 -n 1 -c $THREADS \ + sbatch -p $PARTITION -w $NODE -N 1 -n 1 -c $THREADS \ $RUN_SCRIPT \ $EXE $FFTW_PLAN $START_THREAD_POW $THREAD_POW $BASE_SIZE $LOOP done @@ -84,7 +88,7 @@ RUN_SCRIPT="$SCRIPT_DIR/run_fftw_strong_shared.sh" for EXE in "${FFTW_EXECUTABLES[@]}"; do echo "Submitting job for executable: $EXE" - sbatch -p $PARTITION -N 1 -n 1 -c $THREADS \ + sbatch -p $PARTITION -w $NODE -N 1 -n 1 -c $THREADS \ $RUN_SCRIPT \ $EXE $FFTW_PLAN $START_THREAD_POW $THREAD_POW $BASE_SIZE $LOOP done @@ -97,7 +101,7 @@ mkdir -p $RESULT_DIR/weak_scaling/plans cd $RESULT_DIR/weak_scaling # Config -LOOP=2 +LOOP=10 BASE_SIZE=512 START_THREAD_POW=0 @@ -106,7 +110,7 @@ RUN_SCRIPT="$SCRIPT_DIR/run_hpxfft_weak_shared.sh" for EXE in "${HPXFFT_EXECUTABLES[@]}"; do echo "Submitting job for executable: $EXE" - sbatch -p $PARTITION -N 1 -n 1 -c $THREADS \ + sbatch -p $PARTITION -w $NODE -N 1 -n 1 -c $THREADS \ $RUN_SCRIPT \ $EXE $FFTW_PLAN $START_THREAD_POW $THREAD_POW $BASE_SIZE $LOOP done @@ -117,7 +121,7 @@ RUN_SCRIPT="$SCRIPT_DIR/run_fftw_weak_shared.sh" for EXE in "${FFTW_EXECUTABLES[@]}"; do echo "Submitting job for executable: $EXE" - sbatch -p $PARTITION -N 1 -n 1 -c $THREADS \ + sbatch -p $PARTITION -w $NODE -N 1 -n 1 -c $THREADS \ $RUN_SCRIPT \ $EXE $FFTW_PLAN $START_THREAD_POW $THREAD_POW $BASE_SIZE $LOOP done @@ -130,15 +134,15 @@ mkdir -p $RESULT_DIR/size_scaling/plans cd $RESULT_DIR/size_scaling # Config -LOOP=2 -START_SIZE_POW=5 -STOP_SIZE_POW=12 +LOOP=10 +START_SIZE_POW=3 +STOP_SIZE_POW=16 # Loop over HPX-FFT executables RUN_SCRIPT="$SCRIPT_DIR/run_hpxfft_size_shared.sh" for EXE in "${HPXFFT_EXECUTABLES[@]}"; do echo "Submitting job for executable: $EXE" - sbatch -p $PARTITION -N 1 -n 1 -c $THREADS \ + sbatch -p $PARTITION -w $NODE -N 1 -n 1 -c $THREADS \ $RUN_SCRIPT \ $EXE $FFTW_PLAN $THREADS $START_SIZE_POW $STOP_SIZE_POW $LOOP done @@ -149,7 +153,7 @@ RUN_SCRIPT="$SCRIPT_DIR/run_fftw_size_shared.sh" for EXE in "${FFTW_EXECUTABLES[@]}"; do echo "Submitting job for executable: $EXE" - sbatch -p $PARTITION -N 1 -n 1 -c $THREADS \ + sbatch -p $PARTITION -w $NODE -N 1 -n 1 -c $THREADS \ $RUN_SCRIPT \ $EXE $FFTW_PLAN $THREADS $START_SIZE_POW $STOP_SIZE_POW $LOOP done diff --git a/compile.sh b/compile.sh index ab28f455..0f366b5c 100755 --- a/compile.sh +++ b/compile.sh @@ -34,6 +34,14 @@ if command -v spack &> /dev/null; then echo "Found ${SPACK_ENV} environment, activating it." spack env activate ${SPACK_ENV} fi + elif [[ "$HOSTNAME" == buran0[0-9] || "$HOSTNAME" == buran1[0-5] ]]; then + module load gcc/14.2.0 + # Check if the spack environment exists + SPACK_ENV=hpxfft_x86_buran + if spack env list | grep -q "${SPACK_ENV}"; then + echo "Found ${SPACK_ENV} environment, activating it." + spack env activate ${SPACK_ENV} + fi else echo "Hostname is $HOSTNAME — no action taken." fi diff --git a/compile_fftw_examples.sh b/compile_fftw_examples.sh index 92e970a9..8d6c6245 100755 --- a/compile_fftw_examples.sh +++ b/compile_fftw_examples.sh @@ -34,6 +34,14 @@ if command -v spack &> /dev/null; then echo "Found ${SPACK_ENV} environment, activating it." spack env activate ${SPACK_ENV} fi + elif [[ "$HOSTNAME" == buran0[0-9] || "$HOSTNAME" == buran1[0-5] ]]; then + module load gcc/14.2.0 + # Check if the spack environment exists + SPACK_ENV=hpxfft_x86_buran + if spack env list | grep -q "${SPACK_ENV}"; then + echo "Found ${SPACK_ENV} environment, activating it." + spack env activate ${SPACK_ENV} + fi else echo "Hostname is $HOSTNAME — no action taken." fi diff --git a/compile_hpxfft_examples.sh b/compile_hpxfft_examples.sh index 37bea4a9..fa18cab2 100755 --- a/compile_hpxfft_examples.sh +++ b/compile_hpxfft_examples.sh @@ -37,6 +37,15 @@ if command -v spack &> /dev/null; then echo "Found ${SPACK_ENV} environment, activating it." spack env activate ${SPACK_ENV} fi + elif [[ "$HOSTNAME" == buran0[0-9] || "$HOSTNAME" == buran1[0-5] ]]; then + module load gcc/14.2.0 + LIB_DIR=lib64 + # Check if the spack environment exists + SPACK_ENV=hpxfft_x86_buran + if spack env list | grep -q "${SPACK_ENV}"; then + echo "Found ${SPACK_ENV} environment, activating it." + spack env activate ${SPACK_ENV} + fi else echo "Hostname is $HOSTNAME — no action taken." fi diff --git a/spack-repo/environments/hpxfft_x86_buran.yaml b/spack-repo/environments/hpxfft_x86_buran.yaml new file mode 100644 index 00000000..9dae2f45 --- /dev/null +++ b/spack-repo/environments/hpxfft_x86_buran.yaml @@ -0,0 +1,8 @@ +spack: + specs: + - hpx@1.11.0%gcc@14.2.0 malloc=system networking=all max_cpu_count=256 instrumentation=none + - fftw@3.3.10%gcc@14.2.0 +mpi +openmp + - openmpi@5.0.5 + view: true + concretizer: + unify: true diff --git a/spack-repo/environments/hpxfft_x86_simcl.yaml b/spack-repo/environments/hpxfft_x86_simcl.yaml deleted file mode 100644 index dfe7d373..00000000 --- a/spack-repo/environments/hpxfft_x86_simcl.yaml +++ /dev/null @@ -1,8 +0,0 @@ -spack: - specs: - - hpx@1.11.0%gcc@14.1.0 malloc=system networking=mpi max_cpu_count=256 instrumentation=none - - fftw@3.3.10%gcc@14.1.0 +mpi +openmp - - cmake@3.30.5%gcc@14.1.0 - view: true - concretizer: - unify: true diff --git a/spack-repo/packages/hpx/__pycache__/package.cpython-39.pyc b/spack-repo/packages/hpx/__pycache__/package.cpython-39.pyc new file mode 100644 index 00000000..75c3a058 Binary files /dev/null and b/spack-repo/packages/hpx/__pycache__/package.cpython-39.pyc differ diff --git a/spack-repo/packages/hpx/git_external.patch b/spack-repo/packages/hpx/git_external.patch new file mode 100644 index 00000000..72d203e7 --- /dev/null +++ b/spack-repo/packages/hpx/git_external.patch @@ -0,0 +1,81 @@ +--- hpx/cmake/GitExternal.cmake.orig 2019-08-16 17:27:04.856315000 +0200 ++++ hpx/cmake/GitExternal.cmake 2019-08-16 17:27:47.362709000 +0200 +@@ -19,12 +19,12 @@ + # + # [optional] Flags which control behaviour + # NO_UPDATE +-# When set, GitExternal will not change a repo that has already been checked out. +-# The purpose of this is to allow one to set a default branch to be checked out, +-# but stop GitExternal from changing back to that branch if the user has checked ++# When set, GitExternal will not change a repo that has already been checked out. ++# The purpose of this is to allow one to set a default branch to be checked out, ++# but stop GitExternal from changing back to that branch if the user has checked + # out and is working on another. +-# VERBOSE +-# When set, displays information about git commands that are executed ++# VERBOSE ++# When set, displays information about git commands that are executed + # + + find_package(Git) +@@ -55,17 +55,21 @@ + if(nok) + message(FATAL_ERROR "${DIR} git clone failed: ${error}\n") + endif() +- endif() + +- if(IS_DIRECTORY "${DIR}/.git") ++ # checkout requested tag ++ GIT_EXTERNAL_MESSAGE("git checkout -q ${TAG}") ++ execute_process( ++ COMMAND "${GIT_EXECUTABLE}" checkout -q "${TAG}" ++ RESULT_VARIABLE nok ERROR_VARIABLE error ++ WORKING_DIRECTORY "${DIR}" ++ ) ++ if(nok) ++ message(STATUS "${DIR} git checkout ${TAG} failed: ${error}\n") ++ endif() ++ elseif(IS_DIRECTORY "${DIR}/.git") + if (${GIT_EXTERNAL_NO_UPDATE}) + GIT_EXTERNAL_MESSAGE("Update branch disabled by user") + else() +- GIT_EXTERNAL_MESSAGE("current ref is \"${currentref}\" and tag is \"${TAG}\"") +- if(currentref STREQUAL TAG) # nothing to do +- return() +- endif() +- + # reset generated files + foreach(GIT_EXTERNAL_RESET_FILE ${GIT_EXTERNAL_RESET}) + GIT_EXTERNAL_MESSAGE("git reset -q ${GIT_EXTERNAL_RESET_FILE}") +@@ -100,15 +104,24 @@ + message(STATUS "${DIR} git checkout ${TAG} failed: ${error}\n") + endif() + +- # update tag +- GIT_EXTERNAL_MESSAGE("git rebase FETCH_HEAD") +- execute_process(COMMAND ${GIT_EXECUTABLE} rebase FETCH_HEAD +- RESULT_VARIABLE RESULT OUTPUT_VARIABLE OUTPUT ERROR_VARIABLE OUTPUT ++ # check if this is a branch ++ GIT_EXTERNAL_MESSAGE("git symbolic-ref -q HEAD") ++ execute_process(COMMAND "${GIT_EXECUTABLE}" symbolic-ref -q HEAD ++ RESULT_VARIABLE nok ERROR_VARIABLE error + WORKING_DIRECTORY "${DIR}") +- if(RESULT) +- message(STATUS "git rebase failed, aborting ${DIR} merge") +- execute_process(COMMAND ${GIT_EXECUTABLE} rebase --abort ++ if(nok) ++ message(STATUS "${TAG} is not a branch") ++ else() ++ # update tag ++ GIT_EXTERNAL_MESSAGE("git rebase FETCH_HEAD") ++ execute_process(COMMAND ${GIT_EXECUTABLE} rebase FETCH_HEAD ++ RESULT_VARIABLE RESULT OUTPUT_VARIABLE OUTPUT ERROR_VARIABLE OUTPUT + WORKING_DIRECTORY "${DIR}") ++ if(RESULT) ++ message(STATUS "git rebase failed, aborting ${DIR} merge") ++ execute_process(COMMAND ${GIT_EXECUTABLE} rebase --abort ++ WORKING_DIRECTORY "${DIR}") ++ endif() + endif() + endif() + else() diff --git a/spack-repo/packages/hpx/mimalloc_no_version_requirement.patch b/spack-repo/packages/hpx/mimalloc_no_version_requirement.patch new file mode 100644 index 00000000..ff7363da --- /dev/null +++ b/spack-repo/packages/hpx/mimalloc_no_version_requirement.patch @@ -0,0 +1,12 @@ +diff --git a/cmake/HPX_SetupAllocator.cmake b/cmake/HPX_SetupAllocator.cmake +index 221c988d997..2e9d760a778 100644 +--- a/cmake/HPX_SetupAllocator.cmake ++++ b/cmake/HPX_SetupAllocator.cmake +@@ -78,6 +78,6 @@ if(NOT TARGET hpx_dependencies_allocator) + # ########################################################################## + # MIMALLOC + if("${HPX_WITH_MALLOC_UPPER}" STREQUAL "MIMALLOC") +- find_package(mimalloc 1.0) ++ find_package(mimalloc) + if(NOT mimalloc_FOUND) + hpx_error(${allocator_error}) diff --git a/spack-repo/packages/hpx/package.py b/spack-repo/packages/hpx/package.py new file mode 100644 index 00000000..f1a7b7da --- /dev/null +++ b/spack-repo/packages/hpx/package.py @@ -0,0 +1,309 @@ +# Copyright 2013-2024 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + + +import sys + +from spack.package import * +from spack.pkg.builtin.boost import Boost + + +class Hpx(CMakePackage, CudaPackage, ROCmPackage): + """C++ runtime system for parallel and distributed applications.""" + + homepage = "https://hpx.stellar-group.org/" + url = "https://github.com/STEllAR-GROUP/hpx/archive/v0.0.0.tar.gz" + git = "https://github.com/STEllAR-GROUP/hpx.git" + maintainers("msimberg", "albestro", "teonnik", "hkaiser", "diehlpk") + + license("BSL-1.0") + + tags = ["e4s"] + + version("master", branch="master") + version("stable", tag="stable", commit="103a7b8e3719a0db948d1abde29de0ff91e070be") + version("1.11.0", sha256="01ec47228a2253b41e318bb09c83325a75021eb6ef3262400fbda30ac7389279") + version("1.10.0", sha256="5720ed7d2460fa0b57bd8cb74fa4f70593fe8675463897678160340526ec3c19") + version("1.9.1", sha256="1adae9d408388a723277290ddb33c699aa9ea72defadf3f12d4acc913a0ff22d") + version("1.9.0", sha256="2a8dca78172fbb15eae5a5e9facf26ab021c845f9c09e61b1912e6cf9e72915a") + version("1.8.1", sha256="2fc4c10f55e2e6bcdc6f6ff950e26c6d8e218e138fdbd885ee71ccf5c5549054") + version("1.8.0", sha256="93f147ab7cf0ab4161f37680ea720d3baeb86540a95382f2fb591645b2a9b135") + version("1.7.1", sha256="008a0335def3c551cba31452eda035d7e914e3e4f77eec679eea070ac71bd83b") + version("1.7.0", sha256="05099b860410aa5d8a10d6915b1a8818733aa1aa2d5f2b9774730ca7e6de5fac") + version("1.6.0", sha256="4ab715613c1e1808edc93451781cc9bc98feec4e422ccd4322858a680f6d9017") + version("1.5.1", sha256="b2f9358ce2a9446b9d8fb1998c30913e7199b007aa82e46d0aa05c763331c635") + version("1.5.0", sha256="de2901d8ae017592c513e0af9cf58de295abc9802e55ece00424cbd8a3801920") + version("1.4.1", sha256="965dabe44d17480e326d92da4eec56722d98b33943c53d2b0f8f4655cb208023") + version("1.4.0", sha256="241a1c47fafba751848fac12446e7bf4ad3d342d5eb2fa1ef94dd904acc329ed") + version("1.3.0", sha256="cd34da674064c4cc4a331402edbd65c5a1f8058fb46003314ca18fa08423c5ad") + version("1.2.1", sha256="8cba9b48e919035176d3b7bbfc2c110df6f07803256626f1dad8d9dde16ab77a") + version("1.2.0", sha256="20942314bd90064d9775f63b0e58a8ea146af5260a4c84d0854f9f968077c170") + version("1.1.0", sha256="1f28bbe58d8f0da600d60c3a74a644d75ac777b20a018a5c1c6030a470e8a1c9") + + depends_on("c", type="build") + depends_on("cxx", type="build") + + generator("ninja") + + map_cxxstd = lambda cxxstd: "2a" if cxxstd == "20" else cxxstd + cxxstds = ("11", "14", "17", "20") + variant( + "cxxstd", + default="17", + values=cxxstds, + description="Use the specified C++ standard when building.", + ) + + variant( + "malloc", + default="tcmalloc", + description="Define which allocator will be linked in", + values=("system", "jemalloc", "mimalloc", "tbbmalloc", "tcmalloc"), + ) + + variant( + "max_cpu_count", + default="auto", + description="Max number of OS-threads for HPX applications", + values=lambda x: isinstance(x, str) and (x.isdigit() or x == "auto"), + ) + + instrumentation_values = ("apex", "google_perftools", "papi", "valgrind", "thread_debug") + variant( + "instrumentation", + values=any_combination_of(*instrumentation_values), + description="Add support for various kind of instrumentation", + ) + + variant( + "networking", + values=any_combination_of("tcp", "mpi", "lci", "all").with_default("tcp"), + description="Support for networking through parcelports", + ) + + default_generic_coroutines = True + if sys.platform.startswith("linux") or sys.platform == "win32": + default_generic_coroutines = False + variant( + "generic_coroutines", + default=default_generic_coroutines, + description="Use Boost.Context as the underlying coroutines" + " context switch implementation.", + ) + + variant("tools", default=False, description="Build HPX tools") + variant("examples", default=False, description="Build examples") + variant("async_mpi", default=False, description="Enable MPI Futures.") + variant("async_cuda", default=False, description="Enable CUDA Futures.") + variant("static", default=False, description='Compile HPX statically linked libraries') + + # Build dependencies + depends_on("python", type=("build", "test", "run")) + depends_on("pkgconfig", type="build") + depends_on("git", type="build") + depends_on("cmake", type="build") + + # Other dependecies + depends_on("hwloc") + depends_on(Boost.with_default_variants) + depends_on("boost +context", when="+generic_coroutines") + for cxxstd in cxxstds: + depends_on("boost cxxstd={0}".format(map_cxxstd(cxxstd)), when="cxxstd={0}".format(cxxstd)) + + with when("@1.7:"): + for cxxstd in cxxstds: + depends_on(f"asio cxxstd={map_cxxstd(cxxstd)}", when=f"cxxstd={cxxstd}") + + depends_on("gperftools", when="malloc=tcmalloc") + depends_on("jemalloc", when="malloc=jemalloc") + depends_on("mimalloc", when="malloc=mimalloc") + depends_on("tbb", when="malloc=tbbmalloc") + + depends_on("mpi", when="networking=mpi") + depends_on("mpi", when="networking=all") + depends_on("mpi", when="+async_mpi") + depends_on("lci", when="networking=lci") + depends_on("lci", when="networking=all") + + depends_on("cuda", when="+async_cuda") + + depends_on("otf2", when="instrumentation=apex") + depends_on("gperftools", when="instrumentation=google_perftools") + depends_on("papi", when="instrumentation=papi") + depends_on("valgrind", when="instrumentation=valgrind") + + # Only ROCm or CUDA maybe be enabled at once + conflicts("+rocm", when="+cuda") + + # Restrictions for 1.9.X + with when("@1.9:"): + conflicts("%gcc@:8") + conflicts("%clang@:9") + + # Restrictions for 1.8.X + with when("@1.8:"): + conflicts("cxxstd=14") + conflicts("%gcc@:7") + conflicts("%clang@:8") + depends_on("cuda@11:", when="+cuda") + + # Restrictions for 1.7.X + with when("@1.7:"): + depends_on("cmake@3.18.0:", type="build") + depends_on("boost@1.71.0:") + depends_on("asio@1.12.0:") + conflicts("%gcc@:6") + conflicts("%clang@:6") + + # Restrictions for 1.6.X + conflicts("+rocm", when="@:1.5") + + # Restrictions for 1.5.x + conflicts("cxxstd=11", when="@1.5:") + + # Restrictions for 1.2.X + with when("@:1.2.1"): + depends_on("cmake@3.9.0:", type="build") + depends_on("boost@1.62.0:") + depends_on("hwloc@1.11:") + + # Restrictions before 1.2 + with when("@:1.1.0"): + depends_on("boost@1.55.0:") + depends_on("hwloc@1.6:") + + # Patches and one-off conflicts + + # Asio 1.34.0 removed io_context::work, used by HPX: + # https://github.com/chriskohlhoff/asio/commit/a70f2df321ff40c1809773c2c09986745abf8d20. + conflicts("^asio@1.34:") + + # Certain Asio headers don't compile with nvcc from 1.17.0 onwards with + # C++17. Starting with CUDA 11.3 they compile again. + conflicts("^asio@1.17.0:", when="+cuda cxxstd=17 ^cuda@:11.2") + + # Starting from ROCm 5.0.0 hipcc miscompiles asio 1.17.0 and newer + conflicts("^asio@1.17.0:", when="+rocm ^hip@5:") + + # Boost and HIP don't work together in certain versions: + # https://github.com/boostorg/config/issues/392. Boost 1.78.0 and HPX 1.8.0 + # both include a fix. + conflicts("^boost@:1.77.0", when="@:1.7 +rocm") + + # libstdc++ has a broken valarray in some versions that clang/hipcc refuses + # to compile: + # https://github.com/spack/spack/issues/38104 + # https://gcc.gnu.org/bugzilla/show_bug.cgi?id=103022 + conflicts("%gcc@9.1:9.4", when="+rocm") + conflicts("%gcc@10.1:10.3", when="+rocm") + conflicts("%gcc@11.2", when="+rocm") + + # boost 1.73.0 build problem with HPX 1.4.0 and 1.4.1 + # https://github.com/STEllAR-GROUP/hpx/issues/4728#issuecomment-640685308 + depends_on("boost@:1.72.0", when="@:1.4") + + # COROUTINES + # ~generic_coroutines conflict is not fully implemented + # for additional information see: + # https://github.com/spack/spack/pull/17654 + # https://github.com/STEllAR-GROUP/hpx/issues/4829 + depends_on("boost+context", when="+generic_coroutines") + + _msg_generic_coroutines_platform = "This platform requires +generic_coroutines" + conflicts("~generic_coroutines", when="platform=darwin", msg=_msg_generic_coroutines_platform) + + _msg_generic_coroutines_target = "This target requires +generic_coroutines" + conflicts("~generic_coroutines", when="target=aarch64:", msg=_msg_generic_coroutines_target) + conflicts("~generic_coroutines", when="target=arm:", msg=_msg_generic_coroutines_target) + + # Patches APEX + patch("git_external.patch", when="@1.3.0 instrumentation=apex") + patch("mimalloc_no_version_requirement.patch", when="@:1.8.0 malloc=mimalloc") + + def url_for_version(self, version): + if version >= Version("1.9.0"): + return "https://github.com/STEllAR-GROUP/hpx/archive/v{}.tar.gz".format(version) + return "https://github.com/STEllAR-GROUP/hpx/archive/{}.tar.gz".format(version) + + def instrumentation_args(self): + args = [] + for value in self.instrumentation_values: + condition = "instrumentation={0}".format(value) + args.append(self.define("HPX_WITH_{0}".format(value.upper()), condition in self.spec)) + return args + + def cmake_args(self): + spec, args = self.spec, [] + + format_max_cpu_count = lambda max_cpu_count: ( + "" if max_cpu_count == "auto" else max_cpu_count + ) + args += [ + self.define("HPX_WITH_CXX{0}".format(spec.variants["cxxstd"].value), True), + self.define_from_variant("HPX_WITH_MALLOC", "malloc"), + self.define_from_variant("HPX_WITH_CUDA", "cuda"), + self.define_from_variant("HPX_WITH_HIP", "rocm"), + self.define_from_variant("HPX_WITH_TOOLS", "tools"), + self.define_from_variant("HPX_WITH_EXAMPLES", "examples"), + self.define_from_variant("HPX_WITH_ASYNC_MPI", "async_mpi"), + self.define_from_variant("HPX_WITH_ASYNC_CUDA", "async_cuda"), + self.define_from_variant("HPX_WITH_STATIC_LINKING", "static"), + self.define("HPX_WITH_TESTS", self.run_tests), + self.define("HPX_WITH_NETWORKING", "networking=none" not in spec), + self.define("HPX_WITH_DISTRIBUTED_RUNTIME", "networking=none" not in spec), + self.define("HPX_WITH_PARCELPORT_TCP", spec.satisfies("networking=tcp") or spec.satisfies("networking=all")), + self.define("HPX_WITH_PARCELPORT_MPI", spec.satisfies("networking=mpi") or spec.satisfies("networking=all")), + self.define( + "HPX_WITH_MAX_CPU_COUNT", + format_max_cpu_count(spec.variants["max_cpu_count"].value), + ), + self.define("HPX_WITH_PARCELPORT_LCI", spec.satisfies("networking=lci") or spec.satisfies("networking=all")), + self.define_from_variant("HPX_WITH_GENERIC_CONTEXT_COROUTINES", "generic_coroutines"), + self.define("BOOST_ROOT", spec["boost"].prefix), + self.define("HWLOC_ROOT", spec["hwloc"].prefix), + self.define("HPX_WITH_THREAD_IDLE_RATES", True), + self.define("HPX_WITH_BOOST_ALL_DYNAMIC_LINK", True), + self.define("BUILD_SHARED_LIBS", True), + self.define("HPX_DATASTRUCTURES_WITH_ADAPT_STD_TUPLE", False), + ] + + # Enable unity builds when available + if spec.satisfies("@1.7:"): + args += [self.define("HPX_WITH_UNITY_BUILD", True)] + + # HIP support requires compiling with hipcc + if self.spec.satisfies("+rocm"): + args += [self.define("CMAKE_CXX_COMPILER", self.spec["hip"].hipcc)] + if self.spec.satisfies("^cmake@3.21.0:3.21.2"): + args += [self.define("__skip_rocmclang", True)] + + # Instrumentation + args += self.instrumentation_args() + + if spec.satisfies("instrumentation=thread_debug"): + args += [ + self.define("HPX_WITH_THREAD_DEBUG_INFO", True), + self.define("HPX_WITH_LOGGING", True), + ] + + if spec.satisfies("instrumentation=apex"): + args += [ + self.define("HPX_WITH_DISTRIBUTED_RUNTIME", True), + self.define("HPX_WITH_FETCH_APEX", True), + self.define("APEX_WITH_OTF2", True), + self.define("OTF2_ROOT", spec["otf2"].prefix), + ] + + # it seems like there was a bug in the default version of APEX in 1.5.x + if spec.satisfies("@1.5"): + args += [self.define("HPX_WITH_APEX_TAG", "v2.3.0")] + + # Custom CMake options to use Clang as CUDA compiler + if self.spec.satisfies("+cuda"): + if self.spec.satisfies("%clang"): + args += [self.define("CMAKE_CUDA_COMPILER", "clang")] + args += [self.define("NVCC_WRAPPER_DEFAULT_COMPILER", "clang")] + + return args