diff --git a/configs/sites/tier1/ursa/packages.yaml b/configs/sites/tier1/ursa/packages.yaml index fd82657501..381c191542 100644 --- a/configs/sites/tier1/ursa/packages.yaml +++ b/configs/sites/tier1/ursa/packages.yaml @@ -14,6 +14,11 @@ packages: - +kokkos - +arborx + # To avoid duplicate packages - py-arch and py-statsmodels need py-setuptools-scm@8 + py-setuptools-scm: + require: + - '@8.2.1' + # GPUs on ursa are H100 # add: '+cuda cuda_arch=90' to kokkos / kokkos-kernels / arborx if building with CUDA support # add: '+wrapper' to kokkos @@ -60,13 +65,6 @@ packages: externals: - spec: doxygen@1.11.0+graphviz~mscgen prefix: /apps/spack-2024-12/linux-rocky9-x86_64/gcc-11.4.1/doxygen-1.11.0-dbhxblaakcrhvmlrbgixvapnagjb2vkx - ecflow: - buildable: False - externals: - - spec: ecflow@5.11.4 - prefix: /apps/spack-2024-12/linux-rocky9-x86_64/gcc-11.4.1/ecflow-5.11.4-a5ggcmayx6yesbqgbacifgvkstamchoy - modules: - - ecflow/5.11.4 file: externals: - spec: file@5.39 diff --git a/configs/sites/tier1/ursa/packages_nvhpc-26.9.yaml b/configs/sites/tier1/ursa/packages_nvhpc-26.9.yaml new file mode 100644 index 0000000000..4939b9e0e7 --- /dev/null +++ b/configs/sites/tier1/ursa/packages_nvhpc-26.9.yaml @@ -0,0 +1,207 @@ +packages: + all: + require: + - any_of: ['%nvhpc@=26.9'] + when: '%nvhpc' + - any_of: ['%gcc@=11.5.0'] + when: '%gcc' + + mpi: + buildable: false + require: + - 'openmpi@5.0.10rc2' + + nvhpc: + buildable: false + externals: + - spec: nvhpc@26.9+blas+lapack~mpi + prefix: /apps/spack-2026-10/linux-x86_64/nvhpc-26.9-5ltv4xi + modules: + - spack-apps/2026.10 + - nvhpc/26.9 + extra_attributes: + compilers: + c: /apps/spack-2026-10/linux-x86_64/nvhpc-26.9-5ltv4xi/Linux_x86_64/26.9/compilers/bin/nvc + cxx: /apps/spack-2026-10/linux-x86_64/nvhpc-26.9-5ltv4xi/Linux_x86_64/26.9/compilers/bin/nvc++ + fortran: /apps/spack-2026-10/linux-x86_64/nvhpc-26.9-5ltv4xi/Linux_x86_64/26.9/compilers/bin/nvfortran + + openmpi: + buildable: false + require:: + - '+internal-hwloc' + externals: + - spec: openmpi@5.0.10rc2 +internal-hwloc +internal-pmix +internal-libevent +two_level_namespace + prefix: /apps/spack-2026-10/linux-x86_64/nvhpc-26.9-5ltv4xi/Linux_x86_64/26.9/comm_libs/13.3/hpcx/hpcx-2.50/ompi5 + modules: + - spack-apps/2026.10 + - nvhpc/26.9 + + # System gcc (no module needed, nvhpc module stays loaded); also the CUDA 13.3 host compiler. + gcc: + buildable: false + require: + - '@11.5.0' + externals: + - spec: gcc@11.5.0 languages:='c,c++,fortran' + prefix: /usr + extra_attributes: + compilers: + c: /usr/bin/gcc + cxx: /usr/bin/g++ + fortran: /usr/bin/gfortran + + cuda: + buildable: false + externals: + - spec: cuda@13.3 + prefix: /apps/spack-2026-10/linux-x86_64/nvhpc-26.9-5ltv4xi/Linux_x86_64/26.9/cuda/13.3 + modules: + - spack-apps/2026.10 + - nvhpc/26.9 + + # libsm and others include , which ossp-uuid does not provide + uuid: + require: + - util-linux-uuid + + # Ursa GPUs H100: CUDA build for H100 (cuda_arch=90) + kokkos: + require: + - '+cuda cuda_arch=90' + - '+wrapper' + kokkos-kernels: + require: + - '+cuda cuda_arch=90' + arborx: + require: + - '+cuda cuda_arch=90' + + # Fortran flags for nvfortran + # gfsio, landsfcutil, sfcio, sigio: free-form sources with .f suffix; + # upstream CMake sets free-form/big-endian flags only for Intel/GNU + gfsio: + require: + - "fflags='-Mfree -byteswapio'" + # hdf5 2.x hl/fortran targets miss the MPI Fortran module dir (HPC-X keeps mpi_f08.mod in lib/) + hdf5: + require: + - "fflags='-I/apps/spack-2026-10/linux-x86_64/nvhpc-26.9-5ltv4xi/Linux_x86_64/26.9/comm_libs/13.3/hpcx/hpcx-2.50/ompi5/lib'" + # landsfcutil_d is not built with -r8 under NVHPC (CMake sets it for Intel only); needs a patch + landsfcutil: + require: + - "fflags='-Mfree'" + sfcio: + require: + - "fflags='-Mfree -byteswapio'" + sigio: + require: + - "fflags='-Mfree -byteswapio'" + + # Packages that nvc/nvc++ cannot build: use gcc + # nvc rejects GNU computed goto and inline asm .previous directives + alsa-lib: + require: + - '%c=gcc' + # Build with gcc; ecflow+ui (boost, qt) is C++ heavy + ecflow: + require: + - '%c,cxx=gcc' + # nvc rejects -Wold-style-definition + elfutils: + require: + - '%c=gcc' + # GNU-specific inline assembly; depends on alsa-lib + ffmpeg: + require: + - '%c,cxx=gcc' + # nvc++ rejects GNU warning flags (-Wold-style-cast, -Wnon-virtual-dtor) + flatbuffers: + require: + - '%cxx=gcc' + # nvc rejects -Wmissing-declarations + fontconfig: + require: + - '%c=gcc' + # nvc rejects -ansi + fribidi: + require: + - '%c=gcc' + # nvc++ cannot parse libertiff templates ("type name is not allowed") + gdal: + require: + - '%c,cxx=gcc' + # nvhpc-export-symbols.patch does not apply to gettext@1.0 + gettext: + require: + - '%c,cxx=gcc' + # meson does not add a PIC flag for nvc ("Can't link non-PIC static library") + glib: + require: + - '%c,cxx=gcc' + glib-bootstrap: + require: + - '%c,cxx=gcc' + # nvc rejects gnulib's GCC warning flags (-Wimplicit-fallthrough, -Wno-sign-conversion, ...) + gperf: + require: + - '%c,cxx=gcc' + # nvc rejects -nodefaultlibs when linking libicudata + icu4c: + require: + - '%c,cxx=gcc' + # nvc rejects -Wno-psabi; build also parses GCC preprocessor output (gpg-error.def) + libgpg-error: + require: + - '%c=gcc' + # nvc++ rejects GNU alias attribute in pcrecpp.cc + pcre: + require: + - '%c,cxx=gcc' + # meson does not add a PIC flag for nvc ("Can't link non-PIC static library") + pixman: + require: + - '%c=gcc' + # nvc++ 26.9 crashes: "LLVM ERROR: Broken module found" + proj: + require: + - '%c,cxx=gcc' + # Python is built with gcc; pip passes its CFLAGS (-fno-strict-overflow) to nvc + py-awscrt: + require: + - '%c,cxx=gcc' + py-cffi: + require: + - '%c=gcc' + py-crc32c: + require: + - '%c=gcc' + py-kiwisolver: + require: + - '%c,cxx=gcc' + py-lxml: + require: + - '%c=gcc' + py-msgpack: + require: + - '%c=gcc' + py-pillow: + require: + - '%c=gcc' + py-psutil: + require: + - '%c=gcc' + py-ruamel-yaml-clib: + require: + - '%c=gcc' + # util-linux-uuid is C only; '::' replaces common's '%c,cxx=gcc', which cannot be satisfied + util-linux-uuid: + require:: + - '%c=gcc' + # nvc++ rejects -Weffc++ and -pedantic-errors + yaml-cpp: + require: + - '%c,cxx=gcc' + # zlib has its own require:: in packages.yaml, bypassing the 'all' rule + zlib: + require: + - '%c=gcc' diff --git a/util/epic/batch_install.sh b/util/epic/batch_install.sh index 3e0942a995..789c65e300 100755 --- a/util/epic/batch_install.sh +++ b/util/epic/batch_install.sh @@ -34,13 +34,15 @@ usage() { echo " if not set, the default location is used" echo " -c Provide location of build caches as BUILDCACHE_DIR;" echo " if not set, authoritative build caches are used" - echo " -b Set compiler build option, can be 'gcc', 'oneapi', or 'all' - the default;" + echo " -b Set compiler build option, can be 'gcc', 'oneapi', 'nvhpc', or 'all' - the default;" + echo " 'all' does not include optional compilers (e.g. nvhpc on ursa), select these explicitly" echo " -u Flag to update bootstrap and source caches;" echo " -e Continue builds/install in existing environments;" echo " by default, exit with an error if already exist" echo " -s Submit 'spack install' to batch scheduler" echo " -t Run tests for specific thirdparty dependencies;" echo " these are currently hardcoded in batch_install.sh" + echo " -v Verbose output, print commands as they are executed (set -x)" echo " -h display this help" echo } @@ -72,6 +74,9 @@ do t) SPACK_STACK_RUN_TESTS="true" ;; + v) + SPACK_STACK_VERBOSE="true" + ;; *) usage exit 1 @@ -79,15 +84,22 @@ do esac done +SPACK_STACK_COMPILER_BUILD_OPT=${SPACK_STACK_COMPILER_BUILD_OPT:-all} + echo "INFO: $0 options:" echo " SPACK_STACK_MODE: ${SPACK_STACK_MODE:-not set}" echo " SPACK_STACK_ENVIRONMENT_DIRS: ${SPACK_STACK_ENVIRONMENT_DIRS:-${SPACK_STACK_DIR}/envs}" echo " SPACK_STACK_BUILDCACHE_DIR: ${SPACK_STACK_BUILDCACHE_DIR:-use default caches}" -echo " SPACK_STACK_COMPILER_BUILD_OPT: ${SPACK_STACK_COMPILER_BUILD_OPT-all}" +echo " SPACK_STACK_COMPILER_BUILD_OPT: ${SPACK_STACK_COMPILER_BUILD_OPT}" echo " SPACK_STACK_UPDATE_DEV_CACHES: ${SPACK_STACK_UPDATE_DEV_CACHES:-false}" echo " SPACK_STACK_IGNORE_ENV_EXIST: ${SPACK_STACK_IGNORE_ENV_EXIST:-false}" echo " SPACK_STACK_SUBMIT_TO_SCHEDULER: ${SPACK_STACK_SUBMIT_TO_SCHEDULER:-false}" echo " SPACK_STACK_RUN_TESTS: ${SPACK_STACK_RUN_TESTS:-false}" +echo " SPACK_STACK_VERBOSE: ${SPACK_STACK_VERBOSE:-false}" + +if [[ ${SPACK_STACK_VERBOSE} == "true" ]]; then + set -x +fi if [[ -z ${SPACK_STACK_MODE} ]]; then echo "ERROR, SPACK_STACK_MODE not defined. Provide -m MODE as argument" @@ -107,11 +119,11 @@ fi # Check for valid compiler build options case "${SPACK_STACK_COMPILER_BUILD_OPT}" in - all|gcc|oneapi) + all|gcc|oneapi|nvhpc) echo "Valid compiler build option: ${SPACK_STACK_COMPILER_BUILD_OPT}" ;; *) - echo "ERROR, invalid compiler build option: ${SPACK_STACK_COMPILER_BUILD_OPT}. Allowed values are: all, gcc, oneapi" + echo "ERROR, invalid compiler build option: ${SPACK_STACK_COMPILER_BUILD_OPT}. Allowed values are: all, gcc, oneapi, nvhpc" exit 1 ;; esac @@ -157,6 +169,8 @@ case ${SPACK_STACK_BATCH_HOST} in ;; ufe) # ursa SPACK_STACK_BATCH_COMPILERS=("oneapi@=2025.3.1" "oneapi@=2025.3.1-hpcx" "gcc@=12.4.0") + # Built only when selected explicitly with -b nvhpc + SPACK_STACK_BATCH_OPTIONAL_COMPILERS=("nvhpc@=26.9") SPACK_STACK_BATCH_TEMPLATES=("unified-dev") SPACK_STACK_MODULE_CHOICE="lmod" SPACK_STACK_BOOTSTRAP_MIRROR="/contrib/spack-stack/bootstrap-mirror" @@ -170,6 +184,23 @@ case ${SPACK_STACK_BATCH_HOST} in ;; esac +# Optional compilers are only built when selected explicitly, not with 'all' +if [[ ! "${SPACK_STACK_COMPILER_BUILD_OPT}" == "all" ]]; then + SPACK_STACK_BATCH_COMPILERS+=("${SPACK_STACK_BATCH_OPTIONAL_COMPILERS[@]}") +fi + +# Check that at least one compiler on this host matches the compiler build option +compiler_selected="false" +for compiler in "${SPACK_STACK_BATCH_COMPILERS[@]}"; do + if [[ "${SPACK_STACK_COMPILER_BUILD_OPT}" == "all" || "${compiler%%@*}" == "${SPACK_STACK_COMPILER_BUILD_OPT}" ]]; then + compiler_selected="true" + fi +done +if [[ ! ${compiler_selected} == "true" ]]; then + echo "ERROR, no '${SPACK_STACK_COMPILER_BUILD_OPT}' compiler configured for host ${SPACK_STACK_BATCH_HOST}" + exit 1 +fi + ################################################################################################## function fix_permissions() { @@ -425,17 +456,15 @@ for compiler in "${SPACK_STACK_BATCH_COMPILERS[@]}"; do fi # Check compiler build argument to build the desired spack-stack - if [[ ${compiler} =~ "${SPACK_STACK_COMPILER_BUILD_OPT}" || "all" == "${SPACK_STACK_COMPILER_BUILD_OPT}" ]]; then - # Success, do nothing - : - else + compiler_name=$(echo ${compiler} | cut -d "@" -f 1) + compiler_version=$(echo ${compiler} | cut -d "=" -f 2) + + # Check compiler build argument to build the desired spack-stack + if [[ ! "${SPACK_STACK_COMPILER_BUILD_OPT}" == "all" && ! "${compiler_name}" == "${SPACK_STACK_COMPILER_BUILD_OPT}" ]]; then # Compiler not selected, try next item continue fi - compiler_name=$(echo ${compiler} | cut -d "@" -f 1) - compiler_version=$(echo ${compiler} | cut -d "=" -f 2) - for template in "${SPACK_STACK_BATCH_TEMPLATES[@]}"; do echo @@ -539,6 +568,8 @@ for compiler in "${SPACK_STACK_BATCH_COMPILERS[@]}"; do ;; oneapi@=2025.3.1-hpcx) ;; + nvhpc@=26.9) + ;; esac ;; *)