From 24c21ae5ed8bad0e672fad1e7d60c6eb825cb68a Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Tue, 20 Oct 2026 06:45:10 -0600 Subject: [PATCH 01/14] .gitmodules: branch release/2.2 --- .gitmodules | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitmodules b/.gitmodules index 2f6cb05661..1bcbf999c3 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,8 +1,8 @@ [submodule "spack"] path = spack url = https://github.com/jcsda/spack - branch = spack-stack-dev + branch = release/2.2 [submodule "repos/builtin"] path = repos/builtin url = https://github.com/jcsda/spack-packages - branch = spack-stack-dev + branch = release/2.2 From 28326015098cfa6fcc1962e617e9fa6e6499e1a8 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Wed, 21 Oct 2026 21:24:00 -0600 Subject: [PATCH 02/14] [release/2.2] Update spack-packages: HDF5 compiler wrappers bug fix (#2180) * Update submodule pointer for repos/builtin --- repos/builtin | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/repos/builtin b/repos/builtin index e03613d824..8cb08dc006 160000 --- a/repos/builtin +++ b/repos/builtin @@ -1 +1 @@ -Subproject commit e03613d8246aefa7f971c2b4fc8b7006c8387c9f +Subproject commit 8cb08dc0064a79804662d70484026cfa472aea16 From b8fa7f6e68f72bad6cb3eefdff8b945366680f7c Mon Sep 17 00:00:00 2001 From: Matt Thompson Date: Thu, 22 Oct 2026 07:49:35 -0400 Subject: [PATCH 03/14] discover-gmao: set target x86_64_v3 and disable arrow mimalloc/gdal arrow for air-gapped builds (#2182) --- configs/sites/tier2/discover-gmao/packages.yaml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/configs/sites/tier2/discover-gmao/packages.yaml b/configs/sites/tier2/discover-gmao/packages.yaml index 56d60208dc..6cdbc101ce 100644 --- a/configs/sites/tier2/discover-gmao/packages.yaml +++ b/configs/sites/tier2/discover-gmao/packages.yaml @@ -1,4 +1,6 @@ packages: + all: + target: [x86_64_v3] ### Modification of common packages ewok-env: require: @@ -26,6 +28,13 @@ packages: variants: ~shared met: variants: +python +grib2 +graphics +lidar2nc +modis + # mimalloc is vendored and downloaded via FetchContent; fails on air-gapped compute nodes + arrow: + require: + - '~mimalloc' + gdal: + require: + - '~arrow' ### All other external packages listed alphabetically autoconf: externals: From 616e3ce8d020104f0a6ad6cd9c96b2c3843f5736 Mon Sep 17 00:00:00 2001 From: Matt Thompson Date: Mon, 26 Oct 2026 18:10:00 -0400 Subject: [PATCH 04/14] util/gmao/batch_install.sh: use private TMPDIR on discover (noexec /tmp) (#2185) --- util/gmao/batch_install.sh | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/util/gmao/batch_install.sh b/util/gmao/batch_install.sh index 85da989d60..492baa5ed6 100755 --- a/util/gmao/batch_install.sh +++ b/util/gmao/batch_install.sh @@ -1378,6 +1378,18 @@ fi export SPACK_CARGO_HOME=${cargo_mirror_path} export CARGO_NET_OFFLINE=true +# Discover's /tmp may disallow execution, which breaks configure checks such +# as FFmpeg's temporary shell script. Use a private directory per install job +# on the shared filesystem, independently of Spack's build stage setting. +if [[ "${host}" == "discover" || "${host}" == "discover-gmao" ]]; then + build_tmpdir_root=/discover/nobackup/projects/gmao/SIteam/spack-stack/build-tmpdir + mkdir -p "\${build_tmpdir_root}" + build_tmpdir=\$(mktemp -d "\${build_tmpdir_root}/\${USER}.\${SLURM_JOB_ID:-local}.XXXXXX") + export TMPDIR="\${build_tmpdir}" + trap 'rm -rf -- "\${build_tmpdir}"' EXIT + echo "INFO: using TMPDIR=\${TMPDIR}" +fi + if [[ "${SPACK_STACK_DISABLE_LOCKS}" == "true" ]]; then # Spack's package workers do not inherit command-line configuration or the # environment's config scopes. Give every worker a private user config scope From 49930ab3d4db16eb5d44741d86aaceabb2ac2a53 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Mon, 26 Oct 2026 16:13:01 -0600 Subject: [PATCH 05/14] Atlantis: llvm-22.1.7 --> llvm-23.1.2 (#2184) --- .../tier1/atlantis/packages_clang-22.1.7.yaml | 30 ------------------- .../tier1/atlantis/packages_clang-23.1.2.yaml | 30 +++++++++++++++++++ 2 files changed, 30 insertions(+), 30 deletions(-) delete mode 100644 configs/sites/tier1/atlantis/packages_clang-22.1.7.yaml create mode 100644 configs/sites/tier1/atlantis/packages_clang-23.1.2.yaml diff --git a/configs/sites/tier1/atlantis/packages_clang-22.1.7.yaml b/configs/sites/tier1/atlantis/packages_clang-22.1.7.yaml deleted file mode 100644 index 5f5b0f2497..0000000000 --- a/configs/sites/tier1/atlantis/packages_clang-22.1.7.yaml +++ /dev/null @@ -1,30 +0,0 @@ -packages: - all: - require: - - any_of: ['%llvm@=22.1.7'] - when: '%llvm' - mpi: - buildable: False - require: - - "openmpi@5.0.11" - llvm: - externals: - - spec: llvm@22.1.7 +flang ~libomptarget - prefix: /gpfs/neptune/spack-stack/llvm-22.1.7 - modules: - - llvm/22.1.7 - extra_attributes: - compilers: - c: /gpfs/neptune/spack-stack/llvm-22.1.7/bin/clang - cxx: /gpfs/neptune/spack-stack/llvm-22.1.7/bin/clang++ - fortran: /gpfs/neptune/spack-stack/llvm-22.1.7/bin/flang-new - extra_rpaths: - - /gpfs/neptune/spack-stack/llvm-22.1.7/lib - - /gpfs/neptune/spack-stack/llvm-22.1.7/lib/x86_64-unknown-linux-gnu - openmpi: - externals: - - spec: openmpi@5.0.11 ~internal-hwloc +two_level_namespace - prefix: /gpfs/neptune/spack-stack/openmpi-5.0.11/llvm-22.1.7 - modules: - - llvm/22.1.7 - - openmpi/5.0.11 diff --git a/configs/sites/tier1/atlantis/packages_clang-23.1.2.yaml b/configs/sites/tier1/atlantis/packages_clang-23.1.2.yaml new file mode 100644 index 0000000000..7c353499c9 --- /dev/null +++ b/configs/sites/tier1/atlantis/packages_clang-23.1.2.yaml @@ -0,0 +1,30 @@ +packages: + all: + require: + - any_of: ['%llvm@=23.1.2'] + when: '%llvm' + mpi: + buildable: False + require: + - "openmpi@5.0.11" + llvm: + externals: + - spec: llvm@23.1.2 +flang ~libomptarget + prefix: /gpfs/neptune/spack-stack/llvm-23.1.2 + modules: + - llvm/23.1.2 + extra_attributes: + compilers: + c: /gpfs/neptune/spack-stack/llvm-23.1.2/bin/clang + cxx: /gpfs/neptune/spack-stack/llvm-23.1.2/bin/clang++ + fortran: /gpfs/neptune/spack-stack/llvm-23.1.2/bin/flang-new + extra_rpaths: + - /gpfs/neptune/spack-stack/llvm-23.1.2/lib + - /gpfs/neptune/spack-stack/llvm-23.1.2/lib/x86_64-unknown-linux-gnu + openmpi: + externals: + - spec: openmpi@5.0.11 ~internal-hwloc +two_level_namespace + prefix: /gpfs/neptune/spack-stack/openmpi-5.0.11/llvm-23.1.2 + modules: + - llvm/23.1.2 + - openmpi/5.0.11 From 4201925250a38d55f14d702ef5f423d3a9225eb2 Mon Sep 17 00:00:00 2001 From: Natalie Perlin <68030316+natalie-perlin@users.noreply.github.com> Date: Wed, 28 Oct 2026 19:45:07 -0400 Subject: [PATCH 06/14] [release/2.2] Few fixes for NOAA Tier1 machines (#2191) * Fixes for oneapi-compilers +fix_rt_linkage on Tier1 machines * Fix ecflow for Ursa * Fix intel/2025.3.0 compiler names for modules: noaa-aws and noaa-azure --- .../gaea-c6/packages_oneapi-2025.2.1.yaml | 2 +- .../gaea-c7/packages_oneapi-2025.3.1.yaml | 2 +- .../gaea-c7/packages_oneapi-2026.0.0.yaml | 2 +- .../hercules/packages_oneapi-2025.3.1.yaml | 2 +- .../noaa-aws/packages_oneapi-2025.3.0.yaml | 21 +++++++------------ .../noaa-azure/packages_oneapi-2025.3.0.yaml | 21 +++++++------------ .../tier1/orion/packages_oneapi-2025.3.1.yaml | 2 +- configs/sites/tier1/ursa/packages.yaml | 10 +++------ .../ursa/packages_oneapi-2025.2.1-hpcx.yaml | 2 +- .../tier1/ursa/packages_oneapi-2025.2.1.yaml | 2 +- .../ursa/packages_oneapi-2025.3.1-hpcx.yaml | 2 +- .../tier1/ursa/packages_oneapi-2025.3.1.yaml | 2 +- 12 files changed, 28 insertions(+), 42 deletions(-) diff --git a/configs/sites/tier1/gaea-c6/packages_oneapi-2025.2.1.yaml b/configs/sites/tier1/gaea-c6/packages_oneapi-2025.2.1.yaml index 35b4b4a754..fb7220f860 100644 --- a/configs/sites/tier1/gaea-c6/packages_oneapi-2025.2.1.yaml +++ b/configs/sites/tier1/gaea-c6/packages_oneapi-2025.2.1.yaml @@ -22,7 +22,7 @@ packages: intel-oneapi-compilers: buildable: False externals: - - spec: intel-oneapi-compilers@2025.2.1 + - spec: intel-oneapi-compilers@2025.2.1 +fix_rt_linkage prefix: /opt/intel/oneapi modules: - PrgEnv-intel/8.6.0 diff --git a/configs/sites/tier1/gaea-c7/packages_oneapi-2025.3.1.yaml b/configs/sites/tier1/gaea-c7/packages_oneapi-2025.3.1.yaml index db2f4fab49..bb16e66229 100644 --- a/configs/sites/tier1/gaea-c7/packages_oneapi-2025.3.1.yaml +++ b/configs/sites/tier1/gaea-c7/packages_oneapi-2025.3.1.yaml @@ -20,7 +20,7 @@ packages: intel-oneapi-compilers: buildable: False externals: - - spec: intel-oneapi-compilers@2025.3.1 + - spec: intel-oneapi-compilers@2025.3.1 +fix_rt_linkage prefix: /sw/gaea-c7/oneapi/2025.3.1 modules: - oneapi/2025.3.1 diff --git a/configs/sites/tier1/gaea-c7/packages_oneapi-2026.0.0.yaml b/configs/sites/tier1/gaea-c7/packages_oneapi-2026.0.0.yaml index 69ce8d4266..edc8f479a7 100644 --- a/configs/sites/tier1/gaea-c7/packages_oneapi-2026.0.0.yaml +++ b/configs/sites/tier1/gaea-c7/packages_oneapi-2026.0.0.yaml @@ -20,7 +20,7 @@ packages: intel-oneapi-compilers: buildable: False externals: - - spec: intel-oneapi-compilers@2026.0.0 + - spec: intel-oneapi-compilers@2026.0.0 +fix_rt_linkage prefix: /sw/gaea-c7/oneapi/2026.0.0 modules: - oneapi/2026.0.0 diff --git a/configs/sites/tier1/hercules/packages_oneapi-2025.3.1.yaml b/configs/sites/tier1/hercules/packages_oneapi-2025.3.1.yaml index e14516e786..81c5b8a361 100644 --- a/configs/sites/tier1/hercules/packages_oneapi-2025.3.1.yaml +++ b/configs/sites/tier1/hercules/packages_oneapi-2025.3.1.yaml @@ -14,7 +14,7 @@ packages: intel-oneapi-compilers: buildable: false externals: - - spec: intel-oneapi-compilers@2025.3.1 + - spec: intel-oneapi-compilers@2025.3.1 +fix_rt_linkage prefix: /apps/contrib/spack-stack/installs/oneapi-2025.3.1 modules: - intel-oneapi-compilers/2025.3.1 diff --git a/configs/sites/tier1/noaa-aws/packages_oneapi-2025.3.0.yaml b/configs/sites/tier1/noaa-aws/packages_oneapi-2025.3.0.yaml index f1d9155d28..2099836175 100644 --- a/configs/sites/tier1/noaa-aws/packages_oneapi-2025.3.0.yaml +++ b/configs/sites/tier1/noaa-aws/packages_oneapi-2025.3.0.yaml @@ -12,44 +12,39 @@ packages: intel-oneapi-compilers: buildable: false externals: - - spec: intel-oneapi-compilers@2025.3.0 + - spec: intel-oneapi-compilers@2025.3.0 +fix_rt_linkage prefix: /pw/apps/intel/2025.3.0 modules: - - tbb/2022.3 - - compiler-rt/2025.3.0 - - umf/1.0.2 - - compiler/2025.3.0 + - intel/2025.3.0/tbb/2022.3 + - intel/2025.3.0/compiler-rt/2025.3.0 + - intel/2025.3.0/umf/1.0.2 + - intel/2025.3.0/compiler/2025.3.0 extra_attributes: compilers: c: /pw/apps/intel/2025.3.0/compiler/2025.3/bin/icx cxx: /pw/apps/intel/2025.3.0/compiler/2025.3/bin/icpx fortran: /pw/apps/intel/2025.3.0/compiler/2025.3/bin/ifx - environment: - set: - MODULES_AUTO_HANDLING: '1' - prepend_path: - MODULEPATH: /pw/apps/modules/intel/2025.3.0 intel-oneapi-mpi: buildable: false externals: - spec: intel-oneapi-mpi@2021.17 prefix: /pw/apps/intel/2025.3.0 modules: - - mpi/2021.17 + - intel/2025.3.0/mpi/2021.17 intel-oneapi-mkl: buildable: false externals: - spec: intel-oneapi-mkl@2025.3 prefix: /pw/apps/intel/2025.3.0 modules: - - mkl/2025.3 + - intel/2025.3.0/mkl/2025.3 intel-oneapi-tbb: buildable: false externals: - spec: intel-oneapi-tbb@2022.3 prefix: /pw/apps/intel/2025.3.0 modules: - - tbb/2022.3 + - intel/2025.3.0/tbb/2022.3 gcc: buildable: false externals: diff --git a/configs/sites/tier1/noaa-azure/packages_oneapi-2025.3.0.yaml b/configs/sites/tier1/noaa-azure/packages_oneapi-2025.3.0.yaml index f1d9155d28..2099836175 100644 --- a/configs/sites/tier1/noaa-azure/packages_oneapi-2025.3.0.yaml +++ b/configs/sites/tier1/noaa-azure/packages_oneapi-2025.3.0.yaml @@ -12,44 +12,39 @@ packages: intel-oneapi-compilers: buildable: false externals: - - spec: intel-oneapi-compilers@2025.3.0 + - spec: intel-oneapi-compilers@2025.3.0 +fix_rt_linkage prefix: /pw/apps/intel/2025.3.0 modules: - - tbb/2022.3 - - compiler-rt/2025.3.0 - - umf/1.0.2 - - compiler/2025.3.0 + - intel/2025.3.0/tbb/2022.3 + - intel/2025.3.0/compiler-rt/2025.3.0 + - intel/2025.3.0/umf/1.0.2 + - intel/2025.3.0/compiler/2025.3.0 extra_attributes: compilers: c: /pw/apps/intel/2025.3.0/compiler/2025.3/bin/icx cxx: /pw/apps/intel/2025.3.0/compiler/2025.3/bin/icpx fortran: /pw/apps/intel/2025.3.0/compiler/2025.3/bin/ifx - environment: - set: - MODULES_AUTO_HANDLING: '1' - prepend_path: - MODULEPATH: /pw/apps/modules/intel/2025.3.0 intel-oneapi-mpi: buildable: false externals: - spec: intel-oneapi-mpi@2021.17 prefix: /pw/apps/intel/2025.3.0 modules: - - mpi/2021.17 + - intel/2025.3.0/mpi/2021.17 intel-oneapi-mkl: buildable: false externals: - spec: intel-oneapi-mkl@2025.3 prefix: /pw/apps/intel/2025.3.0 modules: - - mkl/2025.3 + - intel/2025.3.0/mkl/2025.3 intel-oneapi-tbb: buildable: false externals: - spec: intel-oneapi-tbb@2022.3 prefix: /pw/apps/intel/2025.3.0 modules: - - tbb/2022.3 + - intel/2025.3.0/tbb/2022.3 gcc: buildable: false externals: diff --git a/configs/sites/tier1/orion/packages_oneapi-2025.3.1.yaml b/configs/sites/tier1/orion/packages_oneapi-2025.3.1.yaml index 6e97117c49..d950e47751 100644 --- a/configs/sites/tier1/orion/packages_oneapi-2025.3.1.yaml +++ b/configs/sites/tier1/orion/packages_oneapi-2025.3.1.yaml @@ -17,7 +17,7 @@ packages: - spec: intel-oneapi-compilers@2025.3.1 prefix: /apps/contrib/spack-stack/installs/oneapi-2025.3.1 modules: - - intel-oneapi-compilers/2025.3.1 + - intel-oneapi-compilers/2025.3.1 +fix_rt_linkage extra_attributes: compilers: c: /apps/contrib/spack-stack/installs/oneapi-2025.3.1/compiler/2025.3/bin/icx diff --git a/configs/sites/tier1/ursa/packages.yaml b/configs/sites/tier1/ursa/packages.yaml index fd82657501..18d837f567 100644 --- a/configs/sites/tier1/ursa/packages.yaml +++ b/configs/sites/tier1/ursa/packages.yaml @@ -26,6 +26,9 @@ packages: arborx: require: - '%cxx=gcc' + ecflow: + require: + - '%c,cxx=gcc' # All other packages listed alphabetically autoconf: @@ -60,13 +63,6 @@ packages: externals: - spec: doxygen@1.11.0+graphviz~mscgen prefix: /apps/spack-2024-12/linux-rocky9-x86_64/gcc-11.4.1/doxygen-1.11.0-dbhxblaakcrhvmlrbgixvapnagjb2vkx - ecflow: - buildable: False - externals: - - spec: ecflow@5.11.4 - prefix: /apps/spack-2024-12/linux-rocky9-x86_64/gcc-11.4.1/ecflow-5.11.4-a5ggcmayx6yesbqgbacifgvkstamchoy - modules: - - ecflow/5.11.4 file: externals: - spec: file@5.39 diff --git a/configs/sites/tier1/ursa/packages_oneapi-2025.2.1-hpcx.yaml b/configs/sites/tier1/ursa/packages_oneapi-2025.2.1-hpcx.yaml index afa6be6d36..3a91658a4c 100644 --- a/configs/sites/tier1/ursa/packages_oneapi-2025.2.1-hpcx.yaml +++ b/configs/sites/tier1/ursa/packages_oneapi-2025.2.1-hpcx.yaml @@ -13,7 +13,7 @@ packages: intel-oneapi-compilers: buildable: False externals: - - spec: intel-oneapi-compilers@2025.2.1 + - spec: intel-oneapi-compilers@2025.2.1 +fix_rt_linkage prefix: /apps/spack-2024-12/linux-rocky9-x86_64/gcc-11.4.1/intel-oneapi-compilers-2025.2.1-5nijquf65qj5mmbn3k2gxkjkrjkpjtw7 modules: - intel-oneapi-compilers/2025.2.1 diff --git a/configs/sites/tier1/ursa/packages_oneapi-2025.2.1.yaml b/configs/sites/tier1/ursa/packages_oneapi-2025.2.1.yaml index 9f887da79c..688e136665 100644 --- a/configs/sites/tier1/ursa/packages_oneapi-2025.2.1.yaml +++ b/configs/sites/tier1/ursa/packages_oneapi-2025.2.1.yaml @@ -6,7 +6,7 @@ packages: intel-oneapi-compilers: buildable: False externals: - - spec: intel-oneapi-compilers@2025.2.1 + - spec: intel-oneapi-compilers@2025.2.1 +fix_rt_linkage prefix: /apps/spack-2024-12/linux-rocky9-x86_64/gcc-11.4.1/intel-oneapi-compilers-2025.2.1-5nijquf65qj5mmbn3k2gxkjkrjkpjtw7 modules: - intel-oneapi-compilers/2025.2.1 diff --git a/configs/sites/tier1/ursa/packages_oneapi-2025.3.1-hpcx.yaml b/configs/sites/tier1/ursa/packages_oneapi-2025.3.1-hpcx.yaml index 8c7c9f5a1b..0a27a70caa 100644 --- a/configs/sites/tier1/ursa/packages_oneapi-2025.3.1-hpcx.yaml +++ b/configs/sites/tier1/ursa/packages_oneapi-2025.3.1-hpcx.yaml @@ -20,7 +20,7 @@ packages: intel-oneapi-compilers: buildable: False externals: - - spec: intel-oneapi-compilers@2025.3.1 + - spec: intel-oneapi-compilers@2025.3.1 +fix_rt_linkage prefix: /apps/spack-2024-12/linux-rocky9-x86_64/gcc-11.4.1/intel-oneapi-compilers-2025.3.1-yqgrg5v3jonwexcjwd3xu3rzvwcp4ejk modules: - intel-oneapi-compilers/2025.3.1 diff --git a/configs/sites/tier1/ursa/packages_oneapi-2025.3.1.yaml b/configs/sites/tier1/ursa/packages_oneapi-2025.3.1.yaml index 3da2e8f3f7..b3e9f50759 100644 --- a/configs/sites/tier1/ursa/packages_oneapi-2025.3.1.yaml +++ b/configs/sites/tier1/ursa/packages_oneapi-2025.3.1.yaml @@ -12,7 +12,7 @@ packages: intel-oneapi-compilers: buildable: False externals: - - spec: intel-oneapi-compilers@2025.3.1 + - spec: intel-oneapi-compilers@2025.3.1 +fix_rt_linkage prefix: /apps/spack-2024-12/linux-rocky9-x86_64/gcc-11.4.1/intel-oneapi-compilers-2025.3.1-yqgrg5v3jonwexcjwd3xu3rzvwcp4ejk modules: - intel-oneapi-compilers/2025.3.1 From 3b07d7b4dda595cad071149b7484fa2a53600f37 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Thu, 29 Oct 2026 09:36:44 -0600 Subject: [PATCH 07/14] Update spack-packages: fms: add -fno-tree-loop-if-convert for gcc >= 12 (#2192) --- repos/builtin | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/repos/builtin b/repos/builtin index 8cb08dc006..aef82ca35e 160000 --- a/repos/builtin +++ b/repos/builtin @@ -1 +1 @@ -Subproject commit 8cb08dc0064a79804662d70484026cfa472aea16 +Subproject commit aef82ca35e9abecfcc814246d0467349af0b0307 From d0a22fb2259b1fb88fe8043ef98e289a06f10a87 Mon Sep 17 00:00:00 2001 From: Matt Thompson Date: Thu, 29 Oct 2026 11:37:03 -0400 Subject: [PATCH 08/14] util/gmao/batch_install.sh: restore private TMPDIR after module preload (#2193) --- util/gmao/batch_install.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/util/gmao/batch_install.sh b/util/gmao/batch_install.sh index 492baa5ed6..9b5db4fafa 100755 --- a/util/gmao/batch_install.sh +++ b/util/gmao/batch_install.sh @@ -1435,6 +1435,13 @@ elif [[ "${host}" == "discover-gmao" ]] && \ module load comp/gcc/12.3.0 comp/nag/7.2-7238 mpi/openmpi/4.1.6/nag_7.2.7238-gcc_12.3.0 fi +# Some Discover MPI modules (e.g. mpi/openmpi/5.0.10/gcc-15.2.0) run +# setenv("TMPDIR", "/tmp"), so restore our private TMPDIR after module loads. +if [[ -n "\${build_tmpdir:-}" ]]; then + export TMPDIR="\${build_tmpdir}" + echo "INFO: TMPDIR after module preload: \${TMPDIR}" +fi + $(declare -p test_packages) # If no tests are required, install everything From a629830b6600f6f2075352ea4dc2e3de19748fb9 Mon Sep 17 00:00:00 2001 From: Evan Parker Date: Thu, 29 Oct 2026 13:27:23 -0600 Subject: [PATCH 09/14] Promote aws-ubuntu site to tier1 and update for Ubuntu 26.04 (#2194) --- configs/sites/tier1/aws-ubuntu/README.md | 421 ++++++++++++++++ configs/sites/tier1/aws-ubuntu/config.yaml | 7 + .../aws-ubuntu}/modules.yaml | 0 configs/sites/tier1/aws-ubuntu/packages.yaml | 121 +++++ .../sites/tier1/aws-ubuntu/packages_gcc.yaml | 13 + .../aws-ubuntu}/packages_oneapi.yaml | 31 +- configs/sites/tier2/aws-ubuntu2404/README.md | 463 ------------------ .../sites/tier2/aws-ubuntu2404/config.yaml | 9 - .../sites/tier2/aws-ubuntu2404/packages.yaml | 117 ----- .../tier2/aws-ubuntu2404/packages_gcc.yaml | 15 - .../tier2/ubuntu2404-nvhpc/packages.yaml | 2 +- 11 files changed, 586 insertions(+), 613 deletions(-) create mode 100644 configs/sites/tier1/aws-ubuntu/README.md create mode 100644 configs/sites/tier1/aws-ubuntu/config.yaml rename configs/sites/{tier2/aws-ubuntu2404 => tier1/aws-ubuntu}/modules.yaml (100%) create mode 100644 configs/sites/tier1/aws-ubuntu/packages.yaml create mode 100644 configs/sites/tier1/aws-ubuntu/packages_gcc.yaml rename configs/sites/{tier2/aws-ubuntu2404 => tier1/aws-ubuntu}/packages_oneapi.yaml (68%) delete mode 100644 configs/sites/tier2/aws-ubuntu2404/README.md delete mode 100644 configs/sites/tier2/aws-ubuntu2404/config.yaml delete mode 100644 configs/sites/tier2/aws-ubuntu2404/packages.yaml delete mode 100644 configs/sites/tier2/aws-ubuntu2404/packages_gcc.yaml diff --git a/configs/sites/tier1/aws-ubuntu/README.md b/configs/sites/tier1/aws-ubuntu/README.md new file mode 100644 index 0000000000..31562e81a3 --- /dev/null +++ b/configs/sites/tier1/aws-ubuntu/README.md @@ -0,0 +1,421 @@ +## spack-stack AMI (Ubuntu 26.04) + +This document covers building and using the spack-stack 2.2 AMI for Ubuntu 26.04 LTS. This image has two environments: gnu (gcc-14.3.0), and intel (intel-oneapi 2026.1.1). You can use either one of these environments for development purposes. + +### Using the Snapshot + +Here is an example `aws cli` command-line to run a pre-built snapshot with your designated key-pair. + +```bash +aws ec2 run-instances \ + --image-id "ami-0eec3117bdb2e6147" --instance-type "m6i.4xlarge" \ + --key-name YOUR-KEYPAIR \ + --block-device-mappings '{"DeviceName":"/dev/sda1","Ebs":{"Encrypted":false,"DeleteOnTermination":true,"Iops":3000,"SnapshotId":"snap-XXXXXXXXXXXXXXXXX","VolumeSize":200,"VolumeType":"gp3","Throughput":125}}' \ + --network-interfaces '{"SubnetId":"subnet-072fb62ff85b32a7a","AssociatePublicIpAddress":true,"DeviceIndex":0,"Groups":["sg-0091fa8e748fbe355"]}' \ + --tag-specifications '{"ResourceType":"instance","Tags":[{"Key":"Name","Value":"ubuntu2604-spack-stack-2.2-gcc-oneapi"}]}' \ + --metadata-options '{"HttpEndpoint":"enabled","HttpPutResponseHopLimit":2,"HttpTokens":"required"}' \ + --private-dns-name-options '{"HostnameType":"ip-name","EnableResourceNameDnsARecord":false,"EnableResourceNameDnsAAAARecord":false}' \ + --count 1 +``` + +## Using this Site Config + +This site tracks the current Ubuntu LTS. It started on 20.04, was +`tier2/aws-ubuntu2404` for spack-stack 2.0–2.1, and is 26.04 as of 2.2. When the +next LTS comes out, update this directory in place; don't create a new +versioned site. + +JCSDA publishes a fully configured and built installation of spack stack derived +from this config as an AWS Snapshot. The easiest way to use this +configuration of spack stack is to launch a VM using that AMI. The instructions +below are included for maintainance of the history of this site config and +their possible relevance to debugging issues should they arise. + +### Base Instance + +This AMI was built on an instance with these properties: + +* OS: Ubuntu 26.04.1 LTS (Resolute Raccoon), kernel 7.0.0-1014-aws +* AMI Name / ID: TODO +* 8 vCPU, 30 GiB RAM +* 200GB of gp3 storage as / + +### Differences from the Ubuntu 24.04 site + +* **Lmod comes from apt.** Ubuntu 26.04 packages Lmod 8.7.60 (on Lua 5.3), + which meets spack's requirement, so the manual Lua 5.1 + Lmod 8.7 source + build is gone. The package installs `/etc/profile.d/lmod.sh`, so `module` + is available in every login shell. +* **The default system gcc is 15.2.0**, which is newer than spack-stack + supports (up to 14.x). We install gcc/g++/gfortran 14.3.0 next to it and + point spack at the `-14` binaries by full path. `/usr/bin/gcc` stays + `gcc-15`. We don't use `update-alternatives` anymore: `/usr/bin/gcc` and + `/usr/bin/g++` are symlinks owned by the `gcc`/`g++` packages, and an + `apt upgrade` would reset them. The stack-gcc meta-module still exports + `CC/CXX/FC` as the gcc-14 paths, so builds that load it use gcc-14. No + unversioned `gfortran` is installed at all. +* **llvm-14 is not available** on 26.04 and nothing in the stack needs it, + so it has been dropped, along with the `llvm-config` alternative. +* **coreutils is uutils (Rust)**, Ubuntu's default since 25.10 + (`ls (uutils coreutils) 0.10.0`). We keep the system default: switching to + `coreutils-from-gnu` would remove the `build-essential` metapackage, which + depends on `coreutils-from-uutils`. Spack doesn't detect uutils as a + coreutils external. If a build fails in an `install`/`cp`/`sort`/`date` + step, suspect uutils first. +* **`/tmp` is a RAM-backed tmpfs** (half of RAM, with per-user quotas). The + 24.04 site's `config.yaml` staged builds and cached sources in `/tmp`, + which fills it with `Disk quota exceeded`. The site now uses the common + default `${SPACK_STACK_DIR}/cache` on the root disk. +* The system ships Python 3.14 and CMake 4.2. Spack runs fine on Python 3.14, + and both are still excluded from `spack external find`. + +## Pre-requisites (For All Compilers) + +### Installing Packages + +```bash +# Update system software. +sudo apt update +sudo apt upgrade -y + +# Build tools. build-essential pulls in the default gcc-15. We add gcc-14 +# alongside it, since spack-stack supports gcc up to 14.x. +sudo apt install -y build-essential gcc-14 g++-14 gfortran-14 cpp-14 make cmake automake autoconf apt-utils + +# Other requirements. +sudo apt install -y libgomp1 git git-lfs autopoint mysql-server libmysqlclient-dev qtbase5-dev qt5-qmake libqt5svg5-dev qt5dxcb-plugin wget curl file tcl-dev gnupg2 iproute2 locales unzip less bzip2 gettext libtree pkg-config libcurl4-openssl-dev libtool flex + +# Lmod (8.7.60 on 26.04; installs /etc/profile.d/lmod.sh). +sudo apt install -y lmod + +# Editors +sudo apt install -y vim nano + +# Python develop. +sudo apt install -y python3 python3-pip python3-setuptools + +# Configure git credential caching and git lfs for the ubuntu user and root. +git config --global credential.helper 'cache --timeout=3600' +git lfs install +sudo -H bash -c "git config --global credential.helper 'cache --timeout=3600' && git lfs install" + +# Log out and back in (or `source /etc/profile.d/lmod.sh`), then check: +module --version # Modules based on Lua: Version 8.7.60 +gcc-14 --version # gcc-14 (Ubuntu 14.3.0-14ubuntu1) 14.3.0 +``` + +### Clone `spack-stack` + +```bash +cd /opt +sudo git clone -b release/2.2 --depth 1 --recursive https://github.com/jcsda/spack-stack.git +sudo chown -R ubuntu:ubuntu /opt/spack-stack +``` + +Everything from here on runs as the `ubuntu` user. Use `sudo` only where it's +shown. These AMIs are single-user dev and test machines and everyone logs in +as `ubuntu`, so building as that user keeps the environments user-owned and +avoids root-owned files in `envs/`. + +## Install Spack-Stack Steps by Compiler + +
+GCC Installation + +**Shortcut:** the committed site config reproduces this environment exactly +(verified: identical concretized hashes). On a prepared image you can skip +the external/compiler derivation and the `spack config add` steps below: + +```bash +umask 0022 +cd /opt/spack-stack && source setup.sh +spack stack create env --site aws-ubuntu --template unified-dev --name unified-gcc --compiler gcc +cd envs/unified-gcc && spack env activate -p . +spack concretize --fresh 2>&1 | tee log.concretize +``` + +The full steps below show how the site config was derived from +`linux.default`. Use them when moving to a new Ubuntu LTS. + +```bash +# Ubuntu gives regular users umask 0002; spack-stack expects 0022 or 0027. +umask 0022 +cd /opt/spack-stack +source setup.sh +# Swap default module type for default linux. +sed -i 's/tcl/lmod/g' configs/sites/tier2/linux.default/modules.yaml +spack stack create env --site linux.default --template unified-dev --name unified-gcc --compiler gcc +cd envs/unified-gcc +spack env activate -p . +unset SPACK_DISABLE_LOCAL_CONFIG +export SPACK_SYSTEM_CONFIG_PATH="$(pwd)/site" +spack external find --scope system \ + --exclude cmake \ + --exclude curl --exclude openssl \ + --exclude openssh --exclude python \ + --exclude zlib +# zlib: the system external (prefix /usr) breaks pmix's configure, which +# looks for libz in /usr/lib{,64} and not the multiarch +# /usr/lib/x86_64-linux-gnu ("ZLIB SUPPORT REQUESTED AND NOT FOUND"). +spack external find --scope system wget grep +spack compiler find --scope system +# ACTION: Edit the generated compiler config (site/compilers.yaml and/or the +# compiler entries in site/packages.yaml) with the following. +# 1) Keep a single gcc@14.3.0 spec and delete the system default gcc@15.2.0 +# and any empty or duplicate gcc specs. Redundant/empty specs cause +# cryptic build errors later. The entry should read: +# gcc: +# buildable: false +# externals: +# - spec: gcc@14.3.0 languages:='c,c++,fortran' +# prefix: /usr +# extra_attributes: +# compilers: +# c: /usr/bin/gcc-14 +# cxx: /usr/bin/g++-14 +# fortran: /usr/bin/gfortran-14 +# 2) Delete or comment clang/llvm refs. +# 3) Add `buildable: false` to the tar and wget entries. +# coreutils (uutils on stock 26.04) and binutils (2.46, no gold) are not +# detected; let spack build them if needed. qt5 isn't detected either (qmake +# is a qtchooser shim), so declare the apt qtbase5-dev install by hand. +# common/packages.yaml requires qt@5. +cat << 'EOF' >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml + qt: + buildable: false + externals: + - spec: qt@5.15.18 + prefix: /usr +EOF +export SPACK_DISABLE_LOCAL_CONFIG=true +unset SPACK_SYSTEM_CONFIG_PATH + +# Continue configuration. Don't use "packages:all:prefer:['%gcc']": gcc-14 is +# the only compiler, and that setting acts as a dependency constraint that +# pushes spack to rebuild system externals (gmake, perl, m4, ...) instead of +# using them. Pin MPI the way the Ubuntu CI does. +spack config add "packages:mpi:require:['openmpi@5.0.10']" +spack config add "packages:fontconfig:variants:+pic" +spack config add "packages:pixman:variants:+pic" +spack config add "packages:cairo:variants:+pic" +# met: with gcc 14, +lidar2nc and +modis can't be honored (they need hdf-eos2, +# and common pins hdf-eos2@2.20v1.00, which conflicts with %gcc@14:), so met +# builds without them. +spack config add "packages:met:variants:+python +grib2 +graphics +lidar2nc +modis" +spack config add "packages:ewok-env:require:[+ecflow]" +# Pin py-netcdf4 +spack config add "packages:py-netcdf4:require:[\"@1.7.2\"]" +# Avoid duplicates: jedi-ufs-env needs jedi-python-env +bufrquery. Left +# unconstrained, jedi-tools-env falls back to py-myst-parser@1 and py-sphinx@6, +# which cap py-attrs@:21 and py-docutils@:0.19 and split py-attrs, py-eccodes, +# py-rasterio, jedi-python-env and awscli-v2. myst-parser 5 + sphinx 9 share +# docutils 0.22 with the rest of the stack. +spack config add "packages:jedi-python-env:require:[+bufrquery]" +spack config add "packages:py-myst-parser:require:['@5:']" +spack config add "packages:py-sphinx:require:['@9:']" + +# Concretize and install +spack concretize --fresh 2>&1 | tee log.concretize +# py-versioneer is a known, unavoidable duplicate (see common/packages.yaml). +${SPACK_STACK_DIR}/util/show_duplicate_packages.py -i fms -i crtm -i crtm-fix -i esmf -i mapl -i py-cython -i py-versioneer +spack stack check-preferred-compiler +spack install --fail-fast -j 8 2>&1 | tee log.install + +# Install modules +spack module lmod refresh -y --delete-tree +spack stack setup-meta-modules + +# Add a number of default module locations to the lmod startup script. +echo 'module use /opt/spack-stack/envs/unified-gcc/modules/Core' | sudo tee -a /etc/profile.d/z01_lmod.sh +``` + +
+ +
+Intel OneAPI Installation + +#### Install Intel OneAPI Compiler + +Intel's `all` apt repo works unchanged on 26.04. 2026.1.1 is the latest +2026.1.x and is the release-tested version. Don't pick up 2026.2, which is +also in the repo. + +```bash +wget -O- https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB | gpg --dearmor | sudo tee /usr/share/keyrings/oneapi-archive-keyring.gpg > /dev/null +echo "deb [signed-by=/usr/share/keyrings/oneapi-archive-keyring.gpg] https://apt.repos.intel.com/oneapi all main" | sudo tee /etc/apt/sources.list.d/oneAPI.list +sudo apt update +sudo apt install -y intel-oneapi-compiler-dpcpp-cpp-2026.1 intel-oneapi-compiler-fortran-2026.1 intel-oneapi-mpi-devel-2021.18 intel-oneapi-tbb-devel-2023.1 intel-oneapi-mkl-devel-2026.1 +``` + +**GCC toolchain note:** icx/icpx use the newest installed GCC for libstdc++ +headers and libraries, and ifx uses it to link its runtime. On 26.04 that's +gcc-15, not the gcc-14 this stack uses (`icpx -v` prints +`Selected GCC installation: /usr/lib/gcc/x86_64-linux-gnu/15`). +`packages_oneapi.yaml` pins them back to gcc 14 with compiler `flags:` +(`--gcc-install-dir=.../14` for C/C++, `-gcc-name`/`-gxx-name` for ifx), the +same way acorn and wcoss2 pin theirs. + +#### Setup OneAPI Modules + +```bash +# Create all modulefiles. +sudo /opt/intel/oneapi/modulefiles-setup.sh --output-dir=/opt/intel/oneapi/modulefiles +module use /opt/intel/oneapi/modulefiles + +# Add the oneapi module files to lmod init (z01 sorts after the apt lmod.sh). +echo 'module use /opt/intel/oneapi/modulefiles' | sudo tee -a /etc/profile.d/z01_lmod.sh + +# Create combined module file. Note that the compiler modulefile has a "tcm" +# prerequisite; omitting it fails the load with "Cannot load module +# compiler/2026.1.1. At least one of these module(s) must be loaded: tcm". +sudo mkdir -p /opt/intel/oneapi/modulefiles/intel-oneapi-full-env/ +sudo tee /opt/intel/oneapi/modulefiles/intel-oneapi-full-env/2026.1.1 > /dev/null << 'EOF' +#%Module1.0 +## +## intel-oneapi-full-env/2026.1.1 +## Intel oneAPI full module environment + +proc ModulesHelp { } { + puts stderr "intel-oneapi-full-env defines the entire module set used for spack-stack intel builds" +} +module-whatis "intel-oneapi-full-env defines the entire module set used for spack-stack intel builds" +module load umf/1.1.0 +module load tcm/1.5 +module load tbb/2023.1 +module load compiler-rt/2026.1.1 +module load compiler/2026.1.1 +module load mkl/2026.1 +module load compiler-intel-llvm/2026.1.1 +EOF +``` + +#### Install Intel OneAPI Spack-Stack Environment + +The committed `packages_oneapi.yaml` is complete. Create the environment +straight from the site; don't re-run `spack external find` or +`spack compiler find` against it. Run everything in a **login shell with no +oneAPI modules loaded** (see "The module-load trap" below). Spack loads the +compiler's modules itself, even during concretization, so the oneAPI +modulefiles must be on `MODULEPATH` via `/etc/profile.d/z01_lmod.sh`. + +```bash +module purge +umask 0022 +cd /opt/spack-stack +source setup.sh + +spack stack create env --site aws-ubuntu --template unified-dev --name unified-oneapi --compiler oneapi +cd envs/unified-oneapi +spack env activate -p . + +spack concretize --fresh 2>&1 | tee log.concretize +${SPACK_STACK_DIR}/util/show_duplicate_packages.py -i fms -i crtm -i crtm-fix -i esmf -i mapl -i py-cython -i py-versioneer +spack stack check-preferred-compiler +spack install --fail-fast -j 8 2>&1 | tee log.install + +spack module lmod refresh -y --delete-tree +spack stack setup-meta-modules + +echo 'module use /opt/spack-stack/envs/unified-oneapi/modules/Core' | sudo tee -a /etc/profile.d/z01_lmod.sh +``` + +How `packages_oneapi.yaml` relates to the 24.04 version (check these when +moving to a new LTS or oneAPI release): + +* `intel-oneapi-compilers` external: the 2026.1.1 compilers, `+fix_rt_linkage` + (common requires it, PR #2150), the module chain + `umf -> tcm -> tbb -> compiler-rt -> compiler`, and the gcc-14 `flags:` + above. To regenerate it, `module load intel-oneapi-full-env/2026.1.1` and run + `spack compiler find` in a scratch env, then copy the entry and `module purge`. +* `gcc` external: gcc@14.3.0 using the `-14` binaries, as in `packages_gcc.yaml`. + common/packages_oneapi.yaml builds a handful of packages (cmake, rust, py-numpy, ...) with gcc. +* MPI is `intel-oneapi-mpi@2021.18` (`mpi: require:`, openmpi/mpich not + buildable). MKL and TBB are externals, and MKL deliberately has no `modules:`. +* Dropped `all: prefer: ['%oneapi']`. common/packages_oneapi.yaml already + prefers oneAPI per language (c/cxx/fortran), and the `all:` form makes spack + rebuild system externals. +* New for 26.04, to avoid duplicates: `py-dask@2024.12.1` (otherwise + py-setuptools-scm splits 8/9 and cascades) and `met +graphics +shared-intel` + (otherwise one met concretizes `~graphics`). met's `+lidar2nc +modis` stay + off, as in the gcc env (hdf-eos2 pin vs gcc 14). +* The shared `packages.yaml` (externals, jedi-python-env `+bufrquery`, + myst-parser/sphinx) applies to both compilers. + +#### The module-load trap + +Spack loads an external's `modules:` itself and treats a load that does not change +`LOADEDMODULES` as a failure. So any oneAPI module already loaded in your shell +makes `spack install` die with a misleading `ModuleLoadError` partway through. +Purge before installing, and let spack do the loading. + +For the same reason no two externals may list the same module. That is why +`intel-oneapi-mkl` above has no `modules:`: it would load before the compilers +entry and miss its `tbb`/`compiler-rt` prereqs, but listing those on mkl collides +with the compilers entry. It needs neither -- the recipe sets `MKLROOT` from +`prefix`. + +
+ +## Test Installation + +
+GCC + +```bash +# Example given for building jedi-bundle +module use /opt/spack-stack/envs/unified-gcc/modules/Core +module load stack-gcc/14.3.0 +module load stack-openmpi/5.0.10 +module load base-env +module load jedi-mpas-env +module load jedi-fv3-env +module load ewok-env +module load sp + +mkdir ~/jedi +cd ~/jedi +git clone https://github.com/JCSDA-internal/jedi-bundle.git +cd jedi-bundle +mkdir build && cd build +ecbuild ../ +make update +make -j10 +ctest +``` + +
+ +
+Intel OneAPI + +```bash +# Build jedi-bundle with oneapi +# stack-intel-oneapi-compilers exports CFLAGS/CXXFLAGS/FFLAGS that pin icx/icpx/ifx +# to gcc 14 (see packages_oneapi.yaml). Keep them: if you override them, the +# compilers fall back to the system gcc 15. +# ifx puts large temporary arrays on the stack. With the default 8 MB limit, the +# saber GSI and dirac_multiscale tests segfault. Raise the limit before running: +ulimit -s unlimited +module use /opt/spack-stack/envs/unified-oneapi/modules/Core +module load stack-intel-oneapi-compilers/2026.1.1 +module load stack-intel-oneapi-mpi/2021.18 +module load base-env +module load jedi-mpas-env +module load jedi-fv3-env +module load ewok-env +module load sp + +mkdir ~/jedi +cd ~/jedi +git clone https://github.com/JCSDA-internal/jedi-bundle.git +cd jedi-bundle +mkdir build && cd build +ecbuild ../ +make update +make -j10 2>&1 | tee log.make +ctest +``` + +
+ +The installation and configuration is now complete for the instance. diff --git a/configs/sites/tier1/aws-ubuntu/config.yaml b/configs/sites/tier1/aws-ubuntu/config.yaml new file mode 100644 index 0000000000..b372c83669 --- /dev/null +++ b/configs/sites/tier1/aws-ubuntu/config.yaml @@ -0,0 +1,7 @@ +config: + build_jobs: 8 + + # Build stage and caches use the common defaults under + # ${SPACK_STACK_DIR}/cache on the root disk. Don't point them at /tmp: on + # Ubuntu 26.04 /tmp is a RAM-backed tmpfs (half of RAM, with per-user quotas), + # and a unified-env build fills it ("Disk quota exceeded"). diff --git a/configs/sites/tier2/aws-ubuntu2404/modules.yaml b/configs/sites/tier1/aws-ubuntu/modules.yaml similarity index 100% rename from configs/sites/tier2/aws-ubuntu2404/modules.yaml rename to configs/sites/tier1/aws-ubuntu/modules.yaml diff --git a/configs/sites/tier1/aws-ubuntu/packages.yaml b/configs/sites/tier1/aws-ubuntu/packages.yaml new file mode 100644 index 0000000000..22ec66785f --- /dev/null +++ b/configs/sites/tier1/aws-ubuntu/packages.yaml @@ -0,0 +1,121 @@ +packages: + # Package constraints. + ewok-env: + require: + - +ecflow + # Soft preference, as on other tier1 sites. With gcc 14, +lidar2nc and +modis + # are dropped: they need hdf-eos2, and common pins hdf-eos2@2.20v1.00, which + # conflicts with %gcc@14: (3.0+ is required). + met: + variants: +python +grib2 +graphics +lidar2nc +modis + py-netcdf4: + require: + - "@1.7.2" + jedi-base-env: + require: [+fftw, +hdf4] + # jedi-ufs-env needs +bufrquery; unconstrained, the root splits in two. + jedi-python-env: + require: [+bufrquery] + # Unconstrained, jedi-tools-env falls back to py-myst-parser@1/py-sphinx@6, + # which cap py-attrs@:21 and py-docutils@:0.19 and split py-attrs, py-eccodes, + # py-rasterio, jedi-python-env and awscli-v2. common/packages_oneapi.yaml + # already requires py-sphinx@9: for the same reason. + py-myst-parser: + require: ['@5:'] + py-sphinx: + require: ['@9:'] + fontconfig: + variants: +pic + pixman: + variants: +pic + cairo: + variants: +pic + + # System packages (Ubuntu 26.04). Derived with `spack external find` + # excluding cmake, curl, openssl, openssh, python and zlib (the multiarch + # libdir breaks pmix's zlib detection). coreutils (uutils) and binutils + # (2.46, no gold) aren't detected and are left for spack to build if needed. + autoconf: + externals: + - spec: autoconf@2.72 + prefix: /usr + automake: + externals: + - spec: automake@1.18.1 + prefix: /usr + diffutils: + externals: + - spec: diffutils@3.12 + prefix: /usr + findutils: + externals: + - spec: findutils@4.10.0 + prefix: /usr + flex: + externals: + - spec: flex@2.6.4+lex + prefix: /usr + gawk: + externals: + - spec: gawk@5.3.2 + prefix: /usr + gettext: + externals: + - spec: gettext@0.23.2 + prefix: /usr + git: + externals: + - spec: git@2.53.0~tcltk + prefix: /usr + git-lfs: + externals: + - spec: git-lfs@3.7.1 + prefix: /usr + gmake: + externals: + - spec: gmake@4.4.1 + prefix: /usr + grep: + externals: + - spec: grep@3.12 + prefix: /usr + groff: + externals: + - spec: groff@1.23.0 + prefix: /usr + libtool: + externals: + - spec: libtool@2.5.4 + prefix: /usr + m4: + externals: + - spec: m4@1.4.21 + prefix: /usr + perl: + externals: + - spec: perl@5.40.1~cpanm+opcode+open+shared+threads + prefix: /usr + pkgconf: + externals: + - spec: pkgconf@2.5.1 + prefix: /usr + sed: + externals: + - spec: sed@4.9 + prefix: /usr + tar: + buildable: false + externals: + - spec: tar@1.35 + prefix: /usr + wget: + buildable: false + externals: + - spec: wget@1.25.0 + prefix: /usr + # Not auto-detected (qmake is a qtchooser shim); apt qtbase5-dev. common requires qt@5. + qt: + buildable: false + externals: + - spec: qt@5.15.18 + prefix: /usr diff --git a/configs/sites/tier1/aws-ubuntu/packages_gcc.yaml b/configs/sites/tier1/aws-ubuntu/packages_gcc.yaml new file mode 100644 index 0000000000..d1638e17a4 --- /dev/null +++ b/configs/sites/tier1/aws-ubuntu/packages_gcc.yaml @@ -0,0 +1,13 @@ +packages: + gcc: + buildable: false + externals: + - spec: gcc@14.3.0 languages:='c,c++,fortran' + prefix: /usr + extra_attributes: + compilers: + c: /usr/bin/gcc-14 + cxx: /usr/bin/g++-14 + fortran: /usr/bin/gfortran-14 + mpi: + require: ['openmpi@5.0.10'] diff --git a/configs/sites/tier2/aws-ubuntu2404/packages_oneapi.yaml b/configs/sites/tier1/aws-ubuntu/packages_oneapi.yaml similarity index 68% rename from configs/sites/tier2/aws-ubuntu2404/packages_oneapi.yaml rename to configs/sites/tier1/aws-ubuntu/packages_oneapi.yaml index d56b47ec40..6c2018614f 100644 --- a/configs/sites/tier2/aws-ubuntu2404/packages_oneapi.yaml +++ b/configs/sites/tier1/aws-ubuntu/packages_oneapi.yaml @@ -1,9 +1,10 @@ packages: # Compilers. + # Language preferences for oneAPI come from common/packages_oneapi.yaml. Don't + # add "all: prefer: ['%oneapi']": it acts as a dependency constraint and makes + # spack rebuild system externals. all: - prefer: - - '%oneapi' conflict: - '%c=oneapi %fortran=gcc' - '%c,cxx=oneapi %fortran=gcc' @@ -13,8 +14,6 @@ packages: - '%fortran=oneapi %c,cxx=gcc' - '%fortran=gcc %c,cxx=oneapi' - '%fortran=gcc %c=oneapi' - providers: - mpi:: [intel-oneapi-mpi@2021.18] intel-oneapi-compilers: buildable: false @@ -33,6 +32,13 @@ packages: fortran: /opt/intel/oneapi/compiler/2026.1/bin/ifx c: /opt/intel/oneapi/compiler/2026.1/bin/icx cxx: /opt/intel/oneapi/compiler/2026.1/bin/icpx + # Ubuntu 26.04's default gcc is 15. Left alone, icx/icpx pick the newest + # installed GCC for libstdc++ headers and libs, and ifx links its + # runtime. Pin them to gcc 14 so they match the gcc env. + flags: + cflags: --gcc-install-dir=/usr/lib/gcc/x86_64-linux-gnu/14 + cxxflags: --gcc-install-dir=/usr/lib/gcc/x86_64-linux-gnu/14 + fflags: -gcc-name=/usr/bin/gcc-14 -gxx-name=/usr/bin/g++-14 openmpi: buildable: false mpich: @@ -43,13 +49,13 @@ packages: - intel-oneapi-mpi@2021.18 gcc: externals: - - spec: gcc@13.3.0 languages:='c,c++,fortran' + - spec: gcc@14.3.0 languages:='c,c++,fortran' prefix: /usr extra_attributes: compilers: - c: /usr/bin/gcc - cxx: /usr/bin/g++ - fortran: /usr/bin/gfortran + c: /usr/bin/gcc-14 + cxx: /usr/bin/g++-14 + fortran: /usr/bin/gfortran-14 buildable: false intel-oneapi-mkl: buildable: false @@ -83,6 +89,15 @@ packages: - tbb/2023.1 # Package constraints. + # Avoid duplicates. Unpinned, gmao-swell-env and py-pyresample pull two + # py-dask versions, which splits py-setuptools-scm (8 vs 9) and cascades into + # py-netcdf4, py-xarray, py-zarr and jedi-python-env. The gcc env settles on + # 2024.12.1 without a pin. + py-dask: + require: ['@2024.12.1'] + # Unconstrained, one met concretizes ~graphics. Keep common's +shared-intel. + met: + require: ['+graphics +shared-intel'] jedi-base-env: require: [+fftw, +hdf4] jedi-python-env: diff --git a/configs/sites/tier2/aws-ubuntu2404/README.md b/configs/sites/tier2/aws-ubuntu2404/README.md deleted file mode 100644 index c5527c8bb0..0000000000 --- a/configs/sites/tier2/aws-ubuntu2404/README.md +++ /dev/null @@ -1,463 +0,0 @@ -## spack-stack AMI (Ubuntu 24.04) - -This document is to go over the the running and usage of this specific AMI for Ubuntu 24.04 LTS. This image has two environments: gnu (gcc-13.3.0), and intel (intel-oneapi 2026.1.1). You can use either one of these environments for development purposes. - -### Using the Snapshot - -Here is an example `aws cli` command-line to run a pre-built snapshot with your designated key-pair. - -```bash -aws ec2 run-instances \ - --image-id "ami-0ea3c35c5c3284d82" --instance-type "m6i.4xlarge" \ - --key-name YOUR-KEYPAIR \ - --block-device-mappings '{"DeviceName":"/dev/sda1","Ebs":{"Encrypted":false,"DeleteOnTermination":true,"Iops":3000,"SnapshotId":"snap-05fb00e35af5550e7","VolumeSize":150,"VolumeType":"gp3","Throughput":125}}' \ - --network-interfaces '{"SubnetId":"subnet-072fb62ff85b32a7a","AssociatePublicIpAddress":true,"DeviceIndex":0,"Groups":["sg-0091fa8e748fbe355"]}' \ - --tag-specifications '{"ResourceType":"instance","Tags":[{"Key":"Name","Value":"ubuntu2404-spack-stack-1.9-gcc-oneapi"}]}' \ - --metadata-options '{"HttpEndpoint":"enabled","HttpPutResponseHopLimit":2,"HttpTokens":"required"}' \ - --private-dns-name-options '{"HostnameType":"ip-name","EnableResourceNameDnsARecord":false,"EnableResourceNameDnsAAAARecord":false}' \ - --count 1 -``` - -## Using this Site Config - -JCSDA publishes a fully configured and built installation of spack stack derived -from this config as an AWS Snapshot. The easiest way to use this -configuration of spack stack is to launch a VM using that AMI. The instructions -below are included for maintainance of the history of this site config and -their possible relevance to debugging issues should they arise. - -### Base Instance - -This AMI was built on an instance with these properties: - -* AMI Name: ubuntu/images/hvm-ssd-gp3/ubuntu-noble-24.04-amd64-server-20240927 -* AMI ID: ami-0ea3c35c5c3284d82 -* Instance m6i.4xlarge (uses Intel Xeon processor) -* 150GB of gp3 storage as / - -## Pre-requisites (For All Compilers) - -### Installing Packages - -```bash -# Update system software and start a tmux session. -sudo su - -apt update -apt upgrade -y - -# Build tools -apt install -y build-essential g++-13 gcc-13 gfortran-13 make cmake automake autoconf apt-utils - -#Install other requirements. -apt install -y cpp-13 libgomp1 git git-lfs autopoint mysql-server libmysqlclient-dev qtbase5-dev qt5-qmake libqt5svg5-dev qt5dxcb-plugin wget curl file tcl-dev gnupg2 iproute2 locales unzip less bzip2 gettext libtree pkg-config libcurl4-openssl-dev mysql-server libtool flex llvm-14 - -# Set llvm config. -update-alternatives --install /usr/bin/llvm-config llvm-config /usr/bin/llvm-config-14 10 - -# Editors -apt install -y vim nano - -# Python develop. -apt install -y python3 python3-pip python3-setuptools - -# Configure git credential caching and git lfs for the rocky user and root. -git config --global credential.helper 'cache --timeout=3600' -git lfs install -# Change the gcc, g++, and gfortran version to 13 and give it the highest priority -update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-13 100 -update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-13 100 -update-alternatives --install /usr/bin/gfortran gfortran /usr/bin/gfortran-13 100 - -exit # Exit root access -``` - -**Important**: Running the `update-alternatives` changes defaults, and if you need to use a different version of gcc/g++/gfortran you can run: `update-alternatives --config [gcc|g++|gfortran]` and select the version you want that is installed. - -### Install Lmod - -```bash -# Install lua/lmod manually because apt only has older versions -# that are not compatible with the modern lua modules spack produces -# https://lmod.readthedocs.io/en/latest/030_installing.html#install-lua-x-y-z-tar-gz -sudo su - -mkdir -p /opt/lua/5.1.4.9/src && cd $_ -wget https://sourceforge.net/projects/lmod/files/lua-5.1.4.9.tar.bz2 -tar -xvf lua-5.1.4.9.tar.bz2 -cd lua-5.1.4.9 -./configure --prefix=/opt/lua/5.1.4.9 2>&1 | tee log.config -make VERBOSE=1 2>&1 | tee log.make -make install 2>&1 | tee log.install - -cat << 'EOF' >> /etc/profile.d/02-lua.sh -# Set environment variables for lua -export PATH="/opt/lua/5.1.4.9/bin:$PATH" -export LD_LIBRARY_PATH="/opt/lua/5.1.4.9/lib:$LD_LIBRARY_PATH" -export CPATH="/opt/lua/5.1.4.9/include:$CPATH" -export MANPATH="/opt/lua/5.1.4.9/man:$MANPATH" -EOF - -source /etc/profile.d/02-lua.sh -mkdir -p /opt/lmod/8.7/src -cd /opt/lmod/8.7/src -wget https://sourceforge.net/projects/lmod/files/Lmod-8.7.tar.bz2 -tar -xvf Lmod-8.7.tar.bz2 -cd Lmod-8.7 -# Note the weird prefix, lmod installs in PREFIX/lmod/X.Y automatically -./configure --prefix=/opt/ \ - --with-lmodConfigDir=/opt/lmod/8.7/config \ - 2>&1 | tee log.config -make install 2>&1 | tee log.install -ln -sf /opt/lmod/lmod/init/profile /etc/profile.d/z00_lmod.sh -ln -sf /opt/lmod/lmod/init/cshrc /etc/profile.d/z00_lmod.csh -ln -sf /opt/lmod/lmod/init/profile.fish /etc/profile.d/z00_lmod.fish - -# Log out completely, ssh back into the instance and check if lua modules work -exit -exit -``` - -### Clone `spack-stack` - -```bash -cd /opt -sudo git clone -b release/2.1 --depth 1 --recursive https://github.com/jcsda/spack-stack.git -``` - -## Install Spack-Stack Steps by Compiler - -
-GCC Installation - -```bash -sudo su - - -cd /opt/spack-stack -source setup.sh -# Swap default module type for default linux. -sed -i 's/tcl/lmod/g' configs/sites/tier2/linux.default/modules.yaml -spack stack create env --site linux.default --template unified-dev --name unified-gcc --compiler gcc -cd envs/unified-gcc -spack env activate -p . -unset SPACK_DISABLE_LOCAL_CONFIG -export SPACK_SYSTEM_CONFIG_PATH="$(pwd)/site" -spack external find --scope system \ - --exclude cmake \ - --exclude curl --exclude openssl \ - --exclude openssh --exclude python -spack compiler find --scope system -export SPACK_DISABLE_LOCAL_CONFIG=true -unset SPACK_SYSTEM_CONFIG_PATH -# ACTION: Edit the generated compiler config (site/compilers.yaml and/or the -# compiler entries in site/packages.yaml) with the following. -# 1) Keep a single gcc@13.3.0 spec (the system compiler installed above); -# delete any empty or duplicate gcc specs. Redundant/empty specs cause -# cryptic build errors later. -# 2) Delete or comment clang/llvm refs. - -# Continue configuration. -spack config add "packages:all:prefer:['%gcc']" -spack config add "packages:all:providers:mpi:[openmpi@5.0.8]" -spack config add "packages:fontconfig:variants:+pic" -spack config add "packages:pixman:variants:+pic" -spack config add "packages:cairo:variants:+pic" -spack config add "packages:met:variants:+python +grib2 +graphics +lidar2nc +modis" -spack config add "packages:ewok-env:require:[+ecflow]" -# Pin py-netcdf4 -spack config add "packages:py-netcdf4:require:[\"@1.7.2\"]" - -# Concretize and install -spack concretize 2>&1 | tee log.concretize -${SPACK_STACK_DIR}/util/show_duplicate_packages.py -i fms -i crtm -i crtm-fix -i esmf -i mapl -i py-cython -spack install --fail-fast -j 14 2>&1 | tee log.install - -# Install modules -spack module lmod refresh -spack stack setup-meta-modules - -# Add a number of default module locations to the lmod startup script. -cat << 'EOF' >> /etc/profile.d/z01_lmod.sh -module use /opt/spack-stack/envs/unified-gcc/install/modulefiles/Core -EOF -``` - -
- -
-Intel OneAPI Installation - -#### Install Intel OneAPI Compiler - -```bash -sudo su - - -wget -O- https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB | gpg --dearmor | tee /usr/share/keyrings/oneapi-archive-keyring.gpg > /dev/null -echo "deb [signed-by=/usr/share/keyrings/oneapi-archive-keyring.gpg] https://apt.repos.intel.com/oneapi all main" | tee /etc/apt/sources.list.d/oneAPI.list -apt update -apt install intel-oneapi-compiler-dpcpp-cpp-2026.1 intel-oneapi-compiler-fortran-2026.1 intel-oneapi-mpi-devel-2021.18 intel-oneapi-tbb-devel-2023.1 intel-oneapi-mkl-devel-2026.1 -y - -exit -``` - -#### Setup OneAPI Modules - -```bash -sudo su - -# Create all modulefiles. -/opt/intel/oneapi/modulefiles-setup.sh --output-dir=/opt/intel/oneapi/modulefiles -module use /opt/intel/oneapi/modulefiles - -# Add the oneapi module files to lmod init (confirm that this file does not exist) -cat << 'EOF' >> /etc/profile.d/z01_lmod.sh -module use /opt/intel/oneapi/modulefiles -EOF - -# Create combined module file. Note that the compiler modulefile has a "tcm" -# prerequisite; omitting it fails the load with "Cannot load module -# compiler/2026.1.1. At least one of these module(s) must be loaded: tcm". -mkdir /opt/intel/oneapi/modulefiles/intel-oneapi-full-env/ -cat << 'EOF' >> /opt/intel/oneapi/modulefiles/intel-oneapi-full-env/2026.1.1 -#%Module1.0 -## -## intel-oneapi-full-env/2026.1.1 -## Intel oneAPI full module environment - -proc ModulesHelp { } { - puts stderr "intel-oneapi-full-env defines the entire module set used for spack-stack intel builds" -} -module-whatis "intel-oneapi-full-env defines the entire module set used for spack-stack intel builds" -module load umf/1.1.0 -module load tcm/1.5 -module load tbb/2023.1 -module load compiler-rt/2026.1.1 -module load compiler/2026.1.1 -module load mkl/2026.1 -module load compiler-intel-llvm/2026.1.1 -EOF -``` - -#### Install Intel OneAPI Spack-Stack Environment - -The steps below are how this site config was generated. Note that the oneAPI -modules must **not** be loaded during `spack install` -- see "The module-load -trap" at the end of this section. - -```bash -sudo su - - -module load intel-oneapi-full-env/2026.1.1 -export FC=ifx -export CXX=icpx -export CC=icx - -cd /opt/spack-stack -source ./setup.sh - -spack stack create env --site linux.default --template unified-dev --name unified-oneapi --compiler oneapi -cd envs/unified-oneapi -spack env activate -p . - - -# Find external packages for the site config. -unset SPACK_DISABLE_LOCAL_CONFIG -export SPACK_SYSTEM_CONFIG_PATH="$(pwd)/site" -spack external find --scope system --exclude bison --exclude openssl --exclude python --exclude gettext --exclude m4 --exclude cmake --exclude curl -spack external find --scope system wget -spack external find --scope system grep - -# Here we are doing some manual configuration to address the -# following tricky situations -# - External find doesn't work well for pre-installed intel-oneapi-mpi -# and we are using an external module load for this. -# - Disable "buildable" on all intel modules. -cat << 'EOF' >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml - intel-oneapi-runtime: - buildable: false - externals: - - spec: intel-oneapi-runtime@2026.1.1 - prefix: /opt/intel/oneapi - modules: - - compiler-rt/2026.1.1 - intel-oneapi-mkl: - buildable: false - externals: - # No "modules:" here on purpose - see "The module-load trap" below. - - spec: intel-oneapi-mkl@2026.1 - prefix: /opt/intel/oneapi - intel-oneapi-mpi: - buildable: false - externals: - - spec: intel-oneapi-mpi@2021.18 - prefix: /opt/intel/oneapi - modules: - - mpi/2021.18 - intel-oneapi-tbb: - buildable: false - externals: - - spec: intel-oneapi-tbb@2023.1 - prefix: /opt/intel/oneapi - modules: - - tbb/2023.1 -EOF - -spack compiler find --scope system - -# Edit site/packages.yaml -pico ${PWD}/site/packages.yaml -# Your intel compiler should look something like this below -# and any non-preferred GCC toolchains should be removed. Note -# that GCC should reference languages c, c++, but not fortran. -# -# NOTE! watch out for redundant empty gcc specs which will cause -# unintelligible build errors later. There should only be one spec -# under "externals:". -# -# intel-oneapi-compilers: -# buildable: false -# externals: -# - spec: intel-oneapi-compilers@2026.1.1 +fix_rt_linkage -# prefix: /opt/intel/oneapi -# modules: -# - umf/1.1.0 -# - tcm/1.5 -# - tbb/2023.1 -# - compiler-rt/2026.1.1 -# - compiler/2026.1.1 -# extra_attributes: -# compilers: -# c: /opt/intel/oneapi/compiler/2026.1/bin/icx -# fortran: /opt/intel/oneapi/compiler/2026.1/bin/ifx -# cxx: /opt/intel/oneapi/compiler/2026.1/bin/icpx -# gcc: -# buildable: false -# externals: -# - spec: gcc@13.3.0 languages:='c,c++,fortran' -# .... - -# Edit the spack.yaml to include these clauses. Note this must happen before -# SPACK_SYSTEM_CONFIG_PATH is unset, or it writes to /packages.yaml. -cat << 'EOF' >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml - all: - prefer: - - '%oneapi' - conflict: - - '%c=oneapi %fortran=gcc' - - '%c,cxx=oneapi %fortran=gcc' - - '%c=gcc %fortran=oneapi' - - '%c,cxx=gcc %fortran=oneapi' - - '%fortran=oneapi %c=gcc' - - '%fortran=oneapi %c,cxx=gcc' - - '%fortran=gcc %c,cxx=oneapi' - - '%fortran=gcc %c=oneapi' - providers: - mpi: [intel-oneapi-mpi@2021.18] - met: - variants: +python +grib2 +graphics +lidar2nc +modis - openmpi: - buildable: false - mpich: - buildable: false - py-scipy: - require: - #- '%c,cxx,fortran=gcc' - - 'cxxflags="-O1"' - jedi-base-env: - require: - - +fftw - - +hdf4 - jedi-python-env: - require: - - ~bufrquery - py-pyyaml: - require: - - +libyaml -EOF - -export SPACK_DISABLE_LOCAL_CONFIG=true -unset SPACK_SYSTEM_CONFIG_PATH - -# Purge the oneAPI modules before installing - see the trap below. -module purge - -spack concretize 2>&1 | tee log.concretize -${SPACK_STACK_DIR}/util/show_duplicate_packages.py -spack install --fail-fast -j 12 2>&1 | tee log.install -spack module lmod refresh && \ -spack stack setup-meta-modules - -cat << 'EOF' >> /etc/profile.d/z01_lmod.sh -module use /opt/spack-stack/envs/unified-oneapi/install/modulefiles/Core -EOF -``` - -#### The module-load trap - -Spack loads an external's `modules:` itself and treats a load that does not change -`LOADEDMODULES` as a failure. So any oneAPI module already loaded in your shell -makes `spack install` die with a misleading `ModuleLoadError` partway through. -Purge before installing, and let spack do the loading. - -For the same reason no two externals may list the same module. That is why -`intel-oneapi-mkl` above has no `modules:`: it would load before the compilers -entry and miss its `tbb`/`compiler-rt` prereqs, but listing those on mkl collides -with the compilers entry. It needs neither -- the recipe sets `MKLROOT` from -`prefix`. - -
- -## Test Installation - -
-GCC - -```bash -# Example given for building jedi-bundle -module use /opt/spack-stack/envs/unified-gcc/install/modulefiles/Core -module load stack-gcc/13.3.0 -module load stack-openmpi/5.0.8 -module load base-env -module load jedi-mpas-env -module load jedi-fv3-env -module load ewok-env -module load sp - -mkdir ~/jedi -cd ~/jedi -git clone https://github.com/JCSDA-internal/jedi-bundle.git -cd jedi-bundle -mkdir build && cd build -ecbuild ../ -make update -make -j10 -ctest -``` - -
- -
-Intel OneAPI - -```bash -# Build jedi-bundle with oneapi -module use /opt/spack-stack/envs/unified-oneapi/install/modulefiles/Core -module load stack-intel-oneapi-compilers/2026.1.1 -module load stack-intel-oneapi-mpi/2021.18 -module load base-env -module load jedi-mpas-env -module load jedi-fv3-env -module load ewok-env -module load sp - -mkdir /opt/jedi -cd /opt/jedi -git clone https://github.com/JCSDA-internal/jedi-bundle.git -cd jedi-bundle -mkdir build && cd build -ecbuild ../ -make update -make -j10 2>&1 | tee log.make -ctest -``` - -
- -The installation and configuration is now complete for the instance. diff --git a/configs/sites/tier2/aws-ubuntu2404/config.yaml b/configs/sites/tier2/aws-ubuntu2404/config.yaml deleted file mode 100644 index cf54e6360f..0000000000 --- a/configs/sites/tier2/aws-ubuntu2404/config.yaml +++ /dev/null @@ -1,9 +0,0 @@ -config: - build_jobs: 8 - - # Overrides for spack build and staging areas to speed up builds - # by using a local directory instead of the EFS shared filesystem - build_stage: /tmp/spack-stack/cache/build_stage - test_stage: /tmp/spack-stack/cache/test_stage - source_cache: /tmp/spack-stack/cache/source_cache - misc_cache: /tmp/spack-stack/cache/misc_cache diff --git a/configs/sites/tier2/aws-ubuntu2404/packages.yaml b/configs/sites/tier2/aws-ubuntu2404/packages.yaml deleted file mode 100644 index 8ee1d27dd4..0000000000 --- a/configs/sites/tier2/aws-ubuntu2404/packages.yaml +++ /dev/null @@ -1,117 +0,0 @@ -packages: - # Package constraints. - ewok-env: - require: - - +ecflow - met: - require: [+python, +grib2, +graphics, +lidar2nc, +modis] - py-awscrt: - require: - - "@0.19.19" - py-botocore: - require: - - "@1.34.162" - py-boto3: - require: - - "@1.34.162" - py-netcdf4: - require: - - "@1.7.2" - jedi-base-env: - require: [+fftw, +hdf4] - jedi-python-env: - require: [+bufrquery] - - # System packages. - autoconf: - externals: - - spec: autoconf@2.71 - prefix: /usr - automake: - externals: - - spec: automake@1.16.5 - prefix: /usr - binutils: - externals: - - spec: binutils@2.42+gold~headers - prefix: /usr - coreutils: - externals: - - spec: coreutils@9.4 - prefix: /usr - diffutils: - externals: - - spec: diffutils@3.10 - prefix: /usr - findutils: - externals: - - spec: findutils@4.9.0 - prefix: /usr - flex: - externals: - - spec: flex@2.6.4+lex - prefix: /usr - gawk: - externals: - - spec: gawk@5.2.1 - prefix: /usr - gettext: - externals: - - spec: gettext@0.21 - prefix: /usr - git: - externals: - - spec: git@2.43.0~tcltk - prefix: /usr - git-lfs: - externals: - - spec: git-lfs@3.4.1 - prefix: /usr - grep: - externals: - - spec: grep@3.11 - prefix: /usr - gmake: - externals: - - spec: gmake@4.3 - prefix: /usr - groff: - externals: - - spec: groff@1.23.0 - prefix: /usr - libtool: - externals: - - spec: libtool@2.4.7 - prefix: /usr - m4: - externals: - - spec: m4@1.4.19 - prefix: /usr - perl: - externals: - - spec: perl@5.38.2~cpanm+opcode+open+shared+threads - prefix: /usr - pkgconf: - externals: - - spec: pkgconf@1.8.1 - prefix: /usr - qt:: - buildable: false - externals: - - spec: qt@5.15.3 - prefix: /usr - version: [5.15.3] - sed: - externals: - - spec: sed@4.9 - prefix: /usr - tar:: - buildable: false - externals: - - spec: tar@1.35 - prefix: /usr - wget:: - buildable: false - externals: - - spec: wget@1.21.4 - prefix: /usr diff --git a/configs/sites/tier2/aws-ubuntu2404/packages_gcc.yaml b/configs/sites/tier2/aws-ubuntu2404/packages_gcc.yaml deleted file mode 100644 index c0451c82d2..0000000000 --- a/configs/sites/tier2/aws-ubuntu2404/packages_gcc.yaml +++ /dev/null @@ -1,15 +0,0 @@ -packages: - gcc: - buildable: false - externals: - - spec: gcc@13.3.0 languages:='c,c++,fortran' - prefix: /usr - extra_attributes: - compilers: - c: /usr/bin/gcc - cxx: /usr/bin/g++ - fortran: /usr/bin/gfortran - gcc-runtime: - externals: - - spec: gcc-runtime@13.3.0 - prefix: /usr diff --git a/configs/sites/tier2/ubuntu2404-nvhpc/packages.yaml b/configs/sites/tier2/ubuntu2404-nvhpc/packages.yaml index c4241e9caa..ff0c62809e 100644 --- a/configs/sites/tier2/ubuntu2404-nvhpc/packages.yaml +++ b/configs/sites/tier2/ubuntu2404-nvhpc/packages.yaml @@ -116,7 +116,7 @@ packages: pkg-config: buildable: False # No system python external: Ubuntu 24.04 ships 3.12 and common requires - # python@3.13, so spack builds it (as on aws-ubuntu2404). + # python@3.13, so spack builds it (the tier1 aws-ubuntu site also excludes it). sed: externals: - spec: sed@4.9 From 0d5d7343b0668a268e3c6c236b5a71dd61a7e91b Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Fri, 30 Oct 2026 09:59:46 -0600 Subject: [PATCH 10/14] [release/2.2] Add mapl@2.69.1 ^esmf@8.9.1 to unified environment (#2195) For release/2.2: Add mapl@2.69.1 ^esmf@8.9.1 to unified environment (configs/templates/unified-dev/spack.yaml). --- configs/common/modules_lmod.yaml | 4 ++++ configs/common/modules_tcl.yaml | 4 ++++ configs/common/packages.yaml | 1 + configs/templates/unified-dev/spack.yaml | 2 ++ 4 files changed, 11 insertions(+) diff --git a/configs/common/modules_lmod.yaml b/configs/common/modules_lmod.yaml index d93368772a..47a0c59efe 100644 --- a/configs/common/modules_lmod.yaml +++ b/configs/common/modules_lmod.yaml @@ -143,6 +143,10 @@ modules: 'MADIS_ROOT': '{prefix}' mapl: suffixes: + ^esmf@=8.9.1~debug ~openmp: 'esmf-8.9.1-noopenmp' + ^esmf@=8.9.1+debug ~openmp: 'esmf-8.9.1-debug-noopenmp' + ^esmf@=8.9.1~debug +openmp: 'esmf-8.9.1' + ^esmf@=8.9.1+debug +openmp: 'esmf-8.9.1-debug' ^esmf@=9.0.0~debug ~openmp: 'esmf-9.0.0-noopenmp' ^esmf@=9.0.0+debug ~openmp: 'esmf-9.0.0-debug-noopenmp' ^esmf@=9.0.0~debug +openmp: 'esmf-9.0.0' diff --git a/configs/common/modules_tcl.yaml b/configs/common/modules_tcl.yaml index 0249e04ac1..bd237abdae 100644 --- a/configs/common/modules_tcl.yaml +++ b/configs/common/modules_tcl.yaml @@ -162,6 +162,10 @@ modules: 'MADIS_ROOT': '{prefix}' mapl: suffixes: + ^esmf@=8.9.1~debug ~openmp: 'esmf-8.9.1-noopenmp' + ^esmf@=8.9.1+debug ~openmp: 'esmf-8.9.1-debug-noopenmp' + ^esmf@=8.9.1~debug +openmp: 'esmf-8.9.1' + ^esmf@=8.9.1+debug +openmp: 'esmf-8.9.1-debug' ^esmf@=9.0.0~debug ~openmp: 'esmf-9.0.0-noopenmp' ^esmf@=9.0.0+debug ~openmp: 'esmf-9.0.0-debug-noopenmp' ^esmf@=9.0.0~debug +openmp: 'esmf-9.0.0' diff --git a/configs/common/packages.yaml b/configs/common/packages.yaml index 49042148f9..91d18f7817 100644 --- a/configs/common/packages.yaml +++ b/configs/common/packages.yaml @@ -151,6 +151,7 @@ packages: - +shared - +external-parallelio - any_of: + - '@=8.9.1' - '@=9.0.0b17' - '@=9.0.0b18' - any_of: diff --git a/configs/templates/unified-dev/spack.yaml b/configs/templates/unified-dev/spack.yaml index dc25760ad7..eb5bc6868a 100644 --- a/configs/templates/unified-dev/spack.yaml +++ b/configs/templates/unified-dev/spack.yaml @@ -28,9 +28,11 @@ spack: - ufs-weather-model-env ^esmf@=9.0.0b17 ^crtm@=3.1.5 # Mapl with specific esmf tags (list all to avoid duplicate packages) + - mapl@2.69.1 ^esmf@=8.9.1 - mapl@2.69.1 ^esmf@=9.0.0b17 # Various esmf tags (list all to avoid duplicate packages) + - esmf@=8.9.1 - esmf@=9.0.0b17 - esmf@=9.0.0b18 From 8fa42858ef9c51136ddbb2c84562974a93dbf1f1 Mon Sep 17 00:00:00 2001 From: Evan Parker Date: Fri, 30 Oct 2026 10:00:51 -0600 Subject: [PATCH 11/14] [release/2.2] Container site updates; move to new compilers and Ubuntu 26.04 (#2197) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Brings the container site in line with tier1/aws-ubuntu for spack-stack 2.2. Ubuntu 26.04, GCC 14.3.0, OneAPI 2026.1. Other changes: - qt is built with gcc, because Qt 5.15 doesn't compile with icpx. - The install step runs with build_language: C.UTF-8. Intel's umf modulefile prints "®" as a bare non-UTF-8 byte which spack can't decode otherwise. - Builder stage sets MODULESHOME and MODULEPATH explcitly to fix spack module discovery. - Updated load_jedi_bundle_env: for new meta-modules and some unexpected python odds and ends to fix the jedi build environment (as far as I can tell this is idiosyncratic to the container build; the base modules work better in the aws-ubuntu site, I will debug this issue later but for now the fixed loader enables successful jedi builds) --- configs/sites/tier1/container/Dockerfile.gcc | 47 ++++--- .../sites/tier1/container/Dockerfile.oneapi | 68 ++++++---- configs/sites/tier1/container/README.md | 63 ++++++++- configs/sites/tier1/container/config.yaml | 5 +- configs/sites/tier1/container/modules.yaml | 6 + configs/sites/tier1/container/packages.yaml | 39 +++--- .../sites/tier1/container/packages_gcc.yaml | 14 +- .../tier1/container/packages_oneapi.yaml | 126 +++++++++++++----- 8 files changed, 265 insertions(+), 103 deletions(-) diff --git a/configs/sites/tier1/container/Dockerfile.gcc b/configs/sites/tier1/container/Dockerfile.gcc index 22da603579..c86b20f7b8 100644 --- a/configs/sites/tier1/container/Dockerfile.gcc +++ b/configs/sites/tier1/container/Dockerfile.gcc @@ -2,9 +2,9 @@ # See the README.md for building instructions. ## ---------- build base container used by builder and runtime ---------- -FROM ubuntu:24.04 AS ubuntu_base +FROM ubuntu:26.04 AS ubuntu_base -ARG BUILD_JOBS=4 +ARG BUILD_JOBS=8 ARG SPACK_STACK_TEMPLATE=unified-dev SHELL ["/bin/bash", "-c"] @@ -22,14 +22,14 @@ RUN set -euo pipefail; \ apt-get -yqq update && \ apt-get -yqq upgrade && \ apt-get -yqq install --no-install-recommends \ - # Core build tools. + # Core build tools. build-essential pulls in the default gcc-15; we add + # gcc-14 alongside it, since spack-stack supports gcc up to 14.x. build-essential \ - gcc-13 \ - gcc++-13 \ - gfortran-13 \ - cpp-13 \ + gcc-14 \ + g++-14 \ + gfortran-14 \ + cpp-14 \ make \ - llvm-14 \ autoconf \ # External dependencies. libcurl4-openssl-dev \ @@ -55,10 +55,11 @@ RUN set -euo pipefail; \ locales && \ locale-gen en_US.UTF-8 && \ # Update alternatives for compilers and tools - update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-13 100 && \ - update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-13 100 && \ - update-alternatives --install /usr/bin/gfortran gfortran /usr/bin/gfortran-13 100 && \ - update-alternatives --install /usr/bin/llvm-config llvm-config /usr/bin/llvm-config-14 100 && \ + # Make gcc-14 the default gcc/g++/gfortran for container users. Spack uses + # the -14 binaries by full path (see packages_gcc.yaml). + update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-14 100 && \ + update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-14 100 && \ + update-alternatives --install /usr/bin/gfortran gfortran /usr/bin/gfortran-14 100 && \ rm -rf /var/lib/apt/lists/* @@ -80,7 +81,8 @@ COPY --exclude=configs/sites/tier1/container/Dockerfile.oneapi \ # Create spack-stack environment WORKDIR /opt/spack-stack -RUN mkdir -p /tmp/spack-stack && \ +RUN set -o pipefail && \ + mkdir -p /tmp/spack-stack && \ source setup.sh && \ spack stack create env \ --site container \ @@ -93,7 +95,7 @@ RUN mkdir -p /tmp/spack-stack && \ spack concretize 2>&1 | tee log.concretize # Build the spack environment. -RUN source setup.sh && cd ${SPACK_STACK_DIR}/envs/container && spack env activate . && \ +RUN set -o pipefail && source setup.sh && cd ${SPACK_STACK_DIR}/envs/container && spack env activate . && \ spack install --fail-fast -j ${BUILD_JOBS} 2>&1 | tee log.install # Setup modules and meta-modules. @@ -126,16 +128,19 @@ RUN echo "ulimit -s unlimited" > /etc/spack_container_rc.sh \ && echo "export PRTE_MCA_rmaps_default_mapping_policy=:oversubscribe" >> /etc/spack_container_rc.sh \ && echo "# TCL module path for the spack-stack environment." >> /etc/spack_container_rc.sh \ && echo "source /etc/profile.d/modules.sh" >> /etc/spack_container_rc.sh \ - && echo "module use /opt/spack-software/modulefiles" >> /etc/spack_container_rc.sh \ && echo "module use /opt/spack-software/modulefiles/Core" >> /etc/spack_container_rc.sh \ && echo "# Bash function to load modules necessary to build the JEDI bundle." >> /etc/spack_container_rc.sh \ && echo "load_jedi_bundle_env() {" >> /etc/spack_container_rc.sh \ && echo " module purge" >> /etc/spack_container_rc.sh \ - && echo " module load stack-gcc" >> /etc/spack_container_rc.sh \ - && echo " module load stack-openmpi" >> /etc/spack_container_rc.sh \ + && echo " module load stack-gcc/14.3.0" >> /etc/spack_container_rc.sh \ + && echo " module load stack-openmpi/5.0.10" >> /etc/spack_container_rc.sh \ && echo " module load base-env" >> /etc/spack_container_rc.sh \ && echo " module load jedi-mpas-env" >> /etc/spack_container_rc.sh \ + && echo " module load jedi-python-env" >> /etc/spack_container_rc.sh \ && echo " module load py-fortranformat" >> /etc/spack_container_rc.sh \ + && echo " module load py-matplotlib" >> /etc/spack_container_rc.sh \ + && echo " module load py-cartopy" >> /etc/spack_container_rc.sh \ + && echo " module load py-pyproj" >> /etc/spack_container_rc.sh \ && echo " module load jedi-fv3-env" >> /etc/spack_container_rc.sh \ && echo " module load ewok-env" >> /etc/spack_container_rc.sh \ && echo " module load ip" >> /etc/spack_container_rc.sh \ @@ -153,6 +158,14 @@ RUN useradd -U -k /etc/skel -s /bin/bash -d /home/nonroot -m nonroot --uid 43891 && mkdir /home/nonroot/.pmix \ && chown -R nonroot:nonroot /home/nonroot/.gitconfig /home/nonroot/.pmix +# Fail the build if the stack meta-modules don't resolve to real compiler and +# MPI modules (and nothing else) in the final image. +RUN source /etc/spack_container_rc.sh && \ + module load stack-gcc/14.3.0 stack-openmpi/5.0.10 && \ + module is-loaded gcc/14.3.0 && module is-loaded openmpi/5.0.10 && \ + ! module is-loaded ufs-weather-model-env && \ + which mpicc mpif90 && mpif90 --version | grep -q " 14\.3\." + # Ensure the container rc is run by non-login shells too. ENV BASH_ENV=/etc/spack_container_rc.sh diff --git a/configs/sites/tier1/container/Dockerfile.oneapi b/configs/sites/tier1/container/Dockerfile.oneapi index f0fe200b82..a536e06c2d 100644 --- a/configs/sites/tier1/container/Dockerfile.oneapi +++ b/configs/sites/tier1/container/Dockerfile.oneapi @@ -2,9 +2,9 @@ # See the README.md for building instructions. ## ---------- build base container used by builder and runtime ---------- -FROM ubuntu:24.04 AS ubuntu_base +FROM ubuntu:26.04 AS ubuntu_base -ARG BUILD_JOBS=4 +ARG BUILD_JOBS=8 ARG SPACK_STACK_TEMPLATE=unified-dev SHELL ["/bin/bash", "-c"] @@ -22,14 +22,14 @@ RUN set -euo pipefail; \ apt-get -yqq update && \ apt-get -yqq upgrade && \ apt-get -yqq install --no-install-recommends \ - # Core build tools. + # Core build tools. build-essential pulls in the default gcc-15; we add + # gcc-14 alongside it, since spack-stack supports gcc up to 14.x. build-essential \ - gcc-13 \ - gcc++-13 \ - gfortran-13 \ - cpp-13 \ + gcc-14 \ + g++-14 \ + gfortran-14 \ + cpp-14 \ make \ - llvm-14 \ autoconf \ # External dependencies. libcurl4-openssl-dev \ @@ -55,10 +55,11 @@ RUN set -euo pipefail; \ locales && \ locale-gen en_US.UTF-8 && \ # Update alternatives for compilers and tools - update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-13 100 && \ - update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-13 100 && \ - update-alternatives --install /usr/bin/gfortran gfortran /usr/bin/gfortran-13 100 && \ - update-alternatives --install /usr/bin/llvm-config llvm-config /usr/bin/llvm-config-14 100 && \ + # Make gcc-14 the default gcc/g++/gfortran for container users. Spack uses + # the -14 binaries by full path (see packages_gcc.yaml). + update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-14 100 && \ + update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-14 100 && \ + update-alternatives --install /usr/bin/gfortran gfortran /usr/bin/gfortran-14 100 && \ # Install the intel apt install -y apt-utils gpg && \ wget -O- https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB \ @@ -68,11 +69,11 @@ RUN set -euo pipefail; \ | tee /etc/apt/sources.list.d/oneAPI.list && \ apt update && \ apt install -y \ - intel-oneapi-compiler-dpcpp-cpp-2025.3 \ - intel-oneapi-compiler-fortran-2025.3 \ - intel-oneapi-mpi-devel-2021.17 \ - intel-oneapi-tbb-devel-2022.3 \ - intel-oneapi-mkl-devel-2025.3 && \ + intel-oneapi-compiler-dpcpp-cpp-2026.1 \ + intel-oneapi-compiler-fortran-2026.1 \ + intel-oneapi-mpi-devel-2021.18 \ + intel-oneapi-tbb-devel-2023.1 \ + intel-oneapi-mkl-devel-2026.1 && \ /opt/intel/oneapi/modulefiles-setup.sh --output-dir=/opt/intel/oneapi/modulefiles && \ echo "source /etc/profile.d/modules.sh && module use /opt/intel/oneapi/modulefiles" > /etc/profile.d/z01_oneapi_modules.sh && \ rm -rf /var/lib/apt/lists/* @@ -82,7 +83,12 @@ RUN set -euo pipefail; \ FROM ubuntu_base AS builder ENV SPACK_STACK_DIR=/opt/spack-stack \ - SPACK_ROOT=/opt/spack-stack/spack + SPACK_ROOT=/opt/spack-stack/spack \ + # Spack loads the oneAPI compiler/MPI modules itself (also during + # concretization). RUN shells don't source /etc/profile.d, so point spack at + # environment-modules and the oneAPI modulefiles explicitly. + MODULESHOME=/usr/share/modules \ + MODULEPATH=/opt/intel/oneapi/modulefiles # Copy spack-stack from build context. Note the build context must be the @@ -97,7 +103,8 @@ COPY --exclude=configs/sites/tier1/container/Dockerfile.oneapi \ # Create spack-stack environment WORKDIR /opt/spack-stack -RUN mkdir -p /tmp/spack-stack && \ +RUN set -o pipefail && \ + mkdir -p /tmp/spack-stack && \ source setup.sh && \ spack stack create env \ --site container \ @@ -110,8 +117,11 @@ RUN mkdir -p /tmp/spack-stack && \ spack concretize 2>&1 | tee log.concretize # Build the spack environment. -RUN source setup.sh && cd ${SPACK_STACK_DIR}/envs/container && spack env activate . && \ - spack install --fail-fast -j ${BUILD_JOBS} 2>&1 | tee log.install +# Spack loads the external oneAPI modules under LC_ALL= (default +# C). In the C locale environment-modules prints the "®" in Intel's umf +# modulefile as a bare Latin-1 byte, which spack fails to decode as UTF-8. +RUN set -o pipefail && source setup.sh && cd ${SPACK_STACK_DIR}/envs/container && spack env activate . && \ + spack -c config:build_language:C.UTF-8 install --fail-fast -j ${BUILD_JOBS} 2>&1 | tee log.install # Setup modules and meta-modules. RUN source setup.sh && cd ${SPACK_STACK_DIR}/envs/container && spack env activate . && \ @@ -140,16 +150,19 @@ RUN echo "ulimit -s unlimited" > /etc/spack_container_rc.sh \ && echo "# TCL module path for the spack-stack environment." >> /etc/spack_container_rc.sh \ && echo "source /etc/profile.d/modules.sh" >> /etc/spack_container_rc.sh \ && echo "module use /opt/intel/oneapi/modulefiles" >> /etc/spack_container_rc.sh \ - && echo "module use /opt/spack-software/modulefiles" >> /etc/spack_container_rc.sh \ && echo "module use /opt/spack-software/modulefiles/Core" >> /etc/spack_container_rc.sh \ && echo "# Bash function to load modules necessary to build the JEDI bundle." >> /etc/spack_container_rc.sh \ && echo "load_jedi_bundle_env() {" >> /etc/spack_container_rc.sh \ && echo " module purge" >> /etc/spack_container_rc.sh \ - && echo " module load stack-intel-oneapi-compilers" >> /etc/spack_container_rc.sh \ - && echo " module load stack-intel-oneapi-mpi" >> /etc/spack_container_rc.sh \ + && echo " module load stack-intel-oneapi-compilers/2026.1.1" >> /etc/spack_container_rc.sh \ + && echo " module load stack-intel-oneapi-mpi/2021.18" >> /etc/spack_container_rc.sh \ && echo " module load base-env" >> /etc/spack_container_rc.sh \ && echo " module load jedi-mpas-env" >> /etc/spack_container_rc.sh \ + && echo " module load jedi-python-env" >> /etc/spack_container_rc.sh \ && echo " module load py-fortranformat" >> /etc/spack_container_rc.sh \ + && echo " module load py-matplotlib" >> /etc/spack_container_rc.sh \ + && echo " module load py-cartopy" >> /etc/spack_container_rc.sh \ + && echo " module load py-pyproj" >> /etc/spack_container_rc.sh \ && echo " module load jedi-fv3-env" >> /etc/spack_container_rc.sh \ && echo " module load ewok-env" >> /etc/spack_container_rc.sh \ && echo " module load ip" >> /etc/spack_container_rc.sh \ @@ -166,6 +179,13 @@ RUN useradd -U -k /etc/skel -s /bin/bash -d /home/nonroot -m nonroot --uid 43891 && mkdir /home/nonroot/.pmix \ && chown -R nonroot:nonroot /home/nonroot/.gitconfig /home/nonroot/.pmix +# Fail the build if the stack meta-modules don't load the oneAPI compilers and +# MPI in the final image. +RUN source /etc/spack_container_rc.sh && \ + module load stack-intel-oneapi-compilers/2026.1.1 stack-intel-oneapi-mpi/2021.18 && \ + module is-loaded compiler/2026.1.1 && module is-loaded mpi/2021.18 && \ + which icx icpx ifx mpiifx + # Ensure the container rc is run by non-login shells too. ENV BASH_ENV=/etc/spack_container_rc.sh diff --git a/configs/sites/tier1/container/README.md b/configs/sites/tier1/container/README.md index c388bd8fa0..0b69bc1e4b 100644 --- a/configs/sites/tier1/container/README.md +++ b/configs/sites/tier1/container/README.md @@ -24,7 +24,7 @@ docker build \ -t spack-stack-gcc:local \ -f "$(git rev-parse --show-toplevel)/configs/sites/tier1/container/Dockerfile.gcc" \ --build-arg SPACK_STACK_TEMPLATE=unified-dev \ - --build-arg BUILD_JOBS=10 \ + --build-arg BUILD_JOBS=8 \ "$(git rev-parse --show-toplevel)" 2>&1 | tee ${HOME}/log.docker_gcc @@ -32,6 +32,65 @@ docker build \ -t spack-stack-oneapi:local \ -f "$(git rev-parse --show-toplevel)/configs/sites/tier1/container/Dockerfile.oneapi" \ --build-arg SPACK_STACK_TEMPLATE=unified-dev \ - --build-arg BUILD_JOBS=10 \ + --build-arg BUILD_JOBS=8 \ "$(git rev-parse --show-toplevel)" 2>&1 | tee ${HOME}/log.docker_oneapi ``` + +`BUILD_JOBS` is the number of make jobs per package. `config.yaml` sets +`concurrent_packages: 4`, so a build runs up to 4 x `BUILD_JOBS` jobs at once +(32 on a 16-core machine). A full `unified-dev` build takes about 4 hours on 16 +cores and needs well over 100 GB of free disk for the build cache. + +The `COPY` of the repository includes `.git` (`spack stack create env` records +the spack-stack and spack commit hashes), so staging or committing a change +invalidates the Docker cache from that layer on and starts the spack install +over. + +## Image Contents + +Both images are based on Ubuntu 26.04. + +* **gcc**: gcc 14.3.0 and openmpi 5.0.10. Ubuntu 26.04's default gcc is 15.2, + which is newer than spack-stack supports. The image installs gcc-14 next to + it and makes it the default `gcc`/`g++`/`gfortran` with `update-alternatives`. + Spack uses the `-14` binaries by full path. +* **oneapi**: Intel oneAPI 2026.1.1 compilers (icx/icpx/ifx), Intel MPI 2021.18 + and MKL 2026.1, installed from Intel's apt repository into `/opt/intel/oneapi`. + The compilers are pinned to the gcc-14 toolchain with compiler flags (see + `packages_oneapi.yaml` and the tier1/aws-ubuntu README). gcc 14.3.0 builds a + few packages (cmake, rust, py-numpy, ...). + +The site configs follow tier1/aws-ubuntu, which tracks the same Ubuntu LTS. +Differences: tcl modules (environment-modules) instead of Lmod, the +container-specific Python pins in `packages.yaml`, and build stages under +`/tmp` (not a tmpfs inside a container). + +## Using the Container + +The images are development images: they include the compilers, MPI, and the +spack-stack modules under `/opt/spack-software/modulefiles`. Every bash shell +sources `/etc/spack_container_rc.sh`, which puts the `Core` module directory on +`MODULEPATH` and defines `load_jedi_bundle_env`: + +```bash +load_jedi_bundle_env # stack-gcc + stack-openmpi (or the oneAPI meta-modules) and the JEDI modules +module list +``` + +To load modules by hand, start from the compiler and MPI meta-modules: + +```bash +# gcc +module load stack-gcc/14.3.0 stack-openmpi/5.0.10 +# oneapi +module load stack-intel-oneapi-compilers/2026.1.1 stack-intel-oneapi-mpi/2021.18 +``` + +Load modules such as `py-cartopy` and `py-pyproj` **before** running `ecbuild` +on jedi-bundle: some tests record `PYTHONPATH` at configure time. + +OpenMPI is set up to run as root and to oversubscribe, so MPI tests work with +the default `docker run`. Intel MPI uses `/dev/shm` for on-node communication, +and Docker's default of 64 MB is too small: 12-task jedi-bundle tests die with a +bus error. Run the oneAPI image with a larger `/dev/shm`, for example +`docker run --shm-size=16g ...`. diff --git a/configs/sites/tier1/container/config.yaml b/configs/sites/tier1/container/config.yaml index 52ecf712aa..46fcc67dcf 100644 --- a/configs/sites/tier1/container/config.yaml +++ b/configs/sites/tier1/container/config.yaml @@ -1,5 +1,8 @@ config: - build_jobs: 10 + # Old installer (see common/config.yaml): build_jobs is per package, so this + # runs up to concurrent_packages x build_jobs compile jobs (32 on 16 cores). + build_jobs: 8 + concurrent_packages: 4 # Container-specific paths for build and staging areas. # Use /tmp for transient build data (not preserved in final image). diff --git a/configs/sites/tier1/container/modules.yaml b/configs/sites/tier1/container/modules.yaml index b134d36696..38df2ef704 100644 --- a/configs/sites/tier1/container/modules.yaml +++ b/configs/sites/tier1/container/modules.yaml @@ -5,3 +5,9 @@ modules: tcl: include: - python + # common/modules_tcl.yaml excludes compiler and MPI modules. The stack-gcc + # and stack-openmpi meta-modules load gcc/ and openmpi/, so the + # gcc env needs them. The oneAPI meta-modules load Intel's own modulefiles + # (the externals' modules:), so those packages stay excluded. + - gcc + - openmpi diff --git a/configs/sites/tier1/container/packages.yaml b/configs/sites/tier1/container/packages.yaml index 82869920b4..8ec49ef669 100644 --- a/configs/sites/tier1/container/packages.yaml +++ b/configs/sites/tier1/container/packages.yaml @@ -13,6 +13,18 @@ packages: parallelio: require: - +pnetcdf + jedi-base-env: + require: [+fftw, +hdf4] + # jedi-ufs-env needs +bufrquery; unconstrained, the root splits in two. + jedi-python-env: + require: [+bufrquery] + # Unconstrained, jedi-tools-env falls back to py-myst-parser@1/py-sphinx@6, + # which cap py-attrs@:21 and py-docutils@:0.19 and split py-attrs, py-eccodes, + # py-rasterio, jedi-python-env and awscli-v2 (see tier1/aws-ubuntu). + py-myst-parser: + require: ['@5:'] + py-sphinx: + require: ['@9:'] pixman: variants: +pic py-awscrt: @@ -28,45 +40,38 @@ packages: require: - "@1.7.2" - # System packages installed during the container build. + # System packages installed during the container build (Ubuntu 26.04). + # coreutils is uutils on 26.04 and isn't detected by spack; llvm-14 is no + # longer available and nothing in the stack needs it. autoconf: externals: - - spec: autoconf@2.71 - prefix: /usr - coreutils: - externals: - - spec: coreutils@9.4 + - spec: autoconf@2.72 prefix: /usr diffutils: buildable: false externals: - - spec: diffutils@3.10 + - spec: diffutils@3.12 prefix: /usr findutils: externals: - - spec: findutils@4.9.0 + - spec: findutils@4.10.0 prefix: /usr git: buildable: false externals: - - spec: git@2.43.0~tcltk + - spec: git@2.53.0~tcltk prefix: /usr git-lfs: buildable: false externals: - - spec: git-lfs@3.4.1 + - spec: git-lfs@3.7.1 prefix: /usr grep: externals: - - spec: grep@3.11 - prefix: /usr - llvm: - buildable: false - externals: - - spec: llvm@14.0.6 + - spec: grep@3.12 prefix: /usr perl: buildable: false externals: - - spec: perl@5.38.2 + - spec: perl@5.40.1~cpanm+opcode+open+shared+threads prefix: /usr diff --git a/configs/sites/tier1/container/packages_gcc.yaml b/configs/sites/tier1/container/packages_gcc.yaml index d17749d313..e16a9b1799 100644 --- a/configs/sites/tier1/container/packages_gcc.yaml +++ b/configs/sites/tier1/container/packages_gcc.yaml @@ -1,18 +1,16 @@ packages: all: - prefer: - - '%gcc' providers: - mpi: [openmpi@5.0.8] + mpi: [openmpi@5.0.10] mpi: - require: [openmpi@5.0.8] + require: [openmpi@5.0.10] gcc: externals: - - spec: gcc@13.3.0 languages:='c,c++,fortran' + - spec: gcc@14.3.0 languages:='c,c++,fortran' prefix: /usr extra_attributes: compilers: - c: /usr/bin/gcc - cxx: /usr/bin/g++ - fortran: /usr/bin/gfortran + c: /usr/bin/gcc-14 + cxx: /usr/bin/g++-14 + fortran: /usr/bin/gfortran-14 buildable: false diff --git a/configs/sites/tier1/container/packages_oneapi.yaml b/configs/sites/tier1/container/packages_oneapi.yaml index 88edc96092..0b6843615b 100644 --- a/configs/sites/tier1/container/packages_oneapi.yaml +++ b/configs/sites/tier1/container/packages_oneapi.yaml @@ -1,65 +1,123 @@ packages: + + # Compilers. + # Language preferences for oneAPI come from common/packages_oneapi.yaml. Don't + # add "all: prefer: ['%oneapi']": it acts as a dependency constraint and makes + # spack rebuild system externals. all: - prefer: - - '%oneapi' - providers: - mpi: [intel-oneapi-mpi@2021.17] - mpi: - require: [intel-oneapi-mpi@2021.17] - # Compilers and meta packages. + conflict: + - '%c=oneapi %fortran=gcc' + - '%c,cxx=oneapi %fortran=gcc' + - '%c=gcc %fortran=oneapi' + - '%c,cxx=gcc %fortran=oneapi' + - '%fortran=oneapi %c=gcc' + - '%fortran=oneapi %c,cxx=gcc' + - '%fortran=gcc %c,cxx=oneapi' + - '%fortran=gcc %c=oneapi' + intel-oneapi-compilers: buildable: false externals: - - spec: intel-oneapi-compilers@2025.3.0 +fix_rt_linkage + - spec: intel-oneapi-compilers@2026.1.1 +fix_rt_linkage prefix: /opt/intel/oneapi modules: - - umf/1.0.2 - - tbb/2022.3 - - compiler-rt/2025.3.2 - - compiler/2025.3.2 + - umf/1.1.0 + # 2026.1's compiler modulefile adds a "tcm" prereq that 2025.3 lacks + - tcm/1.5 + - tbb/2023.1 + - compiler-rt/2026.1.1 + - compiler/2026.1.1 extra_attributes: compilers: - c: /opt/intel/oneapi/compiler/2025.3/bin/icx - cxx: /opt/intel/oneapi/compiler/2025.3/bin/icpx - fortran: /opt/intel/oneapi/compiler/2025.3/bin/ifx + fortran: /opt/intel/oneapi/compiler/2026.1/bin/ifx + c: /opt/intel/oneapi/compiler/2026.1/bin/icx + cxx: /opt/intel/oneapi/compiler/2026.1/bin/icpx + # Ubuntu 26.04's default gcc is 15. Left alone, icx/icpx pick the newest + # installed GCC for libstdc++ headers and libs, and ifx links its + # runtime. Pin them to gcc 14 so they match the gcc env. + flags: + cflags: --gcc-install-dir=/usr/lib/gcc/x86_64-linux-gnu/14 + cxxflags: --gcc-install-dir=/usr/lib/gcc/x86_64-linux-gnu/14 + fflags: -gcc-name=/usr/bin/gcc-14 -gxx-name=/usr/bin/g++-14 + openmpi: + buildable: false + mpich: + buildable: false + mpi: + buildable: false + require: + - intel-oneapi-mpi@2021.18 gcc: externals: - - spec: gcc@13.3.0 languages:='c,c++,fortran' + - spec: gcc@14.3.0 languages:='c,c++,fortran' prefix: /usr extra_attributes: compilers: - c: /usr/bin/gcc - cxx: /usr/bin/g++ - fortran: /usr/bin/gfortran + c: /usr/bin/gcc-14 + cxx: /usr/bin/g++-14 + fortran: /usr/bin/gfortran-14 buildable: false intel-oneapi-mkl: buildable: false externals: - - spec: intel-oneapi-mkl@2025.3 + # Deliberately NO "modules:" here. The mkl/2026.1 modulefile declares prereqs + # on tbb and compiler-rt, and spack loads externals in DAG order -- for e.g. + # "ip" it loads intel-oneapi-mkl BEFORE intel-oneapi-compilers, so mkl would + # be loaded without its prereqs and fail with + # ModuleLoadError: Module 'mkl/2026.1' could not be loaded. + # Adding tbb/compiler-rt here instead collides with the compilers entry + # below: spack's load_module() treats an already-loaded module (no change to + # LOADEDMODULES) as a failure, so the second load of tbb would error. + # None of it is needed -- the intel-oneapi-mkl package sets MKLROOT and + # PKG_CONFIG_PATH from component_prefix (= prefix/mkl/) under the + # default +envmods, and resolves libs/headers from prefix. + - spec: intel-oneapi-mkl@2026.1 prefix: /opt/intel/oneapi - modules: - - mkl/2025.3 intel-oneapi-mpi: buildable: false externals: - - spec: intel-oneapi-mpi@2021.17 + - spec: intel-oneapi-mpi@2021.18 modules: - - mpi/2021.17 + - mpi/2021.18 prefix: /opt/intel/oneapi intel-oneapi-tbb: buildable: false externals: - - spec: intel-oneapi-tbb@2022.3 + - spec: intel-oneapi-tbb@2023.1 prefix: /opt/intel/oneapi modules: - - tbb/2022.3 - mpich: - buildable: false - openmpi: - buildable: false - mkl: + - tbb/2023.1 + + # Package constraints. + # Avoid duplicates. Unpinned, gmao-swell-env and py-pyresample pull two + # py-dask versions, which splits py-setuptools-scm (8 vs 9) and cascades into + # py-netcdf4, py-xarray, py-zarr and jedi-python-env. The gcc env settles on + # 2024.12.1 without a pin. + py-dask: + require: ['@2024.12.1'] + # Unconstrained, one met concretizes ~graphics. Keep common's +shared-intel. + met: + require: ['+graphics +shared-intel'] + jedi-base-env: + require: [+fftw, +hdf4] + jedi-python-env: + require: [+bufrquery] + eccodes: + require: + - "jp2k=jasper" + jasper: + require: + - +jpeg + curl: + require: ['+nghttp2'] + py-pyyaml: require: - - intel-oneapi-mkl - py-numpy: + - +libyaml + # aws-ubuntu uses the apt qt5 as an external; the container builds qt, and + # qt 5.15 fails to compile with icpx (qfloat16_f16c.c redefinitions). Build it + # with gcc as common/packages_oneapi.yaml suggests. '@5' repeats common's + # requirement, which this one replaces. + qt: require: - - '^intel-oneapi-mkl' + - '@5' + - '%c,cxx=gcc' From ea892e859351c3367bb5fc197da0fb9cf928a8b8 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Fri, 30 Oct 2026 14:03:54 -0600 Subject: [PATCH 12/14] Update .gitmodules: use spack-stack-2.2.0 tags (#2199) --- .gitmodules | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitmodules b/.gitmodules index 1bcbf999c3..387f1dc031 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,8 +1,8 @@ [submodule "spack"] path = spack url = https://github.com/jcsda/spack - branch = release/2.2 + branch = spack-stack-2.2.0 [submodule "repos/builtin"] path = repos/builtin url = https://github.com/jcsda/spack-packages - branch = release/2.2 + branch = spack-stack-2.2.0 From 9b1c8857b6ba450d6e4abd1404fe3577437e79c2 Mon Sep 17 00:00:00 2001 From: RatkoVasic-NOAA <37597874+RatkoVasic-NOAA@users.noreply.github.com> Date: Tue, 3 Nov 2026 14:41:17 -0500 Subject: [PATCH 13/14] [release/2.2]: Gaea C7: fix cairo problem and add loading INTEL before MPI. (#2205) --- configs/sites/tier1/gaea-c7/packages.yaml | 3 +++ configs/sites/tier1/gaea-c7/packages_oneapi-2025.3.1.yaml | 1 + configs/sites/tier1/gaea-c7/packages_oneapi-2026.0.0.yaml | 1 + 3 files changed, 5 insertions(+) diff --git a/configs/sites/tier1/gaea-c7/packages.yaml b/configs/sites/tier1/gaea-c7/packages.yaml index e7c6f88d9f..98e0569e08 100644 --- a/configs/sites/tier1/gaea-c7/packages.yaml +++ b/configs/sites/tier1/gaea-c7/packages.yaml @@ -5,6 +5,9 @@ packages: zlib: require:: - '@1.2.13' + cairo: + # /usr/lib64/libgdk-x11-2.0.so: undefined reference to `cairo_xlib_surface_set_size' + require: +X autoconf: externals: diff --git a/configs/sites/tier1/gaea-c7/packages_oneapi-2025.3.1.yaml b/configs/sites/tier1/gaea-c7/packages_oneapi-2025.3.1.yaml index bb16e66229..b1ae54bd05 100644 --- a/configs/sites/tier1/gaea-c7/packages_oneapi-2025.3.1.yaml +++ b/configs/sites/tier1/gaea-c7/packages_oneapi-2025.3.1.yaml @@ -42,6 +42,7 @@ packages: - spec: intel-oneapi-mkl@2025.3.1 prefix: /sw/gaea-c7/spack/envs/env_26.06/opt/intel-oneapi-compilers-2025.3.1/intel-oneapi-mkl-2025.3.1-xxdwsuy4jxt34coagmd66yhi33u2fy6v modules: + - oneapi/2025.3.1 - intel-oneapi-mkl/2025.3.1 gcc: buildable: False diff --git a/configs/sites/tier1/gaea-c7/packages_oneapi-2026.0.0.yaml b/configs/sites/tier1/gaea-c7/packages_oneapi-2026.0.0.yaml index edc8f479a7..fa6b161ef2 100644 --- a/configs/sites/tier1/gaea-c7/packages_oneapi-2026.0.0.yaml +++ b/configs/sites/tier1/gaea-c7/packages_oneapi-2026.0.0.yaml @@ -42,6 +42,7 @@ packages: - spec: intel-oneapi-mkl@2026.0.0 prefix: /sw/gaea-c7/spack/envs/env_26.06/opt/intel-oneapi-compilers-2026.0.0/intel-oneapi-mkl-2026.0.0-ferm7nss4cxphhlu5rbcxzwblz7xyg53 modules: + - oneapi/2026.0.0 - intel-oneapi-mkl/2026.0.0 gcc: buildable: False From 7c6814fe2103cf2c52c2647ea8d6ef533cfa3678 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Wed, 4 Nov 2026 08:02:35 -0700 Subject: [PATCH 14/14] Update .gitmodules and submodule pointer for repos/builtin for code review and testing --- .gitmodules | 3 ++- repos/builtin | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.gitmodules b/.gitmodules index 2f6cb05661..1d98385d5e 100644 --- a/.gitmodules +++ b/.gitmodules @@ -5,4 +5,5 @@ [submodule "repos/builtin"] path = repos/builtin url = https://github.com/jcsda/spack-packages - branch = spack-stack-dev + #branch = spack-stack-dev + branch = feature/merge_rel22_to_spackstackdev diff --git a/repos/builtin b/repos/builtin index 7e51ad8614..5be3fc546f 160000 --- a/repos/builtin +++ b/repos/builtin @@ -1 +1 @@ -Subproject commit 7e51ad86144fb59b25dab27931284ba71a70fe89 +Subproject commit 5be3fc546f0a0dafa74dc4cd32ba55eb92b5a32b