diff --git a/.gitignore b/.gitignore index cc8a36bda..473c2af6e 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,8 @@ doc/build deploy_logs/ logs/ log.* +spack-install.* +tmp-bootstrap-mirror/ +spack.nas.* +spack.nas-toss5.* +spack.discover* diff --git a/configs/common/packages.yaml b/configs/common/packages.yaml index 159a470fa..bf657fb73 100644 --- a/configs/common/packages.yaml +++ b/configs/common/packages.yaml @@ -261,7 +261,7 @@ packages: - generator=make libpng: require: - - '@1.6.37' + - '@1.6.55' - +pic # libtirpc +gssapi doesn't build with oneapi@2025 # https://github.com/JCSDA/spack-stack/issues/1441 @@ -414,7 +414,8 @@ packages: # https://github.com/JCSDA/spack-stack/issues/1276 py-matplotlib: require: - - '@3.7.4' + - '@3.10.8' + #- '@3.7.4' #- '@3.8.0' # https://github.com/JCSDA/spack-stack/issues/1971 py-netcdf4: diff --git a/configs/sites/tier1/discover-scu17/mirrors.yaml b/configs/sites/tier1/discover-scu17/mirrors.yaml deleted file mode 100644 index ed2a0f626..000000000 --- a/configs/sites/tier1/discover-scu17/mirrors.yaml +++ /dev/null @@ -1,2 +0,0 @@ -mirrors: - local-source: file:///discover/swdev/jcsda/spack-stack/source-cache diff --git a/configs/sites/tier1/discover-scu17/packages_gcc.yaml b/configs/sites/tier1/discover-scu17/packages_gcc.yaml deleted file mode 100644 index bb5679dc4..000000000 --- a/configs/sites/tier1/discover-scu17/packages_gcc.yaml +++ /dev/null @@ -1,27 +0,0 @@ -packages: - all: - compiler:: [gcc@13.4.0] - providers: - mpi:: [openmpi@4.1.6] - mpi: - buildable: False - openmpi: - buildable: False - externals: - - spec: openmpi@4.1.6 ~cuda~cxx~cxx_exceptions~java~memchecker+pmi~static~wrapper-rpath+two_level_namespace - fabrics=ucx schedulers=slurm - prefix: /discover/swdev/gmao_SIteam/MPI/openmpi/4.1.6-SLES15/gcc-13.4.0 - modules: - - mpi/openmpi/4.1.6/gcc-13.4.0 - gcc: - buildable: false - externals: - - spec: gcc@13.4.0 languages='c,c++,fortran' - prefix: /discover/swdev/gmao_SIteam/other/SLES15.4/GCC/13.4.0 - modules: - - comp/gcc/13.4.0 - extra_attributes: - compilers: - c: /discover/swdev/gmao_SIteam/other/SLES15.4/GCC/13.4.0/bin/gcc - cxx: /discover/swdev/gmao_SIteam/other/SLES15.4/GCC/13.4.0/bin/g++ - fortran: /discover/swdev/gmao_SIteam/other/SLES15.4/GCC/13.4.0/bin/gfortran diff --git a/configs/sites/tier1/discover-scu17/config.yaml b/configs/sites/tier1/discover/config.yaml similarity index 100% rename from configs/sites/tier1/discover-scu17/config.yaml rename to configs/sites/tier1/discover/config.yaml diff --git a/configs/sites/tier1/discover/mirrors.yaml b/configs/sites/tier1/discover/mirrors.yaml new file mode 100644 index 000000000..1cd8ba7db --- /dev/null +++ b/configs/sites/tier1/discover/mirrors.yaml @@ -0,0 +1,8 @@ +mirrors: + local-source: + url: file:///discover/swdev/jcsda/spack-stack/source-cache + binary: false + local-binary: + url: file:///discover/swdev/jcsda/spack-stack/build-cache + binary: true + diff --git a/configs/sites/tier1/discover-scu17/modules.yaml b/configs/sites/tier1/discover/modules.yaml similarity index 100% rename from configs/sites/tier1/discover-scu17/modules.yaml rename to configs/sites/tier1/discover/modules.yaml diff --git a/configs/sites/tier1/discover-scu17/packages.yaml b/configs/sites/tier1/discover/packages.yaml similarity index 86% rename from configs/sites/tier1/discover-scu17/packages.yaml rename to configs/sites/tier1/discover/packages.yaml index 331a68b39..0f2f94601 100644 --- a/configs/sites/tier1/discover-scu17/packages.yaml +++ b/configs/sites/tier1/discover/packages.yaml @@ -31,8 +31,10 @@ packages: prefix: /usr cmake: externals: - - spec: cmake@3.28.2 - prefix: /usr/local/other/cmake/3.28.2 + - spec: cmake@3.30.3 + prefix: /usr/local/other/cmake/3.30.3 + - spec: cmake@4.3.0 + prefix: /usr/local/other/cmake/4.3.0 coreutils: externals: - spec: coreutils@8.32 @@ -69,15 +71,10 @@ packages: externals: - spec: git@2.35.3+tcltk prefix: /usr - - spec: git@2.42.0+tcltk - prefix: /usr/local/other/git/2.42.0/gcc-7.5.0 + - spec: git@2.49.0+tcltk + prefix: /usr/local/other/git/2.49.0/gcc-7.5.0 modules: - - git/2.42.0 - # DH* - #- spec: git@2.42.0~tcltk - # prefix: /usr/local/other/git/2.42.0/gcc-7.5.0/libexec/git-core - # modules: - # - git/2.42.0 + - git/2.49.0 git-lfs: externals: - spec: git-lfs@3.4.0 @@ -143,3 +140,12 @@ packages: externals: - spec: texinfo@6.5 prefix: /usr + wget: + externals: + - spec: wget@1.20.3 + prefix: /usr + zlib: + buildable: False + externals: + - spec: zlib@1.2.11 + prefix: /usr diff --git a/configs/sites/tier1/discover/packages_gcc-14.2.0.yaml b/configs/sites/tier1/discover/packages_gcc-14.2.0.yaml new file mode 100644 index 000000000..2e0723aff --- /dev/null +++ b/configs/sites/tier1/discover/packages_gcc-14.2.0.yaml @@ -0,0 +1,48 @@ +packages: + all: + require: + - any_of: ['%gcc@=14.2.0'] + when: '%gcc' + mpi: + buildable: false + require: + - "openmpi@4.1.6" + gcc: + buildable: false + externals: + - spec: gcc@14.2.0 languages:='c,c++,fortran' + prefix: /usr/local/other/gcc/gcc-14.2.0 + modules: + - comp/gcc/14.2.0 + extra_attributes: + compilers: + c: /usr/local/other/gcc/gcc-14.2.0/bin/gcc + cxx: /usr/local/other/gcc/gcc-14.2.0/bin/g++ + fortran: /usr/local/other/gcc/gcc-14.2.0/bin/gfortran + openmpi: + buildable: false + externals: + - spec: openmpi@4.1.6 ~cuda~cxx~cxx_exceptions~java~memchecker+pmi~static~wrapper-rpath+lustre~internal-hwloc+two_level_namespace fabrics=ucx schedulers=slurm %gcc@14.2.0 + prefix: /discover/swdev/gmao_SIteam/MPI/openmpi/4.1.6-SLES15/gcc-14.2.0 + modules: + - mpi/openmpi/4.1.6/gcc-14.2.0 + extra_attributes: + environment: + set: + OMPI_MCA_orte_tmpdir_base: /tmp + OMP_STACKSIZE: 1G + OMPI_MCA_mca_base_component_show_load_errors: 0 + PMIX_MCA_mca_base_component_show_load_errors: 0 + OMPI_MCA_coll_hcoll_enable: 0 + OPENMPI: /discover/swdev/gmao_SIteam/MPI/openmpi/4.1.6-SLES15/gcc-14.2.0 + prepend_path: + PATH: /discover/swdev/gmao_SIteam/MPI/openmpi/4.1.6-SLES15/gcc-14.2.0/bin + LIBRARY_PATH: /discover/swdev/gmao_SIteam/MPI/openmpi/4.1.6-SLES15/gcc-14.2.0/lib + LD_LIBRARY_PATH: /discover/swdev/gmao_SIteam/MPI/openmpi/4.1.6-SLES15/gcc-14.2.0/lib + INCLUDE: /discover/swdev/gmao_SIteam/MPI/openmpi/4.1.6-SLES15/gcc-14.2.0/include + MANPATH: /discover/swdev/gmao_SIteam/MPI/openmpi/4.1.6-SLES15/gcc-14.2.0 + + # Testing shows we now need to pass in -lm for GCC + p4est: + require: + - 'ldflags="-lm"' diff --git a/configs/sites/tier1/discover-scu17/packages_oneapi.yaml b/configs/sites/tier1/discover/packages_oneapi-2024.2.0.yaml similarity index 71% rename from configs/sites/tier1/discover-scu17/packages_oneapi.yaml rename to configs/sites/tier1/discover/packages_oneapi-2024.2.0.yaml index 5bf4354c4..68ef40e65 100644 --- a/configs/sites/tier1/discover-scu17/packages_oneapi.yaml +++ b/configs/sites/tier1/discover/packages_oneapi-2024.2.0.yaml @@ -1,13 +1,16 @@ packages: all: - prefer: - - '%oneapi' - providers: - mpi:: [intel-oneapi-mpi@2021.13] + require: + - any_of: ['%intel-oneapi-compilers@=2024.2.0'] + when: '%intel-oneapi-compilers' + - any_of: ['%gcc@=12.3.0'] + when: '%gcc' mpi: - buildable: False + buildable: false + require: + - "intel-oneapi-mpi@2021.13" intel-oneapi-compilers: - buildable: False + buildable: false externals: - spec: intel-oneapi-compilers@2024.2.0 prefix: /usr/local/intel/oneapi/2024 @@ -18,32 +21,18 @@ packages: c: /usr/local/intel/oneapi/2024/compiler/2024.2/bin/icx cxx: /usr/local/intel/oneapi/2024/compiler/2024.2/bin/icpx fortran: /usr/local/intel/oneapi/2024/compiler/2024.2/bin/ifort - f77: /usr/local/intel/oneapi/2024/compiler/2024.2/bin/ifort environment: - set: - # Force ifort not ifx - INTEL_COMPILER_TYPE: 'RECOMMENDED' prepend_path: PATH: /usr/local/other/gcc/12.3.0/bin CPATH: /usr/local/other/gcc/12.3.0/include - LD_LIBRARY_PATH: '/usr/local/intel/oneapi/2024/compiler/2024.2/lib/:/usr/local/other/gcc/12.3.0/lib64' + LD_LIBRARY_PATH: '/usr/local/intel/oneapi/2024/compiler/2024.2/compiler/lib:/usr/local/other/gcc/12.3.0/lib64' extra_rpaths: - /usr/local/other/gcc/12.3.0/lib64 - intel-oneapi-mpi: - buildable: False - externals: - - spec: intel-oneapi-mpi@2021.13 - prefix: /usr/local/intel/oneapi/2024 - modules: - - mpi/impi/2021.13 - intel-oneapi-mkl: - buildable: false - externals: - - spec: intel-oneapi-mkl@2024.2.0 - prefix: /usr/local/intel/oneapi/2024 + - /usr/local/other/gcc/12.3.0/lib gcc: + buildable: false externals: - - spec: gcc@12.3.0 languages:='c,c++,fortran' + - spec: gcc@12.3.0 languages:='c,c++' prefix: /usr/local/other/gcc/12.3.0 modules: - comp/gcc/12.3.0 @@ -52,6 +41,15 @@ packages: c: /usr/local/other/gcc/12.3.0/bin/gcc cxx: /usr/local/other/gcc/12.3.0/bin/g++ fortran: /usr/local/other/gcc/12.3.0/bin/gfortran - flags: {} - environment: {} - extra_rpaths: [] + intel-oneapi-mpi: + buildable: false + externals: + - spec: intel-oneapi-mpi@2021.13 + prefix: /usr/local/intel/oneapi/2024 + modules: + - mpi/impi/2021.13 + intel-oneapi-mkl: + buildable: false + externals: + - spec: intel-oneapi-mkl@2024.2 + prefix: /usr/local/intel/oneapi/2024 diff --git a/configs/sites/tier1/discover/packages_oneapi-2025.3.0.yaml b/configs/sites/tier1/discover/packages_oneapi-2025.3.0.yaml new file mode 100644 index 000000000..c0fdbedfe --- /dev/null +++ b/configs/sites/tier1/discover/packages_oneapi-2025.3.0.yaml @@ -0,0 +1,55 @@ +packages: + all: + require: + - any_of: ['%intel-oneapi-compilers@=2025.3.0'] + when: '%intel-oneapi-compilers' + - any_of: ['%gcc@=12.3.0'] + when: '%gcc' + mpi: + buildable: false + require: + - "intel-oneapi-mpi@2021.17" + intel-oneapi-compilers: + buildable: false + externals: + - spec: intel-oneapi-compilers@2025.3.0 + prefix: /usr/local/intel/oneapi/2025 + modules: + - comp/intel/2025.3.0 + extra_attributes: + compilers: + c: /usr/local/intel/oneapi/2025/compiler/2025.3/bin/icx + cxx: /usr/local/intel/oneapi/2025/compiler/2025.3/bin/icpx + fortran: /usr/local/intel/oneapi/2025/compiler/2025.3/bin/ifx + environment: + prepend_path: + PATH: /usr/local/other/gcc/12.3.0/bin + CPATH: /usr/local/other/gcc/12.3.0/include + LD_LIBRARY_PATH: '/usr/local/intel/oneapi/2025/compiler/2025.3/compiler/lib:/usr/local/other/gcc/12.3.0/lib64' + extra_rpaths: + - /usr/local/other/gcc/12.3.0/lib64 + - /usr/local/other/gcc/12.3.0/lib + gcc: + buildable: false + externals: + - spec: gcc@12.3.0 languages:='c,c++' + prefix: /usr/local/other/gcc/12.3.0 + modules: + - comp/gcc/12.3.0 + extra_attributes: + compilers: + c: /usr/local/other/gcc/12.3.0/bin/gcc + cxx: /usr/local/other/gcc/12.3.0/bin/g++ + fortran: /usr/local/other/gcc/12.3.0/bin/gfortran + intel-oneapi-mpi: + buildable: false + externals: + - spec: intel-oneapi-mpi@2021.17 + prefix: /usr/local/intel/oneapi/2025 + modules: + - mpi/impi/2021.17 + intel-oneapi-mkl: + buildable: false + externals: + - spec: intel-oneapi-mkl@2025.3 + prefix: /usr/local/intel/oneapi/2025 diff --git a/configs/sites/tier1/nas-toss5/README.md b/configs/sites/tier1/nas-toss5/README.md index 208d88ced..bec7fd3dd 100644 --- a/configs/sites/tier1/nas-toss5/README.md +++ b/configs/sites/tier1/nas-toss5/README.md @@ -161,7 +161,7 @@ spack mirror create -a -d /swbuild/gmao_SIteam/spack-stack/source-cache Rust packages frequently require network access during build. Pre-fetch their dependencies: ```bash -export CARGO_HOME=/swbuild/gmao_SIteam/spack-stack/cargo-cache +export CARGO_HOME=/swbuild/gmao_SIteam/spack-stack/cargo-mirror ../../util/fetch_cargo_deps.py ``` @@ -174,7 +174,7 @@ export CARGO_HOME=/swbuild/gmao_SIteam/spack-stack/cargo-cache Run installation on a **compute node**: ```bash -export CARGO_HOME=/swbuild/gmao_SIteam/spack-stack/cargo-cache +export CARGO_HOME=/swbuild/gmao_SIteam/spack-stack/cargo-mirror spack install -j 16 --verbose --fail-fast --show-log-on-error --no-check-signature 2>&1 | tee log.install ; bell ``` @@ -229,17 +229,17 @@ Typical commands were: ```bash # Step 1 (compute node) -export CARGO_HOME=/swbuild/gmao_SIteam/spack-stack/cargo-cache +export CARGO_HOME=/swbuild/gmao_SIteam/spack-stack/cargo-mirror spack install -j 16 --verbose --fail-fast --show-log-on-error --no-check-signature \ --only dependencies py-cryptography py-maturin py-rpds-py ecflow 2>&1 | tee log.install.deps-for-rust-and-ecflow ; bell # Step 2 (athfe login node) -export CARGO_HOME=/swbuild/gmao_SIteam/spack-stack/cargo-cache +export CARGO_HOME=/swbuild/gmao_SIteam/spack-stack/cargo-mirror spack install -j 2 -p 1 --verbose --fail-fast --show-log-on-error --no-check-signature \ py-cryptography py-maturin py-rpds-py ecflow 2>&1 | tee log.install.rust-and-ecflow ; bell # Step 3 (compute node) -export CARGO_HOME=/swbuild/gmao_SIteam/spack-stack/cargo-cache +export CARGO_HOME=/swbuild/gmao_SIteam/spack-stack/cargo-mirror spack install -j 16 --verbose --fail-fast --show-log-on-error --no-check-signature 2>&1 | tee log.install.after-cargo ; bell ``` diff --git a/configs/sites/tier1/nas/README.md b/configs/sites/tier1/nas/README.md index fb6fdf2b2..7e1e99367 100644 --- a/configs/sites/tier1/nas/README.md +++ b/configs/sites/tier1/nas/README.md @@ -113,8 +113,8 @@ cd envs/ue-oneapi-2024.2.0 ### GCC Environment ```bash -spack stack create env --name ue-gcc-13.2.0 --template unified-dev --site nas --compiler=gcc-13.2.0 -cd envs/ue-gcc-13.2.0 +spack stack create env --name ue-gcc-14.2.0 --template unified-dev --site nas --compiler=gcc-14.2.0 +cd envs/ue-gcc-14.2.0 ``` --- @@ -163,7 +163,7 @@ spack mirror create -a -d /swbuild/gmao_SIteam/spack-stack/source-cache Rust packages frequently require network access during build. Pre-fetch their dependencies: ```bash -export CARGO_HOME=/swbuild/gmao_SIteam/spack-stack/cargo-cache +export CARGO_HOME=/swbuild/gmao_SIteam/spack-stack/cargo-mirror ../../util/fetch_cargo_deps.py ``` @@ -176,7 +176,7 @@ export CARGO_HOME=/swbuild/gmao_SIteam/spack-stack/cargo-cache Run installation on a **compute node**: ```bash -export CARGO_HOME=/swbuild/gmao_SIteam/spack-stack/cargo-cache +export CARGO_HOME=/swbuild/gmao_SIteam/spack-stack/cargo-mirror spack install -j 16 --verbose --fail-fast --show-log-on-error --no-check-signature 2>&1 | tee log.install ; bell ``` @@ -231,17 +231,17 @@ Typical commands were: ```bash # Step 1 (compute node) -export CARGO_HOME=/swbuild/gmao_SIteam/spack-stack/cargo-cache +export CARGO_HOME=/swbuild/gmao_SIteam/spack-stack/cargo-mirror spack install -j 16 --verbose --fail-fast --show-log-on-error --no-check-signature \ --only dependencies py-cryptography py-maturin py-rpds-py ecflow 2>&1 | tee log.install.deps-for-rust-and-ecflow ; bell # Step 2 (afe login node) -export CARGO_HOME=/swbuild/gmao_SIteam/spack-stack/cargo-cache +export CARGO_HOME=/swbuild/gmao_SIteam/spack-stack/cargo-mirror spack install -j 2 -p 1 --verbose --fail-fast --show-log-on-error --no-check-signature \ py-cryptography py-maturin py-rpds-py ecflow 2>&1 | tee log.install.rust-and-ecflow ; bell # Step 3 (compute node) -export CARGO_HOME=/swbuild/gmao_SIteam/spack-stack/cargo-cache +export CARGO_HOME=/swbuild/gmao_SIteam/spack-stack/cargo-mirror spack install -j 16 --verbose --fail-fast --show-log-on-error --no-check-signature 2>&1 | tee log.install.after-cargo ; bell ``` diff --git a/configs/sites/tier1/nas/packages_gcc-13.2.0.yaml b/configs/sites/tier1/nas/packages_gcc-14.2.0.yaml similarity index 69% rename from configs/sites/tier1/nas/packages_gcc-13.2.0.yaml rename to configs/sites/tier1/nas/packages_gcc-14.2.0.yaml index f2d3fb4b7..9dea27286 100644 --- a/configs/sites/tier1/nas/packages_gcc-13.2.0.yaml +++ b/configs/sites/tier1/nas/packages_gcc-14.2.0.yaml @@ -1,7 +1,7 @@ packages: all: require: - - any_of: ['%gcc@=13.2.0'] + - any_of: ['%gcc@=14.2.0'] when: '%gcc' mpi: buildable: false @@ -10,26 +10,26 @@ packages: gcc: buildable: false externals: - - spec: gcc@13.2.0 languages:='c,c++,fortran' - prefix: /nobackup/gmao_SIteam/gcc/gcc-13.2.0 + - spec: gcc@14.2.0 languages:='c,c++,fortran' + prefix: /nobackup/gmao_SIteam/gcc/gcc-14.2.0 modules: - - comp-gcc/13.2.0 + - comp-gcc/14.2.0 extra_attributes: compilers: - c: /nobackup/gmao_SIteam/gcc/gcc-13.2.0/bin/gcc - cxx: /nobackup/gmao_SIteam/gcc/gcc-13.2.0/bin/g++ - fortran: /nobackup/gmao_SIteam/gcc/gcc-13.2.0/bin/gfortran + c: /nobackup/gmao_SIteam/gcc/gcc-14.2.0/bin/gcc + cxx: /nobackup/gmao_SIteam/gcc/gcc-14.2.0/bin/g++ + fortran: /nobackup/gmao_SIteam/gcc/gcc-14.2.0/bin/gfortran openmpi: buildable: false externals: - - spec: openmpi@4.1.8 ~cuda~cxx~cxx_exceptions~java~memchecker+pmi~static~wrapper-rpath+lustre~internal-hwloc+two_level_namespace fabrics=ucx schedulers=tm %gcc@13.2.0 - prefix: /nobackup/gmao_SIteam/MPI/openmpi/4.1.8/gcc-13.2.0 + - spec: openmpi@4.1.8 ~cuda~cxx~cxx_exceptions~java~memchecker+pmi~static~wrapper-rpath+lustre~internal-hwloc+two_level_namespace fabrics=ucx schedulers=tm %gcc@14.2.0 + prefix: /nobackup/gmao_SIteam/MPI/openmpi/4.1.8/gcc-14.2.0 modules: - - mpi-openmpi/4.1.8/gcc/13.2.0 + - mpi-openmpi/4.1.8/gcc/14.2.0 extra_attributes: environment: set: - OPAL_PREFIX: /nobackup/gmao_SIteam/MPI/openmpi/4.1.8/gcc-13.2.0 + OPAL_PREFIX: /nobackup/gmao_SIteam/MPI/openmpi/4.1.8/gcc-14.2.0 UCX_MAX_EAGER_LANES: 1 UCX_MAX_RNDV_LANES: 1 OMPI_MCA_pml: ucx @@ -40,14 +40,14 @@ packages: OMPI_MCA_opal_warn_on_missing_libcuda: 0 UCX_IB_SUBNET_PREFIX: fec0:0000:0000:0008 OMPI_MCA_shmem_mmap_enable_nfs_warning: 0 - OPENMPI: /nobackup/gmao_SIteam/MPI/openmpi/4.1.8/gcc-13.2.0 + OPENMPI: /nobackup/gmao_SIteam/MPI/openmpi/4.1.8/gcc-14.2.0 prepend_path: - PATH: /nobackup/gmao_SIteam/MPI/openmpi/4.1.8/gcc-13.2.0/bin - LIBRARY_PATH: /nobackup/gmao_SIteam/MPI/openmpi/4.1.8/gcc-13.2.0/lib - LD_LIBRARY_PATH: /nobackup/gmao_SIteam/MPI/openmpi/4.1.8/gcc-13.2.0/lib - CPATH: /nobackup/gmao_SIteam/MPI/openmpi/4.1.8/gcc-13.2.0/include - FPATH: /nobackup/gmao_SIteam/MPI/openmpi/4.1.8/gcc-13.2.0/include - MANPATH: /nobackup/gmao_SIteam/MPI/openmpi/4.1.8/gcc-13.2.0/share/man + PATH: /nobackup/gmao_SIteam/MPI/openmpi/4.1.8/gcc-14.2.0/bin + LIBRARY_PATH: /nobackup/gmao_SIteam/MPI/openmpi/4.1.8/gcc-14.2.0/lib + LD_LIBRARY_PATH: /nobackup/gmao_SIteam/MPI/openmpi/4.1.8/gcc-14.2.0/lib + CPATH: /nobackup/gmao_SIteam/MPI/openmpi/4.1.8/gcc-14.2.0/include + FPATH: /nobackup/gmao_SIteam/MPI/openmpi/4.1.8/gcc-14.2.0/include + MANPATH: /nobackup/gmao_SIteam/MPI/openmpi/4.1.8/gcc-14.2.0/share/man # Testing shows we now need to pass in -lm for GCC p4est: diff --git a/configs/sites/tier2/discover-gmao/config.yaml b/configs/sites/tier2/discover-gmao/config.yaml new file mode 100644 index 000000000..b7f1f5624 --- /dev/null +++ b/configs/sites/tier2/discover-gmao/config.yaml @@ -0,0 +1,9 @@ +config: + build_jobs: 6 + + # Overrides for spack build and staging areas to speed up builds + # and avoid errors with Lustre file locking and xattr issues + build_stage: /discover/nobackup/projects/gmao/SIteam/spack-stack/cache/build_stage + test_stage: /discover/nobackup/projects/gmao/SIteam/spack-stack/cache/test_stage + source_cache: /discover/nobackup/projects/gmao/SIteam/spack-stack/cache/source_cache + misc_cache: /discover/nobackup/projects/gmao/SIteam/spack-stack/cache/misc_cache diff --git a/configs/sites/tier2/discover-gmao/mirrors.yaml b/configs/sites/tier2/discover-gmao/mirrors.yaml new file mode 100644 index 000000000..9c06031b7 --- /dev/null +++ b/configs/sites/tier2/discover-gmao/mirrors.yaml @@ -0,0 +1,8 @@ +mirrors: + local-source: + url: file:///discover/nobackup/projects/gmao/SIteam/spack-stack/source-cache + binary: false + local-binary: + url: file:///discover/nobackup/projects/gmao/SIteam/spack-stack/build-cache + binary: true + diff --git a/configs/sites/tier2/discover-gmao/modules.yaml b/configs/sites/tier2/discover-gmao/modules.yaml new file mode 100644 index 000000000..4e9771f38 --- /dev/null +++ b/configs/sites/tier2/discover-gmao/modules.yaml @@ -0,0 +1,4 @@ +modules: + default: + enable:: + - lmod diff --git a/configs/sites/tier2/discover-gmao/packages.yaml b/configs/sites/tier2/discover-gmao/packages.yaml new file mode 100644 index 000000000..0f2f94601 --- /dev/null +++ b/configs/sites/tier2/discover-gmao/packages.yaml @@ -0,0 +1,151 @@ +packages: +### Modification of common packages + ewok-env: + require: + - '+ecflow' + # Problems building shared hdf-eos2 with Intel, not needed + hdf-eos2: + variants: ~shared + met: + variants: +python +grib2 +graphics +lidar2nc +modis +### All other external packages listed alphabetically + autoconf: + externals: + - spec: autoconf@2.69 + prefix: /usr + automake: + externals: + - spec: automake@1.15.1 + prefix: /usr + binutils: + externals: + - spec: binutils@2.41.0 + prefix: /usr + bison: + externals: + - spec: bison@3.0.4 + prefix: /usr + ccache: + externals: + - spec: ccache@3.4.2 + prefix: /usr + cmake: + externals: + - spec: cmake@3.30.3 + prefix: /usr/local/other/cmake/3.30.3 + - spec: cmake@4.3.0 + prefix: /usr/local/other/cmake/4.3.0 + coreutils: + externals: + - spec: coreutils@8.32 + prefix: /usr + curl: + externals: + - spec: curl@8.0.1+gssapi+ldap+nghttp2 + prefix: /usr + cvs: + externals: + - spec: cvs@1.12.13 + prefix: /usr + diffutils: + externals: + - spec: diffutils@3.6 + prefix: /usr + findutils: + externals: + - spec: findutils@4.8.0 + prefix: /usr + flex: + externals: + - spec: flex@2.6.4+lex + prefix: /usr + gawk: + externals: + - spec: gawk@4.2.1 + prefix: /usr + gettext: + externals: + - spec: gettext@0.20.2 + prefix: /usr + git: + externals: + - spec: git@2.35.3+tcltk + prefix: /usr + - spec: git@2.49.0+tcltk + prefix: /usr/local/other/git/2.49.0/gcc-7.5.0 + modules: + - git/2.49.0 + git-lfs: + externals: + - spec: git-lfs@3.4.0 + prefix: /usr/local/other/git-lfs/3.4.0 + gmake: + externals: + - spec: gmake@4.2.1 + prefix: /usr + grep: + externals: + - spec: grep@3.1 + prefix: /usr + groff: + externals: + - spec: groff@1.22.4 + prefix: /usr + libtool: + externals: + - spec: libtool@2.4.6 + prefix: /usr + m4: + externals: + - spec: m4@1.4.18 + prefix: /usr + openssh: + externals: + - spec: openssh@8.4p1 + prefix: /usr + openssl: + externals: + - spec: openssl@1.1.1l + prefix: /usr + perl: + externals: + - spec: perl@5.26.1~cpanm+opcode+open+shared+threads + prefix: /usr + pkg-config: + externals: + - spec: pkg-config@0.29.2 + prefix: /usr + qt: + buildable: False + externals: + - spec: qt@5.15.10 + prefix: /usr/local/other/xpdf/4.04/Deps + sed: + externals: + - spec: sed@4.4 + prefix: /usr + subversion: + externals: + - spec: subversion@1.14.1 + prefix: /usr + swig: + externals: + - spec: swig@3.0.12 + prefix: /usr + tar: + externals: + - spec: tar@1.34 + prefix: /usr + texinfo: + externals: + - spec: texinfo@6.5 + prefix: /usr + wget: + externals: + - spec: wget@1.20.3 + prefix: /usr + zlib: + buildable: False + externals: + - spec: zlib@1.2.11 + prefix: /usr diff --git a/configs/sites/tier2/discover-gmao/packages_gcc-15.2.0.yaml b/configs/sites/tier2/discover-gmao/packages_gcc-15.2.0.yaml new file mode 100644 index 000000000..cc82d25e6 --- /dev/null +++ b/configs/sites/tier2/discover-gmao/packages_gcc-15.2.0.yaml @@ -0,0 +1,59 @@ +packages: + all: + require: + - any_of: ['%gcc@=15.2.0'] + when: '%gcc' + mpi: + buildable: false + require: + - "openmpi@5.0.10" + gcc: + buildable: false + externals: + - spec: gcc@15.2.0 languages:='c,c++,fortran' + prefix: /usr/local/other/gcc/gcc-15.2.0 + modules: + - comp/gcc/15.2.0 + extra_attributes: + compilers: + c: /usr/local/other/gcc/gcc-15.2.0/bin/gcc + cxx: /usr/local/other/gcc/gcc-15.2.0/bin/g++ + fortran: /usr/local/other/gcc/gcc-15.2.0/bin/gfortran + openmpi: + buildable: false + externals: + - spec: openmpi@5.0.10 ~cuda~cxx~cxx_exceptions~java~memchecker+pmi~static~wrapper-rpath+lustre~internal-hwloc+two_level_namespace fabrics=ucx schedulers=slurm %gcc@15.2.0 + prefix: /discover/swdev/gmao_SIteam/MPI/openmpi/5.0.10/gcc-15.2.0 + modules: + - mpi/openmpi/5.0.10/gcc-15.2.0 + extra_attributes: + environment: + set: + OMPI_MCA_orte_tmpdir_base: /tmp + OMP_STACKSIZE: 1G + OMPI_MCA_mca_base_component_show_load_errors: 0 + PMIX_MCA_mca_base_component_show_load_errors: 0 + OMPI_MCA_coll_hcoll_enable: 0 + OPENMPI: /discover/swdev/gmao_SIteam/MPI/openmpi/5.0.10/gcc-15.2.0 + prepend_path: + PATH: /discover/swdev/gmao_SIteam/MPI/openmpi/5.0.10/gcc-15.2.0/bin + LIBRARY_PATH: /discover/swdev/gmao_SIteam/MPI/openmpi/5.0.10/gcc-15.2.0/lib + LD_LIBRARY_PATH: /discover/swdev/gmao_SIteam/MPI/openmpi/5.0.10/gcc-15.2.0/lib + INCLUDE: /discover/swdev/gmao_SIteam/MPI/openmpi/5.0.10/gcc-15.2.0/include + MANPATH: /discover/swdev/gmao_SIteam/MPI/openmpi/5.0.10/gcc-15.2.0 + + # Testing shows we now need to pass in -lm for GCC + p4est: + require: + - 'ldflags="-lm"' + + # GCC 15 defaults to -std=c23 where 'false' is a keyword; parallel-netcdf 1.12.3 + # uses 'false' as an enum constant in ncmpidump.h, so force C11 mode. + parallel-netcdf: + require: + - 'cflags="-std=gnu11"' + + # ncurses 6.5 fails with GCC 15 due to NCURSES_BOOL conversion issues; 6.6 fixes this. + ncurses: + require: + - '@6.6' diff --git a/configs/sites/tier2/discover-gmao/packages_oneapi-2024.2.0.yaml b/configs/sites/tier2/discover-gmao/packages_oneapi-2024.2.0.yaml new file mode 100644 index 000000000..68ef40e65 --- /dev/null +++ b/configs/sites/tier2/discover-gmao/packages_oneapi-2024.2.0.yaml @@ -0,0 +1,55 @@ +packages: + all: + require: + - any_of: ['%intel-oneapi-compilers@=2024.2.0'] + when: '%intel-oneapi-compilers' + - any_of: ['%gcc@=12.3.0'] + when: '%gcc' + mpi: + buildable: false + require: + - "intel-oneapi-mpi@2021.13" + intel-oneapi-compilers: + buildable: false + externals: + - spec: intel-oneapi-compilers@2024.2.0 + prefix: /usr/local/intel/oneapi/2024 + modules: + - comp/intel/2024.2.0 + extra_attributes: + compilers: + c: /usr/local/intel/oneapi/2024/compiler/2024.2/bin/icx + cxx: /usr/local/intel/oneapi/2024/compiler/2024.2/bin/icpx + fortran: /usr/local/intel/oneapi/2024/compiler/2024.2/bin/ifort + environment: + prepend_path: + PATH: /usr/local/other/gcc/12.3.0/bin + CPATH: /usr/local/other/gcc/12.3.0/include + LD_LIBRARY_PATH: '/usr/local/intel/oneapi/2024/compiler/2024.2/compiler/lib:/usr/local/other/gcc/12.3.0/lib64' + extra_rpaths: + - /usr/local/other/gcc/12.3.0/lib64 + - /usr/local/other/gcc/12.3.0/lib + gcc: + buildable: false + externals: + - spec: gcc@12.3.0 languages:='c,c++' + prefix: /usr/local/other/gcc/12.3.0 + modules: + - comp/gcc/12.3.0 + extra_attributes: + compilers: + c: /usr/local/other/gcc/12.3.0/bin/gcc + cxx: /usr/local/other/gcc/12.3.0/bin/g++ + fortran: /usr/local/other/gcc/12.3.0/bin/gfortran + intel-oneapi-mpi: + buildable: false + externals: + - spec: intel-oneapi-mpi@2021.13 + prefix: /usr/local/intel/oneapi/2024 + modules: + - mpi/impi/2021.13 + intel-oneapi-mkl: + buildable: false + externals: + - spec: intel-oneapi-mkl@2024.2 + prefix: /usr/local/intel/oneapi/2024 diff --git a/configs/sites/tier2/discover-gmao/packages_oneapi-2025.3.0.yaml b/configs/sites/tier2/discover-gmao/packages_oneapi-2025.3.0.yaml new file mode 100644 index 000000000..c0fdbedfe --- /dev/null +++ b/configs/sites/tier2/discover-gmao/packages_oneapi-2025.3.0.yaml @@ -0,0 +1,55 @@ +packages: + all: + require: + - any_of: ['%intel-oneapi-compilers@=2025.3.0'] + when: '%intel-oneapi-compilers' + - any_of: ['%gcc@=12.3.0'] + when: '%gcc' + mpi: + buildable: false + require: + - "intel-oneapi-mpi@2021.17" + intel-oneapi-compilers: + buildable: false + externals: + - spec: intel-oneapi-compilers@2025.3.0 + prefix: /usr/local/intel/oneapi/2025 + modules: + - comp/intel/2025.3.0 + extra_attributes: + compilers: + c: /usr/local/intel/oneapi/2025/compiler/2025.3/bin/icx + cxx: /usr/local/intel/oneapi/2025/compiler/2025.3/bin/icpx + fortran: /usr/local/intel/oneapi/2025/compiler/2025.3/bin/ifx + environment: + prepend_path: + PATH: /usr/local/other/gcc/12.3.0/bin + CPATH: /usr/local/other/gcc/12.3.0/include + LD_LIBRARY_PATH: '/usr/local/intel/oneapi/2025/compiler/2025.3/compiler/lib:/usr/local/other/gcc/12.3.0/lib64' + extra_rpaths: + - /usr/local/other/gcc/12.3.0/lib64 + - /usr/local/other/gcc/12.3.0/lib + gcc: + buildable: false + externals: + - spec: gcc@12.3.0 languages:='c,c++' + prefix: /usr/local/other/gcc/12.3.0 + modules: + - comp/gcc/12.3.0 + extra_attributes: + compilers: + c: /usr/local/other/gcc/12.3.0/bin/gcc + cxx: /usr/local/other/gcc/12.3.0/bin/g++ + fortran: /usr/local/other/gcc/12.3.0/bin/gfortran + intel-oneapi-mpi: + buildable: false + externals: + - spec: intel-oneapi-mpi@2021.17 + prefix: /usr/local/intel/oneapi/2025 + modules: + - mpi/impi/2021.17 + intel-oneapi-mkl: + buildable: false + externals: + - spec: intel-oneapi-mkl@2025.3 + prefix: /usr/local/intel/oneapi/2025 diff --git a/configs/sites/tier2/macos.gmao/packages_gcc-15.2.0.yaml.template b/configs/sites/tier2/macos.gmao/packages_gcc-15.2.0.yaml.template index 20c3ac212..4014ae07c 100644 --- a/configs/sites/tier2/macos.gmao/packages_gcc-15.2.0.yaml.template +++ b/configs/sites/tier2/macos.gmao/packages_gcc-15.2.0.yaml.template @@ -35,8 +35,9 @@ packages: # ESMF by default for Darwin.gfortranclang turns off OpenMP, so # we do the same here to avoid build failures. - # We also build as +debug only because testing shows that (so far) - # esmf + Darwin + gfortranclang needs ESMF_BOPT=g to work with GEOS + # We also build as +debug only because esmf ~debug does not work with + # gfortranclang/GEOS on macOS. The geos-gcm-env ~debug spec is removed + # from the environment spack.yaml by batch_install.sh on macOS. esmf: require: - '~openmp' diff --git a/configs/sites/tier2/macos.gmao/packages_nag.yaml.template b/configs/sites/tier2/macos.gmao/packages_nag.yaml.template index 6a6a024af..3bec5923f 100644 --- a/configs/sites/tier2/macos.gmao/packages_nag.yaml.template +++ b/configs/sites/tier2/macos.gmao/packages_nag.yaml.template @@ -53,6 +53,15 @@ packages: require:: - 'fflags="-fpp -mismatch_all"' + # openblas CMake's shared library build passes GCC-specific flags (-x f95-cpp-input) + # that NAG does not support. Build static-only to avoid that code path. + # A fix has been submitted upstream: https://github.com/OpenMathLib/OpenBLAS/pull/5824 + # Once that PR is merged and a new OpenBLAS release is available, this workaround + # can be removed. + openblas: + require: + - '~shared' + gdbm: externals: diff --git a/configs/templates/geos-dev-nag/spack.yaml b/configs/templates/geos-dev-nag/spack.yaml index a61c1007d..5137c6ac5 100644 --- a/configs/templates/geos-dev-nag/spack.yaml +++ b/configs/templates/geos-dev-nag/spack.yaml @@ -8,12 +8,13 @@ spack: include: [] specs: - # NOTE: Do not explicitly add ~debug to the first geos-gcm-env spec below. - # On some platforms (like macOS with gcc-15/gfortranclang), ESMF requires - # +debug to build successfully, which is enforced via the site's packages.yaml. - # Explicitly requesting ~debug here will cause a concretization conflict on those platforms. - - geos-gcm-env ^esmf@=9.0.0b11 - - geos-gcm-env +debug ^esmf@=9.0.0b11 + # Two builds: release (esmf ~debug) and debug (esmf +debug). + # The esmf debug variant is driven solely by the geos-gcm-env debug variant + # (see geos_gcm_env/package.py). On platforms where only +debug is supported + # (e.g., macOS), the site packages.yaml requires geos-gcm-env +debug, which + # causes both specs to resolve to +debug (only one unique build). + - geos-gcm-env ~debug + - geos-gcm-env +debug packages: @@ -21,3 +22,4 @@ spack: esmf: require: - +python + - '@=9.0.0b11' diff --git a/configs/templates/geos-dev/spack.yaml b/configs/templates/geos-dev/spack.yaml index 0e317ec2f..c313a64cc 100644 --- a/configs/templates/geos-dev/spack.yaml +++ b/configs/templates/geos-dev/spack.yaml @@ -8,12 +8,13 @@ spack: include: [] specs: - # NOTE: Do not explicitly add ~debug to the first geos-gcm-env spec below. - # On some platforms (like macOS with gcc-15/gfortranclang), ESMF requires - # +debug to build successfully, which is enforced via the site's packages.yaml. - # Explicitly requesting ~debug here will cause a concretization conflict on those platforms. - - geos-gcm-env ^esmf@=9.0.0b11 - - geos-gcm-env +debug ^esmf@=9.0.0b11 + # Two builds: release (esmf ~debug) and debug (esmf +debug). + # The esmf debug variant is driven solely by the geos-gcm-env debug variant + # (see geos_gcm_env/package.py). On platforms where only +debug is supported + # (e.g., macOS), the site packages.yaml requires geos-gcm-env +debug, which + # causes both specs to resolve to +debug (only one unique build). + - geos-gcm-env ~debug + - geos-gcm-env +debug # Various fms builds - fms@=2024.03 ~gfs_phys constants=GEOS @@ -23,3 +24,4 @@ spack: esmf: require: - +python + - '@=9.0.0b11' diff --git a/repos/spack_stack/spack_repo/spack_stack/packages/geos_gcm_env/package.py b/repos/spack_stack/spack_repo/spack_stack/packages/geos_gcm_env/package.py index 91576e9e5..8985ce217 100644 --- a/repos/spack_stack/spack_repo/spack_stack/packages/geos_gcm_env/package.py +++ b/repos/spack_stack/spack_repo/spack_stack/packages/geos_gcm_env/package.py @@ -44,4 +44,6 @@ class GeosGcmEnv(BundlePackage): depends_on("perl", type="run") + depends_on("py-cmocean", type="run") + # There is no need for install() since there is no code. diff --git a/spack-ext/lib/jcsda-emc/spack-stack/stack/compiler_utils.py b/spack-ext/lib/jcsda-emc/spack-stack/stack/compiler_utils.py index fcb54871c..e3343a816 100644 --- a/spack-ext/lib/jcsda-emc/spack-stack/stack/compiler_utils.py +++ b/spack-ext/lib/jcsda-emc/spack-stack/stack/compiler_utils.py @@ -41,7 +41,7 @@ def get_compiler_choice(string): "cxx,fortran,c=|" + \ "c,fortran,cxx=|" + \ "cxx,c,fortran=|" + \ - "fortran,cxx,c=)(\S+)\s*$" + r"fortran,cxx,c=)(\S+)\s*$" COMPILER_CHOICE_REGEX = re.compile(COMPILER_CHOICE_REGEX_STRING) match = COMPILER_CHOICE_REGEX.match(string) if match: diff --git a/spack-ext/lib/jcsda-emc/spack-stack/stack/stack_env.py b/spack-ext/lib/jcsda-emc/spack-stack/stack/stack_env.py index 36e01e6b8..f666af7ac 100644 --- a/spack-ext/lib/jcsda-emc/spack-stack/stack/stack_env.py +++ b/spack-ext/lib/jcsda-emc/spack-stack/stack/stack_env.py @@ -155,7 +155,7 @@ def _copy_or_merge_includes(self, section, default_in_path, update_in_path, resu # - replace "- packages:" with "packages:" (similar for modules) sanitized_output = re.sub(r"- {}:".format(section), "{}:".format(section), stream.getvalue()) # - get rid of annoying single quotes and !!omap - sanitized_output = re.sub(r"!!omap", "", re.sub(r"'(\S+):':", "\g<1>::", sanitized_output)) + sanitized_output = re.sub(r"!!omap", "", re.sub(r"'(\S+):':", r"\g<1>::", sanitized_output)) with open(result_out_path, "w") as f: f.write(sanitized_output) else: diff --git a/util/gmao/batch_install.sh b/util/gmao/batch_install.sh index c00450964..92813329a 100755 --- a/util/gmao/batch_install.sh +++ b/util/gmao/batch_install.sh @@ -70,6 +70,8 @@ usage() { echo " overrides the default compilers for the site" echo " -N Path to nagfor executable (e.g. /opt/nag/bin/nagfor);" echo " forces NAG stack to be built using this specific compiler" + echo " -a Set PBS/SLURM account (default: s1873);" + echo " overrides the ACCOUNT environment variable" echo " -s Submit 'spack install' to batch scheduler" echo " -t Run tests for specific thirdparty dependencies;" echo " these are currently hardcoded in batch_install.sh" @@ -80,7 +82,7 @@ usage() { echo } -while getopts r:m:d:c:C:N:H:nuesth flag +while getopts r:m:d:c:C:N:H:a:nuesth flag do case "${flag}" in r) @@ -104,6 +106,9 @@ do H) SPACK_STACK_BATCH_HOST_OPT=${OPTARG} ;; + a) + ACCOUNT=${OPTARG} + ;; n) SPACK_STACK_DRY_RUN="true" ;; @@ -137,6 +142,10 @@ echo " SPACK_STACK_UPDATE_DEV_CACHES: ${SPACK_STACK_UPDATE_DEV_CA echo " SPACK_STACK_IGNORE_ENV_EXIST: ${SPACK_STACK_IGNORE_ENV_EXIST:-false}" echo " SPACK_STACK_SUBMIT_TO_SCHEDULER: ${SPACK_STACK_SUBMIT_TO_SCHEDULER:-false}" echo " SPACK_STACK_RUN_TESTS: ${SPACK_STACK_RUN_TESTS:-false}" +echo " ACCOUNT: ${ACCOUNT:-s1873 (default)}" + +# Set default account if not provided via -a or environment +ACCOUNT=${ACCOUNT:-s1873} if [[ -z ${SPACK_STACK_ROLE} ]]; then echo "ERROR, SPACK_STACK_ROLE not defined. Provide -r ROLE as argument" @@ -184,22 +193,32 @@ case ${SPACK_STACK_BATCH_HOST} in SPACK_STACK_BATCH_COMPILERS=("oneapi@=2024.2.0" "oneapi@=2025.3.0" "gcc@=13.2.0") SPACK_STACK_BATCH_TEMPLATES=("unified-dev") SPACK_STACK_MODULE_CHOICE="tcl" - SPACK_STACK_BOOTSTRAP_MIRROR="/swbuild/gmao_SIteam/spack-stack/bootstrap-mirror" + SPACK_STACK_BOOTSTRAP_MIRROR="/swbuild/gmao_SIteam/spack-stack/bootstrap-mirror-toss4" SPACK_STACK_CARGO_MIRROR="/swbuild/gmao_SIteam/spack-stack/cargo-mirror" + SPACK_STACK_ENVIRONMENT_DIRS=${SPACK_STACK_ENVIRONMENT_DIRS:-${PWD}/envs/toss4} ;; nas-toss5) SPACK_STACK_BATCH_COMPILERS=("oneapi@=2024.2.0" "oneapi@=2025.3.0" "gcc@=14.2.1") SPACK_STACK_BATCH_TEMPLATES=("unified-dev") SPACK_STACK_MODULE_CHOICE="tcl" - SPACK_STACK_BOOTSTRAP_MIRROR="/swbuild/gmao_SIteam/spack-stack/bootstrap-mirror" + SPACK_STACK_BOOTSTRAP_MIRROR="/swbuild/gmao_SIteam/spack-stack/bootstrap-mirror-toss5" SPACK_STACK_CARGO_MIRROR="/swbuild/gmao_SIteam/spack-stack/cargo-mirror" + SPACK_STACK_ENVIRONMENT_DIRS=${SPACK_STACK_ENVIRONMENT_DIRS:-${PWD}/envs/toss5} ;; - discover-gmao) - SPACK_STACK_BATCH_COMPILERS=("oneapi@=2024.2.0" "oneapi@=2025.3.0" "gcc@=14.2.1") + discover) + SPACK_STACK_BATCH_COMPILERS=("oneapi@=2024.2.0" "oneapi@=2025.3.0" "gcc@=14.2.0") SPACK_STACK_BATCH_TEMPLATES=("unified-dev") SPACK_STACK_MODULE_CHOICE="lmod" - SPACK_STACK_BOOTSTRAP_MIRROR="/swbuild/gmao_SIteam/spack-stack/bootstrap-mirror" - SPACK_STACK_CARGO_MIRROR="/swbuild/gmao_SIteam/spack-stack/cargo-mirror" + SPACK_STACK_BOOTSTRAP_MIRROR="/discover/swdev/jcsda/spack-stack/bootstrap-mirror" + SPACK_STACK_CARGO_MIRROR="/discover/swdev/jcsda/spack-stack/cargo-mirror" + ;; + discover-gmao) + SPACK_STACK_BATCH_COMPILERS=("oneapi@=2024.2.0" "oneapi@=2025.3.0" "gcc@=15.2.0") + #SPACK_STACK_BATCH_TEMPLATES=("unified-dev") + SPACK_STACK_BATCH_TEMPLATES=("geos-dev") + SPACK_STACK_MODULE_CHOICE="lmod" + SPACK_STACK_BOOTSTRAP_MIRROR="/discover/nobackup/projects/gmao/SIteam/spack-stack/bootstrap-mirror" + SPACK_STACK_CARGO_MIRROR="/discover/nobackup/projects/gmao/SIteam/spack-stack/cargo-mirror" ;; macos.gmao) # Detect NAG Fortran Compiler @@ -216,13 +235,9 @@ case ${SPACK_STACK_BATCH_HOST} in export MAC_GMAO_NAG_PREFIX=$(dirname $(dirname "${MAC_GMAO_NAG_PATH}")) fi - if [[ -n "${SPACK_STACK_COMPILER_OPT}" ]]; then - IFS=',' read -r -a SPACK_STACK_BATCH_COMPILERS <<< "${SPACK_STACK_COMPILER_OPT}" - else - SPACK_STACK_BATCH_COMPILERS=("gcc@=15.2.0") - if [[ -n "${MAC_GMAO_NAG_VERSION}" ]]; then - SPACK_STACK_BATCH_COMPILERS+=("nag@=${MAC_GMAO_NAG_VERSION}") - fi + SPACK_STACK_BATCH_COMPILERS=("gcc@=15.2.0") + if [[ -n "${MAC_GMAO_NAG_VERSION}" ]]; then + SPACK_STACK_BATCH_COMPILERS+=("nag@=${MAC_GMAO_NAG_VERSION}") fi # Auto-detect Apple Clang version @@ -243,6 +258,11 @@ case ${SPACK_STACK_BATCH_HOST} in ;; esac +# Apply -C compiler override for all hosts (not just macos.gmao) +if [[ -n "${SPACK_STACK_COMPILER_OPT}" ]]; then + IFS=',' read -r -a SPACK_STACK_BATCH_COMPILERS <<< "${SPACK_STACK_COMPILER_OPT}" +fi + ################################################################################################## function fix_permissions() { @@ -266,6 +286,13 @@ function fix_permissions() { fi nice -n 19 find ${dir} -type f -print0 | xargs --null chmod a+r ;; + discover) + nice -n 19 find ${dir} -type d -print0 | xargs --null chmod a+rx + if [[ ${executables} -eq 1 ]]; then + nice -n 19 find ${dir} -type f -executable -print0 | xargs --null chmod a+rx + fi + nice -n 19 find ${dir} -type f -print0 | xargs --null chmod a+r + ;; discover-gmao) nice -n 19 find ${dir} -type d -print0 | xargs --null chmod a+rx if [[ ${executables} -eq 1 ]]; then @@ -289,13 +316,16 @@ function tasks_per_node() { host=$1 case ${host} in nas) - tpn=128 + tpn=120 ;; nas-toss5) - tpn=256 + tpn=240 + ;; + discover) + tpn=120 ;; discover-gmao) - tpn=128 + tpn=120 ;; *) echo "ERROR, tasks_per_node command not configured for ${host}" @@ -311,28 +341,89 @@ function run_interactive_job() { host=$1 script=$2 reuse_build_cache=$3 + env_name=$4 tpn=$(tasks_per_node ${host}) - if [[ "${reuse_build_cache}" == "true" ]]; then - walltime="120" - else - walltime="720" - fi - if [[ ! -n "${ACCOUNT}" ]]; then - echo "ERROR, environment variable ACCOUNT not set" - exit 1 - fi - echo "Starting interactive job on ${host} with ${tpn} tasks and a walltime of ${walltime} minutes for ${script} ..." + walltime="08:00:00" + job_name="spack.${host}.${env_name}" + echo "Starting batch job on ${host} with ${tpn} tasks, walltime ${walltime}, account ${ACCOUNT} for ${script} ..." case ${host} in nas) - module load slurm - salloc --exclusive --nodes=1 --ntasks-per-node=${tpn} --time=${walltime} bash ${script} - module unload slurm + # Determine PBS model based on login node name + login_node=$(hostname | cut -d "." -f 1) + case ${login_node} in + pfe*) + pbs_model="rom_ait" + ;; + afe*) + pbs_model="mil_ait" + ;; + *) + echo "ERROR, cannot determine PBS model from login node '${login_node}' on ${host}" + echo "Expected login node name starting with 'pfe' or 'afe'" + exit 1 + ;; + esac + echo " Login node: ${login_node}, PBS model: ${pbs_model}" + qsub -V \ + -l select=1:ncpus=${tpn}:mpiprocs=${tpn}:model=${pbs_model} \ + -l walltime=${walltime} \ + -l site=needed=/home3+/nobackupp18+/nobackupp28+/vast_swbuild/swbuild4 \ + -W group_list=${ACCOUNT} \ + -W block=true \ + -W umask=0022 \ + -j oe -k oed \ + -N ${job_name} \ + ${script} ;; nas-toss5) - salloc --exclusive --nodes=1 --ntasks-per-node=${tpn} --time=${walltime} --qos=serial --account=${ACCOUNT} bash ${script} + # All nas-toss5 login nodes start with athfe + login_node=$(hostname | cut -d "." -f 1) + if [[ ! ${login_node} == athfe* ]]; then + echo "WARNING, expected login node name starting with 'athfe' on ${host}, got '${login_node}'" + fi + qsub -V \ + -l select=1:ncpus=${tpn}:mpiprocs=${tpn}:model=tur_ath \ + -q normal \ + -l walltime=${walltime} \ + -l site=needed=/home3+/nobackupp18+/nobackupp28+/vast_swbuild/swbuild4 \ + -W group_list=${ACCOUNT} \ + -W block=true \ + -W umask=0022 \ + -j oe -k oed \ + -N ${job_name} \ + ${script} + ;; + discover) + slurm_constraint="--constraint=mil" + if [[ "${ACCOUNT}" == "s1873" ]]; then + slurm_partition="--partition=preops --qos=benchmark" + else + slurm_partition="" + fi + slurm_log="${job_name}.log" + echo "INFO: salloc output redirected to ${slurm_log}" + salloc --nodes=1 --ntasks-per-node=${tpn} --time=${walltime} \ + ${slurm_constraint} ${slurm_partition} \ + --job-name=${job_name} \ + --account=${ACCOUNT} bash ${script} \ + > "${slurm_log}" 2>&1 + echo "INFO: salloc job complete, log: ${slurm_log}" ;; discover-gmao) - salloc --exclusive --nodes=1 --ntasks-per-node=${tpn} --time=${walltime} --qos=compute --account=${ACCOUNT} bash ${script} + slurm_constraint="--constraint=mil" + if [[ "${ACCOUNT}" == "s1873" ]]; then + slurm_partition="--partition=preops --qos=benchmark" + else + slurm_partition="" + fi + slurm_log="${job_name}.log" + echo "INFO: salloc output redirected to ${slurm_log}" + salloc --nodes=1 --ntasks-per-node=${tpn} --time=${walltime} \ + ${slurm_constraint} ${slurm_partition} \ + --job-name=${job_name} \ + --account=${ACCOUNT} bash ${script} \ + > "${slurm_log}" 2>&1 + echo "INFO: salloc job complete, log: ${slurm_log}" ;; *) echo "ERROR, run_interactive_job command not configured for ${host}" @@ -444,23 +535,6 @@ for compiler in "${SPACK_STACK_BATCH_COMPILERS[@]}"; do ############################################################# # Add excluded combinations of compilers and templates here # ############################################################# - # cylc-dev only with gcc - #if [[ "${template}" == "cylc-dev" && ! "${compiler_name}" == "gcc" ]]; then - #echo "Skipping template ${template} with compiler ${compiler}" - #continue - ## With clang, only neptune-dev-llvm - #elif [[ "${compiler_name}" == "clang" && ! "${template}" == "neptune-dev-llvm" ]]; then - #echo "Skipping template ${template} with compiler ${compiler}" - #continue - ## With other compilers, skip neptune-dev-llvm - #elif [[ ! "${compiler_name}" == "clang" && "${template}" == "neptune-dev-llvm" ]]; then - #echo "Skipping template ${template} with compiler ${compiler}" - #continue - ## FMS compiler ICE: https://github.com/NOAA-GFDL/FMS/issues/1680 - #elif [[ "${compiler_name}" == "oneapi" && "${compiler_version}" == "2025.1"* && "${template}" == "unified-dev" ]]; then - #echo "Skipping template ${template} with compiler ${compiler}" - #continue - #fi if [[ "${template}" == "geos-dev" && "${compiler_name}" == "nag" ]]; then echo "Skipping template ${template} with compiler ${compiler} (fms not supported by nag)" continue @@ -528,14 +602,22 @@ for compiler in "${SPACK_STACK_BATCH_COMPILERS[@]}"; do echo "[DRY-RUN] spack stack create env --name=${env_name} \\" echo " --site=${host} --compiler=${compiler_name}-${compiler_version} \\" echo " --template=${template} --dir=${environment_dirs} --treat-warnings-as-errors" + if [[ "${host}" == "macos.gmao" ]]; then + echo "[DRY-RUN] grep -v 'geos-gcm-env ~debug' ${env_dir}/spack.yaml # remove ~debug spec (esmf ~debug unsupported on macOS)" + fi fi echo "[DRY-RUN] spack env activate -p ${env_dir}" if [[ "${host}" == "macos.gmao" && ! ${env_exists} == "true" ]]; then echo "[DRY-RUN] spack external find --not-buildable autoconf automake bash cmake cvs doxygen gawk git-lfs groff libtool ninja npm subversion swig texinfo" echo "[DRY-RUN] generating spack-macos-externals.yaml and applying with 'spack config add -f'" fi + echo "[DRY-RUN] spack bootstrap list # add local-sources and local-binaries if missing" + if [[ "${update_build_cache}" == "true" ]]; then + echo "[DRY-RUN] spack config add config:install_tree:padded_length:200" + fi echo "[DRY-RUN] spack bootstrap now" echo "[DRY-RUN] spack concretize --force --fresh" + echo "[DRY-RUN] ./util/show_duplicate_packages.py -i crtm -i crtm-fix -i esmf -i mapl -i neptune-env -i py-cython -i ip -i fms -i geos-gcm-env" if [[ "${update_source_cache}" == "true"* ]]; then echo "[DRY-RUN] spack mirror create -a -d " @@ -544,9 +626,71 @@ for compiler in "${SPACK_STACK_BATCH_COMPILERS[@]}"; do echo "[DRY-RUN] ./util/fetch_cargo_deps.py" fi + if [[ "${host}" == "nas" || "${host}" == "nas-toss5" ]] && \ + [[ "${env_name_prefix}" == "ue" ]] && \ + [[ "${compiler_name}" == "oneapi" ]]; then + echo "[DRY-RUN] # ectrans/ecbuild workaround (NAS oneapi only):" + echo "[DRY-RUN] # See: https://github.com/JCSDA/spack-stack/issues/1775#issuecomment-3898802720" + echo "[DRY-RUN] spack install ecbuild" + echo "[DRY-RUN] ./util/gmao/patch_ecbuild_ectrans.py --patch \$(spack location -i ecbuild)/.../ecbuild_add_lang_flags.cmake" + echo "[DRY-RUN] spack install ectrans" + echo "[DRY-RUN] ./util/gmao/patch_ecbuild_ectrans.py --revert \$(spack location -i ecbuild)/.../ecbuild_add_lang_flags.cmake" + fi + echo "[DRY-RUN] Generating spack-install.${env_name}.sh and executing via:" if [[ "${submit_to_scheduler}" == "true" ]]; then - echo "[DRY-RUN] run_interactive_job ${host} spack-install.${env_name}.sh ${reuse_build_cache}" + tpn_dry=$(tasks_per_node ${host}) + case ${host} in + nas) + login_node=$(hostname | cut -d "." -f 1) + case ${login_node} in + pfe*) pbs_model_dry="rom_ait" ;; + afe*) pbs_model_dry="mil_ait" ;; + *) pbs_model_dry="" ;; + esac + echo "[DRY-RUN] qsub -V \\" + echo "[DRY-RUN] -l select=1:ncpus=${tpn_dry}:mpiprocs=${tpn_dry}:model=${pbs_model_dry} \\" + echo "[DRY-RUN] -l walltime=08:00:00 \\" + echo "[DRY-RUN] -l site=needed=/home3+/nobackupp18+/nobackupp28+/vast_swbuild/swbuild4 \\" + echo "[DRY-RUN] -W group_list=${ACCOUNT} -W block=true -W umask=0022 \\" + echo "[DRY-RUN] -j oe -k oed -N spack.${host}.${env_name} \\" + echo "[DRY-RUN] spack-install.${env_name}.sh" + ;; + nas-toss5) + echo "[DRY-RUN] qsub -V \\" + echo "[DRY-RUN] -l select=1:ncpus=${tpn_dry}:mpiprocs=${tpn_dry}:model=tur_ath \\" + echo "[DRY-RUN] -q normal -l walltime=08:00:00 \\" + echo "[DRY-RUN] -l site=needed=/home3+/nobackupp18+/nobackupp28+/vast_swbuild/swbuild4 \\" + echo "[DRY-RUN] -W group_list=${ACCOUNT} -W block=true -W umask=0022 \\" + echo "[DRY-RUN] -j oe -k oed -N spack.${host}.${env_name} \\" + echo "[DRY-RUN] spack-install.${env_name}.sh" + ;; + discover) + if [[ "${ACCOUNT}" == "s1873" ]]; then + slurm_extra_dry="--partition=preops --qos=benchmark" + else + slurm_extra_dry="(default partition/qos)" + fi + echo "[DRY-RUN] salloc --nodes=1 --ntasks-per-node=${tpn_dry} --time=08:00:00 \\" + echo "[DRY-RUN] --constraint=mil ${slurm_extra_dry} \\" + echo "[DRY-RUN] --job-name=spack.${host}.${env_name} \\" + echo "[DRY-RUN] --account=${ACCOUNT} bash spack-install.${env_name}.sh" + ;; + discover-gmao) + if [[ "${ACCOUNT}" == "s1873" ]]; then + slurm_extra_dry="--partition=preops --qos=benchmark" + else + slurm_extra_dry="(default partition/qos)" + fi + echo "[DRY-RUN] salloc --nodes=1 --ntasks-per-node=${tpn_dry} --time=08:00:00 \\" + echo "[DRY-RUN] --constraint=mil ${slurm_extra_dry} \\" + echo "[DRY-RUN] --job-name=spack.${host}.${env_name} \\" + echo "[DRY-RUN] --account=${ACCOUNT} bash spack-install.${env_name}.sh" + ;; + *) + echo "[DRY-RUN] run_interactive_job ${host} spack-install.${env_name}.sh ${reuse_build_cache}" + ;; + esac else echo "[DRY-RUN] bash spack-install.${env_name}.sh" fi @@ -554,10 +698,17 @@ for compiler in "${SPACK_STACK_BATCH_COMPILERS[@]}"; do if [[ "${update_build_cache}" == "true" ]]; then echo "[DRY-RUN] spack buildcache push -u " echo "[DRY-RUN] spack buildcache update-index local-binary" + echo "[DRY-RUN] fix_permissions ${host} 0" else echo "[DRY-RUN] spack module ${module_choice} refresh --yes --upstream-modules" echo "[DRY-RUN] spack stack setup-meta-modules" fi + if [[ "${update_source_cache}" == "true" ]]; then + echo "[DRY-RUN] fix_permissions ${host} 0" + fi + if [[ "${update_cargo_mirror}" == "true" ]]; then + echo "[DRY-RUN] fix_permissions ${host} ${cargo_mirror_path} 0" + fi echo "[DRY-RUN] spack clean -d -f -m -p -s" echo "[DRY-RUN] spack env deactivate" @@ -598,6 +749,12 @@ for compiler in "${SPACK_STACK_BATCH_COMPILERS[@]}"; do module purge set -e ;; + discover) + umask 0022 + set +e + module purge + set -e + ;; discover-gmao) umask 0022 set +e @@ -697,6 +854,16 @@ for compiler in "${SPACK_STACK_BATCH_COMPILERS[@]}"; do --treat-warnings-as-errors \ 2>&1 | tee ${SPACK_STACK_DIR}/logs/log.create.${env_name}.${LOG_TIMESTAMP} + # On macOS, esmf ~debug does not work with gfortranclang/GEOS, so remove + # the geos-gcm-env ~debug spec from the environment spack.yaml if present. + if [[ "${host}" == "macos.gmao" ]]; then + env_spack_yaml="${env_dir}/spack.yaml" + if grep -q "geos-gcm-env ~debug" "${env_spack_yaml}" 2>/dev/null; then + echo "INFO: macOS: removing 'geos-gcm-env ~debug' spec from ${env_spack_yaml}" + grep -v "geos-gcm-env ~debug" "${env_spack_yaml}" > "${env_spack_yaml}.tmp" && mv "${env_spack_yaml}.tmp" "${env_spack_yaml}" + fi + fi + # Clean up the generated yamls in the site configuration now that the env is created if [[ "${host}" == "macos.gmao" && ! ${env_exists} == "true" ]]; then rm -f "${SPACK_STACK_DIR}/configs/sites/tier2/${host}/mirrors.yaml" @@ -744,24 +911,25 @@ EOF spack bootstrap add --trust local-binaries ${bootstrap_mirror_path}/metadata/binaries # Check that the site has mirrors configured for local source and build caches, - # and extract the local path on disk. Need to strip leading "file://" from path + # and extract the local path on disk. Need to strip leading "file://" from path. + # Use awk to grab the last field since spack mirror list uses variable-width columns. result=$(spack mirror list | grep local-source) || \ (echo "ERROR, no local source cache configured" && exit 1) - source_mirror_path=$(echo ${result} | cut -d " " -f 3) - source_mirror_path=${source_mirror_path:7} + source_mirror_path=$(echo ${result} | awk '{print $NF}') + source_mirror_path=${source_mirror_path#file://} echo "Spack source mirror path: ${source_mirror_path}" # For build caches, additional logic is needed. If buildcache_dir is defined, # update the location of the default build cache to this directory. result=$(spack mirror list | grep local-binary) || \ (echo "ERROR, no local binary cache configured" && exit 1) - binary_mirror_path=$(echo ${result} | cut -d " " -f 3) - binary_mirror_path=${binary_mirror_path:7} + binary_mirror_path=$(echo ${result} | awk '{print $NF}') + binary_mirror_path=${binary_mirror_path#file://} # If buildcache_dir is set, update binary_mirror_path if [[ ! -z ${buildcache_dir} ]]; then sed -i "s#${binary_mirror_path}#${buildcache_dir}#g" ${env_dir}/site/mirrors.yaml result=$(spack mirror list | grep local-binary) - binary_mirror_path=$(echo ${result} | cut -d " " -f 3) - binary_mirror_path=${binary_mirror_path:7} + binary_mirror_path=$(echo ${result} | awk '{print $NF}') + binary_mirror_path=${binary_mirror_path#file://} fi echo "Spack binary mirror path: ${binary_mirror_path}" @@ -794,6 +962,11 @@ EOF if [[ "${update_cargo_mirror}" == "true"* ]]; then set +e echo "Updating local cargo mirror ..." + export CARGO_HTTP_MULTIPLEXING=false + export CARGO_HTTP_TIMEOUT=600 + export CARGO_HTTP_LOW_SPEED_LIMIT=1 + export CARGO_HTTP_LOW_SPEED_TIMEOUT=600 + export CARGO_NET_RETRY=10 ./util/fetch_cargo_deps.py set -e fi @@ -815,7 +988,7 @@ EOF case ${submit_to_scheduler} in "true") jobs=$(tasks_per_node ${host}) - parallel_install_flags="--concurrent-packages=10 --jobs=${jobs}" + parallel_install_flags="--concurrent-packages=2 --jobs=${jobs}" ;; "false") parallel_install_flags="" @@ -840,6 +1013,27 @@ set -e $(declare -p test_packages) +# Workaround for ectrans build failure with oneapi at NAS (nas/nas-toss5). +# ecbuild's flag checker incorrectly rejects valid Fortran flags (-march=core-avx2 -no-fma). +# Fix: patch ecbuild cmake to force-add the flags even when the check fails, +# install ectrans, then revert the patch. Spack skips already-installed packages, +# so the subsequent full install proceeds normally. +# See: https://github.com/JCSDA/spack-stack/issues/1775#issuecomment-3898802720 +if [[ "${host}" == "nas" || "${host}" == "nas-toss5" ]] && \ + [[ "${env_name_prefix}" == "ue" ]] && \ + [[ "${compiler_name}" == "oneapi" ]]; then + set -o pipefail + echo "Installing ecbuild before ectrans workaround ..." + spack install --verbose ${buildcache_install_flags} ecbuild 2>&1 | tee ${SPACK_STACK_DIR}/logs/log.install.${env_name}.${LOG_TIMESTAMP}.ecbuild + ecbuild_flags_cmake=\$(spack location -i ecbuild)/share/ecbuild/cmake/ecbuild_add_lang_flags.cmake + echo "Applying ectrans/ecbuild workaround to \${ecbuild_flags_cmake} ..." + ${SPACK_STACK_DIR}/util/gmao/patch_ecbuild_ectrans.py --patch \${ecbuild_flags_cmake} + spack install --verbose ${buildcache_install_flags} ectrans 2>&1 | tee ${SPACK_STACK_DIR}/logs/log.install.${env_name}.${LOG_TIMESTAMP}.ectrans + set +o pipefail + echo "Reverting ectrans/ecbuild workaround ..." + ${SPACK_STACK_DIR}/util/gmao/patch_ecbuild_ectrans.py --revert \${ecbuild_flags_cmake} +fi + # If no tests are required, install everything if [[ \${#test_packages[@]} -eq 0 ]]; then set -o pipefail @@ -868,7 +1062,7 @@ fi EOF chmod u+x ${install_script} if [[ "${submit_to_scheduler}" == "true" ]]; then - run_interactive_job ${host} ${install_script} ${reuse_build_cache} + run_interactive_job ${host} ${install_script} ${reuse_build_cache} ${env_name} else bash ${install_script} fi diff --git a/util/gmao/monitor_install.py b/util/gmao/monitor_install.py new file mode 100755 index 000000000..e46b3dea3 --- /dev/null +++ b/util/gmao/monitor_install.py @@ -0,0 +1,111 @@ +#!/usr/bin/env python3 +""" +Monitor a spack install log file for progress. + +Usage: + python3 monitor_install.py # monitor from current end (live) + python3 monitor_install.py -f # scan from beginning of file (live) + python3 monitor_install.py -r # read full file, print report, exit +""" + +import sys +import re +import time +import argparse + +pattern_installing = re.compile(r'==> Installing (\S+) \[(\d+)/(\d+)\]') +pattern_success = re.compile(r'^\[.\]\s+\S+.*Installing (\S+)') +pattern_error = re.compile(r'==> Error: Installation request failed') +pattern_failed_pkg = re.compile(r'^\s+(\S+)\s+\[FAILED\]') + + +def format_bar(current, total, bar_len=30): + pct = int(current) / int(total) * 100 + filled = int(bar_len * int(current) / int(total)) + bar = '#' * filled + '-' * (bar_len - filled) + return f"[{bar}] {current:>4}/{total} {pct:5.1f}%" + + +def monitor(logfile, from_start=False): + with open(logfile) as f: + if not from_start: + f.seek(0, 2) # seek to end for live monitoring + while True: + line = f.readline() + if not line: + time.sleep(1) + continue + m = pattern_installing.search(line) + if m: + pkg, current, total = m.group(1), m.group(2), m.group(3) + print(f"{format_bar(current, total)} {pkg}", flush=True) + + +def report(logfile): + """Read entire file, show progress lines, then print a summary.""" + installs = [] # list of (pkg, current, total) + errors = [] # packages that failed + last_current = 0 + last_total = 0 + + with open(logfile) as f: + lines = f.readlines() + + for line in lines: + m = pattern_installing.search(line) + if m: + pkg, current, total = m.group(1), int(m.group(2)), int(m.group(3)) + installs.append((pkg, current, total)) + last_current = current + last_total = total + print(f"{format_bar(current, total)} {pkg}", flush=True) + continue + + m = pattern_failed_pkg.search(line) + if m: + errors.append(m.group(1)) + + # Summary + print() + print("=" * 60) + if last_total: + print(f"Packages attempted : {last_current}/{last_total}") + else: + print("No '==> Installing' lines found in log.") + if errors: + print(f"FAILED packages : {len(errors)}") + for e in errors: + print(f" - {e}") + else: + # Check for a generic error marker even without a parsed package name + had_error = any(pattern_error.search(l) for l in lines) + if had_error: + print("FAILED : yes (see log for details)") + else: + print("Result : all attempted packages succeeded") + print("=" * 60) + + +def main(): + parser = argparse.ArgumentParser(description="Monitor spack install log progress") + parser.add_argument("logfile", help="Path to the spack PBS/SLURM log file") + parser.add_argument("-f", "--from-start", action="store_true", + help="Scan from beginning of file instead of tailing live") + parser.add_argument("-r", "--report", action="store_true", + help="Read full file, show progress, print summary, then exit") + args = parser.parse_args() + + try: + if args.report: + report(args.logfile) + else: + monitor(args.logfile, from_start=args.from_start) + except KeyboardInterrupt: + print("\nDone monitoring.") + except FileNotFoundError: + print(f"Error: file not found: {args.logfile}", file=sys.stderr) + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/util/gmao/patch_ecbuild_ectrans.py b/util/gmao/patch_ecbuild_ectrans.py new file mode 100755 index 000000000..acd3c159e --- /dev/null +++ b/util/gmao/patch_ecbuild_ectrans.py @@ -0,0 +1,84 @@ +#!/usr/bin/env python3 +""" +Patch/revert ecbuild_add_lang_flags.cmake to work around an ectrans build +failure with oneapi at NAS (nas/nas-toss5). + +The problem: ecbuild's Fortran flag checker incorrectly determines that +'-march=core-avx2 -no-fma' is not supported by ifx (via spack compiler +wrapper), and calls ecbuild_critical() which aborts the build. The flags +are actually valid -- the check is a false negative. + +The fix: replace the ecbuild_critical() call in the else() block with +the same logic as the if(_flag_ok) block, so the flags are always added +regardless of the check result. + +See: https://github.com/JCSDA/spack-stack/issues/1775#issuecomment-3898802720 + +Usage: + patch_ecbuild_ectrans.py --patch + patch_ecbuild_ectrans.py --revert +""" + +import argparse +import shutil +import sys +from pathlib import Path + +OLD = """ else() + ecbuild_critical( "${_lang} compiler ${CMAKE_${_lang}_COMPILER} does not recognise ${_lang} flag '${_flags}'" )""" + +NEW = """ else() + ecbuild_info( "${_lang} compiler ${CMAKE_${_lang}_COMPILER} does not recognise ${_lang} flag '${_flags}' -- forcing anyway (NAS oneapi workaround)" ) + if( _PAR_BUILD ) + set( CMAKE_${_lang}_FLAGS_${_PAR_BUILD} "${CMAKE_${_lang}_FLAGS_${_PAR_BUILD}} ${_flags}" PARENT_SCOPE ) + ecbuild_info( "Added ${_lang} flag [${_flags}] to build type ${_PAR_BUILD}" ) + else() + set( CMAKE_${_lang}_FLAGS "${CMAKE_${_lang}_FLAGS} ${_flags}" PARENT_SCOPE ) + ecbuild_info( "Added ${_lang} flag [${_flags}]" ) + endif()""" + + +def patch(cmake_file: Path): + backup = Path(str(cmake_file) + ".orig") + content = cmake_file.read_text() + if OLD not in content: + if NEW in content: + print(f"Patch already applied to {cmake_file} -- skipping") + return + print(f"WARNING: expected patch target not found in {cmake_file} -- skipping", file=sys.stderr) + return + shutil.copy(cmake_file, backup) + cmake_file.write_text(content.replace(OLD, NEW, 1)) + print(f"Patched {cmake_file} (backup: {backup})") + + +def revert(cmake_file: Path): + backup = Path(str(cmake_file) + ".orig") + if not backup.exists(): + print(f"WARNING: backup {backup} not found -- cannot revert", file=sys.stderr) + return + shutil.copy(backup, cmake_file) + backup.unlink() + print(f"Reverted {cmake_file} (removed {backup})") + + +def main(): + parser = argparse.ArgumentParser(description="Patch/revert ecbuild for ectrans NAS oneapi workaround") + group = parser.add_mutually_exclusive_group(required=True) + group.add_argument("--patch", action="store_true", help="Apply the patch") + group.add_argument("--revert", action="store_true", help="Revert the patch") + parser.add_argument("cmake_file", type=Path, help="Path to ecbuild_add_lang_flags.cmake") + args = parser.parse_args() + + if not args.cmake_file.exists(): + print(f"ERROR: file not found: {args.cmake_file}", file=sys.stderr) + sys.exit(1) + + if args.patch: + patch(args.cmake_file) + else: + revert(args.cmake_file) + + +if __name__ == "__main__": + main()