diff --git a/CMakeLists.txt b/CMakeLists.txt index 78ae10f..de3a107 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,4 +1,4 @@ -# (C) Copyright 2017-2024 UCAR +# (C) Copyright 2017-2026 UCAR # # This software is licensed under the terms of the Apache Licence Version 2.0 # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. @@ -8,7 +8,7 @@ # # -# Usage: cmake -DCMAKE_BUILD_TYPE=RelWithDebINfo -DCMAKE_VERBOSE_MAKEFILE=ON -DSNAPSHOT_DATE=2024-06-01 -DMPAS_DOUBLE_PRECISION=OFF +# Usage: cmake -DCMAKE_BUILD_TYPE=RelWithDebINfo -DCMAKE_VERBOSE_MAKEFILE=ON -DMPAS_DOUBLE_PRECISION=OFF cmake_minimum_required( VERSION 3.14 ) project( mpas-bundle VERSION 3.0.0 LANGUAGES C CXX Fortran ) @@ -25,59 +25,68 @@ include( ecbuild_bundle ) ecbuild_bundle_initialize() # Use external jedi-cmake or build in bundle -if(DEFINED ENV{jedi_cmake_ROOT}) - include( $ENV{jedi_cmake_ROOT}/share/jedicmake/Functions/git_functions.cmake ) - list( APPEND CMAKE_MODULE_PATH $ENV{jedi_cmake_ROOT}/share/jedicmake/Modules ) - message (INFO "CMAKE_MODULE_PATH") +# If the gpu toolchain is loaded, only configure MPAS-Model +if(CMAKE_Fortran_COMPILER_ID MATCHES NVHPC) + ecbuild_bundle( PROJECT MPAS GIT "https://github.com/MPAS-Dev/MPAS-Model.git" BRANCH develop UPDATE ) else() - ecbuild_bundle( PROJECT jedicmake GIT "https://github.com/JCSDA/jedi-cmake.git" BRANCH develop UPDATE RECURSIVE ) - include( jedicmake/cmake/Functions/git_functions.cmake ) - list( APPEND CMAKE_MODULE_PATH ${CMAKE_SOURCE_DIR}/jedicmake/CMakeModules/Modules ) -endif() - -option(BUNDLE_SKIP_ECKIT "Don't build eckit" "ON" ) # Skip eckit build unless user passes -DBUNDLE_SKIP_ECKIT=OFF -option(BUNDLE_SKIP_FCKIT "Don't build fckit" "ON" ) # Skip eckit build unless user passes -DBUNDLE_SKIP_FCKIT=OFF -option(BUNDLE_SKIP_ATLAS "Don't build atlas" "ON" ) # Skip atlas build unless user passes -DBUNDLE_SKIP_ATLAS=OFF -ecbuild_bundle( PROJECT eckit GIT "https://github.com/ecmwf/eckit.git" TAG 1.24.4 ) -ecbuild_bundle( PROJECT fckit GIT "https://github.com/ecmwf/fckit.git" TAG 0.11.0 ) -ecbuild_bundle( PROJECT atlas GIT "https://github.com/ecmwf/atlas.git" TAG 0.34.0 ) - -#TODO: When mpas-bundle becomes a public repo, consider changing the default value of BUNDLE_SKIP_ROPP-UFO to "ON" -option(BUNDLE_SKIP_ROPP-UFO "Don't build ROPP-UFO" "ON") # Build ropp-ufo unless user passes -DBUNDLE_SKIP_ROPP-UFO=ON -ecbuild_bundle( PROJECT ropp-ufo GIT "https://github.com/JCSDA-internal/ropp-test.git" BRANCH develop UPDATE ) -option(BUNDLE_SKIP_RTTOV "Don't build rttov" "ON") # Skip rttov build unless user passes -DBUNDLE_SKIP_RTTOV=OFF -ecbuild_bundle( PROJECT rttov GIT "https://github.com/JCSDA-internal/rttov.git" BRANCH develop UPDATE ) - -ecbuild_bundle( PROJECT oops GIT "https://github.com/JCSDA/oops.git" BRANCH develop UPDATE ) -ecbuild_bundle( PROJECT vader GIT "https://github.com/JCSDA/vader.git" BRANCH develop UPDATE ) -ecbuild_bundle( PROJECT saber GIT "https://github.com/JCSDA/saber.git" BRANCH develop UPDATE) -ecbuild_bundle( PROJECT crtm GIT "https://github.com/JCSDA/CRTMv3.git" BRANCH develop UPDATE ) - -option(ENABLE_IODA_DATA "Obtain ioda test data from ioda-data repository (vs tarball)" ON) -ecbuild_bundle( PROJECT ioda-data GIT "https://github.com/JCSDA-internal/ioda-data.git" BRANCH develop UPDATE ) -ecbuild_bundle( PROJECT ioda GIT "https://github.com/JCSDA/ioda.git" BRANCH develop UPDATE ) -option(ENABLE_UFO_DATA "Obtain ufo test data from ufo-data repository (vs tarball)" ON) -ecbuild_bundle( PROJECT ufo-data GIT "https://github.com/JCSDA-internal/ufo-data.git" BRANCH develop UPDATE ) -ecbuild_bundle( PROJECT ufo GIT "https://github.com/JCSDA/ufo.git" BRANCH develop UPDATE ) - - -# Find external ESMF for mpas-model (optional) -find_package(ESMF 8.3.0 MODULE) - -set(MPAS_DOUBLE_PRECISION "ON" CACHE STRING "MPAS-Model: Use double precision 64-bit Floating point.") -set(MPAS_CORES init_atmosphere atmosphere CACHE STRING "MPAS-Model: cores to build.") - -ecbuild_bundle( PROJECT MPAS GIT "https://github.com/MPAS-Dev/MPAS-Model" BRANCH develop UPDATE ) -option(ENABLE_MPAS_JEDI_DATA "Obtain mpas-jedi test data from mpas-jedi-data repository (vs tarball)" ON) -ecbuild_bundle( PROJECT mpas-jedi-data GIT "https://github.com/JCSDA-internal/mpas-jedi-data.git" BRANCH develop UPDATE ) -ecbuild_bundle( PROJECT mpas-jedi GIT "https://github.com/JCSDA/mpas-jedi" BRANCH develop UPDATE ) - -# Set GIT_BRANCH_FUNC to MPAS-JEDI's current branch so that it can be used for mpas-jedi-data -find_branch_name(REPO_DIR_NAME mpas-jedi) - -# If mpas-jedi's current branch is available in mpas-jedi-data repo, that branch will be checked out -branch_checkout (REPO_DIR_NAME mpas-jedi-data + if(DEFINED ENV{jedi_cmake_ROOT}) + include( $ENV{jedi_cmake_ROOT}/share/jedicmake/Functions/git_functions.cmake ) + list( APPEND CMAKE_MODULE_PATH $ENV{jedi_cmake_ROOT}/share/jedicmake/Modules ) + message (INFO "CMAKE_MODULE_PATH") + else() + ecbuild_bundle( PROJECT jedicmake GIT "https://github.com/JCSDA/jedi-cmake.git" BRANCH develop UPDATE RECURSIVE ) + include( jedicmake/cmake/Functions/git_functions.cmake ) + list( APPEND CMAKE_MODULE_PATH ${CMAKE_SOURCE_DIR}/jedicmake/CMakeModules/Modules ) + endif() + + option(BUNDLE_SKIP_ECKIT "Don't build eckit" "ON" ) # Skip eckit build unless user passes -DBUNDLE_SKIP_ECKIT=OFF + option(BUNDLE_SKIP_FCKIT "Don't build fckit" "ON" ) # Skip eckit build unless user passes -DBUNDLE_SKIP_FCKIT=OFF + option(BUNDLE_SKIP_ATLAS "Don't build atlas" "ON" ) # Skip atlas build unless user passes -DBUNDLE_SKIP_ATLAS=OFF + ecbuild_bundle( PROJECT eckit GIT "https://github.com/ecmwf/eckit.git" TAG 1.24.4 ) + ecbuild_bundle( PROJECT fckit GIT "https://github.com/ecmwf/fckit.git" TAG 0.11.0 ) + ecbuild_bundle( PROJECT atlas GIT "https://github.com/ecmwf/atlas.git" TAG 0.34.0 ) + + #TODO: When mpas-bundle becomes a public repo, consider changing the default value of BUNDLE_SKIP_ROPP-UFO to "ON" + option(BUNDLE_SKIP_ROPP-UFO "Don't build ROPP-UFO" "ON") # Build ropp-ufo unless user passes -DBUNDLE_SKIP_ROPP-UFO=ON + ecbuild_bundle( PROJECT ropp-ufo GIT "https://github.com/JCSDA-internal/ropp-test.git" BRANCH develop UPDATE ) + option(BUNDLE_SKIP_RTTOV "Don't build rttov" "ON") # Skip rttov build unless user passes -DBUNDLE_SKIP_RTTOV=OFF + ecbuild_bundle( PROJECT rttov GIT "https://github.com/JCSDA-internal/rttov.git" BRANCH develop UPDATE ) + + ecbuild_bundle( PROJECT oops GIT "https://github.com/JCSDA/oops.git" BRANCH develop UPDATE ) + ecbuild_bundle( PROJECT vader GIT "https://github.com/JCSDA/vader.git" BRANCH develop UPDATE ) + ecbuild_bundle( PROJECT saber GIT "https://github.com/JCSDA/saber.git" BRANCH develop UPDATE ) + ecbuild_bundle( PROJECT crtm GIT "https://github.com/JCSDA/CRTMv3.git" BRANCH develop UPDATE ) + + option(ENABLE_IODA_DATA "Obtain ioda test data from ioda-data repository (vs tarball)" ON) + ecbuild_bundle( PROJECT ioda-data GIT "https://github.com/JCSDA-internal/ioda-data.git" BRANCH develop UPDATE ) + ecbuild_bundle( PROJECT ioda GIT "https://github.com/JCSDA/ioda.git" BRANCH develop UPDATE ) + option(ENABLE_UFO_DATA "Obtain ufo test data from ufo-data repository (vs tarball)" ON) + ecbuild_bundle( PROJECT ufo-data GIT "https://github.com/JCSDA-internal/ufo-data.git" BRANCH develop UPDATE ) + ecbuild_bundle( PROJECT ufo GIT "https://github.com/JCSDA/ufo.git" BRANCH develop UPDATE ) + + # Build IODA converters if requested + option(BUILD_IODA_CONVERTERS "Build IODA Converters" OFF) + if(BUILD_IODA_CONVERTERS) + ecbuild_bundle( PROJECT iodaconv GIT "https://github.com/jcsda-internal/ioda-converters.git" BRANCH develop UPDATE ) + endif() + + # Find external ESMF for mpas-model (optional) + find_package(ESMF 8.3.0 MODULE) + + set(MPAS_DOUBLE_PRECISION "ON" CACHE STRING "MPAS-Model: Use double precision 64-bit Floating point.") + set(MPAS_CORES init_atmosphere atmosphere CACHE STRING "MPAS-Model: cores to build.") + + ecbuild_bundle( PROJECT MPAS GIT "https://github.com/MPAS-Dev/MPAS-Model" BRANCH develop UPDATE ) + option(ENABLE_MPAS_JEDI_DATA "Obtain mpas-jedi test data from mpas-jedi-data repository (vs tarball)" ON) + ecbuild_bundle( PROJECT mpas-jedi-data GIT "https://github.com/JCSDA-internal/mpas-jedi-data.git" BRANCH develop UPDATE ) + ecbuild_bundle( PROJECT mpas-jedi GIT "https://github.com/JCSDA/mpas-jedi" BRANCH develop UPDATE ) + + # Set GIT_BRANCH_FUNC to MPAS-JEDI's current branch so that it can be used for mpas-jedi-data + find_branch_name(REPO_DIR_NAME mpas-jedi) + + # If mpas-jedi's current branch is available in mpas-jedi-data repo, that branch will be checked out + branch_checkout (REPO_DIR_NAME mpas-jedi-data BRANCH ${GIT_BRANCH_FUNC} ) - +endif() ecbuild_bundle_finalize() diff --git a/README.md b/README.md index 1503994..2fbb115 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -(C) Copyright 2017-2021 UCAR +(C) Copyright 2017-2026 UCAR This software is licensed under the terms of the Apache Licence Version 2.0 which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. @@ -45,6 +45,8 @@ _**For performance and memory reasons, it is recommended to compile ```mpas-bund ```bash git clone https://github.com/JCSDA/mpas-bundle.git + ``` + ```bash cd mpas-bundle ``` @@ -60,8 +62,12 @@ _**For performance and memory reasons, it is recommended to compile ```mpas-bund | __csh/tcsh__ | `source /env-setup/gnu-derecho.csh` | `source /env-setup/intel-derecho.csh` | * Create and navigate into the build directory. + ```bash mkdir -p + ``` + ```bash cd + ``` * To configure the build using CMake, set the `MPAS_DOUBLE_PRECISION` flag according to your usage needs: enable it (`-DMPAS_DOUBLE_PRECISION=ON`) for running the `mpas-jedi` test suite, or disable it for `MPAS-Workflow` calculations when using the `mpas-bundle` build. diff --git a/env-setup/crontab.sh b/env-setup/crontab.sh deleted file mode 100644 index 9dcfe56..0000000 --- a/env-setup/crontab.sh +++ /dev/null @@ -1,61 +0,0 @@ - -# run scripts to build mpas-bundle and run ctest -# to use this, -# 1. log on to cron.hpc.ucar.edu -# 2. run: crontab bundle-crontab - -# set this to the directory the mpas-bundle repository has been cloned to -bundle_dir=/glade/work/jwittig/repos1/mpas-bundle-cron/mpas-bundle/ -# the directory where the single precision bundle build is -bundle_build_dir=/glade/work/jwittig/repos1/mpas-bundle-cron/build-gnu-1p/ -# this is the build script to execute, relative to the mpas-bundle source directory -build_script=env-setup/mpas-bundle-cron.sh - -# the workflow directory with the script to run a workflow and make graphs -workflow_dir=/glade/derecho/scratch/jwittig/repos-s/MPAS-Workflow-cron -# the script to run a cylc job and to create graphs, relative to the workflow directory -workflow_script=env-setup/run_cylc.sh -# the workflow scenario to run, relative to the workflow directory -workflow_scenario=scenarios/3denvar_OIE120km_WarmStart_VarBC_cron.yaml - -# the directory with the graphics scripts -graphs_dir=/glade/derecho/scratch/jwittig/repos-s/mpas-jedi-cron/graphics -# the directory where the workflow results graphs shold be placed -graphs_out_dir=/glade/derecho/scratch/jwittig/graphs/data/ - -# derecho hpc -derecho=derecho.hpc.ucar.edu -# mmm web server -webserver=whitedwarf.mmm.ucar.edu -# destination for graphs -web_graphs_dir=/web/htdocs/projects/mpas-jedi/weekly-cycling/cylc_graphs - -# start at 11:05 PM and clean up log files -05 23 * * * ssh $derecho "cd ~/my_cron_logs && (gunzip mpas-bundle-cron.log.tar.gz ; tar --remove-files -uf mpas-bundle-cron.log.tar mpas-bundle-cron.log.2* ; tar --remove-files -uf mpas-bundle-cron.log.tar git_shas* ; gzip mpas-bundle-cron.log.tar)" - -# start at 12:05 AM Sun through Fri -# change 'gnu' to 'intel' to use the intel build toolchain, or 'both' to build both -# double precision (-p 2), to be used for ctests. -# don't do anything if no source code changed from previous run (don't provide -f) -#05 00 * * 0-5 $bundle_dir/$build_script -d $bundle_dir -q main@desched1 -c gnu -p 2 -#05 00 * * 0-5 $bundle_dir/$build_script -d $bundle_dir -q develop@desched1 -c gnu -p 2 -l $bundle_dir/$build_script.lock - -# start at 12:05 AM on Sat -# always build, even if no source change from previous run (-f) -05 00 * * 6 $bundle_dir/$build_script -d $bundle_dir -q develop@desched1 -c gnu -p 2 -f - -# start at 1:05 AM on Sat -# always build, even if no source change from previous run (-f) -# single precision (-p 1), to be used for cylc experiment. -05 01 * * 6 $bundle_dir/$build_script -d $bundle_dir -q develop@desched1 -c gnu -p 1 -f -l $bundle_dir/$build_script.lock - -# at 12:05 am on Sun run the workflow -suffix=$(date +%F) -05 00 * * 7 ssh $derecho $workflow_dir/$workflow_script -w $workflow_dir -d $bundle_build_dir -k $bundle_dir/$build_script.lock -s $workflow_scenario -x $suffix - -# at 12:05 am Mon-Fri try to graph results from the completed workflow runs -05 00 * * 1-5 ssh $derecho $workflow_dir/$workflow_script -w $workflow_dir -g $graphs_dir -o $graphs_out_dir - -# at 6:05 am weekdays tar any graphics files and copy them to an mmm web server -05 06 * * 1-5 echo "`date` checking $graphs_out_dir" >> ~/my_cron_logs/cp_graphs.log && for dir in $(/bin/ls $graphs_out_dir); do echo "`date` dir to copy:$dir" >> ~/my_cron_logs/cp_graphs.log; echo "ssh $derecho cd $graphs_out_dir && tar --remove-files -czf $dir.tgz $dir" >> ~/my_cron_logs/cp_graphs.log; ssh $derecho "cd $graphs_out_dir && tar --remove-files -czf $dir.tgz $dir"; echo "scp $graphs_out_dir/$dir.tgz $webserver:$web_graphs_dir/" >> ~/my_cron_logs/cp_graphs.log; scp $graphs_out_dir/$dir.tgz $webserver:$web_graphs_dir/; echo "ssh $webserver cd $web_graphs_dir/ && tar -xzf $dir.tgz" >> ~/my_cron_logs/cp_graphs.log; ssh $webserver "cd $web_graphs_dir/ && tar -xzf $dir.tgz"; mv $graphs_out_dir/$dir.tgz $graphs_out_dir/../copied; echo "finished $dir" >> ~/my_cron_logs/cp_graphs.log; done - diff --git a/env-setup/gnu-derecho.csh b/env-setup/gnu-derecho.csh index 1d210e0..c378b1d 100644 --- a/env-setup/gnu-derecho.csh +++ b/env-setup/gnu-derecho.csh @@ -10,16 +10,33 @@ endif module purge # ignore that the sticky module ncarenv/... is not unloaded -module load ncarenv/23.09 +module load ncarenv/24.12 module use /glade/work/epicufsrt/contrib/spack-stack/derecho/modulefiles module load ecflow/5.8.4 module load mysql/8.0.33 -module use /glade/work/epicufsrt/contrib/spack-stack/derecho/spack-stack-1.6.0/envs/unified-env/install/modulefiles/Core -module load stack-gcc/12.2.0 -module load stack-cray-mpich/8.1.25 -module load stack-python/3.10.13 -module load jedi-mpas-env +module use /glade/work/epicufsrt/contrib/spack-stack/derecho/spack-stack-1.9.3/envs/ue-gcc-12.4.0/install/modulefiles/Core + +module load stack-gcc/12.4.0 +module load stack-cray-mpich/8.1.29 +module load stack-python/3.11.7 +module load py-pycodestyle/2.11.0 +# module load jedi-mpas-env is commented out because it loads many modules not needed for +# building and running mpas-bundle on derecho. +# To load all of the modules which were loaded prior to 01/15/2026, +# uncomment out "module load jedi-mpas-env" +# module load jedi-mpas-env +module load atlas/0.40.0 +module load ecbuild/3.7.2 +module load netcdf-cxx4/4.3.1 +module load parallelio/2.6.2 +module load gsl-lite/0.37.0 +module load nccmp/1.9.0.1 +module load udunits/2.2.28 +# Following modules are required for ioda-converters build +module load bufr/12.1.0 +module load py-pybind11/2.13.5 +module list limit stacksize unlimited setenv F_UFMTENDIAN 'big_endian:101-200' diff --git a/env-setup/gnu-derecho.sh b/env-setup/gnu-derecho.sh index 8eae3fc..55c3036 100644 --- a/env-setup/gnu-derecho.sh +++ b/env-setup/gnu-derecho.sh @@ -11,19 +11,35 @@ fi module purge # ignore that the sticky module ncarenv/... is not unloaded -module load ncarenv/23.09 +module load ncarenv/24.12 module use /glade/work/epicufsrt/contrib/spack-stack/derecho/modulefiles module load ecflow/5.8.4 module load mysql/8.0.33 -module use /glade/work/epicufsrt/contrib/spack-stack/derecho/spack-stack-1.6.0/envs/unified-env/install/modulefiles/Core -module load stack-gcc/12.2.0 -module load stack-cray-mpich/8.1.25 -module load stack-python/3.10.13 -module load jedi-mpas-env +module use /glade/work/epicufsrt/contrib/spack-stack/derecho/spack-stack-1.9.3/envs/ue-gcc-12.4.0/install/modulefiles/Core + +module load stack-gcc/12.4.0 +module load stack-cray-mpich/8.1.29 +module load stack-python/3.11.7 +# module load jedi-mpas-env is commented out because it loads many modules not needed for +# building and running mpas-bundle on derecho. +# To load all of the modules which were loaded prior to 01/15/2026, +# uncomment out "module load jedi-mpas-env" +# module load jedi-mpas-env +module load py-pycodestyle/2.11.0 +module load atlas/0.40.0 +module load ecbuild/3.7.2 +module load netcdf-cxx4/4.3.1 +module load parallelio/2.6.2 +module load gsl-lite/0.37.0 +module load nccmp/1.9.0.1 +module load udunits/2.2.28 +# Following modules are required for ioda-converters build +module load bufr/12.1.0 +module load py-pybind11/2.13.5 module list ulimit -s unlimited -export F_UFMTENDIAN='big_endian:101-200' +export GFORTRAN_CONVERT_UNIT='big_endian:101-200' export LD_LIBRARY_PATH=`pwd`/lib:$LD_LIBRARY_PATH diff --git a/env-setup/intel-derecho.csh b/env-setup/intel-derecho.csh index 286167e..fd09f6d 100644 --- a/env-setup/intel-derecho.csh +++ b/env-setup/intel-derecho.csh @@ -13,17 +13,29 @@ endif # module purge -module use /lustre/desc1/scratch/epicufsrt/contrib/modulefiles -module use /lustre/desc1/scratch/epicufsrt/contrib/modulefiles_extra +module load ncarenv/24.12 +module use /glade/work/epicufsrt/contrib/spack-stack/derecho/modulefiles module load ecflow/5.8.4 module load mysql/8.0.33 -module use /glade/work/epicufsrt/contrib/spack-stack/derecho/spack-stack-1.6.0/envs/unified-env/install/modulefiles/Core +module use /glade/work/epicufsrt/contrib/spack-stack/derecho/spack-stack-1.9.3/envs/ue-oneapi-2024.2.1/install/modulefiles/Core -module load stack-intel/2021.10.0 -module load stack-cray-mpich/8.1.25 -module load stack-python/3.10.13 -module load jedi-mpas-env +module load stack-oneapi/2024.2.1 +module load stack-cray-mpich/8.1.29 +module load stack-python/3.11.7 +# module load jedi-mpas-env is commented out because it loads many modules not needed for +# building and running mpas-bundle on derecho. +# To load all of the modules which were loaded prior to 01/15/2026, +# uncomment out "module load jedi-mpas-env" +#module load jedi-mpas-env +module load py-pycodestyle/2.11.0 +module load atlas/0.40.0 +module load ecbuild/3.7.2 +module load netcdf-cxx4/4.3.1 +module load parallelio/2.6.2 +module load gsl-lite/0.37.0 +module load nccmp/1.9.0.1 +module load udunits/2.2.28 module list limit stacksize unlimited diff --git a/env-setup/intel-derecho.sh b/env-setup/intel-derecho.sh index aecca3b..554b659 100644 --- a/env-setup/intel-derecho.sh +++ b/env-setup/intel-derecho.sh @@ -10,17 +10,29 @@ if command -v conda >/dev/null 2>&1; then fi module purge -module use /lustre/desc1/scratch/epicufsrt/contrib/modulefiles -module use /lustre/desc1/scratch/epicufsrt/contrib/modulefiles_extra +module load ncarenv/24.12 +module use /glade/work/epicufsrt/contrib/spack-stack/derecho/modulefiles module load ecflow/5.8.4 module load mysql/8.0.33 -module use /glade/work/epicufsrt/contrib/spack-stack/derecho/spack-stack-1.6.0/envs/unified-env/install/modulefiles/Core +module use /glade/work/epicufsrt/contrib/spack-stack/derecho/spack-stack-1.9.3/envs/ue-oneapi-2024.2.1/install/modulefiles/Core -module load stack-intel/2021.10.0 -module load stack-cray-mpich/8.1.25 -module load stack-python/3.10.13 -module load jedi-mpas-env +module load stack-oneapi/2024.2.1 +module load stack-cray-mpich/8.1.29 +module load stack-python/3.11.7 +# module load jedi-mpas-env is commented out because it loads many modules not needed for +# building and running mpas-bundle on derecho. +# To load all of the modules which were loaded prior to 01/15/2026, +# uncomment out "module load jedi-mpas-env" +#module load jedi-mpas-env +module load py-pycodestyle/2.11.0 +module load atlas/0.40.0 +module load ecbuild/3.7.2 +module load netcdf-cxx4/4.3.1 +module load parallelio/2.6.2 +module load gsl-lite/0.37.0 +module load nccmp/1.9.0.1 +module load udunits/2.2.28 module list ulimit -s unlimited diff --git a/env-setup/mpas-bundle-cron.sh b/env-setup/mpas-bundle-cron.sh deleted file mode 100755 index 8f564b6..0000000 --- a/env-setup/mpas-bundle-cron.sh +++ /dev/null @@ -1,548 +0,0 @@ -#!/bin/bash -l - -#-------------------------------------------------------------- -# Script to build mpas-bundle and run mpas-jedi ctests. -# mpas-bundle may be built with both the gnu and intel tool chains. -# After running mpas-jedi ctests the results will be put into an html document, -# which will be copied to the web pages directory on whitedwarf.mmm.ucar.edu -# -# This script assumes the mpas-bundle repository has been cloned into -# / # directory mpas-bundle has been cloned to -# This script will make the following directory layout if it doesn't exist: -# /build-gnu-2p # directory to build the bundle with the gnu tool chain, double precision -# /build-gnu-1p # directory to build the bundle with the gnu tool chain, single precision -# /build-intel-2p # directory to build the bundle with the intel tool chain, double preecision -# /build-intel-1p # directory to build the bundle with the intel tool chain, single preecision -# Note the build directories are adjacent to the bundle source directory. -# -# This script is expected to be run via cron, but can be run manually as well. -# See usage() for invocation - -declare -r timestamp="$(date +%F-%H-%M)" -declare -r lock_file_name="mpas_bundle_cron.lock" -LOCK_FILE="" -LOGFILE="" - -init_logs() -{ - local cron_logdir="$1" - mkdir -p ${cron_logdir} || exit 1 - LOGFILE="${cron_logdir}/mpas-bundle-cron.log.${timestamp}" - touch $LOGFILE - HTML_BODY_FILE="${cron_logdir}/body.html" -} - -# where to copy html output files on whitedwarf.mmm.ucar.edu -html_dir="/web/htdocs/projects/mpas-jedi/weekly-ctests" -# available queues to use -# normal cpu hours are charged -MAIN_Q="main@desched1" -# charged @ .3 * regular rate, job may not run to completion -PREEMPT_Q="preempt@desched1" -# shared nodes, charged only for the no. of cpu's used -DEV_Q="develop@desched1" - -usage() -{ - args="-d -c [-x ] [-q ] [-a ] [ -p ] [-l ] [-o ] [-f] [-h] [-n]" - log "usage:" - log " mpas-bundle-cron.sh ${args}" - log " -d is where the mpas-bundle repo has been cloned to" - log " -c is one of gnu, intel or both" - log " -x is a suffix to add to the build directory" - log " -q is ${MAIN_Q}, ${DEV_Q} or ${PREEMPT_Q}, defaults to ${MAIN_Q}" - log " -a is the account to use when submitting PBS jbs (e.g. 'nmmm0015')" - log " -p is 1 or 2, defaults to 2 (double)" - log " -l is absolute path to the lock file which indicates the build is running," - log " default is /$lock_file_name" - log " -o is where the html files will be copied to, default is $html_dir" - log " -f means force-build, always build even if no source changed since last build" - log " -h prints help and exits" - log " -n means don't run cmake" - exit 1 -} - -log() -{ - if [ -f "${LOGFILE}" ]; then - echo -e "$1" | tee -a ${LOGFILE} - else - echo -e "$1" - fi -} - -remove_lock() -{ - rm -f "${LOCK_FILE}" -} - -another_instance() -{ - log "Cannot acquire lock on ${LOCK_FILE}" - log "There is another instance running, exiting" - exit 1 -} - -# check the return code of a pbs job, return 0 if it succeeded, else return non-zero -check_pbs_return() -{ - local jobno=$1 - local pbs_status=$(qstat -x -f $jobno | grep 'Exit_status') - - if [ ! -z "${pbs_status}" ]; then - retcode=$(echo $pbs_status|awk '{print $3}') - return $retcode - else - return 1 - fi -} - -# wait for the provided job to either show in the queue ($2 is 1) -# or for the job to leave the queue ($2 is 0) -queue_wait() -{ - local job=$1 - local pcode=$2 - local secs=$3 - - qstat ${job} &> /dev/null - local rc=$? - while [ ${rc} == "${pcode}" ]; do - sleep ${secs} - qstat ${job} &> /dev/null - rc=$? - done -} - -# write html header, including the table directive -print_header() -{ - outfile=$1 - local html_header=(" " \ - "

mpas-bundle mpas-jedi ctest results

" \ - "") - - for line in ${!html_header[@]}; do - echo -n ${html_header[$line]} >> ${outfile} - echo >> ${outfile} - done -} - -# write html to end the page -print_footer() -{ - outfile=$1 - echo "
Date Results Spack Compiler Tools Stats
" >> ${outfile} -} - -# create a script which will execute cmake and run it on derecho -run_cmake() -{ - cmake_script="${LOG_DIR}/run_cmake.sh" - -cat > $cmake_script << CMAKE_EOF -#!/bin/bash -# -cd $1 && source ../mpas-bundle/env-setup/$2-derecho.sh && if [ -f Makefile ]; then echo "make update" && make update |& tee make.update.log; fi && cmake -DCMAKE_VERBOSE_MAKEFILE=ON -DBUNDLE_SKIP_RTTOV=ON -DMPAS_DOUBLE_PRECISION=$3 ctest_update ../mpas-bundle; -CMAKE_EOF - - chmod 755 ${cmake_script} - ssh derecho.hpc.ucar.edu ${cmake_script} |& tee ${cmake_script}.log -} - -# create a script which will check for changes in git repositories, and run it on derecho -check_git_changes() -{ - check_changes_log=$2 - - check_script="${LOG_DIR}/check_changes.sh" - update_file="${LOG_DIR}/files_changed.lock" - rm ${update_file} - -cat > $check_script << EOF -#!/bin/bash -l -# -# check to see if any source files changed in git. -# this is meant to be called from automated scripts. - -# touch $update_file if sources changed -check_for_changes() { - # traverse subdirectories - dirs=\$(ls -d */) - for subdir in \$dirs; - do - - # skip directories which don't have a git repo - if [ ! -d "\${subdir}/.git" ]; then - continue - fi - - echo \$subdir - cd "\$subdir" - - # get latest code and see if it has changed - git fetch >& /dev/null - local branch=\$(git rev-parse --abbrev-ref HEAD) - local remote=\$(git remote get-url origin) - local local_commit=\$(git rev-parse --short HEAD) - local remote_commit=\$(git rev-parse --short origin/\${branch}) - if [ "\$local_commit" != "\$remote_commit" ]; then - echo "updating \${remote} in branch \${branch} to sha \$remote_commit." &>> $check_changes_log - git pull &>> $check_changes_log - echo "\${remote} in branch \${branch} has been updated to sha \$remote_commit." - touch $update_file - else - echo "No updates found for \${remote} in branch \${branch}, sha \$local_commit." - fi - - cd .. - done -} - -echo "work_dir $1" -echo "sync_file $update_file" -cd $1 -check_for_changes -EOF - - # run a script on derecho, it needs git which is not on the cron server - chmod 755 ${check_script} - log "ssh derecho.hpc.ucar.edu ${check_script} ${BUNDLE_DIR} ${update_file} |& tee ${check_changes_log}" - ssh derecho.hpc.ucar.edu "${check_script} |& tee ${check_changes_log}" - local ret_code=0 - if [ -f ${update_file} ]; then - log "update file $update_file exists" - ret_code=1 - fi - rm $update_file - return $ret_code -} - -# run the ctests -run_ctests() -{ - # input params - local scripts=$1 - local cc=$2 - local make_job=$3 - local timestamp=$4 - - # output params - local lsummary="" # 5th param - local lctest_time="" # 6th param - local lhref="href=\"./ctest.results.log.${timestamp}\"" # 7th param - - # create script to run ctest and run it, holding it until the make job finishes - mv ./${CTEST_LOGFILE} ./${CTEST_LOGFILE}.old - log "${scripts}/run_make.bundle.sh -A ${ACCOUNT} -q ${QUEUE} -p economy -x ctest -c ${cc} -N cron-${cc}-ctest-mpas -m -n" - ${scripts}/run_make.bundle.sh -A ${ACCOUNT} -q ${QUEUE} -p economy -x ctest -c ${cc} -N "cron-${cc}-ctest-mpas" -m -n - local ctest_job=$(qsub -W depend=afterok:${make_job} ./ctest.pbs.sh) || \ - { log "cannot connect to Derecho PBS" ; exit 1; } - log "${cc} ctest: ${ctest_job}" - - # wait for the ctest job to show up in the queue, check every 5 seconds - queue_wait ${ctest_job} 1 5 - - # wait for the ctest job to finish, check every 60 seconds - queue_wait ${ctest_job} 0 60 - - # get the runtimes for the PBS ctest job - lctest_time=$(qstat -xf ${ctest_job} | grep used.walltime | awk '{print $3}') - - # if there's no log file, the make was probably aborted, so ctest didn't run - if [ ! -f ${CTEST_LOGFILE} ]; then - # check the make job for success - check_pbs_return $make_job - local make_rc=$? - if [ "$make_rc" -ne 0 ]; then - lsummary="make job failed return code $make_rc" - lctest_time="NA" - lhref="" - else - check_pbs_return $ctest_job - local ctest_rc=$? - if [ "$ctest_rc" -ne 0 ]; then - lsummary="ctest job failed return code $ctest_rc" - lctest_time="NA" - lhref="" - else - lsummary="ctests didn't run, unknown failure" - lctest_time="NA" - lhref="" - fi - fi - else - lsummary=`grep 'tests .*passed' ${CTEST_LOGFILE}` - if [ -z "$lsummary" ]; then - lsummary="ctest run didn't complete" - fi - fi - log "run_ctests() summary:$lsummary href:$lhref ctest_time:$lctest_time" - eval "$5=\"$lsummary\"" - eval "$6=\"$lctest_time\"" - eval "$7=\"$lhref\"" -} - -# build the html file and copy it to the web server -make_html() -{ - local dest_dir=$1 - local make_job=$2 - local cc=$3 - local timestamp=$4 - local ctest_time=$5 - local summary=$6 - local href=$7 - local sha_file=$8 - #local dest_dir="/web/htdocs/projects/mpas-jedi/weekly-ctests" - - log "make_html() dest_dir: $dest_dir make_job=$make_job ctest_time=$ctest_time" - log " cc=$cc timestamp=$timestamp href=$href summary=$summary" - log " chref=$href sha_file=$sha_file sha_file:t=${sha_file##*/}" - - # get tool versions to put into summary table - local spack_stack=$(grep spack-stack- ../mpas-bundle/env-setup/${cc}-derecho.sh | awk -F/ '{print $8}') - log "spack-stack: ${spack_stack}" - if [ ${cc} == "gnu" ]; then - local comp="gcc" - else - local comp="intel" - fi - local compiler=$(grep "load *stack-${comp}" ../mpas-bundle/env-setup/${cc}-derecho.sh | awk '{print $3}') - log "compiler: ${compiler}" - local mpich=$(grep "load *stack-cray" ../mpas-bundle/env-setup/${cc}-derecho.sh | awk '{print $3}') - log "mpich: ${mpich}" - - - # get the runtime for the PBS make job - local make_time=$(qstat -xf ${make_job} | grep used.walltime | awk '{print $3}') - - # prepend current results to the table, then add the old entries - local stats=" Q:${QUEUE%"@desched1"} make:${make_time} ctest:${ctest_time}" - echo "${timestamp} ${summary} ${spack_stack} ${cc} ${compiler} ${mpich} ${stats} " > body.html - if [ -f ${HTML_BODY_FILE} ]; then - cat ${HTML_BODY_FILE} >> body.html - fi - cp body.html ${HTML_BODY_FILE} - - rm -f index.html - print_header index.html - cat body.html >> index.html - print_footer index.html - rm body.html - - local host="whitedwarf.mmm.ucar.edu" - local dest="${host}:${dest_dir}" - local index_tarfile="index.tar" - - if [ -f ${CTEST_LOGFILE} ]; then - local resultsfile="ctest.results.tmp" - echo ${timestamp} > ${resultsfile} - echo >> ${resultsfile} - egrep -A 20 'Test|Start' ${CTEST_LOGFILE} >> ./${resultsfile} - mv ${resultsfile} ctest.results.log.${timestamp} - log "scp ./ctest.results.log.${timestamp} ${dest}" - scp ./ctest.results.log.${timestamp} ${dest} || { log "scp ctest_results failed"; } - log "scp ${sha_file} ${dest}" - scp ${sha_file} ${dest} || { log "scp $sha_file failed"; } - mv ctest.results.log.${timestamp} ctest.results.log.old - else - log "${CTEST_LOGFILE} doesn't exist" - log "scp ${sha_file} ${dest}" - scp ${sha_file} ${dest} || { log "scp $sha_file failed"; } - fi - # clean up old index files by putting them in a tar file - log "ssh ${host} cd ${dest_dir} && tar --remove-files -rf ${index_tarfile} index.2" - ssh ${host} "cd ${dest_dir} && tar --remove-files -rf ${index_tarfile} index.2*" - mv index.html index.${timestamp}.html - log "scp ./index.${timestamp}.html ${dest}" - scp ./index.${timestamp}.html ${dest} ||{ log "scp index failed"; } - rm index.${timestamp}.html - log "ssh ${host} cd ${dest_dir} && rm -f index.html && ln -s index.${timestamp}.html index.html" - ssh ${host} "cd ${dest_dir} && rm -f index.html && ln -s index.${timestamp}.html index.html" -} - -# run cmake, make, and ctest for the provided build chain -build_and_test() -{ - local cc=$1 - local dbl_p=$2 - local html_dir=$3 - local run_cmake=$4 - local sha_file=$5 - local suffix=$6 - - local build_dir_suffix="" - if [ $dbl_p == "ON" ]; then - build_dir_suffix="2p" - else - build_dir_suffix="1p" - fi - local scripts=${BUNDLE_DIR}/env-setup/ - - #-------------------------------------------------------------- - # go to build directory, exit on failure: - BUILD_DIR="${BUNDLE_DIR}/../build-${cc}-${build_dir_suffix}" - if [ "$suffix" != "" ]; then - BUILD_DIR="${BUILD_DIR}_${suffix}" - fi - - mkdir -p ${BUILD_DIR} - log "BUNDLE_DIR ${BUNDLE_DIR}" - log "BUILD_DIR ${BUILD_DIR}" - cd ${BUILD_DIR} || { log "cannot cd ${BUILD_DIR}"; exit 1; } - log "building in: ${PWD}" - - # run cmake on a login node (compute nodes have poor internet transmission) - # block until cmake finishes - if [[ "$run_cmake" == "yes" ]]; then - run_cmake ${BUILD_DIR} ${cc} $dbl_p - fi - - # create script to run gnu make and run it - mv make.pbs.sh.log make.pbs.sh.log.old - log "${scripts}/run_make.bundle.sh -A ${ACCOUNT} -q ${QUEUE} -p economy -x make -c ${cc} -N cron-${cc}-make-mpas -m -n" - ${scripts}/run_make.bundle.sh -A ${ACCOUNT} -q ${QUEUE} -p economy -x make -c ${cc} -N "cron-${cc}-make-mpas" -m -n - local make_job=$(qsub make.pbs.sh) || { log "cannot connect to Derecho PBS"; exit 1; } - log "${cc} make: ${make_job}" - local ctest_time="" - local summary="" - local href="" - - # only run ctest for double precision builds - if [ "${dbl_p}" == "ON" ]; then - run_ctests $scripts $cc $make_job $timestamp summary ctest_time href - log "build_and_test() summary:$summary href:$href ctest_time:$ctest_time" - else - # wait for the make job - # wait for the make job to show up in the queue, check every 5 seconds - queue_wait ${make_job} 1 5 - - # wait for the make job to finish, check every 60 seconds - queue_wait ${make_job} 0 60 - summary="Single precision build - no ctests run" - ctest_time="NA" - fi - - # - # create and copy the docs to the web page directory - # - log "calling make_html() dir: $html_dir make_job $make_job ctest_time $ctest_time cc $cc timestamp $timestamp href $href summary $summary" - make_html $html_dir $make_job $cc $timestamp $ctest_time "$summary" $href $sha_file -} - -CTEST_LOGFILE="ctest.pbs.sh.log" -QUEUE=$MAIN_Q -BUNDLE_DIR="" -tools="" -precision="2" -force_build=0 -LOG_DIR="${HOME}/my_cron_logs/" -help="" -run_cmake="yes" -suffix="" -ACCOUNT="nmmm0015" - -# get comamnd line args -while getopts d:q:a:p:c:l:o:x:fhn flag -do - case "${flag}" in - d) BUNDLE_DIR="${OPTARG}";; - q) QUEUE="${OPTARG}";; - a) ACCOUNT="${OPTARG}";; - p) precision=${OPTARG};; - c) tools=${OPTARG};; - l) LOCK_FILE=${OPTARG};; - o) html_dir=${OPTARG};; - x) suffix=${OPTARG};; - f) force_build=1;; - h) help="help";; - n) run_cmake="no";; - esac -done - -init_logs $LOG_DIR -log "commandline: $0 $*" - -if [ "$help" != "" ]; then - usage -fi - -# location of the mpas-bundle source dir -if [ "$BUNDLE_DIR" = "" ]; then - log "source dir <-d source_dir> is required" - usage -fi -if [ ! -d ${BUNDLE_DIR} ]; then - log "source dir ${BUNDLE_DIR} does not exist" - usage -fi - -if [[ "$QUEUE" != "$MAIN_Q" && "$QUEUE" != "$PREEMPT_Q" && "$QUEUE" != "$DEV_Q" ]]; then - log "queue \"${QUEUE}\" is invalid" - usage -fi - -if [ "$tools" = "" ]; then - log "compiler <-c compiler> is required" - usage -fi - -if [ "$precision" = "2" ]; then - dbl_precision="ON" -else - dbl_precision="OFF" -fi - -log "q:$QUEUE precision: $precision $dbl_precision force: $force_build" - -# acquire an exclusive lock on our ${LOCK_FILE} file to make sure -# only one copy of this script is running at a time. -# wait 10 seconds between attempts to get the lockfile, -# retry 360 times ( 1 hr) , delete lockfile which is over 20 hours old. -# Also, the compiles can get stuck in PBS queues for lengthy periods, so we want to queue up -# single and double precision builds simultaneously. -if [ "$LOCK_FILE" = "" ]; then - LOCK_FILE="${BUNDLE_DIR}/$lock_file_name" -fi -lockfile -10 -r 360 -l 72000 "${LOCK_FILE}" || another_instance -trap remove_lock EXIT -log "lockfile: ${LOCK_FILE}" - -scriptdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" -log "[${timestamp}]: Running ${0} on $(hostname)\n\tfrom $(pwd)\n\tscriptdir=${scriptdir}" -log "using job queue ${QUEUE}" - -# check for any changed source, store all current git sha's in a file -declare -r sha_file="${LOG_DIR}/git_shas.$timestamp" -check_git_changes ${BUNDLE_DIR} $sha_file -declare -r git_changes=$? -log "check_git_changes returned $git_changes" - -# check to see if any source files changed, do nothing if no changes -if [ "$force_build" -eq 0 ]; then - if [ "$git_changes" -eq 1 ]; then - force_build=1 - log "forcing build, check_git_changes returned 1" - fi -fi - -if [ "$force_build" -eq 0 ]; then - log "no source changes, not running" -else - log "building with tools ${tools}" - # build gnu version and run ctest - if [[ "$tools" == "gnu" || "$tools" == "both" ]]; then - log "build_and_test gnu $dbl_precision $html_dir $run_cmake $sha_file $suffix" - build_and_test "gnu" $dbl_precision $html_dir $run_cmake $sha_file $suffix - fi - - # build intel version and run ctest - if [[ "$tools" == "intel" || "$tools" == "both" ]]; then - build_and_test "intel" $dbl_precision $html_dir $run_cmake $sha_file $suffix - fi -fi - diff --git a/env-setup/nvhpc-derecho.csh b/env-setup/nvhpc-derecho.csh new file mode 100644 index 0000000..5fd4fde --- /dev/null +++ b/env-setup/nvhpc-derecho.csh @@ -0,0 +1,21 @@ +#!/bin/csh +# + +source /etc/profile.d/z00_modules.csh + +module --force purge +module load ncarenv/24.12 + +module use /glade/work/epicufsrt/contrib/spack-stack/derecho/spack-stack-1.9.3/envs/ue-gcc-12.4.0/install/modulefiles/Core +module load stack-gcc/12.4.0 +module load ecbuild/3.7.2 +module load cmake/3.27.9 + +module load craype/2.7.31 +module load nvhpc/24.11 +module load ncarcompilers/1.0.0 +module load cray-mpich/8.1.29 +module load cuda/12.3.2 +module load parallel-netcdf/1.14.0 +module list + diff --git a/env-setup/nvhpc-derecho.sh b/env-setup/nvhpc-derecho.sh new file mode 100644 index 0000000..8f83fc0 --- /dev/null +++ b/env-setup/nvhpc-derecho.sh @@ -0,0 +1,21 @@ +#!/bin/bash +# + +source /etc/profile.d/z00_modules.sh + +module --force purge +module load ncarenv/24.12 + +module use /glade/work/epicufsrt/contrib/spack-stack/derecho/spack-stack-1.9.3/envs/ue-gcc-12.4.0/install/modulefiles/Core +module load stack-gcc/12.4.0 +module load ecbuild/3.7.2 +module load cmake/3.27.9 + +module load craype/2.7.31 +module load nvhpc/24.11 +module load ncarcompilers/1.0.0 +module load cray-mpich/8.1.29 +module load cuda/12.3.2 +module load parallel-netcdf/1.14.0 +module list + diff --git a/env-setup/run_make.bundle.sh b/env-setup/run_make.bundle.sh index 3e1be7b..a52d3c0 100755 --- a/env-setup/run_make.bundle.sh +++ b/env-setup/run_make.bundle.sh @@ -36,7 +36,7 @@ LOG="" VERBOSE="" # derecho params -DERECHO_CC="gnu intel" +DERECHO_CC="gnu intel nvhpc" DERECHO_CC_INTEL="intel" DERECHO_Q=("main" "develop" ) QUEUE_OPTS=${DERECHO_Q[@]} @@ -47,7 +47,7 @@ export GFORTRAN_CONVERT_UNIT='big_endian:101-200' usage() { - echo "usage: $0 -A account -c gnu|intel [-x make|ctest|both|echo] [-l] " + echo "usage: $0 -A account -c gnu|intel|nvhpc [-x make|ctest|both|echo] [-l] " echo " [-q queue] [-p priority][-t threads] [-N name] [-f job-file] [-m] [-n] [-e env-dir] [-h] " echo echo " account is the HPC account number" @@ -79,7 +79,7 @@ elif [ "cron" == "$HOST" ]; then fi if [ "$HPC" = "derecho" ]; then - QUEUE="-q ${DERECHO_Q[0]}" + QUEUE="-q ${DERECHO_Q[1]}" QUEUE_OPTS=${DERECHO_Q[@]} else echo "unsupported HPC, must run on HPC login node" @@ -113,14 +113,14 @@ done if [ "$ACCOUNT" = "" ]; then echo "account (-A) is required" - echo " something like nmmm0004" + echo " something like nmmm0015" echo usage fi if [ "$COMPILER" = "" ]; then echo "compiler (-c) is required" - echo "gnu or intel" + echo " one of $DERECHO_CC" echo usage fi @@ -159,8 +159,10 @@ if [ "$HPC" == "derecho" ]; then MODFILE="${ENV_DIR}/gnu-derecho.sh" elif [ "$COMPILER" == "intel" ]; then MODFILE="${ENV_DIR}/intel-derecho.sh" + elif [ "$COMPILER" == "nvhpc" ]; then + MODFILE="${ENV_DIR}/nvhpc-derecho.sh" else - echo unknown compiler: $COMPILER, must be either "gnu" or "intel" + echo "unknown compiler: $COMPILER, must be one of $DERECHO_CC" echo usage fi