From d1822a2cded56231ddc6fb0e85ae7ac1cb08825a Mon Sep 17 00:00:00 2001 From: JeffBeck-NOAA <55201531+JeffBeck-NOAA@users.noreply.github.com> Date: Tue, 22 Sep 2020 16:07:23 -0600 Subject: [PATCH] Move to a single UFS_UTILS repository; remove utility builds from regional_workflow (#17) * Changes necessary to run with a single UFS_UTILS repository and with the new chgres_cube * Changes to Externals.cfg to point to the EMC_post release branch and remove the second UFS_UTILS repo * Change UFS_UTILS source directory name (remove _develop) * Changes to build regional_grid on Jet * Remove builds for utilities previously in regional_workflow * Add new UFS_UTILS utilities to install_all.sh * Remove spaces. * Changes to Externals.cfg to point to new UFS_UTILS and regional_workflow, plus develop branch of ufs_weather_model and release branch of EMC_post * Revert back to NCAR fork of ufs_weather_model, since develop branch build of ufs-community/ufs_weather_model fails on Hera * Point to Larissa's fork of UFS_UTILS for new bug fixes * Point to my fork of UFS_UTILS --- Externals.cfg | 21 ++--- src/build_UFS_UTILS.sh | 50 ++++++++++++ src/build_all.sh | 12 +-- src/build_regional_grid.sh | 128 ------------------------------ src/build_utils.sh | 158 ------------------------------------- src/install_all.sh | 29 ++++--- src/partial_build.sh | 3 +- src/regional_build.cfg | 7 +- 8 files changed, 78 insertions(+), 330 deletions(-) create mode 100755 src/build_UFS_UTILS.sh delete mode 100755 src/build_regional_grid.sh delete mode 100755 src/build_utils.sh diff --git a/Externals.cfg b/Externals.cfg index 7b86c5f2ae..bf1e19230d 100644 --- a/Externals.cfg +++ b/Externals.cfg @@ -8,20 +8,11 @@ required = True [ufs_utils] protocol = git -repo_url = https://github.com/NCAR/UFS_UTILS +repo_url = https://github.com/JeffBeck-NOAA/UFS_UTILS # Specify either a branch name or a hash but not both. -#branch = dtc/develop -hash = 6c9db728 -local_path = src/UFS_UTILS_develop -required = True - -[ufs_utils_chgres] -protocol = git -repo_url = https://github.com/NCAR/UFS_UTILS -# Specify either a branch name or a hash but not both. -#branch = feature/chgres_grib2 -hash = 53193694 -local_path = src/UFS_UTILS_chgres_grib2 +branch = feature/regional_release +#hash = 6c9db728 +local_path = src/UFS_UTILS required = True [ufs_weather_model] @@ -37,8 +28,8 @@ required = True protocol = git repo_url = https://github.com/NOAA-EMC/EMC_post # Specify either a branch name or a hash but not both. -#branch = develop -hash = dceca26 +branch = release/public-v2 +#hash = local_path = src/EMC_post required = True diff --git a/src/build_UFS_UTILS.sh b/src/build_UFS_UTILS.sh new file mode 100755 index 0000000000..512f5bcff8 --- /dev/null +++ b/src/build_UFS_UTILS.sh @@ -0,0 +1,50 @@ +#!/bin/sh +#========================================================================== +# +# Description: Builds chgres, chgres_cube, orog, fre-nctools, sfc_climo_gen, +# regional_grid, global_equiv_resol, and mosaic_file. +# +# Usage: ./build_UFS_UTILS.sh +# +#========================================================================== +set -eux +cwd=`pwd` + +build_dir="${cwd}" +UFS_UTILS="${cwd}/UFS_UTILS" +export USE_PREINST_LIBS="true" + +#------------------------------------ +# END USER DEFINED STUFF +#------------------------------------ + +logs_dir=${cwd}/logs +if [ ! -d $logs_dir ]; then + echo "Creating logs folder" + mkdir $logs_dir +fi + +#------------------------------------ +# INCLUDE PARTIAL BUILD +#------------------------------------ + +. ./partial_build.sh + +#------------------------------------ +# build UFS_UTILS +#------------------------------------ +$Build_UFS_UTILS && { +echo " .... Building UFS_UTILS .... " +cd $UFS_UTILS +./build_all.sh > $logs_dir/build_UFS_UTILS.log 2>&1 +if [ $? -eq 0 ] ; then + echo "UFS_UTILS build SUCCEEDED" +else + echo "UFS_UTILS build FAILED see $logs_dir/build_UFS_UTILS.log" + exit 1 +fi +} + +cd $build_dir + +echo 'Building UFS_UTILS done' diff --git a/src/build_all.sh b/src/build_all.sh index 7fa30e85f5..2e54f5e502 100755 --- a/src/build_all.sh +++ b/src/build_all.sh @@ -78,15 +78,15 @@ fi } #------------------------------------ -# build utils +# build UTILS #------------------------------------ -$Build_utils && { -echo " .... Building utils .... " -./build_utils.sh > $logs_dir/build_utils.log 2>&1 +$Build_UFS_UTILS && { +echo " .... Building UFS_UTILS .... " +./build_UFS_UTILS.sh > $logs_dir/build_UFS_UTILS.log 2>&1 if [ $? -eq 0 ] ; then - echo "Utils build SUCCEEDED" + echo "UFS_UTILS build SUCCEEDED" else - echo "Utils build FAILED see $logs_dir/build_utils.log" + echo "UFS_UTILS build FAILED see $logs_dir/build_UFS_UTILS.log" exit 1 fi } diff --git a/src/build_regional_grid.sh b/src/build_regional_grid.sh deleted file mode 100755 index d17c5f2b0d..0000000000 --- a/src/build_regional_grid.sh +++ /dev/null @@ -1,128 +0,0 @@ -#!/bin/sh -set -eux -# -# Check for input argument: this should be the "platform" if it exists. -# -if [ $# -eq 0 ]; then - echo - echo "No 'platform' argument supplied" - echo "Using directory structure to determine machine settings" - platform='' -else - platform=$1 -fi -# -source ./machine-setup.sh > /dev/null 2>&1 -if [ $platform = "wcoss_cray" ]; then - platform="cray" -fi -# -# Set the name of the package. This will also be the name of the execu- -# table that will be built. -# -package_name="regional_grid" -# -# Make an exec folder if it doesn't already exist. -# -exec_dir=`pwd`/../exec -mkdir -p ${exec_dir} -# -# Change directory to where the source code is located. -# -srcDir=`pwd`/../regional_workflow/sorc/${package_name}.fd/ -cd ${srcDir} -# -# The build will be performed in a temporary directory. If the build is -# successful, the temporary directory will be removed. -# -tmpDir=`pwd`/build -mkdir -p $tmpDir -cd $tmpDir -# -# Load modules. -# -set +x -module list -#module use ../../../modulefiles/regional_workflow -#module load ${package_name}.${target} -module use ../../../modulefiles/codes/${target} -module load ${package_name} -module list -set -x -# -MPICH_UNEX_BUFFER_SIZE=256m -MPICH_MAX_SHORT_MSG_SIZE=64000 -MPICH_PTL_UNEX_EVENTS=160k -KMP_STACKSIZE=2g -F_UFMTENDIAN=big -# -# HDF5 and NetCDF directories. -# -if [ $platform = "cray" ]; then - HDF5=${HDF5_DIR} - NETCDF=${NETCDF_DIR} -elif [ $platform = "theia" ]; then - HDF5_DIR=$HDF5 - NETCDF_DIR=$NETCDF -elif [ $platform = "hera" ]; then - HDF5_DIR=$HDF5 - NETCDF_DIR=$NETCDF -elif [ $platform = "cheyenne" ]; then - NETCDF_DIR=$NETCDF - HDF5_DIR=$NETCDF #HDF5 resides with NETCDF on Cheyenne - export HDF5=$NETCDF #HDF5 used in Makefile_cheyenne -elif [ $platform = "jet" ]; then - HDF5_DIR=$HDF5 - NETCDF_DIR=$NETCDF -elif [ $platform = "stampede" ]; then - HDF5_DIR=$TACC_HDF5_DIR - NETCDF_DIR=$TACC_NETCDF_DIR -fi -# -# Create alias for "make". -# -if [ $platform = "cray" ]; then - alias make="make HDF5_HOME=${HDF5} NETCDF_HOME=${NETCDF} NC_BLKSZ=64K SITE=${platform}" -else - alias make="make HDF5_HOME=${HDF5_DIR} NETCDF_HOME=${NETCDF_DIR} NC_BLKSZ=64K SITE=${platform}" -fi - -set +x -echo -echo "////////////////////////////////////////////////////////////////////////////////" -echo "Building package \"$package_name\" for platform \"$platform\" ..." -echo "////////////////////////////////////////////////////////////////////////////////" -echo -set -x -# -# Copy all source code and the makefile to the temporary directory. -# Then clean and build from scratch. -# -cp $srcDir/*.f90 $tmpDir -cp $srcDir/Makefile_${platform} $tmpDir/Makefile -make clean -make -# -# Check if the executable was successfully built. If so, move it to the -# exec subdirectory under the home directory. If not, exit with an er- -# ror message. -# -target="${package_name}" -if [ -f $target ]; then - mv $target $exec_dir -else - echo "Error during '$target' build" - exit 1 -fi -# -# Remove the temporary build directory. -# -set +x -echo -echo "Removing temporary build directory ..." -echo -rm -fr $tmpDir - -echo "Done." - -exit diff --git a/src/build_utils.sh b/src/build_utils.sh deleted file mode 100755 index 77289fc6a6..0000000000 --- a/src/build_utils.sh +++ /dev/null @@ -1,158 +0,0 @@ -#!/bin/sh -#========================================================================== -# -# Description: Builds chgres, chgres_cube, orog, fre-nctools, sfc_climo_gen, -# regional_grid, global_equiv_resol, and mosaic_file. -# -# Note: This scripts build two chgres_cube from two different -# branches of the UFS_UTILS repository. -# -# Note: The global_equiv_resol, mosaic_file and regional_grid -# reside in the ../regional_workflow/sorc directory and are -# cloned with that repository. -# -# Usage: ./build_utils.sh -# -#========================================================================== -set -eux -cwd=`pwd` - -build_dir="${cwd}" -UFS_UTILS_DEV="${cwd}/UFS_UTILS_develop/sorc" -UFS_UTILS_CHGRES_GRIB2="${cwd}/UFS_UTILS_chgres_grib2/sorc" -build_dir="${cwd}" -export USE_PREINST_LIBS="true" - -#------------------------------------ -# END USER DEFINED STUFF -#------------------------------------ - -logs_dir=${cwd}/logs -if [ ! -d $logs_dir ]; then - echo "Creating logs folder" - mkdir $logs_dir -fi - -#------------------------------------ -# INCLUDE PARTIAL BUILD -#------------------------------------ - -. ./partial_build.sh - -#------------------------------------ -# build chgres -#------------------------------------ -$Build_chgres && { -echo " .... Chgres build not currently supported .... " -#echo " .... Building chgres .... " -#./build_chgres.sh > $logs_dir/build_chgres.log 2>&1 -} - -#------------------------------------ -# build chgres_cube -#------------------------------------ -$Build_chgres_cube && { -echo " .... Building chgres_cube .... " -cd $UFS_UTILS_CHGRES_GRIB2 -./build_chgres_cube.sh > $logs_dir/build_chgres_cube.log 2>&1 -if [ $? -eq 0 ] ; then - echo "chgres_cube build SUCCEEDED" -else - echo "chgres_cube build FAILED see $logs_dir/build_chgres_cube.log" - exit 1 -fi -} - -#------------------------------------ -# build orog -#------------------------------------ -$Build_orog && { -echo " .... Building orog .... " -cd $UFS_UTILS_DEV -./build_orog.sh > $logs_dir/build_orog.log 2>&1 -if [ $? -eq 0 ] ; then - echo "orog build SUCCEEDED" -else - echo "orog build FAILED see $logs_dir/build_orog.log" - exit 1 -fi -} - -#------------------------------------ -# build fre-nctools -#------------------------------------ -$Build_nctools && { -echo " .... Building fre-nctools .... " -cd $UFS_UTILS_DEV -./build_fre-nctools.sh > $logs_dir/build_fre-nctools.log 2>&1 -if [ $? -eq 0 ] ; then - echo "fre-nctools build SUCCEEDED" -else - echo "fre-nctools build FAILED see $logs_dir/build_fre-nctools.log" - exit 1 -fi -} - -#------------------------------------ -# build sfc_climo_gen -#------------------------------------ -$Build_sfc_climo_gen && { -echo " .... Building sfc_climo_gen .... " -cd $UFS_UTILS_DEV -./build_sfc_climo_gen.sh > $logs_dir/build_sfc_climo_gen.log 2>&1 -if [ $? -eq 0 ] ; then - echo "sfc_climo_gen build SUCCEEDED" -else - echo "sfc_climo_gen build FAILED see $logs_dir/build_sfc_climo_gen.log" - exit 1 -fi -} - -#------------------------------------ -# build regional_grid -#------------------------------------ -$Build_regional_grid && { -echo " .... Building regional_grid .... " -cd $build_dir -./build_regional_grid.sh > $logs_dir/build_regional_grid.log 2>&1 -if [ $? -eq 0 ] ; then - echo "regional_grid build SUCCEEDED" -else - echo "regional_grid build FAILED see $logs_dir/build_regional_grid.log" - exit 1 -fi -} - -#------------------------------------ -# build global_equiv_resol -#------------------------------------ -$Build_global_equiv_resol && { -echo " .... Building global_equiv_resol .... " -cd $build_dir -./build_global_equiv_resol.sh > $logs_dir/build_global_equiv_resol.log 2>&1 -if [ $? -eq 0 ] ; then - echo "global_equiv_resol build SUCCEEDED" -else - echo "global_equiv_resol build FAILED see $logs_dir/build_global_equiv_resol.log" - exit 1 -fi -} - -#------------------------------------ -# build mosaic file -#------------------------------------ -$Build_mosaic_file && { -echo " .... Building mosaic_file .... " -cd $build_dir -./build_mosaic_file.sh > $logs_dir/build_mosaic_file.log 2>&1 -if [ $? -eq 0 ] ; then - echo "mosaic_file build SUCCEEDED" -else - echo "mosaic_file build FAILED see $logs_dir/build_mosaic.log" - exit 1 -fi -} - -cd $build_dir - -echo 'Building utils done' diff --git a/src/install_all.sh b/src/install_all.sh index b6ce2c73d7..c2763d73bb 100755 --- a/src/install_all.sh +++ b/src/install_all.sh @@ -20,12 +20,12 @@ fi #------------------------------------ # install forecast #------------------------------------ -#${CP} NEMSfv3gfs/fv3.exe ../exec/regional_forecast.x +#${CP} NEMSfv3gfs/fv3.exe ../exec/regional_forecast.x #------------------------------------ # install post #------------------------------------ - ${CP} EMC_post/exec/* ../exec/ncep_post + ${CP} EMC_post/exec/* ../exec/ncep_post #------------------------------------ # install chgres @@ -35,33 +35,32 @@ fi #------------------------------------ # install chgres_cube #------------------------------------ - ${CP} UFS_UTILS_chgres_grib2/exec/chgres_cube.exe ../exec/chgres_cube.exe + ${CP} UFS_UTILS/exec/chgres_cube ../exec/chgres_cube.exe #------------------------------------ # install orog #------------------------------------ - ${CP} UFS_UTILS_develop/exec/orog.x ../exec/orog.x + ${CP} UFS_UTILS/exec/orog ../exec/orog.x #------------------------------------ # install sfc_climo_gen #------------------------------------ - ${CP} UFS_UTILS_develop/exec/sfc_climo_gen ../exec/sfc_climo_gen + ${CP} UFS_UTILS/exec/sfc_climo_gen ../exec/sfc_climo_gen #------------------------------------ -# install regional_grid +# install regional_esg_grid #------------------------------------ -# ${CP} regional_utils.fd/exec/regional_grid ../exec/regional_grid.x + ${CP} UFS_UTILS/exec/regional_esg_grid ../exec/regional_grid #------------------------------------ -# install fre-nctools +# install other utilities #------------------------------------ - ${CP} UFS_UTILS_develop/exec/make_hgrid ../exec/make_hgrid - ${CP} UFS_UTILS_develop/exec/make_hgrid_parallel ../exec/make_hgrid_parallel - ${CP} UFS_UTILS_develop/exec/make_solo_mosaic ../exec/make_solo_mosaic - ${CP} UFS_UTILS_develop/exec/fregrid ../exec/fregrid - ${CP} UFS_UTILS_develop/exec/fregrid_parallel ../exec/fregrid_parallel - ${CP} UFS_UTILS_develop/exec/filter_topo ../exec/filter_topo - ${CP} UFS_UTILS_develop/exec/shave.x ../exec/shave.x + ${CP} UFS_UTILS/exec/make_hgrid ../exec/make_hgrid + ${CP} UFS_UTILS/exec/make_solo_mosaic ../exec/make_solo_mosaic + ${CP} UFS_UTILS/exec/fregrid ../exec/fregrid + ${CP} UFS_UTILS/exec/filter_topo ../exec/filter_topo + ${CP} UFS_UTILS/exec/shave ../exec/shave.x + ${CP} UFS_UTILS/exec/global_equiv_resol ../exec/global_equiv_resol #------------------------------------ # install gsi diff --git a/src/partial_build.sh b/src/partial_build.sh index 5e2b815bb3..3579eee046 100755 --- a/src/partial_build.sh +++ b/src/partial_build.sh @@ -2,8 +2,7 @@ # define the array of the name of build program # declare -a Build_prg=("Build_libs" "Build_forecast" "Build_gsi" \ - "Build_post" "Build_utils" "Build_chgres" "Build_chgres_cube" \ - "Build_orog" "Build_sfc_climo_gen" "Build_regional_grid" "Build_nctools" \ + "Build_post" "Build_UFS_UTILS" "Build_regional_grid" \ "Build_mosaic_file" "Build_global_equiv_resol") # diff --git a/src/regional_build.cfg b/src/regional_build.cfg index b2ea405fff..917493a06a 100644 --- a/src/regional_build.cfg +++ b/src/regional_build.cfg @@ -5,13 +5,8 @@ Building forecast (forecast) .......................... yes Building gsi (gsi) .................................... no Building post (post) .................................. yes - Building utils (utils) ................................ yes - Building chgres (chgres) .............................. no - Building chgres_cube (chgres_cube) .................... yes - Building sfc_climo_gen (sfc_climo_gen) ................ yes + Building UFS_UTILS (UFS_UTILS) ........................ yes Building regional_grid (regional_grid) ................ yes - Building orog (orog) .................................. yes - Building fre-nctools (nctools) ........................ yes Building mosaic_file (mosaic_file) .................... yes Building global_equiv_resol (global_equiv_resol) ...... yes