Merge remote-tracking branch 'emc/develop' into feature/spack-stack

DavidHuber-NOAA · Nov 7, 2023 · 67fa16f · 67fa16f
2 parents 69513ea + 5183c43
commit 67fa16f
Show file tree

Hide file tree

Showing 126 changed files with 1,864 additions and 1,927 deletions.
diff --git a/Externals.cfg b/Externals.cfg
@@ -8,7 +8,7 @@ protocol = git
 required = True
 
 [UFS]
-tag = 4d05445
+tag = 63a43d9
 local_path = sorc/ufs_model.fd
 repo_url = https://github.com/ufs-community/ufs-weather-model.git
 protocol = git
@@ -57,7 +57,7 @@ protocol = git
 required = False
 
 [GDASApp]
-hash = d347d22
+hash = 7659c10
 local_path = sorc/gdas.cd
 repo_url = https://github.com/NOAA-EMC/GDASApp.git
 protocol = git

diff --git a/ci/platforms/config.hera b/ci/platforms/config.hera
@@ -4,5 +4,5 @@ export GFS_CI_ROOT=/scratch1/NCEPDEV/global/Terry.McGuinness/GFS_CI_ROOT
 export ICSDIR_ROOT=/scratch1/NCEPDEV/global/glopara/data/ICSDIR
 export STMP="/scratch1/NCEPDEV/stmp2/${USER}"
 export SLURM_ACCOUNT=nems
-export max_concurrent_cases=2
-export max_concurrent_pr=2
+export max_concurrent_cases=5
+export max_concurrent_pr=4
diff --git a/ci/platforms/config.orion b/ci/platforms/config.orion
@@ -4,5 +4,5 @@ export GFS_CI_ROOT=/work2/noaa/stmp/GFS_CI_ROOT
 export ICSDIR_ROOT=/work/noaa/global/glopara/data/ICSDIR
 export STMP="/work/noaa/stmp/${USER}"
 export SLURM_ACCOUNT=nems
-export max_concurrent_cases=2
-export max_concurrent_pr=2
+export max_concurrent_cases=5
+export max_concurrent_pr=4
diff --git a/ci/scripts/check_ci.sh b/ci/scripts/check_ci.sh
@@ -70,6 +70,8 @@ fi
 
 for pr in ${pr_list}; do
   id=$("${GH}" pr view "${pr}" --repo "${REPO_URL}" --json id --jq '.id')
+  output_ci="${GFS_CI_ROOT}/PR/${pr}/output_runtime_${id}"
+  output_ci_single="${GFS_CI_ROOT}/PR/${pr}/output_runtime_single.log"
   echo "Processing Pull Request #${pr} and looking for cases"
   pr_dir="${GFS_CI_ROOT}/PR/${pr}"
 
@@ -83,15 +85,16 @@ for pr in ${pr_list}; do
   # shellcheck disable=SC2312
   if [[ -z $(ls -A "${pr_dir}/RUNTESTS/EXPDIR") ]] ; then
     "${GH}" pr edit --repo "${REPO_URL}" "${pr}" --remove-label "CI-${MACHINE_ID^}-Running" --add-label "CI-${MACHINE_ID^}-Passed"
-    sed -i "s/\`\`\`//2g" "${GFS_CI_ROOT}/PR/${pr}/output_${id}"
-    "${GH}" pr comment "${pr}" --repo "${REPO_URL}" --body-file "${GFS_CI_ROOT}/PR/${pr}/output_${id}"
+    sed -i "1 i\`\`\`" "${output_ci}"
+    sed -i "1 i\All CI Test Cases Passed:" "${output_ci}"
+    "${GH}" pr comment "${pr}" --repo "${REPO_URL}" --body-file "${output_ci}"
     "${ROOT_DIR}/ci/scripts/pr_list_database.py" --remove_pr "${pr}" --dbfile "${pr_list_dbfile}"
     # Check to see if this PR that was opened by the weekly tests and if so close it if it passed on all platforms
     weekly_labels=$(${GH} pr view "${pr}" --repo "${REPO_URL}"  --json headRefName,labels,author --jq 'select(.author.login | contains("emcbot")) | select(.headRefName | contains("weekly_ci")) | .labels[].name ') || true
     if [[ -n "${weekly_labels}" ]]; then
-      num_platforms=$(find ../platforms -type f -name "config.*" | wc -l)
+      num_platforms=$(find "${ROOT_DIR}/ci/platforms" -type f -name "config.*" | wc -l)
       passed=0
-      for platforms in ../platforms/config.*; do
+      for platforms in "${ROOT_DIR}"/ci/platforms/config.*; do
         machine=$(basename "${platforms}" | cut -d. -f2)
         if [[ "${weekly_labels}" == *"CI-${machine^}-Passed"* ]]; then
           ((passed=passed+1))
@@ -121,22 +124,20 @@ for pr in ${pr_list}; do
     rocoto_stat_output=$("${rocotostat}" -w "${xml}" -d "${db}" -s | grep -v CYCLE) || true
     num_cycles=$(echo "${rocoto_stat_output}" | wc -l) || true
     num_done=$(echo "${rocoto_stat_output}" | grep -c Done) || true
-    num_succeeded=$("${rocotostat}" -w "${xml}" -d "${db}" -a | grep -c SUCCEEDED) || true
+    # num_succeeded=$("${rocotostat}" -w "${xml}" -d "${db}" -a | grep -c SUCCEEDED) || true
     echo "${pslot} Total Cycles: ${num_cycles} number done: ${num_done}" || true
     num_failed=$("${rocotostat}" -w "${xml}" -d "${db}" -a | grep -c -E 'FAIL|DEAD') || true
     if [[ ${num_failed} -ne 0 ]]; then
-      {
-        echo "Experiment ${pslot} Terminated: *FAILED*"
-        echo "Experiment ${pslot} Terminated with ${num_failed} tasks failed at $(date)" || true
-      } >> "${GFS_CI_ROOT}/PR/${pr}/output_${id}"
-      error_logs=$("${rocotostat}" -d "${db}" -w "${xml}" | grep -E 'FAIL|DEAD' | awk '{print "-c", $1, "-t", $2}' | xargs "${rocotocheck}" -d "${db}" -w "${xml}" | grep join | awk '{print $2}') || true
       "${GH}" pr edit --repo "${REPO_URL}" "${pr}" --remove-label "CI-${MACHINE_ID^}-Running" --add-label "CI-${MACHINE_ID^}-Failed"
+      error_logs=$("${rocotostat}" -d "${db}" -w "${xml}" | grep -E 'FAIL|DEAD' | awk '{print "-c", $1, "-t", $2}' | xargs "${rocotocheck}" -d "${db}" -w "${xml}" | grep join | awk '{print $2}') || true
       {
+       echo "Experiment ${pslot} Terminated: *** FAILED ***"
+       echo "Experiment ${pslot} Terminated with ${num_failed} tasks failed at $(date)" || true
        echo "Error logs:"
        echo "${error_logs}"
-      } >> "${GFS_CI_ROOT}/PR/${pr}/output_${id}"
-      sed -i "s/\`\`\`//2g" "${GFS_CI_ROOT}/PR/${pr}/output_${id}"
-      "${GH}" pr comment "${pr}" --repo "${REPO_URL}" --body-file "${GFS_CI_ROOT}/PR/${pr}/output_${id}"
+      } >> "${output_ci}"
+      sed -i "1 i\`\`\`" "${output_ci}"
+      "${GH}" pr comment "${pr}" --repo "${REPO_URL}" --body-file "${output_ci}"
       "${ROOT_DIR}/ci/scripts/pr_list_database.py" --remove_pr "${pr}" --dbfile "${pr_list_dbfile}"
       for kill_cases in "${pr_dir}/RUNTESTS/"*; do
          pslot=$(basename "${kill_cases}")
@@ -145,16 +146,16 @@ for pr in ${pr_list}; do
       break
     fi
     if [[ "${num_done}" -eq  "${num_cycles}" ]]; then
-      {
-        echo "Experiment ${pslot} completed: *SUCCESS*"
-        echo "Experiment ${pslot} Completed at $(date)" || true
-        echo "with ${num_succeeded} successfully completed jobs" || true
-      } >> "${GFS_CI_ROOT}/PR/${pr}/output_${id}"
-      sed -i "s/\`\`\`//2g" "${GFS_CI_ROOT}/PR/${pr}/output_${id}"
-      "${GH}" pr comment "${pr}" --repo "${REPO_URL}" --body-file "${GFS_CI_ROOT}/PR/${pr}/output_${id}"
       #Remove Experment cases that completed successfully
       rm -Rf "${pslot_dir}"
       rm -Rf "${pr_dir}/RUNTESTS/COMROT/${pslot}"
+      rm -f "${output_ci_single}"
+      # echo "\`\`\`" > "${output_ci_single}"
+      DATE=$(date)
+      echo "Experiment ${pslot} **SUCCESS** ${DATE}" >> "${output_ci_single}"
+      echo "Experiment ${pslot} **SUCCESS** at ${DATE}" >> "${output_ci}"
+      "${GH}" pr comment "${pr}" --repo "${REPO_URL}" --body-file "${output_ci_single}"
+
     fi
   done
 done
diff --git a/ci/scripts/clone-build_ci.sh b/ci/scripts/clone-build_ci.sh
@@ -40,21 +40,13 @@ while getopts "p:d:o:h" opt; do
 done
 
 cd "${repodir}" || exit 1
-# clone copy of repo
 if [[ -d global-workflow ]]; then
   rm -Rf global-workflow
 fi
 
 git clone "${REPO_URL}"
 cd global-workflow || exit 1
 
-pr_state=$("${GH}" pr view "${PR}" --json state --jq '.state')
-if [[ "${pr_state}" != "OPEN" ]]; then
-  title=$("${GH}" pr view "${PR}" --json title --jq '.title')
-  echo "PR ${title} is no longer open, state is ${pr_state} ... quitting"
-  exit 1
-fi
-
 # checkout pull request
 "${GH}" pr checkout "${PR}" --repo "${REPO_URL}"
 HOMEgfs="${PWD}"
@@ -78,19 +70,17 @@ echo "${commit}" > "../commit"
 # run checkout script
 cd sorc || exit 1
 set +e
-# TODO enable -u later when GDASApp tests are added
-./checkout.sh -c -g >> log.checkout 2>&1
+./checkout.sh -c -g -u >> log.checkout 2>&1
 checkout_status=$?
 if [[ ${checkout_status} != 0 ]]; then
   {
-    echo "Checkout:                      *FAILED*"
+    echo "Checkout: *** FAILED ***"
     echo "Checkout: Failed at $(date)" || true
     echo "Checkout: see output at ${PWD}/log.checkout"
   } >> "${outfile}"
   exit "${checkout_status}"
 else
   {
-    echo "Checkout:                      *SUCCESS*"
     echo "Checkout: Completed at $(date)" || true
   } >> "${outfile}"
 fi
@@ -104,19 +94,26 @@ build_status=$?
 
 if [[ ${build_status} != 0 ]]; then
   {
-    echo "Build:                         *FAILED*"
+    echo "Build: *** FAILED ***"
     echo "Build: Failed at $(date)" || true
     echo "Build: see output at ${PWD}/log.build"
   } >> "${outfile}"
   exit "${build_status}"
 else
   {
-    echo "Build:                         *SUCCESS*"
     echo "Build: Completed at $(date)" || true
   } >> "${outfile}"
 fi
 
 ./link_workflow.sh
+link_status=$?
+if [[ ${link_status} != 0 ]]; then
+  {
+    echo "Link: *** FAILED ***"
+    echo "Link: Failed at $(date)" || true
+  } >> "${outfile}"
+  exit "${link_status}"
+fi
 
 echo "check/build/link test completed"
 exit "${build_status}"
diff --git a/ci/scripts/driver.sh b/ci/scripts/driver.sh
@@ -119,7 +119,10 @@ for pr in ${pr_list}; do
   # call clone-build_ci to clone and build PR
   id=$("${GH}" pr view "${pr}" --repo "${REPO_URL}" --json id --jq '.id')
   set +e
-  "${ROOT_DIR}/ci/scripts/clone-build_ci.sh" -p "${pr}" -d "${pr_dir}" -o "${pr_dir}/output_${id}"
+  output_ci="${pr_dir}/output_build_${id}"
+  rm -f "${output_ci}"
+  "${ROOT_DIR}/ci/scripts/clone-build_ci.sh" -p "${pr}" -d "${pr_dir}" -o "${output_ci}"
+  #echo "SKIPPING: ${ROOT_DIR}/ci/scripts/clone-build_ci.sh"
   ci_status=$?
   ##################################################################
   # Checking for special case when Ready label was updated
@@ -138,7 +141,7 @@ for pr in ${pr_list}; do
     #setup space to put an experiment
     # export RUNTESTS for yaml case files to pickup
     export RUNTESTS="${pr_dir}/RUNTESTS"
-    #rm -Rf "${pr_dir:?}/RUNTESTS/"*
+    rm -Rf "${pr_dir:?}/RUNTESTS/"*
 
     #############################################################
     # loop over every yaml file in the PR's ci/cases
@@ -154,39 +157,47 @@ for pr in ${pr_list}; do
       export pslot="${case}_${pr_sha}"
       rm -Rf "${STMP}/RUNDIRS/${pslot}"
       set +e
-      "${HOMEgfs}/workflow/create_experiment.py" --yaml "${HOMEgfs}/ci/cases/pr/${case}.yaml"
+      export LOGFILE_PATH="${HOMEgfs}/ci/scripts/create_experiment.log"
+      rm -f "${LOGFILE_PATH}"
+      "${HOMEgfs}/workflow/create_experiment.py" --yaml "${HOMEgfs}/ci/cases/pr/${case}.yaml" 2>&1 "${LOGFILE_PATH}"
       ci_status=$?
       set -e
       if [[ ${ci_status} -eq 0 ]]; then
+        last_line=$(tail -1 "${LOGFILE_PATH}")
+        if [[ "${last_line}" == *"Skipping creation"* ]]; then
+          action="Skipped"
+        else
+          action="Completed"
+        fi
         {
-          echo "Created experiment:            *SUCCESS*"
-          echo "Case setup: Completed at $(date) for experiment ${pslot}" || true
-        } >> "${GFS_CI_ROOT}/PR/${pr}/output_${id}"
-        "${GH}" pr edit --repo "${REPO_URL}" "${pr}" --remove-label "CI-${MACHINE_ID^}-Building" --add-label "CI-${MACHINE_ID^}-Running"
-        "${ROOT_DIR}/ci/scripts/pr_list_database.py" --dbfile "${pr_list_dbfile}" --update_pr "${pr}" Open Running
+          echo "Case setup: ${action} for experiment ${pslot}" || true
+        } >> "${output_ci}"
       else
         {
-          echo "Failed to create experiment:  *FAIL* ${pslot}"
-          echo "Experiment setup: failed at $(date) for experiment ${pslot}" || true
+          echo "*** Failed *** to create experiment: ${pslot}"
           echo ""
-          cat "${HOMEgfs}/ci/scripts/"setup_*.std*
-        } >> "${GFS_CI_ROOT}/PR/${pr}/output_${id}"
+          cat "${LOGFILE_PATH}"
+        } >> "${output_ci}"
         "${GH}" pr edit "${pr}" --repo "${REPO_URL}" --remove-label "CI-${MACHINE_ID^}-Building" --add-label "CI-${MACHINE_ID^}-Failed"
         "${ROOT_DIR}/ci/scripts/pr_list_database.py" --remove_pr "${pr}" --dbfile "${pr_list_dbfile}"
+        "${GH}" pr comment "${pr}" --repo "${REPO_URL}" --body-file "${output_ci}"
+        exit 1
       fi
     done
 
+    "${GH}" pr edit --repo "${REPO_URL}" "${pr}" --remove-label "CI-${MACHINE_ID^}-Building" --add-label "CI-${MACHINE_ID^}-Running"
+    "${ROOT_DIR}/ci/scripts/pr_list_database.py" --dbfile "${pr_list_dbfile}" --update_pr "${pr}" Open Running
+    "${GH}" pr comment "${pr}" --repo "${REPO_URL}" --body-file "${output_ci}"
+
   else
     {
-      echo '```'
       echo "Failed on cloning and building global-workflowi PR: ${pr}"
       echo "CI on ${MACHINE_ID^} failed to build on $(date) for repo ${REPO_URL}" || true
-    } >> "${GFS_CI_ROOT}/PR/${pr}/output_${id}"
+    } >> "${output_ci}"
     "${GH}" pr edit "${pr}" --repo "${REPO_URL}" --remove-label "CI-${MACHINE_ID^}-Building" --add-label "CI-${MACHINE_ID^}-Failed"
     "${ROOT_DIR}/ci/scripts/pr_list_database.py" --remove_pr "${pr}" --dbfile "${pr_list_dbfile}"
+    "${GH}" pr comment "${pr}" --repo "${REPO_URL}" --body-file "${output_ci}"
   fi
-  sed -i "s/\`\`\`//2g" "${GFS_CI_ROOT}/PR/${pr}/output_${id}"
-  "${GH}" pr comment "${pr}" --repo "${REPO_URL}" --body-file "${GFS_CI_ROOT}/PR/${pr}/output_${id}"
 
 done # looping over each open and labeled PR
 

diff --git a/docs/source/index.rst b/docs/source/index.rst
@@ -10,7 +10,7 @@ Status
 ======
 
 * State of develop (HEAD) branch: GFSv17+ development
-* State of operations (dev/gfs.v16 branch): GFS v16.3.9 `tag: [gfs.v16.3.9] <https://github.com/NOAA-EMC/global-workflow/releases/tag/gfs.v16.3.9>`_
+* State of operations (dev/gfs.v16 branch): GFS v16.3.10 `tag: [gfs.v16.3.10] <https://github.com/NOAA-EMC/global-workflow/releases/tag/gfs.v16.3.10>`_
 
 =============
 Code managers

diff --git a/docs/source/noaa_csp.rst b/docs/source/noaa_csp.rst
@@ -183,14 +183,14 @@ the global-workflow.
 
 The software stack supporting the ``develop`` branch of the
 global-workflow is provided for the user and is located beneath
-``/contrib/global-workflow/spack-stack``. The modules required for the
+``/contrib/emc_static/spack-stack``. The modules required for the
 global-workflow execution may be loaded as follows.
 
 .. code-block:: bash
 
    user@host:$ module unuse /opt/cray/craype/default/modulefiles
    user@host:$ module unuse /opt/cray/modulefiles
-   user@host:$ module use /contrib/global-workflow/spack-stack/miniconda/modulefiles/miniconda
+   user@host:$ module use /contrib/emc_static/spack-stack/miniconda/modulefiles/miniconda
    user@host:$ module load py39_4.12.0
    user@host:$ module load rocoto/1.3.3
 

diff --git a/env/WCOSS2.env b/env/WCOSS2.env
@@ -35,6 +35,7 @@ elif [[ "${step}" = "preplandobs" ]]; then
 
 elif [[ "${step}" = "waveinit" ]] || [[ "${step}" = "waveprep" ]] || [[ "${step}" = "wavepostsbs" ]] || [[ "${step}" = "wavepostbndpnt" ]] || [[ "${step}" = "wavepostbndpntbll"  ]] || [[ "${step}" = "wavepostpnt" ]]; then
 
+    export USE_CFP="YES"
     if [[ "${step}" = "waveprep" ]] && [[ "${CDUMP}" = "gfs" ]]; then export NTASKS=${NTASKS_gfs} ; fi
     export wavempexec="${launcher} -np"
     export wave_mpmd=${mpmd_opt}

diff --git a/jobs/JGDAS_ATMOS_VERFOZN b/jobs/JGDAS_ATMOS_VERFOZN
@@ -4,7 +4,7 @@
 # Set up environment for GDAS Ozone Monitor job
 #############################################################
 source "${HOMEgfs}/ush/preamble.sh"
-source "${HOMEgfs}/ush/jjob_header.sh" -e "vrfy" -c "base vrfy"
+source "${HOMEgfs}/ush/jjob_header.sh" -e "verfozn" -c "base verfozn"
 
 export OZNMON_SUFFIX=${OZNMON_SUFFIX:-${NET}}
 
@@ -42,15 +42,12 @@ export p_cyc=${pdate:8:2}
 #---------------------------------------------
 # OZN_TANKDIR - WHERE OUTPUT DATA WILL RESIDE
 #
-export OZN_TANKDIR=${OZN_TANKDIR:-$(compath.py ${envir}/${NET}/${gfs_ver})}
-export TANKverf_ozn=${TANKverf_ozn:-${OZN_TANKDIR}/${RUN}.${PDY}/${cyc}/atmos/oznmon}
-export TANKverf_oznM1=${TANKverf_oznM1:-${OZN_TANKDIR}/${RUN}.${P_PDY}/${p_cyc}/atmos/oznmon}
-
 YMD=${PDY} HH=${cyc} generate_com -rx COM_ATMOS_ANALYSIS
+YMD=${PDY} HH=${cyc} generate_com -rx COM_ATMOS_OZNMON
 
-if [[ ! -d ${TANKverf_ozn} ]]; then
-   mkdir -p -m 775 ${TANKverf_ozn}
-fi
+export TANKverf_ozn=${TANKverf_ozn:-${COM_ATMOS_OZNMON}}
+
+if [[ ! -d ${TANKverf_ozn} ]]; then mkdir -p -m 775 ${TANKverf_ozn} ; fi
 
 #---------------------------------------
 #  set up validation file
@@ -77,10 +74,6 @@ err=$?
 ################################
 # Remove the Working Directory
 ################################
-KEEPDATA=${KEEPDATA:-NO}
-cd ${DATAROOT}
-if [ ${KEEPDATA} = NO ] ; then
-  rm -rf ${DATA}
-fi
+[[ "${KEEPDATA}" = "NO" ]] && rm -rf "${DATA}"
 
 exit 0