From 755df7638f67a4de1c5aac651c239185a28b7155 Mon Sep 17 00:00:00 2001 From: Dusan Jovic Date: Tue, 7 Jul 2020 19:27:07 +0000 Subject: [PATCH 01/11] Allow multiple regtest ecflow suites to run on the same server --- tests/rt.sh | 2 +- tests/rt_utils.sh | 49 ++++++++++++++++++++--------------------------- 2 files changed, 22 insertions(+), 29 deletions(-) diff --git a/tests/rt.sh b/tests/rt.sh index 0fd4afaa7e..eeeb521fbd 100755 --- a/tests/rt.sh +++ b/tests/rt.sh @@ -492,7 +492,7 @@ fi if [[ $ECFLOW == true ]]; then ECFLOW_RUN=${PATHRT}/ecflow_run - ECFLOW_SUITE=regtest + ECFLOW_SUITE=regtest_$$ rm -rf ${ECFLOW_RUN} mkdir -p ${ECFLOW_RUN}/${ECFLOW_SUITE} cp head.h tail.h ${ECFLOW_RUN} diff --git a/tests/rt_utils.sh b/tests/rt_utils.sh index a6c239b47a..4829568836 100755 --- a/tests/rt_utils.sh +++ b/tests/rt_utils.sh @@ -513,37 +513,24 @@ ecflow_run() { ECF_HOST=$( hostname ) - sh ${ECFLOW_START} -d ${ECFLOW_RUN} -p ${ECF_PORT} >> ${ECFLOW_RUN}/ecflow.log 2>&1 - -# set +e -# i=0 -# max_atempts=5 -# while [[ $i -lt $max_atempts ]]; do -# ecflow_client --ping --host=${ECF_HOST} --port=${ECF_PORT} -# not_running=$? -# if [[ $not_running -eq 1 ]]; then -# echo "ecflow_server is NOT running on ${ECF_HOST}:${ECF_PORT}" -# sh ${ECFLOW_START} -d ${ECFLOW_RUN} -p ${ECF_PORT} >> ${ECFLOW_RUN}/ecflow.log 2>&1 -# break -# else -# echo "ecflow_server is already running on ${ECF_HOST}:${ECF_PORT}" -# ECF_PORT=$(( ECF_PORT + 1 )) -# fi -# i=$(( i + 1 )) -# if [[ $i -eq $max_atempts ]]; then -# echo "You already have $max_atempts ecFlow servers running on this node" -# exit 1 -# fi -# done -# set -e + set +e + ecflow_client --ping --host=${ECF_HOST} --port=${ECF_PORT} + not_running=$? + if [[ $not_running -eq 1 ]]; then + echo "ecflow_server is NOT running on ${ECF_HOST}:${ECF_PORT}" + sh ${ECFLOW_START} -p ${ECF_PORT} + else + echo "ecflow_server is already running on ${ECF_HOST}:${ECF_PORT}" + fi + set -e ECFLOW_RUNNING=true export ECF_PORT export ECF_HOST - ecflow_client --load=${ECFLOW_RUN}/${ECFLOW_SUITE}.def >> ${ECFLOW_RUN}/ecflow.log 2>&1 - ecflow_client --begin=${ECFLOW_SUITE} >> ${ECFLOW_RUN}/ecflow.log 2>&1 + ecflow_client --load=${ECFLOW_RUN}/${ECFLOW_SUITE}.def + ecflow_client --begin=${ECFLOW_SUITE} active_tasks=1 while [[ $active_tasks -ne 0 ]] @@ -554,6 +541,8 @@ ecflow_run() { ${PATHRT}/abort_dep_tasks.py done sleep 65 # wait one ECF_INTERVAL plus 5 seconds + ecflow_client --delete=yes /${ECFLOW_SUITE} + sleep 5 } ecflow_kill() { @@ -568,7 +557,11 @@ ecflow_stop() { [[ ${ECFLOW_RUNNING:-false} == true ]] || return set +e wait - ecflow_client --halt=yes - ecflow_client --check_pt - ecflow_client --terminate=yes + SUITES=$( ecflow_client --get | grep "^suite" ) + echo "SUITES=${SUITES}" + if [ -z "${SUITES}" ]; then + ecflow_client --halt=yes + ecflow_client --check_pt + ecflow_client --terminate=yes + fi } From 53095c760f99315be2a60c2a7b18ab4558e543c5 Mon Sep 17 00:00:00 2001 From: Dusan Jovic Date: Fri, 10 Jul 2020 12:37:28 +0000 Subject: [PATCH 02/11] Update FV3 submodule --- .gitmodules | 6 ++++-- FV3 | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.gitmodules b/.gitmodules index a6770b7e98..1c6895e298 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,7 +1,9 @@ [submodule "FV3"] path = FV3 - url = https://github.com/NOAA-EMC/fv3atm - branch = develop + #url = https://github.com/NOAA-EMC/fv3atm + #branch = develop + url = https://github.com/climbfuji/fv3atm + branch = cleanup_work_20200709 [submodule "NEMS"] path = NEMS url = https://github.com/NOAA-EMC/NEMS diff --git a/FV3 b/FV3 index 16e36152e4..8c063714ad 160000 --- a/FV3 +++ b/FV3 @@ -1 +1 @@ -Subproject commit 16e36152e4a08ab1b5e69c1cadb15855af5dc5c9 +Subproject commit 8c063714ad770f5525b260af38c6eea79d93cc46 From d4144d3ba965236ccfc573d593f52d8e5520a15a Mon Sep 17 00:00:00 2001 From: Dusan Jovic Date: Wed, 15 Jul 2020 12:36:30 +0000 Subject: [PATCH 03/11] Update FV3 submodule --- FV3 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/FV3 b/FV3 index 8c063714ad..3ce9859559 160000 --- a/FV3 +++ b/FV3 @@ -1 +1 @@ -Subproject commit 8c063714ad770f5525b260af38c6eea79d93cc46 +Subproject commit 3ce9859559b0bd117a077e581de01996fd23d34e From b05284445818cf6eaee348e9ec288bea9b04c30e Mon Sep 17 00:00:00 2001 From: MinsukJi-NOAA Date: Thu, 2 Jul 2020 16:30:59 +0000 Subject: [PATCH 04/11] implement running a single rt test --- .gitignore | 1 + tests/rt.sh | 41 ++++++++++++++++++++++++++++++++++++++++- 2 files changed, 41 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index c7ab7602ae..07e72778a7 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ coupled*.mk +rt.conf.single diff --git a/tests/rt.sh b/tests/rt.sh index b73bc3525b..efa43aed30 100755 --- a/tests/rt.sh +++ b/tests/rt.sh @@ -27,6 +27,34 @@ usage() { [[ $# -eq 0 ]] && usage +rt_single() { + local compile_line='' + local run_line='' + while read -r line; do + line="${line#"${line%%[![:space:]]*}"}" + [[ ${#line} == 0 ]] && continue + [[ $line == \#* ]] && continue + + if [[ $line =~ COMPILE && $line =~ ${MACHINE_ID} ]]; then + compile_line=$line + fi + + if [[ $line =~ RUN ]]; then + tmp_test=$(echo $line | cut -d'|' -f2 | sed -e 's/^ *//' -e 's/ *$//') + if [[ $SINGLE_NAME == $tmp_test && $compile_line != '' ]]; then + echo $compile_line >$TESTS_FILE + echo $line >>$TESTS_FILE + break + fi + fi + done <'rt.conf' + + if [[ ! -f $TESTS_FILE ]]; then + echo "$SINGLE_NAME does not exist or cannot be run on $MACHINE_ID" + exit 1 + fi +} + rt_trap() { [[ ${ROCOTO:-false} == true ]] && rocoto_kill cleanup @@ -323,6 +351,7 @@ CREATE_BASELINE=false ROCOTO=false ECFLOW=false KEEP_RUNDIR=false +SINGLE_NAME='' TESTS_FILE='rt.conf' @@ -332,7 +361,7 @@ fi SET_ID='standard' -while getopts ":cfsl:mkreh" opt; do +while getopts ":cfsl:mn:kreh" opt; do case $opt in c) CREATE_BASELINE=true @@ -352,6 +381,12 @@ while getopts ":cfsl:mkreh" opt; do # redefine RTPWD to point to newly created baseline outputs RTPWD=${NEW_BASELINE} ;; + n) + SINGLE_NAME=$OPTARG + TESTS_FILE='rt.conf.single' + SET_ID=' ' + rm -f $TESTS_FILE + ;; k) KEEP_RUNDIR=true ;; @@ -377,6 +412,10 @@ while getopts ":cfsl:mkreh" opt; do esac done +if [[ $SINGLE_NAME != '' ]]; then + rt_single +fi + if [[ $MACHINE_ID = hera.* ]] || [[ $MACHINE_ID = orion.* ]] || [[ $MACHINE_ID = cheyenne.* ]]; then RTPWD=${RTPWD:-$DISKNM/NEMSfv3gfs/develop-20200713/${COMPILER^^}} else From 16a201f8e1427372c71fe72aae980d15c27b4444 Mon Sep 17 00:00:00 2001 From: Dusan Jovic Date: Wed, 15 Jul 2020 13:47:05 +0000 Subject: [PATCH 05/11] Update 'usage' function in rt.sh (added -n option) --- tests/rt.sh | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/tests/rt.sh b/tests/rt.sh index efa43aed30..53a616ff04 100755 --- a/tests/rt.sh +++ b/tests/rt.sh @@ -9,17 +9,18 @@ die() { echo "$@" >&2; exit 1; } usage() { set +x echo - echo "Usage: $0 -c | -f | -s | -l | -m | -k | -r | -e | -h" + echo "Usage: $0 -c | -e | -f | -h | -k | -l | -m | -n | -r | -s" echo - echo " -c create new baseline results for " + echo " -c create new baseline results" + echo " -e use ecFlow workflow manager" echo " -f run full suite of regression tests" - echo " -s run standard suite of regression tests" + echo " -h display this help" + echo " -k keep run directory" echo " -l runs test specified in " echo " -m compare against new baseline results" - echo " -k keep run directory" + echo " -n run single test " echo " -r use Rocoto workflow manager" - echo " -e use ecFlow workflow manager" - echo " -h display this help" + echo " -s run standard suite of regression tests" echo set -x exit 1 @@ -37,7 +38,7 @@ rt_single() { if [[ $line =~ COMPILE && $line =~ ${MACHINE_ID} ]]; then compile_line=$line - fi + fi if [[ $line =~ RUN ]]; then tmp_test=$(echo $line | cut -d'|' -f2 | sed -e 's/^ *//' -e 's/ *$//') @@ -45,14 +46,14 @@ rt_single() { echo $compile_line >$TESTS_FILE echo $line >>$TESTS_FILE break - fi - fi + fi + fi done <'rt.conf' if [[ ! -f $TESTS_FILE ]]; then echo "$SINGLE_NAME does not exist or cannot be run on $MACHINE_ID" exit 1 - fi + fi } rt_trap() { From cec5a2be8a073348d3a7ee6b294df8b700ebdd7a Mon Sep 17 00:00:00 2001 From: Dusan Jovic Date: Wed, 15 Jul 2020 13:52:33 +0000 Subject: [PATCH 06/11] Update FV3 submodule --- FV3 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/FV3 b/FV3 index 3ce9859559..4eb2b376ff 160000 --- a/FV3 +++ b/FV3 @@ -1 +1 @@ -Subproject commit 3ce9859559b0bd117a077e581de01996fd23d34e +Subproject commit 4eb2b376ff7cb4f82392cb9f691a761321548bf1 From 156e675a2c5af1d257642b9d8a6bcceec64ad392 Mon Sep 17 00:00:00 2001 From: Dusan Jovic Date: Wed, 15 Jul 2020 14:53:06 +0000 Subject: [PATCH 07/11] Add regtest logs on WCOSS Dell --- tests/Compile_wcoss_dell_p3.log | 619 ++++++++++++------------ tests/RegressionTests_wcoss_dell_p3.log | 96 ++-- 2 files changed, 357 insertions(+), 358 deletions(-) diff --git a/tests/Compile_wcoss_dell_p3.log b/tests/Compile_wcoss_dell_p3.log index 04b6059952..32f581e28e 100644 --- a/tests/Compile_wcoss_dell_p3.log +++ b/tests/Compile_wcoss_dell_p3.log @@ -1,22 +1,22 @@ + SECONDS=0 + [[ 4 != 4 ]] -+ export PATHRT=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -+ PATHRT=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -+ export RUNDIR_ROOT=/gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_69502 -+ RUNDIR_ROOT=/gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_69502 ++ export PATHRT=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests ++ PATHRT=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests ++ export RUNDIR_ROOT=/gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_160616 ++ RUNDIR_ROOT=/gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_160616 + export 'MAKE_OPT=CCPP=Y SUITES=FV3_GSD_v0,FV3_GFS_v15_thompson 32BIT=Y DEBUG=Y' + MAKE_OPT='CCPP=Y SUITES=FV3_GSD_v0,FV3_GFS_v15_thompson 32BIT=Y DEBUG=Y' + export COMPILE_NR=10 + COMPILE_NR=10 -+ cd /gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -+ [[ -e /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_69502/compile_10.env ]] -+ source /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_69502/compile_10.env ++ cd /gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests ++ [[ -e /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_160616/compile_10.env ]] ++ source /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_160616/compile_10.env ++ export MACHINE_ID=wcoss_dell_p3 ++ MACHINE_ID=wcoss_dell_p3 -++ export PATHRT=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -++ PATHRT=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -++ export PATHTR=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr154/ufs-weather-model -++ PATHTR=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr154/ufs-weather-model +++ export PATHRT=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests +++ PATHRT=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests +++ export PATHTR=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model +++ PATHTR=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model ++ export SCHEDULER=lsf ++ SCHEDULER=lsf ++ export ACCNR=GFS-DEV @@ -29,10 +29,10 @@ ++ ROCOTO=false ++ export ECFLOW=true ++ ECFLOW=true -++ export REGRESSIONTEST_LOG=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/RegressionTests_wcoss_dell_p3.log -++ REGRESSIONTEST_LOG=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/RegressionTests_wcoss_dell_p3.log -++ export LOG_DIR=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/log_wcoss_dell_p3 -++ LOG_DIR=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/log_wcoss_dell_p3 +++ export REGRESSIONTEST_LOG=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/RegressionTests_wcoss_dell_p3.log +++ REGRESSIONTEST_LOG=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/RegressionTests_wcoss_dell_p3.log +++ export LOG_DIR=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/log_wcoss_dell_p3 +++ LOG_DIR=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/log_wcoss_dell_p3 + source default_vars.sh ++ '[' wcoss_dell_p3 = wcoss ']' ++ '[' wcoss_dell_p3 = wcoss_cray ']' @@ -63,15 +63,15 @@ + TEST_NR=10 + export JBNME=compile_10 + JBNME=compile_10 -+ export RUNDIR=/gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_69502/compile_10 -+ RUNDIR=/gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_69502/compile_10 ++ export RUNDIR=/gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_160616/compile_10 ++ RUNDIR=/gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_160616/compile_10 + source rt_utils.sh ++ set -eu -++ [[ /gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/run_compile.sh = \r\t\_\u\t\i\l\s\.\s\h ]] +++ [[ /gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/run_compile.sh = \r\t\_\u\t\i\l\s\.\s\h ]] ++ UNIT_TEST=false + source atparse.bash -+ mkdir -p /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_69502/compile_10 -+ cd /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_69502/compile_10 ++ mkdir -p /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_160616/compile_10 ++ cd /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_160616/compile_10 + [[ lsf = \s\l\u\r\m ]] + [[ lsf = \l\s\f ]] + atparse @@ -86,7 +86,7 @@ + set_x='set -x' + set +x ESUB: MEMORY LIMIT RLIMIT_RSS set to 8192 - with rusage set in shared job. -Job id 29942971 +Job id 30062260 TEST 10 compile is waiting to enter the queue 1 min. TEST 10 compile is running, Status: RUN 2 min. TEST 10 compile is running, Status: RUN @@ -96,31 +96,29 @@ TEST 10 compile is waiting to enter the queue 6 min. TEST 10 compile is running, Status: RUN 7 min. TEST 10 compile is running, Status: RUN 8 min. TEST 10 compile is running, Status: RUN -9 min. TEST 10 compile is running, Status: RUN -10 min. TEST 10 compile is running, Status: RUN -11 min. TEST 10 compile is finished, Status: DONE -+ elapsed=667 -+ echo 'Elapsed time 667 seconds. Compile 10' -Elapsed time 667 seconds. Compile 10 +9 min. TEST 10 compile is finished, Status: DONE ++ elapsed=546 ++ echo 'Elapsed time 546 seconds. Compile 10' +Elapsed time 546 seconds. Compile 10 + SECONDS=0 + [[ 4 != 4 ]] -+ export PATHRT=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -+ PATHRT=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -+ export RUNDIR_ROOT=/gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_69502 -+ RUNDIR_ROOT=/gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_69502 ++ export PATHRT=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests ++ PATHRT=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests ++ export RUNDIR_ROOT=/gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_160616 ++ RUNDIR_ROOT=/gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_160616 + export 'MAKE_OPT=CCPP=Y SUITES=FV3_GFS_v16_csawmg' + MAKE_OPT='CCPP=Y SUITES=FV3_GFS_v16_csawmg' + export COMPILE_NR=11 + COMPILE_NR=11 -+ cd /gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -+ [[ -e /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_69502/compile_11.env ]] -+ source /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_69502/compile_11.env ++ cd /gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests ++ [[ -e /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_160616/compile_11.env ]] ++ source /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_160616/compile_11.env ++ export MACHINE_ID=wcoss_dell_p3 ++ MACHINE_ID=wcoss_dell_p3 -++ export PATHRT=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -++ PATHRT=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -++ export PATHTR=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr154/ufs-weather-model -++ PATHTR=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr154/ufs-weather-model +++ export PATHRT=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests +++ PATHRT=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests +++ export PATHTR=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model +++ PATHTR=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model ++ export SCHEDULER=lsf ++ SCHEDULER=lsf ++ export ACCNR=GFS-DEV @@ -133,10 +131,10 @@ Elapsed time 667 seconds. Compile 10 ++ ROCOTO=false ++ export ECFLOW=true ++ ECFLOW=true -++ export REGRESSIONTEST_LOG=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/RegressionTests_wcoss_dell_p3.log -++ REGRESSIONTEST_LOG=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/RegressionTests_wcoss_dell_p3.log -++ export LOG_DIR=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/log_wcoss_dell_p3 -++ LOG_DIR=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/log_wcoss_dell_p3 +++ export REGRESSIONTEST_LOG=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/RegressionTests_wcoss_dell_p3.log +++ REGRESSIONTEST_LOG=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/RegressionTests_wcoss_dell_p3.log +++ export LOG_DIR=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/log_wcoss_dell_p3 +++ LOG_DIR=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/log_wcoss_dell_p3 + source default_vars.sh ++ '[' wcoss_dell_p3 = wcoss ']' ++ '[' wcoss_dell_p3 = wcoss_cray ']' @@ -167,15 +165,15 @@ Elapsed time 667 seconds. Compile 10 + TEST_NR=11 + export JBNME=compile_11 + JBNME=compile_11 -+ export RUNDIR=/gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_69502/compile_11 -+ RUNDIR=/gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_69502/compile_11 ++ export RUNDIR=/gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_160616/compile_11 ++ RUNDIR=/gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_160616/compile_11 + source rt_utils.sh ++ set -eu -++ [[ /gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/run_compile.sh = \r\t\_\u\t\i\l\s\.\s\h ]] +++ [[ /gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/run_compile.sh = \r\t\_\u\t\i\l\s\.\s\h ]] ++ UNIT_TEST=false + source atparse.bash -+ mkdir -p /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_69502/compile_11 -+ cd /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_69502/compile_11 ++ mkdir -p /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_160616/compile_11 ++ cd /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_160616/compile_11 + [[ lsf = \s\l\u\r\m ]] + [[ lsf = \l\s\f ]] + atparse @@ -190,7 +188,7 @@ Elapsed time 667 seconds. Compile 10 + set_x='set -x' + set +x ESUB: MEMORY LIMIT RLIMIT_RSS set to 8192 - with rusage set in shared job. -Job id 29946387 +Job id 30063001 TEST 11 compile is waiting to enter the queue 1 min. TEST 11 compile is running, Status: RUN 2 min. TEST 11 compile is running, Status: RUN @@ -215,30 +213,29 @@ TEST 11 compile is waiting to enter the queue 21 min. TEST 11 compile is running, Status: RUN 22 min. TEST 11 compile is running, Status: RUN 23 min. TEST 11 compile is running, Status: RUN -24 min. TEST 11 compile is running, Status: RUN -25 min. TEST 11 compile is finished, Status: DONE -+ elapsed=1507 -+ echo 'Elapsed time 1507 seconds. Compile 11' -Elapsed time 1507 seconds. Compile 11 +24 min. TEST 11 compile is finished, Status: DONE ++ elapsed=1447 ++ echo 'Elapsed time 1447 seconds. Compile 11' +Elapsed time 1447 seconds. Compile 11 + SECONDS=0 + [[ 4 != 4 ]] -+ export PATHRT=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -+ PATHRT=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -+ export RUNDIR_ROOT=/gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_69502 -+ RUNDIR_ROOT=/gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_69502 ++ export PATHRT=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests ++ PATHRT=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests ++ export RUNDIR_ROOT=/gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_160616 ++ RUNDIR_ROOT=/gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_160616 + export 'MAKE_OPT=CCPP=Y SUITES=FV3_GFS_2017_gfdlmp,FV3_GFS_2017_gfdlmp_noahmp' + MAKE_OPT='CCPP=Y SUITES=FV3_GFS_2017_gfdlmp,FV3_GFS_2017_gfdlmp_noahmp' + export COMPILE_NR=12 + COMPILE_NR=12 -+ cd /gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -+ [[ -e /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_69502/compile_12.env ]] -+ source /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_69502/compile_12.env ++ cd /gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests ++ [[ -e /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_160616/compile_12.env ]] ++ source /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_160616/compile_12.env ++ export MACHINE_ID=wcoss_dell_p3 ++ MACHINE_ID=wcoss_dell_p3 -++ export PATHRT=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -++ PATHRT=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -++ export PATHTR=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr154/ufs-weather-model -++ PATHTR=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr154/ufs-weather-model +++ export PATHRT=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests +++ PATHRT=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests +++ export PATHTR=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model +++ PATHTR=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model ++ export SCHEDULER=lsf ++ SCHEDULER=lsf ++ export ACCNR=GFS-DEV @@ -251,10 +248,10 @@ Elapsed time 1507 seconds. Compile 11 ++ ROCOTO=false ++ export ECFLOW=true ++ ECFLOW=true -++ export REGRESSIONTEST_LOG=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/RegressionTests_wcoss_dell_p3.log -++ REGRESSIONTEST_LOG=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/RegressionTests_wcoss_dell_p3.log -++ export LOG_DIR=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/log_wcoss_dell_p3 -++ LOG_DIR=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/log_wcoss_dell_p3 +++ export REGRESSIONTEST_LOG=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/RegressionTests_wcoss_dell_p3.log +++ REGRESSIONTEST_LOG=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/RegressionTests_wcoss_dell_p3.log +++ export LOG_DIR=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/log_wcoss_dell_p3 +++ LOG_DIR=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/log_wcoss_dell_p3 + source default_vars.sh ++ '[' wcoss_dell_p3 = wcoss ']' ++ '[' wcoss_dell_p3 = wcoss_cray ']' @@ -285,15 +282,15 @@ Elapsed time 1507 seconds. Compile 11 + TEST_NR=12 + export JBNME=compile_12 + JBNME=compile_12 -+ export RUNDIR=/gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_69502/compile_12 -+ RUNDIR=/gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_69502/compile_12 ++ export RUNDIR=/gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_160616/compile_12 ++ RUNDIR=/gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_160616/compile_12 + source rt_utils.sh ++ set -eu -++ [[ /gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/run_compile.sh = \r\t\_\u\t\i\l\s\.\s\h ]] +++ [[ /gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/run_compile.sh = \r\t\_\u\t\i\l\s\.\s\h ]] ++ UNIT_TEST=false + source atparse.bash -+ mkdir -p /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_69502/compile_12 -+ cd /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_69502/compile_12 ++ mkdir -p /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_160616/compile_12 ++ cd /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_160616/compile_12 + [[ lsf = \s\l\u\r\m ]] + [[ lsf = \l\s\f ]] + atparse @@ -308,7 +305,7 @@ Elapsed time 1507 seconds. Compile 11 + set_x='set -x' + set +x ESUB: MEMORY LIMIT RLIMIT_RSS set to 8192 - with rusage set in shared job. -Job id 29946388 +Job id 30063029 TEST 12 compile is waiting to enter the queue 1 min. TEST 12 compile is running, Status: RUN 2 min. TEST 12 compile is running, Status: RUN @@ -333,29 +330,31 @@ TEST 12 compile is waiting to enter the queue 21 min. TEST 12 compile is running, Status: RUN 22 min. TEST 12 compile is running, Status: RUN 23 min. TEST 12 compile is running, Status: RUN -24 min. TEST 12 compile is finished, Status: DONE -+ elapsed=1447 -+ echo 'Elapsed time 1447 seconds. Compile 12' -Elapsed time 1447 seconds. Compile 12 +24 min. TEST 12 compile is running, Status: RUN +25 min. TEST 12 compile is running, Status: RUN +26 min. TEST 12 compile is finished, Status: DONE ++ elapsed=1568 ++ echo 'Elapsed time 1568 seconds. Compile 12' +Elapsed time 1568 seconds. Compile 12 + SECONDS=0 + [[ 4 != 4 ]] -+ export PATHRT=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -+ PATHRT=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -+ export RUNDIR_ROOT=/gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_69502 -+ RUNDIR_ROOT=/gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_69502 ++ export PATHRT=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests ++ PATHRT=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests ++ export RUNDIR_ROOT=/gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_160616 ++ RUNDIR_ROOT=/gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_160616 + export 'MAKE_OPT=CCPP=Y SUITES=FV3_GFS_2017' + MAKE_OPT='CCPP=Y SUITES=FV3_GFS_2017' + export COMPILE_NR=1 + COMPILE_NR=1 -+ cd /gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -+ [[ -e /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_69502/compile_1.env ]] -+ source /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_69502/compile_1.env ++ cd /gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests ++ [[ -e /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_160616/compile_1.env ]] ++ source /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_160616/compile_1.env ++ export MACHINE_ID=wcoss_dell_p3 ++ MACHINE_ID=wcoss_dell_p3 -++ export PATHRT=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -++ PATHRT=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -++ export PATHTR=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr154/ufs-weather-model -++ PATHTR=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr154/ufs-weather-model +++ export PATHRT=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests +++ PATHRT=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests +++ export PATHTR=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model +++ PATHTR=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model ++ export SCHEDULER=lsf ++ SCHEDULER=lsf ++ export ACCNR=GFS-DEV @@ -368,10 +367,10 @@ Elapsed time 1447 seconds. Compile 12 ++ ROCOTO=false ++ export ECFLOW=true ++ ECFLOW=true -++ export REGRESSIONTEST_LOG=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/RegressionTests_wcoss_dell_p3.log -++ REGRESSIONTEST_LOG=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/RegressionTests_wcoss_dell_p3.log -++ export LOG_DIR=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/log_wcoss_dell_p3 -++ LOG_DIR=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/log_wcoss_dell_p3 +++ export REGRESSIONTEST_LOG=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/RegressionTests_wcoss_dell_p3.log +++ REGRESSIONTEST_LOG=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/RegressionTests_wcoss_dell_p3.log +++ export LOG_DIR=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/log_wcoss_dell_p3 +++ LOG_DIR=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/log_wcoss_dell_p3 + source default_vars.sh ++ '[' wcoss_dell_p3 = wcoss ']' ++ '[' wcoss_dell_p3 = wcoss_cray ']' @@ -402,15 +401,15 @@ Elapsed time 1447 seconds. Compile 12 + TEST_NR=1 + export JBNME=compile_1 + JBNME=compile_1 -+ export RUNDIR=/gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_69502/compile_1 -+ RUNDIR=/gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_69502/compile_1 ++ export RUNDIR=/gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_160616/compile_1 ++ RUNDIR=/gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_160616/compile_1 + source rt_utils.sh ++ set -eu -++ [[ /gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/run_compile.sh = \r\t\_\u\t\i\l\s\.\s\h ]] +++ [[ /gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/run_compile.sh = \r\t\_\u\t\i\l\s\.\s\h ]] ++ UNIT_TEST=false + source atparse.bash -+ mkdir -p /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_69502/compile_1 -+ cd /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_69502/compile_1 ++ mkdir -p /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_160616/compile_1 ++ cd /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_160616/compile_1 + [[ lsf = \s\l\u\r\m ]] + [[ lsf = \l\s\f ]] + atparse @@ -425,7 +424,7 @@ Elapsed time 1447 seconds. Compile 12 + set_x='set -x' + set +x ESUB: MEMORY LIMIT RLIMIT_RSS set to 8192 - with rusage set in shared job. -Job id 29942967 +Job id 30062264 TEST 1 compile is waiting to enter the queue 1 min. TEST 1 compile is running, Status: RUN 2 min. TEST 1 compile is running, Status: RUN @@ -451,30 +450,29 @@ TEST 1 compile is waiting to enter the queue 22 min. TEST 1 compile is running, Status: RUN 23 min. TEST 1 compile is running, Status: RUN 24 min. TEST 1 compile is running, Status: RUN -25 min. TEST 1 compile is running, Status: RUN -26 min. TEST 1 compile is finished, Status: DONE -+ elapsed=1567 -+ echo 'Elapsed time 1567 seconds. Compile 1' -Elapsed time 1567 seconds. Compile 1 +25 min. TEST 1 compile is finished, Status: DONE ++ elapsed=1507 ++ echo 'Elapsed time 1507 seconds. Compile 1' +Elapsed time 1507 seconds. Compile 1 + SECONDS=0 + [[ 4 != 4 ]] -+ export PATHRT=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -+ PATHRT=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -+ export RUNDIR_ROOT=/gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_69502 -+ RUNDIR_ROOT=/gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_69502 ++ export PATHRT=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests ++ PATHRT=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests ++ export RUNDIR_ROOT=/gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_160616 ++ RUNDIR_ROOT=/gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_160616 + export 'MAKE_OPT=CCPP=Y SUITES=FV3_GFS_2017,FV3_GFS_2017_gfdlmp WW3=Y' + MAKE_OPT='CCPP=Y SUITES=FV3_GFS_2017,FV3_GFS_2017_gfdlmp WW3=Y' + export COMPILE_NR=2 + COMPILE_NR=2 -+ cd /gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -+ [[ -e /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_69502/compile_2.env ]] -+ source /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_69502/compile_2.env ++ cd /gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests ++ [[ -e /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_160616/compile_2.env ]] ++ source /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_160616/compile_2.env ++ export MACHINE_ID=wcoss_dell_p3 ++ MACHINE_ID=wcoss_dell_p3 -++ export PATHRT=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -++ PATHRT=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -++ export PATHTR=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr154/ufs-weather-model -++ PATHTR=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr154/ufs-weather-model +++ export PATHRT=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests +++ PATHRT=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests +++ export PATHTR=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model +++ PATHTR=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model ++ export SCHEDULER=lsf ++ SCHEDULER=lsf ++ export ACCNR=GFS-DEV @@ -487,10 +485,10 @@ Elapsed time 1567 seconds. Compile 1 ++ ROCOTO=false ++ export ECFLOW=true ++ ECFLOW=true -++ export REGRESSIONTEST_LOG=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/RegressionTests_wcoss_dell_p3.log -++ REGRESSIONTEST_LOG=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/RegressionTests_wcoss_dell_p3.log -++ export LOG_DIR=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/log_wcoss_dell_p3 -++ LOG_DIR=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/log_wcoss_dell_p3 +++ export REGRESSIONTEST_LOG=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/RegressionTests_wcoss_dell_p3.log +++ REGRESSIONTEST_LOG=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/RegressionTests_wcoss_dell_p3.log +++ export LOG_DIR=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/log_wcoss_dell_p3 +++ LOG_DIR=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/log_wcoss_dell_p3 + source default_vars.sh ++ '[' wcoss_dell_p3 = wcoss ']' ++ '[' wcoss_dell_p3 = wcoss_cray ']' @@ -521,15 +519,15 @@ Elapsed time 1567 seconds. Compile 1 + TEST_NR=2 + export JBNME=compile_2 + JBNME=compile_2 -+ export RUNDIR=/gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_69502/compile_2 -+ RUNDIR=/gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_69502/compile_2 ++ export RUNDIR=/gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_160616/compile_2 ++ RUNDIR=/gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_160616/compile_2 + source rt_utils.sh ++ set -eu -++ [[ /gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/run_compile.sh = \r\t\_\u\t\i\l\s\.\s\h ]] +++ [[ /gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/run_compile.sh = \r\t\_\u\t\i\l\s\.\s\h ]] ++ UNIT_TEST=false + source atparse.bash -+ mkdir -p /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_69502/compile_2 -+ cd /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_69502/compile_2 ++ mkdir -p /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_160616/compile_2 ++ cd /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_160616/compile_2 + [[ lsf = \s\l\u\r\m ]] + [[ lsf = \l\s\f ]] + atparse @@ -544,9 +542,9 @@ Elapsed time 1567 seconds. Compile 1 + set_x='set -x' + set +x ESUB: MEMORY LIMIT RLIMIT_RSS set to 8192 - with rusage set in shared job. -Job id 29942974 +Job id 30062266 TEST 2 compile is waiting to enter the queue -1 min. TEST 2 compile is waiting in a queue, Status: PEND +1 min. TEST 2 compile is running, Status: RUN 2 min. TEST 2 compile is running, Status: RUN 3 min. TEST 2 compile is running, Status: RUN 4 min. TEST 2 compile is running, Status: RUN @@ -572,29 +570,33 @@ TEST 2 compile is waiting to enter the queue 24 min. TEST 2 compile is running, Status: RUN 25 min. TEST 2 compile is running, Status: RUN 26 min. TEST 2 compile is running, Status: RUN -27 min. TEST 2 compile is finished, Status: DONE -+ elapsed=1628 -+ echo 'Elapsed time 1628 seconds. Compile 2' -Elapsed time 1628 seconds. Compile 2 +27 min. TEST 2 compile is running, Status: RUN +28 min. TEST 2 compile is running, Status: RUN +29 min. TEST 2 compile is running, Status: RUN +30 min. TEST 2 compile is running, Status: RUN +31 min. TEST 2 compile is finished, Status: DONE ++ elapsed=1867 ++ echo 'Elapsed time 1867 seconds. Compile 2' +Elapsed time 1867 seconds. Compile 2 + SECONDS=0 + [[ 4 != 4 ]] -+ export PATHRT=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -+ PATHRT=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -+ export RUNDIR_ROOT=/gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_69502 -+ RUNDIR_ROOT=/gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_69502 ++ export PATHRT=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests ++ PATHRT=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests ++ export RUNDIR_ROOT=/gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_160616 ++ RUNDIR_ROOT=/gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_160616 + export 'MAKE_OPT=CCPP=Y SUITES=FV3_GFS_2017_fv3wam 32BIT=Y MULTI_GASES=Y' + MAKE_OPT='CCPP=Y SUITES=FV3_GFS_2017_fv3wam 32BIT=Y MULTI_GASES=Y' + export COMPILE_NR=3 + COMPILE_NR=3 -+ cd /gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -+ [[ -e /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_69502/compile_3.env ]] -+ source /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_69502/compile_3.env ++ cd /gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests ++ [[ -e /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_160616/compile_3.env ]] ++ source /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_160616/compile_3.env ++ export MACHINE_ID=wcoss_dell_p3 ++ MACHINE_ID=wcoss_dell_p3 -++ export PATHRT=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -++ PATHRT=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -++ export PATHTR=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr154/ufs-weather-model -++ PATHTR=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr154/ufs-weather-model +++ export PATHRT=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests +++ PATHRT=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests +++ export PATHTR=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model +++ PATHTR=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model ++ export SCHEDULER=lsf ++ SCHEDULER=lsf ++ export ACCNR=GFS-DEV @@ -607,10 +609,10 @@ Elapsed time 1628 seconds. Compile 2 ++ ROCOTO=false ++ export ECFLOW=true ++ ECFLOW=true -++ export REGRESSIONTEST_LOG=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/RegressionTests_wcoss_dell_p3.log -++ REGRESSIONTEST_LOG=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/RegressionTests_wcoss_dell_p3.log -++ export LOG_DIR=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/log_wcoss_dell_p3 -++ LOG_DIR=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/log_wcoss_dell_p3 +++ export REGRESSIONTEST_LOG=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/RegressionTests_wcoss_dell_p3.log +++ REGRESSIONTEST_LOG=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/RegressionTests_wcoss_dell_p3.log +++ export LOG_DIR=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/log_wcoss_dell_p3 +++ LOG_DIR=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/log_wcoss_dell_p3 + source default_vars.sh ++ '[' wcoss_dell_p3 = wcoss ']' ++ '[' wcoss_dell_p3 = wcoss_cray ']' @@ -641,15 +643,15 @@ Elapsed time 1628 seconds. Compile 2 + TEST_NR=3 + export JBNME=compile_3 + JBNME=compile_3 -+ export RUNDIR=/gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_69502/compile_3 -+ RUNDIR=/gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_69502/compile_3 ++ export RUNDIR=/gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_160616/compile_3 ++ RUNDIR=/gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_160616/compile_3 + source rt_utils.sh ++ set -eu -++ [[ /gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/run_compile.sh = \r\t\_\u\t\i\l\s\.\s\h ]] +++ [[ /gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/run_compile.sh = \r\t\_\u\t\i\l\s\.\s\h ]] ++ UNIT_TEST=false + source atparse.bash -+ mkdir -p /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_69502/compile_3 -+ cd /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_69502/compile_3 ++ mkdir -p /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_160616/compile_3 ++ cd /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_160616/compile_3 + [[ lsf = \s\l\u\r\m ]] + [[ lsf = \l\s\f ]] + atparse @@ -664,9 +666,9 @@ Elapsed time 1628 seconds. Compile 2 + set_x='set -x' + set +x ESUB: MEMORY LIMIT RLIMIT_RSS set to 8192 - with rusage set in shared job. -Job id 29942972 +Job id 30062263 TEST 3 compile is waiting to enter the queue -1 min. TEST 3 compile is waiting in a queue, Status: PEND +1 min. TEST 3 compile is running, Status: RUN 2 min. TEST 3 compile is running, Status: RUN 3 min. TEST 3 compile is running, Status: RUN 4 min. TEST 3 compile is running, Status: RUN @@ -691,29 +693,30 @@ TEST 3 compile is waiting to enter the queue 23 min. TEST 3 compile is running, Status: RUN 24 min. TEST 3 compile is running, Status: RUN 25 min. TEST 3 compile is running, Status: RUN -26 min. TEST 3 compile is finished, Status: DONE -+ elapsed=1567 -+ echo 'Elapsed time 1567 seconds. Compile 3' -Elapsed time 1567 seconds. Compile 3 +26 min. TEST 3 compile is running, Status: RUN +27 min. TEST 3 compile is finished, Status: DONE ++ elapsed=1627 ++ echo 'Elapsed time 1627 seconds. Compile 3' +Elapsed time 1627 seconds. Compile 3 + SECONDS=0 + [[ 4 != 4 ]] -+ export PATHRT=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -+ PATHRT=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -+ export RUNDIR_ROOT=/gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_69502 -+ RUNDIR_ROOT=/gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_69502 ++ export PATHRT=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests ++ PATHRT=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests ++ export RUNDIR_ROOT=/gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_160616 ++ RUNDIR_ROOT=/gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_160616 + export 'MAKE_OPT=CCPP=Y SUITES=FV3_GFS_2017,FV3_GFS_2017_stretched 32BIT=Y' + MAKE_OPT='CCPP=Y SUITES=FV3_GFS_2017,FV3_GFS_2017_stretched 32BIT=Y' + export COMPILE_NR=4 + COMPILE_NR=4 -+ cd /gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -+ [[ -e /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_69502/compile_4.env ]] -+ source /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_69502/compile_4.env ++ cd /gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests ++ [[ -e /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_160616/compile_4.env ]] ++ source /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_160616/compile_4.env ++ export MACHINE_ID=wcoss_dell_p3 ++ MACHINE_ID=wcoss_dell_p3 -++ export PATHRT=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -++ PATHRT=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -++ export PATHTR=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr154/ufs-weather-model -++ PATHTR=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr154/ufs-weather-model +++ export PATHRT=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests +++ PATHRT=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests +++ export PATHTR=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model +++ PATHTR=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model ++ export SCHEDULER=lsf ++ SCHEDULER=lsf ++ export ACCNR=GFS-DEV @@ -726,10 +729,10 @@ Elapsed time 1567 seconds. Compile 3 ++ ROCOTO=false ++ export ECFLOW=true ++ ECFLOW=true -++ export REGRESSIONTEST_LOG=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/RegressionTests_wcoss_dell_p3.log -++ REGRESSIONTEST_LOG=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/RegressionTests_wcoss_dell_p3.log -++ export LOG_DIR=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/log_wcoss_dell_p3 -++ LOG_DIR=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/log_wcoss_dell_p3 +++ export REGRESSIONTEST_LOG=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/RegressionTests_wcoss_dell_p3.log +++ REGRESSIONTEST_LOG=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/RegressionTests_wcoss_dell_p3.log +++ export LOG_DIR=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/log_wcoss_dell_p3 +++ LOG_DIR=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/log_wcoss_dell_p3 + source default_vars.sh ++ '[' wcoss_dell_p3 = wcoss ']' ++ '[' wcoss_dell_p3 = wcoss_cray ']' @@ -760,15 +763,15 @@ Elapsed time 1567 seconds. Compile 3 + TEST_NR=4 + export JBNME=compile_4 + JBNME=compile_4 -+ export RUNDIR=/gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_69502/compile_4 -+ RUNDIR=/gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_69502/compile_4 ++ export RUNDIR=/gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_160616/compile_4 ++ RUNDIR=/gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_160616/compile_4 + source rt_utils.sh ++ set -eu -++ [[ /gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/run_compile.sh = \r\t\_\u\t\i\l\s\.\s\h ]] +++ [[ /gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/run_compile.sh = \r\t\_\u\t\i\l\s\.\s\h ]] ++ UNIT_TEST=false + source atparse.bash -+ mkdir -p /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_69502/compile_4 -+ cd /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_69502/compile_4 ++ mkdir -p /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_160616/compile_4 ++ cd /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_160616/compile_4 + [[ lsf = \s\l\u\r\m ]] + [[ lsf = \l\s\f ]] + atparse @@ -783,9 +786,9 @@ Elapsed time 1567 seconds. Compile 3 + set_x='set -x' + set +x ESUB: MEMORY LIMIT RLIMIT_RSS set to 8192 - with rusage set in shared job. -Job id 29942973 +Job id 30062265 TEST 4 compile is waiting to enter the queue -1 min. TEST 4 compile is waiting in a queue, Status: PEND +1 min. TEST 4 compile is running, Status: RUN 2 min. TEST 4 compile is running, Status: RUN 3 min. TEST 4 compile is running, Status: RUN 4 min. TEST 4 compile is running, Status: RUN @@ -809,30 +812,29 @@ TEST 4 compile is waiting to enter the queue 22 min. TEST 4 compile is running, Status: RUN 23 min. TEST 4 compile is running, Status: RUN 24 min. TEST 4 compile is running, Status: RUN -25 min. TEST 4 compile is running, Status: RUN -26 min. TEST 4 compile is finished, Status: DONE -+ elapsed=1567 -+ echo 'Elapsed time 1567 seconds. Compile 4' -Elapsed time 1567 seconds. Compile 4 +25 min. TEST 4 compile is finished, Status: DONE ++ elapsed=1507 ++ echo 'Elapsed time 1507 seconds. Compile 4' +Elapsed time 1507 seconds. Compile 4 + SECONDS=0 + [[ 4 != 4 ]] -+ export PATHRT=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -+ PATHRT=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -+ export RUNDIR_ROOT=/gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_69502 -+ RUNDIR_ROOT=/gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_69502 ++ export PATHRT=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests ++ PATHRT=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests ++ export RUNDIR_ROOT=/gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_160616 ++ RUNDIR_ROOT=/gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_160616 + export 'MAKE_OPT=CCPP=Y SUITES=FV3_GFS_2017_gfdlmp_regional,FV3_GFS_2017_gfdlmp_regional_c768 32BIT=Y' + MAKE_OPT='CCPP=Y SUITES=FV3_GFS_2017_gfdlmp_regional,FV3_GFS_2017_gfdlmp_regional_c768 32BIT=Y' + export COMPILE_NR=5 + COMPILE_NR=5 -+ cd /gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -+ [[ -e /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_69502/compile_5.env ]] -+ source /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_69502/compile_5.env ++ cd /gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests ++ [[ -e /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_160616/compile_5.env ]] ++ source /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_160616/compile_5.env ++ export MACHINE_ID=wcoss_dell_p3 ++ MACHINE_ID=wcoss_dell_p3 -++ export PATHRT=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -++ PATHRT=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -++ export PATHTR=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr154/ufs-weather-model -++ PATHTR=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr154/ufs-weather-model +++ export PATHRT=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests +++ PATHRT=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests +++ export PATHTR=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model +++ PATHTR=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model ++ export SCHEDULER=lsf ++ SCHEDULER=lsf ++ export ACCNR=GFS-DEV @@ -845,10 +847,10 @@ Elapsed time 1567 seconds. Compile 4 ++ ROCOTO=false ++ export ECFLOW=true ++ ECFLOW=true -++ export REGRESSIONTEST_LOG=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/RegressionTests_wcoss_dell_p3.log -++ REGRESSIONTEST_LOG=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/RegressionTests_wcoss_dell_p3.log -++ export LOG_DIR=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/log_wcoss_dell_p3 -++ LOG_DIR=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/log_wcoss_dell_p3 +++ export REGRESSIONTEST_LOG=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/RegressionTests_wcoss_dell_p3.log +++ REGRESSIONTEST_LOG=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/RegressionTests_wcoss_dell_p3.log +++ export LOG_DIR=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/log_wcoss_dell_p3 +++ LOG_DIR=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/log_wcoss_dell_p3 + source default_vars.sh ++ '[' wcoss_dell_p3 = wcoss ']' ++ '[' wcoss_dell_p3 = wcoss_cray ']' @@ -879,15 +881,15 @@ Elapsed time 1567 seconds. Compile 4 + TEST_NR=5 + export JBNME=compile_5 + JBNME=compile_5 -+ export RUNDIR=/gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_69502/compile_5 -+ RUNDIR=/gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_69502/compile_5 ++ export RUNDIR=/gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_160616/compile_5 ++ RUNDIR=/gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_160616/compile_5 + source rt_utils.sh ++ set -eu -++ [[ /gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/run_compile.sh = \r\t\_\u\t\i\l\s\.\s\h ]] +++ [[ /gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/run_compile.sh = \r\t\_\u\t\i\l\s\.\s\h ]] ++ UNIT_TEST=false + source atparse.bash -+ mkdir -p /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_69502/compile_5 -+ cd /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_69502/compile_5 ++ mkdir -p /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_160616/compile_5 ++ cd /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_160616/compile_5 + [[ lsf = \s\l\u\r\m ]] + [[ lsf = \l\s\f ]] + atparse @@ -902,7 +904,7 @@ Elapsed time 1567 seconds. Compile 4 + set_x='set -x' + set +x ESUB: MEMORY LIMIT RLIMIT_RSS set to 8192 - with rusage set in shared job. -Job id 29942965 +Job id 30062262 TEST 5 compile is waiting to enter the queue 1 min. TEST 5 compile is running, Status: RUN 2 min. TEST 5 compile is running, Status: RUN @@ -935,23 +937,23 @@ TEST 5 compile is waiting to enter the queue Elapsed time 1567 seconds. Compile 5 + SECONDS=0 + [[ 4 != 4 ]] -+ export PATHRT=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -+ PATHRT=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -+ export RUNDIR_ROOT=/gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_69502 -+ RUNDIR_ROOT=/gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_69502 ++ export PATHRT=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests ++ PATHRT=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests ++ export RUNDIR_ROOT=/gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_160616 ++ RUNDIR_ROOT=/gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_160616 + export 'MAKE_OPT=CCPP=Y SUITES=FV3_GFS_2017,FV3_GFS_2017_stretched 32BIT=Y DEBUG=Y' + MAKE_OPT='CCPP=Y SUITES=FV3_GFS_2017,FV3_GFS_2017_stretched 32BIT=Y DEBUG=Y' + export COMPILE_NR=6 + COMPILE_NR=6 -+ cd /gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -+ [[ -e /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_69502/compile_6.env ]] -+ source /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_69502/compile_6.env ++ cd /gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests ++ [[ -e /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_160616/compile_6.env ]] ++ source /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_160616/compile_6.env ++ export MACHINE_ID=wcoss_dell_p3 ++ MACHINE_ID=wcoss_dell_p3 -++ export PATHRT=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -++ PATHRT=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -++ export PATHTR=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr154/ufs-weather-model -++ PATHTR=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr154/ufs-weather-model +++ export PATHRT=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests +++ PATHRT=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests +++ export PATHTR=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model +++ PATHTR=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model ++ export SCHEDULER=lsf ++ SCHEDULER=lsf ++ export ACCNR=GFS-DEV @@ -964,10 +966,10 @@ Elapsed time 1567 seconds. Compile 5 ++ ROCOTO=false ++ export ECFLOW=true ++ ECFLOW=true -++ export REGRESSIONTEST_LOG=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/RegressionTests_wcoss_dell_p3.log -++ REGRESSIONTEST_LOG=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/RegressionTests_wcoss_dell_p3.log -++ export LOG_DIR=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/log_wcoss_dell_p3 -++ LOG_DIR=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/log_wcoss_dell_p3 +++ export REGRESSIONTEST_LOG=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/RegressionTests_wcoss_dell_p3.log +++ REGRESSIONTEST_LOG=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/RegressionTests_wcoss_dell_p3.log +++ export LOG_DIR=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/log_wcoss_dell_p3 +++ LOG_DIR=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/log_wcoss_dell_p3 + source default_vars.sh ++ '[' wcoss_dell_p3 = wcoss ']' ++ '[' wcoss_dell_p3 = wcoss_cray ']' @@ -998,15 +1000,15 @@ Elapsed time 1567 seconds. Compile 5 + TEST_NR=6 + export JBNME=compile_6 + JBNME=compile_6 -+ export RUNDIR=/gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_69502/compile_6 -+ RUNDIR=/gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_69502/compile_6 ++ export RUNDIR=/gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_160616/compile_6 ++ RUNDIR=/gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_160616/compile_6 + source rt_utils.sh ++ set -eu -++ [[ /gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/run_compile.sh = \r\t\_\u\t\i\l\s\.\s\h ]] +++ [[ /gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/run_compile.sh = \r\t\_\u\t\i\l\s\.\s\h ]] ++ UNIT_TEST=false + source atparse.bash -+ mkdir -p /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_69502/compile_6 -+ cd /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_69502/compile_6 ++ mkdir -p /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_160616/compile_6 ++ cd /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_160616/compile_6 + [[ lsf = \s\l\u\r\m ]] + [[ lsf = \l\s\f ]] + atparse @@ -1021,7 +1023,7 @@ Elapsed time 1567 seconds. Compile 5 + set_x='set -x' + set +x ESUB: MEMORY LIMIT RLIMIT_RSS set to 8192 - with rusage set in shared job. -Job id 29942970 +Job id 30062268 TEST 6 compile is waiting to enter the queue 1 min. TEST 6 compile is running, Status: RUN 2 min. TEST 6 compile is running, Status: RUN @@ -1030,32 +1032,29 @@ TEST 6 compile is waiting to enter the queue 5 min. TEST 6 compile is running, Status: RUN 6 min. TEST 6 compile is running, Status: RUN 7 min. TEST 6 compile is running, Status: RUN -8 min. TEST 6 compile is running, Status: RUN -9 min. TEST 6 compile is running, Status: RUN -10 min. TEST 6 compile is running, Status: RUN -11 min. TEST 6 compile is finished, Status: DONE -+ elapsed=667 -+ echo 'Elapsed time 667 seconds. Compile 6' -Elapsed time 667 seconds. Compile 6 +8 min. TEST 6 compile is finished, Status: DONE ++ elapsed=486 ++ echo 'Elapsed time 486 seconds. Compile 6' +Elapsed time 486 seconds. Compile 6 + SECONDS=0 + [[ 4 != 4 ]] -+ export PATHRT=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -+ PATHRT=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -+ export RUNDIR_ROOT=/gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_69502 -+ RUNDIR_ROOT=/gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_69502 ++ export PATHRT=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests ++ PATHRT=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests ++ export RUNDIR_ROOT=/gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_160616 ++ RUNDIR_ROOT=/gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_160616 + export 'MAKE_OPT=CCPP=Y SUITES=FV3_GFS_2017_gfdlmp,FV3_GFS_2017_gfdlmp_noahmp' + MAKE_OPT='CCPP=Y SUITES=FV3_GFS_2017_gfdlmp,FV3_GFS_2017_gfdlmp_noahmp' + export COMPILE_NR=7 + COMPILE_NR=7 -+ cd /gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -+ [[ -e /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_69502/compile_7.env ]] -+ source /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_69502/compile_7.env ++ cd /gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests ++ [[ -e /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_160616/compile_7.env ]] ++ source /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_160616/compile_7.env ++ export MACHINE_ID=wcoss_dell_p3 ++ MACHINE_ID=wcoss_dell_p3 -++ export PATHRT=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -++ PATHRT=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -++ export PATHTR=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr154/ufs-weather-model -++ PATHTR=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr154/ufs-weather-model +++ export PATHRT=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests +++ PATHRT=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests +++ export PATHTR=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model +++ PATHTR=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model ++ export SCHEDULER=lsf ++ SCHEDULER=lsf ++ export ACCNR=GFS-DEV @@ -1068,10 +1067,10 @@ Elapsed time 667 seconds. Compile 6 ++ ROCOTO=false ++ export ECFLOW=true ++ ECFLOW=true -++ export REGRESSIONTEST_LOG=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/RegressionTests_wcoss_dell_p3.log -++ REGRESSIONTEST_LOG=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/RegressionTests_wcoss_dell_p3.log -++ export LOG_DIR=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/log_wcoss_dell_p3 -++ LOG_DIR=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/log_wcoss_dell_p3 +++ export REGRESSIONTEST_LOG=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/RegressionTests_wcoss_dell_p3.log +++ REGRESSIONTEST_LOG=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/RegressionTests_wcoss_dell_p3.log +++ export LOG_DIR=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/log_wcoss_dell_p3 +++ LOG_DIR=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/log_wcoss_dell_p3 + source default_vars.sh ++ '[' wcoss_dell_p3 = wcoss ']' ++ '[' wcoss_dell_p3 = wcoss_cray ']' @@ -1102,15 +1101,15 @@ Elapsed time 667 seconds. Compile 6 + TEST_NR=7 + export JBNME=compile_7 + JBNME=compile_7 -+ export RUNDIR=/gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_69502/compile_7 -+ RUNDIR=/gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_69502/compile_7 ++ export RUNDIR=/gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_160616/compile_7 ++ RUNDIR=/gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_160616/compile_7 + source rt_utils.sh ++ set -eu -++ [[ /gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/run_compile.sh = \r\t\_\u\t\i\l\s\.\s\h ]] +++ [[ /gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/run_compile.sh = \r\t\_\u\t\i\l\s\.\s\h ]] ++ UNIT_TEST=false + source atparse.bash -+ mkdir -p /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_69502/compile_7 -+ cd /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_69502/compile_7 ++ mkdir -p /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_160616/compile_7 ++ cd /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_160616/compile_7 + [[ lsf = \s\l\u\r\m ]] + [[ lsf = \l\s\f ]] + atparse @@ -1125,7 +1124,7 @@ Elapsed time 667 seconds. Compile 6 + set_x='set -x' + set +x ESUB: MEMORY LIMIT RLIMIT_RSS set to 8192 - with rusage set in shared job. -Job id 29942969 +Job id 30062267 TEST 7 compile is waiting to enter the queue 1 min. TEST 7 compile is running, Status: RUN 2 min. TEST 7 compile is running, Status: RUN @@ -1150,32 +1149,29 @@ TEST 7 compile is waiting to enter the queue 21 min. TEST 7 compile is running, Status: RUN 22 min. TEST 7 compile is running, Status: RUN 23 min. TEST 7 compile is running, Status: RUN -24 min. TEST 7 compile is running, Status: RUN -25 min. TEST 7 compile is running, Status: RUN -26 min. TEST 7 compile is running, Status: RUN -27 min. TEST 7 compile is finished, Status: DONE -+ elapsed=1627 -+ echo 'Elapsed time 1627 seconds. Compile 7' -Elapsed time 1627 seconds. Compile 7 +24 min. TEST 7 compile is finished, Status: DONE ++ elapsed=1447 ++ echo 'Elapsed time 1447 seconds. Compile 7' +Elapsed time 1447 seconds. Compile 7 + SECONDS=0 + [[ 4 != 4 ]] -+ export PATHRT=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -+ PATHRT=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -+ export RUNDIR_ROOT=/gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_69502 -+ RUNDIR_ROOT=/gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_69502 ++ export PATHRT=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests ++ PATHRT=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests ++ export RUNDIR_ROOT=/gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_160616 ++ RUNDIR_ROOT=/gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_160616 + export 'MAKE_OPT=CCPP=Y SUITES=FV3_GFS_2017_csawmgshoc,FV3_GFS_2017_csawmg,FV3_GFS_2017_satmedmf,FV3_GFS_2017_satmedmfq' + MAKE_OPT='CCPP=Y SUITES=FV3_GFS_2017_csawmgshoc,FV3_GFS_2017_csawmg,FV3_GFS_2017_satmedmf,FV3_GFS_2017_satmedmfq' + export COMPILE_NR=8 + COMPILE_NR=8 -+ cd /gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -+ [[ -e /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_69502/compile_8.env ]] -+ source /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_69502/compile_8.env ++ cd /gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests ++ [[ -e /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_160616/compile_8.env ]] ++ source /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_160616/compile_8.env ++ export MACHINE_ID=wcoss_dell_p3 ++ MACHINE_ID=wcoss_dell_p3 -++ export PATHRT=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -++ PATHRT=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -++ export PATHTR=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr154/ufs-weather-model -++ PATHTR=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr154/ufs-weather-model +++ export PATHRT=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests +++ PATHRT=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests +++ export PATHTR=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model +++ PATHTR=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model ++ export SCHEDULER=lsf ++ SCHEDULER=lsf ++ export ACCNR=GFS-DEV @@ -1188,10 +1184,10 @@ Elapsed time 1627 seconds. Compile 7 ++ ROCOTO=false ++ export ECFLOW=true ++ ECFLOW=true -++ export REGRESSIONTEST_LOG=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/RegressionTests_wcoss_dell_p3.log -++ REGRESSIONTEST_LOG=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/RegressionTests_wcoss_dell_p3.log -++ export LOG_DIR=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/log_wcoss_dell_p3 -++ LOG_DIR=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/log_wcoss_dell_p3 +++ export REGRESSIONTEST_LOG=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/RegressionTests_wcoss_dell_p3.log +++ REGRESSIONTEST_LOG=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/RegressionTests_wcoss_dell_p3.log +++ export LOG_DIR=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/log_wcoss_dell_p3 +++ LOG_DIR=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/log_wcoss_dell_p3 + source default_vars.sh ++ '[' wcoss_dell_p3 = wcoss ']' ++ '[' wcoss_dell_p3 = wcoss_cray ']' @@ -1222,15 +1218,15 @@ Elapsed time 1627 seconds. Compile 7 + TEST_NR=8 + export JBNME=compile_8 + JBNME=compile_8 -+ export RUNDIR=/gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_69502/compile_8 -+ RUNDIR=/gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_69502/compile_8 ++ export RUNDIR=/gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_160616/compile_8 ++ RUNDIR=/gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_160616/compile_8 + source rt_utils.sh ++ set -eu -++ [[ /gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/run_compile.sh = \r\t\_\u\t\i\l\s\.\s\h ]] +++ [[ /gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/run_compile.sh = \r\t\_\u\t\i\l\s\.\s\h ]] ++ UNIT_TEST=false + source atparse.bash -+ mkdir -p /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_69502/compile_8 -+ cd /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_69502/compile_8 ++ mkdir -p /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_160616/compile_8 ++ cd /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_160616/compile_8 + [[ lsf = \s\l\u\r\m ]] + [[ lsf = \l\s\f ]] + atparse @@ -1245,7 +1241,7 @@ Elapsed time 1627 seconds. Compile 7 + set_x='set -x' + set +x ESUB: MEMORY LIMIT RLIMIT_RSS set to 8192 - with rusage set in shared job. -Job id 29942966 +Job id 30062261 TEST 8 compile is waiting to enter the queue 1 min. TEST 8 compile is running, Status: RUN 2 min. TEST 8 compile is running, Status: RUN @@ -1276,29 +1272,30 @@ TEST 8 compile is waiting to enter the queue 27 min. TEST 8 compile is running, Status: RUN 28 min. TEST 8 compile is running, Status: RUN 29 min. TEST 8 compile is running, Status: RUN -30 min. TEST 8 compile is finished, Status: DONE -+ elapsed=1808 -+ echo 'Elapsed time 1808 seconds. Compile 8' -Elapsed time 1808 seconds. Compile 8 +30 min. TEST 8 compile is running, Status: RUN +31 min. TEST 8 compile is finished, Status: DONE ++ elapsed=1867 ++ echo 'Elapsed time 1867 seconds. Compile 8' +Elapsed time 1867 seconds. Compile 8 + SECONDS=0 + [[ 4 != 4 ]] -+ export PATHRT=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -+ PATHRT=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -+ export RUNDIR_ROOT=/gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_69502 -+ RUNDIR_ROOT=/gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_69502 ++ export PATHRT=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests ++ PATHRT=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests ++ export RUNDIR_ROOT=/gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_160616 ++ RUNDIR_ROOT=/gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_160616 + export 'MAKE_OPT=CCPP=Y SUITES=FV3_GFS_2017_gfdlmp,FV3_CPT_v0,FV3_GSD_v0,FV3_GFS_v15_thompson,FV3_RAP,FV3_HRRR 32BIT=Y' + MAKE_OPT='CCPP=Y SUITES=FV3_GFS_2017_gfdlmp,FV3_CPT_v0,FV3_GSD_v0,FV3_GFS_v15_thompson,FV3_RAP,FV3_HRRR 32BIT=Y' + export COMPILE_NR=9 + COMPILE_NR=9 -+ cd /gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -+ [[ -e /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_69502/compile_9.env ]] -+ source /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_69502/compile_9.env ++ cd /gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests ++ [[ -e /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_160616/compile_9.env ]] ++ source /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_160616/compile_9.env ++ export MACHINE_ID=wcoss_dell_p3 ++ MACHINE_ID=wcoss_dell_p3 -++ export PATHRT=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -++ PATHRT=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -++ export PATHTR=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr154/ufs-weather-model -++ PATHTR=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr154/ufs-weather-model +++ export PATHRT=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests +++ PATHRT=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests +++ export PATHTR=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model +++ PATHTR=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model ++ export SCHEDULER=lsf ++ SCHEDULER=lsf ++ export ACCNR=GFS-DEV @@ -1311,10 +1308,10 @@ Elapsed time 1808 seconds. Compile 8 ++ ROCOTO=false ++ export ECFLOW=true ++ ECFLOW=true -++ export REGRESSIONTEST_LOG=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/RegressionTests_wcoss_dell_p3.log -++ REGRESSIONTEST_LOG=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/RegressionTests_wcoss_dell_p3.log -++ export LOG_DIR=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/log_wcoss_dell_p3 -++ LOG_DIR=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/log_wcoss_dell_p3 +++ export REGRESSIONTEST_LOG=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/RegressionTests_wcoss_dell_p3.log +++ REGRESSIONTEST_LOG=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/RegressionTests_wcoss_dell_p3.log +++ export LOG_DIR=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/log_wcoss_dell_p3 +++ LOG_DIR=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/log_wcoss_dell_p3 + source default_vars.sh ++ '[' wcoss_dell_p3 = wcoss ']' ++ '[' wcoss_dell_p3 = wcoss_cray ']' @@ -1345,15 +1342,15 @@ Elapsed time 1808 seconds. Compile 8 + TEST_NR=9 + export JBNME=compile_9 + JBNME=compile_9 -+ export RUNDIR=/gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_69502/compile_9 -+ RUNDIR=/gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_69502/compile_9 ++ export RUNDIR=/gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_160616/compile_9 ++ RUNDIR=/gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_160616/compile_9 + source rt_utils.sh ++ set -eu -++ [[ /gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/run_compile.sh = \r\t\_\u\t\i\l\s\.\s\h ]] +++ [[ /gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/run_compile.sh = \r\t\_\u\t\i\l\s\.\s\h ]] ++ UNIT_TEST=false + source atparse.bash -+ mkdir -p /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_69502/compile_9 -+ cd /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_69502/compile_9 ++ mkdir -p /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_160616/compile_9 ++ cd /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_160616/compile_9 + [[ lsf = \s\l\u\r\m ]] + [[ lsf = \l\s\f ]] + atparse @@ -1368,7 +1365,7 @@ Elapsed time 1808 seconds. Compile 8 + set_x='set -x' + set +x ESUB: MEMORY LIMIT RLIMIT_RSS set to 8192 - with rusage set in shared job. -Job id 29942968 +Job id 30062269 TEST 9 compile is waiting to enter the queue 1 min. TEST 9 compile is running, Status: RUN 2 min. TEST 9 compile is running, Status: RUN @@ -1400,7 +1397,9 @@ TEST 9 compile is waiting to enter the queue 28 min. TEST 9 compile is running, Status: RUN 29 min. TEST 9 compile is running, Status: RUN 30 min. TEST 9 compile is running, Status: RUN -31 min. TEST 9 compile is finished, Status: DONE -+ elapsed=1868 -+ echo 'Elapsed time 1868 seconds. Compile 9' -Elapsed time 1868 seconds. Compile 9 +31 min. TEST 9 compile is running, Status: RUN +32 min. TEST 9 compile is running, Status: RUN +33 min. TEST 9 compile is finished, Status: DONE ++ elapsed=1987 ++ echo 'Elapsed time 1987 seconds. Compile 9' +Elapsed time 1987 seconds. Compile 9 diff --git a/tests/RegressionTests_wcoss_dell_p3.log b/tests/RegressionTests_wcoss_dell_p3.log index a867a47786..759bf2ec5a 100644 --- a/tests/RegressionTests_wcoss_dell_p3.log +++ b/tests/RegressionTests_wcoss_dell_p3.log @@ -1,9 +1,9 @@ -Tue Jul 14 01:31:15 UTC 2020 +Wed Jul 15 14:03:40 UTC 2020 Start Regression test baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200713/fv3_control_ccpp -working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_69502/fv3_ccpp_control_prod +working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_160616/fv3_ccpp_control_prod Checking test 001 fv3_ccpp_control results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -71,7 +71,7 @@ Test 001 fv3_ccpp_control PASS baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200713/fv3_control_ccpp -working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_69502/fv3_ccpp_decomp_prod +working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_160616/fv3_ccpp_decomp_prod Checking test 002 fv3_ccpp_decomp results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -139,7 +139,7 @@ Test 002 fv3_ccpp_decomp PASS baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200713/fv3_control_ccpp -working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_69502/fv3_ccpp_2threads_prod +working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_160616/fv3_ccpp_2threads_prod Checking test 003 fv3_ccpp_2threads results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -207,7 +207,7 @@ Test 003 fv3_ccpp_2threads PASS baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200713/fv3_restart_ccpp -working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_69502/fv3_ccpp_restart_prod +working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_160616/fv3_ccpp_restart_prod Checking test 004 fv3_ccpp_restart results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -275,7 +275,7 @@ Test 004 fv3_ccpp_restart PASS baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200713/fv3_read_inc_ccpp -working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_69502/fv3_ccpp_read_inc_prod +working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_160616/fv3_ccpp_read_inc_prod Checking test 005 fv3_ccpp_read_inc results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -343,7 +343,7 @@ Test 005 fv3_ccpp_read_inc PASS baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200713/fv3_wrtGauss_netcdf_esmf_ccpp -working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_69502/fv3_ccpp_wrtGauss_netcdf_esmf_prod +working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_160616/fv3_ccpp_wrtGauss_netcdf_esmf_prod Checking test 006 fv3_ccpp_wrtGauss_netcdf_esmf results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -391,7 +391,7 @@ Test 006 fv3_ccpp_wrtGauss_netcdf_esmf PASS baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200713/fv3_wrtGauss_netcdf_ccpp -working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_69502/fv3_ccpp_wrtGauss_netcdf_prod +working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_160616/fv3_ccpp_wrtGauss_netcdf_prod Checking test 007 fv3_ccpp_wrtGauss_netcdf results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -439,7 +439,7 @@ Test 007 fv3_ccpp_wrtGauss_netcdf PASS baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200713/fv3_wrtGlatlon_netcdf_ccpp -working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_69502/fv3_ccpp_wrtGlatlon_netcdf_prod +working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_160616/fv3_ccpp_wrtGlatlon_netcdf_prod Checking test 008 fv3_ccpp_wrtGlatlon_netcdf results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -487,7 +487,7 @@ Test 008 fv3_ccpp_wrtGlatlon_netcdf PASS baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200713/fv3_wrtGauss_nemsio_ccpp -working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_69502/fv3_ccpp_wrtGauss_nemsio_prod +working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_160616/fv3_ccpp_wrtGauss_nemsio_prod Checking test 009 fv3_ccpp_wrtGauss_nemsio results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -535,7 +535,7 @@ Test 009 fv3_ccpp_wrtGauss_nemsio PASS baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200713/fv3_wrtGauss_nemsio_c192_ccpp -working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_69502/fv3_ccpp_wrtGauss_nemsio_c192_prod +working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_160616/fv3_ccpp_wrtGauss_nemsio_c192_prod Checking test 010 fv3_ccpp_wrtGauss_nemsio_c192 results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -583,7 +583,7 @@ Test 010 fv3_ccpp_wrtGauss_nemsio_c192 PASS baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200713/fv3_stochy_ccpp -working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_69502/fv3_ccpp_stochy_prod +working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_160616/fv3_ccpp_stochy_prod Checking test 011 fv3_ccpp_stochy results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -651,7 +651,7 @@ Test 011 fv3_ccpp_stochy PASS baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200713/fv3_iau_ccpp -working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_69502/fv3_ccpp_iau_prod +working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_160616/fv3_ccpp_iau_prod Checking test 012 fv3_ccpp_iau results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -719,7 +719,7 @@ Test 012 fv3_ccpp_iau PASS baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200713/fv3_lheatstrg_ccpp -working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_69502/fv3_ccpp_lheatstrg_prod +working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_160616/fv3_ccpp_lheatstrg_prod Checking test 013 fv3_ccpp_lheatstrg results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -767,7 +767,7 @@ Test 013 fv3_ccpp_lheatstrg PASS baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200713/fv3_gfdlmprad_ccpp -working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_69502/fv3_ccpp_gfdlmprad_prod +working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_160616/fv3_ccpp_gfdlmprad_prod Checking test 014 fv3_ccpp_gfdlmprad results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -816,7 +816,7 @@ Test 014 fv3_ccpp_gfdlmprad PASS baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200713/fv3_gfdlmprad_atmwav_ccpp -working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_69502/fv3_ccpp_gfdlmprad_atmwav_prod +working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_160616/fv3_ccpp_gfdlmprad_atmwav_prod Checking test 015 fv3_ccpp_gfdlmprad_atmwav results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -865,7 +865,7 @@ Test 015 fv3_ccpp_gfdlmprad_atmwav PASS baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200713/fv3_multigases_ccpp -working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_69502/fv3_ccpp_multigases_prod +working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_160616/fv3_ccpp_multigases_prod Checking test 016 fv3_ccpp_multigases results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -939,7 +939,7 @@ Test 016 fv3_ccpp_multigases PASS baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200713/fv3_control_32bit_ccpp -working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_69502/fv3_ccpp_control_32bit_prod +working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_160616/fv3_ccpp_control_32bit_prod Checking test 017 fv3_ccpp_control_32bit results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1007,7 +1007,7 @@ Test 017 fv3_ccpp_control_32bit PASS baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200713/fv3_stretched_ccpp -working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_69502/fv3_ccpp_stretched_prod +working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_160616/fv3_ccpp_stretched_prod Checking test 018 fv3_ccpp_stretched results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1063,7 +1063,7 @@ Test 018 fv3_ccpp_stretched PASS baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200713/fv3_stretched_nest_ccpp -working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_69502/fv3_ccpp_stretched_nest_prod +working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_160616/fv3_ccpp_stretched_nest_prod Checking test 019 fv3_ccpp_stretched_nest results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1130,7 +1130,7 @@ Test 019 fv3_ccpp_stretched_nest PASS baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200713/fv3_regional_control_ccpp -working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_69502/fv3_ccpp_regional_control_prod +working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_160616/fv3_ccpp_regional_control_prod Checking test 020 fv3_ccpp_regional_control results .... Comparing atmos_4xdaily.nc .........OK Comparing fv3_history2d.nc .........OK @@ -1141,7 +1141,7 @@ Test 020 fv3_ccpp_regional_control PASS baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200713/fv3_regional_restart_ccpp -working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_69502/fv3_ccpp_regional_restart_prod +working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_160616/fv3_ccpp_regional_restart_prod Checking test 021 fv3_ccpp_regional_restart results .... Comparing atmos_4xdaily.nc .........OK Comparing fv3_history2d.nc .........OK @@ -1150,7 +1150,7 @@ Test 021 fv3_ccpp_regional_restart PASS baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200713/fv3_regional_quilt_ccpp -working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_69502/fv3_ccpp_regional_quilt_prod +working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_160616/fv3_ccpp_regional_quilt_prod Checking test 022 fv3_ccpp_regional_quilt results .... Comparing atmos_4xdaily.nc .........OK Comparing dynf000.nc .........OK @@ -1161,7 +1161,7 @@ Test 022 fv3_ccpp_regional_quilt PASS baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200713/fv3_regional_c768_ccpp -working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_69502/fv3_ccpp_regional_c768_prod +working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_160616/fv3_ccpp_regional_c768_prod Checking test 023 fv3_ccpp_regional_c768 results .... Comparing atmos_4xdaily.nc .........OK Comparing dynf000.nc .........OK @@ -1172,19 +1172,19 @@ Test 023 fv3_ccpp_regional_c768 PASS baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200713/fv3_control_ccpp -working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_69502/fv3_ccpp_control_debug_prod +working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_160616/fv3_ccpp_control_debug_prod Checking test 024 fv3_ccpp_control_debug results .... Test 024 fv3_ccpp_control_debug PASS baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200713/fv3_stretched_nest_ccpp -working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_69502/fv3_ccpp_stretched_nest_debug_prod +working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_160616/fv3_ccpp_stretched_nest_debug_prod Checking test 025 fv3_ccpp_stretched_nest_debug results .... Test 025 fv3_ccpp_stretched_nest_debug PASS baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200713/fv3_gfdlmp_ccpp -working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_69502/fv3_ccpp_gfdlmp_prod +working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_160616/fv3_ccpp_gfdlmp_prod Checking test 026 fv3_ccpp_gfdlmp results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1232,7 +1232,7 @@ Test 026 fv3_ccpp_gfdlmp PASS baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200713/fv3_gfdlmprad_gwd_ccpp -working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_69502/fv3_ccpp_gfdlmprad_gwd_prod +working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_160616/fv3_ccpp_gfdlmprad_gwd_prod Checking test 027 fv3_ccpp_gfdlmprad_gwd results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1280,7 +1280,7 @@ Test 027 fv3_ccpp_gfdlmprad_gwd PASS baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200713/fv3_gfdlmprad_noahmp_ccpp -working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_69502/fv3_ccpp_gfdlmprad_noahmp_prod +working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_160616/fv3_ccpp_gfdlmprad_noahmp_prod Checking test 028 fv3_ccpp_gfdlmprad_noahmp results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1328,7 +1328,7 @@ Test 028 fv3_ccpp_gfdlmprad_noahmp PASS baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200713/fv3_gfdlmp_hwrfsas_ccpp -working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_69502/fv3_ccpp_gfdlmp_hwrfsas_prod +working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_160616/fv3_ccpp_gfdlmp_hwrfsas_prod Checking test 029 fv3_ccpp_gfdlmp_hwrfsas results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1376,7 +1376,7 @@ Test 029 fv3_ccpp_gfdlmp_hwrfsas PASS baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200713/fv3_csawmg_ccpp -working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_69502/fv3_ccpp_csawmg_prod +working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_160616/fv3_ccpp_csawmg_prod Checking test 030 fv3_ccpp_csawmg results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1424,7 +1424,7 @@ Test 030 fv3_ccpp_csawmg PASS baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200713/fv3_satmedmf_ccpp -working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_69502/fv3_ccpp_satmedmf_prod +working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_160616/fv3_ccpp_satmedmf_prod Checking test 031 fv3_ccpp_satmedmf results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1472,7 +1472,7 @@ Test 031 fv3_ccpp_satmedmf PASS baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200713/fv3_satmedmfq_ccpp -working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_69502/fv3_ccpp_satmedmfq_prod +working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_160616/fv3_ccpp_satmedmfq_prod Checking test 032 fv3_ccpp_satmedmfq results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1520,7 +1520,7 @@ Test 032 fv3_ccpp_satmedmfq PASS baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200713/fv3_gfdlmp_32bit_ccpp -working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_69502/fv3_ccpp_gfdlmp_32bit_prod +working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_160616/fv3_ccpp_gfdlmp_32bit_prod Checking test 033 fv3_ccpp_gfdlmp_32bit results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1568,7 +1568,7 @@ Test 033 fv3_ccpp_gfdlmp_32bit PASS baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200713/fv3_gfdlmprad_32bit_post_ccpp -working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_69502/fv3_ccpp_gfdlmprad_32bit_post_prod +working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_160616/fv3_ccpp_gfdlmprad_32bit_post_prod Checking test 034 fv3_ccpp_gfdlmprad_32bit_post results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1620,7 +1620,7 @@ Test 034 fv3_ccpp_gfdlmprad_32bit_post PASS baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200713/fv3_cpt_ccpp -working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_69502/fv3_ccpp_cpt_prod +working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_160616/fv3_ccpp_cpt_prod Checking test 035 fv3_ccpp_cpt results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1674,7 +1674,7 @@ Test 035 fv3_ccpp_cpt PASS baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200713/fv3_gsd_ccpp -working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_69502/fv3_ccpp_gsd_prod +working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_160616/fv3_ccpp_gsd_prod Checking test 036 fv3_ccpp_gsd results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1766,7 +1766,7 @@ Test 036 fv3_ccpp_gsd PASS baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200713/fv3_thompson_ccpp -working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_69502/fv3_ccpp_thompson_prod +working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_160616/fv3_ccpp_thompson_prod Checking test 037 fv3_ccpp_thompson results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1834,7 +1834,7 @@ Test 037 fv3_ccpp_thompson PASS baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200713/fv3_thompson_no_aero_ccpp -working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_69502/fv3_ccpp_thompson_no_aero_prod +working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_160616/fv3_ccpp_thompson_no_aero_prod Checking test 038 fv3_ccpp_thompson_no_aero results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1902,7 +1902,7 @@ Test 038 fv3_ccpp_thompson_no_aero PASS baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200713/fv3_gsd_debug_ccpp -working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_69502/fv3_ccpp_gsd_debug_prod +working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_160616/fv3_ccpp_gsd_debug_prod Checking test 039 fv3_ccpp_gsd_debug results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1970,7 +1970,7 @@ Test 039 fv3_ccpp_gsd_debug PASS baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200713/fv3_gsd_diag3d_debug_ccpp -working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_69502/fv3_ccpp_gsd_diag3d_debug_prod +working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_160616/fv3_ccpp_gsd_diag3d_debug_prod Checking test 040 fv3_ccpp_gsd_diag3d_debug results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -2038,7 +2038,7 @@ Test 040 fv3_ccpp_gsd_diag3d_debug PASS baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200713/fv3_thompson_debug_ccpp -working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_69502/fv3_ccpp_thompson_debug_prod +working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_160616/fv3_ccpp_thompson_debug_prod Checking test 041 fv3_ccpp_thompson_debug results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -2106,7 +2106,7 @@ Test 041 fv3_ccpp_thompson_debug PASS baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200713/fv3_thompson_no_aero_debug_ccpp -working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_69502/fv3_ccpp_thompson_no_aero_debug_prod +working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_160616/fv3_ccpp_thompson_no_aero_debug_prod Checking test 042 fv3_ccpp_thompson_no_aero_debug results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -2174,7 +2174,7 @@ Test 042 fv3_ccpp_thompson_no_aero_debug PASS baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200713/fv3_gfsv16_csawmg_ccpp -working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_69502/fv3_ccpp_gfsv16_csawmg_prod +working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_160616/fv3_ccpp_gfsv16_csawmg_prod Checking test 043 fv3_ccpp_gfsv16_csawmg results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -2222,7 +2222,7 @@ Test 043 fv3_ccpp_gfsv16_csawmg PASS baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200713/fv3_gfsv16_csawmgt_ccpp -working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_69502/fv3_ccpp_gfsv16_csawmgt_prod +working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_160616/fv3_ccpp_gfsv16_csawmgt_prod Checking test 044 fv3_ccpp_gfsv16_csawmgt results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -2270,7 +2270,7 @@ Test 044 fv3_ccpp_gfsv16_csawmgt PASS baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200713/fv3_gocart_clm_ccpp -working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_69502/fv3_ccpp_gocart_clm_prod +working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_160616/fv3_ccpp_gocart_clm_prod Checking test 045 fv3_ccpp_gocart_clm results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -2318,5 +2318,5 @@ Test 045 fv3_ccpp_gocart_clm PASS REGRESSION TEST WAS SUCCESSFUL -Tue Jul 14 02:14:30 UTC 2020 -Elapsed time: 00h:43m:21s. Have a nice day! +Wed Jul 15 14:45:52 UTC 2020 +Elapsed time: 00h:42m:16s. Have a nice day! From bd94f334a08da7374409f8844e86da9807e69455 Mon Sep 17 00:00:00 2001 From: Dusan Jovic Date: Wed, 15 Jul 2020 14:55:39 +0000 Subject: [PATCH 08/11] Add regtest logs on WCOSS Cray --- tests/Compile_wcoss_cray.log | 591 +++++++++++++-------------- tests/RegressionTests_wcoss_cray.log | 94 ++--- 2 files changed, 341 insertions(+), 344 deletions(-) diff --git a/tests/Compile_wcoss_cray.log b/tests/Compile_wcoss_cray.log index 73390bb07d..a5eedfbd23 100644 --- a/tests/Compile_wcoss_cray.log +++ b/tests/Compile_wcoss_cray.log @@ -1,22 +1,22 @@ + SECONDS=0 + [[ 4 != 4 ]] -+ export PATHRT=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -+ PATHRT=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -+ export RUNDIR_ROOT=/gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39969 -+ RUNDIR_ROOT=/gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39969 ++ export PATHRT=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests ++ PATHRT=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests ++ export RUNDIR_ROOT=/gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_47014 ++ RUNDIR_ROOT=/gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_47014 + export 'MAKE_OPT=CCPP=Y SUITES=FV3_GSD_v0,FV3_GFS_v15_thompson 32BIT=Y DEBUG=Y' + MAKE_OPT='CCPP=Y SUITES=FV3_GSD_v0,FV3_GFS_v15_thompson 32BIT=Y DEBUG=Y' + export COMPILE_NR=10 + COMPILE_NR=10 -+ cd /gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -+ [[ -e /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39969/compile_10.env ]] -+ source /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39969/compile_10.env ++ cd /gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests ++ [[ -e /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_47014/compile_10.env ]] ++ source /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_47014/compile_10.env ++ export MACHINE_ID=wcoss_cray ++ MACHINE_ID=wcoss_cray -++ export PATHRT=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -++ PATHRT=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -++ export PATHTR=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr154/ufs-weather-model -++ PATHTR=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr154/ufs-weather-model +++ export PATHRT=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests +++ PATHRT=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests +++ export PATHTR=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model +++ PATHTR=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model ++ export SCHEDULER=lsf ++ SCHEDULER=lsf ++ export ACCNR=GFS-DEV @@ -29,10 +29,10 @@ ++ ROCOTO=false ++ export ECFLOW=true ++ ECFLOW=true -++ export REGRESSIONTEST_LOG=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/RegressionTests_wcoss_cray.log -++ REGRESSIONTEST_LOG=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/RegressionTests_wcoss_cray.log -++ export LOG_DIR=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/log_wcoss_cray -++ LOG_DIR=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/log_wcoss_cray +++ export REGRESSIONTEST_LOG=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/RegressionTests_wcoss_cray.log +++ REGRESSIONTEST_LOG=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/RegressionTests_wcoss_cray.log +++ export LOG_DIR=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/log_wcoss_cray +++ LOG_DIR=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/log_wcoss_cray + source default_vars.sh ++ '[' wcoss_cray = wcoss ']' ++ '[' wcoss_cray = wcoss_cray ']' @@ -62,15 +62,15 @@ + TEST_NR=10 + export JBNME=compile_10 + JBNME=compile_10 -+ export RUNDIR=/gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39969/compile_10 -+ RUNDIR=/gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39969/compile_10 ++ export RUNDIR=/gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_47014/compile_10 ++ RUNDIR=/gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_47014/compile_10 + source rt_utils.sh ++ set -eu -++ [[ /gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/run_compile.sh = \.\/\r\t\_\u\t\i\l\s\.\s\h ]] +++ [[ /gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/run_compile.sh = \.\/\r\t\_\u\t\i\l\s\.\s\h ]] ++ UNIT_TEST=false + source atparse.bash -+ mkdir -p /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39969/compile_10 -+ cd /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39969/compile_10 ++ mkdir -p /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_47014/compile_10 ++ cd /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_47014/compile_10 + [[ lsf = \s\l\u\r\m ]] + [[ lsf = \l\s\f ]] + atparse @@ -86,7 +86,7 @@ + set +x ESUB: OMP_NUM_THREADS value changed to 1 ESUB: MEMORY LIMIT rusage set to "rusage[mem=500]" - with rusage set in shared job. -Job id 714139 +Job id 750369 TEST 10 compile is waiting to enter the queue 1 min. TEST 10 compile is running, Status: RUN 2 min. TEST 10 compile is running, Status: RUN @@ -96,30 +96,29 @@ TEST 10 compile is waiting to enter the queue 6 min. TEST 10 compile is running, Status: RUN 7 min. TEST 10 compile is running, Status: RUN 8 min. TEST 10 compile is running, Status: RUN -9 min. TEST 10 compile is running, Status: RUN -10 min. TEST 10 compile is finished, Status: DONE -+ elapsed=606 -+ echo 'Elapsed time 606 seconds. Compile 10' -Elapsed time 606 seconds. Compile 10 +9 min. TEST 10 compile is finished, Status: DONE ++ elapsed=546 ++ echo 'Elapsed time 546 seconds. Compile 10' +Elapsed time 546 seconds. Compile 10 + SECONDS=0 + [[ 4 != 4 ]] -+ export PATHRT=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -+ PATHRT=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -+ export RUNDIR_ROOT=/gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39969 -+ RUNDIR_ROOT=/gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39969 ++ export PATHRT=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests ++ PATHRT=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests ++ export RUNDIR_ROOT=/gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_47014 ++ RUNDIR_ROOT=/gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_47014 + export 'MAKE_OPT=CCPP=Y SUITES=FV3_GFS_v16_csawmg' + MAKE_OPT='CCPP=Y SUITES=FV3_GFS_v16_csawmg' + export COMPILE_NR=11 + COMPILE_NR=11 -+ cd /gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -+ [[ -e /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39969/compile_11.env ]] -+ source /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39969/compile_11.env ++ cd /gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests ++ [[ -e /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_47014/compile_11.env ]] ++ source /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_47014/compile_11.env ++ export MACHINE_ID=wcoss_cray ++ MACHINE_ID=wcoss_cray -++ export PATHRT=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -++ PATHRT=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -++ export PATHTR=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr154/ufs-weather-model -++ PATHTR=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr154/ufs-weather-model +++ export PATHRT=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests +++ PATHRT=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests +++ export PATHTR=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model +++ PATHTR=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model ++ export SCHEDULER=lsf ++ SCHEDULER=lsf ++ export ACCNR=GFS-DEV @@ -132,10 +131,10 @@ Elapsed time 606 seconds. Compile 10 ++ ROCOTO=false ++ export ECFLOW=true ++ ECFLOW=true -++ export REGRESSIONTEST_LOG=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/RegressionTests_wcoss_cray.log -++ REGRESSIONTEST_LOG=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/RegressionTests_wcoss_cray.log -++ export LOG_DIR=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/log_wcoss_cray -++ LOG_DIR=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/log_wcoss_cray +++ export REGRESSIONTEST_LOG=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/RegressionTests_wcoss_cray.log +++ REGRESSIONTEST_LOG=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/RegressionTests_wcoss_cray.log +++ export LOG_DIR=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/log_wcoss_cray +++ LOG_DIR=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/log_wcoss_cray + source default_vars.sh ++ '[' wcoss_cray = wcoss ']' ++ '[' wcoss_cray = wcoss_cray ']' @@ -165,15 +164,15 @@ Elapsed time 606 seconds. Compile 10 + TEST_NR=11 + export JBNME=compile_11 + JBNME=compile_11 -+ export RUNDIR=/gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39969/compile_11 -+ RUNDIR=/gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39969/compile_11 ++ export RUNDIR=/gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_47014/compile_11 ++ RUNDIR=/gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_47014/compile_11 + source rt_utils.sh ++ set -eu -++ [[ /gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/run_compile.sh = \.\/\r\t\_\u\t\i\l\s\.\s\h ]] +++ [[ /gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/run_compile.sh = \.\/\r\t\_\u\t\i\l\s\.\s\h ]] ++ UNIT_TEST=false + source atparse.bash -+ mkdir -p /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39969/compile_11 -+ cd /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39969/compile_11 ++ mkdir -p /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_47014/compile_11 ++ cd /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_47014/compile_11 + [[ lsf = \s\l\u\r\m ]] + [[ lsf = \l\s\f ]] + atparse @@ -189,7 +188,7 @@ Elapsed time 606 seconds. Compile 10 + set +x ESUB: OMP_NUM_THREADS value changed to 1 ESUB: MEMORY LIMIT rusage set to "rusage[mem=500]" - with rusage set in shared job. -Job id 714240 +Job id 750564 TEST 11 compile is waiting to enter the queue 1 min. TEST 11 compile is running, Status: RUN 2 min. TEST 11 compile is running, Status: RUN @@ -210,23 +209,23 @@ TEST 11 compile is waiting to enter the queue Elapsed time 846 seconds. Compile 11 + SECONDS=0 + [[ 4 != 4 ]] -+ export PATHRT=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -+ PATHRT=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -+ export RUNDIR_ROOT=/gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39969 -+ RUNDIR_ROOT=/gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39969 ++ export PATHRT=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests ++ PATHRT=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests ++ export RUNDIR_ROOT=/gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_47014 ++ RUNDIR_ROOT=/gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_47014 + export 'MAKE_OPT=CCPP=Y SUITES=FV3_GFS_2017_gfdlmp,FV3_GFS_2017_gfdlmp_noahmp' + MAKE_OPT='CCPP=Y SUITES=FV3_GFS_2017_gfdlmp,FV3_GFS_2017_gfdlmp_noahmp' + export COMPILE_NR=12 + COMPILE_NR=12 -+ cd /gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -+ [[ -e /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39969/compile_12.env ]] -+ source /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39969/compile_12.env ++ cd /gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests ++ [[ -e /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_47014/compile_12.env ]] ++ source /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_47014/compile_12.env ++ export MACHINE_ID=wcoss_cray ++ MACHINE_ID=wcoss_cray -++ export PATHRT=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -++ PATHRT=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -++ export PATHTR=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr154/ufs-weather-model -++ PATHTR=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr154/ufs-weather-model +++ export PATHRT=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests +++ PATHRT=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests +++ export PATHTR=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model +++ PATHTR=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model ++ export SCHEDULER=lsf ++ SCHEDULER=lsf ++ export ACCNR=GFS-DEV @@ -239,10 +238,10 @@ Elapsed time 846 seconds. Compile 11 ++ ROCOTO=false ++ export ECFLOW=true ++ ECFLOW=true -++ export REGRESSIONTEST_LOG=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/RegressionTests_wcoss_cray.log -++ REGRESSIONTEST_LOG=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/RegressionTests_wcoss_cray.log -++ export LOG_DIR=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/log_wcoss_cray -++ LOG_DIR=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/log_wcoss_cray +++ export REGRESSIONTEST_LOG=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/RegressionTests_wcoss_cray.log +++ REGRESSIONTEST_LOG=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/RegressionTests_wcoss_cray.log +++ export LOG_DIR=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/log_wcoss_cray +++ LOG_DIR=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/log_wcoss_cray + source default_vars.sh ++ '[' wcoss_cray = wcoss ']' ++ '[' wcoss_cray = wcoss_cray ']' @@ -272,15 +271,15 @@ Elapsed time 846 seconds. Compile 11 + TEST_NR=12 + export JBNME=compile_12 + JBNME=compile_12 -+ export RUNDIR=/gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39969/compile_12 -+ RUNDIR=/gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39969/compile_12 ++ export RUNDIR=/gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_47014/compile_12 ++ RUNDIR=/gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_47014/compile_12 + source rt_utils.sh ++ set -eu -++ [[ /gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/run_compile.sh = \.\/\r\t\_\u\t\i\l\s\.\s\h ]] +++ [[ /gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/run_compile.sh = \.\/\r\t\_\u\t\i\l\s\.\s\h ]] ++ UNIT_TEST=false + source atparse.bash -+ mkdir -p /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39969/compile_12 -+ cd /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39969/compile_12 ++ mkdir -p /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_47014/compile_12 ++ cd /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_47014/compile_12 + [[ lsf = \s\l\u\r\m ]] + [[ lsf = \l\s\f ]] + atparse @@ -296,7 +295,7 @@ Elapsed time 846 seconds. Compile 11 + set +x ESUB: OMP_NUM_THREADS value changed to 1 ESUB: MEMORY LIMIT rusage set to "rusage[mem=500]" - with rusage set in shared job. -Job id 714241 +Job id 750565 TEST 12 compile is waiting to enter the queue 1 min. TEST 12 compile is running, Status: RUN 2 min. TEST 12 compile is running, Status: RUN @@ -319,23 +318,23 @@ TEST 12 compile is waiting to enter the queue Elapsed time 966 seconds. Compile 12 + SECONDS=0 + [[ 4 != 4 ]] -+ export PATHRT=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -+ PATHRT=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -+ export RUNDIR_ROOT=/gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39969 -+ RUNDIR_ROOT=/gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39969 ++ export PATHRT=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests ++ PATHRT=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests ++ export RUNDIR_ROOT=/gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_47014 ++ RUNDIR_ROOT=/gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_47014 + export 'MAKE_OPT=CCPP=Y SUITES=FV3_GFS_2017' + MAKE_OPT='CCPP=Y SUITES=FV3_GFS_2017' + export COMPILE_NR=1 + COMPILE_NR=1 -+ cd /gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -+ [[ -e /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39969/compile_1.env ]] -+ source /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39969/compile_1.env ++ cd /gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests ++ [[ -e /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_47014/compile_1.env ]] ++ source /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_47014/compile_1.env ++ export MACHINE_ID=wcoss_cray ++ MACHINE_ID=wcoss_cray -++ export PATHRT=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -++ PATHRT=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -++ export PATHTR=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr154/ufs-weather-model -++ PATHTR=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr154/ufs-weather-model +++ export PATHRT=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests +++ PATHRT=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests +++ export PATHTR=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model +++ PATHTR=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model ++ export SCHEDULER=lsf ++ SCHEDULER=lsf ++ export ACCNR=GFS-DEV @@ -348,10 +347,10 @@ Elapsed time 966 seconds. Compile 12 ++ ROCOTO=false ++ export ECFLOW=true ++ ECFLOW=true -++ export REGRESSIONTEST_LOG=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/RegressionTests_wcoss_cray.log -++ REGRESSIONTEST_LOG=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/RegressionTests_wcoss_cray.log -++ export LOG_DIR=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/log_wcoss_cray -++ LOG_DIR=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/log_wcoss_cray +++ export REGRESSIONTEST_LOG=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/RegressionTests_wcoss_cray.log +++ REGRESSIONTEST_LOG=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/RegressionTests_wcoss_cray.log +++ export LOG_DIR=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/log_wcoss_cray +++ LOG_DIR=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/log_wcoss_cray + source default_vars.sh ++ '[' wcoss_cray = wcoss ']' ++ '[' wcoss_cray = wcoss_cray ']' @@ -381,15 +380,15 @@ Elapsed time 966 seconds. Compile 12 + TEST_NR=1 + export JBNME=compile_1 + JBNME=compile_1 -+ export RUNDIR=/gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39969/compile_1 -+ RUNDIR=/gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39969/compile_1 ++ export RUNDIR=/gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_47014/compile_1 ++ RUNDIR=/gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_47014/compile_1 + source rt_utils.sh ++ set -eu -++ [[ /gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/run_compile.sh = \.\/\r\t\_\u\t\i\l\s\.\s\h ]] +++ [[ /gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/run_compile.sh = \.\/\r\t\_\u\t\i\l\s\.\s\h ]] ++ UNIT_TEST=false + source atparse.bash -+ mkdir -p /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39969/compile_1 -+ cd /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39969/compile_1 ++ mkdir -p /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_47014/compile_1 ++ cd /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_47014/compile_1 + [[ lsf = \s\l\u\r\m ]] + [[ lsf = \l\s\f ]] + atparse @@ -405,7 +404,7 @@ Elapsed time 966 seconds. Compile 12 + set +x ESUB: OMP_NUM_THREADS value changed to 1 ESUB: MEMORY LIMIT rusage set to "rusage[mem=500]" - with rusage set in shared job. -Job id 714144 +Job id 750361 TEST 1 compile is waiting to enter the queue 1 min. TEST 1 compile is running, Status: RUN 2 min. TEST 1 compile is running, Status: RUN @@ -420,29 +419,30 @@ TEST 1 compile is waiting to enter the queue 11 min. TEST 1 compile is running, Status: RUN 12 min. TEST 1 compile is running, Status: RUN 13 min. TEST 1 compile is running, Status: RUN -14 min. TEST 1 compile is finished, Status: DONE -+ elapsed=846 -+ echo 'Elapsed time 846 seconds. Compile 1' -Elapsed time 846 seconds. Compile 1 +14 min. TEST 1 compile is running, Status: RUN +15 min. TEST 1 compile is finished, Status: DONE ++ elapsed=906 ++ echo 'Elapsed time 906 seconds. Compile 1' +Elapsed time 906 seconds. Compile 1 + SECONDS=0 + [[ 4 != 4 ]] -+ export PATHRT=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -+ PATHRT=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -+ export RUNDIR_ROOT=/gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39969 -+ RUNDIR_ROOT=/gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39969 ++ export PATHRT=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests ++ PATHRT=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests ++ export RUNDIR_ROOT=/gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_47014 ++ RUNDIR_ROOT=/gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_47014 + export 'MAKE_OPT=CCPP=Y SUITES=FV3_GFS_2017,FV3_GFS_2017_gfdlmp WW3=Y' + MAKE_OPT='CCPP=Y SUITES=FV3_GFS_2017,FV3_GFS_2017_gfdlmp WW3=Y' + export COMPILE_NR=2 + COMPILE_NR=2 -+ cd /gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -+ [[ -e /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39969/compile_2.env ]] -+ source /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39969/compile_2.env ++ cd /gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests ++ [[ -e /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_47014/compile_2.env ]] ++ source /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_47014/compile_2.env ++ export MACHINE_ID=wcoss_cray ++ MACHINE_ID=wcoss_cray -++ export PATHRT=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -++ PATHRT=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -++ export PATHTR=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr154/ufs-weather-model -++ PATHTR=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr154/ufs-weather-model +++ export PATHRT=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests +++ PATHRT=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests +++ export PATHTR=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model +++ PATHTR=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model ++ export SCHEDULER=lsf ++ SCHEDULER=lsf ++ export ACCNR=GFS-DEV @@ -455,10 +455,10 @@ Elapsed time 846 seconds. Compile 1 ++ ROCOTO=false ++ export ECFLOW=true ++ ECFLOW=true -++ export REGRESSIONTEST_LOG=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/RegressionTests_wcoss_cray.log -++ REGRESSIONTEST_LOG=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/RegressionTests_wcoss_cray.log -++ export LOG_DIR=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/log_wcoss_cray -++ LOG_DIR=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/log_wcoss_cray +++ export REGRESSIONTEST_LOG=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/RegressionTests_wcoss_cray.log +++ REGRESSIONTEST_LOG=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/RegressionTests_wcoss_cray.log +++ export LOG_DIR=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/log_wcoss_cray +++ LOG_DIR=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/log_wcoss_cray + source default_vars.sh ++ '[' wcoss_cray = wcoss ']' ++ '[' wcoss_cray = wcoss_cray ']' @@ -488,15 +488,15 @@ Elapsed time 846 seconds. Compile 1 + TEST_NR=2 + export JBNME=compile_2 + JBNME=compile_2 -+ export RUNDIR=/gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39969/compile_2 -+ RUNDIR=/gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39969/compile_2 ++ export RUNDIR=/gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_47014/compile_2 ++ RUNDIR=/gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_47014/compile_2 + source rt_utils.sh ++ set -eu -++ [[ /gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/run_compile.sh = \.\/\r\t\_\u\t\i\l\s\.\s\h ]] +++ [[ /gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/run_compile.sh = \.\/\r\t\_\u\t\i\l\s\.\s\h ]] ++ UNIT_TEST=false + source atparse.bash -+ mkdir -p /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39969/compile_2 -+ cd /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39969/compile_2 ++ mkdir -p /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_47014/compile_2 ++ cd /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_47014/compile_2 + [[ lsf = \s\l\u\r\m ]] + [[ lsf = \l\s\f ]] + atparse @@ -512,7 +512,7 @@ Elapsed time 846 seconds. Compile 1 + set +x ESUB: OMP_NUM_THREADS value changed to 1 ESUB: MEMORY LIMIT rusage set to "rusage[mem=500]" - with rusage set in shared job. -Job id 714140 +Job id 750370 TEST 2 compile is waiting to enter the queue 1 min. TEST 2 compile is running, Status: RUN 2 min. TEST 2 compile is running, Status: RUN @@ -529,29 +529,34 @@ TEST 2 compile is waiting to enter the queue 13 min. TEST 2 compile is running, Status: RUN 14 min. TEST 2 compile is running, Status: RUN 15 min. TEST 2 compile is running, Status: RUN -16 min. TEST 2 compile is finished, Status: DONE -+ elapsed=966 -+ echo 'Elapsed time 966 seconds. Compile 2' -Elapsed time 966 seconds. Compile 2 +16 min. TEST 2 compile is running, Status: RUN +17 min. TEST 2 compile is running, Status: RUN +18 min. TEST 2 compile is running, Status: RUN +19 min. TEST 2 compile is running, Status: RUN +20 min. TEST 2 compile is running, Status: RUN +21 min. TEST 2 compile is finished, Status: DONE ++ elapsed=1266 ++ echo 'Elapsed time 1266 seconds. Compile 2' +Elapsed time 1266 seconds. Compile 2 + SECONDS=0 + [[ 4 != 4 ]] -+ export PATHRT=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -+ PATHRT=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -+ export RUNDIR_ROOT=/gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39969 -+ RUNDIR_ROOT=/gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39969 ++ export PATHRT=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests ++ PATHRT=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests ++ export RUNDIR_ROOT=/gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_47014 ++ RUNDIR_ROOT=/gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_47014 + export 'MAKE_OPT=CCPP=Y SUITES=FV3_GFS_2017_fv3wam 32BIT=Y MULTI_GASES=Y' + MAKE_OPT='CCPP=Y SUITES=FV3_GFS_2017_fv3wam 32BIT=Y MULTI_GASES=Y' + export COMPILE_NR=3 + COMPILE_NR=3 -+ cd /gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -+ [[ -e /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39969/compile_3.env ]] -+ source /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39969/compile_3.env ++ cd /gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests ++ [[ -e /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_47014/compile_3.env ]] ++ source /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_47014/compile_3.env ++ export MACHINE_ID=wcoss_cray ++ MACHINE_ID=wcoss_cray -++ export PATHRT=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -++ PATHRT=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -++ export PATHTR=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr154/ufs-weather-model -++ PATHTR=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr154/ufs-weather-model +++ export PATHRT=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests +++ PATHRT=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests +++ export PATHTR=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model +++ PATHTR=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model ++ export SCHEDULER=lsf ++ SCHEDULER=lsf ++ export ACCNR=GFS-DEV @@ -564,10 +569,10 @@ Elapsed time 966 seconds. Compile 2 ++ ROCOTO=false ++ export ECFLOW=true ++ ECFLOW=true -++ export REGRESSIONTEST_LOG=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/RegressionTests_wcoss_cray.log -++ REGRESSIONTEST_LOG=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/RegressionTests_wcoss_cray.log -++ export LOG_DIR=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/log_wcoss_cray -++ LOG_DIR=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/log_wcoss_cray +++ export REGRESSIONTEST_LOG=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/RegressionTests_wcoss_cray.log +++ REGRESSIONTEST_LOG=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/RegressionTests_wcoss_cray.log +++ export LOG_DIR=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/log_wcoss_cray +++ LOG_DIR=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/log_wcoss_cray + source default_vars.sh ++ '[' wcoss_cray = wcoss ']' ++ '[' wcoss_cray = wcoss_cray ']' @@ -597,15 +602,15 @@ Elapsed time 966 seconds. Compile 2 + TEST_NR=3 + export JBNME=compile_3 + JBNME=compile_3 -+ export RUNDIR=/gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39969/compile_3 -+ RUNDIR=/gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39969/compile_3 ++ export RUNDIR=/gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_47014/compile_3 ++ RUNDIR=/gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_47014/compile_3 + source rt_utils.sh ++ set -eu -++ [[ /gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/run_compile.sh = \.\/\r\t\_\u\t\i\l\s\.\s\h ]] +++ [[ /gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/run_compile.sh = \.\/\r\t\_\u\t\i\l\s\.\s\h ]] ++ UNIT_TEST=false + source atparse.bash -+ mkdir -p /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39969/compile_3 -+ cd /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39969/compile_3 ++ mkdir -p /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_47014/compile_3 ++ cd /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_47014/compile_3 + [[ lsf = \s\l\u\r\m ]] + [[ lsf = \l\s\f ]] + atparse @@ -621,7 +626,7 @@ Elapsed time 966 seconds. Compile 2 + set +x ESUB: OMP_NUM_THREADS value changed to 1 ESUB: MEMORY LIMIT rusage set to "rusage[mem=500]" - with rusage set in shared job. -Job id 714137 +Job id 750362 TEST 3 compile is waiting to enter the queue 1 min. TEST 3 compile is running, Status: RUN 2 min. TEST 3 compile is running, Status: RUN @@ -637,31 +642,29 @@ TEST 3 compile is waiting to enter the queue 12 min. TEST 3 compile is running, Status: RUN 13 min. TEST 3 compile is running, Status: RUN 14 min. TEST 3 compile is running, Status: RUN -15 min. TEST 3 compile is running, Status: RUN -16 min. TEST 3 compile is running, Status: RUN -17 min. TEST 3 compile is finished, Status: DONE -+ elapsed=1026 -+ echo 'Elapsed time 1026 seconds. Compile 3' -Elapsed time 1026 seconds. Compile 3 +15 min. TEST 3 compile is finished, Status: DONE ++ elapsed=906 ++ echo 'Elapsed time 906 seconds. Compile 3' +Elapsed time 906 seconds. Compile 3 + SECONDS=0 + [[ 4 != 4 ]] -+ export PATHRT=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -+ PATHRT=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -+ export RUNDIR_ROOT=/gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39969 -+ RUNDIR_ROOT=/gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39969 ++ export PATHRT=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests ++ PATHRT=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests ++ export RUNDIR_ROOT=/gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_47014 ++ RUNDIR_ROOT=/gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_47014 + export 'MAKE_OPT=CCPP=Y SUITES=FV3_GFS_2017,FV3_GFS_2017_stretched 32BIT=Y' + MAKE_OPT='CCPP=Y SUITES=FV3_GFS_2017,FV3_GFS_2017_stretched 32BIT=Y' + export COMPILE_NR=4 + COMPILE_NR=4 -+ cd /gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -+ [[ -e /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39969/compile_4.env ]] -+ source /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39969/compile_4.env ++ cd /gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests ++ [[ -e /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_47014/compile_4.env ]] ++ source /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_47014/compile_4.env ++ export MACHINE_ID=wcoss_cray ++ MACHINE_ID=wcoss_cray -++ export PATHRT=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -++ PATHRT=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -++ export PATHTR=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr154/ufs-weather-model -++ PATHTR=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr154/ufs-weather-model +++ export PATHRT=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests +++ PATHRT=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests +++ export PATHTR=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model +++ PATHTR=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model ++ export SCHEDULER=lsf ++ SCHEDULER=lsf ++ export ACCNR=GFS-DEV @@ -674,10 +677,10 @@ Elapsed time 1026 seconds. Compile 3 ++ ROCOTO=false ++ export ECFLOW=true ++ ECFLOW=true -++ export REGRESSIONTEST_LOG=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/RegressionTests_wcoss_cray.log -++ REGRESSIONTEST_LOG=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/RegressionTests_wcoss_cray.log -++ export LOG_DIR=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/log_wcoss_cray -++ LOG_DIR=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/log_wcoss_cray +++ export REGRESSIONTEST_LOG=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/RegressionTests_wcoss_cray.log +++ REGRESSIONTEST_LOG=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/RegressionTests_wcoss_cray.log +++ export LOG_DIR=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/log_wcoss_cray +++ LOG_DIR=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/log_wcoss_cray + source default_vars.sh ++ '[' wcoss_cray = wcoss ']' ++ '[' wcoss_cray = wcoss_cray ']' @@ -707,15 +710,15 @@ Elapsed time 1026 seconds. Compile 3 + TEST_NR=4 + export JBNME=compile_4 + JBNME=compile_4 -+ export RUNDIR=/gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39969/compile_4 -+ RUNDIR=/gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39969/compile_4 ++ export RUNDIR=/gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_47014/compile_4 ++ RUNDIR=/gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_47014/compile_4 + source rt_utils.sh ++ set -eu -++ [[ /gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/run_compile.sh = \.\/\r\t\_\u\t\i\l\s\.\s\h ]] +++ [[ /gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/run_compile.sh = \.\/\r\t\_\u\t\i\l\s\.\s\h ]] ++ UNIT_TEST=false + source atparse.bash -+ mkdir -p /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39969/compile_4 -+ cd /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39969/compile_4 ++ mkdir -p /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_47014/compile_4 ++ cd /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_47014/compile_4 + [[ lsf = \s\l\u\r\m ]] + [[ lsf = \l\s\f ]] + atparse @@ -731,7 +734,7 @@ Elapsed time 1026 seconds. Compile 3 + set +x ESUB: OMP_NUM_THREADS value changed to 1 ESUB: MEMORY LIMIT rusage set to "rusage[mem=500]" - with rusage set in shared job. -Job id 714136 +Job id 750364 TEST 4 compile is waiting to enter the queue 1 min. TEST 4 compile is running, Status: RUN 2 min. TEST 4 compile is running, Status: RUN @@ -746,31 +749,29 @@ TEST 4 compile is waiting to enter the queue 11 min. TEST 4 compile is running, Status: RUN 12 min. TEST 4 compile is running, Status: RUN 13 min. TEST 4 compile is running, Status: RUN -14 min. TEST 4 compile is running, Status: RUN -15 min. TEST 4 compile is running, Status: RUN -16 min. TEST 4 compile is finished, Status: DONE -+ elapsed=966 -+ echo 'Elapsed time 966 seconds. Compile 4' -Elapsed time 966 seconds. Compile 4 +14 min. TEST 4 compile is finished, Status: DONE ++ elapsed=846 ++ echo 'Elapsed time 846 seconds. Compile 4' +Elapsed time 846 seconds. Compile 4 + SECONDS=0 + [[ 4 != 4 ]] -+ export PATHRT=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -+ PATHRT=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -+ export RUNDIR_ROOT=/gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39969 -+ RUNDIR_ROOT=/gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39969 ++ export PATHRT=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests ++ PATHRT=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests ++ export RUNDIR_ROOT=/gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_47014 ++ RUNDIR_ROOT=/gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_47014 + export 'MAKE_OPT=CCPP=Y SUITES=FV3_GFS_2017_gfdlmp_regional,FV3_GFS_2017_gfdlmp_regional_c768 32BIT=Y' + MAKE_OPT='CCPP=Y SUITES=FV3_GFS_2017_gfdlmp_regional,FV3_GFS_2017_gfdlmp_regional_c768 32BIT=Y' + export COMPILE_NR=5 + COMPILE_NR=5 -+ cd /gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -+ [[ -e /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39969/compile_5.env ]] -+ source /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39969/compile_5.env ++ cd /gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests ++ [[ -e /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_47014/compile_5.env ]] ++ source /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_47014/compile_5.env ++ export MACHINE_ID=wcoss_cray ++ MACHINE_ID=wcoss_cray -++ export PATHRT=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -++ PATHRT=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -++ export PATHTR=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr154/ufs-weather-model -++ PATHTR=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr154/ufs-weather-model +++ export PATHRT=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests +++ PATHRT=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests +++ export PATHTR=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model +++ PATHTR=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model ++ export SCHEDULER=lsf ++ SCHEDULER=lsf ++ export ACCNR=GFS-DEV @@ -783,10 +784,10 @@ Elapsed time 966 seconds. Compile 4 ++ ROCOTO=false ++ export ECFLOW=true ++ ECFLOW=true -++ export REGRESSIONTEST_LOG=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/RegressionTests_wcoss_cray.log -++ REGRESSIONTEST_LOG=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/RegressionTests_wcoss_cray.log -++ export LOG_DIR=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/log_wcoss_cray -++ LOG_DIR=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/log_wcoss_cray +++ export REGRESSIONTEST_LOG=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/RegressionTests_wcoss_cray.log +++ REGRESSIONTEST_LOG=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/RegressionTests_wcoss_cray.log +++ export LOG_DIR=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/log_wcoss_cray +++ LOG_DIR=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/log_wcoss_cray + source default_vars.sh ++ '[' wcoss_cray = wcoss ']' ++ '[' wcoss_cray = wcoss_cray ']' @@ -816,15 +817,15 @@ Elapsed time 966 seconds. Compile 4 + TEST_NR=5 + export JBNME=compile_5 + JBNME=compile_5 -+ export RUNDIR=/gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39969/compile_5 -+ RUNDIR=/gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39969/compile_5 ++ export RUNDIR=/gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_47014/compile_5 ++ RUNDIR=/gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_47014/compile_5 + source rt_utils.sh ++ set -eu -++ [[ /gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/run_compile.sh = \.\/\r\t\_\u\t\i\l\s\.\s\h ]] +++ [[ /gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/run_compile.sh = \.\/\r\t\_\u\t\i\l\s\.\s\h ]] ++ UNIT_TEST=false + source atparse.bash -+ mkdir -p /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39969/compile_5 -+ cd /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39969/compile_5 ++ mkdir -p /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_47014/compile_5 ++ cd /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_47014/compile_5 + [[ lsf = \s\l\u\r\m ]] + [[ lsf = \l\s\f ]] + atparse @@ -840,7 +841,7 @@ Elapsed time 966 seconds. Compile 4 + set +x ESUB: OMP_NUM_THREADS value changed to 1 ESUB: MEMORY LIMIT rusage set to "rusage[mem=500]" - with rusage set in shared job. -Job id 714138 +Job id 750363 TEST 5 compile is waiting to enter the queue 1 min. TEST 5 compile is running, Status: RUN 2 min. TEST 5 compile is running, Status: RUN @@ -855,30 +856,29 @@ TEST 5 compile is waiting to enter the queue 11 min. TEST 5 compile is running, Status: RUN 12 min. TEST 5 compile is running, Status: RUN 13 min. TEST 5 compile is running, Status: RUN -14 min. TEST 5 compile is running, Status: RUN -15 min. TEST 5 compile is finished, Status: DONE -+ elapsed=906 -+ echo 'Elapsed time 906 seconds. Compile 5' -Elapsed time 906 seconds. Compile 5 +14 min. TEST 5 compile is finished, Status: DONE ++ elapsed=846 ++ echo 'Elapsed time 846 seconds. Compile 5' +Elapsed time 846 seconds. Compile 5 + SECONDS=0 + [[ 4 != 4 ]] -+ export PATHRT=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -+ PATHRT=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -+ export RUNDIR_ROOT=/gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39969 -+ RUNDIR_ROOT=/gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39969 ++ export PATHRT=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests ++ PATHRT=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests ++ export RUNDIR_ROOT=/gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_47014 ++ RUNDIR_ROOT=/gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_47014 + export 'MAKE_OPT=CCPP=Y SUITES=FV3_GFS_2017,FV3_GFS_2017_stretched 32BIT=Y DEBUG=Y' + MAKE_OPT='CCPP=Y SUITES=FV3_GFS_2017,FV3_GFS_2017_stretched 32BIT=Y DEBUG=Y' + export COMPILE_NR=6 + COMPILE_NR=6 -+ cd /gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -+ [[ -e /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39969/compile_6.env ]] -+ source /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39969/compile_6.env ++ cd /gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests ++ [[ -e /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_47014/compile_6.env ]] ++ source /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_47014/compile_6.env ++ export MACHINE_ID=wcoss_cray ++ MACHINE_ID=wcoss_cray -++ export PATHRT=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -++ PATHRT=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -++ export PATHTR=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr154/ufs-weather-model -++ PATHTR=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr154/ufs-weather-model +++ export PATHRT=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests +++ PATHRT=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests +++ export PATHTR=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model +++ PATHTR=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model ++ export SCHEDULER=lsf ++ SCHEDULER=lsf ++ export ACCNR=GFS-DEV @@ -891,10 +891,10 @@ Elapsed time 906 seconds. Compile 5 ++ ROCOTO=false ++ export ECFLOW=true ++ ECFLOW=true -++ export REGRESSIONTEST_LOG=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/RegressionTests_wcoss_cray.log -++ REGRESSIONTEST_LOG=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/RegressionTests_wcoss_cray.log -++ export LOG_DIR=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/log_wcoss_cray -++ LOG_DIR=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/log_wcoss_cray +++ export REGRESSIONTEST_LOG=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/RegressionTests_wcoss_cray.log +++ REGRESSIONTEST_LOG=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/RegressionTests_wcoss_cray.log +++ export LOG_DIR=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/log_wcoss_cray +++ LOG_DIR=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/log_wcoss_cray + source default_vars.sh ++ '[' wcoss_cray = wcoss ']' ++ '[' wcoss_cray = wcoss_cray ']' @@ -924,15 +924,15 @@ Elapsed time 906 seconds. Compile 5 + TEST_NR=6 + export JBNME=compile_6 + JBNME=compile_6 -+ export RUNDIR=/gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39969/compile_6 -+ RUNDIR=/gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39969/compile_6 ++ export RUNDIR=/gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_47014/compile_6 ++ RUNDIR=/gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_47014/compile_6 + source rt_utils.sh ++ set -eu -++ [[ /gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/run_compile.sh = \.\/\r\t\_\u\t\i\l\s\.\s\h ]] +++ [[ /gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/run_compile.sh = \.\/\r\t\_\u\t\i\l\s\.\s\h ]] ++ UNIT_TEST=false + source atparse.bash -+ mkdir -p /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39969/compile_6 -+ cd /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39969/compile_6 ++ mkdir -p /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_47014/compile_6 ++ cd /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_47014/compile_6 + [[ lsf = \s\l\u\r\m ]] + [[ lsf = \l\s\f ]] + atparse @@ -948,7 +948,7 @@ Elapsed time 906 seconds. Compile 5 + set +x ESUB: OMP_NUM_THREADS value changed to 1 ESUB: MEMORY LIMIT rusage set to "rusage[mem=500]" - with rusage set in shared job. -Job id 714141 +Job id 750368 TEST 6 compile is waiting to enter the queue 1 min. TEST 6 compile is running, Status: RUN 2 min. TEST 6 compile is running, Status: RUN @@ -958,30 +958,29 @@ TEST 6 compile is waiting to enter the queue 6 min. TEST 6 compile is running, Status: RUN 7 min. TEST 6 compile is running, Status: RUN 8 min. TEST 6 compile is running, Status: RUN -9 min. TEST 6 compile is running, Status: RUN -10 min. TEST 6 compile is finished, Status: DONE -+ elapsed=606 -+ echo 'Elapsed time 606 seconds. Compile 6' -Elapsed time 606 seconds. Compile 6 +9 min. TEST 6 compile is finished, Status: DONE ++ elapsed=546 ++ echo 'Elapsed time 546 seconds. Compile 6' +Elapsed time 546 seconds. Compile 6 + SECONDS=0 + [[ 4 != 4 ]] -+ export PATHRT=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -+ PATHRT=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -+ export RUNDIR_ROOT=/gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39969 -+ RUNDIR_ROOT=/gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39969 ++ export PATHRT=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests ++ PATHRT=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests ++ export RUNDIR_ROOT=/gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_47014 ++ RUNDIR_ROOT=/gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_47014 + export 'MAKE_OPT=CCPP=Y SUITES=FV3_GFS_2017_gfdlmp,FV3_GFS_2017_gfdlmp_noahmp' + MAKE_OPT='CCPP=Y SUITES=FV3_GFS_2017_gfdlmp,FV3_GFS_2017_gfdlmp_noahmp' + export COMPILE_NR=7 + COMPILE_NR=7 -+ cd /gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -+ [[ -e /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39969/compile_7.env ]] -+ source /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39969/compile_7.env ++ cd /gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests ++ [[ -e /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_47014/compile_7.env ]] ++ source /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_47014/compile_7.env ++ export MACHINE_ID=wcoss_cray ++ MACHINE_ID=wcoss_cray -++ export PATHRT=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -++ PATHRT=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -++ export PATHTR=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr154/ufs-weather-model -++ PATHTR=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr154/ufs-weather-model +++ export PATHRT=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests +++ PATHRT=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests +++ export PATHTR=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model +++ PATHTR=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model ++ export SCHEDULER=lsf ++ SCHEDULER=lsf ++ export ACCNR=GFS-DEV @@ -994,10 +993,10 @@ Elapsed time 606 seconds. Compile 6 ++ ROCOTO=false ++ export ECFLOW=true ++ ECFLOW=true -++ export REGRESSIONTEST_LOG=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/RegressionTests_wcoss_cray.log -++ REGRESSIONTEST_LOG=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/RegressionTests_wcoss_cray.log -++ export LOG_DIR=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/log_wcoss_cray -++ LOG_DIR=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/log_wcoss_cray +++ export REGRESSIONTEST_LOG=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/RegressionTests_wcoss_cray.log +++ REGRESSIONTEST_LOG=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/RegressionTests_wcoss_cray.log +++ export LOG_DIR=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/log_wcoss_cray +++ LOG_DIR=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/log_wcoss_cray + source default_vars.sh ++ '[' wcoss_cray = wcoss ']' ++ '[' wcoss_cray = wcoss_cray ']' @@ -1027,15 +1026,15 @@ Elapsed time 606 seconds. Compile 6 + TEST_NR=7 + export JBNME=compile_7 + JBNME=compile_7 -+ export RUNDIR=/gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39969/compile_7 -+ RUNDIR=/gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39969/compile_7 ++ export RUNDIR=/gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_47014/compile_7 ++ RUNDIR=/gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_47014/compile_7 + source rt_utils.sh ++ set -eu -++ [[ /gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/run_compile.sh = \.\/\r\t\_\u\t\i\l\s\.\s\h ]] +++ [[ /gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/run_compile.sh = \.\/\r\t\_\u\t\i\l\s\.\s\h ]] ++ UNIT_TEST=false + source atparse.bash -+ mkdir -p /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39969/compile_7 -+ cd /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39969/compile_7 ++ mkdir -p /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_47014/compile_7 ++ cd /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_47014/compile_7 + [[ lsf = \s\l\u\r\m ]] + [[ lsf = \l\s\f ]] + atparse @@ -1051,7 +1050,7 @@ Elapsed time 606 seconds. Compile 6 + set +x ESUB: OMP_NUM_THREADS value changed to 1 ESUB: MEMORY LIMIT rusage set to "rusage[mem=500]" - with rusage set in shared job. -Job id 714145 +Job id 750366 TEST 7 compile is waiting to enter the queue 1 min. TEST 7 compile is running, Status: RUN 2 min. TEST 7 compile is running, Status: RUN @@ -1066,30 +1065,29 @@ TEST 7 compile is waiting to enter the queue 11 min. TEST 7 compile is running, Status: RUN 12 min. TEST 7 compile is running, Status: RUN 13 min. TEST 7 compile is running, Status: RUN -14 min. TEST 7 compile is running, Status: RUN -15 min. TEST 7 compile is finished, Status: DONE -+ elapsed=906 -+ echo 'Elapsed time 906 seconds. Compile 7' -Elapsed time 906 seconds. Compile 7 +14 min. TEST 7 compile is finished, Status: DONE ++ elapsed=846 ++ echo 'Elapsed time 846 seconds. Compile 7' +Elapsed time 846 seconds. Compile 7 + SECONDS=0 + [[ 4 != 4 ]] -+ export PATHRT=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -+ PATHRT=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -+ export RUNDIR_ROOT=/gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39969 -+ RUNDIR_ROOT=/gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39969 ++ export PATHRT=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests ++ PATHRT=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests ++ export RUNDIR_ROOT=/gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_47014 ++ RUNDIR_ROOT=/gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_47014 + export 'MAKE_OPT=CCPP=Y SUITES=FV3_GFS_2017_csawmgshoc,FV3_GFS_2017_csawmg,FV3_GFS_2017_satmedmf,FV3_GFS_2017_satmedmfq' + MAKE_OPT='CCPP=Y SUITES=FV3_GFS_2017_csawmgshoc,FV3_GFS_2017_csawmg,FV3_GFS_2017_satmedmf,FV3_GFS_2017_satmedmfq' + export COMPILE_NR=8 + COMPILE_NR=8 -+ cd /gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -+ [[ -e /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39969/compile_8.env ]] -+ source /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39969/compile_8.env ++ cd /gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests ++ [[ -e /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_47014/compile_8.env ]] ++ source /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_47014/compile_8.env ++ export MACHINE_ID=wcoss_cray ++ MACHINE_ID=wcoss_cray -++ export PATHRT=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -++ PATHRT=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -++ export PATHTR=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr154/ufs-weather-model -++ PATHTR=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr154/ufs-weather-model +++ export PATHRT=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests +++ PATHRT=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests +++ export PATHTR=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model +++ PATHTR=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model ++ export SCHEDULER=lsf ++ SCHEDULER=lsf ++ export ACCNR=GFS-DEV @@ -1102,10 +1100,10 @@ Elapsed time 906 seconds. Compile 7 ++ ROCOTO=false ++ export ECFLOW=true ++ ECFLOW=true -++ export REGRESSIONTEST_LOG=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/RegressionTests_wcoss_cray.log -++ REGRESSIONTEST_LOG=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/RegressionTests_wcoss_cray.log -++ export LOG_DIR=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/log_wcoss_cray -++ LOG_DIR=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/log_wcoss_cray +++ export REGRESSIONTEST_LOG=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/RegressionTests_wcoss_cray.log +++ REGRESSIONTEST_LOG=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/RegressionTests_wcoss_cray.log +++ export LOG_DIR=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/log_wcoss_cray +++ LOG_DIR=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/log_wcoss_cray + source default_vars.sh ++ '[' wcoss_cray = wcoss ']' ++ '[' wcoss_cray = wcoss_cray ']' @@ -1135,15 +1133,15 @@ Elapsed time 906 seconds. Compile 7 + TEST_NR=8 + export JBNME=compile_8 + JBNME=compile_8 -+ export RUNDIR=/gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39969/compile_8 -+ RUNDIR=/gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39969/compile_8 ++ export RUNDIR=/gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_47014/compile_8 ++ RUNDIR=/gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_47014/compile_8 + source rt_utils.sh ++ set -eu -++ [[ /gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/run_compile.sh = \.\/\r\t\_\u\t\i\l\s\.\s\h ]] +++ [[ /gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/run_compile.sh = \.\/\r\t\_\u\t\i\l\s\.\s\h ]] ++ UNIT_TEST=false + source atparse.bash -+ mkdir -p /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39969/compile_8 -+ cd /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39969/compile_8 ++ mkdir -p /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_47014/compile_8 ++ cd /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_47014/compile_8 + [[ lsf = \s\l\u\r\m ]] + [[ lsf = \l\s\f ]] + atparse @@ -1159,7 +1157,7 @@ Elapsed time 906 seconds. Compile 7 + set +x ESUB: OMP_NUM_THREADS value changed to 1 ESUB: MEMORY LIMIT rusage set to "rusage[mem=500]" - with rusage set in shared job. -Job id 714142 +Job id 750367 TEST 8 compile is waiting to enter the queue 1 min. TEST 8 compile is running, Status: RUN 2 min. TEST 8 compile is running, Status: RUN @@ -1181,23 +1179,23 @@ TEST 8 compile is waiting to enter the queue Elapsed time 906 seconds. Compile 8 + SECONDS=0 + [[ 4 != 4 ]] -+ export PATHRT=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -+ PATHRT=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -+ export RUNDIR_ROOT=/gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39969 -+ RUNDIR_ROOT=/gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39969 ++ export PATHRT=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests ++ PATHRT=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests ++ export RUNDIR_ROOT=/gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_47014 ++ RUNDIR_ROOT=/gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_47014 + export 'MAKE_OPT=CCPP=Y SUITES=FV3_GFS_2017_gfdlmp,FV3_CPT_v0,FV3_GSD_v0,FV3_GFS_v15_thompson,FV3_RAP,FV3_HRRR 32BIT=Y' + MAKE_OPT='CCPP=Y SUITES=FV3_GFS_2017_gfdlmp,FV3_CPT_v0,FV3_GSD_v0,FV3_GFS_v15_thompson,FV3_RAP,FV3_HRRR 32BIT=Y' + export COMPILE_NR=9 + COMPILE_NR=9 -+ cd /gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -+ [[ -e /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39969/compile_9.env ]] -+ source /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39969/compile_9.env ++ cd /gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests ++ [[ -e /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_47014/compile_9.env ]] ++ source /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_47014/compile_9.env ++ export MACHINE_ID=wcoss_cray ++ MACHINE_ID=wcoss_cray -++ export PATHRT=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -++ PATHRT=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -++ export PATHTR=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr154/ufs-weather-model -++ PATHTR=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr154/ufs-weather-model +++ export PATHRT=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests +++ PATHRT=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests +++ export PATHTR=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model +++ PATHTR=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model ++ export SCHEDULER=lsf ++ SCHEDULER=lsf ++ export ACCNR=GFS-DEV @@ -1210,10 +1208,10 @@ Elapsed time 906 seconds. Compile 8 ++ ROCOTO=false ++ export ECFLOW=true ++ ECFLOW=true -++ export REGRESSIONTEST_LOG=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/RegressionTests_wcoss_cray.log -++ REGRESSIONTEST_LOG=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/RegressionTests_wcoss_cray.log -++ export LOG_DIR=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/log_wcoss_cray -++ LOG_DIR=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/log_wcoss_cray +++ export REGRESSIONTEST_LOG=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/RegressionTests_wcoss_cray.log +++ REGRESSIONTEST_LOG=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/RegressionTests_wcoss_cray.log +++ export LOG_DIR=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/log_wcoss_cray +++ LOG_DIR=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/log_wcoss_cray + source default_vars.sh ++ '[' wcoss_cray = wcoss ']' ++ '[' wcoss_cray = wcoss_cray ']' @@ -1243,15 +1241,15 @@ Elapsed time 906 seconds. Compile 8 + TEST_NR=9 + export JBNME=compile_9 + JBNME=compile_9 -+ export RUNDIR=/gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39969/compile_9 -+ RUNDIR=/gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39969/compile_9 ++ export RUNDIR=/gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_47014/compile_9 ++ RUNDIR=/gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_47014/compile_9 + source rt_utils.sh ++ set -eu -++ [[ /gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/run_compile.sh = \.\/\r\t\_\u\t\i\l\s\.\s\h ]] +++ [[ /gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/run_compile.sh = \.\/\r\t\_\u\t\i\l\s\.\s\h ]] ++ UNIT_TEST=false + source atparse.bash -+ mkdir -p /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39969/compile_9 -+ cd /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39969/compile_9 ++ mkdir -p /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_47014/compile_9 ++ cd /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_47014/compile_9 + [[ lsf = \s\l\u\r\m ]] + [[ lsf = \l\s\f ]] + atparse @@ -1267,7 +1265,7 @@ Elapsed time 906 seconds. Compile 8 + set +x ESUB: OMP_NUM_THREADS value changed to 1 ESUB: MEMORY LIMIT rusage set to "rusage[mem=500]" - with rusage set in shared job. -Job id 714143 +Job id 750365 TEST 9 compile is waiting to enter the queue 1 min. TEST 9 compile is running, Status: RUN 2 min. TEST 9 compile is running, Status: RUN @@ -1283,8 +1281,7 @@ TEST 9 compile is waiting to enter the queue 12 min. TEST 9 compile is running, Status: RUN 13 min. TEST 9 compile is running, Status: RUN 14 min. TEST 9 compile is running, Status: RUN -15 min. TEST 9 compile is running, Status: RUN -16 min. TEST 9 compile is finished, Status: DONE -+ elapsed=966 -+ echo 'Elapsed time 966 seconds. Compile 9' -Elapsed time 966 seconds. Compile 9 +15 min. TEST 9 compile is finished, Status: DONE ++ elapsed=906 ++ echo 'Elapsed time 906 seconds. Compile 9' +Elapsed time 906 seconds. Compile 9 diff --git a/tests/RegressionTests_wcoss_cray.log b/tests/RegressionTests_wcoss_cray.log index c224ec3044..2fa52d89ec 100644 --- a/tests/RegressionTests_wcoss_cray.log +++ b/tests/RegressionTests_wcoss_cray.log @@ -1,9 +1,9 @@ -Tue Jul 14 01:13:00 UTC 2020 +Wed Jul 15 14:08:49 UTC 2020 Start Regression test baseline dir = /gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200713/fv3_control_ccpp -working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39969/fv3_ccpp_control_prod +working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_47014/fv3_ccpp_control_prod Checking test 001 fv3_ccpp_control results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -71,7 +71,7 @@ Test 001 fv3_ccpp_control PASS baseline dir = /gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200713/fv3_control_ccpp -working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39969/fv3_ccpp_decomp_prod +working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_47014/fv3_ccpp_decomp_prod Checking test 002 fv3_ccpp_decomp results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -139,7 +139,7 @@ Test 002 fv3_ccpp_decomp PASS baseline dir = /gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200713/fv3_control_ccpp -working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39969/fv3_ccpp_2threads_prod +working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_47014/fv3_ccpp_2threads_prod Checking test 003 fv3_ccpp_2threads results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -207,7 +207,7 @@ Test 003 fv3_ccpp_2threads PASS baseline dir = /gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200713/fv3_restart_ccpp -working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39969/fv3_ccpp_restart_prod +working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_47014/fv3_ccpp_restart_prod Checking test 004 fv3_ccpp_restart results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -275,7 +275,7 @@ Test 004 fv3_ccpp_restart PASS baseline dir = /gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200713/fv3_read_inc_ccpp -working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39969/fv3_ccpp_read_inc_prod +working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_47014/fv3_ccpp_read_inc_prod Checking test 005 fv3_ccpp_read_inc results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -343,7 +343,7 @@ Test 005 fv3_ccpp_read_inc PASS baseline dir = /gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200713/fv3_wrtGauss_netcdf_esmf_ccpp -working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39969/fv3_ccpp_wrtGauss_netcdf_esmf_prod +working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_47014/fv3_ccpp_wrtGauss_netcdf_esmf_prod Checking test 006 fv3_ccpp_wrtGauss_netcdf_esmf results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -391,7 +391,7 @@ Test 006 fv3_ccpp_wrtGauss_netcdf_esmf PASS baseline dir = /gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200713/fv3_wrtGauss_netcdf_ccpp -working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39969/fv3_ccpp_wrtGauss_netcdf_prod +working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_47014/fv3_ccpp_wrtGauss_netcdf_prod Checking test 007 fv3_ccpp_wrtGauss_netcdf results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -439,7 +439,7 @@ Test 007 fv3_ccpp_wrtGauss_netcdf PASS baseline dir = /gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200713/fv3_wrtGlatlon_netcdf_ccpp -working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39969/fv3_ccpp_wrtGlatlon_netcdf_prod +working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_47014/fv3_ccpp_wrtGlatlon_netcdf_prod Checking test 008 fv3_ccpp_wrtGlatlon_netcdf results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -487,7 +487,7 @@ Test 008 fv3_ccpp_wrtGlatlon_netcdf PASS baseline dir = /gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200713/fv3_wrtGauss_nemsio_ccpp -working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39969/fv3_ccpp_wrtGauss_nemsio_prod +working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_47014/fv3_ccpp_wrtGauss_nemsio_prod Checking test 009 fv3_ccpp_wrtGauss_nemsio results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -535,7 +535,7 @@ Test 009 fv3_ccpp_wrtGauss_nemsio PASS baseline dir = /gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200713/fv3_wrtGauss_nemsio_c192_ccpp -working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39969/fv3_ccpp_wrtGauss_nemsio_c192_prod +working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_47014/fv3_ccpp_wrtGauss_nemsio_c192_prod Checking test 010 fv3_ccpp_wrtGauss_nemsio_c192 results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -583,7 +583,7 @@ Test 010 fv3_ccpp_wrtGauss_nemsio_c192 PASS baseline dir = /gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200713/fv3_stochy_ccpp -working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39969/fv3_ccpp_stochy_prod +working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_47014/fv3_ccpp_stochy_prod Checking test 011 fv3_ccpp_stochy results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -651,7 +651,7 @@ Test 011 fv3_ccpp_stochy PASS baseline dir = /gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200713/fv3_iau_ccpp -working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39969/fv3_ccpp_iau_prod +working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_47014/fv3_ccpp_iau_prod Checking test 012 fv3_ccpp_iau results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -719,7 +719,7 @@ Test 012 fv3_ccpp_iau PASS baseline dir = /gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200713/fv3_lheatstrg_ccpp -working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39969/fv3_ccpp_lheatstrg_prod +working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_47014/fv3_ccpp_lheatstrg_prod Checking test 013 fv3_ccpp_lheatstrg results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -767,7 +767,7 @@ Test 013 fv3_ccpp_lheatstrg PASS baseline dir = /gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200713/fv3_gfdlmprad_ccpp -working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39969/fv3_ccpp_gfdlmprad_prod +working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_47014/fv3_ccpp_gfdlmprad_prod Checking test 014 fv3_ccpp_gfdlmprad results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -816,7 +816,7 @@ Test 014 fv3_ccpp_gfdlmprad PASS baseline dir = /gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200713/fv3_gfdlmprad_atmwav_ccpp -working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39969/fv3_ccpp_gfdlmprad_atmwav_prod +working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_47014/fv3_ccpp_gfdlmprad_atmwav_prod Checking test 015 fv3_ccpp_gfdlmprad_atmwav results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -865,7 +865,7 @@ Test 015 fv3_ccpp_gfdlmprad_atmwav PASS baseline dir = /gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200713/fv3_multigases_ccpp -working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39969/fv3_ccpp_multigases_prod +working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_47014/fv3_ccpp_multigases_prod Checking test 016 fv3_ccpp_multigases results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -939,7 +939,7 @@ Test 016 fv3_ccpp_multigases PASS baseline dir = /gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200713/fv3_control_32bit_ccpp -working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39969/fv3_ccpp_control_32bit_prod +working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_47014/fv3_ccpp_control_32bit_prod Checking test 017 fv3_ccpp_control_32bit results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1007,7 +1007,7 @@ Test 017 fv3_ccpp_control_32bit PASS baseline dir = /gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200713/fv3_stretched_ccpp -working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39969/fv3_ccpp_stretched_prod +working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_47014/fv3_ccpp_stretched_prod Checking test 018 fv3_ccpp_stretched results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1063,7 +1063,7 @@ Test 018 fv3_ccpp_stretched PASS baseline dir = /gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200713/fv3_stretched_nest_ccpp -working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39969/fv3_ccpp_stretched_nest_prod +working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_47014/fv3_ccpp_stretched_nest_prod Checking test 019 fv3_ccpp_stretched_nest results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1130,7 +1130,7 @@ Test 019 fv3_ccpp_stretched_nest PASS baseline dir = /gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200713/fv3_regional_control_ccpp -working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39969/fv3_ccpp_regional_control_prod +working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_47014/fv3_ccpp_regional_control_prod Checking test 020 fv3_ccpp_regional_control results .... Comparing atmos_4xdaily.nc .........OK Comparing fv3_history2d.nc .........OK @@ -1141,7 +1141,7 @@ Test 020 fv3_ccpp_regional_control PASS baseline dir = /gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200713/fv3_regional_restart_ccpp -working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39969/fv3_ccpp_regional_restart_prod +working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_47014/fv3_ccpp_regional_restart_prod Checking test 021 fv3_ccpp_regional_restart results .... Comparing atmos_4xdaily.nc .........OK Comparing fv3_history2d.nc .........OK @@ -1150,7 +1150,7 @@ Test 021 fv3_ccpp_regional_restart PASS baseline dir = /gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200713/fv3_regional_quilt_ccpp -working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39969/fv3_ccpp_regional_quilt_prod +working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_47014/fv3_ccpp_regional_quilt_prod Checking test 022 fv3_ccpp_regional_quilt results .... Comparing atmos_4xdaily.nc .........OK Comparing dynf000.nc .........OK @@ -1161,19 +1161,19 @@ Test 022 fv3_ccpp_regional_quilt PASS baseline dir = /gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200713/fv3_control_ccpp -working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39969/fv3_ccpp_control_debug_prod +working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_47014/fv3_ccpp_control_debug_prod Checking test 023 fv3_ccpp_control_debug results .... Test 023 fv3_ccpp_control_debug PASS baseline dir = /gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200713/fv3_stretched_nest_ccpp -working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39969/fv3_ccpp_stretched_nest_debug_prod +working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_47014/fv3_ccpp_stretched_nest_debug_prod Checking test 024 fv3_ccpp_stretched_nest_debug results .... Test 024 fv3_ccpp_stretched_nest_debug PASS baseline dir = /gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200713/fv3_gfdlmp_ccpp -working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39969/fv3_ccpp_gfdlmp_prod +working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_47014/fv3_ccpp_gfdlmp_prod Checking test 025 fv3_ccpp_gfdlmp results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1221,7 +1221,7 @@ Test 025 fv3_ccpp_gfdlmp PASS baseline dir = /gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200713/fv3_gfdlmprad_gwd_ccpp -working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39969/fv3_ccpp_gfdlmprad_gwd_prod +working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_47014/fv3_ccpp_gfdlmprad_gwd_prod Checking test 026 fv3_ccpp_gfdlmprad_gwd results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1269,7 +1269,7 @@ Test 026 fv3_ccpp_gfdlmprad_gwd PASS baseline dir = /gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200713/fv3_gfdlmprad_noahmp_ccpp -working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39969/fv3_ccpp_gfdlmprad_noahmp_prod +working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_47014/fv3_ccpp_gfdlmprad_noahmp_prod Checking test 027 fv3_ccpp_gfdlmprad_noahmp results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1317,7 +1317,7 @@ Test 027 fv3_ccpp_gfdlmprad_noahmp PASS baseline dir = /gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200713/fv3_gfdlmp_hwrfsas_ccpp -working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39969/fv3_ccpp_gfdlmp_hwrfsas_prod +working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_47014/fv3_ccpp_gfdlmp_hwrfsas_prod Checking test 028 fv3_ccpp_gfdlmp_hwrfsas results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1365,7 +1365,7 @@ Test 028 fv3_ccpp_gfdlmp_hwrfsas PASS baseline dir = /gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200713/fv3_csawmg_ccpp -working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39969/fv3_ccpp_csawmg_prod +working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_47014/fv3_ccpp_csawmg_prod Checking test 029 fv3_ccpp_csawmg results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1413,7 +1413,7 @@ Test 029 fv3_ccpp_csawmg PASS baseline dir = /gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200713/fv3_satmedmf_ccpp -working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39969/fv3_ccpp_satmedmf_prod +working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_47014/fv3_ccpp_satmedmf_prod Checking test 030 fv3_ccpp_satmedmf results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1461,7 +1461,7 @@ Test 030 fv3_ccpp_satmedmf PASS baseline dir = /gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200713/fv3_satmedmfq_ccpp -working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39969/fv3_ccpp_satmedmfq_prod +working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_47014/fv3_ccpp_satmedmfq_prod Checking test 031 fv3_ccpp_satmedmfq results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1509,7 +1509,7 @@ Test 031 fv3_ccpp_satmedmfq PASS baseline dir = /gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200713/fv3_gfdlmp_32bit_ccpp -working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39969/fv3_ccpp_gfdlmp_32bit_prod +working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_47014/fv3_ccpp_gfdlmp_32bit_prod Checking test 032 fv3_ccpp_gfdlmp_32bit results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1557,7 +1557,7 @@ Test 032 fv3_ccpp_gfdlmp_32bit PASS baseline dir = /gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200713/fv3_gfdlmprad_32bit_post_ccpp -working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39969/fv3_ccpp_gfdlmprad_32bit_post_prod +working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_47014/fv3_ccpp_gfdlmprad_32bit_post_prod Checking test 033 fv3_ccpp_gfdlmprad_32bit_post results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1609,7 +1609,7 @@ Test 033 fv3_ccpp_gfdlmprad_32bit_post PASS baseline dir = /gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200713/fv3_cpt_ccpp -working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39969/fv3_ccpp_cpt_prod +working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_47014/fv3_ccpp_cpt_prod Checking test 034 fv3_ccpp_cpt results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1663,7 +1663,7 @@ Test 034 fv3_ccpp_cpt PASS baseline dir = /gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200713/fv3_gsd_ccpp -working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39969/fv3_ccpp_gsd_prod +working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_47014/fv3_ccpp_gsd_prod Checking test 035 fv3_ccpp_gsd results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1755,7 +1755,7 @@ Test 035 fv3_ccpp_gsd PASS baseline dir = /gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200713/fv3_thompson_ccpp -working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39969/fv3_ccpp_thompson_prod +working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_47014/fv3_ccpp_thompson_prod Checking test 036 fv3_ccpp_thompson results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1823,7 +1823,7 @@ Test 036 fv3_ccpp_thompson PASS baseline dir = /gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200713/fv3_thompson_no_aero_ccpp -working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39969/fv3_ccpp_thompson_no_aero_prod +working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_47014/fv3_ccpp_thompson_no_aero_prod Checking test 037 fv3_ccpp_thompson_no_aero results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1891,7 +1891,7 @@ Test 037 fv3_ccpp_thompson_no_aero PASS baseline dir = /gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200713/fv3_gsd_debug_ccpp -working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39969/fv3_ccpp_gsd_debug_prod +working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_47014/fv3_ccpp_gsd_debug_prod Checking test 038 fv3_ccpp_gsd_debug results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1959,7 +1959,7 @@ Test 038 fv3_ccpp_gsd_debug PASS baseline dir = /gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200713/fv3_gsd_diag3d_debug_ccpp -working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39969/fv3_ccpp_gsd_diag3d_debug_prod +working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_47014/fv3_ccpp_gsd_diag3d_debug_prod Checking test 039 fv3_ccpp_gsd_diag3d_debug results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -2027,7 +2027,7 @@ Test 039 fv3_ccpp_gsd_diag3d_debug PASS baseline dir = /gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200713/fv3_thompson_debug_ccpp -working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39969/fv3_ccpp_thompson_debug_prod +working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_47014/fv3_ccpp_thompson_debug_prod Checking test 040 fv3_ccpp_thompson_debug results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -2095,7 +2095,7 @@ Test 040 fv3_ccpp_thompson_debug PASS baseline dir = /gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200713/fv3_thompson_no_aero_debug_ccpp -working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39969/fv3_ccpp_thompson_no_aero_debug_prod +working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_47014/fv3_ccpp_thompson_no_aero_debug_prod Checking test 041 fv3_ccpp_thompson_no_aero_debug results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -2163,7 +2163,7 @@ Test 041 fv3_ccpp_thompson_no_aero_debug PASS baseline dir = /gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200713/fv3_gfsv16_csawmg_ccpp -working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39969/fv3_ccpp_gfsv16_csawmg_prod +working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_47014/fv3_ccpp_gfsv16_csawmg_prod Checking test 042 fv3_ccpp_gfsv16_csawmg results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -2211,7 +2211,7 @@ Test 042 fv3_ccpp_gfsv16_csawmg PASS baseline dir = /gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200713/fv3_gfsv16_csawmgt_ccpp -working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39969/fv3_ccpp_gfsv16_csawmgt_prod +working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_47014/fv3_ccpp_gfsv16_csawmgt_prod Checking test 043 fv3_ccpp_gfsv16_csawmgt results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -2259,7 +2259,7 @@ Test 043 fv3_ccpp_gfsv16_csawmgt PASS baseline dir = /gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200713/fv3_gocart_clm_ccpp -working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39969/fv3_ccpp_gocart_clm_prod +working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_47014/fv3_ccpp_gocart_clm_prod Checking test 044 fv3_ccpp_gocart_clm results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -2307,5 +2307,5 @@ Test 044 fv3_ccpp_gocart_clm PASS REGRESSION TEST WAS SUCCESSFUL -Tue Jul 14 02:00:12 UTC 2020 -Elapsed time: 00h:47m:14s. Have a nice day! +Wed Jul 15 14:52:04 UTC 2020 +Elapsed time: 00h:43m:15s. Have a nice day! From 5f8db589bc2796e7289046a17e90bfa62bd000c8 Mon Sep 17 00:00:00 2001 From: Dusan Jovic Date: Wed, 15 Jul 2020 09:56:54 -0500 Subject: [PATCH 09/11] Add regtest logs on Orion --- tests/Compile_orion.intel.log | 1034 +++++++++++-------------- tests/RegressionTests_orion.intel.log | 102 +-- 2 files changed, 499 insertions(+), 637 deletions(-) diff --git a/tests/Compile_orion.intel.log b/tests/Compile_orion.intel.log index e2eaffd668..5bac650bc8 100644 --- a/tests/Compile_orion.intel.log +++ b/tests/Compile_orion.intel.log @@ -1,22 +1,22 @@ + SECONDS=0 + [[ 4 != 4 ]] -+ export PATHRT=/work/noaa/fv3-cam/djovic/ufs/pr154/ufs-weather-model/tests -+ PATHRT=/work/noaa/fv3-cam/djovic/ufs/pr154/ufs-weather-model/tests -+ export RUNDIR_ROOT=/work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_369221 -+ RUNDIR_ROOT=/work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_369221 ++ export PATHRT=/work/noaa/fv3-cam/djovic/ufs/rt_ecf_suite/ufs-weather-model/tests ++ PATHRT=/work/noaa/fv3-cam/djovic/ufs/rt_ecf_suite/ufs-weather-model/tests ++ export RUNDIR_ROOT=/work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_339981 ++ RUNDIR_ROOT=/work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_339981 + export 'MAKE_OPT=CCPP=Y SUITES=FV3_GFS_v15p2,FV3_GFS_v16beta,FV3_GFS_2017_RRTMGP DEBUG=Y' + MAKE_OPT='CCPP=Y SUITES=FV3_GFS_v15p2,FV3_GFS_v16beta,FV3_GFS_2017_RRTMGP DEBUG=Y' + export COMPILE_NR=10 + COMPILE_NR=10 -+ cd /work/noaa/fv3-cam/djovic/ufs/pr154/ufs-weather-model/tests -+ [[ -e /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_369221/compile_10.env ]] -+ source /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_369221/compile_10.env ++ cd /work/noaa/fv3-cam/djovic/ufs/rt_ecf_suite/ufs-weather-model/tests ++ [[ -e /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_339981/compile_10.env ]] ++ source /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_339981/compile_10.env ++ export MACHINE_ID=orion.intel ++ MACHINE_ID=orion.intel -++ export PATHRT=/work/noaa/fv3-cam/djovic/ufs/pr154/ufs-weather-model/tests -++ PATHRT=/work/noaa/fv3-cam/djovic/ufs/pr154/ufs-weather-model/tests -++ export PATHTR=/work/noaa/fv3-cam/djovic/ufs/pr154/ufs-weather-model -++ PATHTR=/work/noaa/fv3-cam/djovic/ufs/pr154/ufs-weather-model +++ export PATHRT=/work/noaa/fv3-cam/djovic/ufs/rt_ecf_suite/ufs-weather-model/tests +++ PATHRT=/work/noaa/fv3-cam/djovic/ufs/rt_ecf_suite/ufs-weather-model/tests +++ export PATHTR=/work/noaa/fv3-cam/djovic/ufs/rt_ecf_suite/ufs-weather-model +++ PATHTR=/work/noaa/fv3-cam/djovic/ufs/rt_ecf_suite/ufs-weather-model ++ export SCHEDULER=slurm ++ SCHEDULER=slurm ++ export ACCNR=nems @@ -29,10 +29,10 @@ ++ ROCOTO=false ++ export ECFLOW=true ++ ECFLOW=true -++ export REGRESSIONTEST_LOG=/work/noaa/fv3-cam/djovic/ufs/pr154/ufs-weather-model/tests/RegressionTests_orion.intel.log -++ REGRESSIONTEST_LOG=/work/noaa/fv3-cam/djovic/ufs/pr154/ufs-weather-model/tests/RegressionTests_orion.intel.log -++ export LOG_DIR=/work/noaa/fv3-cam/djovic/ufs/pr154/ufs-weather-model/tests/log_orion.intel -++ LOG_DIR=/work/noaa/fv3-cam/djovic/ufs/pr154/ufs-weather-model/tests/log_orion.intel +++ export REGRESSIONTEST_LOG=/work/noaa/fv3-cam/djovic/ufs/rt_ecf_suite/ufs-weather-model/tests/RegressionTests_orion.intel.log +++ REGRESSIONTEST_LOG=/work/noaa/fv3-cam/djovic/ufs/rt_ecf_suite/ufs-weather-model/tests/RegressionTests_orion.intel.log +++ export LOG_DIR=/work/noaa/fv3-cam/djovic/ufs/rt_ecf_suite/ufs-weather-model/tests/log_orion.intel +++ LOG_DIR=/work/noaa/fv3-cam/djovic/ufs/rt_ecf_suite/ufs-weather-model/tests/log_orion.intel + source default_vars.sh ++ '[' orion.intel = wcoss ']' ++ '[' orion.intel = wcoss_cray ']' @@ -64,15 +64,15 @@ + TEST_NR=10 + export JBNME=compile_10 + JBNME=compile_10 -+ export RUNDIR=/work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_369221/compile_10 -+ RUNDIR=/work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_369221/compile_10 ++ export RUNDIR=/work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_339981/compile_10 ++ RUNDIR=/work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_339981/compile_10 + source rt_utils.sh ++ set -eu -++ [[ /work/noaa/fv3-cam/djovic/ufs/pr154/ufs-weather-model/tests/run_compile.sh = \r\t\_\u\t\i\l\s\.\s\h ]] +++ [[ /work/noaa/fv3-cam/djovic/ufs/rt_ecf_suite/ufs-weather-model/tests/run_compile.sh = \r\t\_\u\t\i\l\s\.\s\h ]] ++ UNIT_TEST=false + source atparse.bash -+ mkdir -p /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_369221/compile_10 -+ cd /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_369221/compile_10 ++ mkdir -p /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_339981/compile_10 ++ cd /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_339981/compile_10 + [[ slurm = \s\l\u\r\m ]] + atparse + local __set_x @@ -85,55 +85,42 @@ + '[' -o xtrace ']' + set_x='set -x' + set +x -Job id 174426 +Job id 177590 TEST 10 compile is waiting to enter the queue -1 min. TEST 10 compile is pending, Status: PD -2 min. TEST 10 compile is pending, Status: PD -3 min. TEST 10 compile is pending, Status: PD -4 min. TEST 10 compile is pending, Status: PD -5 min. TEST 10 compile is pending, Status: PD -6 min. TEST 10 compile is pending, Status: PD -7 min. TEST 10 compile is pending, Status: PD -8 min. TEST 10 compile is pending, Status: PD -9 min. TEST 10 compile is pending, Status: PD -10 min. TEST 10 compile is pending, Status: PD -11 min. TEST 10 compile is pending, Status: PD -12 min. TEST 10 compile is pending, Status: PD -13 min. TEST 10 compile is pending, Status: PD -14 min. TEST 10 compile is pending, Status: PD -15 min. TEST 10 compile is pending, Status: PD -16 min. TEST 10 compile is pending, Status: PD -17 min. TEST 10 compile is running, Status: R -18 min. TEST 10 compile is running, Status: R +1 min. TEST 10 compile is running, Status: R +2 min. TEST 10 compile is running, Status: R +3 min. TEST 10 compile is running, Status: R +4 min. TEST 10 compile is running, Status: R +5 min. TEST 10 compile is running, Status: R Slurm unknown status -. Check sacct ... -174426 COMPLETED compile_10 -174426.batch COMPLETED batch -174426.exte+ COMPLETED extern -19 min. TEST 10 compile is COMPLETED +177590 COMPLETED compile_10 +177590.batch COMPLETED batch +177590.exte+ COMPLETED extern +6 min. TEST 10 compile is COMPLETED COMPLETED COMPLETED -+ elapsed=1148 -+ echo 'Elapsed time 1148 seconds. Compile 10' -Elapsed time 1148 seconds. Compile 10 ++ elapsed=366 ++ echo 'Elapsed time 366 seconds. Compile 10' +Elapsed time 366 seconds. Compile 10 + SECONDS=0 + [[ 4 != 4 ]] -+ export PATHRT=/work/noaa/fv3-cam/djovic/ufs/pr154/ufs-weather-model/tests -+ PATHRT=/work/noaa/fv3-cam/djovic/ufs/pr154/ufs-weather-model/tests -+ export RUNDIR_ROOT=/work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_369221 -+ RUNDIR_ROOT=/work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_369221 ++ export PATHRT=/work/noaa/fv3-cam/djovic/ufs/rt_ecf_suite/ufs-weather-model/tests ++ PATHRT=/work/noaa/fv3-cam/djovic/ufs/rt_ecf_suite/ufs-weather-model/tests ++ export RUNDIR_ROOT=/work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_339981 ++ RUNDIR_ROOT=/work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_339981 + export 'MAKE_OPT=CCPP=Y SUITES=FV3_GSD_v0,FV3_GFS_v15_thompson 32BIT=Y DEBUG=Y' + MAKE_OPT='CCPP=Y SUITES=FV3_GSD_v0,FV3_GFS_v15_thompson 32BIT=Y DEBUG=Y' + export COMPILE_NR=11 + COMPILE_NR=11 -+ cd /work/noaa/fv3-cam/djovic/ufs/pr154/ufs-weather-model/tests -+ [[ -e /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_369221/compile_11.env ]] -+ source /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_369221/compile_11.env ++ cd /work/noaa/fv3-cam/djovic/ufs/rt_ecf_suite/ufs-weather-model/tests ++ [[ -e /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_339981/compile_11.env ]] ++ source /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_339981/compile_11.env ++ export MACHINE_ID=orion.intel ++ MACHINE_ID=orion.intel -++ export PATHRT=/work/noaa/fv3-cam/djovic/ufs/pr154/ufs-weather-model/tests -++ PATHRT=/work/noaa/fv3-cam/djovic/ufs/pr154/ufs-weather-model/tests -++ export PATHTR=/work/noaa/fv3-cam/djovic/ufs/pr154/ufs-weather-model -++ PATHTR=/work/noaa/fv3-cam/djovic/ufs/pr154/ufs-weather-model +++ export PATHRT=/work/noaa/fv3-cam/djovic/ufs/rt_ecf_suite/ufs-weather-model/tests +++ PATHRT=/work/noaa/fv3-cam/djovic/ufs/rt_ecf_suite/ufs-weather-model/tests +++ export PATHTR=/work/noaa/fv3-cam/djovic/ufs/rt_ecf_suite/ufs-weather-model +++ PATHTR=/work/noaa/fv3-cam/djovic/ufs/rt_ecf_suite/ufs-weather-model ++ export SCHEDULER=slurm ++ SCHEDULER=slurm ++ export ACCNR=nems @@ -146,10 +133,10 @@ Elapsed time 1148 seconds. Compile 10 ++ ROCOTO=false ++ export ECFLOW=true ++ ECFLOW=true -++ export REGRESSIONTEST_LOG=/work/noaa/fv3-cam/djovic/ufs/pr154/ufs-weather-model/tests/RegressionTests_orion.intel.log -++ REGRESSIONTEST_LOG=/work/noaa/fv3-cam/djovic/ufs/pr154/ufs-weather-model/tests/RegressionTests_orion.intel.log -++ export LOG_DIR=/work/noaa/fv3-cam/djovic/ufs/pr154/ufs-weather-model/tests/log_orion.intel -++ LOG_DIR=/work/noaa/fv3-cam/djovic/ufs/pr154/ufs-weather-model/tests/log_orion.intel +++ export REGRESSIONTEST_LOG=/work/noaa/fv3-cam/djovic/ufs/rt_ecf_suite/ufs-weather-model/tests/RegressionTests_orion.intel.log +++ REGRESSIONTEST_LOG=/work/noaa/fv3-cam/djovic/ufs/rt_ecf_suite/ufs-weather-model/tests/RegressionTests_orion.intel.log +++ export LOG_DIR=/work/noaa/fv3-cam/djovic/ufs/rt_ecf_suite/ufs-weather-model/tests/log_orion.intel +++ LOG_DIR=/work/noaa/fv3-cam/djovic/ufs/rt_ecf_suite/ufs-weather-model/tests/log_orion.intel + source default_vars.sh ++ '[' orion.intel = wcoss ']' ++ '[' orion.intel = wcoss_cray ']' @@ -181,15 +168,15 @@ Elapsed time 1148 seconds. Compile 10 + TEST_NR=11 + export JBNME=compile_11 + JBNME=compile_11 -+ export RUNDIR=/work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_369221/compile_11 -+ RUNDIR=/work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_369221/compile_11 ++ export RUNDIR=/work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_339981/compile_11 ++ RUNDIR=/work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_339981/compile_11 + source rt_utils.sh ++ set -eu -++ [[ /work/noaa/fv3-cam/djovic/ufs/pr154/ufs-weather-model/tests/run_compile.sh = \r\t\_\u\t\i\l\s\.\s\h ]] +++ [[ /work/noaa/fv3-cam/djovic/ufs/rt_ecf_suite/ufs-weather-model/tests/run_compile.sh = \r\t\_\u\t\i\l\s\.\s\h ]] ++ UNIT_TEST=false + source atparse.bash -+ mkdir -p /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_369221/compile_11 -+ cd /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_369221/compile_11 ++ mkdir -p /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_339981/compile_11 ++ cd /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_339981/compile_11 + [[ slurm = \s\l\u\r\m ]] + atparse + local __set_x @@ -202,51 +189,41 @@ Elapsed time 1148 seconds. Compile 10 + '[' -o xtrace ']' + set_x='set -x' + set +x -Job id 174439 +Job id 177594 TEST 11 compile is waiting to enter the queue -1 min. TEST 11 compile is pending, Status: PD -2 min. TEST 11 compile is pending, Status: PD -3 min. TEST 11 compile is pending, Status: PD -4 min. TEST 11 compile is pending, Status: PD -5 min. TEST 11 compile is pending, Status: PD -6 min. TEST 11 compile is pending, Status: PD -7 min. TEST 11 compile is pending, Status: PD -8 min. TEST 11 compile is pending, Status: PD -9 min. TEST 11 compile is pending, Status: PD -10 min. TEST 11 compile is pending, Status: PD -11 min. TEST 11 compile is pending, Status: PD -12 min. TEST 11 compile is running, Status: R -13 min. TEST 11 compile is running, Status: R -14 min. TEST 11 compile is running, Status: R +1 min. TEST 11 compile is running, Status: R +2 min. TEST 11 compile is running, Status: R +3 min. TEST 11 compile is running, Status: R +4 min. TEST 11 compile is running, Status: R Slurm unknown status -. Check sacct ... -174439 COMPLETED compile_11 -174439.batch COMPLETED batch -174439.exte+ COMPLETED extern -15 min. TEST 11 compile is COMPLETED +177594 COMPLETED compile_11 +177594.batch COMPLETED batch +177594.exte+ COMPLETED extern +5 min. TEST 11 compile is COMPLETED COMPLETED COMPLETED -+ elapsed=907 -+ echo 'Elapsed time 907 seconds. Compile 11' -Elapsed time 907 seconds. Compile 11 ++ elapsed=306 ++ echo 'Elapsed time 306 seconds. Compile 11' +Elapsed time 306 seconds. Compile 11 + SECONDS=0 + [[ 4 != 4 ]] -+ export PATHRT=/work/noaa/fv3-cam/djovic/ufs/pr154/ufs-weather-model/tests -+ PATHRT=/work/noaa/fv3-cam/djovic/ufs/pr154/ufs-weather-model/tests -+ export RUNDIR_ROOT=/work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_369221 -+ RUNDIR_ROOT=/work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_369221 ++ export PATHRT=/work/noaa/fv3-cam/djovic/ufs/rt_ecf_suite/ufs-weather-model/tests ++ PATHRT=/work/noaa/fv3-cam/djovic/ufs/rt_ecf_suite/ufs-weather-model/tests ++ export RUNDIR_ROOT=/work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_339981 ++ RUNDIR_ROOT=/work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_339981 + export 'MAKE_OPT=CCPP=Y SUITES=FV3_GFS_v16_csawmg' + MAKE_OPT='CCPP=Y SUITES=FV3_GFS_v16_csawmg' + export COMPILE_NR=12 + COMPILE_NR=12 -+ cd /work/noaa/fv3-cam/djovic/ufs/pr154/ufs-weather-model/tests -+ [[ -e /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_369221/compile_12.env ]] -+ source /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_369221/compile_12.env ++ cd /work/noaa/fv3-cam/djovic/ufs/rt_ecf_suite/ufs-weather-model/tests ++ [[ -e /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_339981/compile_12.env ]] ++ source /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_339981/compile_12.env ++ export MACHINE_ID=orion.intel ++ MACHINE_ID=orion.intel -++ export PATHRT=/work/noaa/fv3-cam/djovic/ufs/pr154/ufs-weather-model/tests -++ PATHRT=/work/noaa/fv3-cam/djovic/ufs/pr154/ufs-weather-model/tests -++ export PATHTR=/work/noaa/fv3-cam/djovic/ufs/pr154/ufs-weather-model -++ PATHTR=/work/noaa/fv3-cam/djovic/ufs/pr154/ufs-weather-model +++ export PATHRT=/work/noaa/fv3-cam/djovic/ufs/rt_ecf_suite/ufs-weather-model/tests +++ PATHRT=/work/noaa/fv3-cam/djovic/ufs/rt_ecf_suite/ufs-weather-model/tests +++ export PATHTR=/work/noaa/fv3-cam/djovic/ufs/rt_ecf_suite/ufs-weather-model +++ PATHTR=/work/noaa/fv3-cam/djovic/ufs/rt_ecf_suite/ufs-weather-model ++ export SCHEDULER=slurm ++ SCHEDULER=slurm ++ export ACCNR=nems @@ -259,10 +236,10 @@ Elapsed time 907 seconds. Compile 11 ++ ROCOTO=false ++ export ECFLOW=true ++ ECFLOW=true -++ export REGRESSIONTEST_LOG=/work/noaa/fv3-cam/djovic/ufs/pr154/ufs-weather-model/tests/RegressionTests_orion.intel.log -++ REGRESSIONTEST_LOG=/work/noaa/fv3-cam/djovic/ufs/pr154/ufs-weather-model/tests/RegressionTests_orion.intel.log -++ export LOG_DIR=/work/noaa/fv3-cam/djovic/ufs/pr154/ufs-weather-model/tests/log_orion.intel -++ LOG_DIR=/work/noaa/fv3-cam/djovic/ufs/pr154/ufs-weather-model/tests/log_orion.intel +++ export REGRESSIONTEST_LOG=/work/noaa/fv3-cam/djovic/ufs/rt_ecf_suite/ufs-weather-model/tests/RegressionTests_orion.intel.log +++ REGRESSIONTEST_LOG=/work/noaa/fv3-cam/djovic/ufs/rt_ecf_suite/ufs-weather-model/tests/RegressionTests_orion.intel.log +++ export LOG_DIR=/work/noaa/fv3-cam/djovic/ufs/rt_ecf_suite/ufs-weather-model/tests/log_orion.intel +++ LOG_DIR=/work/noaa/fv3-cam/djovic/ufs/rt_ecf_suite/ufs-weather-model/tests/log_orion.intel + source default_vars.sh ++ '[' orion.intel = wcoss ']' ++ '[' orion.intel = wcoss_cray ']' @@ -294,15 +271,15 @@ Elapsed time 907 seconds. Compile 11 + TEST_NR=12 + export JBNME=compile_12 + JBNME=compile_12 -+ export RUNDIR=/work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_369221/compile_12 -+ RUNDIR=/work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_369221/compile_12 ++ export RUNDIR=/work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_339981/compile_12 ++ RUNDIR=/work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_339981/compile_12 + source rt_utils.sh ++ set -eu -++ [[ /work/noaa/fv3-cam/djovic/ufs/pr154/ufs-weather-model/tests/run_compile.sh = \r\t\_\u\t\i\l\s\.\s\h ]] +++ [[ /work/noaa/fv3-cam/djovic/ufs/rt_ecf_suite/ufs-weather-model/tests/run_compile.sh = \r\t\_\u\t\i\l\s\.\s\h ]] ++ UNIT_TEST=false + source atparse.bash -+ mkdir -p /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_369221/compile_12 -+ cd /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_369221/compile_12 ++ mkdir -p /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_339981/compile_12 ++ cd /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_339981/compile_12 + [[ slurm = \s\l\u\r\m ]] + atparse + local __set_x @@ -315,55 +292,43 @@ Elapsed time 907 seconds. Compile 11 + '[' -o xtrace ']' + set_x='set -x' + set +x -Job id 174438 +Job id 177599 TEST 12 compile is waiting to enter the queue -1 min. TEST 12 compile is pending, Status: PD -2 min. TEST 12 compile is pending, Status: PD -3 min. TEST 12 compile is pending, Status: PD -4 min. TEST 12 compile is pending, Status: PD -5 min. TEST 12 compile is pending, Status: PD -6 min. TEST 12 compile is pending, Status: PD -7 min. TEST 12 compile is pending, Status: PD -8 min. TEST 12 compile is pending, Status: PD -9 min. TEST 12 compile is pending, Status: PD -10 min. TEST 12 compile is pending, Status: PD -11 min. TEST 12 compile is pending, Status: PD -12 min. TEST 12 compile is running, Status: R -13 min. TEST 12 compile is running, Status: R -14 min. TEST 12 compile is running, Status: R -15 min. TEST 12 compile is running, Status: R -16 min. TEST 12 compile is running, Status: R -17 min. TEST 12 compile is running, Status: R -18 min. TEST 12 compile is running, Status: R +1 min. TEST 12 compile is running, Status: R +2 min. TEST 12 compile is running, Status: R +3 min. TEST 12 compile is running, Status: R +4 min. TEST 12 compile is running, Status: R +5 min. TEST 12 compile is running, Status: R +6 min. TEST 12 compile is running, Status: R Slurm unknown status -. Check sacct ... -174438 COMPLETED compile_12 -174438.batch COMPLETED batch -174438.exte+ COMPLETED extern -19 min. TEST 12 compile is COMPLETED +177599 COMPLETED compile_12 +177599.batch COMPLETED batch +177599.exte+ COMPLETED extern +7 min. TEST 12 compile is COMPLETED COMPLETED COMPLETED -+ elapsed=1147 -+ echo 'Elapsed time 1147 seconds. Compile 12' -Elapsed time 1147 seconds. Compile 12 ++ elapsed=426 ++ echo 'Elapsed time 426 seconds. Compile 12' +Elapsed time 426 seconds. Compile 12 + SECONDS=0 + [[ 4 != 4 ]] -+ export PATHRT=/work/noaa/fv3-cam/djovic/ufs/pr154/ufs-weather-model/tests -+ PATHRT=/work/noaa/fv3-cam/djovic/ufs/pr154/ufs-weather-model/tests -+ export RUNDIR_ROOT=/work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_369221 -+ RUNDIR_ROOT=/work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_369221 ++ export PATHRT=/work/noaa/fv3-cam/djovic/ufs/rt_ecf_suite/ufs-weather-model/tests ++ PATHRT=/work/noaa/fv3-cam/djovic/ufs/rt_ecf_suite/ufs-weather-model/tests ++ export RUNDIR_ROOT=/work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_339981 ++ RUNDIR_ROOT=/work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_339981 + export 'MAKE_OPT=CCPP=Y SUITES=FV3_GFS_2017_gfdlmp,FV3_GFS_2017_gfdlmp_noahmp' + MAKE_OPT='CCPP=Y SUITES=FV3_GFS_2017_gfdlmp,FV3_GFS_2017_gfdlmp_noahmp' + export COMPILE_NR=13 + COMPILE_NR=13 -+ cd /work/noaa/fv3-cam/djovic/ufs/pr154/ufs-weather-model/tests -+ [[ -e /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_369221/compile_13.env ]] -+ source /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_369221/compile_13.env ++ cd /work/noaa/fv3-cam/djovic/ufs/rt_ecf_suite/ufs-weather-model/tests ++ [[ -e /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_339981/compile_13.env ]] ++ source /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_339981/compile_13.env ++ export MACHINE_ID=orion.intel ++ MACHINE_ID=orion.intel -++ export PATHRT=/work/noaa/fv3-cam/djovic/ufs/pr154/ufs-weather-model/tests -++ PATHRT=/work/noaa/fv3-cam/djovic/ufs/pr154/ufs-weather-model/tests -++ export PATHTR=/work/noaa/fv3-cam/djovic/ufs/pr154/ufs-weather-model -++ PATHTR=/work/noaa/fv3-cam/djovic/ufs/pr154/ufs-weather-model +++ export PATHRT=/work/noaa/fv3-cam/djovic/ufs/rt_ecf_suite/ufs-weather-model/tests +++ PATHRT=/work/noaa/fv3-cam/djovic/ufs/rt_ecf_suite/ufs-weather-model/tests +++ export PATHTR=/work/noaa/fv3-cam/djovic/ufs/rt_ecf_suite/ufs-weather-model +++ PATHTR=/work/noaa/fv3-cam/djovic/ufs/rt_ecf_suite/ufs-weather-model ++ export SCHEDULER=slurm ++ SCHEDULER=slurm ++ export ACCNR=nems @@ -376,10 +341,10 @@ Elapsed time 1147 seconds. Compile 12 ++ ROCOTO=false ++ export ECFLOW=true ++ ECFLOW=true -++ export REGRESSIONTEST_LOG=/work/noaa/fv3-cam/djovic/ufs/pr154/ufs-weather-model/tests/RegressionTests_orion.intel.log -++ REGRESSIONTEST_LOG=/work/noaa/fv3-cam/djovic/ufs/pr154/ufs-weather-model/tests/RegressionTests_orion.intel.log -++ export LOG_DIR=/work/noaa/fv3-cam/djovic/ufs/pr154/ufs-weather-model/tests/log_orion.intel -++ LOG_DIR=/work/noaa/fv3-cam/djovic/ufs/pr154/ufs-weather-model/tests/log_orion.intel +++ export REGRESSIONTEST_LOG=/work/noaa/fv3-cam/djovic/ufs/rt_ecf_suite/ufs-weather-model/tests/RegressionTests_orion.intel.log +++ REGRESSIONTEST_LOG=/work/noaa/fv3-cam/djovic/ufs/rt_ecf_suite/ufs-weather-model/tests/RegressionTests_orion.intel.log +++ export LOG_DIR=/work/noaa/fv3-cam/djovic/ufs/rt_ecf_suite/ufs-weather-model/tests/log_orion.intel +++ LOG_DIR=/work/noaa/fv3-cam/djovic/ufs/rt_ecf_suite/ufs-weather-model/tests/log_orion.intel + source default_vars.sh ++ '[' orion.intel = wcoss ']' ++ '[' orion.intel = wcoss_cray ']' @@ -411,15 +376,15 @@ Elapsed time 1147 seconds. Compile 12 + TEST_NR=13 + export JBNME=compile_13 + JBNME=compile_13 -+ export RUNDIR=/work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_369221/compile_13 -+ RUNDIR=/work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_369221/compile_13 ++ export RUNDIR=/work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_339981/compile_13 ++ RUNDIR=/work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_339981/compile_13 + source rt_utils.sh ++ set -eu -++ [[ /work/noaa/fv3-cam/djovic/ufs/pr154/ufs-weather-model/tests/run_compile.sh = \r\t\_\u\t\i\l\s\.\s\h ]] +++ [[ /work/noaa/fv3-cam/djovic/ufs/rt_ecf_suite/ufs-weather-model/tests/run_compile.sh = \r\t\_\u\t\i\l\s\.\s\h ]] ++ UNIT_TEST=false + source atparse.bash -+ mkdir -p /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_369221/compile_13 -+ cd /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_369221/compile_13 ++ mkdir -p /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_339981/compile_13 ++ cd /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_339981/compile_13 + [[ slurm = \s\l\u\r\m ]] + atparse + local __set_x @@ -432,54 +397,43 @@ Elapsed time 1147 seconds. Compile 12 + '[' -o xtrace ']' + set_x='set -x' + set +x -Job id 174456 +Job id 177604 TEST 13 compile is waiting to enter the queue -1 min. TEST 13 compile is pending, Status: PD -2 min. TEST 13 compile is pending, Status: PD -3 min. TEST 13 compile is pending, Status: PD -4 min. TEST 13 compile is pending, Status: PD -5 min. TEST 13 compile is pending, Status: PD -6 min. TEST 13 compile is pending, Status: PD -7 min. TEST 13 compile is pending, Status: PD -8 min. TEST 13 compile is pending, Status: PD -9 min. TEST 13 compile is pending, Status: PD -10 min. TEST 13 compile is pending, Status: PD -11 min. TEST 13 compile is running, Status: R -12 min. TEST 13 compile is running, Status: R -13 min. TEST 13 compile is running, Status: R -14 min. TEST 13 compile is running, Status: R -15 min. TEST 13 compile is running, Status: R -16 min. TEST 13 compile is running, Status: R -17 min. TEST 13 compile is running, Status: R +1 min. TEST 13 compile is running, Status: R +2 min. TEST 13 compile is running, Status: R +3 min. TEST 13 compile is running, Status: R +4 min. TEST 13 compile is running, Status: R +5 min. TEST 13 compile is running, Status: R +6 min. TEST 13 compile is running, Status: R Slurm unknown status -. Check sacct ... -174456 COMPLETED compile_13 -174456.batch COMPLETED batch -174456.exte+ COMPLETED extern -18 min. TEST 13 compile is COMPLETED +177604 COMPLETED compile_13 +177604.batch COMPLETED batch +177604.exte+ COMPLETED extern +7 min. TEST 13 compile is COMPLETED COMPLETED COMPLETED -+ elapsed=1087 -+ echo 'Elapsed time 1087 seconds. Compile 13' -Elapsed time 1087 seconds. Compile 13 ++ elapsed=426 ++ echo 'Elapsed time 426 seconds. Compile 13' +Elapsed time 426 seconds. Compile 13 + SECONDS=0 + [[ 4 != 4 ]] -+ export PATHRT=/work/noaa/fv3-cam/djovic/ufs/pr154/ufs-weather-model/tests -+ PATHRT=/work/noaa/fv3-cam/djovic/ufs/pr154/ufs-weather-model/tests -+ export RUNDIR_ROOT=/work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_369221 -+ RUNDIR_ROOT=/work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_369221 ++ export PATHRT=/work/noaa/fv3-cam/djovic/ufs/rt_ecf_suite/ufs-weather-model/tests ++ PATHRT=/work/noaa/fv3-cam/djovic/ufs/rt_ecf_suite/ufs-weather-model/tests ++ export RUNDIR_ROOT=/work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_339981 ++ RUNDIR_ROOT=/work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_339981 + export 'MAKE_OPT=CCPP=Y SUITES=FV3_GFS_2017_fv3wam 32BIT=Y MULTI_GASES=Y' + MAKE_OPT='CCPP=Y SUITES=FV3_GFS_2017_fv3wam 32BIT=Y MULTI_GASES=Y' + export COMPILE_NR=14 + COMPILE_NR=14 -+ cd /work/noaa/fv3-cam/djovic/ufs/pr154/ufs-weather-model/tests -+ [[ -e /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_369221/compile_14.env ]] -+ source /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_369221/compile_14.env ++ cd /work/noaa/fv3-cam/djovic/ufs/rt_ecf_suite/ufs-weather-model/tests ++ [[ -e /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_339981/compile_14.env ]] ++ source /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_339981/compile_14.env ++ export MACHINE_ID=orion.intel ++ MACHINE_ID=orion.intel -++ export PATHRT=/work/noaa/fv3-cam/djovic/ufs/pr154/ufs-weather-model/tests -++ PATHRT=/work/noaa/fv3-cam/djovic/ufs/pr154/ufs-weather-model/tests -++ export PATHTR=/work/noaa/fv3-cam/djovic/ufs/pr154/ufs-weather-model -++ PATHTR=/work/noaa/fv3-cam/djovic/ufs/pr154/ufs-weather-model +++ export PATHRT=/work/noaa/fv3-cam/djovic/ufs/rt_ecf_suite/ufs-weather-model/tests +++ PATHRT=/work/noaa/fv3-cam/djovic/ufs/rt_ecf_suite/ufs-weather-model/tests +++ export PATHTR=/work/noaa/fv3-cam/djovic/ufs/rt_ecf_suite/ufs-weather-model +++ PATHTR=/work/noaa/fv3-cam/djovic/ufs/rt_ecf_suite/ufs-weather-model ++ export SCHEDULER=slurm ++ SCHEDULER=slurm ++ export ACCNR=nems @@ -492,10 +446,10 @@ Elapsed time 1087 seconds. Compile 13 ++ ROCOTO=false ++ export ECFLOW=true ++ ECFLOW=true -++ export REGRESSIONTEST_LOG=/work/noaa/fv3-cam/djovic/ufs/pr154/ufs-weather-model/tests/RegressionTests_orion.intel.log -++ REGRESSIONTEST_LOG=/work/noaa/fv3-cam/djovic/ufs/pr154/ufs-weather-model/tests/RegressionTests_orion.intel.log -++ export LOG_DIR=/work/noaa/fv3-cam/djovic/ufs/pr154/ufs-weather-model/tests/log_orion.intel -++ LOG_DIR=/work/noaa/fv3-cam/djovic/ufs/pr154/ufs-weather-model/tests/log_orion.intel +++ export REGRESSIONTEST_LOG=/work/noaa/fv3-cam/djovic/ufs/rt_ecf_suite/ufs-weather-model/tests/RegressionTests_orion.intel.log +++ REGRESSIONTEST_LOG=/work/noaa/fv3-cam/djovic/ufs/rt_ecf_suite/ufs-weather-model/tests/RegressionTests_orion.intel.log +++ export LOG_DIR=/work/noaa/fv3-cam/djovic/ufs/rt_ecf_suite/ufs-weather-model/tests/log_orion.intel +++ LOG_DIR=/work/noaa/fv3-cam/djovic/ufs/rt_ecf_suite/ufs-weather-model/tests/log_orion.intel + source default_vars.sh ++ '[' orion.intel = wcoss ']' ++ '[' orion.intel = wcoss_cray ']' @@ -527,15 +481,15 @@ Elapsed time 1087 seconds. Compile 13 + TEST_NR=14 + export JBNME=compile_14 + JBNME=compile_14 -+ export RUNDIR=/work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_369221/compile_14 -+ RUNDIR=/work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_369221/compile_14 ++ export RUNDIR=/work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_339981/compile_14 ++ RUNDIR=/work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_339981/compile_14 + source rt_utils.sh ++ set -eu -++ [[ /work/noaa/fv3-cam/djovic/ufs/pr154/ufs-weather-model/tests/run_compile.sh = \r\t\_\u\t\i\l\s\.\s\h ]] +++ [[ /work/noaa/fv3-cam/djovic/ufs/rt_ecf_suite/ufs-weather-model/tests/run_compile.sh = \r\t\_\u\t\i\l\s\.\s\h ]] ++ UNIT_TEST=false + source atparse.bash -+ mkdir -p /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_369221/compile_14 -+ cd /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_369221/compile_14 ++ mkdir -p /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_339981/compile_14 ++ cd /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_339981/compile_14 + [[ slurm = \s\l\u\r\m ]] + atparse + local __set_x @@ -548,48 +502,43 @@ Elapsed time 1087 seconds. Compile 13 + '[' -o xtrace ']' + set_x='set -x' + set +x -Job id 174465 +Job id 177605 TEST 14 compile is waiting to enter the queue -1 min. TEST 14 compile is pending, Status: PD -2 min. TEST 14 compile is pending, Status: PD -3 min. TEST 14 compile is pending, Status: PD -4 min. TEST 14 compile is pending, Status: PD -5 min. TEST 14 compile is pending, Status: PD +1 min. TEST 14 compile is running, Status: R +2 min. TEST 14 compile is running, Status: R +3 min. TEST 14 compile is running, Status: R +4 min. TEST 14 compile is running, Status: R +5 min. TEST 14 compile is running, Status: R 6 min. TEST 14 compile is running, Status: R -7 min. TEST 14 compile is running, Status: R -8 min. TEST 14 compile is running, Status: R -9 min. TEST 14 compile is running, Status: R -10 min. TEST 14 compile is running, Status: R -11 min. TEST 14 compile is running, Status: R Slurm unknown status -. Check sacct ... -174465 COMPLETED compile_14 -174465.batch COMPLETED batch -174465.exte+ COMPLETED extern -12 min. TEST 14 compile is COMPLETED +177605 COMPLETED compile_14 +177605.batch COMPLETED batch +177605.exte+ COMPLETED extern +7 min. TEST 14 compile is COMPLETED COMPLETED COMPLETED -+ elapsed=726 -+ echo 'Elapsed time 726 seconds. Compile 14' -Elapsed time 726 seconds. Compile 14 ++ elapsed=425 ++ echo 'Elapsed time 425 seconds. Compile 14' +Elapsed time 425 seconds. Compile 14 + SECONDS=0 + [[ 4 != 4 ]] -+ export PATHRT=/work/noaa/fv3-cam/djovic/ufs/pr154/ufs-weather-model/tests -+ PATHRT=/work/noaa/fv3-cam/djovic/ufs/pr154/ufs-weather-model/tests -+ export RUNDIR_ROOT=/work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_369221 -+ RUNDIR_ROOT=/work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_369221 ++ export PATHRT=/work/noaa/fv3-cam/djovic/ufs/rt_ecf_suite/ufs-weather-model/tests ++ PATHRT=/work/noaa/fv3-cam/djovic/ufs/rt_ecf_suite/ufs-weather-model/tests ++ export RUNDIR_ROOT=/work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_339981 ++ RUNDIR_ROOT=/work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_339981 + export 'MAKE_OPT=CCPP=Y SUITES=FV3_GFS_2017' + MAKE_OPT='CCPP=Y SUITES=FV3_GFS_2017' + export COMPILE_NR=1 + COMPILE_NR=1 -+ cd /work/noaa/fv3-cam/djovic/ufs/pr154/ufs-weather-model/tests -+ [[ -e /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_369221/compile_1.env ]] -+ source /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_369221/compile_1.env ++ cd /work/noaa/fv3-cam/djovic/ufs/rt_ecf_suite/ufs-weather-model/tests ++ [[ -e /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_339981/compile_1.env ]] ++ source /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_339981/compile_1.env ++ export MACHINE_ID=orion.intel ++ MACHINE_ID=orion.intel -++ export PATHRT=/work/noaa/fv3-cam/djovic/ufs/pr154/ufs-weather-model/tests -++ PATHRT=/work/noaa/fv3-cam/djovic/ufs/pr154/ufs-weather-model/tests -++ export PATHTR=/work/noaa/fv3-cam/djovic/ufs/pr154/ufs-weather-model -++ PATHTR=/work/noaa/fv3-cam/djovic/ufs/pr154/ufs-weather-model +++ export PATHRT=/work/noaa/fv3-cam/djovic/ufs/rt_ecf_suite/ufs-weather-model/tests +++ PATHRT=/work/noaa/fv3-cam/djovic/ufs/rt_ecf_suite/ufs-weather-model/tests +++ export PATHTR=/work/noaa/fv3-cam/djovic/ufs/rt_ecf_suite/ufs-weather-model +++ PATHTR=/work/noaa/fv3-cam/djovic/ufs/rt_ecf_suite/ufs-weather-model ++ export SCHEDULER=slurm ++ SCHEDULER=slurm ++ export ACCNR=nems @@ -602,10 +551,10 @@ Elapsed time 726 seconds. Compile 14 ++ ROCOTO=false ++ export ECFLOW=true ++ ECFLOW=true -++ export REGRESSIONTEST_LOG=/work/noaa/fv3-cam/djovic/ufs/pr154/ufs-weather-model/tests/RegressionTests_orion.intel.log -++ REGRESSIONTEST_LOG=/work/noaa/fv3-cam/djovic/ufs/pr154/ufs-weather-model/tests/RegressionTests_orion.intel.log -++ export LOG_DIR=/work/noaa/fv3-cam/djovic/ufs/pr154/ufs-weather-model/tests/log_orion.intel -++ LOG_DIR=/work/noaa/fv3-cam/djovic/ufs/pr154/ufs-weather-model/tests/log_orion.intel +++ export REGRESSIONTEST_LOG=/work/noaa/fv3-cam/djovic/ufs/rt_ecf_suite/ufs-weather-model/tests/RegressionTests_orion.intel.log +++ REGRESSIONTEST_LOG=/work/noaa/fv3-cam/djovic/ufs/rt_ecf_suite/ufs-weather-model/tests/RegressionTests_orion.intel.log +++ export LOG_DIR=/work/noaa/fv3-cam/djovic/ufs/rt_ecf_suite/ufs-weather-model/tests/log_orion.intel +++ LOG_DIR=/work/noaa/fv3-cam/djovic/ufs/rt_ecf_suite/ufs-weather-model/tests/log_orion.intel + source default_vars.sh ++ '[' orion.intel = wcoss ']' ++ '[' orion.intel = wcoss_cray ']' @@ -637,15 +586,15 @@ Elapsed time 726 seconds. Compile 14 + TEST_NR=1 + export JBNME=compile_1 + JBNME=compile_1 -+ export RUNDIR=/work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_369221/compile_1 -+ RUNDIR=/work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_369221/compile_1 ++ export RUNDIR=/work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_339981/compile_1 ++ RUNDIR=/work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_339981/compile_1 + source rt_utils.sh ++ set -eu -++ [[ /work/noaa/fv3-cam/djovic/ufs/pr154/ufs-weather-model/tests/run_compile.sh = \r\t\_\u\t\i\l\s\.\s\h ]] +++ [[ /work/noaa/fv3-cam/djovic/ufs/rt_ecf_suite/ufs-weather-model/tests/run_compile.sh = \r\t\_\u\t\i\l\s\.\s\h ]] ++ UNIT_TEST=false + source atparse.bash -+ mkdir -p /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_369221/compile_1 -+ cd /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_369221/compile_1 ++ mkdir -p /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_339981/compile_1 ++ cd /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_339981/compile_1 + [[ slurm = \s\l\u\r\m ]] + atparse + local __set_x @@ -658,7 +607,7 @@ Elapsed time 726 seconds. Compile 14 + '[' -o xtrace ']' + set_x='set -x' + set +x -Job id 174417 +Job id 177584 TEST 1 compile is waiting to enter the queue 1 min. TEST 1 compile is running, Status: R 2 min. TEST 1 compile is running, Status: R @@ -667,9 +616,9 @@ TEST 1 compile is waiting to enter the queue 5 min. TEST 1 compile is running, Status: R 6 min. TEST 1 compile is running, Status: R Slurm unknown status -. Check sacct ... -174417 COMPLETED compile_1 -174417.batch COMPLETED batch -174417.exte+ COMPLETED extern +177584 COMPLETED compile_1 +177584.batch COMPLETED batch +177584.exte+ COMPLETED extern 7 min. TEST 1 compile is COMPLETED COMPLETED COMPLETED @@ -678,23 +627,23 @@ COMPLETED Elapsed time 426 seconds. Compile 1 + SECONDS=0 + [[ 4 != 4 ]] -+ export PATHRT=/work/noaa/fv3-cam/djovic/ufs/pr154/ufs-weather-model/tests -+ PATHRT=/work/noaa/fv3-cam/djovic/ufs/pr154/ufs-weather-model/tests -+ export RUNDIR_ROOT=/work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_369221 -+ RUNDIR_ROOT=/work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_369221 ++ export PATHRT=/work/noaa/fv3-cam/djovic/ufs/rt_ecf_suite/ufs-weather-model/tests ++ PATHRT=/work/noaa/fv3-cam/djovic/ufs/rt_ecf_suite/ufs-weather-model/tests ++ export RUNDIR_ROOT=/work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_339981 ++ RUNDIR_ROOT=/work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_339981 + export 'MAKE_OPT=CCPP=Y SUITES=FV3_GFS_2017,FV3_GFS_2017_gfdlmp WW3=Y' + MAKE_OPT='CCPP=Y SUITES=FV3_GFS_2017,FV3_GFS_2017_gfdlmp WW3=Y' + export COMPILE_NR=2 + COMPILE_NR=2 -+ cd /work/noaa/fv3-cam/djovic/ufs/pr154/ufs-weather-model/tests -+ [[ -e /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_369221/compile_2.env ]] -+ source /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_369221/compile_2.env ++ cd /work/noaa/fv3-cam/djovic/ufs/rt_ecf_suite/ufs-weather-model/tests ++ [[ -e /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_339981/compile_2.env ]] ++ source /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_339981/compile_2.env ++ export MACHINE_ID=orion.intel ++ MACHINE_ID=orion.intel -++ export PATHRT=/work/noaa/fv3-cam/djovic/ufs/pr154/ufs-weather-model/tests -++ PATHRT=/work/noaa/fv3-cam/djovic/ufs/pr154/ufs-weather-model/tests -++ export PATHTR=/work/noaa/fv3-cam/djovic/ufs/pr154/ufs-weather-model -++ PATHTR=/work/noaa/fv3-cam/djovic/ufs/pr154/ufs-weather-model +++ export PATHRT=/work/noaa/fv3-cam/djovic/ufs/rt_ecf_suite/ufs-weather-model/tests +++ PATHRT=/work/noaa/fv3-cam/djovic/ufs/rt_ecf_suite/ufs-weather-model/tests +++ export PATHTR=/work/noaa/fv3-cam/djovic/ufs/rt_ecf_suite/ufs-weather-model +++ PATHTR=/work/noaa/fv3-cam/djovic/ufs/rt_ecf_suite/ufs-weather-model ++ export SCHEDULER=slurm ++ SCHEDULER=slurm ++ export ACCNR=nems @@ -707,10 +656,10 @@ Elapsed time 426 seconds. Compile 1 ++ ROCOTO=false ++ export ECFLOW=true ++ ECFLOW=true -++ export REGRESSIONTEST_LOG=/work/noaa/fv3-cam/djovic/ufs/pr154/ufs-weather-model/tests/RegressionTests_orion.intel.log -++ REGRESSIONTEST_LOG=/work/noaa/fv3-cam/djovic/ufs/pr154/ufs-weather-model/tests/RegressionTests_orion.intel.log -++ export LOG_DIR=/work/noaa/fv3-cam/djovic/ufs/pr154/ufs-weather-model/tests/log_orion.intel -++ LOG_DIR=/work/noaa/fv3-cam/djovic/ufs/pr154/ufs-weather-model/tests/log_orion.intel +++ export REGRESSIONTEST_LOG=/work/noaa/fv3-cam/djovic/ufs/rt_ecf_suite/ufs-weather-model/tests/RegressionTests_orion.intel.log +++ REGRESSIONTEST_LOG=/work/noaa/fv3-cam/djovic/ufs/rt_ecf_suite/ufs-weather-model/tests/RegressionTests_orion.intel.log +++ export LOG_DIR=/work/noaa/fv3-cam/djovic/ufs/rt_ecf_suite/ufs-weather-model/tests/log_orion.intel +++ LOG_DIR=/work/noaa/fv3-cam/djovic/ufs/rt_ecf_suite/ufs-weather-model/tests/log_orion.intel + source default_vars.sh ++ '[' orion.intel = wcoss ']' ++ '[' orion.intel = wcoss_cray ']' @@ -742,15 +691,15 @@ Elapsed time 426 seconds. Compile 1 + TEST_NR=2 + export JBNME=compile_2 + JBNME=compile_2 -+ export RUNDIR=/work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_369221/compile_2 -+ RUNDIR=/work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_369221/compile_2 ++ export RUNDIR=/work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_339981/compile_2 ++ RUNDIR=/work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_339981/compile_2 + source rt_utils.sh ++ set -eu -++ [[ /work/noaa/fv3-cam/djovic/ufs/pr154/ufs-weather-model/tests/run_compile.sh = \r\t\_\u\t\i\l\s\.\s\h ]] +++ [[ /work/noaa/fv3-cam/djovic/ufs/rt_ecf_suite/ufs-weather-model/tests/run_compile.sh = \r\t\_\u\t\i\l\s\.\s\h ]] ++ UNIT_TEST=false + source atparse.bash -+ mkdir -p /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_369221/compile_2 -+ cd /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_369221/compile_2 ++ mkdir -p /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_339981/compile_2 ++ cd /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_339981/compile_2 + [[ slurm = \s\l\u\r\m ]] + atparse + local __set_x @@ -763,7 +712,7 @@ Elapsed time 426 seconds. Compile 1 + '[' -o xtrace ']' + set_x='set -x' + set +x -Job id 174419 +Job id 177583 TEST 2 compile is waiting to enter the queue 1 min. TEST 2 compile is running, Status: R 2 min. TEST 2 compile is running, Status: R @@ -771,35 +720,36 @@ TEST 2 compile is waiting to enter the queue 4 min. TEST 2 compile is running, Status: R 5 min. TEST 2 compile is running, Status: R 6 min. TEST 2 compile is running, Status: R +7 min. TEST 2 compile is running, Status: R Slurm unknown status -. Check sacct ... -174419 COMPLETED compile_2 -174419.batch COMPLETED batch -174419.exte+ COMPLETED extern -7 min. TEST 2 compile is COMPLETED +177583 COMPLETED compile_2 +177583.batch COMPLETED batch +177583.exte+ COMPLETED extern +8 min. TEST 2 compile is COMPLETED COMPLETED COMPLETED -+ elapsed=426 -+ echo 'Elapsed time 426 seconds. Compile 2' -Elapsed time 426 seconds. Compile 2 ++ elapsed=486 ++ echo 'Elapsed time 486 seconds. Compile 2' +Elapsed time 486 seconds. Compile 2 + SECONDS=0 + [[ 4 != 4 ]] -+ export PATHRT=/work/noaa/fv3-cam/djovic/ufs/pr154/ufs-weather-model/tests -+ PATHRT=/work/noaa/fv3-cam/djovic/ufs/pr154/ufs-weather-model/tests -+ export RUNDIR_ROOT=/work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_369221 -+ RUNDIR_ROOT=/work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_369221 ++ export PATHRT=/work/noaa/fv3-cam/djovic/ufs/rt_ecf_suite/ufs-weather-model/tests ++ PATHRT=/work/noaa/fv3-cam/djovic/ufs/rt_ecf_suite/ufs-weather-model/tests ++ export RUNDIR_ROOT=/work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_339981 ++ RUNDIR_ROOT=/work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_339981 + export 'MAKE_OPT=CCPP=Y SUITES=FV3_GFS_2017,FV3_GFS_2017_stretched 32BIT=Y' + MAKE_OPT='CCPP=Y SUITES=FV3_GFS_2017,FV3_GFS_2017_stretched 32BIT=Y' + export COMPILE_NR=3 + COMPILE_NR=3 -+ cd /work/noaa/fv3-cam/djovic/ufs/pr154/ufs-weather-model/tests -+ [[ -e /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_369221/compile_3.env ]] -+ source /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_369221/compile_3.env ++ cd /work/noaa/fv3-cam/djovic/ufs/rt_ecf_suite/ufs-weather-model/tests ++ [[ -e /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_339981/compile_3.env ]] ++ source /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_339981/compile_3.env ++ export MACHINE_ID=orion.intel ++ MACHINE_ID=orion.intel -++ export PATHRT=/work/noaa/fv3-cam/djovic/ufs/pr154/ufs-weather-model/tests -++ PATHRT=/work/noaa/fv3-cam/djovic/ufs/pr154/ufs-weather-model/tests -++ export PATHTR=/work/noaa/fv3-cam/djovic/ufs/pr154/ufs-weather-model -++ PATHTR=/work/noaa/fv3-cam/djovic/ufs/pr154/ufs-weather-model +++ export PATHRT=/work/noaa/fv3-cam/djovic/ufs/rt_ecf_suite/ufs-weather-model/tests +++ PATHRT=/work/noaa/fv3-cam/djovic/ufs/rt_ecf_suite/ufs-weather-model/tests +++ export PATHTR=/work/noaa/fv3-cam/djovic/ufs/rt_ecf_suite/ufs-weather-model +++ PATHTR=/work/noaa/fv3-cam/djovic/ufs/rt_ecf_suite/ufs-weather-model ++ export SCHEDULER=slurm ++ SCHEDULER=slurm ++ export ACCNR=nems @@ -812,10 +762,10 @@ Elapsed time 426 seconds. Compile 2 ++ ROCOTO=false ++ export ECFLOW=true ++ ECFLOW=true -++ export REGRESSIONTEST_LOG=/work/noaa/fv3-cam/djovic/ufs/pr154/ufs-weather-model/tests/RegressionTests_orion.intel.log -++ REGRESSIONTEST_LOG=/work/noaa/fv3-cam/djovic/ufs/pr154/ufs-weather-model/tests/RegressionTests_orion.intel.log -++ export LOG_DIR=/work/noaa/fv3-cam/djovic/ufs/pr154/ufs-weather-model/tests/log_orion.intel -++ LOG_DIR=/work/noaa/fv3-cam/djovic/ufs/pr154/ufs-weather-model/tests/log_orion.intel +++ export REGRESSIONTEST_LOG=/work/noaa/fv3-cam/djovic/ufs/rt_ecf_suite/ufs-weather-model/tests/RegressionTests_orion.intel.log +++ REGRESSIONTEST_LOG=/work/noaa/fv3-cam/djovic/ufs/rt_ecf_suite/ufs-weather-model/tests/RegressionTests_orion.intel.log +++ export LOG_DIR=/work/noaa/fv3-cam/djovic/ufs/rt_ecf_suite/ufs-weather-model/tests/log_orion.intel +++ LOG_DIR=/work/noaa/fv3-cam/djovic/ufs/rt_ecf_suite/ufs-weather-model/tests/log_orion.intel + source default_vars.sh ++ '[' orion.intel = wcoss ']' ++ '[' orion.intel = wcoss_cray ']' @@ -847,15 +797,15 @@ Elapsed time 426 seconds. Compile 2 + TEST_NR=3 + export JBNME=compile_3 + JBNME=compile_3 -+ export RUNDIR=/work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_369221/compile_3 -+ RUNDIR=/work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_369221/compile_3 ++ export RUNDIR=/work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_339981/compile_3 ++ RUNDIR=/work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_339981/compile_3 + source rt_utils.sh ++ set -eu -++ [[ /work/noaa/fv3-cam/djovic/ufs/pr154/ufs-weather-model/tests/run_compile.sh = \r\t\_\u\t\i\l\s\.\s\h ]] +++ [[ /work/noaa/fv3-cam/djovic/ufs/rt_ecf_suite/ufs-weather-model/tests/run_compile.sh = \r\t\_\u\t\i\l\s\.\s\h ]] ++ UNIT_TEST=false + source atparse.bash -+ mkdir -p /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_369221/compile_3 -+ cd /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_369221/compile_3 ++ mkdir -p /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_339981/compile_3 ++ cd /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_339981/compile_3 + [[ slurm = \s\l\u\r\m ]] + atparse + local __set_x @@ -868,53 +818,43 @@ Elapsed time 426 seconds. Compile 2 + '[' -o xtrace ']' + set_x='set -x' + set +x -Job id 174420 +Job id 177586 TEST 3 compile is waiting to enter the queue -1 min. TEST 3 compile is pending, Status: PD -2 min. TEST 3 compile is pending, Status: PD -3 min. TEST 3 compile is pending, Status: PD -4 min. TEST 3 compile is pending, Status: PD -5 min. TEST 3 compile is pending, Status: PD -6 min. TEST 3 compile is pending, Status: PD -7 min. TEST 3 compile is pending, Status: PD -8 min. TEST 3 compile is pending, Status: PD -9 min. TEST 3 compile is pending, Status: PD -10 min. TEST 3 compile is running, Status: R -11 min. TEST 3 compile is running, Status: R -12 min. TEST 3 compile is running, Status: R -13 min. TEST 3 compile is running, Status: R -14 min. TEST 3 compile is running, Status: R -15 min. TEST 3 compile is running, Status: R -16 min. TEST 3 compile is running, Status: R +1 min. TEST 3 compile is running, Status: R +2 min. TEST 3 compile is running, Status: R +3 min. TEST 3 compile is running, Status: R +4 min. TEST 3 compile is running, Status: R +5 min. TEST 3 compile is running, Status: R +6 min. TEST 3 compile is running, Status: R Slurm unknown status -. Check sacct ... -174420 COMPLETED compile_3 -174420.batch COMPLETED batch -174420.exte+ COMPLETED extern -17 min. TEST 3 compile is COMPLETED +177586 COMPLETED compile_3 +177586.batch COMPLETED batch +177586.exte+ COMPLETED extern +7 min. TEST 3 compile is COMPLETED COMPLETED COMPLETED -+ elapsed=1028 -+ echo 'Elapsed time 1028 seconds. Compile 3' -Elapsed time 1028 seconds. Compile 3 ++ elapsed=426 ++ echo 'Elapsed time 426 seconds. Compile 3' +Elapsed time 426 seconds. Compile 3 + SECONDS=0 + [[ 4 != 4 ]] -+ export PATHRT=/work/noaa/fv3-cam/djovic/ufs/pr154/ufs-weather-model/tests -+ PATHRT=/work/noaa/fv3-cam/djovic/ufs/pr154/ufs-weather-model/tests -+ export RUNDIR_ROOT=/work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_369221 -+ RUNDIR_ROOT=/work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_369221 ++ export PATHRT=/work/noaa/fv3-cam/djovic/ufs/rt_ecf_suite/ufs-weather-model/tests ++ PATHRT=/work/noaa/fv3-cam/djovic/ufs/rt_ecf_suite/ufs-weather-model/tests ++ export RUNDIR_ROOT=/work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_339981 ++ RUNDIR_ROOT=/work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_339981 + export 'MAKE_OPT=CCPP=Y SUITES=FV3_GFS_2017_gfdlmp_regional,FV3_GFS_2017_gfdlmp_regional_c768 32BIT=Y' + MAKE_OPT='CCPP=Y SUITES=FV3_GFS_2017_gfdlmp_regional,FV3_GFS_2017_gfdlmp_regional_c768 32BIT=Y' + export COMPILE_NR=4 + COMPILE_NR=4 -+ cd /work/noaa/fv3-cam/djovic/ufs/pr154/ufs-weather-model/tests -+ [[ -e /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_369221/compile_4.env ]] -+ source /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_369221/compile_4.env ++ cd /work/noaa/fv3-cam/djovic/ufs/rt_ecf_suite/ufs-weather-model/tests ++ [[ -e /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_339981/compile_4.env ]] ++ source /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_339981/compile_4.env ++ export MACHINE_ID=orion.intel ++ MACHINE_ID=orion.intel -++ export PATHRT=/work/noaa/fv3-cam/djovic/ufs/pr154/ufs-weather-model/tests -++ PATHRT=/work/noaa/fv3-cam/djovic/ufs/pr154/ufs-weather-model/tests -++ export PATHTR=/work/noaa/fv3-cam/djovic/ufs/pr154/ufs-weather-model -++ PATHTR=/work/noaa/fv3-cam/djovic/ufs/pr154/ufs-weather-model +++ export PATHRT=/work/noaa/fv3-cam/djovic/ufs/rt_ecf_suite/ufs-weather-model/tests +++ PATHRT=/work/noaa/fv3-cam/djovic/ufs/rt_ecf_suite/ufs-weather-model/tests +++ export PATHTR=/work/noaa/fv3-cam/djovic/ufs/rt_ecf_suite/ufs-weather-model +++ PATHTR=/work/noaa/fv3-cam/djovic/ufs/rt_ecf_suite/ufs-weather-model ++ export SCHEDULER=slurm ++ SCHEDULER=slurm ++ export ACCNR=nems @@ -927,10 +867,10 @@ Elapsed time 1028 seconds. Compile 3 ++ ROCOTO=false ++ export ECFLOW=true ++ ECFLOW=true -++ export REGRESSIONTEST_LOG=/work/noaa/fv3-cam/djovic/ufs/pr154/ufs-weather-model/tests/RegressionTests_orion.intel.log -++ REGRESSIONTEST_LOG=/work/noaa/fv3-cam/djovic/ufs/pr154/ufs-weather-model/tests/RegressionTests_orion.intel.log -++ export LOG_DIR=/work/noaa/fv3-cam/djovic/ufs/pr154/ufs-weather-model/tests/log_orion.intel -++ LOG_DIR=/work/noaa/fv3-cam/djovic/ufs/pr154/ufs-weather-model/tests/log_orion.intel +++ export REGRESSIONTEST_LOG=/work/noaa/fv3-cam/djovic/ufs/rt_ecf_suite/ufs-weather-model/tests/RegressionTests_orion.intel.log +++ REGRESSIONTEST_LOG=/work/noaa/fv3-cam/djovic/ufs/rt_ecf_suite/ufs-weather-model/tests/RegressionTests_orion.intel.log +++ export LOG_DIR=/work/noaa/fv3-cam/djovic/ufs/rt_ecf_suite/ufs-weather-model/tests/log_orion.intel +++ LOG_DIR=/work/noaa/fv3-cam/djovic/ufs/rt_ecf_suite/ufs-weather-model/tests/log_orion.intel + source default_vars.sh ++ '[' orion.intel = wcoss ']' ++ '[' orion.intel = wcoss_cray ']' @@ -962,15 +902,15 @@ Elapsed time 1028 seconds. Compile 3 + TEST_NR=4 + export JBNME=compile_4 + JBNME=compile_4 -+ export RUNDIR=/work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_369221/compile_4 -+ RUNDIR=/work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_369221/compile_4 ++ export RUNDIR=/work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_339981/compile_4 ++ RUNDIR=/work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_339981/compile_4 + source rt_utils.sh ++ set -eu -++ [[ /work/noaa/fv3-cam/djovic/ufs/pr154/ufs-weather-model/tests/run_compile.sh = \r\t\_\u\t\i\l\s\.\s\h ]] +++ [[ /work/noaa/fv3-cam/djovic/ufs/rt_ecf_suite/ufs-weather-model/tests/run_compile.sh = \r\t\_\u\t\i\l\s\.\s\h ]] ++ UNIT_TEST=false + source atparse.bash -+ mkdir -p /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_369221/compile_4 -+ cd /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_369221/compile_4 ++ mkdir -p /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_339981/compile_4 ++ cd /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_339981/compile_4 + [[ slurm = \s\l\u\r\m ]] + atparse + local __set_x @@ -983,44 +923,43 @@ Elapsed time 1028 seconds. Compile 3 + '[' -o xtrace ']' + set_x='set -x' + set +x -Job id 174418 +Job id 177585 TEST 4 compile is waiting to enter the queue -1 min. TEST 4 compile is pending, Status: PD +1 min. TEST 4 compile is running, Status: R 2 min. TEST 4 compile is running, Status: R 3 min. TEST 4 compile is running, Status: R 4 min. TEST 4 compile is running, Status: R 5 min. TEST 4 compile is running, Status: R 6 min. TEST 4 compile is running, Status: R -7 min. TEST 4 compile is running, Status: R Slurm unknown status -. Check sacct ... -174418 COMPLETED compile_4 -174418.batch COMPLETED batch -174418.exte+ COMPLETED extern -8 min. TEST 4 compile is COMPLETED +177585 COMPLETED compile_4 +177585.batch COMPLETED batch +177585.exte+ COMPLETED extern +7 min. TEST 4 compile is COMPLETED COMPLETED COMPLETED -+ elapsed=486 -+ echo 'Elapsed time 486 seconds. Compile 4' -Elapsed time 486 seconds. Compile 4 ++ elapsed=426 ++ echo 'Elapsed time 426 seconds. Compile 4' +Elapsed time 426 seconds. Compile 4 + SECONDS=0 + [[ 4 != 4 ]] -+ export PATHRT=/work/noaa/fv3-cam/djovic/ufs/pr154/ufs-weather-model/tests -+ PATHRT=/work/noaa/fv3-cam/djovic/ufs/pr154/ufs-weather-model/tests -+ export RUNDIR_ROOT=/work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_369221 -+ RUNDIR_ROOT=/work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_369221 ++ export PATHRT=/work/noaa/fv3-cam/djovic/ufs/rt_ecf_suite/ufs-weather-model/tests ++ PATHRT=/work/noaa/fv3-cam/djovic/ufs/rt_ecf_suite/ufs-weather-model/tests ++ export RUNDIR_ROOT=/work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_339981 ++ RUNDIR_ROOT=/work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_339981 + export 'MAKE_OPT=CCPP=Y SUITES=FV3_GFS_2017,FV3_GFS_2017_stretched 32BIT=Y DEBUG=Y' + MAKE_OPT='CCPP=Y SUITES=FV3_GFS_2017,FV3_GFS_2017_stretched 32BIT=Y DEBUG=Y' + export COMPILE_NR=5 + COMPILE_NR=5 -+ cd /work/noaa/fv3-cam/djovic/ufs/pr154/ufs-weather-model/tests -+ [[ -e /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_369221/compile_5.env ]] -+ source /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_369221/compile_5.env ++ cd /work/noaa/fv3-cam/djovic/ufs/rt_ecf_suite/ufs-weather-model/tests ++ [[ -e /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_339981/compile_5.env ]] ++ source /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_339981/compile_5.env ++ export MACHINE_ID=orion.intel ++ MACHINE_ID=orion.intel -++ export PATHRT=/work/noaa/fv3-cam/djovic/ufs/pr154/ufs-weather-model/tests -++ PATHRT=/work/noaa/fv3-cam/djovic/ufs/pr154/ufs-weather-model/tests -++ export PATHTR=/work/noaa/fv3-cam/djovic/ufs/pr154/ufs-weather-model -++ PATHTR=/work/noaa/fv3-cam/djovic/ufs/pr154/ufs-weather-model +++ export PATHRT=/work/noaa/fv3-cam/djovic/ufs/rt_ecf_suite/ufs-weather-model/tests +++ PATHRT=/work/noaa/fv3-cam/djovic/ufs/rt_ecf_suite/ufs-weather-model/tests +++ export PATHTR=/work/noaa/fv3-cam/djovic/ufs/rt_ecf_suite/ufs-weather-model +++ PATHTR=/work/noaa/fv3-cam/djovic/ufs/rt_ecf_suite/ufs-weather-model ++ export SCHEDULER=slurm ++ SCHEDULER=slurm ++ export ACCNR=nems @@ -1033,10 +972,10 @@ Elapsed time 486 seconds. Compile 4 ++ ROCOTO=false ++ export ECFLOW=true ++ ECFLOW=true -++ export REGRESSIONTEST_LOG=/work/noaa/fv3-cam/djovic/ufs/pr154/ufs-weather-model/tests/RegressionTests_orion.intel.log -++ REGRESSIONTEST_LOG=/work/noaa/fv3-cam/djovic/ufs/pr154/ufs-weather-model/tests/RegressionTests_orion.intel.log -++ export LOG_DIR=/work/noaa/fv3-cam/djovic/ufs/pr154/ufs-weather-model/tests/log_orion.intel -++ LOG_DIR=/work/noaa/fv3-cam/djovic/ufs/pr154/ufs-weather-model/tests/log_orion.intel +++ export REGRESSIONTEST_LOG=/work/noaa/fv3-cam/djovic/ufs/rt_ecf_suite/ufs-weather-model/tests/RegressionTests_orion.intel.log +++ REGRESSIONTEST_LOG=/work/noaa/fv3-cam/djovic/ufs/rt_ecf_suite/ufs-weather-model/tests/RegressionTests_orion.intel.log +++ export LOG_DIR=/work/noaa/fv3-cam/djovic/ufs/rt_ecf_suite/ufs-weather-model/tests/log_orion.intel +++ LOG_DIR=/work/noaa/fv3-cam/djovic/ufs/rt_ecf_suite/ufs-weather-model/tests/log_orion.intel + source default_vars.sh ++ '[' orion.intel = wcoss ']' ++ '[' orion.intel = wcoss_cray ']' @@ -1068,15 +1007,15 @@ Elapsed time 486 seconds. Compile 4 + TEST_NR=5 + export JBNME=compile_5 + JBNME=compile_5 -+ export RUNDIR=/work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_369221/compile_5 -+ RUNDIR=/work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_369221/compile_5 ++ export RUNDIR=/work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_339981/compile_5 ++ RUNDIR=/work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_339981/compile_5 + source rt_utils.sh ++ set -eu -++ [[ /work/noaa/fv3-cam/djovic/ufs/pr154/ufs-weather-model/tests/run_compile.sh = \r\t\_\u\t\i\l\s\.\s\h ]] +++ [[ /work/noaa/fv3-cam/djovic/ufs/rt_ecf_suite/ufs-weather-model/tests/run_compile.sh = \r\t\_\u\t\i\l\s\.\s\h ]] ++ UNIT_TEST=false + source atparse.bash -+ mkdir -p /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_369221/compile_5 -+ cd /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_369221/compile_5 ++ mkdir -p /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_339981/compile_5 ++ cd /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_339981/compile_5 + [[ slurm = \s\l\u\r\m ]] + atparse + local __set_x @@ -1089,56 +1028,39 @@ Elapsed time 486 seconds. Compile 4 + '[' -o xtrace ']' + set_x='set -x' + set +x -Job id 174422 +Job id 177592 TEST 5 compile is waiting to enter the queue -1 min. TEST 5 compile is pending, Status: PD -2 min. TEST 5 compile is pending, Status: PD -3 min. TEST 5 compile is pending, Status: PD -4 min. TEST 5 compile is pending, Status: PD -5 min. TEST 5 compile is pending, Status: PD -6 min. TEST 5 compile is pending, Status: PD -7 min. TEST 5 compile is pending, Status: PD -8 min. TEST 5 compile is pending, Status: PD -9 min. TEST 5 compile is pending, Status: PD -10 min. TEST 5 compile is pending, Status: PD -11 min. TEST 5 compile is running, Status: R -12 min. TEST 5 compile is running, Status: R +1 min. TEST 5 compile is running, Status: R +2 min. TEST 5 compile is running, Status: R Slurm unknown status -. Check sacct ... -174422 COMPLETED compile_5 -174422.batch COMPLETED batch -174422.exte+ COMPLETED extern -13 min. TEST 5 compile is COMPLETED +177592 COMPLETED compile_5 +177592.batch COMPLETED batch +177592.exte+ COMPLETED extern +3 min. TEST 5 compile is COMPLETED COMPLETED COMPLETED -Slurm unknown status -. Check sacct ... -174422 COMPLETED compile_5 -174422.batch COMPLETED batch -174422.exte+ COMPLETED extern -14 min. TEST 5 compile is COMPLETED -COMPLETED -COMPLETED -+ elapsed=846 -+ echo 'Elapsed time 846 seconds. Compile 5' -Elapsed time 846 seconds. Compile 5 ++ elapsed=185 ++ echo 'Elapsed time 185 seconds. Compile 5' +Elapsed time 185 seconds. Compile 5 + SECONDS=0 + [[ 4 != 4 ]] -+ export PATHRT=/work/noaa/fv3-cam/djovic/ufs/pr154/ufs-weather-model/tests -+ PATHRT=/work/noaa/fv3-cam/djovic/ufs/pr154/ufs-weather-model/tests -+ export RUNDIR_ROOT=/work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_369221 -+ RUNDIR_ROOT=/work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_369221 ++ export PATHRT=/work/noaa/fv3-cam/djovic/ufs/rt_ecf_suite/ufs-weather-model/tests ++ PATHRT=/work/noaa/fv3-cam/djovic/ufs/rt_ecf_suite/ufs-weather-model/tests ++ export RUNDIR_ROOT=/work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_339981 ++ RUNDIR_ROOT=/work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_339981 + export 'MAKE_OPT=CCPP=Y SUITES=FV3_GFS_2017_gfdlmp,FV3_GFS_2017_gfdlmp_noahmp' + MAKE_OPT='CCPP=Y SUITES=FV3_GFS_2017_gfdlmp,FV3_GFS_2017_gfdlmp_noahmp' + export COMPILE_NR=6 + COMPILE_NR=6 -+ cd /work/noaa/fv3-cam/djovic/ufs/pr154/ufs-weather-model/tests -+ [[ -e /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_369221/compile_6.env ]] -+ source /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_369221/compile_6.env ++ cd /work/noaa/fv3-cam/djovic/ufs/rt_ecf_suite/ufs-weather-model/tests ++ [[ -e /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_339981/compile_6.env ]] ++ source /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_339981/compile_6.env ++ export MACHINE_ID=orion.intel ++ MACHINE_ID=orion.intel -++ export PATHRT=/work/noaa/fv3-cam/djovic/ufs/pr154/ufs-weather-model/tests -++ PATHRT=/work/noaa/fv3-cam/djovic/ufs/pr154/ufs-weather-model/tests -++ export PATHTR=/work/noaa/fv3-cam/djovic/ufs/pr154/ufs-weather-model -++ PATHTR=/work/noaa/fv3-cam/djovic/ufs/pr154/ufs-weather-model +++ export PATHRT=/work/noaa/fv3-cam/djovic/ufs/rt_ecf_suite/ufs-weather-model/tests +++ PATHRT=/work/noaa/fv3-cam/djovic/ufs/rt_ecf_suite/ufs-weather-model/tests +++ export PATHTR=/work/noaa/fv3-cam/djovic/ufs/rt_ecf_suite/ufs-weather-model +++ PATHTR=/work/noaa/fv3-cam/djovic/ufs/rt_ecf_suite/ufs-weather-model ++ export SCHEDULER=slurm ++ SCHEDULER=slurm ++ export ACCNR=nems @@ -1151,10 +1073,10 @@ Elapsed time 846 seconds. Compile 5 ++ ROCOTO=false ++ export ECFLOW=true ++ ECFLOW=true -++ export REGRESSIONTEST_LOG=/work/noaa/fv3-cam/djovic/ufs/pr154/ufs-weather-model/tests/RegressionTests_orion.intel.log -++ REGRESSIONTEST_LOG=/work/noaa/fv3-cam/djovic/ufs/pr154/ufs-weather-model/tests/RegressionTests_orion.intel.log -++ export LOG_DIR=/work/noaa/fv3-cam/djovic/ufs/pr154/ufs-weather-model/tests/log_orion.intel -++ LOG_DIR=/work/noaa/fv3-cam/djovic/ufs/pr154/ufs-weather-model/tests/log_orion.intel +++ export REGRESSIONTEST_LOG=/work/noaa/fv3-cam/djovic/ufs/rt_ecf_suite/ufs-weather-model/tests/RegressionTests_orion.intel.log +++ REGRESSIONTEST_LOG=/work/noaa/fv3-cam/djovic/ufs/rt_ecf_suite/ufs-weather-model/tests/RegressionTests_orion.intel.log +++ export LOG_DIR=/work/noaa/fv3-cam/djovic/ufs/rt_ecf_suite/ufs-weather-model/tests/log_orion.intel +++ LOG_DIR=/work/noaa/fv3-cam/djovic/ufs/rt_ecf_suite/ufs-weather-model/tests/log_orion.intel + source default_vars.sh ++ '[' orion.intel = wcoss ']' ++ '[' orion.intel = wcoss_cray ']' @@ -1186,15 +1108,15 @@ Elapsed time 846 seconds. Compile 5 + TEST_NR=6 + export JBNME=compile_6 + JBNME=compile_6 -+ export RUNDIR=/work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_369221/compile_6 -+ RUNDIR=/work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_369221/compile_6 ++ export RUNDIR=/work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_339981/compile_6 ++ RUNDIR=/work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_339981/compile_6 + source rt_utils.sh ++ set -eu -++ [[ /work/noaa/fv3-cam/djovic/ufs/pr154/ufs-weather-model/tests/run_compile.sh = \r\t\_\u\t\i\l\s\.\s\h ]] +++ [[ /work/noaa/fv3-cam/djovic/ufs/rt_ecf_suite/ufs-weather-model/tests/run_compile.sh = \r\t\_\u\t\i\l\s\.\s\h ]] ++ UNIT_TEST=false + source atparse.bash -+ mkdir -p /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_369221/compile_6 -+ cd /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_369221/compile_6 ++ mkdir -p /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_339981/compile_6 ++ cd /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_339981/compile_6 + [[ slurm = \s\l\u\r\m ]] + atparse + local __set_x @@ -1207,60 +1129,43 @@ Elapsed time 846 seconds. Compile 5 + '[' -o xtrace ']' + set_x='set -x' + set +x -Job id 174424 +Job id 177591 TEST 6 compile is waiting to enter the queue -1 min. TEST 6 compile is pending, Status: PD -2 min. TEST 6 compile is pending, Status: PD -3 min. TEST 6 compile is pending, Status: PD -4 min. TEST 6 compile is pending, Status: PD -5 min. TEST 6 compile is pending, Status: PD -6 min. TEST 6 compile is pending, Status: PD -7 min. TEST 6 compile is pending, Status: PD -8 min. TEST 6 compile is pending, Status: PD -9 min. TEST 6 compile is pending, Status: PD -10 min. TEST 6 compile is pending, Status: PD -11 min. TEST 6 compile is pending, Status: PD -12 min. TEST 6 compile is pending, Status: PD -13 min. TEST 6 compile is pending, Status: PD -14 min. TEST 6 compile is pending, Status: PD -15 min. TEST 6 compile is pending, Status: PD -16 min. TEST 6 compile is pending, Status: PD -17 min. TEST 6 compile is running, Status: R -18 min. TEST 6 compile is running, Status: R -19 min. TEST 6 compile is running, Status: R -20 min. TEST 6 compile is running, Status: R -21 min. TEST 6 compile is running, Status: R -22 min. TEST 6 compile is running, Status: R -23 min. TEST 6 compile is running, Status: R +1 min. TEST 6 compile is running, Status: R +2 min. TEST 6 compile is running, Status: R +3 min. TEST 6 compile is running, Status: R +4 min. TEST 6 compile is running, Status: R +5 min. TEST 6 compile is running, Status: R +6 min. TEST 6 compile is running, Status: R Slurm unknown status -. Check sacct ... -174424 COMPLETED compile_6 -174424.batch COMPLETED batch -174424.exte+ COMPLETED extern -24 min. TEST 6 compile is COMPLETED +177591 COMPLETED compile_6 +177591.batch COMPLETED batch +177591.exte+ COMPLETED extern +7 min. TEST 6 compile is COMPLETED COMPLETED COMPLETED -+ elapsed=1450 -+ echo 'Elapsed time 1450 seconds. Compile 6' -Elapsed time 1450 seconds. Compile 6 ++ elapsed=426 ++ echo 'Elapsed time 426 seconds. Compile 6' +Elapsed time 426 seconds. Compile 6 + SECONDS=0 + [[ 4 != 4 ]] -+ export PATHRT=/work/noaa/fv3-cam/djovic/ufs/pr154/ufs-weather-model/tests -+ PATHRT=/work/noaa/fv3-cam/djovic/ufs/pr154/ufs-weather-model/tests -+ export RUNDIR_ROOT=/work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_369221 -+ RUNDIR_ROOT=/work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_369221 ++ export PATHRT=/work/noaa/fv3-cam/djovic/ufs/rt_ecf_suite/ufs-weather-model/tests ++ PATHRT=/work/noaa/fv3-cam/djovic/ufs/rt_ecf_suite/ufs-weather-model/tests ++ export RUNDIR_ROOT=/work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_339981 ++ RUNDIR_ROOT=/work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_339981 + export 'MAKE_OPT=CCPP=Y SUITES=FV3_GFS_2017_csawmgshoc,FV3_GFS_2017_csawmg,FV3_GFS_2017_satmedmf' + MAKE_OPT='CCPP=Y SUITES=FV3_GFS_2017_csawmgshoc,FV3_GFS_2017_csawmg,FV3_GFS_2017_satmedmf' + export COMPILE_NR=7 + COMPILE_NR=7 -+ cd /work/noaa/fv3-cam/djovic/ufs/pr154/ufs-weather-model/tests -+ [[ -e /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_369221/compile_7.env ]] -+ source /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_369221/compile_7.env ++ cd /work/noaa/fv3-cam/djovic/ufs/rt_ecf_suite/ufs-weather-model/tests ++ [[ -e /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_339981/compile_7.env ]] ++ source /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_339981/compile_7.env ++ export MACHINE_ID=orion.intel ++ MACHINE_ID=orion.intel -++ export PATHRT=/work/noaa/fv3-cam/djovic/ufs/pr154/ufs-weather-model/tests -++ PATHRT=/work/noaa/fv3-cam/djovic/ufs/pr154/ufs-weather-model/tests -++ export PATHTR=/work/noaa/fv3-cam/djovic/ufs/pr154/ufs-weather-model -++ PATHTR=/work/noaa/fv3-cam/djovic/ufs/pr154/ufs-weather-model +++ export PATHRT=/work/noaa/fv3-cam/djovic/ufs/rt_ecf_suite/ufs-weather-model/tests +++ PATHRT=/work/noaa/fv3-cam/djovic/ufs/rt_ecf_suite/ufs-weather-model/tests +++ export PATHTR=/work/noaa/fv3-cam/djovic/ufs/rt_ecf_suite/ufs-weather-model +++ PATHTR=/work/noaa/fv3-cam/djovic/ufs/rt_ecf_suite/ufs-weather-model ++ export SCHEDULER=slurm ++ SCHEDULER=slurm ++ export ACCNR=nems @@ -1273,10 +1178,10 @@ Elapsed time 1450 seconds. Compile 6 ++ ROCOTO=false ++ export ECFLOW=true ++ ECFLOW=true -++ export REGRESSIONTEST_LOG=/work/noaa/fv3-cam/djovic/ufs/pr154/ufs-weather-model/tests/RegressionTests_orion.intel.log -++ REGRESSIONTEST_LOG=/work/noaa/fv3-cam/djovic/ufs/pr154/ufs-weather-model/tests/RegressionTests_orion.intel.log -++ export LOG_DIR=/work/noaa/fv3-cam/djovic/ufs/pr154/ufs-weather-model/tests/log_orion.intel -++ LOG_DIR=/work/noaa/fv3-cam/djovic/ufs/pr154/ufs-weather-model/tests/log_orion.intel +++ export REGRESSIONTEST_LOG=/work/noaa/fv3-cam/djovic/ufs/rt_ecf_suite/ufs-weather-model/tests/RegressionTests_orion.intel.log +++ REGRESSIONTEST_LOG=/work/noaa/fv3-cam/djovic/ufs/rt_ecf_suite/ufs-weather-model/tests/RegressionTests_orion.intel.log +++ export LOG_DIR=/work/noaa/fv3-cam/djovic/ufs/rt_ecf_suite/ufs-weather-model/tests/log_orion.intel +++ LOG_DIR=/work/noaa/fv3-cam/djovic/ufs/rt_ecf_suite/ufs-weather-model/tests/log_orion.intel + source default_vars.sh ++ '[' orion.intel = wcoss ']' ++ '[' orion.intel = wcoss_cray ']' @@ -1308,15 +1213,15 @@ Elapsed time 1450 seconds. Compile 6 + TEST_NR=7 + export JBNME=compile_7 + JBNME=compile_7 -+ export RUNDIR=/work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_369221/compile_7 -+ RUNDIR=/work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_369221/compile_7 ++ export RUNDIR=/work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_339981/compile_7 ++ RUNDIR=/work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_339981/compile_7 + source rt_utils.sh ++ set -eu -++ [[ /work/noaa/fv3-cam/djovic/ufs/pr154/ufs-weather-model/tests/run_compile.sh = \r\t\_\u\t\i\l\s\.\s\h ]] +++ [[ /work/noaa/fv3-cam/djovic/ufs/rt_ecf_suite/ufs-weather-model/tests/run_compile.sh = \r\t\_\u\t\i\l\s\.\s\h ]] ++ UNIT_TEST=false + source atparse.bash -+ mkdir -p /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_369221/compile_7 -+ cd /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_369221/compile_7 ++ mkdir -p /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_339981/compile_7 ++ cd /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_339981/compile_7 + [[ slurm = \s\l\u\r\m ]] + atparse + local __set_x @@ -1329,60 +1234,43 @@ Elapsed time 1450 seconds. Compile 6 + '[' -o xtrace ']' + set_x='set -x' + set +x -Job id 174425 +Job id 177589 TEST 7 compile is waiting to enter the queue -1 min. TEST 7 compile is pending, Status: PD -2 min. TEST 7 compile is pending, Status: PD -3 min. TEST 7 compile is pending, Status: PD -4 min. TEST 7 compile is pending, Status: PD -5 min. TEST 7 compile is pending, Status: PD -6 min. TEST 7 compile is pending, Status: PD -7 min. TEST 7 compile is pending, Status: PD -8 min. TEST 7 compile is pending, Status: PD -9 min. TEST 7 compile is pending, Status: PD -10 min. TEST 7 compile is pending, Status: PD -11 min. TEST 7 compile is pending, Status: PD -12 min. TEST 7 compile is pending, Status: PD -13 min. TEST 7 compile is pending, Status: PD -14 min. TEST 7 compile is pending, Status: PD -15 min. TEST 7 compile is pending, Status: PD -16 min. TEST 7 compile is pending, Status: PD -17 min. TEST 7 compile is running, Status: R -18 min. TEST 7 compile is running, Status: R -19 min. TEST 7 compile is running, Status: R -20 min. TEST 7 compile is running, Status: R -21 min. TEST 7 compile is running, Status: R -22 min. TEST 7 compile is running, Status: R -23 min. TEST 7 compile is running, Status: R +1 min. TEST 7 compile is running, Status: R +2 min. TEST 7 compile is running, Status: R +3 min. TEST 7 compile is running, Status: R +4 min. TEST 7 compile is running, Status: R +5 min. TEST 7 compile is running, Status: R +6 min. TEST 7 compile is running, Status: R Slurm unknown status -. Check sacct ... -174425 COMPLETED compile_7 -174425.batch COMPLETED batch -174425.exte+ COMPLETED extern -24 min. TEST 7 compile is COMPLETED +177589 COMPLETED compile_7 +177589.batch COMPLETED batch +177589.exte+ COMPLETED extern +7 min. TEST 7 compile is COMPLETED COMPLETED COMPLETED -+ elapsed=1450 -+ echo 'Elapsed time 1450 seconds. Compile 7' -Elapsed time 1450 seconds. Compile 7 ++ elapsed=426 ++ echo 'Elapsed time 426 seconds. Compile 7' +Elapsed time 426 seconds. Compile 7 + SECONDS=0 + [[ 4 != 4 ]] -+ export PATHRT=/work/noaa/fv3-cam/djovic/ufs/pr154/ufs-weather-model/tests -+ PATHRT=/work/noaa/fv3-cam/djovic/ufs/pr154/ufs-weather-model/tests -+ export RUNDIR_ROOT=/work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_369221 -+ RUNDIR_ROOT=/work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_369221 ++ export PATHRT=/work/noaa/fv3-cam/djovic/ufs/rt_ecf_suite/ufs-weather-model/tests ++ PATHRT=/work/noaa/fv3-cam/djovic/ufs/rt_ecf_suite/ufs-weather-model/tests ++ export RUNDIR_ROOT=/work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_339981 ++ RUNDIR_ROOT=/work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_339981 + export 'MAKE_OPT=CCPP=Y SUITES=FV3_GFS_2017_gfdlmp,FV3_CPT_v0,FV3_GSD_v0,FV3_GFS_v15_thompson 32BIT=Y' + MAKE_OPT='CCPP=Y SUITES=FV3_GFS_2017_gfdlmp,FV3_CPT_v0,FV3_GSD_v0,FV3_GFS_v15_thompson 32BIT=Y' + export COMPILE_NR=8 + COMPILE_NR=8 -+ cd /work/noaa/fv3-cam/djovic/ufs/pr154/ufs-weather-model/tests -+ [[ -e /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_369221/compile_8.env ]] -+ source /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_369221/compile_8.env ++ cd /work/noaa/fv3-cam/djovic/ufs/rt_ecf_suite/ufs-weather-model/tests ++ [[ -e /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_339981/compile_8.env ]] ++ source /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_339981/compile_8.env ++ export MACHINE_ID=orion.intel ++ MACHINE_ID=orion.intel -++ export PATHRT=/work/noaa/fv3-cam/djovic/ufs/pr154/ufs-weather-model/tests -++ PATHRT=/work/noaa/fv3-cam/djovic/ufs/pr154/ufs-weather-model/tests -++ export PATHTR=/work/noaa/fv3-cam/djovic/ufs/pr154/ufs-weather-model -++ PATHTR=/work/noaa/fv3-cam/djovic/ufs/pr154/ufs-weather-model +++ export PATHRT=/work/noaa/fv3-cam/djovic/ufs/rt_ecf_suite/ufs-weather-model/tests +++ PATHRT=/work/noaa/fv3-cam/djovic/ufs/rt_ecf_suite/ufs-weather-model/tests +++ export PATHTR=/work/noaa/fv3-cam/djovic/ufs/rt_ecf_suite/ufs-weather-model +++ PATHTR=/work/noaa/fv3-cam/djovic/ufs/rt_ecf_suite/ufs-weather-model ++ export SCHEDULER=slurm ++ SCHEDULER=slurm ++ export ACCNR=nems @@ -1395,10 +1283,10 @@ Elapsed time 1450 seconds. Compile 7 ++ ROCOTO=false ++ export ECFLOW=true ++ ECFLOW=true -++ export REGRESSIONTEST_LOG=/work/noaa/fv3-cam/djovic/ufs/pr154/ufs-weather-model/tests/RegressionTests_orion.intel.log -++ REGRESSIONTEST_LOG=/work/noaa/fv3-cam/djovic/ufs/pr154/ufs-weather-model/tests/RegressionTests_orion.intel.log -++ export LOG_DIR=/work/noaa/fv3-cam/djovic/ufs/pr154/ufs-weather-model/tests/log_orion.intel -++ LOG_DIR=/work/noaa/fv3-cam/djovic/ufs/pr154/ufs-weather-model/tests/log_orion.intel +++ export REGRESSIONTEST_LOG=/work/noaa/fv3-cam/djovic/ufs/rt_ecf_suite/ufs-weather-model/tests/RegressionTests_orion.intel.log +++ REGRESSIONTEST_LOG=/work/noaa/fv3-cam/djovic/ufs/rt_ecf_suite/ufs-weather-model/tests/RegressionTests_orion.intel.log +++ export LOG_DIR=/work/noaa/fv3-cam/djovic/ufs/rt_ecf_suite/ufs-weather-model/tests/log_orion.intel +++ LOG_DIR=/work/noaa/fv3-cam/djovic/ufs/rt_ecf_suite/ufs-weather-model/tests/log_orion.intel + source default_vars.sh ++ '[' orion.intel = wcoss ']' ++ '[' orion.intel = wcoss_cray ']' @@ -1430,15 +1318,15 @@ Elapsed time 1450 seconds. Compile 7 + TEST_NR=8 + export JBNME=compile_8 + JBNME=compile_8 -+ export RUNDIR=/work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_369221/compile_8 -+ RUNDIR=/work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_369221/compile_8 ++ export RUNDIR=/work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_339981/compile_8 ++ RUNDIR=/work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_339981/compile_8 + source rt_utils.sh ++ set -eu -++ [[ /work/noaa/fv3-cam/djovic/ufs/pr154/ufs-weather-model/tests/run_compile.sh = \r\t\_\u\t\i\l\s\.\s\h ]] +++ [[ /work/noaa/fv3-cam/djovic/ufs/rt_ecf_suite/ufs-weather-model/tests/run_compile.sh = \r\t\_\u\t\i\l\s\.\s\h ]] ++ UNIT_TEST=false + source atparse.bash -+ mkdir -p /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_369221/compile_8 -+ cd /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_369221/compile_8 ++ mkdir -p /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_339981/compile_8 ++ cd /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_339981/compile_8 + [[ slurm = \s\l\u\r\m ]] + atparse + local __set_x @@ -1451,60 +1339,43 @@ Elapsed time 1450 seconds. Compile 7 + '[' -o xtrace ']' + set_x='set -x' + set +x -Job id 174423 +Job id 177587 TEST 8 compile is waiting to enter the queue -1 min. TEST 8 compile is pending, Status: PD -2 min. TEST 8 compile is pending, Status: PD -3 min. TEST 8 compile is pending, Status: PD -4 min. TEST 8 compile is pending, Status: PD -5 min. TEST 8 compile is pending, Status: PD -6 min. TEST 8 compile is pending, Status: PD -7 min. TEST 8 compile is pending, Status: PD -8 min. TEST 8 compile is pending, Status: PD -9 min. TEST 8 compile is pending, Status: PD -10 min. TEST 8 compile is pending, Status: PD -11 min. TEST 8 compile is pending, Status: PD -12 min. TEST 8 compile is pending, Status: PD -13 min. TEST 8 compile is pending, Status: PD -14 min. TEST 8 compile is pending, Status: PD -15 min. TEST 8 compile is pending, Status: PD -16 min. TEST 8 compile is pending, Status: PD -17 min. TEST 8 compile is running, Status: R -18 min. TEST 8 compile is running, Status: R -19 min. TEST 8 compile is running, Status: R -20 min. TEST 8 compile is running, Status: R -21 min. TEST 8 compile is running, Status: R -22 min. TEST 8 compile is running, Status: R -23 min. TEST 8 compile is running, Status: R +1 min. TEST 8 compile is running, Status: R +2 min. TEST 8 compile is running, Status: R +3 min. TEST 8 compile is running, Status: R +4 min. TEST 8 compile is running, Status: R +5 min. TEST 8 compile is running, Status: R +6 min. TEST 8 compile is running, Status: R Slurm unknown status -. Check sacct ... -174423 COMPLETED compile_8 -174423.batch COMPLETED batch -174423.exte+ COMPLETED extern -24 min. TEST 8 compile is COMPLETED +177587 COMPLETED compile_8 +177587.batch COMPLETED batch +177587.exte+ COMPLETED extern +7 min. TEST 8 compile is COMPLETED COMPLETED COMPLETED -+ elapsed=1450 -+ echo 'Elapsed time 1450 seconds. Compile 8' -Elapsed time 1450 seconds. Compile 8 ++ elapsed=426 ++ echo 'Elapsed time 426 seconds. Compile 8' +Elapsed time 426 seconds. Compile 8 + SECONDS=0 + [[ 4 != 4 ]] -+ export PATHRT=/work/noaa/fv3-cam/djovic/ufs/pr154/ufs-weather-model/tests -+ PATHRT=/work/noaa/fv3-cam/djovic/ufs/pr154/ufs-weather-model/tests -+ export RUNDIR_ROOT=/work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_369221 -+ RUNDIR_ROOT=/work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_369221 ++ export PATHRT=/work/noaa/fv3-cam/djovic/ufs/rt_ecf_suite/ufs-weather-model/tests ++ PATHRT=/work/noaa/fv3-cam/djovic/ufs/rt_ecf_suite/ufs-weather-model/tests ++ export RUNDIR_ROOT=/work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_339981 ++ RUNDIR_ROOT=/work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_339981 + export 'MAKE_OPT=CCPP=Y SUITES=FV3_GFS_v15p2,FV3_GFS_v16beta,FV3_GFS_2017_RRTMGP' + MAKE_OPT='CCPP=Y SUITES=FV3_GFS_v15p2,FV3_GFS_v16beta,FV3_GFS_2017_RRTMGP' + export COMPILE_NR=9 + COMPILE_NR=9 -+ cd /work/noaa/fv3-cam/djovic/ufs/pr154/ufs-weather-model/tests -+ [[ -e /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_369221/compile_9.env ]] -+ source /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_369221/compile_9.env ++ cd /work/noaa/fv3-cam/djovic/ufs/rt_ecf_suite/ufs-weather-model/tests ++ [[ -e /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_339981/compile_9.env ]] ++ source /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_339981/compile_9.env ++ export MACHINE_ID=orion.intel ++ MACHINE_ID=orion.intel -++ export PATHRT=/work/noaa/fv3-cam/djovic/ufs/pr154/ufs-weather-model/tests -++ PATHRT=/work/noaa/fv3-cam/djovic/ufs/pr154/ufs-weather-model/tests -++ export PATHTR=/work/noaa/fv3-cam/djovic/ufs/pr154/ufs-weather-model -++ PATHTR=/work/noaa/fv3-cam/djovic/ufs/pr154/ufs-weather-model +++ export PATHRT=/work/noaa/fv3-cam/djovic/ufs/rt_ecf_suite/ufs-weather-model/tests +++ PATHRT=/work/noaa/fv3-cam/djovic/ufs/rt_ecf_suite/ufs-weather-model/tests +++ export PATHTR=/work/noaa/fv3-cam/djovic/ufs/rt_ecf_suite/ufs-weather-model +++ PATHTR=/work/noaa/fv3-cam/djovic/ufs/rt_ecf_suite/ufs-weather-model ++ export SCHEDULER=slurm ++ SCHEDULER=slurm ++ export ACCNR=nems @@ -1517,10 +1388,10 @@ Elapsed time 1450 seconds. Compile 8 ++ ROCOTO=false ++ export ECFLOW=true ++ ECFLOW=true -++ export REGRESSIONTEST_LOG=/work/noaa/fv3-cam/djovic/ufs/pr154/ufs-weather-model/tests/RegressionTests_orion.intel.log -++ REGRESSIONTEST_LOG=/work/noaa/fv3-cam/djovic/ufs/pr154/ufs-weather-model/tests/RegressionTests_orion.intel.log -++ export LOG_DIR=/work/noaa/fv3-cam/djovic/ufs/pr154/ufs-weather-model/tests/log_orion.intel -++ LOG_DIR=/work/noaa/fv3-cam/djovic/ufs/pr154/ufs-weather-model/tests/log_orion.intel +++ export REGRESSIONTEST_LOG=/work/noaa/fv3-cam/djovic/ufs/rt_ecf_suite/ufs-weather-model/tests/RegressionTests_orion.intel.log +++ REGRESSIONTEST_LOG=/work/noaa/fv3-cam/djovic/ufs/rt_ecf_suite/ufs-weather-model/tests/RegressionTests_orion.intel.log +++ export LOG_DIR=/work/noaa/fv3-cam/djovic/ufs/rt_ecf_suite/ufs-weather-model/tests/log_orion.intel +++ LOG_DIR=/work/noaa/fv3-cam/djovic/ufs/rt_ecf_suite/ufs-weather-model/tests/log_orion.intel + source default_vars.sh ++ '[' orion.intel = wcoss ']' ++ '[' orion.intel = wcoss_cray ']' @@ -1552,15 +1423,15 @@ Elapsed time 1450 seconds. Compile 8 + TEST_NR=9 + export JBNME=compile_9 + JBNME=compile_9 -+ export RUNDIR=/work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_369221/compile_9 -+ RUNDIR=/work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_369221/compile_9 ++ export RUNDIR=/work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_339981/compile_9 ++ RUNDIR=/work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_339981/compile_9 + source rt_utils.sh ++ set -eu -++ [[ /work/noaa/fv3-cam/djovic/ufs/pr154/ufs-weather-model/tests/run_compile.sh = \r\t\_\u\t\i\l\s\.\s\h ]] +++ [[ /work/noaa/fv3-cam/djovic/ufs/rt_ecf_suite/ufs-weather-model/tests/run_compile.sh = \r\t\_\u\t\i\l\s\.\s\h ]] ++ UNIT_TEST=false + source atparse.bash -+ mkdir -p /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_369221/compile_9 -+ cd /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_369221/compile_9 ++ mkdir -p /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_339981/compile_9 ++ cd /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_339981/compile_9 + [[ slurm = \s\l\u\r\m ]] + atparse + local __set_x @@ -1573,30 +1444,21 @@ Elapsed time 1450 seconds. Compile 8 + '[' -o xtrace ']' + set_x='set -x' + set +x -Job id 174421 +Job id 177588 TEST 9 compile is waiting to enter the queue -1 min. TEST 9 compile is pending, Status: PD -2 min. TEST 9 compile is pending, Status: PD -3 min. TEST 9 compile is pending, Status: PD -4 min. TEST 9 compile is pending, Status: PD -5 min. TEST 9 compile is pending, Status: PD -6 min. TEST 9 compile is pending, Status: PD -7 min. TEST 9 compile is pending, Status: PD -8 min. TEST 9 compile is pending, Status: PD -9 min. TEST 9 compile is pending, Status: PD -10 min. TEST 9 compile is running, Status: R -11 min. TEST 9 compile is running, Status: R -12 min. TEST 9 compile is running, Status: R -13 min. TEST 9 compile is running, Status: R -14 min. TEST 9 compile is running, Status: R -15 min. TEST 9 compile is running, Status: R +1 min. TEST 9 compile is running, Status: R +2 min. TEST 9 compile is running, Status: R +3 min. TEST 9 compile is running, Status: R +4 min. TEST 9 compile is running, Status: R +5 min. TEST 9 compile is running, Status: R +6 min. TEST 9 compile is running, Status: R Slurm unknown status -. Check sacct ... -174421 COMPLETED compile_9 -174421.batch COMPLETED batch -174421.exte+ COMPLETED extern -16 min. TEST 9 compile is COMPLETED +177588 COMPLETED compile_9 +177588.batch COMPLETED batch +177588.exte+ COMPLETED extern +7 min. TEST 9 compile is COMPLETED COMPLETED COMPLETED -+ elapsed=968 -+ echo 'Elapsed time 968 seconds. Compile 9' -Elapsed time 968 seconds. Compile 9 ++ elapsed=426 ++ echo 'Elapsed time 426 seconds. Compile 9' +Elapsed time 426 seconds. Compile 9 diff --git a/tests/RegressionTests_orion.intel.log b/tests/RegressionTests_orion.intel.log index f1426eb00d..4ead851a8f 100644 --- a/tests/RegressionTests_orion.intel.log +++ b/tests/RegressionTests_orion.intel.log @@ -1,9 +1,9 @@ -Tue Jul 14 08:27:38 CDT 2020 +Wed Jul 15 09:13:28 CDT 2020 Start Regression test baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200713/INTEL/fv3_control_ccpp -working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_369221/fv3_ccpp_control_prod +working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_339981/fv3_ccpp_control_prod Checking test 001 fv3_ccpp_control results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -71,7 +71,7 @@ Test 001 fv3_ccpp_control PASS baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200713/INTEL/fv3_control_ccpp -working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_369221/fv3_ccpp_decomp_prod +working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_339981/fv3_ccpp_decomp_prod Checking test 002 fv3_ccpp_decomp results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -139,7 +139,7 @@ Test 002 fv3_ccpp_decomp PASS baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200713/INTEL/fv3_control_ccpp -working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_369221/fv3_ccpp_2threads_prod +working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_339981/fv3_ccpp_2threads_prod Checking test 003 fv3_ccpp_2threads results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -207,7 +207,7 @@ Test 003 fv3_ccpp_2threads PASS baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200713/INTEL/fv3_restart_ccpp -working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_369221/fv3_ccpp_restart_prod +working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_339981/fv3_ccpp_restart_prod Checking test 004 fv3_ccpp_restart results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -275,7 +275,7 @@ Test 004 fv3_ccpp_restart PASS baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200713/INTEL/fv3_read_inc_ccpp -working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_369221/fv3_ccpp_read_inc_prod +working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_339981/fv3_ccpp_read_inc_prod Checking test 005 fv3_ccpp_read_inc results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -343,7 +343,7 @@ Test 005 fv3_ccpp_read_inc PASS baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200713/INTEL/fv3_wrtGauss_netcdf_esmf_ccpp -working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_369221/fv3_ccpp_wrtGauss_netcdf_esmf_prod +working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_339981/fv3_ccpp_wrtGauss_netcdf_esmf_prod Checking test 006 fv3_ccpp_wrtGauss_netcdf_esmf results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -391,7 +391,7 @@ Test 006 fv3_ccpp_wrtGauss_netcdf_esmf PASS baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200713/INTEL/fv3_wrtGauss_netcdf_ccpp -working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_369221/fv3_ccpp_wrtGauss_netcdf_prod +working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_339981/fv3_ccpp_wrtGauss_netcdf_prod Checking test 007 fv3_ccpp_wrtGauss_netcdf results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -439,7 +439,7 @@ Test 007 fv3_ccpp_wrtGauss_netcdf PASS baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200713/INTEL/fv3_wrtGauss_nemsio_ccpp -working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_369221/fv3_ccpp_wrtGauss_nemsio_prod +working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_339981/fv3_ccpp_wrtGauss_nemsio_prod Checking test 008 fv3_ccpp_wrtGauss_nemsio results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -487,7 +487,7 @@ Test 008 fv3_ccpp_wrtGauss_nemsio PASS baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200713/INTEL/fv3_wrtGauss_nemsio_c192_ccpp -working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_369221/fv3_ccpp_wrtGauss_nemsio_c192_prod +working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_339981/fv3_ccpp_wrtGauss_nemsio_c192_prod Checking test 009 fv3_ccpp_wrtGauss_nemsio_c192 results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -535,7 +535,7 @@ Test 009 fv3_ccpp_wrtGauss_nemsio_c192 PASS baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200713/INTEL/fv3_stochy_ccpp -working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_369221/fv3_ccpp_stochy_prod +working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_339981/fv3_ccpp_stochy_prod Checking test 010 fv3_ccpp_stochy results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -603,7 +603,7 @@ Test 010 fv3_ccpp_stochy PASS baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200713/INTEL/fv3_iau_ccpp -working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_369221/fv3_ccpp_iau_prod +working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_339981/fv3_ccpp_iau_prod Checking test 011 fv3_ccpp_iau results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -671,7 +671,7 @@ Test 011 fv3_ccpp_iau PASS baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200713/INTEL/fv3_ca_ccpp -working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_369221/fv3_ccpp_ca_prod +working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_339981/fv3_ccpp_ca_prod Checking test 012 fv3_ccpp_ca results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -739,7 +739,7 @@ Test 012 fv3_ccpp_ca PASS baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200713/INTEL/fv3_gfdlmprad_ccpp -working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_369221/fv3_ccpp_gfdlmprad_prod +working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_339981/fv3_ccpp_gfdlmprad_prod Checking test 013 fv3_ccpp_gfdlmprad results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -788,7 +788,7 @@ Test 013 fv3_ccpp_gfdlmprad PASS baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200713/INTEL/fv3_gfdlmprad_atmwav_ccpp -working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_369221/fv3_ccpp_gfdlmprad_atmwav_prod +working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_339981/fv3_ccpp_gfdlmprad_atmwav_prod Checking test 014 fv3_ccpp_gfdlmprad_atmwav results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -837,7 +837,7 @@ Test 014 fv3_ccpp_gfdlmprad_atmwav PASS baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200713/INTEL/fv3_wrtGauss_nemsio_c768_ccpp -working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_369221/fv3_ccpp_wrtGauss_nemsio_c768_prod +working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_339981/fv3_ccpp_wrtGauss_nemsio_c768_prod Checking test 015 fv3_ccpp_wrtGauss_nemsio_c768 results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -886,7 +886,7 @@ Test 015 fv3_ccpp_wrtGauss_nemsio_c768 PASS baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200713/INTEL/fv3_control_32bit_ccpp -working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_369221/fv3_ccpp_control_32bit_prod +working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_339981/fv3_ccpp_control_32bit_prod Checking test 016 fv3_ccpp_control_32bit results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -954,7 +954,7 @@ Test 016 fv3_ccpp_control_32bit PASS baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200713/INTEL/fv3_stretched_ccpp -working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_369221/fv3_ccpp_stretched_prod +working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_339981/fv3_ccpp_stretched_prod Checking test 017 fv3_ccpp_stretched results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1010,7 +1010,7 @@ Test 017 fv3_ccpp_stretched PASS baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200713/INTEL/fv3_stretched_nest_ccpp -working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_369221/fv3_ccpp_stretched_nest_prod +working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_339981/fv3_ccpp_stretched_nest_prod Checking test 018 fv3_ccpp_stretched_nest results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1077,7 +1077,7 @@ Test 018 fv3_ccpp_stretched_nest PASS baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200713/INTEL/fv3_regional_control_ccpp -working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_369221/fv3_ccpp_regional_control_prod +working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_339981/fv3_ccpp_regional_control_prod Checking test 019 fv3_ccpp_regional_control results .... Comparing atmos_4xdaily.nc .........OK Comparing fv3_history2d.nc .........OK @@ -1088,7 +1088,7 @@ Test 019 fv3_ccpp_regional_control PASS baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200713/INTEL/fv3_regional_restart_ccpp -working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_369221/fv3_ccpp_regional_restart_prod +working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_339981/fv3_ccpp_regional_restart_prod Checking test 020 fv3_ccpp_regional_restart results .... Comparing atmos_4xdaily.nc .........OK Comparing fv3_history2d.nc .........OK @@ -1097,7 +1097,7 @@ Test 020 fv3_ccpp_regional_restart PASS baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200713/INTEL/fv3_regional_quilt_ccpp -working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_369221/fv3_ccpp_regional_quilt_prod +working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_339981/fv3_ccpp_regional_quilt_prod Checking test 021 fv3_ccpp_regional_quilt results .... Comparing atmos_4xdaily.nc .........OK Comparing dynf000.nc .........OK @@ -1108,7 +1108,7 @@ Test 021 fv3_ccpp_regional_quilt PASS baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200713/INTEL/fv3_regional_c768_ccpp -working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_369221/fv3_ccpp_regional_c768_prod +working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_339981/fv3_ccpp_regional_c768_prod Checking test 022 fv3_ccpp_regional_c768 results .... Comparing atmos_4xdaily.nc .........OK Comparing dynf000.nc .........OK @@ -1119,19 +1119,19 @@ Test 022 fv3_ccpp_regional_c768 PASS baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200713/INTEL/fv3_control_ccpp -working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_369221/fv3_ccpp_control_debug_prod +working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_339981/fv3_ccpp_control_debug_prod Checking test 023 fv3_ccpp_control_debug results .... Test 023 fv3_ccpp_control_debug PASS baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200713/INTEL/fv3_stretched_nest_ccpp -working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_369221/fv3_ccpp_stretched_nest_debug_prod +working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_339981/fv3_ccpp_stretched_nest_debug_prod Checking test 024 fv3_ccpp_stretched_nest_debug results .... Test 024 fv3_ccpp_stretched_nest_debug PASS baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200713/INTEL/fv3_gfdlmp_ccpp -working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_369221/fv3_ccpp_gfdlmp_prod +working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_339981/fv3_ccpp_gfdlmp_prod Checking test 025 fv3_ccpp_gfdlmp results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1179,7 +1179,7 @@ Test 025 fv3_ccpp_gfdlmp PASS baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200713/INTEL/fv3_gfdlmprad_gwd_ccpp -working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_369221/fv3_ccpp_gfdlmprad_gwd_prod +working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_339981/fv3_ccpp_gfdlmprad_gwd_prod Checking test 026 fv3_ccpp_gfdlmprad_gwd results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1227,7 +1227,7 @@ Test 026 fv3_ccpp_gfdlmprad_gwd PASS baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200713/INTEL/fv3_gfdlmprad_noahmp_ccpp -working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_369221/fv3_ccpp_gfdlmprad_noahmp_prod +working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_339981/fv3_ccpp_gfdlmprad_noahmp_prod Checking test 027 fv3_ccpp_gfdlmprad_noahmp results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1275,7 +1275,7 @@ Test 027 fv3_ccpp_gfdlmprad_noahmp PASS baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200713/INTEL/fv3_csawmg_ccpp -working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_369221/fv3_ccpp_csawmg_prod +working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_339981/fv3_ccpp_csawmg_prod Checking test 028 fv3_ccpp_csawmg results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1323,7 +1323,7 @@ Test 028 fv3_ccpp_csawmg PASS baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200713/INTEL/fv3_satmedmf_ccpp -working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_369221/fv3_ccpp_satmedmf_prod +working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_339981/fv3_ccpp_satmedmf_prod Checking test 029 fv3_ccpp_satmedmf results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1371,7 +1371,7 @@ Test 029 fv3_ccpp_satmedmf PASS baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200713/INTEL/fv3_gfdlmp_32bit_ccpp -working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_369221/fv3_ccpp_gfdlmp_32bit_prod +working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_339981/fv3_ccpp_gfdlmp_32bit_prod Checking test 030 fv3_ccpp_gfdlmp_32bit results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1419,7 +1419,7 @@ Test 030 fv3_ccpp_gfdlmp_32bit PASS baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200713/INTEL/fv3_gfdlmprad_32bit_post_ccpp -working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_369221/fv3_ccpp_gfdlmprad_32bit_post_prod +working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_339981/fv3_ccpp_gfdlmprad_32bit_post_prod Checking test 031 fv3_ccpp_gfdlmprad_32bit_post results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1471,7 +1471,7 @@ Test 031 fv3_ccpp_gfdlmprad_32bit_post PASS baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200713/INTEL/fv3_cpt_ccpp -working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_369221/fv3_ccpp_cpt_prod +working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_339981/fv3_ccpp_cpt_prod Checking test 032 fv3_ccpp_cpt results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1525,7 +1525,7 @@ Test 032 fv3_ccpp_cpt PASS baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200713/INTEL/fv3_gsd_ccpp -working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_369221/fv3_ccpp_gsd_prod +working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_339981/fv3_ccpp_gsd_prod Checking test 033 fv3_ccpp_gsd results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1617,7 +1617,7 @@ Test 033 fv3_ccpp_gsd PASS baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200713/INTEL/fv3_thompson_ccpp -working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_369221/fv3_ccpp_thompson_prod +working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_339981/fv3_ccpp_thompson_prod Checking test 034 fv3_ccpp_thompson results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1685,7 +1685,7 @@ Test 034 fv3_ccpp_thompson PASS baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200713/INTEL/fv3_thompson_no_aero_ccpp -working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_369221/fv3_ccpp_thompson_no_aero_prod +working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_339981/fv3_ccpp_thompson_no_aero_prod Checking test 035 fv3_ccpp_thompson_no_aero results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1753,7 +1753,7 @@ Test 035 fv3_ccpp_thompson_no_aero PASS baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200713/INTEL/fv3_gfs_v15p2_ccpp -working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_369221/fv3_ccpp_gfs_v15p2_prod +working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_339981/fv3_ccpp_gfs_v15p2_prod Checking test 036 fv3_ccpp_gfs_v15p2 results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1821,7 +1821,7 @@ Test 036 fv3_ccpp_gfs_v15p2 PASS baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200713/INTEL/fv3_gfs_v16beta_ccpp -working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_369221/fv3_ccpp_gfs_v16beta_prod +working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_339981/fv3_ccpp_gfs_v16beta_prod Checking test 037 fv3_ccpp_gfs_v16beta results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1889,7 +1889,7 @@ Test 037 fv3_ccpp_gfs_v16beta PASS baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200713/INTEL/fv3_rrtmgp_ccpp -working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_369221/fv3_ccpp_rrtmgp_prod +working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_339981/fv3_ccpp_rrtmgp_prod Checking test 038 fv3_ccpp_rrtmgp results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1957,7 +1957,7 @@ Test 038 fv3_ccpp_rrtmgp PASS baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200713/INTEL/fv3_gfs_v15p2_debug_ccpp -working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_369221/fv3_ccpp_gfs_v15p2_debug_prod +working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_339981/fv3_ccpp_gfs_v15p2_debug_prod Checking test 039 fv3_ccpp_gfs_v15p2_debug results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -2025,7 +2025,7 @@ Test 039 fv3_ccpp_gfs_v15p2_debug PASS baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200713/INTEL/fv3_gfs_v16beta_debug_ccpp -working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_369221/fv3_ccpp_gfs_v16beta_debug_prod +working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_339981/fv3_ccpp_gfs_v16beta_debug_prod Checking test 040 fv3_ccpp_gfs_v16beta_debug results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -2093,7 +2093,7 @@ Test 040 fv3_ccpp_gfs_v16beta_debug PASS baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200713/INTEL/fv3_rrtmgp_debug_ccpp -working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_369221/fv3_ccpp_rrtmgp_debug_prod +working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_339981/fv3_ccpp_rrtmgp_debug_prod Checking test 041 fv3_ccpp_rrtmgp_debug results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -2161,7 +2161,7 @@ Test 041 fv3_ccpp_rrtmgp_debug PASS baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200713/INTEL/fv3_gsd_debug_ccpp -working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_369221/fv3_ccpp_gsd_debug_prod +working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_339981/fv3_ccpp_gsd_debug_prod Checking test 042 fv3_ccpp_gsd_debug results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -2229,7 +2229,7 @@ Test 042 fv3_ccpp_gsd_debug PASS baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200713/INTEL/fv3_thompson_debug_ccpp -working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_369221/fv3_ccpp_thompson_debug_prod +working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_339981/fv3_ccpp_thompson_debug_prod Checking test 043 fv3_ccpp_thompson_debug results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -2297,7 +2297,7 @@ Test 043 fv3_ccpp_thompson_debug PASS baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200713/INTEL/fv3_thompson_no_aero_debug_ccpp -working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_369221/fv3_ccpp_thompson_no_aero_debug_prod +working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_339981/fv3_ccpp_thompson_no_aero_debug_prod Checking test 044 fv3_ccpp_thompson_no_aero_debug results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -2365,7 +2365,7 @@ Test 044 fv3_ccpp_thompson_no_aero_debug PASS baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200713/INTEL/fv3_gfsv16_csawmg_ccpp -working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_369221/fv3_ccpp_gfsv16_csawmg_prod +working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_339981/fv3_ccpp_gfsv16_csawmg_prod Checking test 045 fv3_ccpp_gfsv16_csawmg results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -2413,7 +2413,7 @@ Test 045 fv3_ccpp_gfsv16_csawmg PASS baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200713/INTEL/fv3_gfsv16_csawmgt_ccpp -working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_369221/fv3_ccpp_gfsv16_csawmgt_prod +working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_339981/fv3_ccpp_gfsv16_csawmgt_prod Checking test 046 fv3_ccpp_gfsv16_csawmgt results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -2461,7 +2461,7 @@ Test 046 fv3_ccpp_gfsv16_csawmgt PASS baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200713/INTEL/fv3_gocart_clm_ccpp -working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_369221/fv3_ccpp_gocart_clm_prod +working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_339981/fv3_ccpp_gocart_clm_prod Checking test 047 fv3_ccpp_gocart_clm results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -2509,7 +2509,7 @@ Test 047 fv3_ccpp_gocart_clm PASS baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200713/INTEL/fv3_multigases_ccpp -working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_369221/fv3_ccpp_multigases_prod +working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_339981/fv3_ccpp_multigases_prod Checking test 048 fv3_ccpp_multigases results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -2583,5 +2583,5 @@ Test 048 fv3_ccpp_multigases PASS REGRESSION TEST WAS SUCCESSFUL -Tue Jul 14 09:14:16 CDT 2020 -Elapsed time: 00h:46m:39s. Have a nice day! +Wed Jul 15 09:44:41 CDT 2020 +Elapsed time: 00h:31m:14s. Have a nice day! From ca9ab4f0943311a3fa6e056390cbdf18a197387b Mon Sep 17 00:00:00 2001 From: Dusan Jovic Date: Wed, 15 Jul 2020 15:43:52 +0000 Subject: [PATCH 10/11] Add regtest logs on Hera --- tests/Compile_hera.gnu.log | 434 ++++++++----- tests/Compile_hera.intel.log | 924 +++++++++++++-------------- tests/RegressionTests_hera.gnu.log | 30 +- tests/RegressionTests_hera.intel.log | 116 ++-- 4 files changed, 781 insertions(+), 723 deletions(-) diff --git a/tests/Compile_hera.gnu.log b/tests/Compile_hera.gnu.log index 148c87dd67..6639751729 100644 --- a/tests/Compile_hera.gnu.log +++ b/tests/Compile_hera.gnu.log @@ -1,22 +1,22 @@ + SECONDS=0 + [[ 4 != 4 ]] -+ export PATHRT=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/gnu/ufs-weather-model/tests -+ PATHRT=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/gnu/ufs-weather-model/tests -+ export RUNDIR_ROOT=/scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_196893 -+ RUNDIR_ROOT=/scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_196893 ++ export PATHRT=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests ++ PATHRT=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests ++ export RUNDIR_ROOT=/scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_130720 ++ RUNDIR_ROOT=/scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_130720 + export 'MAKE_OPT=CCPP=Y SUITES=FV3_GFS_2017_gfdlmp' + MAKE_OPT='CCPP=Y SUITES=FV3_GFS_2017_gfdlmp' + export COMPILE_NR=1 + COMPILE_NR=1 -+ cd /scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/gnu/ufs-weather-model/tests -+ [[ -e /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_196893/compile_1.env ]] -+ source /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_196893/compile_1.env ++ cd /scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests ++ [[ -e /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_130720/compile_1.env ]] ++ source /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_130720/compile_1.env ++ export MACHINE_ID=hera.gnu ++ MACHINE_ID=hera.gnu -++ export PATHRT=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/gnu/ufs-weather-model/tests -++ PATHRT=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/gnu/ufs-weather-model/tests -++ export PATHTR=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/gnu/ufs-weather-model -++ PATHTR=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/gnu/ufs-weather-model +++ export PATHRT=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests +++ PATHRT=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests +++ export PATHTR=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model +++ PATHTR=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model ++ export SCHEDULER=slurm ++ SCHEDULER=slurm ++ export ACCNR=nems @@ -29,10 +29,10 @@ ++ ROCOTO=false ++ export ECFLOW=true ++ ECFLOW=true -++ export REGRESSIONTEST_LOG=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/gnu/ufs-weather-model/tests/RegressionTests_hera.gnu.log -++ REGRESSIONTEST_LOG=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/gnu/ufs-weather-model/tests/RegressionTests_hera.gnu.log -++ export LOG_DIR=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/gnu/ufs-weather-model/tests/log_hera.gnu -++ LOG_DIR=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/gnu/ufs-weather-model/tests/log_hera.gnu +++ export REGRESSIONTEST_LOG=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/RegressionTests_hera.gnu.log +++ REGRESSIONTEST_LOG=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/RegressionTests_hera.gnu.log +++ export LOG_DIR=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/log_hera.gnu +++ LOG_DIR=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/log_hera.gnu + source default_vars.sh ++ '[' hera.gnu = wcoss ']' ++ '[' hera.gnu = wcoss_cray ']' @@ -64,15 +64,15 @@ + TEST_NR=1 + export JBNME=compile_1 + JBNME=compile_1 -+ export RUNDIR=/scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_196893/compile_1 -+ RUNDIR=/scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_196893/compile_1 ++ export RUNDIR=/scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_130720/compile_1 ++ RUNDIR=/scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_130720/compile_1 + source rt_utils.sh ++ set -eu -++ [[ /scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/gnu/ufs-weather-model/tests/run_compile.sh = \r\t\_\u\t\i\l\s\.\s\h ]] +++ [[ /scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/run_compile.sh = \r\t\_\u\t\i\l\s\.\s\h ]] ++ UNIT_TEST=false + source atparse.bash -+ mkdir -p /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_196893/compile_1 -+ cd /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_196893/compile_1 ++ mkdir -p /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_130720/compile_1 ++ cd /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_130720/compile_1 + [[ slurm = \s\l\u\r\m ]] + atparse + local __set_x @@ -85,39 +85,44 @@ + '[' -o xtrace ']' + set_x='set -x' + set +x -Job id 9562152 +Job id 9592106 TEST 1 compile is waiting to enter the queue -1 min. TEST 1 compile is running, Status: R -2 min. TEST 1 compile is running, Status: R +1 min. TEST 1 compile is pending, Status: PD +2 min. TEST 1 compile is pending, Status: PD +3 min. TEST 1 compile is pending, Status: PD +4 min. TEST 1 compile is pending, Status: PD +5 min. TEST 1 compile is running, Status: R +6 min. TEST 1 compile is running, Status: R +7 min. TEST 1 compile is running, Status: R Slurm unknown status -. Check sacct ... -9562152 COMPLETED compile_1 -9562152.bat+ COMPLETED batch -9562152.ext+ COMPLETED extern -3 min. TEST 1 compile is COMPLETED +9592106 COMPLETED compile_1 +9592106.bat+ COMPLETED batch +9592106.ext+ COMPLETED extern +8 min. TEST 1 compile is COMPLETED COMPLETED COMPLETED -+ elapsed=185 -+ echo 'Elapsed time 185 seconds. Compile 1' -Elapsed time 185 seconds. Compile 1 ++ elapsed=487 ++ echo 'Elapsed time 487 seconds. Compile 1' +Elapsed time 487 seconds. Compile 1 + SECONDS=0 + [[ 4 != 4 ]] -+ export PATHRT=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/gnu/ufs-weather-model/tests -+ PATHRT=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/gnu/ufs-weather-model/tests -+ export RUNDIR_ROOT=/scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_196893 -+ RUNDIR_ROOT=/scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_196893 ++ export PATHRT=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests ++ PATHRT=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests ++ export RUNDIR_ROOT=/scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_130720 ++ RUNDIR_ROOT=/scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_130720 + export 'MAKE_OPT=CCPP=Y SUITES=FV3_GFS_v15p2,FV3_GFS_v16beta,FV3_GFS_2017_RRTMGP' + MAKE_OPT='CCPP=Y SUITES=FV3_GFS_v15p2,FV3_GFS_v16beta,FV3_GFS_2017_RRTMGP' + export COMPILE_NR=2 + COMPILE_NR=2 -+ cd /scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/gnu/ufs-weather-model/tests -+ [[ -e /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_196893/compile_2.env ]] -+ source /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_196893/compile_2.env ++ cd /scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests ++ [[ -e /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_130720/compile_2.env ]] ++ source /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_130720/compile_2.env ++ export MACHINE_ID=hera.gnu ++ MACHINE_ID=hera.gnu -++ export PATHRT=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/gnu/ufs-weather-model/tests -++ PATHRT=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/gnu/ufs-weather-model/tests -++ export PATHTR=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/gnu/ufs-weather-model -++ PATHTR=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/gnu/ufs-weather-model +++ export PATHRT=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests +++ PATHRT=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests +++ export PATHTR=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model +++ PATHTR=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model ++ export SCHEDULER=slurm ++ SCHEDULER=slurm ++ export ACCNR=nems @@ -130,10 +135,10 @@ Elapsed time 185 seconds. Compile 1 ++ ROCOTO=false ++ export ECFLOW=true ++ ECFLOW=true -++ export REGRESSIONTEST_LOG=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/gnu/ufs-weather-model/tests/RegressionTests_hera.gnu.log -++ REGRESSIONTEST_LOG=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/gnu/ufs-weather-model/tests/RegressionTests_hera.gnu.log -++ export LOG_DIR=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/gnu/ufs-weather-model/tests/log_hera.gnu -++ LOG_DIR=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/gnu/ufs-weather-model/tests/log_hera.gnu +++ export REGRESSIONTEST_LOG=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/RegressionTests_hera.gnu.log +++ REGRESSIONTEST_LOG=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/RegressionTests_hera.gnu.log +++ export LOG_DIR=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/log_hera.gnu +++ LOG_DIR=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/log_hera.gnu + source default_vars.sh ++ '[' hera.gnu = wcoss ']' ++ '[' hera.gnu = wcoss_cray ']' @@ -165,15 +170,15 @@ Elapsed time 185 seconds. Compile 1 + TEST_NR=2 + export JBNME=compile_2 + JBNME=compile_2 -+ export RUNDIR=/scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_196893/compile_2 -+ RUNDIR=/scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_196893/compile_2 ++ export RUNDIR=/scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_130720/compile_2 ++ RUNDIR=/scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_130720/compile_2 + source rt_utils.sh ++ set -eu -++ [[ /scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/gnu/ufs-weather-model/tests/run_compile.sh = \r\t\_\u\t\i\l\s\.\s\h ]] +++ [[ /scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/run_compile.sh = \r\t\_\u\t\i\l\s\.\s\h ]] ++ UNIT_TEST=false + source atparse.bash -+ mkdir -p /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_196893/compile_2 -+ cd /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_196893/compile_2 ++ mkdir -p /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_130720/compile_2 ++ cd /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_130720/compile_2 + [[ slurm = \s\l\u\r\m ]] + atparse + local __set_x @@ -186,39 +191,47 @@ Elapsed time 185 seconds. Compile 1 + '[' -o xtrace ']' + set_x='set -x' + set +x -Job id 9562156 +Job id 9592105 TEST 2 compile is waiting to enter the queue -1 min. TEST 2 compile is running, Status: R +1 min. TEST 2 compile is pending, Status: PD 2 min. TEST 2 compile is running, Status: R +3 min. TEST 2 compile is running, Status: R +Slurm unknown status CG. Check sacct ... +9592105 COMPLETED compile_2 +9592105.bat+ COMPLETED batch +9592105.ext+ COMPLETED extern +4 min. TEST 2 compile is COMPLETED +COMPLETED +COMPLETED Slurm unknown status -. Check sacct ... -9562156 COMPLETED compile_2 -9562156.bat+ COMPLETED batch -9562156.ext+ COMPLETED extern -3 min. TEST 2 compile is COMPLETED +9592105 COMPLETED compile_2 +9592105.bat+ COMPLETED batch +9592105.ext+ COMPLETED extern +5 min. TEST 2 compile is COMPLETED COMPLETED COMPLETED -+ elapsed=185 -+ echo 'Elapsed time 185 seconds. Compile 2' -Elapsed time 185 seconds. Compile 2 ++ elapsed=307 ++ echo 'Elapsed time 307 seconds. Compile 2' +Elapsed time 307 seconds. Compile 2 + SECONDS=0 + [[ 4 != 4 ]] -+ export PATHRT=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/gnu/ufs-weather-model/tests -+ PATHRT=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/gnu/ufs-weather-model/tests -+ export RUNDIR_ROOT=/scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_196893 -+ RUNDIR_ROOT=/scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_196893 ++ export PATHRT=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests ++ PATHRT=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests ++ export RUNDIR_ROOT=/scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_130720 ++ RUNDIR_ROOT=/scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_130720 + export 'MAKE_OPT=CCPP=Y SUITES=FV3_GSD_v0,FV3_GFS_v15_thompson 32BIT=Y' + MAKE_OPT='CCPP=Y SUITES=FV3_GSD_v0,FV3_GFS_v15_thompson 32BIT=Y' + export COMPILE_NR=3 + COMPILE_NR=3 -+ cd /scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/gnu/ufs-weather-model/tests -+ [[ -e /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_196893/compile_3.env ]] -+ source /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_196893/compile_3.env ++ cd /scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests ++ [[ -e /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_130720/compile_3.env ]] ++ source /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_130720/compile_3.env ++ export MACHINE_ID=hera.gnu ++ MACHINE_ID=hera.gnu -++ export PATHRT=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/gnu/ufs-weather-model/tests -++ PATHRT=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/gnu/ufs-weather-model/tests -++ export PATHTR=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/gnu/ufs-weather-model -++ PATHTR=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/gnu/ufs-weather-model +++ export PATHRT=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests +++ PATHRT=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests +++ export PATHTR=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model +++ PATHTR=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model ++ export SCHEDULER=slurm ++ SCHEDULER=slurm ++ export ACCNR=nems @@ -231,10 +244,10 @@ Elapsed time 185 seconds. Compile 2 ++ ROCOTO=false ++ export ECFLOW=true ++ ECFLOW=true -++ export REGRESSIONTEST_LOG=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/gnu/ufs-weather-model/tests/RegressionTests_hera.gnu.log -++ REGRESSIONTEST_LOG=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/gnu/ufs-weather-model/tests/RegressionTests_hera.gnu.log -++ export LOG_DIR=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/gnu/ufs-weather-model/tests/log_hera.gnu -++ LOG_DIR=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/gnu/ufs-weather-model/tests/log_hera.gnu +++ export REGRESSIONTEST_LOG=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/RegressionTests_hera.gnu.log +++ REGRESSIONTEST_LOG=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/RegressionTests_hera.gnu.log +++ export LOG_DIR=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/log_hera.gnu +++ LOG_DIR=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/log_hera.gnu + source default_vars.sh ++ '[' hera.gnu = wcoss ']' ++ '[' hera.gnu = wcoss_cray ']' @@ -266,15 +279,15 @@ Elapsed time 185 seconds. Compile 2 + TEST_NR=3 + export JBNME=compile_3 + JBNME=compile_3 -+ export RUNDIR=/scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_196893/compile_3 -+ RUNDIR=/scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_196893/compile_3 ++ export RUNDIR=/scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_130720/compile_3 ++ RUNDIR=/scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_130720/compile_3 + source rt_utils.sh ++ set -eu -++ [[ /scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/gnu/ufs-weather-model/tests/run_compile.sh = \r\t\_\u\t\i\l\s\.\s\h ]] +++ [[ /scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/run_compile.sh = \r\t\_\u\t\i\l\s\.\s\h ]] ++ UNIT_TEST=false + source atparse.bash -+ mkdir -p /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_196893/compile_3 -+ cd /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_196893/compile_3 ++ mkdir -p /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_130720/compile_3 ++ cd /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_130720/compile_3 + [[ slurm = \s\l\u\r\m ]] + atparse + local __set_x @@ -287,39 +300,52 @@ Elapsed time 185 seconds. Compile 2 + '[' -o xtrace ']' + set_x='set -x' + set +x -Job id 9562155 +Job id 9592108 TEST 3 compile is waiting to enter the queue -1 min. TEST 3 compile is running, Status: R -2 min. TEST 3 compile is running, Status: R +1 min. TEST 3 compile is pending, Status: PD +2 min. TEST 3 compile is pending, Status: PD +3 min. TEST 3 compile is pending, Status: PD +4 min. TEST 3 compile is pending, Status: PD +5 min. TEST 3 compile is pending, Status: PD +6 min. TEST 3 compile is pending, Status: PD +7 min. TEST 3 compile is pending, Status: PD +8 min. TEST 3 compile is pending, Status: PD +9 min. TEST 3 compile is pending, Status: PD +10 min. TEST 3 compile is pending, Status: PD +11 min. TEST 3 compile is pending, Status: PD +12 min. TEST 3 compile is pending, Status: PD +13 min. TEST 3 compile is running, Status: R +14 min. TEST 3 compile is running, Status: R +15 min. TEST 3 compile is running, Status: R Slurm unknown status -. Check sacct ... -9562155 COMPLETED compile_3 -9562155.bat+ COMPLETED batch -9562155.ext+ COMPLETED extern -3 min. TEST 3 compile is COMPLETED +9592108 COMPLETED compile_3 +9592108.bat+ COMPLETED batch +9592108.ext+ COMPLETED extern +16 min. TEST 3 compile is COMPLETED COMPLETED COMPLETED -+ elapsed=185 -+ echo 'Elapsed time 185 seconds. Compile 3' -Elapsed time 185 seconds. Compile 3 ++ elapsed=968 ++ echo 'Elapsed time 968 seconds. Compile 3' +Elapsed time 968 seconds. Compile 3 + SECONDS=0 + [[ 4 != 4 ]] -+ export PATHRT=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/gnu/ufs-weather-model/tests -+ PATHRT=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/gnu/ufs-weather-model/tests -+ export RUNDIR_ROOT=/scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_196893 -+ RUNDIR_ROOT=/scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_196893 ++ export PATHRT=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests ++ PATHRT=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests ++ export RUNDIR_ROOT=/scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_130720 ++ RUNDIR_ROOT=/scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_130720 + export 'MAKE_OPT=CCPP=Y SUITES=FV3_GFS_2017 32BIT=Y DEBUG=Y' + MAKE_OPT='CCPP=Y SUITES=FV3_GFS_2017 32BIT=Y DEBUG=Y' + export COMPILE_NR=4 + COMPILE_NR=4 -+ cd /scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/gnu/ufs-weather-model/tests -+ [[ -e /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_196893/compile_4.env ]] -+ source /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_196893/compile_4.env ++ cd /scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests ++ [[ -e /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_130720/compile_4.env ]] ++ source /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_130720/compile_4.env ++ export MACHINE_ID=hera.gnu ++ MACHINE_ID=hera.gnu -++ export PATHRT=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/gnu/ufs-weather-model/tests -++ PATHRT=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/gnu/ufs-weather-model/tests -++ export PATHTR=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/gnu/ufs-weather-model -++ PATHTR=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/gnu/ufs-weather-model +++ export PATHRT=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests +++ PATHRT=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests +++ export PATHTR=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model +++ PATHTR=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model ++ export SCHEDULER=slurm ++ SCHEDULER=slurm ++ export ACCNR=nems @@ -332,10 +358,10 @@ Elapsed time 185 seconds. Compile 3 ++ ROCOTO=false ++ export ECFLOW=true ++ ECFLOW=true -++ export REGRESSIONTEST_LOG=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/gnu/ufs-weather-model/tests/RegressionTests_hera.gnu.log -++ REGRESSIONTEST_LOG=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/gnu/ufs-weather-model/tests/RegressionTests_hera.gnu.log -++ export LOG_DIR=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/gnu/ufs-weather-model/tests/log_hera.gnu -++ LOG_DIR=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/gnu/ufs-weather-model/tests/log_hera.gnu +++ export REGRESSIONTEST_LOG=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/RegressionTests_hera.gnu.log +++ REGRESSIONTEST_LOG=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/RegressionTests_hera.gnu.log +++ export LOG_DIR=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/log_hera.gnu +++ LOG_DIR=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/log_hera.gnu + source default_vars.sh ++ '[' hera.gnu = wcoss ']' ++ '[' hera.gnu = wcoss_cray ']' @@ -367,15 +393,15 @@ Elapsed time 185 seconds. Compile 3 + TEST_NR=4 + export JBNME=compile_4 + JBNME=compile_4 -+ export RUNDIR=/scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_196893/compile_4 -+ RUNDIR=/scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_196893/compile_4 ++ export RUNDIR=/scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_130720/compile_4 ++ RUNDIR=/scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_130720/compile_4 + source rt_utils.sh ++ set -eu -++ [[ /scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/gnu/ufs-weather-model/tests/run_compile.sh = \r\t\_\u\t\i\l\s\.\s\h ]] +++ [[ /scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/run_compile.sh = \r\t\_\u\t\i\l\s\.\s\h ]] ++ UNIT_TEST=false + source atparse.bash -+ mkdir -p /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_196893/compile_4 -+ cd /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_196893/compile_4 ++ mkdir -p /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_130720/compile_4 ++ cd /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_130720/compile_4 + [[ slurm = \s\l\u\r\m ]] + atparse + local __set_x @@ -388,38 +414,58 @@ Elapsed time 185 seconds. Compile 3 + '[' -o xtrace ']' + set_x='set -x' + set +x -Job id 9562154 +Job id 9592109 TEST 4 compile is waiting to enter the queue -1 min. TEST 4 compile is running, Status: R +1 min. TEST 4 compile is pending, Status: PD +2 min. TEST 4 compile is pending, Status: PD +3 min. TEST 4 compile is pending, Status: PD +4 min. TEST 4 compile is pending, Status: PD +5 min. TEST 4 compile is pending, Status: PD +6 min. TEST 4 compile is pending, Status: PD +7 min. TEST 4 compile is pending, Status: PD +8 min. TEST 4 compile is pending, Status: PD +9 min. TEST 4 compile is pending, Status: PD +10 min. TEST 4 compile is pending, Status: PD +11 min. TEST 4 compile is pending, Status: PD +12 min. TEST 4 compile is pending, Status: PD +13 min. TEST 4 compile is pending, Status: PD +14 min. TEST 4 compile is pending, Status: PD +15 min. TEST 4 compile is pending, Status: PD +16 min. TEST 4 compile is pending, Status: PD +17 min. TEST 4 compile is pending, Status: PD +18 min. TEST 4 compile is pending, Status: PD +19 min. TEST 4 compile is pending, Status: PD +20 min. TEST 4 compile is running, Status: R +21 min. TEST 4 compile is running, Status: R Slurm unknown status -. Check sacct ... -9562154 COMPLETED compile_4 -9562154.bat+ COMPLETED batch -9562154.ext+ COMPLETED extern -2 min. TEST 4 compile is COMPLETED +9592109 COMPLETED compile_4 +9592109.bat+ COMPLETED batch +9592109.ext+ COMPLETED extern +22 min. TEST 4 compile is COMPLETED COMPLETED COMPLETED -+ elapsed=125 -+ echo 'Elapsed time 125 seconds. Compile 4' -Elapsed time 125 seconds. Compile 4 ++ elapsed=1328 ++ echo 'Elapsed time 1328 seconds. Compile 4' +Elapsed time 1328 seconds. Compile 4 + SECONDS=0 + [[ 4 != 4 ]] -+ export PATHRT=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/gnu/ufs-weather-model/tests -+ PATHRT=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/gnu/ufs-weather-model/tests -+ export RUNDIR_ROOT=/scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_196893 -+ RUNDIR_ROOT=/scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_196893 ++ export PATHRT=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests ++ PATHRT=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests ++ export RUNDIR_ROOT=/scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_130720 ++ RUNDIR_ROOT=/scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_130720 + export 'MAKE_OPT=CCPP=Y SUITES=FV3_GFS_v15p2,FV3_GFS_v16beta,FV3_GFS_2017_RRTMGP DEBUG=Y' + MAKE_OPT='CCPP=Y SUITES=FV3_GFS_v15p2,FV3_GFS_v16beta,FV3_GFS_2017_RRTMGP DEBUG=Y' + export COMPILE_NR=5 + COMPILE_NR=5 -+ cd /scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/gnu/ufs-weather-model/tests -+ [[ -e /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_196893/compile_5.env ]] -+ source /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_196893/compile_5.env ++ cd /scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests ++ [[ -e /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_130720/compile_5.env ]] ++ source /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_130720/compile_5.env ++ export MACHINE_ID=hera.gnu ++ MACHINE_ID=hera.gnu -++ export PATHRT=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/gnu/ufs-weather-model/tests -++ PATHRT=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/gnu/ufs-weather-model/tests -++ export PATHTR=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/gnu/ufs-weather-model -++ PATHTR=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/gnu/ufs-weather-model +++ export PATHRT=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests +++ PATHRT=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests +++ export PATHTR=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model +++ PATHTR=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model ++ export SCHEDULER=slurm ++ SCHEDULER=slurm ++ export ACCNR=nems @@ -432,10 +478,10 @@ Elapsed time 125 seconds. Compile 4 ++ ROCOTO=false ++ export ECFLOW=true ++ ECFLOW=true -++ export REGRESSIONTEST_LOG=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/gnu/ufs-weather-model/tests/RegressionTests_hera.gnu.log -++ REGRESSIONTEST_LOG=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/gnu/ufs-weather-model/tests/RegressionTests_hera.gnu.log -++ export LOG_DIR=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/gnu/ufs-weather-model/tests/log_hera.gnu -++ LOG_DIR=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/gnu/ufs-weather-model/tests/log_hera.gnu +++ export REGRESSIONTEST_LOG=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/RegressionTests_hera.gnu.log +++ REGRESSIONTEST_LOG=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/RegressionTests_hera.gnu.log +++ export LOG_DIR=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/log_hera.gnu +++ LOG_DIR=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/log_hera.gnu + source default_vars.sh ++ '[' hera.gnu = wcoss ']' ++ '[' hera.gnu = wcoss_cray ']' @@ -467,15 +513,15 @@ Elapsed time 125 seconds. Compile 4 + TEST_NR=5 + export JBNME=compile_5 + JBNME=compile_5 -+ export RUNDIR=/scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_196893/compile_5 -+ RUNDIR=/scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_196893/compile_5 ++ export RUNDIR=/scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_130720/compile_5 ++ RUNDIR=/scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_130720/compile_5 + source rt_utils.sh ++ set -eu -++ [[ /scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/gnu/ufs-weather-model/tests/run_compile.sh = \r\t\_\u\t\i\l\s\.\s\h ]] +++ [[ /scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/run_compile.sh = \r\t\_\u\t\i\l\s\.\s\h ]] ++ UNIT_TEST=false + source atparse.bash -+ mkdir -p /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_196893/compile_5 -+ cd /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_196893/compile_5 ++ mkdir -p /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_130720/compile_5 ++ cd /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_130720/compile_5 + [[ slurm = \s\l\u\r\m ]] + atparse + local __set_x @@ -488,38 +534,52 @@ Elapsed time 125 seconds. Compile 4 + '[' -o xtrace ']' + set_x='set -x' + set +x -Job id 9562153 +Job id 9592107 TEST 5 compile is waiting to enter the queue -1 min. TEST 5 compile is running, Status: R +1 min. TEST 5 compile is pending, Status: PD +2 min. TEST 5 compile is pending, Status: PD +3 min. TEST 5 compile is pending, Status: PD +4 min. TEST 5 compile is pending, Status: PD +5 min. TEST 5 compile is pending, Status: PD +6 min. TEST 5 compile is pending, Status: PD +7 min. TEST 5 compile is pending, Status: PD +8 min. TEST 5 compile is pending, Status: PD +9 min. TEST 5 compile is pending, Status: PD +10 min. TEST 5 compile is pending, Status: PD +11 min. TEST 5 compile is pending, Status: PD +12 min. TEST 5 compile is pending, Status: PD +13 min. TEST 5 compile is running, Status: R +14 min. TEST 5 compile is running, Status: R +15 min. TEST 5 compile is running, Status: R Slurm unknown status -. Check sacct ... -9562153 COMPLETED compile_5 -9562153.bat+ COMPLETED batch -9562153.ext+ COMPLETED extern -2 min. TEST 5 compile is COMPLETED +9592107 COMPLETED compile_5 +9592107.bat+ COMPLETED batch +9592107.ext+ COMPLETED extern +16 min. TEST 5 compile is COMPLETED COMPLETED COMPLETED -+ elapsed=125 -+ echo 'Elapsed time 125 seconds. Compile 5' -Elapsed time 125 seconds. Compile 5 ++ elapsed=968 ++ echo 'Elapsed time 968 seconds. Compile 5' +Elapsed time 968 seconds. Compile 5 + SECONDS=0 + [[ 4 != 4 ]] -+ export PATHRT=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/gnu/ufs-weather-model/tests -+ PATHRT=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/gnu/ufs-weather-model/tests -+ export RUNDIR_ROOT=/scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_196893 -+ RUNDIR_ROOT=/scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_196893 ++ export PATHRT=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests ++ PATHRT=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests ++ export RUNDIR_ROOT=/scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_130720 ++ RUNDIR_ROOT=/scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_130720 + export 'MAKE_OPT=CCPP=Y SUITES=FV3_GFS_2017_fv3wam 32BIT=Y MULTI_GASES=Y' + MAKE_OPT='CCPP=Y SUITES=FV3_GFS_2017_fv3wam 32BIT=Y MULTI_GASES=Y' + export COMPILE_NR=6 + COMPILE_NR=6 -+ cd /scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/gnu/ufs-weather-model/tests -+ [[ -e /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_196893/compile_6.env ]] -+ source /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_196893/compile_6.env ++ cd /scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests ++ [[ -e /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_130720/compile_6.env ]] ++ source /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_130720/compile_6.env ++ export MACHINE_ID=hera.gnu ++ MACHINE_ID=hera.gnu -++ export PATHRT=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/gnu/ufs-weather-model/tests -++ PATHRT=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/gnu/ufs-weather-model/tests -++ export PATHTR=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/gnu/ufs-weather-model -++ PATHTR=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/gnu/ufs-weather-model +++ export PATHRT=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests +++ PATHRT=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests +++ export PATHTR=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model +++ PATHTR=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model ++ export SCHEDULER=slurm ++ SCHEDULER=slurm ++ export ACCNR=nems @@ -532,10 +592,10 @@ Elapsed time 125 seconds. Compile 5 ++ ROCOTO=false ++ export ECFLOW=true ++ ECFLOW=true -++ export REGRESSIONTEST_LOG=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/gnu/ufs-weather-model/tests/RegressionTests_hera.gnu.log -++ REGRESSIONTEST_LOG=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/gnu/ufs-weather-model/tests/RegressionTests_hera.gnu.log -++ export LOG_DIR=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/gnu/ufs-weather-model/tests/log_hera.gnu -++ LOG_DIR=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/gnu/ufs-weather-model/tests/log_hera.gnu +++ export REGRESSIONTEST_LOG=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/RegressionTests_hera.gnu.log +++ REGRESSIONTEST_LOG=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/RegressionTests_hera.gnu.log +++ export LOG_DIR=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/log_hera.gnu +++ LOG_DIR=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/log_hera.gnu + source default_vars.sh ++ '[' hera.gnu = wcoss ']' ++ '[' hera.gnu = wcoss_cray ']' @@ -567,15 +627,15 @@ Elapsed time 125 seconds. Compile 5 + TEST_NR=6 + export JBNME=compile_6 + JBNME=compile_6 -+ export RUNDIR=/scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_196893/compile_6 -+ RUNDIR=/scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_196893/compile_6 ++ export RUNDIR=/scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_130720/compile_6 ++ RUNDIR=/scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_130720/compile_6 + source rt_utils.sh ++ set -eu -++ [[ /scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/gnu/ufs-weather-model/tests/run_compile.sh = \r\t\_\u\t\i\l\s\.\s\h ]] +++ [[ /scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/run_compile.sh = \r\t\_\u\t\i\l\s\.\s\h ]] ++ UNIT_TEST=false + source atparse.bash -+ mkdir -p /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_196893/compile_6 -+ cd /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_196893/compile_6 ++ mkdir -p /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_130720/compile_6 ++ cd /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_130720/compile_6 + [[ slurm = \s\l\u\r\m ]] + atparse + local __set_x @@ -588,17 +648,37 @@ Elapsed time 125 seconds. Compile 5 + '[' -o xtrace ']' + set_x='set -x' + set +x -Job id 9562151 +Job id 9592110 TEST 6 compile is waiting to enter the queue -1 min. TEST 6 compile is running, Status: R -2 min. TEST 6 compile is running, Status: R +1 min. TEST 6 compile is pending, Status: PD +2 min. TEST 6 compile is pending, Status: PD +3 min. TEST 6 compile is pending, Status: PD +4 min. TEST 6 compile is pending, Status: PD +5 min. TEST 6 compile is pending, Status: PD +6 min. TEST 6 compile is pending, Status: PD +7 min. TEST 6 compile is pending, Status: PD +8 min. TEST 6 compile is pending, Status: PD +9 min. TEST 6 compile is pending, Status: PD +10 min. TEST 6 compile is pending, Status: PD +11 min. TEST 6 compile is pending, Status: PD +12 min. TEST 6 compile is pending, Status: PD +13 min. TEST 6 compile is pending, Status: PD +14 min. TEST 6 compile is pending, Status: PD +15 min. TEST 6 compile is pending, Status: PD +16 min. TEST 6 compile is pending, Status: PD +17 min. TEST 6 compile is pending, Status: PD +18 min. TEST 6 compile is pending, Status: PD +19 min. TEST 6 compile is pending, Status: PD +20 min. TEST 6 compile is running, Status: R +21 min. TEST 6 compile is running, Status: R +22 min. TEST 6 compile is running, Status: R Slurm unknown status -. Check sacct ... -9562151 COMPLETED compile_6 -9562151.bat+ COMPLETED batch -9562151.ext+ COMPLETED extern -3 min. TEST 6 compile is COMPLETED +9592110 COMPLETED compile_6 +9592110.bat+ COMPLETED batch +9592110.ext+ COMPLETED extern +23 min. TEST 6 compile is COMPLETED COMPLETED COMPLETED -+ elapsed=185 -+ echo 'Elapsed time 185 seconds. Compile 6' -Elapsed time 185 seconds. Compile 6 ++ elapsed=1388 ++ echo 'Elapsed time 1388 seconds. Compile 6' +Elapsed time 1388 seconds. Compile 6 diff --git a/tests/Compile_hera.intel.log b/tests/Compile_hera.intel.log index c71349223a..55258a7dbe 100644 --- a/tests/Compile_hera.intel.log +++ b/tests/Compile_hera.intel.log @@ -1,22 +1,22 @@ + SECONDS=0 + [[ 4 != 4 ]] -+ export PATHRT=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -+ PATHRT=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -+ export RUNDIR_ROOT=/scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_136435 -+ RUNDIR_ROOT=/scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_136435 ++ export PATHRT=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests ++ PATHRT=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests ++ export RUNDIR_ROOT=/scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_293743 ++ RUNDIR_ROOT=/scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_293743 + export 'MAKE_OPT=CCPP=Y SUITES=FV3_GFS_v15p2,FV3_GFS_v16beta,FV3_GFS_2017_RRTMGP' + MAKE_OPT='CCPP=Y SUITES=FV3_GFS_v15p2,FV3_GFS_v16beta,FV3_GFS_2017_RRTMGP' + export COMPILE_NR=10 + COMPILE_NR=10 -+ cd /scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -+ [[ -e /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_136435/compile_10.env ]] -+ source /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_136435/compile_10.env ++ cd /scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests ++ [[ -e /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_293743/compile_10.env ]] ++ source /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_293743/compile_10.env ++ export MACHINE_ID=hera.intel ++ MACHINE_ID=hera.intel -++ export PATHRT=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -++ PATHRT=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -++ export PATHTR=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/ufs-weather-model -++ PATHTR=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/ufs-weather-model +++ export PATHRT=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests +++ PATHRT=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests +++ export PATHTR=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model +++ PATHTR=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model ++ export SCHEDULER=slurm ++ SCHEDULER=slurm ++ export ACCNR=nems @@ -29,10 +29,10 @@ ++ ROCOTO=false ++ export ECFLOW=true ++ ECFLOW=true -++ export REGRESSIONTEST_LOG=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/RegressionTests_hera.intel.log -++ REGRESSIONTEST_LOG=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/RegressionTests_hera.intel.log -++ export LOG_DIR=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/log_hera.intel -++ LOG_DIR=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/log_hera.intel +++ export REGRESSIONTEST_LOG=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/RegressionTests_hera.intel.log +++ REGRESSIONTEST_LOG=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/RegressionTests_hera.intel.log +++ export LOG_DIR=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/log_hera.intel +++ LOG_DIR=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/log_hera.intel + source default_vars.sh ++ '[' hera.intel = wcoss ']' ++ '[' hera.intel = wcoss_cray ']' @@ -65,15 +65,15 @@ + TEST_NR=10 + export JBNME=compile_10 + JBNME=compile_10 -+ export RUNDIR=/scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_136435/compile_10 -+ RUNDIR=/scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_136435/compile_10 ++ export RUNDIR=/scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_293743/compile_10 ++ RUNDIR=/scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_293743/compile_10 + source rt_utils.sh ++ set -eu -++ [[ /scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/run_compile.sh = \r\t\_\u\t\i\l\s\.\s\h ]] +++ [[ /scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/run_compile.sh = \r\t\_\u\t\i\l\s\.\s\h ]] ++ UNIT_TEST=false + source atparse.bash -+ mkdir -p /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_136435/compile_10 -+ cd /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_136435/compile_10 ++ mkdir -p /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_293743/compile_10 ++ cd /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_293743/compile_10 + [[ slurm = \s\l\u\r\m ]] + atparse + local __set_x @@ -86,43 +86,44 @@ + '[' -o xtrace ']' + set_x='set -x' + set +x -Job id 9560624 +Job id 9590940 TEST 10 compile is waiting to enter the queue -1 min. TEST 10 compile is running, Status: R +1 min. TEST 10 compile is pending, Status: PD 2 min. TEST 10 compile is running, Status: R 3 min. TEST 10 compile is running, Status: R 4 min. TEST 10 compile is running, Status: R 5 min. TEST 10 compile is running, Status: R 6 min. TEST 10 compile is running, Status: R +7 min. TEST 10 compile is running, Status: R Slurm unknown status -. Check sacct ... -9560624 COMPLETED compile_10 -9560624.bat+ COMPLETED batch -9560624.ext+ COMPLETED extern -7 min. TEST 10 compile is COMPLETED +9590940 COMPLETED compile_10 +9590940.bat+ COMPLETED batch +9590940.ext+ COMPLETED extern +8 min. TEST 10 compile is COMPLETED COMPLETED COMPLETED -+ elapsed=426 -+ echo 'Elapsed time 426 seconds. Compile 10' -Elapsed time 426 seconds. Compile 10 ++ elapsed=486 ++ echo 'Elapsed time 486 seconds. Compile 10' +Elapsed time 486 seconds. Compile 10 + SECONDS=0 + [[ 4 != 4 ]] -+ export PATHRT=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -+ PATHRT=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -+ export RUNDIR_ROOT=/scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_136435 -+ RUNDIR_ROOT=/scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_136435 ++ export PATHRT=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests ++ PATHRT=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests ++ export RUNDIR_ROOT=/scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_293743 ++ RUNDIR_ROOT=/scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_293743 + export 'MAKE_OPT=CCPP=Y SUITES=FV3_GFS_v15p2,FV3_GFS_v16beta,FV3_GFS_2017_RRTMGP DEBUG=Y' + MAKE_OPT='CCPP=Y SUITES=FV3_GFS_v15p2,FV3_GFS_v16beta,FV3_GFS_2017_RRTMGP DEBUG=Y' + export COMPILE_NR=11 + COMPILE_NR=11 -+ cd /scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -+ [[ -e /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_136435/compile_11.env ]] -+ source /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_136435/compile_11.env ++ cd /scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests ++ [[ -e /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_293743/compile_11.env ]] ++ source /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_293743/compile_11.env ++ export MACHINE_ID=hera.intel ++ MACHINE_ID=hera.intel -++ export PATHRT=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -++ PATHRT=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -++ export PATHTR=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/ufs-weather-model -++ PATHTR=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/ufs-weather-model +++ export PATHRT=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests +++ PATHRT=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests +++ export PATHTR=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model +++ PATHTR=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model ++ export SCHEDULER=slurm ++ SCHEDULER=slurm ++ export ACCNR=nems @@ -135,10 +136,10 @@ Elapsed time 426 seconds. Compile 10 ++ ROCOTO=false ++ export ECFLOW=true ++ ECFLOW=true -++ export REGRESSIONTEST_LOG=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/RegressionTests_hera.intel.log -++ REGRESSIONTEST_LOG=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/RegressionTests_hera.intel.log -++ export LOG_DIR=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/log_hera.intel -++ LOG_DIR=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/log_hera.intel +++ export REGRESSIONTEST_LOG=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/RegressionTests_hera.intel.log +++ REGRESSIONTEST_LOG=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/RegressionTests_hera.intel.log +++ export LOG_DIR=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/log_hera.intel +++ LOG_DIR=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/log_hera.intel + source default_vars.sh ++ '[' hera.intel = wcoss ']' ++ '[' hera.intel = wcoss_cray ']' @@ -171,15 +172,15 @@ Elapsed time 426 seconds. Compile 10 + TEST_NR=11 + export JBNME=compile_11 + JBNME=compile_11 -+ export RUNDIR=/scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_136435/compile_11 -+ RUNDIR=/scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_136435/compile_11 ++ export RUNDIR=/scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_293743/compile_11 ++ RUNDIR=/scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_293743/compile_11 + source rt_utils.sh ++ set -eu -++ [[ /scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/run_compile.sh = \r\t\_\u\t\i\l\s\.\s\h ]] +++ [[ /scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/run_compile.sh = \r\t\_\u\t\i\l\s\.\s\h ]] ++ UNIT_TEST=false + source atparse.bash -+ mkdir -p /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_136435/compile_11 -+ cd /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_136435/compile_11 ++ mkdir -p /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_293743/compile_11 ++ cd /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_293743/compile_11 + [[ slurm = \s\l\u\r\m ]] + atparse + local __set_x @@ -192,56 +193,43 @@ Elapsed time 426 seconds. Compile 10 + '[' -o xtrace ']' + set_x='set -x' + set +x -Job id 9560765 +Job id 9591144 TEST 11 compile is waiting to enter the queue 1 min. TEST 11 compile is pending, Status: PD 2 min. TEST 11 compile is pending, Status: PD 3 min. TEST 11 compile is pending, Status: PD 4 min. TEST 11 compile is pending, Status: PD -5 min. TEST 11 compile is pending, Status: PD -6 min. TEST 11 compile is pending, Status: PD -7 min. TEST 11 compile is pending, Status: PD -8 min. TEST 11 compile is pending, Status: PD -9 min. TEST 11 compile is pending, Status: PD -10 min. TEST 11 compile is pending, Status: PD -11 min. TEST 11 compile is pending, Status: PD -12 min. TEST 11 compile is pending, Status: PD -13 min. TEST 11 compile is pending, Status: PD -14 min. TEST 11 compile is pending, Status: PD -15 min. TEST 11 compile is pending, Status: PD -16 min. TEST 11 compile is pending, Status: PD -17 min. TEST 11 compile is running, Status: R -18 min. TEST 11 compile is running, Status: R -19 min. TEST 11 compile is running, Status: R +5 min. TEST 11 compile is running, Status: R +6 min. TEST 11 compile is running, Status: R Slurm unknown status -. Check sacct ... -9560765 COMPLETED compile_11 -9560765.bat+ COMPLETED batch -9560765.ext+ COMPLETED extern -20 min. TEST 11 compile is COMPLETED +9591144 COMPLETED compile_11 +9591144.bat+ COMPLETED batch +9591144.ext+ COMPLETED extern +7 min. TEST 11 compile is COMPLETED COMPLETED COMPLETED -+ elapsed=1211 -+ echo 'Elapsed time 1211 seconds. Compile 11' -Elapsed time 1211 seconds. Compile 11 ++ elapsed=426 ++ echo 'Elapsed time 426 seconds. Compile 11' +Elapsed time 426 seconds. Compile 11 + SECONDS=0 + [[ 4 != 4 ]] -+ export PATHRT=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -+ PATHRT=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -+ export RUNDIR_ROOT=/scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_136435 -+ RUNDIR_ROOT=/scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_136435 ++ export PATHRT=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests ++ PATHRT=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests ++ export RUNDIR_ROOT=/scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_293743 ++ RUNDIR_ROOT=/scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_293743 + export 'MAKE_OPT=CCPP=Y SUITES=FV3_GSD_v0,FV3_GFS_v15_thompson 32BIT=Y DEBUG=Y' + MAKE_OPT='CCPP=Y SUITES=FV3_GSD_v0,FV3_GFS_v15_thompson 32BIT=Y DEBUG=Y' + export COMPILE_NR=12 + COMPILE_NR=12 -+ cd /scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -+ [[ -e /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_136435/compile_12.env ]] -+ source /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_136435/compile_12.env ++ cd /scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests ++ [[ -e /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_293743/compile_12.env ]] ++ source /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_293743/compile_12.env ++ export MACHINE_ID=hera.intel ++ MACHINE_ID=hera.intel -++ export PATHRT=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -++ PATHRT=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -++ export PATHTR=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/ufs-weather-model -++ PATHTR=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/ufs-weather-model +++ export PATHRT=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests +++ PATHRT=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests +++ export PATHTR=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model +++ PATHTR=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model ++ export SCHEDULER=slurm ++ SCHEDULER=slurm ++ export ACCNR=nems @@ -254,10 +242,10 @@ Elapsed time 1211 seconds. Compile 11 ++ ROCOTO=false ++ export ECFLOW=true ++ ECFLOW=true -++ export REGRESSIONTEST_LOG=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/RegressionTests_hera.intel.log -++ REGRESSIONTEST_LOG=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/RegressionTests_hera.intel.log -++ export LOG_DIR=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/log_hera.intel -++ LOG_DIR=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/log_hera.intel +++ export REGRESSIONTEST_LOG=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/RegressionTests_hera.intel.log +++ REGRESSIONTEST_LOG=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/RegressionTests_hera.intel.log +++ export LOG_DIR=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/log_hera.intel +++ LOG_DIR=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/log_hera.intel + source default_vars.sh ++ '[' hera.intel = wcoss ']' ++ '[' hera.intel = wcoss_cray ']' @@ -290,15 +278,15 @@ Elapsed time 1211 seconds. Compile 11 + TEST_NR=12 + export JBNME=compile_12 + JBNME=compile_12 -+ export RUNDIR=/scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_136435/compile_12 -+ RUNDIR=/scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_136435/compile_12 ++ export RUNDIR=/scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_293743/compile_12 ++ RUNDIR=/scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_293743/compile_12 + source rt_utils.sh ++ set -eu -++ [[ /scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/run_compile.sh = \r\t\_\u\t\i\l\s\.\s\h ]] +++ [[ /scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/run_compile.sh = \r\t\_\u\t\i\l\s\.\s\h ]] ++ UNIT_TEST=false + source atparse.bash -+ mkdir -p /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_136435/compile_12 -+ cd /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_136435/compile_12 ++ mkdir -p /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_293743/compile_12 ++ cd /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_293743/compile_12 + [[ slurm = \s\l\u\r\m ]] + atparse + local __set_x @@ -311,60 +299,43 @@ Elapsed time 1211 seconds. Compile 11 + '[' -o xtrace ']' + set_x='set -x' + set +x -Job id 9560843 +Job id 9591145 TEST 12 compile is waiting to enter the queue 1 min. TEST 12 compile is pending, Status: PD 2 min. TEST 12 compile is pending, Status: PD 3 min. TEST 12 compile is pending, Status: PD 4 min. TEST 12 compile is pending, Status: PD -5 min. TEST 12 compile is pending, Status: PD -6 min. TEST 12 compile is pending, Status: PD -7 min. TEST 12 compile is pending, Status: PD -8 min. TEST 12 compile is pending, Status: PD -9 min. TEST 12 compile is pending, Status: PD -10 min. TEST 12 compile is pending, Status: PD -11 min. TEST 12 compile is pending, Status: PD -12 min. TEST 12 compile is pending, Status: PD -13 min. TEST 12 compile is pending, Status: PD -14 min. TEST 12 compile is pending, Status: PD -15 min. TEST 12 compile is pending, Status: PD -16 min. TEST 12 compile is pending, Status: PD -17 min. TEST 12 compile is pending, Status: PD -18 min. TEST 12 compile is pending, Status: PD -19 min. TEST 12 compile is pending, Status: PD -20 min. TEST 12 compile is pending, Status: PD -21 min. TEST 12 compile is running, Status: R -22 min. TEST 12 compile is running, Status: R -23 min. TEST 12 compile is running, Status: R +5 min. TEST 12 compile is running, Status: R +6 min. TEST 12 compile is running, Status: R Slurm unknown status -. Check sacct ... -9560843 COMPLETED compile_12 -9560843.bat+ COMPLETED batch -9560843.ext+ COMPLETED extern -24 min. TEST 12 compile is COMPLETED +9591145 COMPLETED compile_12 +9591145.bat+ COMPLETED batch +9591145.ext+ COMPLETED extern +7 min. TEST 12 compile is COMPLETED COMPLETED COMPLETED -+ elapsed=1451 -+ echo 'Elapsed time 1451 seconds. Compile 12' -Elapsed time 1451 seconds. Compile 12 ++ elapsed=426 ++ echo 'Elapsed time 426 seconds. Compile 12' +Elapsed time 426 seconds. Compile 12 + SECONDS=0 + [[ 4 != 4 ]] -+ export PATHRT=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -+ PATHRT=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -+ export RUNDIR_ROOT=/scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_136435 -+ RUNDIR_ROOT=/scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_136435 ++ export PATHRT=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests ++ PATHRT=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests ++ export RUNDIR_ROOT=/scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_293743 ++ RUNDIR_ROOT=/scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_293743 + export 'MAKE_OPT=CCPP=Y SUITES=FV3_GFS_v16_csawmg' + MAKE_OPT='CCPP=Y SUITES=FV3_GFS_v16_csawmg' + export COMPILE_NR=13 + COMPILE_NR=13 -+ cd /scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -+ [[ -e /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_136435/compile_13.env ]] -+ source /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_136435/compile_13.env ++ cd /scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests ++ [[ -e /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_293743/compile_13.env ]] ++ source /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_293743/compile_13.env ++ export MACHINE_ID=hera.intel ++ MACHINE_ID=hera.intel -++ export PATHRT=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -++ PATHRT=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -++ export PATHTR=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/ufs-weather-model -++ PATHTR=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/ufs-weather-model +++ export PATHRT=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests +++ PATHRT=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests +++ export PATHTR=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model +++ PATHTR=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model ++ export SCHEDULER=slurm ++ SCHEDULER=slurm ++ export ACCNR=nems @@ -377,10 +348,10 @@ Elapsed time 1451 seconds. Compile 12 ++ ROCOTO=false ++ export ECFLOW=true ++ ECFLOW=true -++ export REGRESSIONTEST_LOG=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/RegressionTests_hera.intel.log -++ REGRESSIONTEST_LOG=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/RegressionTests_hera.intel.log -++ export LOG_DIR=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/log_hera.intel -++ LOG_DIR=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/log_hera.intel +++ export REGRESSIONTEST_LOG=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/RegressionTests_hera.intel.log +++ REGRESSIONTEST_LOG=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/RegressionTests_hera.intel.log +++ export LOG_DIR=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/log_hera.intel +++ LOG_DIR=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/log_hera.intel + source default_vars.sh ++ '[' hera.intel = wcoss ']' ++ '[' hera.intel = wcoss_cray ']' @@ -413,15 +384,15 @@ Elapsed time 1451 seconds. Compile 12 + TEST_NR=13 + export JBNME=compile_13 + JBNME=compile_13 -+ export RUNDIR=/scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_136435/compile_13 -+ RUNDIR=/scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_136435/compile_13 ++ export RUNDIR=/scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_293743/compile_13 ++ RUNDIR=/scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_293743/compile_13 + source rt_utils.sh ++ set -eu -++ [[ /scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/run_compile.sh = \r\t\_\u\t\i\l\s\.\s\h ]] +++ [[ /scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/run_compile.sh = \r\t\_\u\t\i\l\s\.\s\h ]] ++ UNIT_TEST=false + source atparse.bash -+ mkdir -p /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_136435/compile_13 -+ cd /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_136435/compile_13 ++ mkdir -p /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_293743/compile_13 ++ cd /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_293743/compile_13 + [[ slurm = \s\l\u\r\m ]] + atparse + local __set_x @@ -434,7 +405,7 @@ Elapsed time 1451 seconds. Compile 12 + '[' -o xtrace ']' + set_x='set -x' + set +x -Job id 9560842 +Job id 9591235 TEST 13 compile is waiting to enter the queue 1 min. TEST 13 compile is pending, Status: PD 2 min. TEST 13 compile is pending, Status: PD @@ -444,60 +415,48 @@ TEST 13 compile is waiting to enter the queue 6 min. TEST 13 compile is pending, Status: PD 7 min. TEST 13 compile is pending, Status: PD 8 min. TEST 13 compile is pending, Status: PD -9 min. TEST 13 compile is pending, Status: PD -10 min. TEST 13 compile is pending, Status: PD -11 min. TEST 13 compile is pending, Status: PD -12 min. TEST 13 compile is pending, Status: PD -13 min. TEST 13 compile is pending, Status: PD -14 min. TEST 13 compile is pending, Status: PD -15 min. TEST 13 compile is pending, Status: PD -16 min. TEST 13 compile is pending, Status: PD -17 min. TEST 13 compile is pending, Status: PD -18 min. TEST 13 compile is pending, Status: PD -19 min. TEST 13 compile is pending, Status: PD -20 min. TEST 13 compile is pending, Status: PD -21 min. TEST 13 compile is running, Status: R -22 min. TEST 13 compile is running, Status: R -23 min. TEST 13 compile is running, Status: R -24 min. TEST 13 compile is running, Status: R -25 min. TEST 13 compile is running, Status: R -26 min. TEST 13 compile is running, Status: R +9 min. TEST 13 compile is running, Status: R +10 min. TEST 13 compile is running, Status: R +11 min. TEST 13 compile is running, Status: R +12 min. TEST 13 compile is running, Status: R +13 min. TEST 13 compile is running, Status: R +14 min. TEST 13 compile is running, Status: R Slurm unknown status CG. Check sacct ... -9560842 COMPLETED compile_13 -9560842.bat+ COMPLETED batch -9560842.ext+ COMPLETED extern -27 min. TEST 13 compile is COMPLETED +9591235 COMPLETED compile_13 +9591235.bat+ COMPLETED batch +9591235.ext+ COMPLETED extern +15 min. TEST 13 compile is COMPLETED COMPLETED COMPLETED Slurm unknown status -. Check sacct ... -9560842 COMPLETED compile_13 -9560842.bat+ COMPLETED batch -9560842.ext+ COMPLETED extern -28 min. TEST 13 compile is COMPLETED +9591235 COMPLETED compile_13 +9591235.bat+ COMPLETED batch +9591235.ext+ COMPLETED extern +16 min. TEST 13 compile is COMPLETED COMPLETED COMPLETED -+ elapsed=1692 -+ echo 'Elapsed time 1692 seconds. Compile 13' -Elapsed time 1692 seconds. Compile 13 ++ elapsed=969 ++ echo 'Elapsed time 969 seconds. Compile 13' +Elapsed time 969 seconds. Compile 13 + SECONDS=0 + [[ 4 != 4 ]] -+ export PATHRT=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -+ PATHRT=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -+ export RUNDIR_ROOT=/scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_136435 -+ RUNDIR_ROOT=/scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_136435 ++ export PATHRT=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests ++ PATHRT=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests ++ export RUNDIR_ROOT=/scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_293743 ++ RUNDIR_ROOT=/scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_293743 + export 'MAKE_OPT=CCPP=Y SUITES=FV3_GFS_2017_gfdlmp,FV3_GFS_2017_gfdlmp_noahmp' + MAKE_OPT='CCPP=Y SUITES=FV3_GFS_2017_gfdlmp,FV3_GFS_2017_gfdlmp_noahmp' + export COMPILE_NR=14 + COMPILE_NR=14 -+ cd /scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -+ [[ -e /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_136435/compile_14.env ]] -+ source /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_136435/compile_14.env ++ cd /scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests ++ [[ -e /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_293743/compile_14.env ]] ++ source /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_293743/compile_14.env ++ export MACHINE_ID=hera.intel ++ MACHINE_ID=hera.intel -++ export PATHRT=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -++ PATHRT=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -++ export PATHTR=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/ufs-weather-model -++ PATHTR=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/ufs-weather-model +++ export PATHRT=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests +++ PATHRT=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests +++ export PATHTR=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model +++ PATHTR=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model ++ export SCHEDULER=slurm ++ SCHEDULER=slurm ++ export ACCNR=nems @@ -510,10 +469,10 @@ Elapsed time 1692 seconds. Compile 13 ++ ROCOTO=false ++ export ECFLOW=true ++ ECFLOW=true -++ export REGRESSIONTEST_LOG=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/RegressionTests_hera.intel.log -++ REGRESSIONTEST_LOG=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/RegressionTests_hera.intel.log -++ export LOG_DIR=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/log_hera.intel -++ LOG_DIR=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/log_hera.intel +++ export REGRESSIONTEST_LOG=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/RegressionTests_hera.intel.log +++ REGRESSIONTEST_LOG=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/RegressionTests_hera.intel.log +++ export LOG_DIR=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/log_hera.intel +++ LOG_DIR=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/log_hera.intel + source default_vars.sh ++ '[' hera.intel = wcoss ']' ++ '[' hera.intel = wcoss_cray ']' @@ -546,15 +505,15 @@ Elapsed time 1692 seconds. Compile 13 + TEST_NR=14 + export JBNME=compile_14 + JBNME=compile_14 -+ export RUNDIR=/scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_136435/compile_14 -+ RUNDIR=/scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_136435/compile_14 ++ export RUNDIR=/scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_293743/compile_14 ++ RUNDIR=/scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_293743/compile_14 + source rt_utils.sh ++ set -eu -++ [[ /scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/run_compile.sh = \r\t\_\u\t\i\l\s\.\s\h ]] +++ [[ /scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/run_compile.sh = \r\t\_\u\t\i\l\s\.\s\h ]] ++ UNIT_TEST=false + source atparse.bash -+ mkdir -p /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_136435/compile_14 -+ cd /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_136435/compile_14 ++ mkdir -p /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_293743/compile_14 ++ cd /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_293743/compile_14 + [[ slurm = \s\l\u\r\m ]] + atparse + local __set_x @@ -567,7 +526,7 @@ Elapsed time 1692 seconds. Compile 13 + '[' -o xtrace ']' + set_x='set -x' + set +x -Job id 9560841 +Job id 9591234 TEST 14 compile is waiting to enter the queue 1 min. TEST 14 compile is pending, Status: PD 2 min. TEST 14 compile is pending, Status: PD @@ -577,53 +536,41 @@ TEST 14 compile is waiting to enter the queue 6 min. TEST 14 compile is pending, Status: PD 7 min. TEST 14 compile is pending, Status: PD 8 min. TEST 14 compile is pending, Status: PD -9 min. TEST 14 compile is pending, Status: PD -10 min. TEST 14 compile is pending, Status: PD -11 min. TEST 14 compile is pending, Status: PD -12 min. TEST 14 compile is pending, Status: PD -13 min. TEST 14 compile is pending, Status: PD -14 min. TEST 14 compile is pending, Status: PD -15 min. TEST 14 compile is pending, Status: PD -16 min. TEST 14 compile is pending, Status: PD -17 min. TEST 14 compile is pending, Status: PD -18 min. TEST 14 compile is pending, Status: PD -19 min. TEST 14 compile is pending, Status: PD -20 min. TEST 14 compile is pending, Status: PD -21 min. TEST 14 compile is running, Status: R -22 min. TEST 14 compile is running, Status: R -23 min. TEST 14 compile is running, Status: R -24 min. TEST 14 compile is running, Status: R -25 min. TEST 14 compile is running, Status: R -26 min. TEST 14 compile is running, Status: R +9 min. TEST 14 compile is running, Status: R +10 min. TEST 14 compile is running, Status: R +11 min. TEST 14 compile is running, Status: R +12 min. TEST 14 compile is running, Status: R +13 min. TEST 14 compile is running, Status: R +14 min. TEST 14 compile is running, Status: R Slurm unknown status -. Check sacct ... -9560841 COMPLETED compile_14 -9560841.bat+ COMPLETED batch -9560841.ext+ COMPLETED extern -27 min. TEST 14 compile is COMPLETED +9591234 COMPLETED compile_14 +9591234.bat+ COMPLETED batch +9591234.ext+ COMPLETED extern +15 min. TEST 14 compile is COMPLETED COMPLETED COMPLETED -+ elapsed=1631 -+ echo 'Elapsed time 1631 seconds. Compile 14' -Elapsed time 1631 seconds. Compile 14 ++ elapsed=908 ++ echo 'Elapsed time 908 seconds. Compile 14' +Elapsed time 908 seconds. Compile 14 + SECONDS=0 + [[ 4 != 4 ]] -+ export PATHRT=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -+ PATHRT=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -+ export RUNDIR_ROOT=/scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_136435 -+ RUNDIR_ROOT=/scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_136435 ++ export PATHRT=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests ++ PATHRT=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests ++ export RUNDIR_ROOT=/scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_293743 ++ RUNDIR_ROOT=/scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_293743 + export 'MAKE_OPT=CCPP=Y SUITES=FV3_GFS_2017' + MAKE_OPT='CCPP=Y SUITES=FV3_GFS_2017' + export COMPILE_NR=1 + COMPILE_NR=1 -+ cd /scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -+ [[ -e /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_136435/compile_1.env ]] -+ source /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_136435/compile_1.env ++ cd /scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests ++ [[ -e /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_293743/compile_1.env ]] ++ source /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_293743/compile_1.env ++ export MACHINE_ID=hera.intel ++ MACHINE_ID=hera.intel -++ export PATHRT=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -++ PATHRT=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -++ export PATHTR=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/ufs-weather-model -++ PATHTR=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/ufs-weather-model +++ export PATHRT=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests +++ PATHRT=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests +++ export PATHTR=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model +++ PATHTR=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model ++ export SCHEDULER=slurm ++ SCHEDULER=slurm ++ export ACCNR=nems @@ -636,10 +583,10 @@ Elapsed time 1631 seconds. Compile 14 ++ ROCOTO=false ++ export ECFLOW=true ++ ECFLOW=true -++ export REGRESSIONTEST_LOG=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/RegressionTests_hera.intel.log -++ REGRESSIONTEST_LOG=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/RegressionTests_hera.intel.log -++ export LOG_DIR=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/log_hera.intel -++ LOG_DIR=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/log_hera.intel +++ export REGRESSIONTEST_LOG=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/RegressionTests_hera.intel.log +++ REGRESSIONTEST_LOG=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/RegressionTests_hera.intel.log +++ export LOG_DIR=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/log_hera.intel +++ LOG_DIR=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/log_hera.intel + source default_vars.sh ++ '[' hera.intel = wcoss ']' ++ '[' hera.intel = wcoss_cray ']' @@ -672,15 +619,15 @@ Elapsed time 1631 seconds. Compile 14 + TEST_NR=1 + export JBNME=compile_1 + JBNME=compile_1 -+ export RUNDIR=/scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_136435/compile_1 -+ RUNDIR=/scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_136435/compile_1 ++ export RUNDIR=/scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_293743/compile_1 ++ RUNDIR=/scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_293743/compile_1 + source rt_utils.sh ++ set -eu -++ [[ /scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/run_compile.sh = \r\t\_\u\t\i\l\s\.\s\h ]] +++ [[ /scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/run_compile.sh = \r\t\_\u\t\i\l\s\.\s\h ]] ++ UNIT_TEST=false + source atparse.bash -+ mkdir -p /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_136435/compile_1 -+ cd /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_136435/compile_1 ++ mkdir -p /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_293743/compile_1 ++ cd /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_293743/compile_1 + [[ slurm = \s\l\u\r\m ]] + atparse + local __set_x @@ -693,7 +640,7 @@ Elapsed time 1631 seconds. Compile 14 + '[' -o xtrace ']' + set_x='set -x' + set +x -Job id 9560616 +Job id 9590931 TEST 1 compile is waiting to enter the queue 1 min. TEST 1 compile is running, Status: R 2 min. TEST 1 compile is running, Status: R @@ -702,9 +649,9 @@ TEST 1 compile is waiting to enter the queue 5 min. TEST 1 compile is running, Status: R 6 min. TEST 1 compile is running, Status: R Slurm unknown status -. Check sacct ... -9560616 COMPLETED compile_1 -9560616.bat+ COMPLETED batch -9560616.ext+ COMPLETED extern +9590931 COMPLETED compile_1 +9590931.bat+ COMPLETED batch +9590931.ext+ COMPLETED extern 7 min. TEST 1 compile is COMPLETED COMPLETED COMPLETED @@ -713,23 +660,23 @@ COMPLETED Elapsed time 426 seconds. Compile 1 + SECONDS=0 + [[ 4 != 4 ]] -+ export PATHRT=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -+ PATHRT=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -+ export RUNDIR_ROOT=/scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_136435 -+ RUNDIR_ROOT=/scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_136435 ++ export PATHRT=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests ++ PATHRT=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests ++ export RUNDIR_ROOT=/scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_293743 ++ RUNDIR_ROOT=/scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_293743 + export 'MAKE_OPT=CCPP=Y SUITES=FV3_GFS_2017,FV3_GFS_2017_gfdlmp WW3=Y' + MAKE_OPT='CCPP=Y SUITES=FV3_GFS_2017,FV3_GFS_2017_gfdlmp WW3=Y' + export COMPILE_NR=2 + COMPILE_NR=2 -+ cd /scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -+ [[ -e /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_136435/compile_2.env ]] -+ source /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_136435/compile_2.env ++ cd /scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests ++ [[ -e /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_293743/compile_2.env ]] ++ source /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_293743/compile_2.env ++ export MACHINE_ID=hera.intel ++ MACHINE_ID=hera.intel -++ export PATHRT=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -++ PATHRT=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -++ export PATHTR=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/ufs-weather-model -++ PATHTR=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/ufs-weather-model +++ export PATHRT=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests +++ PATHRT=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests +++ export PATHTR=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model +++ PATHTR=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model ++ export SCHEDULER=slurm ++ SCHEDULER=slurm ++ export ACCNR=nems @@ -742,10 +689,10 @@ Elapsed time 426 seconds. Compile 1 ++ ROCOTO=false ++ export ECFLOW=true ++ ECFLOW=true -++ export REGRESSIONTEST_LOG=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/RegressionTests_hera.intel.log -++ REGRESSIONTEST_LOG=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/RegressionTests_hera.intel.log -++ export LOG_DIR=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/log_hera.intel -++ LOG_DIR=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/log_hera.intel +++ export REGRESSIONTEST_LOG=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/RegressionTests_hera.intel.log +++ REGRESSIONTEST_LOG=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/RegressionTests_hera.intel.log +++ export LOG_DIR=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/log_hera.intel +++ LOG_DIR=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/log_hera.intel + source default_vars.sh ++ '[' hera.intel = wcoss ']' ++ '[' hera.intel = wcoss_cray ']' @@ -778,15 +725,15 @@ Elapsed time 426 seconds. Compile 1 + TEST_NR=2 + export JBNME=compile_2 + JBNME=compile_2 -+ export RUNDIR=/scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_136435/compile_2 -+ RUNDIR=/scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_136435/compile_2 ++ export RUNDIR=/scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_293743/compile_2 ++ RUNDIR=/scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_293743/compile_2 + source rt_utils.sh ++ set -eu -++ [[ /scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/run_compile.sh = \r\t\_\u\t\i\l\s\.\s\h ]] +++ [[ /scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/run_compile.sh = \r\t\_\u\t\i\l\s\.\s\h ]] ++ UNIT_TEST=false + source atparse.bash -+ mkdir -p /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_136435/compile_2 -+ cd /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_136435/compile_2 ++ mkdir -p /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_293743/compile_2 ++ cd /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_293743/compile_2 + [[ slurm = \s\l\u\r\m ]] + atparse + local __set_x @@ -799,7 +746,7 @@ Elapsed time 426 seconds. Compile 1 + '[' -o xtrace ']' + set_x='set -x' + set +x -Job id 9560615 +Job id 9590932 TEST 2 compile is waiting to enter the queue 1 min. TEST 2 compile is running, Status: R 2 min. TEST 2 compile is running, Status: R @@ -807,35 +754,36 @@ TEST 2 compile is waiting to enter the queue 4 min. TEST 2 compile is running, Status: R 5 min. TEST 2 compile is running, Status: R 6 min. TEST 2 compile is running, Status: R +7 min. TEST 2 compile is running, Status: R Slurm unknown status -. Check sacct ... -9560615 COMPLETED compile_2 -9560615.bat+ COMPLETED batch -9560615.ext+ COMPLETED extern -7 min. TEST 2 compile is COMPLETED +9590932 COMPLETED compile_2 +9590932.bat+ COMPLETED batch +9590932.ext+ COMPLETED extern +8 min. TEST 2 compile is COMPLETED COMPLETED COMPLETED -+ elapsed=426 -+ echo 'Elapsed time 426 seconds. Compile 2' -Elapsed time 426 seconds. Compile 2 ++ elapsed=486 ++ echo 'Elapsed time 486 seconds. Compile 2' +Elapsed time 486 seconds. Compile 2 + SECONDS=0 + [[ 4 != 4 ]] -+ export PATHRT=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -+ PATHRT=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -+ export RUNDIR_ROOT=/scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_136435 -+ RUNDIR_ROOT=/scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_136435 ++ export PATHRT=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests ++ PATHRT=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests ++ export RUNDIR_ROOT=/scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_293743 ++ RUNDIR_ROOT=/scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_293743 + export 'MAKE_OPT=CCPP=Y SUITES=FV3_GFS_2017_fv3wam 32BIT=Y MULTI_GASES=Y' + MAKE_OPT='CCPP=Y SUITES=FV3_GFS_2017_fv3wam 32BIT=Y MULTI_GASES=Y' + export COMPILE_NR=3 + COMPILE_NR=3 -+ cd /scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -+ [[ -e /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_136435/compile_3.env ]] -+ source /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_136435/compile_3.env ++ cd /scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests ++ [[ -e /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_293743/compile_3.env ]] ++ source /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_293743/compile_3.env ++ export MACHINE_ID=hera.intel ++ MACHINE_ID=hera.intel -++ export PATHRT=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -++ PATHRT=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -++ export PATHTR=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/ufs-weather-model -++ PATHTR=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/ufs-weather-model +++ export PATHRT=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests +++ PATHRT=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests +++ export PATHTR=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model +++ PATHTR=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model ++ export SCHEDULER=slurm ++ SCHEDULER=slurm ++ export ACCNR=nems @@ -848,10 +796,10 @@ Elapsed time 426 seconds. Compile 2 ++ ROCOTO=false ++ export ECFLOW=true ++ ECFLOW=true -++ export REGRESSIONTEST_LOG=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/RegressionTests_hera.intel.log -++ REGRESSIONTEST_LOG=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/RegressionTests_hera.intel.log -++ export LOG_DIR=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/log_hera.intel -++ LOG_DIR=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/log_hera.intel +++ export REGRESSIONTEST_LOG=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/RegressionTests_hera.intel.log +++ REGRESSIONTEST_LOG=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/RegressionTests_hera.intel.log +++ export LOG_DIR=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/log_hera.intel +++ LOG_DIR=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/log_hera.intel + source default_vars.sh ++ '[' hera.intel = wcoss ']' ++ '[' hera.intel = wcoss_cray ']' @@ -884,15 +832,15 @@ Elapsed time 426 seconds. Compile 2 + TEST_NR=3 + export JBNME=compile_3 + JBNME=compile_3 -+ export RUNDIR=/scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_136435/compile_3 -+ RUNDIR=/scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_136435/compile_3 ++ export RUNDIR=/scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_293743/compile_3 ++ RUNDIR=/scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_293743/compile_3 + source rt_utils.sh ++ set -eu -++ [[ /scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/run_compile.sh = \r\t\_\u\t\i\l\s\.\s\h ]] +++ [[ /scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/run_compile.sh = \r\t\_\u\t\i\l\s\.\s\h ]] ++ UNIT_TEST=false + source atparse.bash -+ mkdir -p /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_136435/compile_3 -+ cd /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_136435/compile_3 ++ mkdir -p /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_293743/compile_3 ++ cd /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_293743/compile_3 + [[ slurm = \s\l\u\r\m ]] + atparse + local __set_x @@ -905,43 +853,46 @@ Elapsed time 426 seconds. Compile 2 + '[' -o xtrace ']' + set_x='set -x' + set +x -Job id 9560618 +Job id 9590934 TEST 3 compile is waiting to enter the queue -1 min. TEST 3 compile is running, Status: R +1 min. TEST 3 compile is pending, Status: PD 2 min. TEST 3 compile is running, Status: R 3 min. TEST 3 compile is running, Status: R 4 min. TEST 3 compile is running, Status: R 5 min. TEST 3 compile is running, Status: R 6 min. TEST 3 compile is running, Status: R +7 min. TEST 3 compile is running, Status: R +8 min. TEST 3 compile is running, Status: R +9 min. TEST 3 compile is running, Status: R Slurm unknown status -. Check sacct ... -9560618 COMPLETED compile_3 -9560618.bat+ COMPLETED batch -9560618.ext+ COMPLETED extern -7 min. TEST 3 compile is COMPLETED +9590934 COMPLETED compile_3 +9590934.bat+ COMPLETED batch +9590934.ext+ COMPLETED extern +10 min. TEST 3 compile is COMPLETED COMPLETED COMPLETED -+ elapsed=426 -+ echo 'Elapsed time 426 seconds. Compile 3' -Elapsed time 426 seconds. Compile 3 ++ elapsed=607 ++ echo 'Elapsed time 607 seconds. Compile 3' +Elapsed time 607 seconds. Compile 3 + SECONDS=0 + [[ 4 != 4 ]] -+ export PATHRT=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -+ PATHRT=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -+ export RUNDIR_ROOT=/scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_136435 -+ RUNDIR_ROOT=/scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_136435 ++ export PATHRT=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests ++ PATHRT=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests ++ export RUNDIR_ROOT=/scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_293743 ++ RUNDIR_ROOT=/scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_293743 + export 'MAKE_OPT=CCPP=Y SUITES=FV3_GFS_2017,FV3_GFS_2017_stretched 32BIT=Y' + MAKE_OPT='CCPP=Y SUITES=FV3_GFS_2017,FV3_GFS_2017_stretched 32BIT=Y' + export COMPILE_NR=4 + COMPILE_NR=4 -+ cd /scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -+ [[ -e /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_136435/compile_4.env ]] -+ source /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_136435/compile_4.env ++ cd /scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests ++ [[ -e /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_293743/compile_4.env ]] ++ source /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_293743/compile_4.env ++ export MACHINE_ID=hera.intel ++ MACHINE_ID=hera.intel -++ export PATHRT=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -++ PATHRT=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -++ export PATHTR=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/ufs-weather-model -++ PATHTR=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/ufs-weather-model +++ export PATHRT=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests +++ PATHRT=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests +++ export PATHTR=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model +++ PATHTR=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model ++ export SCHEDULER=slurm ++ SCHEDULER=slurm ++ export ACCNR=nems @@ -954,10 +905,10 @@ Elapsed time 426 seconds. Compile 3 ++ ROCOTO=false ++ export ECFLOW=true ++ ECFLOW=true -++ export REGRESSIONTEST_LOG=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/RegressionTests_hera.intel.log -++ REGRESSIONTEST_LOG=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/RegressionTests_hera.intel.log -++ export LOG_DIR=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/log_hera.intel -++ LOG_DIR=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/log_hera.intel +++ export REGRESSIONTEST_LOG=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/RegressionTests_hera.intel.log +++ REGRESSIONTEST_LOG=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/RegressionTests_hera.intel.log +++ export LOG_DIR=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/log_hera.intel +++ LOG_DIR=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/log_hera.intel + source default_vars.sh ++ '[' hera.intel = wcoss ']' ++ '[' hera.intel = wcoss_cray ']' @@ -990,15 +941,15 @@ Elapsed time 426 seconds. Compile 3 + TEST_NR=4 + export JBNME=compile_4 + JBNME=compile_4 -+ export RUNDIR=/scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_136435/compile_4 -+ RUNDIR=/scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_136435/compile_4 ++ export RUNDIR=/scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_293743/compile_4 ++ RUNDIR=/scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_293743/compile_4 + source rt_utils.sh ++ set -eu -++ [[ /scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/run_compile.sh = \r\t\_\u\t\i\l\s\.\s\h ]] +++ [[ /scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/run_compile.sh = \r\t\_\u\t\i\l\s\.\s\h ]] ++ UNIT_TEST=false + source atparse.bash -+ mkdir -p /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_136435/compile_4 -+ cd /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_136435/compile_4 ++ mkdir -p /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_293743/compile_4 ++ cd /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_293743/compile_4 + [[ slurm = \s\l\u\r\m ]] + atparse + local __set_x @@ -1011,43 +962,45 @@ Elapsed time 426 seconds. Compile 3 + '[' -o xtrace ']' + set_x='set -x' + set +x -Job id 9560619 +Job id 9590933 TEST 4 compile is waiting to enter the queue -1 min. TEST 4 compile is running, Status: R +1 min. TEST 4 compile is pending, Status: PD 2 min. TEST 4 compile is running, Status: R 3 min. TEST 4 compile is running, Status: R 4 min. TEST 4 compile is running, Status: R 5 min. TEST 4 compile is running, Status: R 6 min. TEST 4 compile is running, Status: R +7 min. TEST 4 compile is running, Status: R +8 min. TEST 4 compile is running, Status: R Slurm unknown status -. Check sacct ... -9560619 COMPLETED compile_4 -9560619.bat+ COMPLETED batch -9560619.ext+ COMPLETED extern -7 min. TEST 4 compile is COMPLETED +9590933 COMPLETED compile_4 +9590933.bat+ COMPLETED batch +9590933.ext+ COMPLETED extern +9 min. TEST 4 compile is COMPLETED COMPLETED COMPLETED -+ elapsed=426 -+ echo 'Elapsed time 426 seconds. Compile 4' -Elapsed time 426 seconds. Compile 4 ++ elapsed=547 ++ echo 'Elapsed time 547 seconds. Compile 4' +Elapsed time 547 seconds. Compile 4 + SECONDS=0 + [[ 4 != 4 ]] -+ export PATHRT=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -+ PATHRT=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -+ export RUNDIR_ROOT=/scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_136435 -+ RUNDIR_ROOT=/scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_136435 ++ export PATHRT=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests ++ PATHRT=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests ++ export RUNDIR_ROOT=/scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_293743 ++ RUNDIR_ROOT=/scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_293743 + export 'MAKE_OPT=CCPP=Y SUITES=FV3_GFS_2017_gfdlmp_regional,FV3_GFS_2017_gfdlmp_regional_c768 32BIT=Y' + MAKE_OPT='CCPP=Y SUITES=FV3_GFS_2017_gfdlmp_regional,FV3_GFS_2017_gfdlmp_regional_c768 32BIT=Y' + export COMPILE_NR=5 + COMPILE_NR=5 -+ cd /scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -+ [[ -e /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_136435/compile_5.env ]] -+ source /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_136435/compile_5.env ++ cd /scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests ++ [[ -e /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_293743/compile_5.env ]] ++ source /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_293743/compile_5.env ++ export MACHINE_ID=hera.intel ++ MACHINE_ID=hera.intel -++ export PATHRT=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -++ PATHRT=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -++ export PATHTR=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/ufs-weather-model -++ PATHTR=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/ufs-weather-model +++ export PATHRT=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests +++ PATHRT=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests +++ export PATHTR=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model +++ PATHTR=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model ++ export SCHEDULER=slurm ++ SCHEDULER=slurm ++ export ACCNR=nems @@ -1060,10 +1013,10 @@ Elapsed time 426 seconds. Compile 4 ++ ROCOTO=false ++ export ECFLOW=true ++ ECFLOW=true -++ export REGRESSIONTEST_LOG=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/RegressionTests_hera.intel.log -++ REGRESSIONTEST_LOG=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/RegressionTests_hera.intel.log -++ export LOG_DIR=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/log_hera.intel -++ LOG_DIR=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/log_hera.intel +++ export REGRESSIONTEST_LOG=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/RegressionTests_hera.intel.log +++ REGRESSIONTEST_LOG=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/RegressionTests_hera.intel.log +++ export LOG_DIR=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/log_hera.intel +++ LOG_DIR=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/log_hera.intel + source default_vars.sh ++ '[' hera.intel = wcoss ']' ++ '[' hera.intel = wcoss_cray ']' @@ -1096,15 +1049,15 @@ Elapsed time 426 seconds. Compile 4 + TEST_NR=5 + export JBNME=compile_5 + JBNME=compile_5 -+ export RUNDIR=/scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_136435/compile_5 -+ RUNDIR=/scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_136435/compile_5 ++ export RUNDIR=/scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_293743/compile_5 ++ RUNDIR=/scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_293743/compile_5 + source rt_utils.sh ++ set -eu -++ [[ /scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/run_compile.sh = \r\t\_\u\t\i\l\s\.\s\h ]] +++ [[ /scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/run_compile.sh = \r\t\_\u\t\i\l\s\.\s\h ]] ++ UNIT_TEST=false + source atparse.bash -+ mkdir -p /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_136435/compile_5 -+ cd /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_136435/compile_5 ++ mkdir -p /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_293743/compile_5 ++ cd /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_293743/compile_5 + [[ slurm = \s\l\u\r\m ]] + atparse + local __set_x @@ -1117,43 +1070,50 @@ Elapsed time 426 seconds. Compile 4 + '[' -o xtrace ']' + set_x='set -x' + set +x -Job id 9560617 +Job id 9590936 TEST 5 compile is waiting to enter the queue -1 min. TEST 5 compile is running, Status: R +1 min. TEST 5 compile is pending, Status: PD 2 min. TEST 5 compile is running, Status: R 3 min. TEST 5 compile is running, Status: R 4 min. TEST 5 compile is running, Status: R 5 min. TEST 5 compile is running, Status: R 6 min. TEST 5 compile is running, Status: R +7 min. TEST 5 compile is running, Status: R +8 min. TEST 5 compile is running, Status: R +9 min. TEST 5 compile is running, Status: R +10 min. TEST 5 compile is running, Status: R +11 min. TEST 5 compile is running, Status: R +12 min. TEST 5 compile is running, Status: R +13 min. TEST 5 compile is running, Status: R Slurm unknown status -. Check sacct ... -9560617 COMPLETED compile_5 -9560617.bat+ COMPLETED batch -9560617.ext+ COMPLETED extern -7 min. TEST 5 compile is COMPLETED +9590936 COMPLETED compile_5 +9590936.bat+ COMPLETED batch +9590936.ext+ COMPLETED extern +14 min. TEST 5 compile is COMPLETED COMPLETED COMPLETED -+ elapsed=426 -+ echo 'Elapsed time 426 seconds. Compile 5' -Elapsed time 426 seconds. Compile 5 ++ elapsed=847 ++ echo 'Elapsed time 847 seconds. Compile 5' +Elapsed time 847 seconds. Compile 5 + SECONDS=0 + [[ 4 != 4 ]] -+ export PATHRT=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -+ PATHRT=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -+ export RUNDIR_ROOT=/scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_136435 -+ RUNDIR_ROOT=/scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_136435 ++ export PATHRT=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests ++ PATHRT=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests ++ export RUNDIR_ROOT=/scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_293743 ++ RUNDIR_ROOT=/scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_293743 + export 'MAKE_OPT=CCPP=Y SUITES=FV3_GFS_2017,FV3_GFS_2017_stretched 32BIT=Y DEBUG=Y' + MAKE_OPT='CCPP=Y SUITES=FV3_GFS_2017,FV3_GFS_2017_stretched 32BIT=Y DEBUG=Y' + export COMPILE_NR=6 + COMPILE_NR=6 -+ cd /scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -+ [[ -e /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_136435/compile_6.env ]] -+ source /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_136435/compile_6.env ++ cd /scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests ++ [[ -e /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_293743/compile_6.env ]] ++ source /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_293743/compile_6.env ++ export MACHINE_ID=hera.intel ++ MACHINE_ID=hera.intel -++ export PATHRT=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -++ PATHRT=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -++ export PATHTR=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/ufs-weather-model -++ PATHTR=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/ufs-weather-model +++ export PATHRT=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests +++ PATHRT=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests +++ export PATHTR=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model +++ PATHTR=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model ++ export SCHEDULER=slurm ++ SCHEDULER=slurm ++ export ACCNR=nems @@ -1166,10 +1126,10 @@ Elapsed time 426 seconds. Compile 5 ++ ROCOTO=false ++ export ECFLOW=true ++ ECFLOW=true -++ export REGRESSIONTEST_LOG=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/RegressionTests_hera.intel.log -++ REGRESSIONTEST_LOG=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/RegressionTests_hera.intel.log -++ export LOG_DIR=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/log_hera.intel -++ LOG_DIR=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/log_hera.intel +++ export REGRESSIONTEST_LOG=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/RegressionTests_hera.intel.log +++ REGRESSIONTEST_LOG=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/RegressionTests_hera.intel.log +++ export LOG_DIR=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/log_hera.intel +++ LOG_DIR=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/log_hera.intel + source default_vars.sh ++ '[' hera.intel = wcoss ']' ++ '[' hera.intel = wcoss_cray ']' @@ -1202,15 +1162,15 @@ Elapsed time 426 seconds. Compile 5 + TEST_NR=6 + export JBNME=compile_6 + JBNME=compile_6 -+ export RUNDIR=/scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_136435/compile_6 -+ RUNDIR=/scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_136435/compile_6 ++ export RUNDIR=/scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_293743/compile_6 ++ RUNDIR=/scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_293743/compile_6 + source rt_utils.sh ++ set -eu -++ [[ /scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/run_compile.sh = \r\t\_\u\t\i\l\s\.\s\h ]] +++ [[ /scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/run_compile.sh = \r\t\_\u\t\i\l\s\.\s\h ]] ++ UNIT_TEST=false + source atparse.bash -+ mkdir -p /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_136435/compile_6 -+ cd /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_136435/compile_6 ++ mkdir -p /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_293743/compile_6 ++ cd /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_293743/compile_6 + [[ slurm = \s\l\u\r\m ]] + atparse + local __set_x @@ -1223,39 +1183,49 @@ Elapsed time 426 seconds. Compile 5 + '[' -o xtrace ']' + set_x='set -x' + set +x -Job id 9560620 +Job id 9590937 TEST 6 compile is waiting to enter the queue -1 min. TEST 6 compile is running, Status: R +1 min. TEST 6 compile is pending, Status: PD 2 min. TEST 6 compile is running, Status: R +3 min. TEST 6 compile is running, Status: R +4 min. TEST 6 compile is running, Status: R +5 min. TEST 6 compile is running, Status: R +Slurm unknown status CG. Check sacct ... +9590937 COMPLETED compile_6 +9590937.bat+ COMPLETED batch +9590937.ext+ COMPLETED extern +6 min. TEST 6 compile is COMPLETED +COMPLETED +COMPLETED Slurm unknown status -. Check sacct ... -9560620 COMPLETED compile_6 -9560620.bat+ COMPLETED batch -9560620.ext+ COMPLETED extern -3 min. TEST 6 compile is COMPLETED +9590937 COMPLETED compile_6 +9590937.bat+ COMPLETED batch +9590937.ext+ COMPLETED extern +7 min. TEST 6 compile is COMPLETED COMPLETED COMPLETED -+ elapsed=186 -+ echo 'Elapsed time 186 seconds. Compile 6' -Elapsed time 186 seconds. Compile 6 ++ elapsed=426 ++ echo 'Elapsed time 426 seconds. Compile 6' +Elapsed time 426 seconds. Compile 6 + SECONDS=0 + [[ 4 != 4 ]] -+ export PATHRT=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -+ PATHRT=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -+ export RUNDIR_ROOT=/scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_136435 -+ RUNDIR_ROOT=/scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_136435 ++ export PATHRT=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests ++ PATHRT=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests ++ export RUNDIR_ROOT=/scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_293743 ++ RUNDIR_ROOT=/scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_293743 + export 'MAKE_OPT=CCPP=Y SUITES=FV3_GFS_2017_gfdlmp,FV3_GFS_2017_gfdlmp_noahmp' + MAKE_OPT='CCPP=Y SUITES=FV3_GFS_2017_gfdlmp,FV3_GFS_2017_gfdlmp_noahmp' + export COMPILE_NR=7 + COMPILE_NR=7 -+ cd /scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -+ [[ -e /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_136435/compile_7.env ]] -+ source /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_136435/compile_7.env ++ cd /scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests ++ [[ -e /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_293743/compile_7.env ]] ++ source /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_293743/compile_7.env ++ export MACHINE_ID=hera.intel ++ MACHINE_ID=hera.intel -++ export PATHRT=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -++ PATHRT=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -++ export PATHTR=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/ufs-weather-model -++ PATHTR=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/ufs-weather-model +++ export PATHRT=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests +++ PATHRT=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests +++ export PATHTR=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model +++ PATHTR=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model ++ export SCHEDULER=slurm ++ SCHEDULER=slurm ++ export ACCNR=nems @@ -1268,10 +1238,10 @@ Elapsed time 186 seconds. Compile 6 ++ ROCOTO=false ++ export ECFLOW=true ++ ECFLOW=true -++ export REGRESSIONTEST_LOG=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/RegressionTests_hera.intel.log -++ REGRESSIONTEST_LOG=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/RegressionTests_hera.intel.log -++ export LOG_DIR=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/log_hera.intel -++ LOG_DIR=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/log_hera.intel +++ export REGRESSIONTEST_LOG=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/RegressionTests_hera.intel.log +++ REGRESSIONTEST_LOG=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/RegressionTests_hera.intel.log +++ export LOG_DIR=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/log_hera.intel +++ LOG_DIR=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/log_hera.intel + source default_vars.sh ++ '[' hera.intel = wcoss ']' ++ '[' hera.intel = wcoss_cray ']' @@ -1304,15 +1274,15 @@ Elapsed time 186 seconds. Compile 6 + TEST_NR=7 + export JBNME=compile_7 + JBNME=compile_7 -+ export RUNDIR=/scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_136435/compile_7 -+ RUNDIR=/scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_136435/compile_7 ++ export RUNDIR=/scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_293743/compile_7 ++ RUNDIR=/scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_293743/compile_7 + source rt_utils.sh ++ set -eu -++ [[ /scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/run_compile.sh = \r\t\_\u\t\i\l\s\.\s\h ]] +++ [[ /scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/run_compile.sh = \r\t\_\u\t\i\l\s\.\s\h ]] ++ UNIT_TEST=false + source atparse.bash -+ mkdir -p /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_136435/compile_7 -+ cd /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_136435/compile_7 ++ mkdir -p /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_293743/compile_7 ++ cd /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_293743/compile_7 + [[ slurm = \s\l\u\r\m ]] + atparse + local __set_x @@ -1325,43 +1295,52 @@ Elapsed time 186 seconds. Compile 6 + '[' -o xtrace ']' + set_x='set -x' + set +x -Job id 9560621 +Job id 9590935 TEST 7 compile is waiting to enter the queue -1 min. TEST 7 compile is running, Status: R +1 min. TEST 7 compile is pending, Status: PD 2 min. TEST 7 compile is running, Status: R 3 min. TEST 7 compile is running, Status: R 4 min. TEST 7 compile is running, Status: R 5 min. TEST 7 compile is running, Status: R 6 min. TEST 7 compile is running, Status: R +7 min. TEST 7 compile is running, Status: R +8 min. TEST 7 compile is running, Status: R +Slurm unknown status CG. Check sacct ... +9590935 COMPLETED compile_7 +9590935.bat+ COMPLETED batch +9590935.ext+ COMPLETED extern +9 min. TEST 7 compile is COMPLETED +COMPLETED +COMPLETED Slurm unknown status -. Check sacct ... -9560621 COMPLETED compile_7 -9560621.bat+ COMPLETED batch -9560621.ext+ COMPLETED extern -7 min. TEST 7 compile is COMPLETED +9590935 COMPLETED compile_7 +9590935.bat+ COMPLETED batch +9590935.ext+ COMPLETED extern +10 min. TEST 7 compile is COMPLETED COMPLETED COMPLETED -+ elapsed=426 -+ echo 'Elapsed time 426 seconds. Compile 7' -Elapsed time 426 seconds. Compile 7 ++ elapsed=607 ++ echo 'Elapsed time 607 seconds. Compile 7' +Elapsed time 607 seconds. Compile 7 + SECONDS=0 + [[ 4 != 4 ]] -+ export PATHRT=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -+ PATHRT=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -+ export RUNDIR_ROOT=/scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_136435 -+ RUNDIR_ROOT=/scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_136435 ++ export PATHRT=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests ++ PATHRT=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests ++ export RUNDIR_ROOT=/scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_293743 ++ RUNDIR_ROOT=/scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_293743 + export 'MAKE_OPT=CCPP=Y SUITES=FV3_GFS_2017_csawmgshoc,FV3_GFS_2017_csawmg,FV3_GFS_2017_satmedmf,FV3_GFS_2017_satmedmfq' + MAKE_OPT='CCPP=Y SUITES=FV3_GFS_2017_csawmgshoc,FV3_GFS_2017_csawmg,FV3_GFS_2017_satmedmf,FV3_GFS_2017_satmedmfq' + export COMPILE_NR=8 + COMPILE_NR=8 -+ cd /scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -+ [[ -e /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_136435/compile_8.env ]] -+ source /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_136435/compile_8.env ++ cd /scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests ++ [[ -e /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_293743/compile_8.env ]] ++ source /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_293743/compile_8.env ++ export MACHINE_ID=hera.intel ++ MACHINE_ID=hera.intel -++ export PATHRT=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -++ PATHRT=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -++ export PATHTR=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/ufs-weather-model -++ PATHTR=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/ufs-weather-model +++ export PATHRT=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests +++ PATHRT=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests +++ export PATHTR=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model +++ PATHTR=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model ++ export SCHEDULER=slurm ++ SCHEDULER=slurm ++ export ACCNR=nems @@ -1374,10 +1353,10 @@ Elapsed time 426 seconds. Compile 7 ++ ROCOTO=false ++ export ECFLOW=true ++ ECFLOW=true -++ export REGRESSIONTEST_LOG=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/RegressionTests_hera.intel.log -++ REGRESSIONTEST_LOG=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/RegressionTests_hera.intel.log -++ export LOG_DIR=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/log_hera.intel -++ LOG_DIR=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/log_hera.intel +++ export REGRESSIONTEST_LOG=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/RegressionTests_hera.intel.log +++ REGRESSIONTEST_LOG=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/RegressionTests_hera.intel.log +++ export LOG_DIR=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/log_hera.intel +++ LOG_DIR=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/log_hera.intel + source default_vars.sh ++ '[' hera.intel = wcoss ']' ++ '[' hera.intel = wcoss_cray ']' @@ -1410,15 +1389,15 @@ Elapsed time 426 seconds. Compile 7 + TEST_NR=8 + export JBNME=compile_8 + JBNME=compile_8 -+ export RUNDIR=/scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_136435/compile_8 -+ RUNDIR=/scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_136435/compile_8 ++ export RUNDIR=/scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_293743/compile_8 ++ RUNDIR=/scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_293743/compile_8 + source rt_utils.sh ++ set -eu -++ [[ /scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/run_compile.sh = \r\t\_\u\t\i\l\s\.\s\h ]] +++ [[ /scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/run_compile.sh = \r\t\_\u\t\i\l\s\.\s\h ]] ++ UNIT_TEST=false + source atparse.bash -+ mkdir -p /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_136435/compile_8 -+ cd /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_136435/compile_8 ++ mkdir -p /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_293743/compile_8 ++ cd /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_293743/compile_8 + [[ slurm = \s\l\u\r\m ]] + atparse + local __set_x @@ -1431,43 +1410,48 @@ Elapsed time 426 seconds. Compile 7 + '[' -o xtrace ']' + set_x='set -x' + set +x -Job id 9560622 +Job id 9590938 TEST 8 compile is waiting to enter the queue -1 min. TEST 8 compile is running, Status: R +1 min. TEST 8 compile is pending, Status: PD 2 min. TEST 8 compile is running, Status: R 3 min. TEST 8 compile is running, Status: R 4 min. TEST 8 compile is running, Status: R 5 min. TEST 8 compile is running, Status: R 6 min. TEST 8 compile is running, Status: R +7 min. TEST 8 compile is running, Status: R +8 min. TEST 8 compile is running, Status: R +9 min. TEST 8 compile is running, Status: R +10 min. TEST 8 compile is running, Status: R +11 min. TEST 8 compile is running, Status: R Slurm unknown status -. Check sacct ... -9560622 COMPLETED compile_8 -9560622.bat+ COMPLETED batch -9560622.ext+ COMPLETED extern -7 min. TEST 8 compile is COMPLETED +9590938 COMPLETED compile_8 +9590938.bat+ COMPLETED batch +9590938.ext+ COMPLETED extern +12 min. TEST 8 compile is COMPLETED COMPLETED COMPLETED -+ elapsed=426 -+ echo 'Elapsed time 426 seconds. Compile 8' -Elapsed time 426 seconds. Compile 8 ++ elapsed=727 ++ echo 'Elapsed time 727 seconds. Compile 8' +Elapsed time 727 seconds. Compile 8 + SECONDS=0 + [[ 4 != 4 ]] -+ export PATHRT=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -+ PATHRT=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -+ export RUNDIR_ROOT=/scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_136435 -+ RUNDIR_ROOT=/scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_136435 ++ export PATHRT=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests ++ PATHRT=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests ++ export RUNDIR_ROOT=/scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_293743 ++ RUNDIR_ROOT=/scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_293743 + export 'MAKE_OPT=CCPP=Y SUITES=FV3_GFS_2017_gfdlmp,FV3_CPT_v0,FV3_GSD_v0,FV3_GFS_v15_thompson,FV3_RAP,FV3_HRRR 32BIT=Y' + MAKE_OPT='CCPP=Y SUITES=FV3_GFS_2017_gfdlmp,FV3_CPT_v0,FV3_GSD_v0,FV3_GFS_v15_thompson,FV3_RAP,FV3_HRRR 32BIT=Y' + export COMPILE_NR=9 + COMPILE_NR=9 -+ cd /scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -+ [[ -e /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_136435/compile_9.env ]] -+ source /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_136435/compile_9.env ++ cd /scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests ++ [[ -e /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_293743/compile_9.env ]] ++ source /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_293743/compile_9.env ++ export MACHINE_ID=hera.intel ++ MACHINE_ID=hera.intel -++ export PATHRT=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -++ PATHRT=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests -++ export PATHTR=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/ufs-weather-model -++ PATHTR=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/ufs-weather-model +++ export PATHRT=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests +++ PATHRT=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests +++ export PATHTR=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model +++ PATHTR=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model ++ export SCHEDULER=slurm ++ SCHEDULER=slurm ++ export ACCNR=nems @@ -1480,10 +1464,10 @@ Elapsed time 426 seconds. Compile 8 ++ ROCOTO=false ++ export ECFLOW=true ++ ECFLOW=true -++ export REGRESSIONTEST_LOG=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/RegressionTests_hera.intel.log -++ REGRESSIONTEST_LOG=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/RegressionTests_hera.intel.log -++ export LOG_DIR=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/log_hera.intel -++ LOG_DIR=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/log_hera.intel +++ export REGRESSIONTEST_LOG=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/RegressionTests_hera.intel.log +++ REGRESSIONTEST_LOG=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/RegressionTests_hera.intel.log +++ export LOG_DIR=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/log_hera.intel +++ LOG_DIR=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/log_hera.intel + source default_vars.sh ++ '[' hera.intel = wcoss ']' ++ '[' hera.intel = wcoss_cray ']' @@ -1516,15 +1500,15 @@ Elapsed time 426 seconds. Compile 8 + TEST_NR=9 + export JBNME=compile_9 + JBNME=compile_9 -+ export RUNDIR=/scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_136435/compile_9 -+ RUNDIR=/scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_136435/compile_9 ++ export RUNDIR=/scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_293743/compile_9 ++ RUNDIR=/scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_293743/compile_9 + source rt_utils.sh ++ set -eu -++ [[ /scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/pr154/ufs-weather-model/tests/run_compile.sh = \r\t\_\u\t\i\l\s\.\s\h ]] +++ [[ /scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ufs/rt_ecf_suite/ufs-weather-model/tests/run_compile.sh = \r\t\_\u\t\i\l\s\.\s\h ]] ++ UNIT_TEST=false + source atparse.bash -+ mkdir -p /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_136435/compile_9 -+ cd /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_136435/compile_9 ++ mkdir -p /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_293743/compile_9 ++ cd /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_293743/compile_9 + [[ slurm = \s\l\u\r\m ]] + atparse + local __set_x @@ -1537,25 +1521,19 @@ Elapsed time 426 seconds. Compile 8 + '[' -o xtrace ']' + set_x='set -x' + set +x -Job id 9560623 +Job id 9590939 TEST 9 compile is waiting to enter the queue -1 min. TEST 9 compile is running, Status: R +1 min. TEST 9 compile is pending, Status: PD 2 min. TEST 9 compile is running, Status: R 3 min. TEST 9 compile is running, Status: R 4 min. TEST 9 compile is running, Status: R 5 min. TEST 9 compile is running, Status: R 6 min. TEST 9 compile is running, Status: R -Slurm unknown status CG. Check sacct ... -9560623 COMPLETED compile_9 -9560623.bat+ COMPLETED batch -9560623.ext+ COMPLETED extern -7 min. TEST 9 compile is COMPLETED -COMPLETED -COMPLETED +7 min. TEST 9 compile is running, Status: R Slurm unknown status -. Check sacct ... -9560623 COMPLETED compile_9 -9560623.bat+ COMPLETED batch -9560623.ext+ COMPLETED extern +9590939 COMPLETED compile_9 +9590939.bat+ COMPLETED batch +9590939.ext+ COMPLETED extern 8 min. TEST 9 compile is COMPLETED COMPLETED COMPLETED diff --git a/tests/RegressionTests_hera.gnu.log b/tests/RegressionTests_hera.gnu.log index 04e8f3ab08..45c61994e0 100644 --- a/tests/RegressionTests_hera.gnu.log +++ b/tests/RegressionTests_hera.gnu.log @@ -1,9 +1,9 @@ -Tue Jul 14 14:52:46 UTC 2020 +Wed Jul 15 14:58:03 UTC 2020 Start Regression test baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200713/GNU/fv3_gfdlmp_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_196893/fv3_ccpp_gfdlmp_prod +working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_130720/fv3_ccpp_gfdlmp_prod Checking test 001 fv3_ccpp_gfdlmp results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -51,7 +51,7 @@ Test 001 fv3_ccpp_gfdlmp PASS baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200713/GNU/fv3_gfs_v15p2_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_196893/fv3_ccpp_gfs_v15p2_prod +working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_130720/fv3_ccpp_gfs_v15p2_prod Checking test 002 fv3_ccpp_gfs_v15p2 results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -119,7 +119,7 @@ Test 002 fv3_ccpp_gfs_v15p2 PASS baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200713/GNU/fv3_gfs_v16beta_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_196893/fv3_ccpp_gfs_v16beta_prod +working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_130720/fv3_ccpp_gfs_v16beta_prod Checking test 003 fv3_ccpp_gfs_v16beta results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -187,7 +187,7 @@ Test 003 fv3_ccpp_gfs_v16beta PASS baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200713/GNU/fv3_rrtmgp_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_196893/fv3_ccpp_rrtmgp_prod +working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_130720/fv3_ccpp_rrtmgp_prod Checking test 004 fv3_ccpp_rrtmgp results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -255,7 +255,7 @@ Test 004 fv3_ccpp_rrtmgp PASS baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200713/GNU/fv3_gsd_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_196893/fv3_ccpp_gsd_prod +working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_130720/fv3_ccpp_gsd_prod Checking test 005 fv3_ccpp_gsd results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -347,7 +347,7 @@ Test 005 fv3_ccpp_gsd PASS baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200713/GNU/fv3_thompson_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_196893/fv3_ccpp_thompson_prod +working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_130720/fv3_ccpp_thompson_prod Checking test 006 fv3_ccpp_thompson results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -415,7 +415,7 @@ Test 006 fv3_ccpp_thompson PASS baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200713/GNU/fv3_thompson_no_aero_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_196893/fv3_ccpp_thompson_no_aero_prod +working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_130720/fv3_ccpp_thompson_no_aero_prod Checking test 007 fv3_ccpp_thompson_no_aero results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -483,13 +483,13 @@ Test 007 fv3_ccpp_thompson_no_aero PASS baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200713/GNU/fv3_control_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_196893/fv3_ccpp_control_debug_prod +working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_130720/fv3_ccpp_control_debug_prod Checking test 008 fv3_ccpp_control_debug results .... Test 008 fv3_ccpp_control_debug PASS baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200713/GNU/fv3_gfs_v15p2_debug_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_196893/fv3_ccpp_gfs_v15p2_debug_prod +working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_130720/fv3_ccpp_gfs_v15p2_debug_prod Checking test 009 fv3_ccpp_gfs_v15p2_debug results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -557,7 +557,7 @@ Test 009 fv3_ccpp_gfs_v15p2_debug PASS baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200713/GNU/fv3_gfs_v16beta_debug_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_196893/fv3_ccpp_gfs_v16beta_debug_prod +working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_130720/fv3_ccpp_gfs_v16beta_debug_prod Checking test 010 fv3_ccpp_gfs_v16beta_debug results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -625,7 +625,7 @@ Test 010 fv3_ccpp_gfs_v16beta_debug PASS baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200713/GNU/fv3_rrtmgp_debug_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_196893/fv3_ccpp_rrtmgp_debug_prod +working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_130720/fv3_ccpp_rrtmgp_debug_prod Checking test 011 fv3_ccpp_rrtmgp_debug results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -693,7 +693,7 @@ Test 011 fv3_ccpp_rrtmgp_debug PASS baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200713/GNU/fv3_multigases_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_196893/fv3_ccpp_multigases_prod +working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_130720/fv3_ccpp_multigases_prod Checking test 012 fv3_ccpp_multigases results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -767,5 +767,5 @@ Test 012 fv3_ccpp_multigases PASS REGRESSION TEST WAS SUCCESSFUL -Tue Jul 14 15:21:39 UTC 2020 -Elapsed time: 00h:28m:55s. Have a nice day! +Wed Jul 15 15:43:13 UTC 2020 +Elapsed time: 00h:45m:12s. Have a nice day! diff --git a/tests/RegressionTests_hera.intel.log b/tests/RegressionTests_hera.intel.log index 1b86ddaa9a..3ad7680f79 100644 --- a/tests/RegressionTests_hera.intel.log +++ b/tests/RegressionTests_hera.intel.log @@ -1,9 +1,9 @@ -Tue Jul 14 13:38:41 UTC 2020 +Wed Jul 15 13:52:58 UTC 2020 Start Regression test baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200713/INTEL/fv3_control_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_136435/fv3_ccpp_control_prod +working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_293743/fv3_ccpp_control_prod Checking test 001 fv3_ccpp_control results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -71,7 +71,7 @@ Test 001 fv3_ccpp_control PASS baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200713/INTEL/fv3_control_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_136435/fv3_ccpp_decomp_prod +working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_293743/fv3_ccpp_decomp_prod Checking test 002 fv3_ccpp_decomp results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -139,7 +139,7 @@ Test 002 fv3_ccpp_decomp PASS baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200713/INTEL/fv3_control_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_136435/fv3_ccpp_2threads_prod +working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_293743/fv3_ccpp_2threads_prod Checking test 003 fv3_ccpp_2threads results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -207,7 +207,7 @@ Test 003 fv3_ccpp_2threads PASS baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200713/INTEL/fv3_restart_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_136435/fv3_ccpp_restart_prod +working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_293743/fv3_ccpp_restart_prod Checking test 004 fv3_ccpp_restart results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -275,7 +275,7 @@ Test 004 fv3_ccpp_restart PASS baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200713/INTEL/fv3_read_inc_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_136435/fv3_ccpp_read_inc_prod +working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_293743/fv3_ccpp_read_inc_prod Checking test 005 fv3_ccpp_read_inc results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -343,7 +343,7 @@ Test 005 fv3_ccpp_read_inc PASS baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200713/INTEL/fv3_wrtGauss_netcdf_esmf_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_136435/fv3_ccpp_wrtGauss_netcdf_esmf_prod +working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_293743/fv3_ccpp_wrtGauss_netcdf_esmf_prod Checking test 006 fv3_ccpp_wrtGauss_netcdf_esmf results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -391,7 +391,7 @@ Test 006 fv3_ccpp_wrtGauss_netcdf_esmf PASS baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200713/INTEL/fv3_wrtGauss_netcdf_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_136435/fv3_ccpp_wrtGauss_netcdf_prod +working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_293743/fv3_ccpp_wrtGauss_netcdf_prod Checking test 007 fv3_ccpp_wrtGauss_netcdf results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -439,7 +439,7 @@ Test 007 fv3_ccpp_wrtGauss_netcdf PASS baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200713/INTEL/fv3_wrtGlatlon_netcdf_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_136435/fv3_ccpp_wrtGlatlon_netcdf_prod +working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_293743/fv3_ccpp_wrtGlatlon_netcdf_prod Checking test 008 fv3_ccpp_wrtGlatlon_netcdf results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -487,7 +487,7 @@ Test 008 fv3_ccpp_wrtGlatlon_netcdf PASS baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200713/INTEL/fv3_wrtGauss_nemsio_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_136435/fv3_ccpp_wrtGauss_nemsio_prod +working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_293743/fv3_ccpp_wrtGauss_nemsio_prod Checking test 009 fv3_ccpp_wrtGauss_nemsio results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -535,7 +535,7 @@ Test 009 fv3_ccpp_wrtGauss_nemsio PASS baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200713/INTEL/fv3_wrtGauss_nemsio_c192_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_136435/fv3_ccpp_wrtGauss_nemsio_c192_prod +working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_293743/fv3_ccpp_wrtGauss_nemsio_c192_prod Checking test 010 fv3_ccpp_wrtGauss_nemsio_c192 results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -583,7 +583,7 @@ Test 010 fv3_ccpp_wrtGauss_nemsio_c192 PASS baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200713/INTEL/fv3_stochy_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_136435/fv3_ccpp_stochy_prod +working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_293743/fv3_ccpp_stochy_prod Checking test 011 fv3_ccpp_stochy results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -651,7 +651,7 @@ Test 011 fv3_ccpp_stochy PASS baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200713/INTEL/fv3_iau_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_136435/fv3_ccpp_iau_prod +working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_293743/fv3_ccpp_iau_prod Checking test 012 fv3_ccpp_iau results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -719,7 +719,7 @@ Test 012 fv3_ccpp_iau PASS baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200713/INTEL/fv3_ca_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_136435/fv3_ccpp_ca_prod +working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_293743/fv3_ccpp_ca_prod Checking test 013 fv3_ccpp_ca results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -787,7 +787,7 @@ Test 013 fv3_ccpp_ca PASS baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200713/INTEL/fv3_lheatstrg_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_136435/fv3_ccpp_lheatstrg_prod +working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_293743/fv3_ccpp_lheatstrg_prod Checking test 014 fv3_ccpp_lheatstrg results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -835,7 +835,7 @@ Test 014 fv3_ccpp_lheatstrg PASS baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200713/INTEL/fv3_gfdlmprad_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_136435/fv3_ccpp_gfdlmprad_prod +working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_293743/fv3_ccpp_gfdlmprad_prod Checking test 015 fv3_ccpp_gfdlmprad results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -884,7 +884,7 @@ Test 015 fv3_ccpp_gfdlmprad PASS baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200713/INTEL/fv3_gfdlmprad_atmwav_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_136435/fv3_ccpp_gfdlmprad_atmwav_prod +working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_293743/fv3_ccpp_gfdlmprad_atmwav_prod Checking test 016 fv3_ccpp_gfdlmprad_atmwav results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -933,7 +933,7 @@ Test 016 fv3_ccpp_gfdlmprad_atmwav PASS baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200713/INTEL/fv3_wrtGauss_nemsio_c768_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_136435/fv3_ccpp_wrtGauss_nemsio_c768_prod +working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_293743/fv3_ccpp_wrtGauss_nemsio_c768_prod Checking test 017 fv3_ccpp_wrtGauss_nemsio_c768 results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -982,7 +982,7 @@ Test 017 fv3_ccpp_wrtGauss_nemsio_c768 PASS baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200713/INTEL/fv3_multigases_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_136435/fv3_ccpp_multigases_prod +working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_293743/fv3_ccpp_multigases_prod Checking test 018 fv3_ccpp_multigases results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1056,7 +1056,7 @@ Test 018 fv3_ccpp_multigases PASS baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200713/INTEL/fv3_control_32bit_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_136435/fv3_ccpp_control_32bit_prod +working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_293743/fv3_ccpp_control_32bit_prod Checking test 019 fv3_ccpp_control_32bit results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1124,7 +1124,7 @@ Test 019 fv3_ccpp_control_32bit PASS baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200713/INTEL/fv3_stretched_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_136435/fv3_ccpp_stretched_prod +working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_293743/fv3_ccpp_stretched_prod Checking test 020 fv3_ccpp_stretched results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1180,7 +1180,7 @@ Test 020 fv3_ccpp_stretched PASS baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200713/INTEL/fv3_stretched_nest_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_136435/fv3_ccpp_stretched_nest_prod +working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_293743/fv3_ccpp_stretched_nest_prod Checking test 021 fv3_ccpp_stretched_nest results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1247,7 +1247,7 @@ Test 021 fv3_ccpp_stretched_nest PASS baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200713/INTEL/fv3_regional_control_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_136435/fv3_ccpp_regional_control_prod +working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_293743/fv3_ccpp_regional_control_prod Checking test 022 fv3_ccpp_regional_control results .... Comparing atmos_4xdaily.nc .........OK Comparing fv3_history2d.nc .........OK @@ -1258,7 +1258,7 @@ Test 022 fv3_ccpp_regional_control PASS baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200713/INTEL/fv3_regional_restart_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_136435/fv3_ccpp_regional_restart_prod +working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_293743/fv3_ccpp_regional_restart_prod Checking test 023 fv3_ccpp_regional_restart results .... Comparing atmos_4xdaily.nc .........OK Comparing fv3_history2d.nc .........OK @@ -1267,7 +1267,7 @@ Test 023 fv3_ccpp_regional_restart PASS baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200713/INTEL/fv3_regional_quilt_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_136435/fv3_ccpp_regional_quilt_prod +working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_293743/fv3_ccpp_regional_quilt_prod Checking test 024 fv3_ccpp_regional_quilt results .... Comparing atmos_4xdaily.nc .........OK Comparing dynf000.nc .........OK @@ -1278,7 +1278,7 @@ Test 024 fv3_ccpp_regional_quilt PASS baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200713/INTEL/fv3_regional_c768_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_136435/fv3_ccpp_regional_c768_prod +working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_293743/fv3_ccpp_regional_c768_prod Checking test 025 fv3_ccpp_regional_c768 results .... Comparing atmos_4xdaily.nc .........OK Comparing dynf000.nc .........OK @@ -1289,19 +1289,19 @@ Test 025 fv3_ccpp_regional_c768 PASS baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200713/INTEL/fv3_control_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_136435/fv3_ccpp_control_debug_prod +working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_293743/fv3_ccpp_control_debug_prod Checking test 026 fv3_ccpp_control_debug results .... Test 026 fv3_ccpp_control_debug PASS baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200713/INTEL/fv3_stretched_nest_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_136435/fv3_ccpp_stretched_nest_debug_prod +working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_293743/fv3_ccpp_stretched_nest_debug_prod Checking test 027 fv3_ccpp_stretched_nest_debug results .... Test 027 fv3_ccpp_stretched_nest_debug PASS baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200713/INTEL/fv3_gfdlmp_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_136435/fv3_ccpp_gfdlmp_prod +working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_293743/fv3_ccpp_gfdlmp_prod Checking test 028 fv3_ccpp_gfdlmp results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1349,7 +1349,7 @@ Test 028 fv3_ccpp_gfdlmp PASS baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200713/INTEL/fv3_gfdlmprad_gwd_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_136435/fv3_ccpp_gfdlmprad_gwd_prod +working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_293743/fv3_ccpp_gfdlmprad_gwd_prod Checking test 029 fv3_ccpp_gfdlmprad_gwd results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1397,7 +1397,7 @@ Test 029 fv3_ccpp_gfdlmprad_gwd PASS baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200713/INTEL/fv3_gfdlmprad_noahmp_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_136435/fv3_ccpp_gfdlmprad_noahmp_prod +working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_293743/fv3_ccpp_gfdlmprad_noahmp_prod Checking test 030 fv3_ccpp_gfdlmprad_noahmp results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1445,7 +1445,7 @@ Test 030 fv3_ccpp_gfdlmprad_noahmp PASS baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200713/INTEL/fv3_gfdlmp_hwrfsas_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_136435/fv3_ccpp_gfdlmp_hwrfsas_prod +working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_293743/fv3_ccpp_gfdlmp_hwrfsas_prod Checking test 031 fv3_ccpp_gfdlmp_hwrfsas results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1493,7 +1493,7 @@ Test 031 fv3_ccpp_gfdlmp_hwrfsas PASS baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200713/INTEL/fv3_csawmg_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_136435/fv3_ccpp_csawmg_prod +working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_293743/fv3_ccpp_csawmg_prod Checking test 032 fv3_ccpp_csawmg results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1541,7 +1541,7 @@ Test 032 fv3_ccpp_csawmg PASS baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200713/INTEL/fv3_satmedmf_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_136435/fv3_ccpp_satmedmf_prod +working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_293743/fv3_ccpp_satmedmf_prod Checking test 033 fv3_ccpp_satmedmf results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1589,7 +1589,7 @@ Test 033 fv3_ccpp_satmedmf PASS baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200713/INTEL/fv3_satmedmfq_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_136435/fv3_ccpp_satmedmfq_prod +working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_293743/fv3_ccpp_satmedmfq_prod Checking test 034 fv3_ccpp_satmedmfq results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1637,7 +1637,7 @@ Test 034 fv3_ccpp_satmedmfq PASS baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200713/INTEL/fv3_gfdlmp_32bit_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_136435/fv3_ccpp_gfdlmp_32bit_prod +working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_293743/fv3_ccpp_gfdlmp_32bit_prod Checking test 035 fv3_ccpp_gfdlmp_32bit results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1685,7 +1685,7 @@ Test 035 fv3_ccpp_gfdlmp_32bit PASS baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200713/INTEL/fv3_gfdlmprad_32bit_post_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_136435/fv3_ccpp_gfdlmprad_32bit_post_prod +working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_293743/fv3_ccpp_gfdlmprad_32bit_post_prod Checking test 036 fv3_ccpp_gfdlmprad_32bit_post results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1737,7 +1737,7 @@ Test 036 fv3_ccpp_gfdlmprad_32bit_post PASS baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200713/INTEL/fv3_cpt_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_136435/fv3_ccpp_cpt_prod +working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_293743/fv3_ccpp_cpt_prod Checking test 037 fv3_ccpp_cpt results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1791,7 +1791,7 @@ Test 037 fv3_ccpp_cpt PASS baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200713/INTEL/fv3_gsd_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_136435/fv3_ccpp_gsd_prod +working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_293743/fv3_ccpp_gsd_prod Checking test 038 fv3_ccpp_gsd results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1883,7 +1883,7 @@ Test 038 fv3_ccpp_gsd PASS baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200713/INTEL/fv3_rap_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_136435/fv3_ccpp_rap_prod +working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_293743/fv3_ccpp_rap_prod Checking test 039 fv3_ccpp_rap results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1951,7 +1951,7 @@ Test 039 fv3_ccpp_rap PASS baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200713/INTEL/fv3_hrrr_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_136435/fv3_ccpp_hrrr_prod +working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_293743/fv3_ccpp_hrrr_prod Checking test 040 fv3_ccpp_hrrr results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -2019,7 +2019,7 @@ Test 040 fv3_ccpp_hrrr PASS baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200713/INTEL/fv3_thompson_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_136435/fv3_ccpp_thompson_prod +working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_293743/fv3_ccpp_thompson_prod Checking test 041 fv3_ccpp_thompson results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -2087,7 +2087,7 @@ Test 041 fv3_ccpp_thompson PASS baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200713/INTEL/fv3_thompson_no_aero_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_136435/fv3_ccpp_thompson_no_aero_prod +working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_293743/fv3_ccpp_thompson_no_aero_prod Checking test 042 fv3_ccpp_thompson_no_aero results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -2155,7 +2155,7 @@ Test 042 fv3_ccpp_thompson_no_aero PASS baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200713/INTEL/fv3_gfs_v15p2_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_136435/fv3_ccpp_gfs_v15p2_prod +working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_293743/fv3_ccpp_gfs_v15p2_prod Checking test 043 fv3_ccpp_gfs_v15p2 results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -2223,7 +2223,7 @@ Test 043 fv3_ccpp_gfs_v15p2 PASS baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200713/INTEL/fv3_gfs_v16beta_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_136435/fv3_ccpp_gfs_v16beta_prod +working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_293743/fv3_ccpp_gfs_v16beta_prod Checking test 044 fv3_ccpp_gfs_v16beta results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -2291,7 +2291,7 @@ Test 044 fv3_ccpp_gfs_v16beta PASS baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200713/INTEL/fv3_rrtmgp_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_136435/fv3_ccpp_rrtmgp_prod +working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_293743/fv3_ccpp_rrtmgp_prod Checking test 045 fv3_ccpp_rrtmgp results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -2359,7 +2359,7 @@ Test 045 fv3_ccpp_rrtmgp PASS baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200713/INTEL/fv3_gfs_v15p2_debug_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_136435/fv3_ccpp_gfs_v15p2_debug_prod +working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_293743/fv3_ccpp_gfs_v15p2_debug_prod Checking test 046 fv3_ccpp_gfs_v15p2_debug results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -2427,7 +2427,7 @@ Test 046 fv3_ccpp_gfs_v15p2_debug PASS baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200713/INTEL/fv3_gfs_v16beta_debug_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_136435/fv3_ccpp_gfs_v16beta_debug_prod +working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_293743/fv3_ccpp_gfs_v16beta_debug_prod Checking test 047 fv3_ccpp_gfs_v16beta_debug results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -2495,7 +2495,7 @@ Test 047 fv3_ccpp_gfs_v16beta_debug PASS baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200713/INTEL/fv3_rrtmgp_debug_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_136435/fv3_ccpp_rrtmgp_debug_prod +working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_293743/fv3_ccpp_rrtmgp_debug_prod Checking test 048 fv3_ccpp_rrtmgp_debug results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -2563,7 +2563,7 @@ Test 048 fv3_ccpp_rrtmgp_debug PASS baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200713/INTEL/fv3_gsd_debug_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_136435/fv3_ccpp_gsd_debug_prod +working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_293743/fv3_ccpp_gsd_debug_prod Checking test 049 fv3_ccpp_gsd_debug results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -2631,7 +2631,7 @@ Test 049 fv3_ccpp_gsd_debug PASS baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200713/INTEL/fv3_gsd_diag3d_debug_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_136435/fv3_ccpp_gsd_diag3d_debug_prod +working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_293743/fv3_ccpp_gsd_diag3d_debug_prod Checking test 050 fv3_ccpp_gsd_diag3d_debug results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -2699,7 +2699,7 @@ Test 050 fv3_ccpp_gsd_diag3d_debug PASS baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200713/INTEL/fv3_thompson_debug_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_136435/fv3_ccpp_thompson_debug_prod +working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_293743/fv3_ccpp_thompson_debug_prod Checking test 051 fv3_ccpp_thompson_debug results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -2767,7 +2767,7 @@ Test 051 fv3_ccpp_thompson_debug PASS baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200713/INTEL/fv3_thompson_no_aero_debug_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_136435/fv3_ccpp_thompson_no_aero_debug_prod +working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_293743/fv3_ccpp_thompson_no_aero_debug_prod Checking test 052 fv3_ccpp_thompson_no_aero_debug results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -2835,7 +2835,7 @@ Test 052 fv3_ccpp_thompson_no_aero_debug PASS baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200713/INTEL/fv3_gfsv16_csawmg_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_136435/fv3_ccpp_gfsv16_csawmg_prod +working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_293743/fv3_ccpp_gfsv16_csawmg_prod Checking test 053 fv3_ccpp_gfsv16_csawmg results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -2883,7 +2883,7 @@ Test 053 fv3_ccpp_gfsv16_csawmg PASS baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200713/INTEL/fv3_gfsv16_csawmgt_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_136435/fv3_ccpp_gfsv16_csawmgt_prod +working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_293743/fv3_ccpp_gfsv16_csawmgt_prod Checking test 054 fv3_ccpp_gfsv16_csawmgt results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -2931,7 +2931,7 @@ Test 054 fv3_ccpp_gfsv16_csawmgt PASS baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200713/INTEL/fv3_gocart_clm_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_136435/fv3_ccpp_gocart_clm_prod +working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_293743/fv3_ccpp_gocart_clm_prod Checking test 055 fv3_ccpp_gocart_clm results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -2979,5 +2979,5 @@ Test 055 fv3_ccpp_gocart_clm PASS REGRESSION TEST WAS SUCCESSFUL -Tue Jul 14 14:33:50 UTC 2020 -Elapsed time: 00h:55m:10s. Have a nice day! +Wed Jul 15 14:54:19 UTC 2020 +Elapsed time: 01h:01m:22s. Have a nice day! From db7525cb48203c7f5bb509c89c134fc2da1c53f5 Mon Sep 17 00:00:00 2001 From: Dusan Jovic Date: Wed, 15 Jul 2020 19:57:54 +0000 Subject: [PATCH 11/11] Update FV3 submodule --- .gitmodules | 6 ++---- FV3 | 2 +- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/.gitmodules b/.gitmodules index 1c6895e298..a6770b7e98 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,9 +1,7 @@ [submodule "FV3"] path = FV3 - #url = https://github.com/NOAA-EMC/fv3atm - #branch = develop - url = https://github.com/climbfuji/fv3atm - branch = cleanup_work_20200709 + url = https://github.com/NOAA-EMC/fv3atm + branch = develop [submodule "NEMS"] path = NEMS url = https://github.com/NOAA-EMC/NEMS diff --git a/FV3 b/FV3 index 4eb2b376ff..f053db29af 160000 --- a/FV3 +++ b/FV3 @@ -1 +1 @@ -Subproject commit 4eb2b376ff7cb4f82392cb9f691a761321548bf1 +Subproject commit f053db29aff82456d081dda17a4962dd93ab39c5