Add timout test

tests/error-test.conf

@@ -11,6 +11,9 @@ COMPILE | atm_dyn32 | intel | -DAPP=ATM -DCCPP_SUITES=FV3_GFS_v16,FV3_GFS_v16_fl
 # This should succeed
 RUN | control_c48.v2.sfc                                |                                      | baseline |
 
+# This should fail due to wall clock timeout
+RUN | control_c48.v2.sfc_timeout                        |                                      | baseline |
+
 # These tests should always fail, and prevent the workflow from completing.
 RUN | fail_to_copy                                      |                                      | baseline |
 RUN | fail_to_run                                       |                                      | baseline |

tests/rt.sh

@@ -1041,6 +1041,7 @@ if [[ ${skip_check_results} == true ]]; then
 else
   REGRESSIONTEST_LOG=${PATHRT}/logs/RegressionTests_${MACHINE_ID}.log
 fi
+rm -f ${REGRESSIONTEST_LOG}
 
 TEST_START_TIME="$(date '+%Y%m%d %T')"
 export TEST_START_TIME

tests/run_compile.sh

@@ -17,7 +17,16 @@ cleanup() {
 
 write_fail_test() {
   echo "${JBNME} failed in run_compile" >> "${PATHRT}/fail_${JBNME}"
-  exit 1
+  if [[ ${ROCOTO:-false} == true ]] || [[ ${ECFLOW:-false} == true ]]; then
+    # if this script has been submitted by a workflow return non-zero exit status
+    # so that workflow can resubmit it
+    exit 1
+  else
+    # if this script has been executed interactively, return zero exit status
+    # so that rt.sh can continue running, and hope that rt.sh's generate_log
+    # will catch failed tests
+    exit 0
+  fi
 }
 
 remove_fail_test() {

tests/tests/control_c48.v2.sfc_timeout

@@ -0,0 +1,4 @@
+source tests/control_c48.v2.sfc
+
+# Intentionally make this test fail due to wall clock timeout. Used by error-test.conf
+export WLCLK=2