Travis: Move regression tests to separate build

This patch removes the regression test from the standard Travis job and
moves it to a new job.  We also run all regression tests before
reporting the status code, and report a fail if any of the test give a
different result.  All files which have changed are reported at the end
of the job.

Code coverage tests have also been moved out of the invariance tests and
into the regression testing.  They continue to be based on symmetric
grids.

This test also combines the whitespace and documentation auditing into a
a single test, in order to stay within 3 concurrent tests.

.testing/Makefile

@@ -84,8 +84,9 @@ SOURCE = $(wildcard $(BASE)/src/*/*.F90 $(BASE)/src/*/*/*.F90 $(BASE)/config_src
 #---
 # Rules
 
-.PHONY: all
+.PHONY: all build.regressions
 all: $(foreach b,$(BUILDS),$(BUILD)/$(b)/MOM6)
+build.regressions: $(foreach b,symmetric target,$(BUILD)/$(b)/MOM6)
 
 # Executable
 BUILD_TARGETS = MOM6 Makefile path_names
@@ -179,7 +180,6 @@ test: $(foreach t,$(TESTS),test.$(t))
 # NOTE: We remove tc3 (OBC) from grid test since it cannot run asymmetric grids
 
 .PHONY: $(foreach t,$(TESTS),test.$(t))
-test.regressions: $(foreach c,$(CONFIGS),$(c).regression $(c).regression.diag)
 test.grids: $(foreach c,$(filter-out tc3,$(CONFIGS)),$(c).grid $(c).grid.diag)
 test.layouts: $(foreach c,$(CONFIGS),$(c).layout $(c).layout.diag)
 test.restarts: $(foreach c,$(CONFIGS),$(c).restart)
@@ -188,6 +188,9 @@ test.openmps: $(foreach c,$(CONFIGS),$(c).openmp $(c).openmp.diag)
 test.nans: $(foreach c,$(CONFIGS),$(c).nan $(c).nan.diag)
 test.dims: $(foreach c,$(CONFIGS),$(foreach d,t l h z,$(c).dim.$(d) $(c).dim.$(d).diag))
 
+test.regressions: $(foreach c,$(CONFIGS),$(c).regression $(c).regression.diag)
+	! ls -1 results/*/*.reg
+
 define CMP_RULE
 .PRECIOUS: $(foreach b,$(2),results/%/ocean.stats.$(b))
 %.$(1): $(foreach b,$(2),results/%/ocean.stats.$(b))
@@ -198,22 +201,30 @@ define CMP_RULE
 	cmp $$^ || diff $$^
 endef
 
-$(eval $(call CMP_RULE,regression,symmetric target))
 $(eval $(call CMP_RULE,grid,symmetric asymmetric))
 $(eval $(call CMP_RULE,layout,symmetric layout))
 $(eval $(call CMP_RULE,repro,symmetric repro))
 $(eval $(call CMP_RULE,openmp,symmetric openmp))
 $(eval $(call CMP_RULE,nan,symmetric nan))
 $(foreach d,t l h z,$(eval $(call CMP_RULE,dim.$(d),symmetric dim.$(d))))
 
+# Custom comparison rules
+
+.PRECIOUS: $(foreach b,symmetric restart target,results/%/ocean.stats.$(b))
+
 # Restart tests only compare the final stat record
-.PRECIOUS: $(foreach b,symmetric restart,results/%/ocean.stats.$(b))
-%.restart: $(foreach b,symmetric restart,results/%/ocean.stats.$(b))
+%.restart: $(foreach b,symmetric restart target,results/%/ocean.stats.$(b))
 	cmp $(foreach f,$^,<(tr -s ' ' < $(f) | cut -d ' ' -f3- | tail -n 1)) \
 		|| diff $^
 
 # TODO: chksum_diag parsing of restart files
 
+# All regression tests must be completed when considering answer changes
+%.regression: $(foreach b,symmetric target,results/%/ocean.stats.$(b))
+	cmp $^ || (diff $^ > $<.reg || true)
+
+%.regression.diag: $(foreach b,symmetric target,results/%/chksum_diag.$(b))
+	cmp $^ || (diff $^ > $<.reg || true)
 
 #---
 # Test run output files

.travis.yml

@@ -17,31 +17,46 @@ addons:
     packages:
     - tcsh pkg-config netcdf-bin libnetcdf-dev libnetcdff-dev openmpi-bin libopenmpi-dev gfortran
     - doxygen graphviz flex bison cmake
+    - python-numpy python-netcdf4
 
 jobs:
   include:
-    - env: JOB="Code style compliance"
+    - env: JOB="Code compliance"
       script:
+        # Whitespace
         - ./.testing/trailer.py -e TEOS10 -l 120 src config_src
-    - env: JOB="Doxygen"
-      script:
+        # API Documentation
         - cd docs && doxygen Doxyfile_nortd
         - grep -v "config_src/solo_driver/coupler_types.F90" doxygen.log | tee doxy_errors
         - test ! -s  doxy_errors
-    - env: JOB="Compile and run"
+
+    - env:
+        - JOB="Configuration testing"
+        - DO_REGRESSION_TESTS=false
+        - MKMF_TEMPLATE=linux-ubuntu-xenial-gnu.mk
       script:
+        - cd .testing
         - echo 'Build executables...' && echo -en 'travis_fold:start:script.1\\r'
-        - TRAVIS_IS_PR=$( [ ${TRAVIS_PULL_REQUEST} = "false" ] || echo "true" )
+        - make all
+        - echo -en 'travis_fold:end:script.1\\r'
+        - echo 'Running tests...' && echo -en 'travis_fold:start:script.2\\r'
+        - make test
+        - echo -en 'travis_fold:end:script.2\\r'
+
+    # NOTE: Code coverage upload is here to reduce load imbalance
+    - if: type = pull_request
+      env:
+        - JOB="Regression testing"
+        - DO_REGRESSION_TESTS=true
+        - REPORT_COVERAGE=true
+        - MKMF_TEMPLATE=linux-ubuntu-xenial-gnu.mk
+        - MOM_TARGET_SLUG=${TRAVIS_REPO_SLUG}
+        - MOM_TARGET_LOCAL_BRANCH=${TRAVIS_BRANCH}
+      script:
         - cd .testing
-        - make \
-            MKMF_TEMPLATE=linux-ubuntu-xenial-gnu.mk \
-            MOM_TARGET_SLUG=${TRAVIS_REPO_SLUG} \
-            MOM_TARGET_LOCAL_BRANCH=${TRAVIS_BRANCH} \
-            DO_REGRESSION_TESTS=${TRAVIS_IS_PR} \
-            REPORT_COVERAGE=true
+        - echo 'Build executables...' && echo -en 'travis_fold:start:script.1\\r'
+        - make build.regressions
         - echo -en 'travis_fold:end:script.1\\r'
         - echo 'Running tests...' && echo -en 'travis_fold:start:script.2\\r'
-        - make test \
-            DO_REGRESSION_TESTS=${TRAVIS_IS_PR} \
-            REPORT_COVERAGE=true
+        - make test.regressions
         - echo -en 'travis_fold:end:script.2\\r'