Merge pull request #1030 from marshallward/travis_regression_fix

Travis: Move regression tests to separate build

.testing/Makefile

@@ -84,8 +84,9 @@ SOURCE = $(wildcard $(BASE)/src/*/*.F90 $(BASE)/src/*/*/*.F90 $(BASE)/config_src
 #---
 # Rules
 
-.PHONY: all
+.PHONY: all build.regressions
 all: $(foreach b,$(BUILDS),$(BUILD)/$(b)/MOM6)
+build.regressions: $(foreach b,symmetric target,$(BUILD)/$(b)/MOM6)
 
 # Executable
 BUILD_TARGETS = MOM6 Makefile path_names
@@ -179,7 +180,6 @@ test: $(foreach t,$(TESTS),test.$(t))
 # NOTE: We remove tc3 (OBC) from grid test since it cannot run asymmetric grids
 
 .PHONY: $(foreach t,$(TESTS),test.$(t))
-test.regressions: $(foreach c,$(CONFIGS),$(c).regression $(c).regression.diag)
 test.grids: $(foreach c,$(filter-out tc3,$(CONFIGS)),$(c).grid $(c).grid.diag)
 test.layouts: $(foreach c,$(CONFIGS),$(c).layout $(c).layout.diag)
 test.restarts: $(foreach c,$(CONFIGS),$(c).restart)
@@ -188,6 +188,9 @@ test.openmps: $(foreach c,$(CONFIGS),$(c).openmp $(c).openmp.diag)
 test.nans: $(foreach c,$(CONFIGS),$(c).nan $(c).nan.diag)
 test.dims: $(foreach c,$(CONFIGS),$(foreach d,t l h z,$(c).dim.$(d) $(c).dim.$(d).diag))
 
+test.regressions: $(foreach c,$(CONFIGS),$(c).regression $(c).regression.diag)
+	! ls -1 results/*/*.reg
+
 define CMP_RULE
 .PRECIOUS: $(foreach b,$(2),results/%/ocean.stats.$(b))
 %.$(1): $(foreach b,$(2),results/%/ocean.stats.$(b))
@@ -198,22 +201,30 @@ define CMP_RULE
 	cmp $$^ || diff $$^
 endef
 
-$(eval $(call CMP_RULE,regression,symmetric target))
 $(eval $(call CMP_RULE,grid,symmetric asymmetric))
 $(eval $(call CMP_RULE,layout,symmetric layout))
 $(eval $(call CMP_RULE,repro,symmetric repro))
 $(eval $(call CMP_RULE,openmp,symmetric openmp))
 $(eval $(call CMP_RULE,nan,symmetric nan))
 $(foreach d,t l h z,$(eval $(call CMP_RULE,dim.$(d),symmetric dim.$(d))))
 
+# Custom comparison rules
+
+.PRECIOUS: $(foreach b,symmetric restart target,results/%/ocean.stats.$(b))
+
 # Restart tests only compare the final stat record
-.PRECIOUS: $(foreach b,symmetric restart,results/%/ocean.stats.$(b))
 %.restart: $(foreach b,symmetric restart,results/%/ocean.stats.$(b))
 	cmp $(foreach f,$^,<(tr -s ' ' < $(f) | cut -d ' ' -f3- | tail -n 1)) \
 		|| diff $^
 
 # TODO: chksum_diag parsing of restart files
 
+# All regression tests must be completed when considering answer changes
+%.regression: $(foreach b,symmetric target,results/%/ocean.stats.$(b))
+	cmp $^ || (diff $^ > $<.reg || true)
+
+%.regression.diag: $(foreach b,symmetric target,results/%/chksum_diag.$(b))
+	cmp $^ || (diff $^ > $<.reg || true)
 
 #---
 # Test run output files

.travis.yml

@@ -17,31 +17,46 @@ addons:
     packages:
     - tcsh pkg-config netcdf-bin libnetcdf-dev libnetcdff-dev openmpi-bin libopenmpi-dev gfortran
     - doxygen graphviz flex bison cmake
+    - python-numpy python-netcdf4
 
 jobs:
   include:
-    - env: JOB="Code style compliance"
+    - env: JOB="Code compliance"
       script:
+        # Whitespace
         - ./.testing/trailer.py -e TEOS10 -l 120 src config_src
-    - env: JOB="Doxygen"
-      script:
+        # API Documentation
         - cd docs && doxygen Doxyfile_nortd
         - grep -v "config_src/solo_driver/coupler_types.F90" doxygen.log | tee doxy_errors
         - test ! -s  doxy_errors
-    - env: JOB="Compile and run"
+
+    - env:
+        - JOB="Configuration testing"
+        - DO_REGRESSION_TESTS=false
+        - MKMF_TEMPLATE=linux-ubuntu-xenial-gnu.mk
       script:
+        - cd .testing
         - echo 'Build executables...' && echo -en 'travis_fold:start:script.1\\r'
-        - TRAVIS_IS_PR=$( [ ${TRAVIS_PULL_REQUEST} = "false" ] || echo "true" )
+        - make all
+        - echo -en 'travis_fold:end:script.1\\r'
+        - echo 'Running tests...' && echo -en 'travis_fold:start:script.2\\r'
+        - make test
+        - echo -en 'travis_fold:end:script.2\\r'
+
+    # NOTE: Code coverage upload is here to reduce load imbalance
+    - if: type = pull_request
+      env:
+        - JOB="Regression testing"
+        - DO_REGRESSION_TESTS=true
+        - REPORT_COVERAGE=true
+        - MKMF_TEMPLATE=linux-ubuntu-xenial-gnu.mk
+        - MOM_TARGET_SLUG=${TRAVIS_REPO_SLUG}
+        - MOM_TARGET_LOCAL_BRANCH=${TRAVIS_BRANCH}
+      script:
         - cd .testing
-        - make \
-            MKMF_TEMPLATE=linux-ubuntu-xenial-gnu.mk \
-            MOM_TARGET_SLUG=${TRAVIS_REPO_SLUG} \
-            MOM_TARGET_LOCAL_BRANCH=${TRAVIS_BRANCH} \
-            DO_REGRESSION_TESTS=${TRAVIS_IS_PR} \
-            REPORT_COVERAGE=true
+        - echo 'Build executables...' && echo -en 'travis_fold:start:script.1\\r'
+        - make build.regressions
         - echo -en 'travis_fold:end:script.1\\r'
         - echo 'Running tests...' && echo -en 'travis_fold:start:script.2\\r'
-        - make test \
-            DO_REGRESSION_TESTS=${TRAVIS_IS_PR} \
-            REPORT_COVERAGE=true
+        - make test.regressions
         - echo -en 'travis_fold:end:script.2\\r'