From f8b75642113d3e5f7b94fce4a8eb246520ccee0f Mon Sep 17 00:00:00 2001 From: Marshall Ward Date: Fri, 15 Nov 2019 11:35:02 -0500 Subject: [PATCH 1/2] Travis: Move regression tests to separate build This patch removes the regression test from the standard Travis job and moves it to a new job. We also run all regression tests before reporting the status code, and report a fail if any of the test give a different result. All files which have changed are reported at the end of the job. Code coverage tests have also been moved out of the invariance tests and into the regression testing. They continue to be based on symmetric grids. This test also combines the whitespace and documentation auditing into a a single test, in order to stay within 3 concurrent tests. --- .testing/Makefile | 21 ++++++++++++++++----- .travis.yml | 43 +++++++++++++++++++++++++++++-------------- 2 files changed, 45 insertions(+), 19 deletions(-) diff --git a/.testing/Makefile b/.testing/Makefile index f032631123..8e6006457e 100644 --- a/.testing/Makefile +++ b/.testing/Makefile @@ -84,8 +84,9 @@ SOURCE = $(wildcard $(BASE)/src/*/*.F90 $(BASE)/src/*/*/*.F90 $(BASE)/config_src #--- # Rules -.PHONY: all +.PHONY: all build.regressions all: $(foreach b,$(BUILDS),$(BUILD)/$(b)/MOM6) +build.regressions: $(foreach b,symmetric target,$(BUILD)/$(b)/MOM6) # Executable BUILD_TARGETS = MOM6 Makefile path_names @@ -179,7 +180,6 @@ test: $(foreach t,$(TESTS),test.$(t)) # NOTE: We remove tc3 (OBC) from grid test since it cannot run asymmetric grids .PHONY: $(foreach t,$(TESTS),test.$(t)) -test.regressions: $(foreach c,$(CONFIGS),$(c).regression $(c).regression.diag) test.grids: $(foreach c,$(filter-out tc3,$(CONFIGS)),$(c).grid $(c).grid.diag) test.layouts: $(foreach c,$(CONFIGS),$(c).layout $(c).layout.diag) test.restarts: $(foreach c,$(CONFIGS),$(c).restart) @@ -188,6 +188,9 @@ test.openmps: $(foreach c,$(CONFIGS),$(c).openmp $(c).openmp.diag) test.nans: $(foreach c,$(CONFIGS),$(c).nan $(c).nan.diag) test.dims: $(foreach c,$(CONFIGS),$(foreach d,t l h z,$(c).dim.$(d) $(c).dim.$(d).diag)) +test.regressions: $(foreach c,$(CONFIGS),$(c).regression $(c).regression.diag) + ! ls -1 results/*/*.reg + define CMP_RULE .PRECIOUS: $(foreach b,$(2),results/%/ocean.stats.$(b)) %.$(1): $(foreach b,$(2),results/%/ocean.stats.$(b)) @@ -198,7 +201,6 @@ define CMP_RULE cmp $$^ || diff $$^ endef -$(eval $(call CMP_RULE,regression,symmetric target)) $(eval $(call CMP_RULE,grid,symmetric asymmetric)) $(eval $(call CMP_RULE,layout,symmetric layout)) $(eval $(call CMP_RULE,repro,symmetric repro)) @@ -206,14 +208,23 @@ $(eval $(call CMP_RULE,openmp,symmetric openmp)) $(eval $(call CMP_RULE,nan,symmetric nan)) $(foreach d,t l h z,$(eval $(call CMP_RULE,dim.$(d),symmetric dim.$(d)))) +# Custom comparison rules + +.PRECIOUS: $(foreach b,symmetric restart target,results/%/ocean.stats.$(b)) + # Restart tests only compare the final stat record -.PRECIOUS: $(foreach b,symmetric restart,results/%/ocean.stats.$(b)) -%.restart: $(foreach b,symmetric restart,results/%/ocean.stats.$(b)) +%.restart: $(foreach b,symmetric restart target,results/%/ocean.stats.$(b)) cmp $(foreach f,$^,<(tr -s ' ' < $(f) | cut -d ' ' -f3- | tail -n 1)) \ || diff $^ # TODO: chksum_diag parsing of restart files +# All regression tests must be completed when considering answer changes +%.regression: $(foreach b,symmetric target,results/%/ocean.stats.$(b)) + cmp $^ || (diff $^ > $<.reg || true) + +%.regression.diag: $(foreach b,symmetric target,results/%/chksum_diag.$(b)) + cmp $^ || (diff $^ > $<.reg || true) #--- # Test run output files diff --git a/.travis.yml b/.travis.yml index 41d9d9b348..2cefbd8771 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,31 +17,46 @@ addons: packages: - tcsh pkg-config netcdf-bin libnetcdf-dev libnetcdff-dev openmpi-bin libopenmpi-dev gfortran - doxygen graphviz flex bison cmake + - python-numpy python-netcdf4 jobs: include: - - env: JOB="Code style compliance" + - env: JOB="Code compliance" script: + # Whitespace - ./.testing/trailer.py -e TEOS10 -l 120 src config_src - - env: JOB="Doxygen" - script: + # API Documentation - cd docs && doxygen Doxyfile_nortd - grep -v "config_src/solo_driver/coupler_types.F90" doxygen.log | tee doxy_errors - test ! -s doxy_errors - - env: JOB="Compile and run" + + - env: + - JOB="Configuration testing" + - DO_REGRESSION_TESTS=false + - MKMF_TEMPLATE=linux-ubuntu-xenial-gnu.mk script: + - cd .testing - echo 'Build executables...' && echo -en 'travis_fold:start:script.1\\r' - - TRAVIS_IS_PR=$( [ ${TRAVIS_PULL_REQUEST} = "false" ] || echo "true" ) + - make all + - echo -en 'travis_fold:end:script.1\\r' + - echo 'Running tests...' && echo -en 'travis_fold:start:script.2\\r' + - make test + - echo -en 'travis_fold:end:script.2\\r' + + # NOTE: Code coverage upload is here to reduce load imbalance + - if: type = pull_request + env: + - JOB="Regression testing" + - DO_REGRESSION_TESTS=true + - REPORT_COVERAGE=true + - MKMF_TEMPLATE=linux-ubuntu-xenial-gnu.mk + - MOM_TARGET_SLUG=${TRAVIS_REPO_SLUG} + - MOM_TARGET_LOCAL_BRANCH=${TRAVIS_BRANCH} + script: - cd .testing - - make \ - MKMF_TEMPLATE=linux-ubuntu-xenial-gnu.mk \ - MOM_TARGET_SLUG=${TRAVIS_REPO_SLUG} \ - MOM_TARGET_LOCAL_BRANCH=${TRAVIS_BRANCH} \ - DO_REGRESSION_TESTS=${TRAVIS_IS_PR} \ - REPORT_COVERAGE=true + - echo 'Build executables...' && echo -en 'travis_fold:start:script.1\\r' + - make build.regressions - echo -en 'travis_fold:end:script.1\\r' - echo 'Running tests...' && echo -en 'travis_fold:start:script.2\\r' - - make test \ - DO_REGRESSION_TESTS=${TRAVIS_IS_PR} \ - REPORT_COVERAGE=true + - make test.regressions - echo -en 'travis_fold:end:script.2\\r' From 558b5e08a6d754e28cace090bb280c4c7ae23043 Mon Sep 17 00:00:00 2001 From: Marshall Ward Date: Fri, 15 Nov 2019 11:51:58 -0500 Subject: [PATCH 2/2] Testing: Makefile typo --- .testing/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.testing/Makefile b/.testing/Makefile index 8e6006457e..66247a252a 100644 --- a/.testing/Makefile +++ b/.testing/Makefile @@ -213,7 +213,7 @@ $(foreach d,t l h z,$(eval $(call CMP_RULE,dim.$(d),symmetric dim.$(d)))) .PRECIOUS: $(foreach b,symmetric restart target,results/%/ocean.stats.$(b)) # Restart tests only compare the final stat record -%.restart: $(foreach b,symmetric restart target,results/%/ocean.stats.$(b)) +%.restart: $(foreach b,symmetric restart,results/%/ocean.stats.$(b)) cmp $(foreach f,$^,<(tr -s ' ' < $(f) | cut -d ' ' -f3- | tail -n 1)) \ || diff $^