diff --git a/.testing/Makefile b/.testing/Makefile index f032631123..66247a252a 100644 --- a/.testing/Makefile +++ b/.testing/Makefile @@ -84,8 +84,9 @@ SOURCE = $(wildcard $(BASE)/src/*/*.F90 $(BASE)/src/*/*/*.F90 $(BASE)/config_src #--- # Rules -.PHONY: all +.PHONY: all build.regressions all: $(foreach b,$(BUILDS),$(BUILD)/$(b)/MOM6) +build.regressions: $(foreach b,symmetric target,$(BUILD)/$(b)/MOM6) # Executable BUILD_TARGETS = MOM6 Makefile path_names @@ -179,7 +180,6 @@ test: $(foreach t,$(TESTS),test.$(t)) # NOTE: We remove tc3 (OBC) from grid test since it cannot run asymmetric grids .PHONY: $(foreach t,$(TESTS),test.$(t)) -test.regressions: $(foreach c,$(CONFIGS),$(c).regression $(c).regression.diag) test.grids: $(foreach c,$(filter-out tc3,$(CONFIGS)),$(c).grid $(c).grid.diag) test.layouts: $(foreach c,$(CONFIGS),$(c).layout $(c).layout.diag) test.restarts: $(foreach c,$(CONFIGS),$(c).restart) @@ -188,6 +188,9 @@ test.openmps: $(foreach c,$(CONFIGS),$(c).openmp $(c).openmp.diag) test.nans: $(foreach c,$(CONFIGS),$(c).nan $(c).nan.diag) test.dims: $(foreach c,$(CONFIGS),$(foreach d,t l h z,$(c).dim.$(d) $(c).dim.$(d).diag)) +test.regressions: $(foreach c,$(CONFIGS),$(c).regression $(c).regression.diag) + ! ls -1 results/*/*.reg + define CMP_RULE .PRECIOUS: $(foreach b,$(2),results/%/ocean.stats.$(b)) %.$(1): $(foreach b,$(2),results/%/ocean.stats.$(b)) @@ -198,7 +201,6 @@ define CMP_RULE cmp $$^ || diff $$^ endef -$(eval $(call CMP_RULE,regression,symmetric target)) $(eval $(call CMP_RULE,grid,symmetric asymmetric)) $(eval $(call CMP_RULE,layout,symmetric layout)) $(eval $(call CMP_RULE,repro,symmetric repro)) @@ -206,14 +208,23 @@ $(eval $(call CMP_RULE,openmp,symmetric openmp)) $(eval $(call CMP_RULE,nan,symmetric nan)) $(foreach d,t l h z,$(eval $(call CMP_RULE,dim.$(d),symmetric dim.$(d)))) +# Custom comparison rules + +.PRECIOUS: $(foreach b,symmetric restart target,results/%/ocean.stats.$(b)) + # Restart tests only compare the final stat record -.PRECIOUS: $(foreach b,symmetric restart,results/%/ocean.stats.$(b)) %.restart: $(foreach b,symmetric restart,results/%/ocean.stats.$(b)) cmp $(foreach f,$^,<(tr -s ' ' < $(f) | cut -d ' ' -f3- | tail -n 1)) \ || diff $^ # TODO: chksum_diag parsing of restart files +# All regression tests must be completed when considering answer changes +%.regression: $(foreach b,symmetric target,results/%/ocean.stats.$(b)) + cmp $^ || (diff $^ > $<.reg || true) + +%.regression.diag: $(foreach b,symmetric target,results/%/chksum_diag.$(b)) + cmp $^ || (diff $^ > $<.reg || true) #--- # Test run output files diff --git a/.travis.yml b/.travis.yml index 41d9d9b348..2cefbd8771 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,31 +17,46 @@ addons: packages: - tcsh pkg-config netcdf-bin libnetcdf-dev libnetcdff-dev openmpi-bin libopenmpi-dev gfortran - doxygen graphviz flex bison cmake + - python-numpy python-netcdf4 jobs: include: - - env: JOB="Code style compliance" + - env: JOB="Code compliance" script: + # Whitespace - ./.testing/trailer.py -e TEOS10 -l 120 src config_src - - env: JOB="Doxygen" - script: + # API Documentation - cd docs && doxygen Doxyfile_nortd - grep -v "config_src/solo_driver/coupler_types.F90" doxygen.log | tee doxy_errors - test ! -s doxy_errors - - env: JOB="Compile and run" + + - env: + - JOB="Configuration testing" + - DO_REGRESSION_TESTS=false + - MKMF_TEMPLATE=linux-ubuntu-xenial-gnu.mk script: + - cd .testing - echo 'Build executables...' && echo -en 'travis_fold:start:script.1\\r' - - TRAVIS_IS_PR=$( [ ${TRAVIS_PULL_REQUEST} = "false" ] || echo "true" ) + - make all + - echo -en 'travis_fold:end:script.1\\r' + - echo 'Running tests...' && echo -en 'travis_fold:start:script.2\\r' + - make test + - echo -en 'travis_fold:end:script.2\\r' + + # NOTE: Code coverage upload is here to reduce load imbalance + - if: type = pull_request + env: + - JOB="Regression testing" + - DO_REGRESSION_TESTS=true + - REPORT_COVERAGE=true + - MKMF_TEMPLATE=linux-ubuntu-xenial-gnu.mk + - MOM_TARGET_SLUG=${TRAVIS_REPO_SLUG} + - MOM_TARGET_LOCAL_BRANCH=${TRAVIS_BRANCH} + script: - cd .testing - - make \ - MKMF_TEMPLATE=linux-ubuntu-xenial-gnu.mk \ - MOM_TARGET_SLUG=${TRAVIS_REPO_SLUG} \ - MOM_TARGET_LOCAL_BRANCH=${TRAVIS_BRANCH} \ - DO_REGRESSION_TESTS=${TRAVIS_IS_PR} \ - REPORT_COVERAGE=true + - echo 'Build executables...' && echo -en 'travis_fold:start:script.1\\r' + - make build.regressions - echo -en 'travis_fold:end:script.1\\r' - echo 'Running tests...' && echo -en 'travis_fold:start:script.2\\r' - - make test \ - DO_REGRESSION_TESTS=${TRAVIS_IS_PR} \ - REPORT_COVERAGE=true + - make test.regressions - echo -en 'travis_fold:end:script.2\\r'