Skip to content

Commit

Permalink
Merge branch 'dev/gfdl' into internal_tide_bathy_params
Browse files Browse the repository at this point in the history
  • Loading branch information
marshallward authored Aug 30, 2021
2 parents 8cd4330 + 13fdbf0 commit 2fbdf35
Show file tree
Hide file tree
Showing 26 changed files with 1,485 additions and 56 deletions.
1 change: 1 addition & 0 deletions .github/actions/testing-setup/action.yml
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ runs:
echo "::group::Install linux packages"
sudo apt-get update
sudo apt-get install netcdf-bin libnetcdf-dev libnetcdff-dev mpich libmpich-dev
sudo apt-get install linux-tools-common
echo "::endgroup::"
- name: Compile FMS library
Expand Down
36 changes: 36 additions & 0 deletions .github/workflows/perfmon.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
name: Performance Monitor

on: [pull_request]

jobs:
build-test-perfmon:

runs-on: ubuntu-latest
defaults:
run:
working-directory: .testing

steps:
- uses: actions/checkout@v2
with:
submodules: recursive

- uses: ./.github/actions/testing-setup

- name: Compile optimized models
run: >-
make -j build.prof
MOM_TARGET_SLUG=$GITHUB_REPOSITORY
MOM_TARGET_LOCAL_BRANCH=$GITHUB_BASE_REF
DO_REGRESSION_TESTS=true
- name: Generate profile data
run: >-
pip install f90nml &&
make profile
DO_REGRESSION_TESTS=true
- name: Generate perf data
run: |
sudo sysctl -w kernel.perf_event_paranoid=2
make perf DO_REGRESSION_TESTS=true
92 changes: 86 additions & 6 deletions .testing/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@
# Build configuration:
# FCFLAGS_DEBUG Testing ("debug") compiler flags
# FCFLAGS_REPRO Production ("repro") compiler flags
# FCFLAGS_OPT Aggressive optimization compiler flags
# FCFLAGS_INIT Variable initialization flags
# FCFLAGS_COVERAGE Code coverage flags
#
Expand Down Expand Up @@ -72,6 +73,7 @@ export MPIFC
# NOTE: FMS will be built using FCFLAGS_DEBUG
FCFLAGS_DEBUG ?= -g -O0
FCFLAGS_REPRO ?= -g -O2
FCFLAGS_OPT ?= -g -O3 -mavx -fno-omit-frame-pointer
FCFLAGS_INIT ?=
FCFLAGS_COVERAGE ?=
# Additional notes:
Expand All @@ -96,6 +98,7 @@ DO_REPRO_TESTS ?=
# Time measurement (configurable by the CI)
TIME ?= time


#---
# Dependencies
DEPS = deps
Expand All @@ -114,6 +117,9 @@ CONFIGS := $(wildcard tc*)
TESTS = grids layouts restarts nans dims openmps rotations
DIMS = t l h z q r

# Set the framework
FRAMEWORK ?= fms1

# REPRO tests enable reproducibility with optimization, and often do not match
# the DEBUG results in older GCCs and vendor compilers, so we can optionally
# disable them.
Expand All @@ -122,6 +128,11 @@ ifeq ($(DO_REPRO_TESTS), true)
TESTS += repros
endif

# Profiling
ifeq ($(DO_PROFILE), false)
BUILDS += opt opt_target
endif

# The following variables are configured by Travis:
# DO_REGRESSION_TESTS: true if $(TRAVIS_PULL_REQUEST) is a PR number
# MOM_TARGET_SLUG: TRAVIS_REPO_SLUG
Expand Down Expand Up @@ -195,6 +206,7 @@ endif
.PHONY: all build.regressions
all: $(foreach b,$(BUILDS),build/$(b)/MOM6) $(VENV_PATH)
build.regressions: $(foreach b,symmetric target,build/$(b)/MOM6)
build.prof: $(foreach b,opt opt_target,build/$(b)/MOM6)

# Executable
BUILD_TARGETS = MOM6 Makefile path_names
Expand All @@ -217,6 +229,7 @@ PATH_FMS = PATH="${PATH}:../../$(DEPS)/bin"
SYMMETRIC_FCFLAGS := FCFLAGS="$(FCFLAGS_DEBUG) $(FCFLAGS_INIT) $(COVERAGE) $(FCFLAGS_FMS)"
ASYMMETRIC_FCFLAGS := FCFLAGS="$(FCFLAGS_DEBUG) $(FCFLAGS_INIT) $(FCFLAGS_FMS)"
REPRO_FCFLAGS := FCFLAGS="$(FCFLAGS_REPRO) $(FCFLAGS_FMS)"
OPT_FCFLAGS := FCFLAGS="$(FCFLAGS_OPT) $(FCFLAGS_FMS)"
OPENMP_FCFLAGS := FCFLAGS="$(FCFLAGS_DEBUG) $(FCFLAGS_INIT) $(FCFLAGS_FMS)"
TARGET_FCFLAGS := FCFLAGS="$(FCFLAGS_DEBUG) $(FCFLAGS_INIT) $(FCFLAGS_FMS)"

Expand All @@ -230,6 +243,8 @@ build/asymmetric/Makefile: MOM_ENV=$(PATH_FMS) $(ASYMMETRIC_FCFLAGS) $(MOM_LDFLA
build/repro/Makefile: MOM_ENV=$(PATH_FMS) $(REPRO_FCFLAGS) $(MOM_LDFLAGS)
build/openmp/Makefile: MOM_ENV=$(PATH_FMS) $(OPENMP_FCFLAGS) $(MOM_LDFLAGS)
build/target/Makefile: MOM_ENV=$(PATH_FMS) $(TARGET_FCFLAGS) $(MOM_LDFLAGS)
build/opt/Makefile: MOM_ENV=$(PATH_FMS) $(OPT_FCFLAGS) $(MOM_LDFLAGS)
build/opt_target/Makefile: MOM_ENV=$(PATH_FMS) $(OPT_FCFLAGS) $(MOM_LDFLAGS)
build/coupled/Makefile: MOM_ENV=$(PATH_FMS) $(SYMMETRIC_FCFLAGS) $(SYMMETRIC_LDFLAGS)
build/nuopc/Makefile: MOM_ENV=$(PATH_FMS) $(SYMMETRIC_FCFLAGS) $(SYMMETRIC_LDFLAGS)
build/mct/Makefile: MOM_ENV=$(PATH_FMS) $(SYMMETRIC_FCFLAGS) $(SYMMETRIC_LDFLAGS)
Expand All @@ -240,12 +255,15 @@ build/asymmetric/Makefile: MOM_ACFLAGS=--enable-asymmetric
build/repro/Makefile: MOM_ACFLAGS=
build/openmp/Makefile: MOM_ACFLAGS=--enable-openmp
build/target/Makefile: MOM_ACFLAGS=
build/opt/Makefile: MOM_ACFLAGS=
build/opt_target/Makefile: MOM_ACFLAGS=
build/coupled/Makefile: MOM_ACFLAGS=--with-driver=coupled_driver
build/nuopc/Makefile: MOM_ACFLAGS=--with-driver=nuopc_driver
build/mct/Makefile: MOM_ACFLAGS=--with-driver=mct_driver

# Fetch regression target source code
build/target/Makefile: | $(TARGET_CODEBASE)
build/opt_target/Makefile: | $(TARGET_CODEBASE)


# Define source code dependencies
Expand All @@ -267,7 +285,7 @@ build/%/MOM6: build/%/Makefile
build/%/Makefile: ../ac/configure ../ac/Makefile.in $(DEPS)/lib/libFMS.a $(MKMF) $(LIST_PATHS)
mkdir -p $(@D)
cd $(@D) \
&& $(MOM_ENV) ../../../ac/configure $(MOM_ACFLAGS) \
&& $(MOM_ENV) ../../../ac/configure $(MOM_ACFLAGS) --with-framework=$(FRAMEWORK) \
|| (cat config.log && false)


Expand All @@ -276,7 +294,8 @@ build/%/Makefile: ../ac/configure ../ac/Makefile.in $(DEPS)/lib/libFMS.a $(MKMF)


# Fetch the regression target codebase
build/target/Makefile: $(TARGET_CODEBASE)/ac/configure $(DEPS)/lib/libFMS.a $(MKMF) $(LIST_PATHS)
build/target/Makefile build/opt_target/Makefile: \
$(TARGET_CODEBASE)/ac/configure $(DEPS)/lib/libFMS.a $(MKMF) $(LIST_PATHS)
mkdir -p $(@D)
cd $(@D) \
&& $(MOM_ENV) ../../$(TARGET_CODEBASE)/ac/configure $(MOM_ACFLAGS) \
Expand Down Expand Up @@ -549,7 +568,11 @@ $(eval $(call STAT_RULE,dim.q,symmetric,,Q_RESCALE_POWER=11,,1))
$(eval $(call STAT_RULE,dim.r,symmetric,,R_RESCALE_POWER=11,,1))


# Restart tests require significant preprocessing, and are handled separately.
# Generate the half-period input namelist as follows:
# 1. Fetch DAYMAX and TIMEUNIT from MOM_input
# 2. Convert DAYMAX from TIMEUNIT to seconds
# 3. Apply seconds to `ocean_solo_nml` inside input.nml.
# NOTE: Assumes that runtime set by DAYMAX, will fail if set by input.nml
work/%/restart/ocean.stats: build/symmetric/MOM6 $(VENV_PATH)
rm -rf $(@D)
mkdir -p $(@D)
Expand All @@ -562,14 +585,13 @@ work/%/restart/ocean.stats: build/symmetric/MOM6 $(VENV_PATH)
cd work/$*/restart; \
fi
mkdir -p $(@D)/RESTART
# Generate the half-period input namelist
# TODO: Assumes that runtime set by DAYMAX, will fail if set by input.nml
# Set the half-period
cd $(@D) \
&& daymax=$$(grep DAYMAX MOM_input | cut -d '!' -f 1 | cut -d '=' -f 2 | xargs) \
&& timeunit=$$(grep TIMEUNIT MOM_input | cut -d '!' -f 1 | cut -d '=' -f 2 | xargs) \
&& if [ -z "$${timeunit}" ]; then timeunit="8.64e4"; fi \
&& printf -v timeunit_int "%.f" "$${timeunit}" \
&& halfperiod=$$(printf "%.f" $$(bc <<< "scale=10; 0.5 * $${daymax} * $${timeunit_int}")) \
&& halfperiod=$$(awk -v t=$${daymax} -v dt=$${timeunit} 'BEGIN {printf "%.f", 0.5*t*dt}') \
&& printf "\n&ocean_solo_nml\n seconds = $${halfperiod}\n/\n" >> input.nml
# Remove any previous archived output
rm -f results/$*/std.restart{1,2}.{out,err}
Expand Down Expand Up @@ -621,6 +643,64 @@ test.summary:
fi


#---
# Profiling
# XXX: This is experimental work to track, log, and report changes in runtime
PCONFIGS = p0

.PHONY: profile
profile: $(foreach p,$(PCONFIGS), prof.$(p))

.PHONY: prof.p0
prof.p0: work/p0/opt/clocks.json work/p0/opt_target/clocks.json
python tools/compare_clocks.py $^

work/p0/%/clocks.json: work/p0/%/std.out
python tools/parse_fms_clocks.py -d $(@D) $^ > $@

work/p0/opt/std.out: build/opt/MOM6
work/p0/opt_target/std.out: build/opt_target/MOM6

work/p0/%/std.out:
mkdir -p $(@D)
cp -RL p0/* $(@D)
mkdir -p $(@D)/RESTART
echo -e "" > $(@D)/MOM_override
cd $(@D) \
&& $(MPIRUN) -n 1 ../../../$< 2> std.err > std.out

#---
# Same but with perf

# TODO: This expects the -e flag, can I handle it in the command?
PERF_EVENTS ?=

.PHONY: perf
perf: $(foreach p,$(PCONFIGS), perf.$(p))

.PHONY: prof.p0
perf.p0: work/p0/opt/profile.json work/p0/opt_target/profile.json
python tools/compare_perf.py $^

work/p0/%/profile.json: work/p0/%/perf.data
python tools/parse_perf.py -f $< > $@

work/p0/opt/perf.data: build/opt/MOM6
work/p0/opt_target/perf.data: build/opt_target/MOM6

work/p0/%/perf.data:
mkdir -p $(@D)
cp -RL p0/* $(@D)
mkdir -p $(@D)/RESTART
echo -e "" > $(@D)/MOM_override
cd $(@D) \
&& perf record \
-F 3999 \
${PERF_EVENTS} \
../../../$< 2> std.perf.err > std.perf.out \
|| cat std.perf.err


#----
# NOTE: These tests assert that we are in the .testing directory.

Expand Down
Loading

0 comments on commit 2fbdf35

Please sign in to comment.