Merge pull request #2 from NOAA-GFDL/dev/gfdl

Updates on 04_21

.gitlab-ci.yml

@@ -31,7 +31,7 @@ setup:
     # Clone regressions directory
     - git clone --recursive http://gitlab.gfdl.noaa.gov/ogrp/Gaea-stats-MOM6-examples.git tests && cd tests
     # Install / update testing scripts
-    - git clone https://github.com/adcroft/MRS.git MRS
+    - git clone -b new-code-struct https://github.com/adcroft/MRS.git MRS
     # Update MOM6-examples and submodules
     - (cd MOM6-examples && git checkout . && git checkout dev/gfdl && git pull && git submodule init && git submodule update)
     - (cd MOM6-examples/src/MOM6 && git submodule update)
@@ -60,7 +60,7 @@ gnu:ocean-only-nolibs:
     - time tar zxf $CACHE_DIR/tests_$CI_PIPELINE_ID.tgz && cd tests
     - make -f MRS/Makefile.build build/gnu/env && cd build/gnu
     # mkdir -p build/gnu/repro/symmetric_dynamic/ocean_only && cd build/gnu/repro/symmetric_dynamic/ocean_only
-    - ../../MOM6-examples/src/mkmf/bin/list_paths -l ../../../config_src/{solo_driver,dynamic_symmetric,ext*} ../../../src ../../MOM6-examples/src/FMS
+    - ../../MOM6-examples/src/mkmf/bin/list_paths -l ../../../config_src/{drivers/solo_driver,memory/dynamic_symmetric,infra/FMS1,ext*} ../../../src ../../MOM6-examples/src/FMS
     - sed -i '/FMS\/.*\/test_/d' path_names
     - ../../MOM6-examples/src/mkmf/bin/mkmf -t ../../MOM6-examples/src/mkmf/templates/ncrc-gnu.mk -p MOM6 -c"-Duse_libMPI -Duse_netCDF" path_names
     - time (source ./env ; make NETCDF=3 REPRO=1 MOM6 -s -j)
@@ -73,7 +73,7 @@ gnu:ice-ocean-nolibs:
     - time tar zxf $CACHE_DIR/tests_$CI_PIPELINE_ID.tgz && cd tests
     - make -f MRS/Makefile.build build/gnu/env && cd build/gnu
     # mkdir -p build/gnu/repro/symmetric_dynamic/ocean_only && cd build/gnu/repro/symmetric_dynamic/ocean_only
-    - ../../MOM6-examples/src/mkmf/bin/list_paths -l ../../../config_src/{coupled_driver,dynamic,ext*} ../../../src ../../MOM6-examples/src/{FMS,coupler,SIS2,icebergs,ice_param,land_null,atmos_null}
+    - ../../MOM6-examples/src/mkmf/bin/list_paths -l ../../../config_src/{drivers/FMS_cap,memory/dynamic_nonsymmetric,infra/FMS1,ext*} ../../../src ../../MOM6-examples/src/{FMS,coupler,SIS2,icebergs,ice_param,land_null,atmos_null}
     - sed -i '/FMS\/.*\/test_/d' path_names
     - ../../MOM6-examples/src/mkmf/bin/mkmf -t ../../MOM6-examples/src/mkmf/templates/ncrc-gnu.mk -p MOM6 -c"-Duse_libMPI -Duse_netCDF -D_USE_LEGACY_LAND_ -Duse_AM3_physics" path_names
     - time (source ./env ; make NETCDF=3 REPRO=1 MOM6 -s -j)

.testing/Makefile

@@ -32,7 +32,6 @@
 #   MPIFC   MPI Fortran compiler
 #
 # Build configuration:
-#
 #   FCFLAGS_DEBUG       Testing ("debug") compiler flags
 #   FCFLAGS_REPRO       Production ("repro") compiler flags
 #   FCFLAGS_INIT        Variable initialization flags
@@ -76,7 +75,6 @@ FCFLAGS_REPRO ?= -g -O2
 FCFLAGS_INIT ?=
 FCFLAGS_COVERAGE ?=
 # Additional notes:
-#
 # - The default values are simple, minimalist flags, supported by nearly all
 #   compilers which are comparable to GFDL's canonical DEBUG and REPRO builds.
 #
@@ -86,14 +84,18 @@ FCFLAGS_COVERAGE ?=
 # - FMS cannot be built with the same aggressive initialization flags as MOM6,
 #   so FCFLAGS_INIT is used to provide additional MOM6 configuration.
 
+# User-defined LDFLAGS (applied to all builds and FMS)
+LDFLAGS_USER ?=
+
 # Set to `true` to require identical results from DEBUG and REPRO builds
+# NOTE: Many compilers (Intel, GCC on ARM64) do not yet produce identical
+#   results across DEBUG and REPRO builds (as defined below), so we disable on
+#   default.
 DO_REPRO_TESTS ?=
 
+# Time measurement (configurable by the CI)
 TIME ?= time
 
-# Many compilers (Intel, GCC on ARM64) do not yet produce identical results
-# across DEBUG and REPRO builds (as defined below), so we disable on default.
-
 #---
 # Dependencies
 DEPS = deps
@@ -154,10 +156,12 @@ SOURCE = \
   $(foreach ext,F90 inc c h,$(wildcard $(1)/*/*.$(ext) $(1)/*/*/*.$(ext)))
 
 MOM_SOURCE = $(call SOURCE,../src) \
-  $(wildcard ../config_src/solo_driver/*.F90) \
+  $(wildcard ../config_src/infra/FMS1/*.F90) \
+  $(wildcard ../config_src/drivers/solo_driver/*.F90) \
   $(wildcard ../config_src/ext*/*/*.F90)
 TARGET_SOURCE = $(call SOURCE,build/target_codebase/src) \
-  $(wildcard build/target_codebase/config_src/solo_driver/*.F90) \
+  $(wildcard build/target_codebase/config_src/infra/FMS1/*.F90) \
+  $(wildcard build/target_codebase/config_src/drivers/solo_driver/*.F90) \
   $(wildcard build/target_codebase/config_src/ext*/*.F90)
 FMS_SOURCE = $(call SOURCE,$(DEPS)/fms/src)
 
@@ -216,8 +220,8 @@ REPRO_FCFLAGS := FCFLAGS="$(FCFLAGS_REPRO) $(FCFLAGS_FMS)"
 OPENMP_FCFLAGS := FCFLAGS="$(FCFLAGS_DEBUG) $(FCFLAGS_INIT) $(FCFLAGS_FMS)"
 TARGET_FCFLAGS := FCFLAGS="$(FCFLAGS_DEBUG) $(FCFLAGS_INIT) $(FCFLAGS_FMS)"
 
-MOM_LDFLAGS := LDFLAGS="$(LDFLAGS_FMS)"
-SYMMETRIC_LDFLAGS := LDFLAGS="$(COVERAGE) $(LDFLAGS_FMS)"
+MOM_LDFLAGS := LDFLAGS="$(LDFLAGS_FMS) $(LDFLAGS_USER)"
+SYMMETRIC_LDFLAGS := LDFLAGS="$(COVERAGE) $(LDFLAGS_FMS) $(LDFLAGS_USER)"
 
 
 # Environment variable configuration
@@ -249,7 +253,7 @@ build/target/Makefile: | $(TARGET_CODEBASE)
 #   Ideally we would want to re-run both Makefile and mkmf, but our mkmf call
 #   is inside ./configure, so we must re-run ./configure as well.
 $(foreach b,$(filter-out target,$(BUILDS)),build/$(b)/Makefile): $(MOM_SOURCE)
-build/target/configure: $(TARGET_SOURCE)
+build/target_codebase/configure: $(TARGET_SOURCE)
 
 
 # Build MOM6
@@ -285,10 +289,7 @@ $(TARGET_CODEBASE)/ac/configure: $(TARGET_CODEBASE)
 
 $(TARGET_CODEBASE):
 	git clone --recursive $(MOM_TARGET_URL) $@
-	cd $@ && git checkout $(MOM_TARGET_BRANCH)
-	# Copy modern autoconf files to target?
-	mkdir -p $(TARGET_CODEBASE)/ac
-	cp -r ../ac/{configure.ac,Makefile.in,m4} $(TARGET_CODEBASE)/ac
+	cd $@ && git checkout --recurse-submodules $(MOM_TARGET_BRANCH)
 
 
 #---

ac/configure.ac

@@ -42,18 +42,18 @@ srcdir=$srcdir/..
 # Default to symmetric grid
 # NOTE: --enable is more properly used to add a feature, rather than to select
 #   a compile-time mode, so this is not exactly being used as intended.
-MEM_LAYOUT=${srcdir}/config_src/dynamic_symmetric
+MEM_LAYOUT=${srcdir}/config_src/memory/dynamic_symmetric
 AC_ARG_ENABLE([asymmetric],
   AS_HELP_STRING([--enable-asymmetric], [Use the asymmetric grid]))
 AS_IF([test "$enable_asymmetric" = yes],
-  [MEM_LAYOUT=${srcdir}/config_src/dynamic])
+  [MEM_LAYOUT=${srcdir}/config_src/memory/dynamic_nonsymmetric])
 
 # Default to solo_driver
-DRIVER_DIR=${srcdir}/config_src/solo_driver
+DRIVER_DIR=${srcdir}/config_src/drivers/solo_driver
 AC_ARG_WITH([driver],
   AS_HELP_STRING([--with-driver=coupled_driver|solo_driver], [Select directory for driver source code]))
 AS_IF([test "x$with_driver" != "x"],
-  [DRIVER_DIR=${srcdir}/config_src/${with_driver}])
+  [DRIVER_DIR=${srcdir}/config_src/drivers/${with_driver}])
 
 # TODO: Rather than point to a pre-configured header file, autoconf could be
 # used to configure a header based on a template.
@@ -216,6 +216,7 @@ AS_IF([test -z "$MKMF"], [
 AC_CONFIG_COMMANDS([path_names],
   [list_paths -l \
     ${srcdir}/src \
+    ${srcdir}/config_src/infra/FMS1 \
     ${srcdir}/config_src/ext* \
     ${DRIVER_DIR} \
     ${MEM_LAYOUT}

config_src/coupled_driver/MOM_surface_forcing_gfdl.F90 → config_src/drivers/FMS_cap/MOM_surface_forcing_gfdl.F90

@@ -36,6 +36,7 @@ module MOM_surface_forcing_gfdl
 use MOM_variables,        only : surface
 use user_revise_forcing,  only : user_alter_forcing, user_revise_forcing_init
 use user_revise_forcing,  only : user_revise_forcing_CS
+use iso_fortran_env, only : int64
 
 implicit none ; private
 
@@ -1622,32 +1623,39 @@ subroutine ice_ocn_bnd_type_chksum(id, timestep, iobt)
   type(ice_ocean_boundary_type), &
                     intent(in) :: iobt   !< An ice-ocean boundary type with fluxes to drive the
                                          !! ocean in a coupled model whose checksums are reported
-  integer ::   n,m, outunit
+  ! Local variables
+  integer(kind=int64) :: chks ! A checksum for the field
+  logical :: root    ! True only on the root PE
+  integer :: outunit ! The output unit to write to
 
   outunit = stdout
-
-  write(outunit,*) "BEGIN CHECKSUM(ice_ocean_boundary_type):: ", id, timestep
-  write(outunit,100) 'iobt%u_flux         ', field_chksum( iobt%u_flux         )
-  write(outunit,100) 'iobt%v_flux         ', field_chksum( iobt%v_flux         )
-  write(outunit,100) 'iobt%t_flux         ', field_chksum( iobt%t_flux         )
-  write(outunit,100) 'iobt%q_flux         ', field_chksum( iobt%q_flux         )
-  write(outunit,100) 'iobt%salt_flux      ', field_chksum( iobt%salt_flux      )
-  write(outunit,100) 'iobt%lw_flux        ', field_chksum( iobt%lw_flux        )
-  write(outunit,100) 'iobt%sw_flux_vis_dir', field_chksum( iobt%sw_flux_vis_dir)
-  write(outunit,100) 'iobt%sw_flux_vis_dif', field_chksum( iobt%sw_flux_vis_dif)
-  write(outunit,100) 'iobt%sw_flux_nir_dir', field_chksum( iobt%sw_flux_nir_dir)
-  write(outunit,100) 'iobt%sw_flux_nir_dif', field_chksum( iobt%sw_flux_nir_dif)
-  write(outunit,100) 'iobt%lprec          ', field_chksum( iobt%lprec          )
-  write(outunit,100) 'iobt%fprec          ', field_chksum( iobt%fprec          )
-  write(outunit,100) 'iobt%runoff         ', field_chksum( iobt%runoff         )
-  write(outunit,100) 'iobt%calving        ', field_chksum( iobt%calving        )
-  write(outunit,100) 'iobt%p              ', field_chksum( iobt%p              )
-  if (associated(iobt%ustar_berg)) &
-    write(outunit,100) 'iobt%ustar_berg     ', field_chksum( iobt%ustar_berg )
-  if (associated(iobt%area_berg)) &
-    write(outunit,100) 'iobt%area_berg      ', field_chksum( iobt%area_berg  )
-  if (associated(iobt%mass_berg)) &
-    write(outunit,100) 'iobt%mass_berg      ', field_chksum( iobt%mass_berg  )
+  root = is_root_pe()
+
+  if (root) write(outunit,*) "BEGIN CHECKSUM(ice_ocean_boundary_type):: ", id, timestep
+  chks = field_chksum( iobt%u_flux         ) ; if (root) write(outunit,100) 'iobt%u_flux         ', chks
+  chks = field_chksum( iobt%v_flux         ) ; if (root) write(outunit,100) 'iobt%v_flux         ', chks
+  chks = field_chksum( iobt%t_flux         ) ; if (root) write(outunit,100) 'iobt%t_flux         ', chks
+  chks = field_chksum( iobt%q_flux         ) ; if (root) write(outunit,100) 'iobt%q_flux         ', chks
+  chks = field_chksum( iobt%salt_flux      ) ; if (root) write(outunit,100) 'iobt%salt_flux      ', chks
+  chks = field_chksum( iobt%lw_flux        ) ; if (root) write(outunit,100) 'iobt%lw_flux        ', chks
+  chks = field_chksum( iobt%sw_flux_vis_dir) ; if (root) write(outunit,100) 'iobt%sw_flux_vis_dir', chks
+  chks = field_chksum( iobt%sw_flux_vis_dif) ; if (root) write(outunit,100) 'iobt%sw_flux_vis_dif', chks
+  chks = field_chksum( iobt%sw_flux_nir_dir) ; if (root) write(outunit,100) 'iobt%sw_flux_nir_dir', chks
+  chks = field_chksum( iobt%sw_flux_nir_dif) ; if (root) write(outunit,100) 'iobt%sw_flux_nir_dif', chks
+  chks = field_chksum( iobt%lprec          ) ; if (root) write(outunit,100) 'iobt%lprec          ', chks
+  chks = field_chksum( iobt%fprec          ) ; if (root) write(outunit,100) 'iobt%fprec          ', chks
+  chks = field_chksum( iobt%runoff         ) ; if (root) write(outunit,100) 'iobt%runoff         ', chks
+  chks = field_chksum( iobt%calving        ) ; if (root) write(outunit,100) 'iobt%calving        ', chks
+  chks = field_chksum( iobt%p              ) ; if (root) write(outunit,100) 'iobt%p              ', chks
+  if (associated(iobt%ustar_berg)) then
+    chks = field_chksum( iobt%ustar_berg ) ; if (root) write(outunit,100) 'iobt%ustar_berg     ', chks
+  endif
+  if (associated(iobt%area_berg)) then
+    chks = field_chksum( iobt%area_berg  ) ; if (root) write(outunit,100) 'iobt%area_berg      ', chks
+  endif
+  if (associated(iobt%mass_berg)) then
+    chks = field_chksum( iobt%mass_berg  ) ; if (root) write(outunit,100) 'iobt%mass_berg      ', chks
+  endif
 100 FORMAT("   CHECKSUM::",A20," = ",Z20)
 
   call coupler_type_write_chksums(iobt%fluxes, outunit, 'iobt%')

config_src/coupled_driver/ocean_model_MOM.F90 → config_src/drivers/FMS_cap/ocean_model_MOM.F90

@@ -56,6 +56,7 @@ module ocean_model_mod
 use MOM_ice_shelf, only : add_shelf_forces, ice_shelf_end, ice_shelf_save_restart
 use MOM_wave_interface, only: wave_parameters_CS, MOM_wave_interface_init
 use MOM_wave_interface, only: MOM_wave_interface_init_lite, Update_Surface_Waves
+use iso_fortran_env, only : int64
 
 #include <MOM_memory.h>
 
@@ -1094,25 +1095,29 @@ subroutine ocean_model_data1D_get(OS, Ocean, name, value)
 
 end subroutine ocean_model_data1D_get
 
-!> Write out FMS-format checsums on fields from the ocean surface state
+!> Write out checksums for fields from the ocean surface state
 subroutine ocean_public_type_chksum(id, timestep, ocn)
 
   character(len=*),        intent(in) :: id  !< An identifying string for this call
   integer,                 intent(in) :: timestep !< The number of elapsed timesteps
   type(ocean_public_type), intent(in) :: ocn !< A structure containing various publicly
                                              !! visible ocean surface fields.
-  integer :: n, m, outunit
+  ! Local variables
+  integer(kind=int64) :: chks ! A checksum for the field
+  logical :: root    ! True only on the root PE
+  integer :: outunit ! The output unit to write to
 
   outunit = stdout
-
-  write(outunit,*) "BEGIN CHECKSUM(ocean_type):: ", id, timestep
-  write(outunit,100) 'ocean%t_surf   ', field_chksum(ocn%t_surf )
-  write(outunit,100) 'ocean%s_surf   ', field_chksum(ocn%s_surf )
-  write(outunit,100) 'ocean%u_surf   ', field_chksum(ocn%u_surf )
-  write(outunit,100) 'ocean%v_surf   ', field_chksum(ocn%v_surf )
-  write(outunit,100) 'ocean%sea_lev  ', field_chksum(ocn%sea_lev)
-  write(outunit,100) 'ocean%frazil   ', field_chksum(ocn%frazil )
-  write(outunit,100) 'ocean%melt_potential  ', field_chksum(ocn%melt_potential)
+  root = is_root_pe()
+
+  if (root) write(outunit,*) "BEGIN CHECKSUM(ocean_type):: ", id, timestep
+  chks = field_chksum(ocn%t_surf ) ; if (root) write(outunit,100) 'ocean%t_surf   ', chks
+  chks = field_chksum(ocn%s_surf ) ; if (root) write(outunit,100) 'ocean%s_surf   ', chks
+  chks = field_chksum(ocn%u_surf ) ; if (root) write(outunit,100) 'ocean%u_surf   ', chks
+  chks = field_chksum(ocn%v_surf ) ; if (root) write(outunit,100) 'ocean%v_surf   ', chks
+  chks = field_chksum(ocn%sea_lev) ; if (root) write(outunit,100) 'ocean%sea_lev  ', chks
+  chks = field_chksum(ocn%frazil ) ; if (root) write(outunit,100) 'ocean%frazil   ', chks
+  chks = field_chksum(ocn%melt_potential) ; if (root) write(outunit,100) 'ocean%melt_potential   ', chks
   call coupler_type_write_chksums(ocn%fields, outunit, 'ocean%')
 100 FORMAT("   CHECKSUM::",A20," = ",Z20)
 
@@ -1162,6 +1167,8 @@ subroutine ocean_model_get_UV_surf(OS, Ocean, name, array2D, isc, jsc)
 
   sfc_state => OS%sfc_state
 
+  call pass_vector(sfc_state%u, sfc_state%v, G%Domain)
+
   select case(name)
   case('ua')
     do j=jsc_bnd,jec_bnd ; do i=isc_bnd,iec_bnd

config_src/ice_solo_driver/ice_shelf_driver.F90 → config_src/drivers/ice_solo_driver/ice_shelf_driver.F90

@@ -185,6 +185,8 @@ program Shelf_main
     endif
   endif
 
+  call Get_MOM_Input(param_file, dirs)
+
   ! Read ocean_solo restart, which can override settings from the namelist.
   if (file_exists(trim(dirs%restart_input_dir)//'ice_solo.res')) then
     call open_ASCII_file(unit, trim(dirs%restart_input_dir)//'ice_solo.res', action=READONLY_FILE)
@@ -215,7 +217,6 @@ program Shelf_main
     Start_time = real_to_time(0.0)
   endif
 
-  call Get_MOM_Input(param_file, dirs)
   ! Determining the internal unit scaling factors for this run.
   call unit_scaling_init(param_file, US)
 

config_src/mct_driver/mom_ocean_model_mct.F90 → config_src/drivers/mct_cap/mom_ocean_model_mct.F90

@@ -15,6 +15,7 @@ module MOM_ocean_model_mct
 use MOM,                      only : extract_surface_state, allocate_surface_state, finish_MOM_initialization
 use MOM,                      only : get_MOM_state_elements, MOM_state_is_synchronized
 use MOM,                      only : get_ocean_stocks, step_offline
+use MOM_coms,                 only : field_chksum
 use MOM_constants,            only : CELSIUS_KELVIN_OFFSET, hlf
 use MOM_diag_mediator,        only : diag_ctrl, enable_averaging, disable_averaging
 use MOM_diag_mediator,        only : diag_mediator_close_registration, diag_mediator_end
@@ -65,6 +66,7 @@ module MOM_ocean_model_mct
 
 ! MCT specfic routines
 use MOM_domains,             only : MOM_infra_end
+use iso_fortran_env,         only : int64
 
 #include <MOM_memory.h>
 
@@ -1042,24 +1044,26 @@ subroutine Ocean_stock_pe(OS, index, value, time_index)
 
 end subroutine Ocean_stock_pe
 
-!> Write out FMS-format checsums on fields from the ocean surface state
+!> Write out checksums for fields from the ocean surface state
 subroutine ocean_public_type_chksum(id, timestep, ocn)
 
   character(len=*),        intent(in) :: id  !< An identifying string for this call
   integer,                 intent(in) :: timestep !< The number of elapsed timesteps
   type(ocean_public_type), intent(in) :: ocn !< A structure containing various publicly
                                              !! visible ocean surface fields.
-  integer :: n, m
-
-  write(stdout,*) "BEGIN CHECKSUM(ocean_type):: ", id, timestep
-  write(stdout,100) 'ocean%t_surf   ',mpp_chksum(ocn%t_surf )
-  write(stdout,100) 'ocean%s_surf   ',mpp_chksum(ocn%s_surf )
-  write(stdout,100) 'ocean%u_surf   ',mpp_chksum(ocn%u_surf )
-  write(stdout,100) 'ocean%v_surf   ',mpp_chksum(ocn%v_surf )
-  write(stdout,100) 'ocean%sea_lev  ',mpp_chksum(ocn%sea_lev)
-  write(stdout,100) 'ocean%frazil   ',mpp_chksum(ocn%frazil )
-  write(stdout,100) 'ocean%melt_potential  ',mpp_chksum(ocn%melt_potential)
-
+  ! Local variables
+  integer(kind=int64) :: chks ! A checksum for the field
+  logical :: root    ! True only on the root PE
+  integer :: outunit ! The output unit to write to
+
+  if (root) write(outunit,*) "BEGIN CHECKSUM(ocean_type):: ", id, timestep
+  chks = field_chksum(ocn%t_surf ) ; if (root) write(outunit,100) 'ocean%t_surf   ', chks
+  chks = field_chksum(ocn%s_surf ) ; if (root) write(outunit,100) 'ocean%s_surf   ', chks
+  chks = field_chksum(ocn%u_surf ) ; if (root) write(outunit,100) 'ocean%u_surf   ', chks
+  chks = field_chksum(ocn%v_surf ) ; if (root) write(outunit,100) 'ocean%v_surf   ', chks
+  chks = field_chksum(ocn%sea_lev) ; if (root) write(outunit,100) 'ocean%sea_lev  ', chks
+  chks = field_chksum(ocn%frazil ) ; if (root) write(outunit,100) 'ocean%frazil   ', chks
+  chks = field_chksum(ocn%melt_potential) ; if (root) write(outunit,100) 'ocean%melt_potential   ', chks
   call coupler_type_write_chksums(ocn%fields, stdout, 'ocean%')
 100 FORMAT("   CHECKSUM::",A20," = ",Z20)