update to latest cmeps master (#22)

* fixes incorrect mapping of ifrac for nems_orig and nems_frac 
* adds ability for diagnostic budgets used in cesm
* adds field packing to increase performance

.github/workflows/extbuild.yml

@@ -0,0 +1,106 @@
+# This is a workflow to compile the cdeps source without cime
+name: extbuild
+
+# Controls when the action will run. Triggers the workflow on push or pull request
+# events but only for the master branch
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    branches: [ master ]
+
+# A workflow run is made up of one or more jobs that can run sequentially or in parallel
+jobs:
+  build-cmeps:
+    runs-on: ubuntu-latest
+    env:
+      CC: mpicc
+      FC: mpifort
+      CXX: mpicxx
+      CPPFLAGS: "-I/usr/include -I/usr/local/include"
+      # Versions of all dependencies can be updated here
+      ESMF_VERSION: ESMF_8_1_0_beta_snapshot_31
+      PNETCDF_VERSION: pnetcdf-1.12.1
+      NETCDF_FORTRAN_VERSION: v4.5.2
+      # PIO version is awkward
+      PIO_VERSION_DIR: pio2_5_2
+      PIO_VERSION: pio-2.5.2
+    steps:
+      - uses: actions/checkout@v2
+        # Build the ESMF library, if the cache contains a previous build
+        # it will be used instead
+      - id: cache-esmf
+        uses: actions/cache@v2
+        with:
+          path: ~/ESMF
+          key: ${{ runner.os }}-${{ env.ESMF_VERSION }}-ESMF
+      - id: load-env
+        run: sudo apt-get install gfortran wget openmpi-bin netcdf-bin libopenmpi-dev libnetcdf-dev
+      - id: build-ESMF
+        if: steps.cache-esmf.outputs.cache-hit != 'true'
+        run: |
+          wget https://github.com/esmf-org/esmf/archive/${{ env.ESMF_VERSION }}.tar.gz
+          tar -xzvf ${{ env.ESMF_VERSION }}.tar.gz
+          pushd esmf-${{ env.ESMF_VERSION }}
+          export ESMF_DIR=`pwd`
+          export ESMF_COMM=openmpi
+          export ESMF_YAMLCPP="internal"
+          export ESMF_INSTALL_PREFIX=$HOME/ESMF
+          export ESMF_BOPT=g
+          make
+          make install
+          popd
+      - id: cache-pnetcdf
+        uses: actions/cache@v2
+        with:
+          path: ~/pnetcdf
+          key: ${{ runner.os }}-${{ env.PNETCDF_VERSION}}-pnetcdf
+      - name: pnetcdf build
+        if: steps.cache-pnetcdf.outputs.cache-hit != 'true'
+        run: |
+          wget https://parallel-netcdf.github.io/Release/${{ env.PNETCDF_VERSION }}.tar.gz
+          tar -xzvf ${{ env.PNETCDF_VERSION }}.tar.gz
+          ls -l
+          pushd ${{ env.PNETCDF_VERSION }}
+          ./configure --prefix=$HOME/pnetcdf --enable-shared --disable-cxx
+          make
+          make install
+          popd
+      - name: Cache netcdf-fortran
+        id: cache-netcdf-fortran
+        uses: actions/cache@v2
+        with:
+          path: ~/netcdf-fortran
+          key: ${{ runner.os }}-${{ env.NETCDF_FORTRAN_VERSION }}-netcdf-fortran
+      - name: netcdf fortran build
+        if: steps.cache-netcdf-fortran.outputs.cache-hit != 'true'
+        run: |
+          wget https://github.com/Unidata/netcdf-fortran/archive/${{ env.NETCDF_FORTRAN_VERSION }}.tar.gz
+          tar -xzvf ${{ env.NETCDF_FORTRAN_VERSION }}.tar.gz
+          ls -l
+          pushd netcdf-fortran-*
+          ./configure --prefix=$HOME/netcdf-fortran
+          make
+          make install
+
+      - name: Build PIO
+        if: steps.cache-PIO.outputs.cache-hit != 'true'
+        run: |
+          git submodule init
+          git submodule update
+          mkdir build-pio
+          pushd build-pio
+          cmake -Wno-dev -DNetCDF_C_LIBRARY=/usr/lib/x86_64-linux-gnu/libnetcdf.so -DNetCDF_C_INCLUDE_DIR=/usr/include -DCMAKE_PREFIX_PATH=/usr -DCMAKE_INSTALL_PREFIX=$HOME/pio -DPIO_HDF5_LOGGING=On -DPIO_USE_MALLOC=On -DPIO_ENABLE_LOGGING=On -DPIO_ENABLE_TIMING=Off -DPIO_ENABLE_TESTS=OFF -DNetCDF_Fortran_PATH=$HOME/netcdf-fortran -DPnetCDF_PATH=$HOME/pnetcdf ../nems/lib/ParallelIO
+          make VERBOSE=1
+          make install
+          popd
+
+      - name: Build CMEPS
+        run: |
+          export ESMFMKFILE=$HOME/ESMF/lib/libg/Linux.gfortran.64.openmpi.default/esmf.mk
+          export PIO=$HOME/pio
+          mkdir build-cmeps
+          pushd build-cmeps
+          cmake -DCMAKE_BUILD_TYPE=DEBUG -DCMAKE_Fortran_FLAGS="-g -Wall -ffree-form -ffree-line-length-none" ../
+          make VERBOSE=1
+          popd

.travis.yml

@@ -6,9 +6,10 @@ install:
 python:
   - '2.7'
   - '3.6'
+  - '3.8'
 
 branches:
   only:
   - master
 
-script: ./tcipylint
+script: ./tcipylint

Makefile

@@ -19,6 +19,11 @@ ifndef CXX
 $(error CXX not defined)
 endif
 
+ifndef INTERNAL_PIO_INIT
+INTERNAL_PIO_INIT := 1
+endif
+$(info INTERNAL_PIO_INIT is set to $(INTERNAL_PIO_INIT))
+
 MEDIATOR_DIR := $(BASE_DIR)/mediator
 LIBRARY_MEDIATOR := $(MEDIATOR_DIR)/libcmeps.a
 LIBRARY_UTIL := $(BASE_DIR)/nems/util/libcmeps_util.a
@@ -48,7 +53,7 @@ endif
 
 $(LIBRARY_MEDIATOR): $(LIBRARY_UTIL) .FORCE
 	cd mediator ;\
-	exec $(MAKE) INTERNAL_PIO_INIT=1
+	exec $(MAKE) PIO_INCLUDE_DIR=$(PIO_INCLUDE_DIR) INTERNAL_PIO_INIT=$(INTERNAL_PIO_INIT)
 
 $(LIBRARY_UTIL): .FORCE
 	cd nems/util ;\

cime_config/buildexe

@@ -6,7 +6,10 @@ build model executable
 
 import sys, os
 
-_CIMEROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..","..","..","..")
+_CIMEROOT = os.environ.get("CIMEROOT")
+if _CIMEROOT is None:
+    raise SystemExit("ERROR: must set CIMEROOT environment variable")
+
 sys.path.append(os.path.join(_CIMEROOT, "scripts", "Tools"))
 
 from standard_script_setup import *
@@ -28,7 +31,6 @@ def _main_func():
 
     with Case(caseroot) as case:
         casetools = case.get_value("CASETOOLS")
-        cimeroot  = case.get_value("CIMEROOT")
         exeroot   = case.get_value("EXEROOT")
         gmake     = case.get_value("GMAKE")
         gmake_j   = case.get_value("GMAKE_J")
@@ -37,7 +39,6 @@ def _main_func():
         ocn_model = case.get_value("COMP_OCN")
         atm_model = case.get_value("COMP_ATM")
         gmake_args = get_standard_makefile_args(case)
-        blddir = os.path.join(case.get_value("EXEROOT"),"cpl","obj")
 
     # Determine valid components
     valid_comps = []
@@ -50,7 +51,7 @@ def _main_func():
         if valid:
             valid_comps.append(item)
 
-    datamodel_in_compset = False 
+    datamodel_in_compset = False
     for comp in comp_classes:
         dcompname = "d"+comp.lower()
         if dcompname in case.get_value("COMP_{}".format(comp)):
@@ -81,10 +82,12 @@ def _main_func():
         os.makedirs(bld_root)
 
     with open(os.path.join(bld_root,'Filepath'), 'w') as out:
-        if not skip_mediator:
-            out.write(os.path.join(cimeroot, "src", "drivers", "nuopc", "mediator") + "\n")
+        cmeps_dir = os.path.join(os.path.dirname(__file__), os.pardir)
+        # SourceMods dir needs to be first listed
         out.write(os.path.join(caseroot, "SourceMods", "src.drv") + "\n")
-        out.write(os.path.join(cimeroot, "src", "drivers", "nuopc", "drivers", "cime") + "\n")
+        if not skip_mediator:
+            out.write(os.path.join(cmeps_dir, "mediator") + "\n")
+        out.write(os.path.join(cmeps_dir, "drivers", "cime") + "\n")
 
     # build model executable
     makefile = os.path.join(casetools, "Makefile")

cime_config/buildnml

@@ -3,7 +3,10 @@
 """
 import os, sys
 
-_CIMEROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..","..","..","..")
+_CIMEROOT = os.environ.get("CIMEROOT")
+if _CIMEROOT is None:
+    raise SystemExit("ERROR: must set CIMEROOT environment variable")
+
 sys.path.append(os.path.join(_CIMEROOT, "scripts", "Tools"))
 
 import shutil, glob, itertools
@@ -237,7 +240,7 @@ def _create_drv_namelists(case, infile, confdir, nmlgen, files):
             valid_comps.append(item)
 
     # Determine if there are any data components in the compset
-    datamodel_in_compset = False 
+    datamodel_in_compset = False
     comp_classes = case.get_values("COMP_CLASSES")
     for comp in comp_classes:
         dcompname = "d"+comp.lower()
@@ -248,14 +251,12 @@ def _create_drv_namelists(case, infile, confdir, nmlgen, files):
     # driver rpointer file if there is only one non-stub component then skip mediator
     if len(valid_comps) == 2 and not datamodel_in_compset:
         # skip the mediator if there is a prognostic component and all other components are stub
-        skip_mediator = True
         valid_comps.remove("CPL")
         nmlgen.set_value('mediator_present', value='.false.')
         nmlgen.set_value("drv_restart_pointer", value="none")
         nmlgen.set_value("component_list", value=" ".join(valid_comps))
     else:
         # do not skip mediator if there is a data component but all other components are stub
-        skip_mediator = False
         nmlgen.set_value("drv_restart_pointer", value="rpointer.cpl")
         valid_comps_string = " ".join(valid_comps)
         nmlgen.set_value("component_list", value=valid_comps_string.replace("CPL","MED"))
@@ -392,7 +393,7 @@ def _create_runseq(case, coupling_times, valid_comps):
             comp_lnd = case.get_value("COMP_LND")
             comp_ocn = case.get_value("COMP_OCN")
 
-            sys.path.append(os.path.join(_CIMEROOT, "src", "drivers", "nuopc", "cime_config", "runseq"))
+            sys.path.append(os.path.join(os.path.dirname(__file__), "runseq"))
 
             if (comp_ice == "cice" and comp_atm == 'datm' and comp_ocn == "docn"):
                 from runseq_D import gen_runseq
@@ -564,14 +565,13 @@ def buildnml(case, caseroot, component):
         shutil.copy(filename, rundir)
 
     # copy fd_cesm.yaml to rundir
-    cimeroot = case.get_value("CIMEROOT")
-    fd_dir = os.path.join(cimeroot, "src","drivers","nuopc","mediator")
+    fd_dir = os.path.join(os.path.dirname(__file__),os.pardir,"mediator")
     coupling_mode = case.get_value('COUPLING_MODE')
     if coupling_mode == 'cesm':
         filename = os.path.join(fd_dir,"fd_cesm.yaml")
     elif coupling_mode == 'hafs':
         filename = os.path.join(fd_dir,"fd_hafs.yaml")
-    elif coupling_mode == 'nems':
+    elif 'nems' in coupling_mode:
         filename = os.path.join(fd_dir,"fd_nems.yaml")
     else:
         expect(False, "coupling mode currently only supports cesm, hafs and nems")

cime_config/config_component.xml

@@ -28,7 +28,7 @@
 
   <entry id="COUPLING_MODE">
     <type>char</type>
-    <valid_values>cesm,nems_orig_active,nems_orig_data,nems_frac,hafs</valid_values>
+    <valid_values>cesm,nems_orig,nems_orig_data,nems_frac,hafs</valid_values>
     <default_value>cesm</default_value>
     <group>run_coupling</group>
     <file>env_run.xml</file>
@@ -987,9 +987,9 @@
     <file>env_run.xml</file>
     <desc>
       Determines what ESMF log files (if any) are generated when
-          USE_ESMF_LIB is TRUE.
+	  USE_ESMF_LIB is TRUE.
       ESMF_LOGKIND_SINGLE: Use a single log file, combining messages from
-          all of the PETs. Not supported on some platforms.
+	  all of the PETs. Not supported on some platforms.
       ESMF_LOGKIND_MULTI: Use multiple log files -- one per PET.
       ESMF_LOGKIND_NONE: Do not issue messages to a log file.
       By default, no ESMF log files are generated.
@@ -2018,7 +2018,7 @@
     <desc>pio rearranger communication max pending requests (io2comp) :
        -2 implies that CIME internally calculates the value ( = 64),
        -1 implies no bound on max pending requests
-        0 implies that MPI_ALLTOALL will be used
+	0 implies that MPI_ALLTOALL will be used
     </desc>
   </entry>
 
@@ -2600,7 +2600,7 @@
  </entry>
 
   <!-- ===================================================================== -->
-  <!-- write ESMF PET log files even without an error -->
+  <!-- Include the AOFLUX calculation for this compset -->
   <!-- ===================================================================== -->
 
  <entry id="ADD_AOFLUX_TO_RUNSEQ">

cime_config/runseq/runseq_general.py

@@ -18,8 +18,10 @@ def gen_runseq(case, coupling_times):
     rundir         = case.get_value("RUNDIR")
     caseroot       = case.get_value("CASEROOT")
     cpl_seq_option = case.get_value('CPL_SEQ_OPTION')
-    cpl_add_aoflux = case.get_value('ADD_AOFLUX_TO_RUNSEQ')
     coupling_mode  = case.get_value('COUPLING_MODE')
+    diag_mode      = case.get_value('BUDGETS')
+    xcompset = case.get_value("COMP_ATM") == 'xatm'
+    cpl_add_aoflux = not xcompset and case.get_value('ADD_AOFLUX_TO_RUNSEQ')
 
     # It is assumed that if a component will be run it will send information to the mediator
     # so the flags run_xxx and xxx_to_med are redundant
@@ -33,6 +35,7 @@ def gen_runseq(case, coupling_times):
     run_rof, med_to_rof, rof_cpl_time = driver_config['rof']
     run_wav, med_to_wav, wav_cpl_time = driver_config['wav']
 
+
     # Note: assume that atm_cpl_dt, lnd_cpl_dt, ice_cpl_dt and wav_cpl_dt are the same
 
     if lnd_cpl_time != atm_cpl_time:
@@ -88,11 +91,12 @@ def gen_runseq(case, coupling_times):
                 runseq.add_action("MED med_phases_aofluxes_run"        , run_ocn and run_atm and (med_to_ocn or med_to_atm))
             runseq.add_action("MED med_phases_prep_ocn_merge"      , med_to_ocn)
             runseq.add_action("MED med_phases_prep_ocn_accum_fast" , med_to_ocn)
-            runseq.add_action("MED med_phases_ocnalb_run"          , run_ocn and run_atm and (med_to_ocn or med_to_atm))
+            runseq.add_action("MED med_phases_ocnalb_run"          , (run_ocn and run_atm and (med_to_ocn or med_to_atm)) and not xcompset)
         runseq.add_action("MED med_phases_prep_lnd"                , med_to_lnd)
         runseq.add_action("MED -> LND :remapMethod=redist"         , med_to_lnd)
         runseq.add_action("MED med_phases_prep_ice"                , med_to_ice)
         runseq.add_action("MED -> ICE :remapMethod=redist"         , med_to_ice)
+        runseq.add_action("MED med_phases_diag_ice_med2ice"        , run_ice and diag_mode)
         runseq.add_action("MED med_phases_prep_wav"                , med_to_wav)
         runseq.add_action("MED -> WAV :remapMethod=redist"         , med_to_wav)
         runseq.add_action("MED med_phases_prep_rof_avg"            , med_to_rof and not rof_outer_loop)
@@ -114,10 +118,13 @@ def gen_runseq(case, coupling_times):
                 runseq.add_action("MED med_phases_aofluxes_run"        , run_ocn and run_atm)
             runseq.add_action("MED med_phases_prep_ocn_merge"      , med_to_ocn)
             runseq.add_action("MED med_phases_prep_ocn_accum_fast" , med_to_ocn)
-            runseq.add_action("MED med_phases_ocnalb_run"          , run_ocn and run_atm)
+            runseq.add_action("MED med_phases_ocnalb_run"          , (run_ocn and run_atm) and not xcompset)
+        runseq.add_action("MED med_phases_diag_ocn"                , run_ocn and diag_mode and not ocn_outer_loop)
         runseq.add_action("LND -> MED :remapMethod=redist"         , run_lnd)
         runseq.add_action("ICE -> MED :remapMethod=redist"         , run_ice)
+        runseq.add_action("MED med_phases_diag_ice_ice2med"        , run_ice and diag_mode)
         runseq.add_action("MED med_fraction_set"                   , run_ice)
+
         runseq.add_action("MED med_phases_prep_rof_accum"          , med_to_rof)
         runseq.add_action("MED med_phases_prep_glc_accum"          , med_to_glc)
         runseq.add_action("MED med_phases_prep_atm"                , med_to_atm)
@@ -126,17 +133,24 @@ def gen_runseq(case, coupling_times):
         runseq.add_action("ATM -> MED :remapMethod=redist"         , run_atm)
         runseq.add_action("WAV -> MED :remapMethod=redist"         , run_wav)
         runseq.add_action("ROF -> MED :remapMethod=redist"         , run_rof and not rof_outer_loop)
-
+        runseq.add_action("MED med_phases_diag_atm"                , run_atm and diag_mode)
+        runseq.add_action("MED med_phases_diag_lnd"                , run_lnd and diag_mode)
+        runseq.add_action("MED med_phases_diag_rof"                , run_rof and diag_mode)
+        runseq.add_action("MED med_phases_diag_glc"                , run_glc and diag_mode)
+        runseq.add_action("MED med_phases_diag_accum"              , diag_mode)
+        runseq.add_action("MED med_phases_diag_print"              , diag_mode)
         #------------------
         runseq.leave_time_loop(inner_loop)
         #------------------
 
         runseq.add_action("OCN"                                    , run_ocn and ocn_outer_loop)
+
         if coupling_mode == 'hafs':
             runseq.add_action("OCN -> MED :remapMethod=redist:ignoreUnmatchedIndices=true"         , run_ocn and ocn_outer_loop)
         else:
             runseq.add_action("OCN -> MED :remapMethod=redist"         , run_ocn and ocn_outer_loop)
 
+
         #------------------
         runseq.leave_time_loop(ocn_outer_loop)
         #------------------