Merge tag 'ctsm5.1.dev112' into subset_mesh_slevis

Rework fates test definitions and add new fates tests This tag includes a number of updates to the fates test definitions and test list to gain more coverage: (1) Reorder and update the fates test definitions so that fates satellite phenology mode can be configured for cases using a compset. (2) Add a long-term exact restart test to catch issues that may arise due to updates to fates procedure calls during end of year simulation dates. (3) Add a no-competition + fixed biogeography, non-satellite phenology test definition to provide additional mode combination configuration. (4) Update the fates externals tag to incorporate a fix a vegetation temperature exact restart issue discovered while implementing (1) above. (5) Truncate all testmods starting with "Fates" to mitigate running over the limits on the length of testnames, typically when specifying custom `test_id` using `run_sys_test`. (6) Updates the expected failures list. Resolved conflicts: python/ctsm/subset_data.py
ESCOMP · Oct 26, 2022 · 623da32 · 623da32
2 parents 60b232f + 616905b
commit 623da32
Show file tree

Hide file tree

Showing 285 changed files with 18,088 additions and 7,251 deletions.
diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs
@@ -0,0 +1,9 @@
+# Ran python directory through black python formatter
+4cd83cb3ee6d85eb909403487abf5eeaf4d98911
+0aa2957c1f8603c63fa30b11295c06cfddff44a5
+2cdb380febb274478e84cd90945aee93f29fa2e6
+e44dc469439e02e9ee582dab274d890ebdfab104
+565e7bcacbb0bba20ff80d680d8ad769676641c6
+b88e1cd1b28e3609684c79a2ec0e88f26cfc362b
+51c102c5df2e0ef971b5f8eeeb477567899af63a
+7dacad70e74e2ec97f6492d4e7a3cb5dd498bcd7
diff --git a/.github/workflows/black.yml b/.github/workflows/black.yml
@@ -0,0 +1,21 @@
+name: black check on push and PR
+#
+# Run the python formatting in check mode
+#
+on: [push, pull_request]
+
+jobs:
+  black-check:
+     runs-on: ubuntu-latest
+     steps:
+        # Checkout the code
+        - uses: actions/checkout@v2
+        # Use the latest stable version of the github action
+        - uses: psf/black@stable
+          with:
+             # Use options and version identical to the conda environment
+             # Using pyproject.toml makes sure this testing is consistent with our python directory testing
+             options: "--check --config python/pyproject.toml"
+             src:     "./python"
+             # Version should be coordinated with the ctsm_py conda environment under the python directory
+             version: "22.3.0"
diff --git a/Externals.cfg b/Externals.cfg
@@ -34,7 +34,7 @@ hash = 34723c2
 required = True
 
 [ccs_config]
-tag = ccs_config_cesm0.0.15
+tag = ccs_config_cesm0.0.38
 protocol = git
 repo_url = https://github.com/ESMCI/ccs_config_cesm.git
 local_path = ccs_config
@@ -44,33 +44,33 @@ required = True
 local_path = cime
 protocol = git
 repo_url = https://github.com/ESMCI/cime
-tag = cime6.0.15
+tag = cime6.0.45
 required = True
 
 [cmeps]
-tag = cmeps0.13.47
+tag = cmeps0.13.71
 protocol = git
 repo_url = https://github.com/ESCOMP/CMEPS.git
 local_path = components/cmeps
 required = True
 
 [cdeps]
-tag = cdeps0.12.41
+tag = cdeps0.12.65
 protocol = git
 repo_url = https://github.com/ESCOMP/CDEPS.git
 local_path = components/cdeps
 externals =  Externals_CDEPS.cfg
 required = True
 
 [cpl7]
-tag = cpl7.0.12
+tag = cpl7.0.14
 protocol = git
 repo_url = https://github.com/ESCOMP/CESM_CPL7andDataComps
 local_path = components/cpl7
 required = True
 
 [share]
-tag = share1.0.10
+tag = share1.0.13
 protocol = git
 repo_url = https://github.com/ESCOMP/CESM_share
 local_path = share
@@ -84,7 +84,7 @@ local_path = libraries/mct
 required = True
 
 [parallelio]
-tag = pio2_5_6
+tag = pio2_5_7
 protocol = git
 repo_url = https://github.com/NCAR/ParallelIO
 local_path = libraries/parallelio

diff --git a/Externals_CLM.cfg b/Externals_CLM.cfg
@@ -2,7 +2,7 @@
 local_path = src/fates
 protocol = git
 repo_url = https://github.com/NGEET/fates
-tag = sci.1.55.4_api.22.1.0
+tag = sci.1.59.7_api.24.1.0
 required = True
 
 [externals_description]

diff --git a/README b/README
@@ -45,6 +45,7 @@ test -------------- CTSM Testing scripts for CTSM offline tools.
 tools ------------- CTSM Offline tools to prepare input datasets and process output.
 cime_config ------- Configuration files of cime for compsets and CTSM settings
 manage_externals -- Script to manage the external source directories
+py_env_create ----- Script to setup the python environment for CTSM python tools using conda
 python ------------ Some python modules mostly for use by run_sys_tests (but could be used elsewhere l
 
 Directory structure only for a CTSM checkout:

diff --git a/bld/CLMBuildNamelist.pm b/bld/CLMBuildNamelist.pm
@@ -741,7 +741,7 @@ sub setup_cmdl_fates_mode {
 
     $var = "use_fates";
     if ( &value_is_true($nl_flags->{$var}) ) {
-      # This section is a place-holder to test for modules that are not allowed with ED
+      # This section is a place-holder to test for modules that are not allowed with FATES
       # the defaults which are set in the logic section of the namelist builder will
       # automatically set these correctly (well that is the assumption), but here we
       # want to set a catch to fail and warn users if they explicitly set incompatible user namelist
@@ -774,7 +774,7 @@ sub setup_cmdl_fates_mode {
        # dis-allow fates specific namelist items with non-fates runs
        my @list  = (  "fates_spitfire_mode", "use_fates_planthydro", "use_fates_ed_st3", "use_fates_ed_prescribed_phys",
                       "use_fates_cohort_age_tracking",
-                      "use_fates_inventory_init","use_fates_fixed_biogeog","use_fates_nocomp","use_fates_sp","fates_inventory_ctrl_filename","use_fates_logging","fates_parteh_mode" );
+                      "use_fates_inventory_init","use_fates_fixed_biogeog","use_fates_nocomp","use_fates_sp","fates_inventory_ctrl_filename","use_fates_logging","fates_parteh_mode","use_fates_tree_damage" );
        # dis-allow fates specific namelist items with non-fates runs
        foreach my $var ( @list ) {
           if ( defined($nl->get_value($var)) ) {
@@ -863,16 +863,34 @@ sub setup_cmdl_bgc {
      }
   }
   #
+  # Set FATES-SP mode
+  #
+  if ( &value_is_true( $nl_flags->{'use_fates'} ) ) {
+     add_default($opts, $nl_flags->{'inputdata_rootdir'}, $definition, $defaults, $nl, 'use_fates_sp', 'use_fates'=>$nl_flags->{'use_fates'} );
+     if ( &value_is_true($nl->get_value('use_fates_sp')) ) {
+        $nl_flags->{'use_fates_sp'} = ".true.";
+     } else {
+        $nl_flags->{'use_fates_sp'} = ".false.";
+     }
+  } else {
+     $nl_flags->{'use_fates_sp'} = ".false.";
+  }
+  #
   # Determine Soil decomposition method
   #
   my $var = "soil_decomp_method";
   add_default($opts, $nl_flags->{'inputdata_rootdir'}, $definition, $defaults, $nl, $var,
-              'phys'=>$nl_flags->{'phys'}, 'use_cn'=>$nl_flags->{'use_cn'}, 'use_fates'=>$nl_flags->{'use_fates'} );
+              'phys'=>$nl_flags->{'phys'}, 'use_cn'=>$nl_flags->{'use_cn'}, 'use_fates'=>$nl_flags->{'use_fates'},
+              'use_fates_sp'=>$nl_flags->{'use_fates_sp'} );
   my $soil_decomp_method = remove_leading_and_trailing_quotes( $nl->get_value( $var ) );
 
-  if ( &value_is_true($nl_flags->{'use_cn'}) ||  &value_is_true($nl_flags->{'use_fates'}))  {
+  if (      &value_is_true($nl_flags->{'use_cn'}) ) {
+     if ( $soil_decomp_method eq "None" ) {
+        $log->fatal_error("$var must NOT be None if use_cn is on");
+     }
+  } elsif ( &value_is_true($nl_flags->{'use_fates'}) && (not &value_is_true($nl_flags->{'use_fates_sp'}))  )  {
      if ( $soil_decomp_method eq "None" ) {
-        $log->fatal_error("$var must NOT be None if use_cn or use_fates are on");
+        $log->fatal_error("$var must NOT be None if use_fates is on and use_fates_sp is not TRUE");
      }
   } elsif ( $soil_decomp_method ne "None" ) {
      $log->fatal_error("$var must be None if use_cn and use_fates are off");
@@ -901,6 +919,12 @@ sub setup_cmdl_bgc {
            $log->warning("$var normally use_nitrif_denitrif should only be FALSE if FATES is on, it has NOT been validated for being off for BGC mode" );
         }
      }
+     # if MIMICS is on and use_fates = .true. then use_lch4 must = .true.
+     if ( (! &value_is_true($nl_flags->{'use_lch4'})) && &value_is_true($nl_flags->{'use_fates'}) ) {
+        if ( $soil_decomp_method eq "MIMICSWieder2015" ) {
+           $log->warning("If MIMICS is on and use_fates = .true. then use_lch4 must be .true. and currently it's not" );
+        }
+     }
   }
   #
   # Set FUN for BGC
@@ -1143,7 +1167,7 @@ sub setup_cmdl_spinup {
      $log->fatal_error("$var has an invalid value ($val). Valid values are: @valid_values");
   }
   if ( $nl_flags->{'bgc_spinup'} eq "on" && (not &value_is_true( $nl_flags->{'use_cn'} ))  && (not &value_is_true($nl_flags->{'use_fates'})) ) {
-     $log->fatal_error("$var can not be '$nl_flags->{'bgc_spinup'}' if neither CN nor ED is turned on (use_cn=$nl_flags->{'use_cn'}, use_fates=$nl_flags->{'use_fates'}).");
+     $log->fatal_error("$var can not be '$nl_flags->{'bgc_spinup'}' if neither CN nor FATES is turned on (use_cn=$nl_flags->{'use_cn'}, use_fates=$nl_flags->{'use_fates'}).");
   }
   if ( $nl->get_value("spinup_state") eq 0 && $nl_flags->{'bgc_spinup'} eq "on" ) {
      $log->fatal_error("Namelist spinup_state contradicts the command line option bgc_spinup" );
@@ -2022,6 +2046,10 @@ sub setup_logic_subgrid {
    add_default($opts, $nl_flags->{'inputdata_rootdir'}, $definition, $defaults, $nl, 'toosmall_lake');
    add_default($opts, $nl_flags->{'inputdata_rootdir'}, $definition, $defaults, $nl, 'toosmall_wetland');
    add_default($opts, $nl_flags->{'inputdata_rootdir'}, $definition, $defaults, $nl, 'toosmall_urban');
+
+   if ( &value_is_true($nl_flags->{'use_fates'}) && $nl->get_value('n_dom_pfts') != 0 ) {
+      $log->fatal_error( "FATES and n_dom_pfts can NOT be set at the same time" );
+   }
 }
 
 #-------------------------------------------------------------------------------
@@ -2587,7 +2615,7 @@ sub setup_logic_do_transient_crops {
       # In principle, use_fates should be compatible with
       # do_transient_crops. However, this hasn't been tested, so to be safe,
       # we are not allowing this combination for now.
-      $cannot_be_true = "$var has not been tested with ED, so for now these two options cannot be combined";
+      $cannot_be_true = "$var has not been tested with FATES, so for now these two options cannot be combined";
    }
 
    if ($cannot_be_true) {
@@ -2698,8 +2726,8 @@ sub setup_logic_do_transient_urban {
    # for them to be unset if that will be their final state):
    # - flanduse_timeseries
    #
-   # NOTE(kwo, 2021-08-11) I based this function on setup_logic_do_transient_lakes. 
-   # As in NOTE(wjs, 2020-08-23) I'm not sure if all of the checks here are truly important 
+   # NOTE(kwo, 2021-08-11) I based this function on setup_logic_do_transient_lakes.
+   # As in NOTE(wjs, 2020-08-23) I'm not sure if all of the checks here are truly important
    # for transient urban (in particular, my guess is that collapse_urban could probably be done with transient
    # urban - as well as transient pfts and transient crops for that matter), but some of
    # the checks probably are needed, and it seems best to keep transient urban consistent
@@ -3636,12 +3664,15 @@ sub setup_logic_dry_deposition {
   my ($opts, $nl_flags, $definition, $defaults, $nl) = @_;
 
   if ($opts->{'drydep'} ) {
+    if ( &value_is_true( $nl_flags->{'use_fates'}) && not &value_is_true( $nl_flags->{'use_fates_sp'}) ) {
+       $log->warning("DryDeposition can NOT be on when FATES is also on unless FATES-SP mode is on.\n" .
+                     "   Use the '--no-drydep' option when '-bgc fates' is activated");
+    }
     add_default($opts,  $nl_flags->{'inputdata_rootdir'}, $definition, $defaults, $nl, 'drydep_list');
-    add_default($opts,  $nl_flags->{'inputdata_rootdir'}, $definition, $defaults, $nl, 'drydep_method');
+    add_default($opts,  $nl_flags->{'inputdata_rootdir'}, $definition, $defaults, $nl, 'dep_data_file');
   } else {
-    if ( defined($nl->get_value('drydep_list')) ||
-         defined($nl->get_value('drydep_method')) ) {
-      $log->fatal_error("drydep_list or drydep_method defined, but drydep option NOT set");
+    if ( defined($nl->get_value('drydep_list')) ) {
+      $log->fatal_error("drydep_list defined, but drydep option NOT set");
     }
   }
 }
@@ -3681,7 +3712,7 @@ sub setup_logic_megan {
 
   if ($nl_flags->{'megan'} ) {
     if ( &value_is_true( $nl_flags->{'use_fates'} ) ) {
-       $log->fatal_error("MEGAN can NOT be on when ED is also on.\n" .
+       $log->warning("MEGAN can NOT be on when FATES is also on.\n" .
                    "   Use the '-no-megan' option when '-bgc fates' is activated");
     }
     add_default($opts,  $nl_flags->{'inputdata_rootdir'}, $definition, $defaults, $nl, 'megan_specifier');
@@ -4081,15 +4112,9 @@ sub setup_logic_fates {
 
     if (&value_is_true( $nl_flags->{'use_fates'})  ) {
         add_default($opts, $nl_flags->{'inputdata_rootdir'}, $definition, $defaults, $nl, 'fates_paramfile', 'phys'=>$nl_flags->{'phys'});
-        add_default($opts, $nl_flags->{'inputdata_rootdir'}, $definition, $defaults, $nl, 'use_fates_sp', 'use_fates'=>$nl_flags->{'use_fates'} );
-        if ( &value_is_true($nl->get_value('use_fates_sp')) ) {
-           $nl_flags->{'use_fates_sp'} = ".true.";
-        } else {
-           $nl_flags->{'use_fates_sp'} = ".false.";
-        }
         my @list  = (  "fates_spitfire_mode", "use_fates_planthydro", "use_fates_ed_st3", "use_fates_ed_prescribed_phys",
                        "use_fates_inventory_init","use_fates_fixed_biogeog","use_fates_nocomp",
-                       "use_fates_logging","fates_parteh_mode", "use_fates_cohort_age_tracking" );
+                       "use_fates_logging","fates_parteh_mode", "use_fates_cohort_age_tracking","use_fates_tree_damage" );
         foreach my $var ( @list ) {
  	  add_default($opts, $nl_flags->{'inputdata_rootdir'}, $definition, $defaults, $nl, $var, 'use_fates'=>$nl_flags->{'use_fates'},
                       'use_fates_sp'=>$nl_flags->{'use_fates_sp'} );

diff --git a/bld/README b/bld/README
@@ -23,7 +23,7 @@ env_run.xml --- Sample case runtime environment variables, so build-namelist can
 listDefaultNamelist.pl -- List the files needed, for a list of resolutions,
                           to run CLM that are currently NOT on your machine.
                           This file can then be used by 
-                            cime/scripts/Tools/check_input_data
+                            cime/CIME/Tools/check_input_data
                           to retreive them from the inputdata repository.
                           Setting up cases with create_newcase also does
                           this -- but only for the exact configuration

diff --git a/bld/listDefaultNamelist.pl b/bld/listDefaultNamelist.pl
@@ -17,7 +17,7 @@
 #
 # To then get the files from the CESM SVN repository:
 #
-# ../cime/scripts/Tools/check_input_data --data-list-dir . --download
+# ../cime/CIME/Tools/check_input_data --data-list-dir . --download
 #
 #=======================================================================
 
@@ -84,7 +84,7 @@ sub usage {
 
   to then read the resulting clm.input_data_list file and retreive the files
 
-  ../cime/scripts/Tools/check_input_data --data-list-dir . --download
+  ../cime/CIME/Tools/check_input_data --data-list-dir . --download
 
 EOF
 }

diff --git a/bld/namelist_files/namelist_defaults_ctsm.xml b/bld/namelist_files/namelist_defaults_ctsm.xml
@@ -491,7 +491,7 @@ attributes from the config_cache.xml file (with keys converted to upper-case).
 <!-- FATES default parameter file                                       -->
 <!-- ================================================================== -->
 
-<fates_paramfile>lnd/clm2/paramdata/fates_params_api.22.1.0_12pft_c220307.nc</fates_paramfile>
+<fates_paramfile>lnd/clm2/paramdata/fates_params_api.24.0.0_12pft_c220608.nc</fates_paramfile>
 
 <!-- ========================================================================================  -->
 <!-- clm 5.0 BGC nitrogen model                                                                -->
@@ -2528,9 +2528,10 @@ lnd/clm2/surfdata_map/release-clm5.0.30/surfdata_ne0np4.CONUS.ne30x8_hist_78pfts
 <!-- Defaults for different BGC/decomp modes    -->
 <!-- =========================================  -->
 
-<soil_decomp_method use_cn=".true."    >CENTURYKoven2013</soil_decomp_method>
-<soil_decomp_method use_fates=".true." >CENTURYKoven2013</soil_decomp_method>
-<soil_decomp_method                    >None</soil_decomp_method>
+<soil_decomp_method use_cn=".true."                          >CENTURYKoven2013</soil_decomp_method>
+<soil_decomp_method use_fates=".true."                       >CENTURYKoven2013</soil_decomp_method>
+<soil_decomp_method use_fates=".true." use_fates_sp=".true." >None</soil_decomp_method>
+<soil_decomp_method                                          >None</soil_decomp_method>
 
 <use_lch4            soil_decomp_method="None" >.false.</use_lch4>
 <use_lch4                                      >.true.</use_lch4>
@@ -2540,6 +2541,7 @@ lnd/clm2/surfdata_map/release-clm5.0.30/surfdata_ne0np4.CONUS.ne30x8_hist_78pfts
 <!-- ===== FATES DEFAULTS =========== -->
 <fates_spitfire_mode           use_fates=".true.">0</fates_spitfire_mode>
 <use_fates_planthydro          use_fates=".true.">.false.</use_fates_planthydro>
+<use_fates_tree_damage         use_fates=".true.">.false.</use_fates_tree_damage>
 <use_fates_cohort_age_tracking use_fates=".true.">.false.</use_fates_cohort_age_tracking>
 <use_fates_ed_st3              use_fates=".true.">.false.</use_fates_ed_st3>
 <use_fates_ed_prescribed_phys  use_fates=".true.">.false.</use_fates_ed_prescribed_phys>

diff --git a/bld/namelist_files/namelist_defaults_drydep.xml b/bld/namelist_files/namelist_defaults_drydep.xml
@@ -17,7 +17,9 @@ attributes from the config_cache.xml file (with keys converted to upper-case).
 
 <drydep_list>'O3','NO2','HNO3','NO','HO2NO2','CH3OOH','CH2O','CO','H2O2','CH3COOOH','PAN','MPAN','C2H5OOH','ONIT','POOH','C3H7OOH','ROOH','CH3COCHO','CH3COCH3','Pb','ONITR','MACROOH','XOOH','ISOPOOH','CH3OH','C2H5OH','CH3CHO','GLYALD','HYAC','HYDRALD','ALKOOH','MEKOOH','TOLOOH','TERPOOH','CH3COOH','CB1','CB2','OC1','OC2','SOA','SO2','SO4','NH3','NH4NO3'
 </drydep_list>
-<drydep_method>xactive_lnd</drydep_method>
+
+<!-- effective Henry's coef data for wet and dry deposition -->
+<dep_data_file>atm/cam/chem/trop_mozart/dvel/dep_data_c201019.nc</dep_data_file>
 
 <!-- Defaults for megan_emis_nl -->
 

diff --git a/bld/namelist_files/namelist_definition_ctsm.xml b/bld/namelist_files/namelist_definition_ctsm.xml
@@ -8,16 +8,6 @@
 <!-- CLM Namelist -->
 <!-- ========================================================================================  -->
 
-<entry id="finidat_interp_dest" 
-       type="char*256" 
-       category="datasets"
-       input_pathname="abs" 
-       group="clm_inparm" 
-       valid_values="" >
-If use_init_interp is set to .true., interpinic will be called to interpolate
-the file given by finidat, creating the output file specified by finidat_interp_dest.
-</entry>
-
 <entry id="finidat" 
        type="char*256" 
        category="datasets"
@@ -648,6 +638,12 @@ Switch deciding which nutrient model to use in FATES.
 (Only relevant if FATES is on)
 </entry>
 
+<entry id="use_fates_tree_damage" type="logical" category="physics"
+        group="clm_inparm" valid_values="" value=".false."> 
+Toggle to turn on the tree damage module in FATES
+(Only relevant if FATES is on)
+</entry>
+
 <entry id="fates_spitfire_mode" type="integer" category="physics"
         group="clm_inparm" valid_values="0,1,2,3,4,5" value=".false.">
 Turn on spitfire module to simulate fire by setting fates_spitfire_mode > 0.
@@ -2124,7 +2120,7 @@ Command line arguement for biogeochemistry mode for CLM4.5
         Nitrification/De-nitrification
         Methane model
         Vertically resolved Carbon
-   fates = FATES/ED ecosystem demography model with below ground BGC:
+   fates = FATES Functionally Assembled Terrestrial Ecosystem Simulator (ecosystem demography model with below ground BGC):
 </entry>
 
 <!-- ========================================================================================  -->