Merge pull request #1467 from jedwards4b/jedwards/neon_updates

improve ui for neon script

Externals.cfg

@@ -41,7 +41,7 @@ tag = cime6.0.11
 required = True
 
 [cmeps]
-tag = cmeps0.13.40
+tag = cmeps0.13.43
 protocol = git
 repo_url = https://github.com/ESCOMP/CMEPS.git
 local_path = components/cmeps
@@ -56,14 +56,14 @@ externals =  Externals_CDEPS.cfg
 required = True
 
 [cpl7]
-tag = cpl7.0.5
+tag = cpl7.0.7
 protocol = git
 repo_url = https://github.com/ESCOMP/CESM_CPL7andDataComps
 local_path = components/cpl7
 required = True
 
 [share]
-tag = share1.0.8
+tag = share1.0.10
 protocol = git
 repo_url = https://github.com/ESCOMP/CESM_share
 local_path = share

bld/CLMBuildNamelist.pm

@@ -4363,7 +4363,7 @@ sub check_input_files {
                 my $pathname = $nl->get_variable_value($group, $var);
                 # Need to strip the quotes
                 $pathname =~ s/['"]//g;
-
+                next if ($pathname =~ /UNSET$/);
                 if ($input_pathname_type eq 'abs') {
                     if ($inputdata_rootdir) {
                         if ( $pathname !~ /^\s*$/ ) {   # If pathname isn't blank or null

bld/namelist_files/namelist_defaults_ctsm.xml

@@ -1812,21 +1812,21 @@ lnd/clm2/surfdata_map/release-clm5.0.30/surfdata_ne0np4.CONUS.ne30x8_hist_78pfts
 
 <!-- mapping files for 1x1_brazil START added on Thu Jul 19 13:06:18 2012-->
 <map frm_hgrid="0.125x0.125" frm_lmask="nomask" to_hgrid="1x1_brazil" to_lmask="nomask"
->lnd/clm2/mappingdata/maps/1x1_brazil/map_0.125x0.125_nomask_to_1x1_brazil_nomask_aave_da_c200206.nc</map>
+>lnd/clm2/mappingdata/maps/1x1_brazil/map_0.125x0.125_nomask_to_1x1_brazil_nomask_aave_da_c211212.nc</map>
 <map frm_hgrid="0.5x0.5"    frm_lmask="nomask"  to_hgrid="1x1_brazil"   to_lmask="nomask"
->lnd/clm2/mappingdata/maps/1x1_brazil/map_0.5x0.5_nomask_to_1x1_brazil_nomask_aave_da_c200206.nc</map>
+>lnd/clm2/mappingdata/maps/1x1_brazil/map_0.5x0.5_nomask_to_1x1_brazil_nomask_aave_da_c211212.nc</map>
 <map frm_hgrid="0.25x0.25"  frm_lmask="nomask"  to_hgrid="1x1_brazil"   to_lmask="nomask"
->lnd/clm2/mappingdata/maps/1x1_brazil/map_0.25x0.25_nomask_to_1x1_brazil_nomask_aave_da_c200309.nc</map>
+>lnd/clm2/mappingdata/maps/1x1_brazil/map_0.25x0.25_nomask_to_1x1_brazil_nomask_aave_da_c211212.nc</map>
 <map frm_hgrid="3x3min"    frm_lmask="nomask"  to_hgrid="1x1_brazil"   to_lmask="nomask"
->lnd/clm2/mappingdata/maps/1x1_brazil/map_3x3min_nomask_to_1x1_brazil_nomask_aave_da_c200309.nc</map>
+>lnd/clm2/mappingdata/maps/1x1_brazil/map_3x3min_nomask_to_1x1_brazil_nomask_aave_da_c211212.nc</map>
 <map frm_hgrid="10x10min"    frm_lmask="nomask"  to_hgrid="1x1_brazil"   to_lmask="nomask"
->lnd/clm2/mappingdata/maps/1x1_brazil/map_10x10min_nomask_to_1x1_brazil_nomask_aave_da_c200206.nc</map>
+>lnd/clm2/mappingdata/maps/1x1_brazil/map_10x10min_nomask_to_1x1_brazil_nomask_aave_da_c211212.nc</map>
 <map frm_hgrid="5x5min"    frm_lmask="nomask"  to_hgrid="1x1_brazil"   to_lmask="nomask"
->lnd/clm2/mappingdata/maps/1x1_brazil/map_5x5min_nomask_to_1x1_brazil_nomask_aave_da_c200309.nc</map>
+>lnd/clm2/mappingdata/maps/1x1_brazil/map_5x5min_nomask_to_1x1_brazil_nomask_aave_da_c211212.nc</map>
 <map frm_hgrid="0.9x1.25"    frm_lmask="nomask"  to_hgrid="1x1_brazil"   to_lmask="nomask"
->lnd/clm2/mappingdata/maps/1x1_brazil/map_0.9x1.25_nomask_to_1x1_brazil_nomask_aave_da_c200206.nc</map>
+>lnd/clm2/mappingdata/maps/1x1_brazil/map_0.9x1.25_nomask_to_1x1_brazil_nomask_aave_da_c211212.nc</map>
 <map frm_hgrid="1km-merge-10min"    frm_lmask="HYDRO1K-merge-nomask"  to_hgrid="1x1_brazil"   to_lmask="nomask"
->lnd/clm2/mappingdata/maps/1x1_brazil/map_1km-merge-10min_HYDRO1K-merge-nomask_to_1x1_brazil_nomask_aave_da_c130403.nc</map>
+>lnd/clm2/mappingdata/maps/1x1_brazil/map_1km-merge-10min_HYDRO1K-merge-nomask_to_1x1_brazil_nomask_aave_da_c211212.nc</map>
 <!-- mapping files for 1x1_brazil END -->
 
 <!-- mapping files for 1x1_mexicocityMEX START added on Thu Jul 19 13:06:18 2012-->

bld/namelist_files/namelist_defaults_ctsm_tools.xml

@@ -95,7 +95,7 @@ attributes from the config_cache.xml file (with keys converted to upper-case).
 
 <!-- Single-point / Regional grids -->
 <scripgriddata hgrid="0.125nldas2"         lmask="nomask" >lnd/clm2/mappingdata/grids/SCRIPgrid_0.125nldas2_nomask_c190328.nc</scripgriddata>
-<scripgriddata hgrid="1x1_brazil"          lmask="nomask" >lnd/clm2/mappingdata/grids/SCRIPgrid_1x1pt_brazil_nomask_c110308.nc</scripgriddata>
+<scripgriddata hgrid="1x1_brazil"          lmask="nomask" >lnd/clm2/mappingdata/grids/SCRIPgrid_1x1pt_brazil_nomask_c20211211.nc</scripgriddata>
 <scripgriddata hgrid="1x1_mexicocityMEX"   lmask="nomask" >lnd/clm2/mappingdata/grids/SCRIPgrid_1x1pt_mexicocityMEX_nomask_c110308.nc</scripgriddata>
 <scripgriddata hgrid="1x1_numaIA"          lmask="nomask" >lnd/clm2/mappingdata/grids/SCRIPgrid_1x1pt_numaIA_nomask_c110308.nc</scripgriddata>
 <scripgriddata hgrid="1x1_smallvilleIA"    lmask="nomask" >lnd/clm2/mappingdata/grids/SCRIPgrid_1x1pt_smallvilleIA_nomask_c110308.nc</scripgriddata>

cime_config/usermods_dirs/NEON/GUAN/shell_commands

@@ -1,3 +1,4 @@
 ./xmlchange NEONSITE=GUAN
 ./xmlchange PTS_LON=293.13112
 ./xmlchange PTS_LAT=17.96882
+./xmlchange RUN_STARTDATE=2018-06-01

cime_config/usermods_dirs/NEON/LAJA/shell_commands

@@ -1,3 +1,4 @@
 ./xmlchange NEONSITE=LAJA
 ./xmlchange PTS_LON=292.92392
 ./xmlchange PTS_LAT=18.02184
+./xmlchange RUN_STARTDATE=2018-05-01

cime_config/usermods_dirs/NEON/SJER/shell_commands

@@ -1,3 +1,4 @@
 ./xmlchange NEONSITE=SJER
 ./xmlchange PTS_LON=240.267
 ./xmlchange PTS_LAT=37.107117
+./xmlchange RUN_STARTDATE=2018-09-01

cime_config/usermods_dirs/NEON/TEAK/shell_commands

@@ -1,3 +1,5 @@
 ./xmlchange NEONSITE=TEAK
 ./xmlchange PTS_LON=240.99424199999999
 ./xmlchange PTS_LAT=37.006472
+# This site is missing data for first half of 2018
+./xmlchange RUN_STARTDATE=2018-06-01

cime_config/usermods_dirs/NEON/YELL/shell_commands

@@ -1,3 +1,4 @@
 ./xmlchange NEONSITE=YELL
 ./xmlchange PTS_LON=249.45803999999998
 ./xmlchange PTS_LAT=44.95597
+./xmlchange RUN_STARTDATE=2018-08-01

cime_config/usermods_dirs/NEON/defaults/shell_commands

@@ -3,4 +3,4 @@
 ./xmlchange CLM_NML_USE_CASE=1850-2100_SSP3-7.0_transient
 ./xmlchange CCSM_CO2_PPMV=408.83
 ./xmlchange DATM_PRESAERO=SSP3-7.0
-./xmlchange DATM_YR_ALIGN=2018,DATM_YR_END=2019,DATM_YR_START=2018
+./xmlchange DATM_YR_ALIGN=2018,DATM_YR_END=2020,DATM_YR_START=2018

cime_config/usermods_dirs/NEON/defaults/user_nl_clm

@@ -19,17 +19,18 @@
 !----------------------------------------------------------------------------------
 
 flanduse_timeseries = ' '   ! This isn't needed for a non transient case, but will be once we start using transient compsets
-fsurdat = "$DIN_LOC_ROOT/lnd/clm2/surfdata_map/NEON/surfdata_hist_78pfts_CMIP6_simyr2000_${NEONSITE}_c210720.nc"
+fsurdat = "$DIN_LOC_ROOT/lnd/clm2/surfdata_map/NEON/surfdata_hist_78pfts_CMIP6_simyr2000_${NEONSITE}_c211102.nc"
 model_year_align_urbantv = 2018
 stream_year_first_urbantv = 2018
-stream_year_last_urbantv = 2019
+stream_year_last_urbantv = 2020
 stream_year_first_ndep = 2018
 model_year_align_ndep = 2018
-stream_year_last_ndep = 2019
+stream_year_last_ndep = 2020
 model_year_align_popdens = 2018
 stream_year_first_popdens = 2018
-stream_year_last_popdens = 2019
+stream_year_last_popdens = 2020
 stream_fldfilename_lightng = '$DIN_LOC_ROOT/atm/datm7/NASA_LIS/clmforc.Li_2016_climo1995-2013.360x720.lnfm_Total_NEONarea_c210625.nc'
+stream_fldfilename_ndep = '$DIN_LOC_ROOT/lnd/clm2/ndepdata/fndep_clm_f09_g17.CMIP6-SSP3-7.0-WACCM_2018-2030_monthly_c210826.nc'
 ! h1 output stream 
 hist_fincl2 = 'AR','ELAI','FCEV','FCTR','FGEV','FIRA','FSA','FSH','GPP','H2OSOI',
              'HR','SNOW_DEPTH','TBOT','TSOI','SOILC_vr','FV','NET_NMIN_vr'

cime_config/usermods_dirs/NEON/defaults/user_nl_datm_streams

@@ -0,0 +1,6 @@
+presaero.SSP3-7.0:datafiles = $DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/aero/aerodep_clm_SSP370_b.e21.BWSSP370cmip6.f09_g17.CMIP6-SSP3-7.0-WACCM.001_2018-2030_monthly_0.9x1.25_c210826.nc
+presaero.SSP3-7.0:year_first=2018
+presaero.SSP3-7.0:year_last=2030
+presaero.SSP3-7.0:year_align=2018
+presaero.SSP3-7.0:dtlimit=30
+

doc/ChangeLog

@@ -1,4 +1,157 @@
 ===============================================================
+Tag name: ctsm5.1.dev067
+Originator(s): jedwards4b/negin513/erik (Erik Kluzek,UCAR/TSS,303-497-1326)
+Date: Mon Dec 13 00:50:01 MST 2021
+One-line Summary: NEON UI update, externals updates, small miscellanouse fixes
+
+Purpose and description of changes
+----------------------------------
+
+Redo options list to remove positional arguments that were difficult to input correctly.
+Transient runs now use run_type startup and get finidat from s3 server unless --run-from-postad option is used (or finidat is not
+available). Use mpi instead of mpi-serial, this mod was recommended for container use. Add a new script neon_finidat_upload which
+allows authorized users to upload finidat files to the s3 server.
+
+This includes the following changes to the script for updating the surface dataset at neon sites using available neon data (i.e.
+modify_singlept_site_neon.py) to address ctsm/issues #1353, #1429, and neon/issue #44:
+
+Update Organic calculation to use the following equation based on discussions in
+https://github.com/ESCOMP/CTSM/pull/1375/files#r669590971 :
+ORGANIC = estimatedOC * bulkDensity / 0.58
+
+Because estimatedOC is rounded to the nearest integer in neon data, it is sometimes bigger than carbonTot. Therefore, in cases where
+estimatedOC > carbonTot, we use carbonTot instead of estimatedOC in the above equation.
+
+Previously, we had missing data on neon files for some neon soil layers (see Modified NEON surface datasets have errors  #1429
+(comment)). Therefore, it caused some missing values in the updated dataset. Here, we interpolate to fill in the missing data using
+different interpolation techniques. Finally, we chose to use linear interpolation to fill in missing data.
+
+This includes the scripts for modification of the surface dataset for neon sites to address #1429.
+Specifically, the following has been addressed in this PR:
+
+Update the calculation of ORGANIC to use the new field (CaCO3) from NEON data.
+For this calculation if CaCO3 data is available, we first calculate inorganic carbon by:
+inorganic carbon = (caco3 /100.0869)*12.0107
+Next, we calculate organic carbon by subtracting inorganic carbon from the total carbon:
+[organic carbon = carbon_tot - inorganic carbon]
+If the CaCO3 is not available then the code uses carbonTot and estimatedOC by NEON.
+
+Discussed here (Modified NEON surface datasets have errors  #1429 (comment))
+For the Ag sites (KONA and STER), it changes the PCT_NATVEG, PCT_CROP, and PCT_NAT_PFT to avoid the error that we previously had in
+spin-up: surfrd_veg_all ERROR: sum of wt_nat_patch not 1.00000000000000 at nl= 1 sum is: 0.000000000000000E+000
+
+Discussed here (Modified NEON surface datasets have errors  #1429 (comment))
+There was a typo previously in the NEON data for ABBY sites caused by mix of sample measurements. Please note that this was updated
+by hand once data was downloaded from NEON site.
+
+With recent versions of CIME, the LILAC build with a user-defined machine was broken for a couple of reasons. This fixes it.
+
+Fix mksurfdata_map for 1x1_brazil. Get tools testing working again. Increase skip_steps by 1, which is needed for a change in CAM
+where balance checks need to occur after the radiation update now rather than before. glob changed for bsd_glob in perl MkDepends
+for mksurfdata_map.
+
+
+Significant changes to scientifically-supported configurations
+--------------------------------------------------------------
+
+Does this tag change answers significantly for any of the following physics configurations?
+(Details of any changes will be given in the "Answer changes" section below.)
+
+    [Put an [X] in the box for any configuration with significant answer changes.]
+
+[ ] clm5_1
+
+[ ] clm5_0
+
+[ ] ctsm5_0-nwp
+
+[ ] clm4_5
+
+
+Bugs fixed or introduced
+------------------------
+
+Issues fixed (include CTSM Issue #):
+  Fixes #1563 increase skip_steps for balance checks by one to permit new CAM physics re-ordering
+  Fixes #1550 In perl code replace glob with bsd_glob
+  Fixes #1574 Trouble with 1x1_brazil for mksurfdata_map because of negative longitude in SCRIP grid file
+  Fixes #1429 Modified NEON surface datasets have errors
+  Fixes #1353 Modify NEON surface data
+  Fixes #1492 Need to update LILAC build process to use cmake macros instead of config_compilers.xml
+
+Known bugs found since the previous tag (include issue #):
+  #1575 -- Build problem for mksurfdata tools testers
+
+Notes of particular relevance for users
+---------------------------------------
+
+Changes made to namelist defaults (e.g., changed parameter values): New 1x1_brazil SCRIP grid file and maps
+   Some NEON namelist settings changed. Last year is now 2020
+
+Changes to the datasets (e.g., parameter, surface or initial files): New NEON surface datasets
+
+Notes of particular relevance for developers:
+---------------------------------------------
+
+Caveats for developers (e.g., code that is duplicated that requires double maintenance):
+   pyproject.toml file added to configure for black python formatter in python directory
+
+Changes to tests or testing:
+   Got tools testing working again.
+
+
+Testing summary:   regular tools
+----------------
+ [PASS means all tests PASS; OK means tests PASS other than expected fails.]
+
+  build-namelist tests (if CLMBuildNamelist.pm has changed):
+
+    cheyenne - PASS (47 compare tests fail because of changes to NEON sites)
+
+  tools-tests (test/tools) (if tools have been changed):
+
+    cheyenne - OK (1x1_brazil mksurfdata changes, run_neon and modify_subset fail as expected)
+
+  python testing (if python code has changed; see instructions in python/README.md; document testing done):
+
+    cheyenne - OK (new black checks do NOT pass as expected)
+
+  regular tests (aux_clm: https://github.com/ESCOMP/CTSM/wiki/System-Testing-Guide#pre-merge-system-testing):
+
+    cheyenne ---- PASS
+    izumi ------- OK
+
+If the tag used for baseline comparisons was NOT the previous tag, note that here:
+
+
+Answer changes
+--------------
+
+Changes answers relative to baseline: No bit-for-bit (other than NEON tests because of updated namelists and surface dataset)
+
+  Summarize any changes to answers, i.e.,
+    - what code configurations: Only NEON sites
+    - what platforms/compilers: all
+    - nature of change: new surface datasets and settings
+
+Other details
+-------------
+List any externals directories updated (cime, rtm, mosart, cism, fates, etc.):
+    Update most externals to version in cesm2_3_alpha07c
+    cmeps to cmeps0.13.43 (version with channel depths)
+    cpl7  to cpl7.0.7
+    share to share1.0.10
+
+Pull Requests that document the changes (include PR ids):
+(https://github.com/ESCOMP/ctsm/pull)
+
+   PR #1467 -- Improve UI for NEON script
+   PR #1474 -- Script for modifying neon surface dataset -- updated  (negin513)
+   PR #1539 -- Neon modify surfurface dataset (negin513)
+   PR #1571 -- Fix LILAC build with user-defined machine with latest CIME (billsacks)
+
+===============================================================
+===============================================================
 Tag name: ctsm5.1.dev066
 Originator(s): rgknox (Ryan Knox,,,)
 Date: Sat Dec  4 01:58:42 MST 2021

doc/ChangeSum

@@ -1,5 +1,6 @@
 Tag                   Who      Date  Summary
 ============================================================================================================================
+    ctsm5.1.dev067 jedwards 12/13/2021 NEON UI update, externals updates, small miscellanouse fixes
     ctsm5.1.dev066   rgknox 12/04/2021 API change with FATES to enable running means inside fates, includes passing in of model timestep
     ctsm5.1.dev065 glemieux 12/02/2021 Refactor static fire data input by moving variables into fire_base_type from cnveg_state_type
     ctsm5.1.dev064  afoster 11/29/2021 Updates to facilitate FATES history variable overhaul

doc/source/lilac/obtaining-building-and-running/obtaining-and-building-ctsm.rst

@@ -41,7 +41,7 @@ On a machine that has *not* been ported to CIME, you will need to provide some a
 information. Run ``./lilac/build_ctsm -h`` for details, but the basic command will look
 like this::
 
-  ./lilac/build_ctsm ~/ctsm_build_dir --os Darwin --compiler gnu --netcdf-path /usr/local --esmf-lib-path /Users/sacks/ESMF/esmf8.0.0/lib/libO/Darwin.gfortranclang.64.mpich3.default --max-mpitasks-per-node 4 --no-pnetcdf
+  ./lilac/build_ctsm ~/ctsm_build_dir --os Darwin --compiler gnu --netcdf-path /usr/local --esmf-mkfile-path /Users/sacks/ESMF/esmf8.0.0/lib/libO/Darwin.gfortranclang.64.mpich3.default/esmf.mk --max-mpitasks-per-node 4 --no-pnetcdf
 
 In both cases, you will then need to include the necessary information in the include and
 link lines of the atmosphere model's build. For a Makefile-based build, this can be done
@@ -205,7 +205,7 @@ above<building-ctsm-and-lilac-prerequisites>`.
 
 The minimal amount of information needed is given by the following::
 
-  ./lilac/build_ctsm /PATH/TO/CTSM/BUILD --os OS --compiler COMPILER --netcdf-path NETCDF_PATH --esmf-lib-path ESMF_LIB_PATH --max-mpitasks-per-node MAX_MPITASKS_PER_NODE --pnetcdf-path PNETCDF_PATH
+  ./lilac/build_ctsm /PATH/TO/CTSM/BUILD --os OS --compiler COMPILER --netcdf-path NETCDF_PATH --esmf-mkfile-path ESMF_MKFILE_PATH --max-mpitasks-per-node MAX_MPITASKS_PER_NODE --pnetcdf-path PNETCDF_PATH
 
 where you should fill in the capitalized arguments with appropriate values for your
 machine. Run ``./lilac/build_ctsm -h`` for details on these arguments, as well as documentation
@@ -229,17 +229,16 @@ model performance.
 
 Example usage for a Mac (a simple case) is::
 
-  ./lilac/build_ctsm ~/ctsm_build_dir --os Darwin --compiler gnu --netcdf-path /usr/local --esmf-lib-path /Users/sacks/ESMF/esmf8.0.0/lib/libO/Darwin.gfortranclang.64.mpich3.default --max-mpitasks-per-node 4 --no-pnetcdf
+  ./lilac/build_ctsm ~/ctsm_build_dir --os Darwin --compiler gnu --netcdf-path /usr/local --esmf-mkfile-path /Users/sacks/ESMF/esmf8.0.0/lib/libO/Darwin.gfortranclang.64.mpich3.default/esmf.mk --max-mpitasks-per-node 4 --no-pnetcdf
 
 Example usage for NCAR's ``cheyenne`` machine (a more complex case) is::
 
   module purge
-  module load ncarenv/1.3 intel/19.0.5 esmf_libs mkl
-  module use /glade/work/himanshu/PROGS/modulefiles/esmfpkgs/intel/19.0.5
-  module load esmf-8.1.0b14-ncdfio-mpt-O mpt/2.21 netcdf/4.7.3 pnetcdf/1.12.1 ncarcompilers/0.5.0
-  module load python
+  module load ncarenv/1.3 python/3.7.9 cmake intel/19.1.1 esmf_libs mkl
+  module use /glade/p/cesmdata/cseg/PROGS/modulefiles/esmfpkgs/intel/19.1.1/
+  module load esmf-8.2.0b23-ncdfio-mpt-O mpt/2.22 netcdf-mpi/4.8.0 pnetcdf/1.12.2 ncarcompilers/0.5.0
 
-  ./lilac/build_ctsm /glade/scratch/$USER/ctsm_build_dir --os linux --compiler intel --netcdf-path '$ENV{NETCDF}' --pio-filesystem-hints gpfs --pnetcdf-path '$ENV{PNETCDF}' --esmf-lib-path '$ENV{ESMF_LIBDIR}' --max-mpitasks-per-node 36 --extra-cflags '-xCORE_AVX2 -no-fma' --extra-fflags '-xCORE_AVX2 -no-fma'
+  ./lilac/build_ctsm /glade/scratch/$USER/ctsm_build_dir --os linux --compiler intel --netcdf-path '$ENV{NETCDF}' --pio-filesystem-hints gpfs --pnetcdf-path '$ENV{PNETCDF}' --esmf-mkfile-path '$ENV{ESMFMKFILE}' --max-mpitasks-per-node 36 --extra-cflags '-xCORE_AVX2 -no-fma' --extra-fflags '-xCORE_AVX2 -no-fma'
 
 (It's better to use the :ref:`alternative process for a CIME-supported
 machine<building-on-a-cime-supported-machine>` in this case, but the above illustrates