diff --git a/Externals.cfg b/Externals.cfg
index d3ee5df893..10963cfd20 100644
--- a/Externals.cfg
+++ b/Externals.cfg
@@ -41,7 +41,7 @@ tag = cime6.0.11
required = True
[cmeps]
-tag = cmeps0.13.40
+tag = cmeps0.13.43
protocol = git
repo_url = https://github.com/ESCOMP/CMEPS.git
local_path = components/cmeps
@@ -56,14 +56,14 @@ externals = Externals_CDEPS.cfg
required = True
[cpl7]
-tag = cpl7.0.5
+tag = cpl7.0.7
protocol = git
repo_url = https://github.com/ESCOMP/CESM_CPL7andDataComps
local_path = components/cpl7
required = True
[share]
-tag = share1.0.8
+tag = share1.0.10
protocol = git
repo_url = https://github.com/ESCOMP/CESM_share
local_path = share
diff --git a/bld/CLMBuildNamelist.pm b/bld/CLMBuildNamelist.pm
index d9417ca8b7..f105491d18 100755
--- a/bld/CLMBuildNamelist.pm
+++ b/bld/CLMBuildNamelist.pm
@@ -4363,7 +4363,7 @@ sub check_input_files {
my $pathname = $nl->get_variable_value($group, $var);
# Need to strip the quotes
$pathname =~ s/['"]//g;
-
+ next if ($pathname =~ /UNSET$/);
if ($input_pathname_type eq 'abs') {
if ($inputdata_rootdir) {
if ( $pathname !~ /^\s*$/ ) { # If pathname isn't blank or null
diff --git a/bld/namelist_files/namelist_defaults_ctsm.xml b/bld/namelist_files/namelist_defaults_ctsm.xml
index cd0da15c89..a61d66360b 100644
--- a/bld/namelist_files/namelist_defaults_ctsm.xml
+++ b/bld/namelist_files/namelist_defaults_ctsm.xml
@@ -1812,21 +1812,21 @@ lnd/clm2/surfdata_map/release-clm5.0.30/surfdata_ne0np4.CONUS.ne30x8_hist_78pfts
+>lnd/clm2/mappingdata/maps/1x1_brazil/map_0.125x0.125_nomask_to_1x1_brazil_nomask_aave_da_c211212.nc
+>lnd/clm2/mappingdata/maps/1x1_brazil/map_0.5x0.5_nomask_to_1x1_brazil_nomask_aave_da_c211212.nc
+>lnd/clm2/mappingdata/maps/1x1_brazil/map_0.25x0.25_nomask_to_1x1_brazil_nomask_aave_da_c211212.nc
+>lnd/clm2/mappingdata/maps/1x1_brazil/map_3x3min_nomask_to_1x1_brazil_nomask_aave_da_c211212.nc
+>lnd/clm2/mappingdata/maps/1x1_brazil/map_10x10min_nomask_to_1x1_brazil_nomask_aave_da_c211212.nc
+>lnd/clm2/mappingdata/maps/1x1_brazil/map_5x5min_nomask_to_1x1_brazil_nomask_aave_da_c211212.nc
+>lnd/clm2/mappingdata/maps/1x1_brazil/map_0.9x1.25_nomask_to_1x1_brazil_nomask_aave_da_c211212.nc
+>lnd/clm2/mappingdata/maps/1x1_brazil/map_1km-merge-10min_HYDRO1K-merge-nomask_to_1x1_brazil_nomask_aave_da_c211212.nc
diff --git a/bld/namelist_files/namelist_defaults_ctsm_tools.xml b/bld/namelist_files/namelist_defaults_ctsm_tools.xml
index 1166e80c9c..ff309c6fc9 100644
--- a/bld/namelist_files/namelist_defaults_ctsm_tools.xml
+++ b/bld/namelist_files/namelist_defaults_ctsm_tools.xml
@@ -95,7 +95,7 @@ attributes from the config_cache.xml file (with keys converted to upper-case).
lnd/clm2/mappingdata/grids/SCRIPgrid_0.125nldas2_nomask_c190328.nc
-lnd/clm2/mappingdata/grids/SCRIPgrid_1x1pt_brazil_nomask_c110308.nc
+lnd/clm2/mappingdata/grids/SCRIPgrid_1x1pt_brazil_nomask_c20211211.nc
lnd/clm2/mappingdata/grids/SCRIPgrid_1x1pt_mexicocityMEX_nomask_c110308.nc
lnd/clm2/mappingdata/grids/SCRIPgrid_1x1pt_numaIA_nomask_c110308.nc
lnd/clm2/mappingdata/grids/SCRIPgrid_1x1pt_smallvilleIA_nomask_c110308.nc
diff --git a/cime_config/usermods_dirs/NEON/GUAN/shell_commands b/cime_config/usermods_dirs/NEON/GUAN/shell_commands
index 4d750b77f8..b2d1b32dbf 100644
--- a/cime_config/usermods_dirs/NEON/GUAN/shell_commands
+++ b/cime_config/usermods_dirs/NEON/GUAN/shell_commands
@@ -1,3 +1,4 @@
./xmlchange NEONSITE=GUAN
./xmlchange PTS_LON=293.13112
./xmlchange PTS_LAT=17.96882
+./xmlchange RUN_STARTDATE=2018-06-01
\ No newline at end of file
diff --git a/cime_config/usermods_dirs/NEON/LAJA/shell_commands b/cime_config/usermods_dirs/NEON/LAJA/shell_commands
index 330690c330..36f01cff81 100644
--- a/cime_config/usermods_dirs/NEON/LAJA/shell_commands
+++ b/cime_config/usermods_dirs/NEON/LAJA/shell_commands
@@ -1,3 +1,4 @@
./xmlchange NEONSITE=LAJA
./xmlchange PTS_LON=292.92392
./xmlchange PTS_LAT=18.02184
+./xmlchange RUN_STARTDATE=2018-05-01
\ No newline at end of file
diff --git a/cime_config/usermods_dirs/NEON/SJER/shell_commands b/cime_config/usermods_dirs/NEON/SJER/shell_commands
index 45de246989..3683443ec0 100644
--- a/cime_config/usermods_dirs/NEON/SJER/shell_commands
+++ b/cime_config/usermods_dirs/NEON/SJER/shell_commands
@@ -1,3 +1,4 @@
./xmlchange NEONSITE=SJER
./xmlchange PTS_LON=240.267
./xmlchange PTS_LAT=37.107117
+./xmlchange RUN_STARTDATE=2018-09-01
\ No newline at end of file
diff --git a/cime_config/usermods_dirs/NEON/TEAK/shell_commands b/cime_config/usermods_dirs/NEON/TEAK/shell_commands
index 53ebedc664..f3a9fd75ef 100644
--- a/cime_config/usermods_dirs/NEON/TEAK/shell_commands
+++ b/cime_config/usermods_dirs/NEON/TEAK/shell_commands
@@ -1,3 +1,5 @@
./xmlchange NEONSITE=TEAK
./xmlchange PTS_LON=240.99424199999999
./xmlchange PTS_LAT=37.006472
+# This site is missing data for first half of 2018
+./xmlchange RUN_STARTDATE=2018-06-01
\ No newline at end of file
diff --git a/cime_config/usermods_dirs/NEON/YELL/shell_commands b/cime_config/usermods_dirs/NEON/YELL/shell_commands
index a40ef81477..3924dff420 100644
--- a/cime_config/usermods_dirs/NEON/YELL/shell_commands
+++ b/cime_config/usermods_dirs/NEON/YELL/shell_commands
@@ -1,3 +1,4 @@
./xmlchange NEONSITE=YELL
./xmlchange PTS_LON=249.45803999999998
./xmlchange PTS_LAT=44.95597
+./xmlchange RUN_STARTDATE=2018-08-01
\ No newline at end of file
diff --git a/cime_config/usermods_dirs/NEON/defaults/shell_commands b/cime_config/usermods_dirs/NEON/defaults/shell_commands
index 2ebe1b4f86..f82278e4b5 100644
--- a/cime_config/usermods_dirs/NEON/defaults/shell_commands
+++ b/cime_config/usermods_dirs/NEON/defaults/shell_commands
@@ -3,4 +3,4 @@
./xmlchange CLM_NML_USE_CASE=1850-2100_SSP3-7.0_transient
./xmlchange CCSM_CO2_PPMV=408.83
./xmlchange DATM_PRESAERO=SSP3-7.0
-./xmlchange DATM_YR_ALIGN=2018,DATM_YR_END=2019,DATM_YR_START=2018
+./xmlchange DATM_YR_ALIGN=2018,DATM_YR_END=2020,DATM_YR_START=2018
diff --git a/cime_config/usermods_dirs/NEON/defaults/user_nl_clm b/cime_config/usermods_dirs/NEON/defaults/user_nl_clm
index 742cd6f65e..eff648d4e5 100644
--- a/cime_config/usermods_dirs/NEON/defaults/user_nl_clm
+++ b/cime_config/usermods_dirs/NEON/defaults/user_nl_clm
@@ -19,17 +19,18 @@
!----------------------------------------------------------------------------------
flanduse_timeseries = ' ' ! This isn't needed for a non transient case, but will be once we start using transient compsets
-fsurdat = "$DIN_LOC_ROOT/lnd/clm2/surfdata_map/NEON/surfdata_hist_78pfts_CMIP6_simyr2000_${NEONSITE}_c210720.nc"
+fsurdat = "$DIN_LOC_ROOT/lnd/clm2/surfdata_map/NEON/surfdata_hist_78pfts_CMIP6_simyr2000_${NEONSITE}_c211102.nc"
model_year_align_urbantv = 2018
stream_year_first_urbantv = 2018
-stream_year_last_urbantv = 2019
+stream_year_last_urbantv = 2020
stream_year_first_ndep = 2018
model_year_align_ndep = 2018
-stream_year_last_ndep = 2019
+stream_year_last_ndep = 2020
model_year_align_popdens = 2018
stream_year_first_popdens = 2018
-stream_year_last_popdens = 2019
+stream_year_last_popdens = 2020
stream_fldfilename_lightng = '$DIN_LOC_ROOT/atm/datm7/NASA_LIS/clmforc.Li_2016_climo1995-2013.360x720.lnfm_Total_NEONarea_c210625.nc'
+stream_fldfilename_ndep = '$DIN_LOC_ROOT/lnd/clm2/ndepdata/fndep_clm_f09_g17.CMIP6-SSP3-7.0-WACCM_2018-2030_monthly_c210826.nc'
! h1 output stream
hist_fincl2 = 'AR','ELAI','FCEV','FCTR','FGEV','FIRA','FSA','FSH','GPP','H2OSOI',
'HR','SNOW_DEPTH','TBOT','TSOI','SOILC_vr','FV','NET_NMIN_vr'
diff --git a/cime_config/usermods_dirs/NEON/defaults/user_nl_datm_streams b/cime_config/usermods_dirs/NEON/defaults/user_nl_datm_streams
new file mode 100644
index 0000000000..41ddbfa611
--- /dev/null
+++ b/cime_config/usermods_dirs/NEON/defaults/user_nl_datm_streams
@@ -0,0 +1,6 @@
+presaero.SSP3-7.0:datafiles = $DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/aero/aerodep_clm_SSP370_b.e21.BWSSP370cmip6.f09_g17.CMIP6-SSP3-7.0-WACCM.001_2018-2030_monthly_0.9x1.25_c210826.nc
+presaero.SSP3-7.0:year_first=2018
+presaero.SSP3-7.0:year_last=2030
+presaero.SSP3-7.0:year_align=2018
+presaero.SSP3-7.0:dtlimit=30
+
diff --git a/doc/ChangeLog b/doc/ChangeLog
index 4a8050bfe1..8c17383973 100644
--- a/doc/ChangeLog
+++ b/doc/ChangeLog
@@ -1,4 +1,157 @@
===============================================================
+Tag name: ctsm5.1.dev067
+Originator(s): jedwards4b/negin513/erik (Erik Kluzek,UCAR/TSS,303-497-1326)
+Date: Mon Dec 13 00:50:01 MST 2021
+One-line Summary: NEON UI update, externals updates, small miscellanouse fixes
+
+Purpose and description of changes
+----------------------------------
+
+Redo options list to remove positional arguments that were difficult to input correctly.
+Transient runs now use run_type startup and get finidat from s3 server unless --run-from-postad option is used (or finidat is not
+available). Use mpi instead of mpi-serial, this mod was recommended for container use. Add a new script neon_finidat_upload which
+allows authorized users to upload finidat files to the s3 server.
+
+This includes the following changes to the script for updating the surface dataset at neon sites using available neon data (i.e.
+modify_singlept_site_neon.py) to address ctsm/issues #1353, #1429, and neon/issue #44:
+
+Update Organic calculation to use the following equation based on discussions in
+https://github.com/ESCOMP/CTSM/pull/1375/files#r669590971 :
+ORGANIC = estimatedOC * bulkDensity / 0.58
+
+Because estimatedOC is rounded to the nearest integer in neon data, it is sometimes bigger than carbonTot. Therefore, in cases where
+estimatedOC > carbonTot, we use carbonTot instead of estimatedOC in the above equation.
+
+Previously, we had missing data on neon files for some neon soil layers (see Modified NEON surface datasets have errors #1429
+(comment)). Therefore, it caused some missing values in the updated dataset. Here, we interpolate to fill in the missing data using
+different interpolation techniques. Finally, we chose to use linear interpolation to fill in missing data.
+
+This includes the scripts for modification of the surface dataset for neon sites to address #1429.
+Specifically, the following has been addressed in this PR:
+
+Update the calculation of ORGANIC to use the new field (CaCO3) from NEON data.
+For this calculation if CaCO3 data is available, we first calculate inorganic carbon by:
+inorganic carbon = (caco3 /100.0869)*12.0107
+Next, we calculate organic carbon by subtracting inorganic carbon from the total carbon:
+[organic carbon = carbon_tot - inorganic carbon]
+If the CaCO3 is not available then the code uses carbonTot and estimatedOC by NEON.
+
+Discussed here (Modified NEON surface datasets have errors #1429 (comment))
+For the Ag sites (KONA and STER), it changes the PCT_NATVEG, PCT_CROP, and PCT_NAT_PFT to avoid the error that we previously had in
+spin-up: surfrd_veg_all ERROR: sum of wt_nat_patch not 1.00000000000000 at nl= 1 sum is: 0.000000000000000E+000
+
+Discussed here (Modified NEON surface datasets have errors #1429 (comment))
+There was a typo previously in the NEON data for ABBY sites caused by mix of sample measurements. Please note that this was updated
+by hand once data was downloaded from NEON site.
+
+With recent versions of CIME, the LILAC build with a user-defined machine was broken for a couple of reasons. This fixes it.
+
+Fix mksurfdata_map for 1x1_brazil. Get tools testing working again. Increase skip_steps by 1, which is needed for a change in CAM
+where balance checks need to occur after the radiation update now rather than before. glob changed for bsd_glob in perl MkDepends
+for mksurfdata_map.
+
+
+Significant changes to scientifically-supported configurations
+--------------------------------------------------------------
+
+Does this tag change answers significantly for any of the following physics configurations?
+(Details of any changes will be given in the "Answer changes" section below.)
+
+ [Put an [X] in the box for any configuration with significant answer changes.]
+
+[ ] clm5_1
+
+[ ] clm5_0
+
+[ ] ctsm5_0-nwp
+
+[ ] clm4_5
+
+
+Bugs fixed or introduced
+------------------------
+
+Issues fixed (include CTSM Issue #):
+ Fixes #1563 increase skip_steps for balance checks by one to permit new CAM physics re-ordering
+ Fixes #1550 In perl code replace glob with bsd_glob
+ Fixes #1574 Trouble with 1x1_brazil for mksurfdata_map because of negative longitude in SCRIP grid file
+ Fixes #1429 Modified NEON surface datasets have errors
+ Fixes #1353 Modify NEON surface data
+ Fixes #1492 Need to update LILAC build process to use cmake macros instead of config_compilers.xml
+
+Known bugs found since the previous tag (include issue #):
+ #1575 -- Build problem for mksurfdata tools testers
+
+Notes of particular relevance for users
+---------------------------------------
+
+Changes made to namelist defaults (e.g., changed parameter values): New 1x1_brazil SCRIP grid file and maps
+ Some NEON namelist settings changed. Last year is now 2020
+
+Changes to the datasets (e.g., parameter, surface or initial files): New NEON surface datasets
+
+Notes of particular relevance for developers:
+---------------------------------------------
+
+Caveats for developers (e.g., code that is duplicated that requires double maintenance):
+ pyproject.toml file added to configure for black python formatter in python directory
+
+Changes to tests or testing:
+ Got tools testing working again.
+
+
+Testing summary: regular tools
+----------------
+ [PASS means all tests PASS; OK means tests PASS other than expected fails.]
+
+ build-namelist tests (if CLMBuildNamelist.pm has changed):
+
+ cheyenne - PASS (47 compare tests fail because of changes to NEON sites)
+
+ tools-tests (test/tools) (if tools have been changed):
+
+ cheyenne - OK (1x1_brazil mksurfdata changes, run_neon and modify_subset fail as expected)
+
+ python testing (if python code has changed; see instructions in python/README.md; document testing done):
+
+ cheyenne - OK (new black checks do NOT pass as expected)
+
+ regular tests (aux_clm: https://github.com/ESCOMP/CTSM/wiki/System-Testing-Guide#pre-merge-system-testing):
+
+ cheyenne ---- PASS
+ izumi ------- OK
+
+If the tag used for baseline comparisons was NOT the previous tag, note that here:
+
+
+Answer changes
+--------------
+
+Changes answers relative to baseline: No bit-for-bit (other than NEON tests because of updated namelists and surface dataset)
+
+ Summarize any changes to answers, i.e.,
+ - what code configurations: Only NEON sites
+ - what platforms/compilers: all
+ - nature of change: new surface datasets and settings
+
+Other details
+-------------
+List any externals directories updated (cime, rtm, mosart, cism, fates, etc.):
+ Update most externals to version in cesm2_3_alpha07c
+ cmeps to cmeps0.13.43 (version with channel depths)
+ cpl7 to cpl7.0.7
+ share to share1.0.10
+
+Pull Requests that document the changes (include PR ids):
+(https://github.com/ESCOMP/ctsm/pull)
+
+ PR #1467 -- Improve UI for NEON script
+ PR #1474 -- Script for modifying neon surface dataset -- updated (negin513)
+ PR #1539 -- Neon modify surfurface dataset (negin513)
+ PR #1571 -- Fix LILAC build with user-defined machine with latest CIME (billsacks)
+
+===============================================================
+===============================================================
Tag name: ctsm5.1.dev066
Originator(s): rgknox (Ryan Knox,,,)
Date: Sat Dec 4 01:58:42 MST 2021
diff --git a/doc/ChangeSum b/doc/ChangeSum
index 73281e5a9c..7742b1f18c 100644
--- a/doc/ChangeSum
+++ b/doc/ChangeSum
@@ -1,5 +1,6 @@
Tag Who Date Summary
============================================================================================================================
+ ctsm5.1.dev067 jedwards 12/13/2021 NEON UI update, externals updates, small miscellanouse fixes
ctsm5.1.dev066 rgknox 12/04/2021 API change with FATES to enable running means inside fates, includes passing in of model timestep
ctsm5.1.dev065 glemieux 12/02/2021 Refactor static fire data input by moving variables into fire_base_type from cnveg_state_type
ctsm5.1.dev064 afoster 11/29/2021 Updates to facilitate FATES history variable overhaul
diff --git a/doc/source/lilac/obtaining-building-and-running/obtaining-and-building-ctsm.rst b/doc/source/lilac/obtaining-building-and-running/obtaining-and-building-ctsm.rst
index a51dcfcf25..99cb908d28 100644
--- a/doc/source/lilac/obtaining-building-and-running/obtaining-and-building-ctsm.rst
+++ b/doc/source/lilac/obtaining-building-and-running/obtaining-and-building-ctsm.rst
@@ -41,7 +41,7 @@ On a machine that has *not* been ported to CIME, you will need to provide some a
information. Run ``./lilac/build_ctsm -h`` for details, but the basic command will look
like this::
- ./lilac/build_ctsm ~/ctsm_build_dir --os Darwin --compiler gnu --netcdf-path /usr/local --esmf-lib-path /Users/sacks/ESMF/esmf8.0.0/lib/libO/Darwin.gfortranclang.64.mpich3.default --max-mpitasks-per-node 4 --no-pnetcdf
+ ./lilac/build_ctsm ~/ctsm_build_dir --os Darwin --compiler gnu --netcdf-path /usr/local --esmf-mkfile-path /Users/sacks/ESMF/esmf8.0.0/lib/libO/Darwin.gfortranclang.64.mpich3.default/esmf.mk --max-mpitasks-per-node 4 --no-pnetcdf
In both cases, you will then need to include the necessary information in the include and
link lines of the atmosphere model's build. For a Makefile-based build, this can be done
@@ -205,7 +205,7 @@ above`.
The minimal amount of information needed is given by the following::
- ./lilac/build_ctsm /PATH/TO/CTSM/BUILD --os OS --compiler COMPILER --netcdf-path NETCDF_PATH --esmf-lib-path ESMF_LIB_PATH --max-mpitasks-per-node MAX_MPITASKS_PER_NODE --pnetcdf-path PNETCDF_PATH
+ ./lilac/build_ctsm /PATH/TO/CTSM/BUILD --os OS --compiler COMPILER --netcdf-path NETCDF_PATH --esmf-mkfile-path ESMF_MKFILE_PATH --max-mpitasks-per-node MAX_MPITASKS_PER_NODE --pnetcdf-path PNETCDF_PATH
where you should fill in the capitalized arguments with appropriate values for your
machine. Run ``./lilac/build_ctsm -h`` for details on these arguments, as well as documentation
@@ -229,17 +229,16 @@ model performance.
Example usage for a Mac (a simple case) is::
- ./lilac/build_ctsm ~/ctsm_build_dir --os Darwin --compiler gnu --netcdf-path /usr/local --esmf-lib-path /Users/sacks/ESMF/esmf8.0.0/lib/libO/Darwin.gfortranclang.64.mpich3.default --max-mpitasks-per-node 4 --no-pnetcdf
+ ./lilac/build_ctsm ~/ctsm_build_dir --os Darwin --compiler gnu --netcdf-path /usr/local --esmf-mkfile-path /Users/sacks/ESMF/esmf8.0.0/lib/libO/Darwin.gfortranclang.64.mpich3.default/esmf.mk --max-mpitasks-per-node 4 --no-pnetcdf
Example usage for NCAR's ``cheyenne`` machine (a more complex case) is::
module purge
- module load ncarenv/1.3 intel/19.0.5 esmf_libs mkl
- module use /glade/work/himanshu/PROGS/modulefiles/esmfpkgs/intel/19.0.5
- module load esmf-8.1.0b14-ncdfio-mpt-O mpt/2.21 netcdf/4.7.3 pnetcdf/1.12.1 ncarcompilers/0.5.0
- module load python
+ module load ncarenv/1.3 python/3.7.9 cmake intel/19.1.1 esmf_libs mkl
+ module use /glade/p/cesmdata/cseg/PROGS/modulefiles/esmfpkgs/intel/19.1.1/
+ module load esmf-8.2.0b23-ncdfio-mpt-O mpt/2.22 netcdf-mpi/4.8.0 pnetcdf/1.12.2 ncarcompilers/0.5.0
- ./lilac/build_ctsm /glade/scratch/$USER/ctsm_build_dir --os linux --compiler intel --netcdf-path '$ENV{NETCDF}' --pio-filesystem-hints gpfs --pnetcdf-path '$ENV{PNETCDF}' --esmf-lib-path '$ENV{ESMF_LIBDIR}' --max-mpitasks-per-node 36 --extra-cflags '-xCORE_AVX2 -no-fma' --extra-fflags '-xCORE_AVX2 -no-fma'
+ ./lilac/build_ctsm /glade/scratch/$USER/ctsm_build_dir --os linux --compiler intel --netcdf-path '$ENV{NETCDF}' --pio-filesystem-hints gpfs --pnetcdf-path '$ENV{PNETCDF}' --esmf-mkfile-path '$ENV{ESMFMKFILE}' --max-mpitasks-per-node 36 --extra-cflags '-xCORE_AVX2 -no-fma' --extra-fflags '-xCORE_AVX2 -no-fma'
(It's better to use the :ref:`alternative process for a CIME-supported
machine` in this case, but the above illustrates
diff --git a/lilac/bld_templates/config_compilers_template.xml b/lilac/bld_templates/config_compilers_template.xml
deleted file mode 100644
index 9fc3358408..0000000000
--- a/lilac/bld_templates/config_compilers_template.xml
+++ /dev/null
@@ -1,44 +0,0 @@
-
-
-
-
-
-
-
-
-
- $GPTL_CPPDEFS
-
-
- $NETCDF_PATH
-
-
- $PIO_FILESYSTEM_HINTS
-
-
- $PNETCDF_PATH
-
- $ESMF_LIBDIR
-
-
- $EXTRA_CFLAGS
-
-
-
- $EXTRA_FFLAGS
-
-
-
-
-
diff --git a/lilac/bld_templates/config_machines_template.xml b/lilac/bld_templates/config_machines_template.xml
index a197e02dfa..ccde849607 100644
--- a/lilac/bld_templates/config_machines_template.xml
+++ b/lilac/bld_templates/config_machines_template.xml
@@ -12,7 +12,7 @@
-->
-
+
Temporary build information for a CTSM build
@@ -111,5 +111,9 @@
-->
+
+ $ESMF_MKFILE_PATH
+
+
diff --git a/lilac/bld_templates/ctsm-build_template.cmake b/lilac/bld_templates/ctsm-build_template.cmake
new file mode 100644
index 0000000000..a4dae78533
--- /dev/null
+++ b/lilac/bld_templates/ctsm-build_template.cmake
@@ -0,0 +1,23 @@
+# This, together with config_machines_template.xml, provides a machine port for building
+# CTSM.
+#
+# If you are looking at the template file: Variable names prefixed with a dollar sign will
+# be replaced with machine-specific values. A double dollar sign gets replaced with a
+# single dollar sign, so something like $$MYVAR refers to the MYVAR cime variable.
+
+if (COMP_NAME STREQUAL gptl)
+ string(APPEND CPPDEFS " $GPTL_CPPDEFS")
+endif()
+
+set(NETCDF_PATH "$NETCDF_PATH")
+
+# If PIO_FILESYSTEM_HINTS is provided, this will set a PIO_FILESYSTEM_HINTS variable; if
+# not provided, this will just be a blank line.
+$PIO_FILESYSTEM_HINTS
+
+# If PNETCDF_PATH is provided, this will set a PNETCDF_PATH variable; if not provided,
+# this will just be a blank line.
+$PNETCDF_PATH
+
+string(APPEND CFLAGS " $EXTRA_CFLAGS")
+string(APPEND FFLAGS " $EXTRA_FFLAGS")
diff --git a/python/Makefile b/python/Makefile
index 6c7e1ab32c..4ea5fba85d 100644
--- a/python/Makefile
+++ b/python/Makefile
@@ -23,7 +23,7 @@ PYLINT_ARGS=-j 4 --rcfile=ctsm/.pylintrc
PYLINT_SRC = \
ctsm
-all: test lint
+all: test lint black
test: utest stest
.PHONY: utest
@@ -38,6 +38,12 @@ stest: FORCE
lint: FORCE
$(PYLINT) $(PYLINT_ARGS) $(PYLINT_SRC)
+.PHONY: black
+# Run black on all of the python files here and undeneath.
+# Use the black configure file to explicitly set a few things and specifiy the exact files.
+black: FORCE
+ black --check --config pyproject.toml .
+
.PHONY: clean
clean: FORCE
find . -name '*.pyc' -exec rm {} \;
diff --git a/python/ctsm/.pylintrc b/python/ctsm/.pylintrc
index 46c4837b6c..bc7ae54dd2 100644
--- a/python/ctsm/.pylintrc
+++ b/python/ctsm/.pylintrc
@@ -140,6 +140,7 @@ disable=print-statement,
deprecated-sys-function,
exception-escape,
comprehension-escape,
+ C0330, # This is an option that the formatter "black" requires us to disable
# --- default list is above here, our own list is below here ---
# While pylint's recommendations to keep the number of arguments, local
# variables and branches low is generally a good one, I don't want it to
diff --git a/python/ctsm/lilac_build_ctsm.py b/python/ctsm/lilac_build_ctsm.py
index 888008897a..be1fbf7350 100644
--- a/python/ctsm/lilac_build_ctsm.py
+++ b/python/ctsm/lilac_build_ctsm.py
@@ -18,7 +18,7 @@
# ========================================================================
# this matches the machine name in config_machines_template.xml
-_MACH_NAME = 'ctsm_build'
+_MACH_NAME = 'ctsm-build'
# these are arbitrary, since we only use the case for its build, not any of the runtime
# settings; they just need to be valid
@@ -71,7 +71,7 @@ def main(cime_path):
machine=args.machine,
os_type=args.os,
netcdf_path=args.netcdf_path,
- esmf_lib_path=args.esmf_lib_path,
+ esmf_mkfile_path=args.esmf_mkfile_path,
max_mpitasks_per_node=args.max_mpitasks_per_node,
gmake=args.gmake,
gmake_j=args.gmake_j,
@@ -92,7 +92,7 @@ def build_ctsm(cime_path,
machine=None,
os_type=None,
netcdf_path=None,
- esmf_lib_path=None,
+ esmf_mkfile_path=None,
max_mpitasks_per_node=None,
gmake=None,
gmake_j=None,
@@ -117,7 +117,7 @@ def build_ctsm(cime_path,
Must be given if machine isn't given; ignored if machine is given
netcdf_path (str or None): path to NetCDF installation
Must be given if machine isn't given; ignored if machine is given
- esmf_lib_path (str or None): path to ESMF library directory
+ esmf_mkfile_path (str or None): path to esmf.mk file (typically within ESMF library directory)
Must be given if machine isn't given; ignored if machine is given
max_mpitasks_per_node (int or None): number of physical processors per shared-memory node
Must be given if machine isn't given; ignored if machine is given
@@ -153,7 +153,7 @@ def build_ctsm(cime_path,
if machine is None:
assert os_type is not None, 'with machine absent, os_type must be given'
assert netcdf_path is not None, 'with machine absent, netcdf_path must be given'
- assert esmf_lib_path is not None, 'with machine absent, esmf_lib_path must be given'
+ assert esmf_mkfile_path is not None, 'with machine absent, esmf_mkfile_path must be given'
assert max_mpitasks_per_node is not None, ('with machine absent '
'max_mpitasks_per_node must be given')
os_type = _check_and_transform_os(os_type)
@@ -161,7 +161,7 @@ def build_ctsm(cime_path,
os_type=os_type,
compiler=compiler,
netcdf_path=netcdf_path,
- esmf_lib_path=esmf_lib_path,
+ esmf_mkfile_path=esmf_mkfile_path,
max_mpitasks_per_node=max_mpitasks_per_node,
gmake=gmake,
gmake_j=gmake_j,
@@ -242,7 +242,7 @@ def _commandline_args(args_to_parse=None):
For a fresh build with a machine that has NOT been ported to cime:
- build_ctsm /path/to/nonexistent/directory --os OS --compiler COMPILER --netcdf-path NETCDF_PATH --esmf-lib-path ESMF_LIB_PATH --max-mpitasks-per-node MAX_MPITASKS_PER_NODE --pnetcdf-path PNETCDF_PATH
+ build_ctsm /path/to/nonexistent/directory --os OS --compiler COMPILER --netcdf-path NETCDF_PATH --esmf-mkfile-path ESMF_MKFILE_PATH --max-mpitasks-per-node MAX_MPITASKS_PER_NODE --pnetcdf-path PNETCDF_PATH
If PNetCDF is not available, set --no-pnetcdf instead of --pnetcdf-path.
@@ -346,10 +346,10 @@ def _commandline_args(args_to_parse=None):
'named lib, include, etc.)')
new_machine_required_list.append('netcdf-path')
- new_machine_required.add_argument('--esmf-lib-path',
- help='Path to ESMF library directory\n'
- 'This directory should include an esmf.mk file')
- new_machine_required_list.append('esmf-lib-path')
+ new_machine_required.add_argument('--esmf-mkfile-path',
+ help='Path to esmf.mk file\n'
+ '(typically within ESMF library directory)')
+ new_machine_required_list.append('esmf-mkfile-path')
new_machine_required.add_argument('--max-mpitasks-per-node', type=int,
help='Number of physical processors per shared-memory node\n'
@@ -499,7 +499,7 @@ def _fill_out_machine_files(build_dir,
os_type,
compiler,
netcdf_path,
- esmf_lib_path,
+ esmf_mkfile_path,
max_mpitasks_per_node,
gmake,
gmake_j,
@@ -512,7 +512,7 @@ def _fill_out_machine_files(build_dir,
For documentation of args, see the documentation in the build_ctsm function
"""
- os.makedirs(os.path.join(build_dir, _MACHINE_CONFIG_DIRNAME))
+ os.makedirs(os.path.join(build_dir, _MACHINE_CONFIG_DIRNAME, "cmake_macros"))
# ------------------------------------------------------------------------
# Fill in config_machines.xml
@@ -526,10 +526,11 @@ def _fill_out_machine_files(build_dir,
'CIME_OUTPUT_ROOT':build_dir,
'GMAKE':gmake,
'GMAKE_J':gmake_j,
- 'MAX_MPITASKS_PER_NODE':max_mpitasks_per_node})
+ 'MAX_MPITASKS_PER_NODE':max_mpitasks_per_node,
+ 'ESMF_MKFILE_PATH':esmf_mkfile_path})
# ------------------------------------------------------------------------
- # Fill in config_compilers.xml
+ # Fill in ctsm-build_template.cmake
# ------------------------------------------------------------------------
if gptl_nano_timers:
@@ -538,27 +539,26 @@ def _fill_out_machine_files(build_dir,
gptl_cppdefs = ''
if pio_filesystem_hints:
- pio_filesystem_hints_tag = '{}'.format(
+ pio_filesystem_hints_addition = 'set(PIO_FILESYSTEM_HINTS "{}")'.format(
pio_filesystem_hints)
else:
- pio_filesystem_hints_tag = ''
+ pio_filesystem_hints_addition = ''
if pnetcdf_path:
- pnetcdf_path_tag = '{}'.format(
+ pnetcdf_path_addition = 'set(PNETCDF_PATH "{}")'.format(
pnetcdf_path)
else:
- pnetcdf_path_tag = ''
+ pnetcdf_path_addition = ''
fill_template_file(
path_to_template=os.path.join(_PATH_TO_TEMPLATES,
- 'config_compilers_template.xml'),
- path_to_final=os.path.join(build_dir, _MACHINE_CONFIG_DIRNAME, 'config_compilers.xml'),
- substitutions={'COMPILER':compiler,
- 'GPTL_CPPDEFS':gptl_cppdefs,
+ 'ctsm-build_template.cmake'),
+ path_to_final=os.path.join(build_dir, _MACHINE_CONFIG_DIRNAME, "cmake_macros",
+ '{}_{}.cmake'.format(compiler, _MACH_NAME)),
+ substitutions={'GPTL_CPPDEFS':gptl_cppdefs,
'NETCDF_PATH':netcdf_path,
- 'PIO_FILESYSTEM_HINTS':pio_filesystem_hints_tag,
- 'PNETCDF_PATH':pnetcdf_path_tag,
- 'ESMF_LIBDIR':esmf_lib_path,
+ 'PIO_FILESYSTEM_HINTS':pio_filesystem_hints_addition,
+ 'PNETCDF_PATH':pnetcdf_path_addition,
'EXTRA_CFLAGS':extra_cflags,
'EXTRA_FFLAGS':extra_fflags})
diff --git a/python/ctsm/test/test_sys_lilac_build_ctsm.py b/python/ctsm/test/test_sys_lilac_build_ctsm.py
index 5a44688171..3c4117fd45 100755
--- a/python/ctsm/test/test_sys_lilac_build_ctsm.py
+++ b/python/ctsm/test/test_sys_lilac_build_ctsm.py
@@ -46,7 +46,7 @@ def test_buildSetup_userDefinedMachine_minimalInfo(self):
no_build=True,
os_type='linux',
netcdf_path='/path/to/netcdf',
- esmf_lib_path='/path/to/esmf/lib',
+ esmf_mkfile_path='/path/to/esmf/lib/esmf.mk',
max_mpitasks_per_node=16,
gmake='gmake',
gmake_j=8,
@@ -76,7 +76,7 @@ def test_buildSetup_userDefinedMachine_allInfo(self):
no_build=True,
os_type='linux',
netcdf_path='/path/to/netcdf',
- esmf_lib_path='/path/to/esmf/lib',
+ esmf_mkfile_path='/path/to/esmf/lib/esmf.mk',
max_mpitasks_per_node=16,
gmake='gmake',
gmake_j=8,
diff --git a/python/ctsm/test/test_unit_lilac_build_ctsm.py b/python/ctsm/test/test_unit_lilac_build_ctsm.py
index 3c1a600326..96f79f3765 100755
--- a/python/ctsm/test/test_unit_lilac_build_ctsm.py
+++ b/python/ctsm/test/test_unit_lilac_build_ctsm.py
@@ -105,7 +105,7 @@ def test_commandlineArgs_noRebuild_valid(self):
'--os', 'linux',
'--compiler', 'intel',
'--netcdf-path', '/path/to/netcdf',
- '--esmf-lib-path', '/path/to/esmf/lib',
+ '--esmf-mkfile-path', '/path/to/esmf/lib/esmf.mk',
'--max-mpitasks-per-node', '16',
'--no-pnetcdf'])
@@ -120,7 +120,7 @@ def test_commandlineArgs_noRebuild_invalid1(self, mock_stderr):
_ = _commandline_args(args_to_parse=['build/directory',
'--os', 'linux',
'--netcdf-path', '/path/to/netcdf',
- '--esmf-lib-path', '/path/to/esmf/lib',
+ '--esmf-mkfile-path', '/path/to/esmf/lib/esmf.mk',
'--max-mpitasks-per-node', '16',
'--no-pnetcdf'])
self.assertRegex(mock_stderr.getvalue(), expected_re)
@@ -136,7 +136,7 @@ def test_commandlineArgs_noRebuild_invalid2(self, mock_stderr):
_ = _commandline_args(args_to_parse=['build/directory',
'--compiler', 'intel',
'--netcdf-path', '/path/to/netcdf',
- '--esmf-lib-path', '/path/to/esmf/lib',
+ '--esmf-mkfile-path', '/path/to/esmf/lib/esmf.mk',
'--max-mpitasks-per-node', '16',
'--no-pnetcdf'])
self.assertRegex(mock_stderr.getvalue(), expected_re)
@@ -150,7 +150,7 @@ def test_commandlineArgs_noRebuild_invalidNeedToDictatePnetcdf(self, mock_stderr
'--os', 'linux',
'--compiler', 'intel',
'--netcdf-path', '/path/to/netcdf',
- '--esmf-lib-path', '/path/to/esmf/lib',
+ '--esmf-mkfile-path', '/path/to/esmf/lib/esmf.mk',
'--max-mpitasks-per-node', '16'])
self.assertRegex(mock_stderr.getvalue(), expected_re)
@@ -163,7 +163,7 @@ def test_commandlineArgs_noRebuild_invalidConflictingPnetcdf(self, mock_stderr):
'--os', 'linux',
'--compiler', 'intel',
'--netcdf-path', '/path/to/netcdf',
- '--esmf-lib-path', '/path/to/esmf/lib',
+ '--esmf-mkfile-path', '/path/to/esmf/lib/esmf.mk',
'--max-mpitasks-per-node', '16',
'--no-pnetcdf',
'--pnetcdf-path', '/path/to/pnetcdf'])
diff --git a/python/pyproject.toml b/python/pyproject.toml
new file mode 100644
index 0000000000..fd8d8ac03c
--- /dev/null
+++ b/python/pyproject.toml
@@ -0,0 +1,13 @@
+#
+# This is a configuration file for python projects.
+# Sepcifically covering build system requirements.
+#
+# Here we are just using a couple options to specify the operation
+# of the python formatter "black".
+#
+[tool.black]
+
+ line-length = 88 # This is the black default
+ target-version = ['py37']
+ include = '(run_ctsm_py_tests|\.py$)' # Files to include
+ exclude = '(\.pylintrc|\.pyc)' # Files to explicitly exclude pylint file and compiled python
diff --git a/src/biogeophys/BalanceCheckMod.F90 b/src/biogeophys/BalanceCheckMod.F90
index 508e118b66..ec928f8645 100644
--- a/src/biogeophys/BalanceCheckMod.F90
+++ b/src/biogeophys/BalanceCheckMod.F90
@@ -82,7 +82,8 @@ subroutine BalanceCheckInit( )
!-----------------------------------------------------------------------
dtime = get_step_size_real()
! Skip a minimum of two time steps, but otherwise skip the number of time-steps in the skip_size rounded to the nearest integer
- skip_steps = max(2, nint( (skip_size / dtime) ) )
+ ! Add an additional step as now required to be after the hourly radiation time-step see github issue #1563
+ skip_steps = max(2, nint( (skip_size / dtime) ) ) + 1
if ( masterproc ) write(iulog,*) ' Skip balance checking for the first ', skip_steps, ' time steps'
diff --git a/src/biogeophys/test/Balance_test/test_Balance.pf b/src/biogeophys/test/Balance_test/test_Balance.pf
index 3d07385ffb..824eebb78c 100644
--- a/src/biogeophys/test/Balance_test/test_Balance.pf
+++ b/src/biogeophys/test/Balance_test/test_Balance.pf
@@ -46,7 +46,7 @@ contains
call unittest_timemgr_setup(dtime=dtime)
call BalanceCheckInit()
nskip = GetBalanceCheckSkipSteps()
- @assertEqual( 2, nskip, message="Ensure standard balance check is 2 time-steps" )
+ @assertEqual( 3, nskip, message="Ensure standard balance check is 3 time-steps" )
end subroutine test_balance_init
@Test
@@ -59,7 +59,7 @@ contains
call unittest_timemgr_setup(dtime=dtime)
call BalanceCheckInit()
nskip = GetBalanceCheckSkipSteps()
- @assertEqual( 2, nskip, message="Ensure even with a long time-step skip is 2 time-steps" )
+ @assertEqual( 3, nskip, message="Ensure even with a long time-step skip is 3 time-steps" )
end subroutine test_balance_longstep
@Test
@@ -72,7 +72,7 @@ contains
call unittest_timemgr_setup(dtime=dtime)
call BalanceCheckInit()
nskip = GetBalanceCheckSkipSteps()
- @assertEqual( 12, nskip, message="Check skip length for 300 sec time-step" )
+ @assertEqual( 13, nskip, message="Check skip length for 300 sec time-step" )
end subroutine test_balance_300sec
@Test
@@ -101,7 +101,7 @@ contains
call unittest_timemgr_setup(dtime=dtime)
call BalanceCheckInit()
nskip = GetBalanceCheckSkipSteps()
- @assertEqual( 100, nskip, message="Ensure with a short step correct number of skip steps is done" )
+ @assertEqual( 101, nskip, message="Ensure with a short step correct number of skip steps is done" )
end subroutine test_balance_shortstep
end module test_balance
diff --git a/test/tools/README b/test/tools/README
index 186f1dfe5d..cb5dcdec34 100644
--- a/test/tools/README
+++ b/test/tools/README
@@ -15,6 +15,11 @@ on cheyenne
qcmd -l walltime=08:00:00 -- ./test_driver.sh -i >& run.out &
+And to for example to compare to another baseline code (in this case ctsm5.1.dev066, which would need to be cloned at the given
+path) ...
+
+qcmd -l walltime=08:00:00 -- env BL_ROOT=/glade/scratch/erik/ctsm5.1.dev066 ./test_driver.sh -i >& run.out &
+
on izumi
nohup ./test_driver.sh -i >& run.out &
diff --git a/test/tools/tests_posttag_hobart_nompi b/test/tools/tests_posttag_hobart_nompi
index 4655f29853..9f07863e4d 100644
--- a/test/tools/tests_posttag_hobart_nompi
+++ b/test/tools/tests_posttag_hobart_nompi
@@ -2,5 +2,3 @@ smc#4 blc#4
smi54 bli54
smi57 bli57
smiT4 bliT4
-smf84 blf84
-smfc4 blfc4
diff --git a/test/tools/tests_posttag_izumi_nompi b/test/tools/tests_posttag_izumi_nompi
index 90be9522dc..62687a7e3d 100644
--- a/test/tools/tests_posttag_izumi_nompi
+++ b/test/tools/tests_posttag_izumi_nompi
@@ -1,5 +1,3 @@
smi54 bli54
smi57 bli57
smiT4 bliT4
-smf84 blf84
-smfc4 blfc4
diff --git a/test/tools/tests_posttag_nompi_regression b/test/tools/tests_posttag_nompi_regression
index 1785b5da47..5b5d76fd60 100644
--- a/test/tools/tests_posttag_nompi_regression
+++ b/test/tools/tests_posttag_nompi_regression
@@ -1,5 +1,4 @@
smc#4 blc#4
-sme14 ble14
smg54 blg54
smi24 bli24
smi53 bli53
@@ -10,9 +9,6 @@ smi74 bli74
smi78 bli78
smiT4 bliT4
smiT2 bliT2
-smf84 blf84
-smfc4 blfc4
-smfg4 blfg4
smiS4 bliS4
smiS8 bliS8
smiS9 bliS9
diff --git a/test/tools/tests_pretag_cheyenne_nompi b/test/tools/tests_pretag_cheyenne_nompi
index f99ab6b691..fec9d08448 100644
--- a/test/tools/tests_pretag_cheyenne_nompi
+++ b/test/tools/tests_pretag_cheyenne_nompi
@@ -1,7 +1,5 @@
smi79 bli79
smc#4 blc#4
-sme14 ble14
-sme@4 ble@4
smg54 blg54
sm0a1 bl0a1
smaa2 blaa2
@@ -17,6 +15,3 @@ smiS4 bliS4
smi74 bli74
smiT4 bliT4
smiT2 bliT2
-smf84 blf84
-smfc4 blfc4
-smfg4 blfg4
diff --git a/tools/mksurfdata_map/src/Mkdepends b/tools/mksurfdata_map/src/Mkdepends
index a75e8fdde0..ddb1682da4 100755
--- a/tools/mksurfdata_map/src/Mkdepends
+++ b/tools/mksurfdata_map/src/Mkdepends
@@ -35,6 +35,7 @@
use Getopt::Std;
use File::Basename;
+use File::Glob ':bsd_glob';
# Check for usage request.
@ARGV >= 2 or usage();
@@ -61,7 +62,7 @@ chomp @file_paths;
unshift(@file_paths,'.');
foreach $dir (@file_paths) { # (could check that directories exist here)
$dir =~ s!/?\s*$!!; # remove / and any whitespace at end of directory name
- ($dir) = glob $dir; # Expand tildes in path names.
+ ($dir) = bsd_glob $dir; # Expand tildes in path names.
}
# Make list of files containing source code.
@@ -91,7 +92,7 @@ my ($dir);
my ($f, $name, $path, $suffix, $mod);
my @suffixes = ('\.mod' );
foreach $dir (@file_paths) {
- @filenames = (glob("$dir/*.mod"));
+ @filenames = (bsd_glob("$dir/*.mod"));
foreach $f (@filenames) {
($name, $path, $suffix) = fileparse($f, @suffixes);
($mod = $name) =~ tr/a-z/A-Z/;
diff --git a/tools/mksurfdata_map/src/mkVICparamsMod.F90 b/tools/mksurfdata_map/src/mkVICparamsMod.F90
index f7cb4946c6..431e43cb28 100644
--- a/tools/mksurfdata_map/src/mkVICparamsMod.F90
+++ b/tools/mksurfdata_map/src/mkVICparamsMod.F90
@@ -129,7 +129,7 @@ subroutine mkVICparams(ldomain, mapfname, datfname, ndiag, &
! Check validity of output data
if (min_bad(binfl_o, min_valid_binfl, 'binfl')) then
- stop
+ call abort()
end if
call output_diagnostics_continuous(data_i, binfl_o, tgridmap, "VIC b parameter", "unitless", ndiag, tdomain%mask, frac_dst)
@@ -144,7 +144,7 @@ subroutine mkVICparams(ldomain, mapfname, datfname, ndiag, &
! Check validity of output data
if (min_bad(ws_o, min_valid_ws, 'Ws')) then
- stop
+ call abort()
end if
call output_diagnostics_continuous(data_i, ws_o, tgridmap, "VIC Ws parameter", "unitless", ndiag, tdomain%mask, frac_dst)
@@ -159,7 +159,7 @@ subroutine mkVICparams(ldomain, mapfname, datfname, ndiag, &
! Check validity of output data
if (min_bad(dsmax_o, min_valid_dsmax, 'Dsmax')) then
- stop
+ call abort()
end if
call output_diagnostics_continuous(data_i, dsmax_o, tgridmap, "VIC Dsmax parameter", "mm/day", ndiag, tdomain%mask, frac_dst)
@@ -174,7 +174,7 @@ subroutine mkVICparams(ldomain, mapfname, datfname, ndiag, &
! Check validity of output data
if (min_bad(ds_o, min_valid_ds, 'Ds')) then
- stop
+ call abort()
end if
call output_diagnostics_continuous(data_i, ds_o, tgridmap, "VIC Ds parameter", "unitless", ndiag, tdomain%mask, frac_dst)
diff --git a/tools/mksurfdata_map/src/mkagfirepkmonthMod.F90 b/tools/mksurfdata_map/src/mkagfirepkmonthMod.F90
index af8001263f..7b58ddffad 100644
--- a/tools/mksurfdata_map/src/mkagfirepkmonthMod.F90
+++ b/tools/mksurfdata_map/src/mkagfirepkmonthMod.F90
@@ -145,7 +145,7 @@ subroutine mkagfirepkmon(ldomain, mapfname, datfname, ndiag, &
! Check validity of output data
if (min_bad(agfirepkmon_o, min_valid, 'agfirepkmon') .or. &
max_bad(agfirepkmon_o, max_valid, 'agfirepkmon')) then
- stop
+ call abort()
end if
diff --git a/tools/mksurfdata_map/src/mkdiagnosticsMod.F90 b/tools/mksurfdata_map/src/mkdiagnosticsMod.F90
index a53d9ca4d2..91769a5823 100644
--- a/tools/mksurfdata_map/src/mkdiagnosticsMod.F90
+++ b/tools/mksurfdata_map/src/mkdiagnosticsMod.F90
@@ -81,7 +81,7 @@ subroutine output_diagnostics_area(data_i, data_o, gridmap, name, percent, ndiag
write(6,*) 'ns_i = ', ns_i
write(6,*) 'size(data_o) = ', size(data_o)
write(6,*) 'ns_o = ', ns_o
- stop
+ call abort()
end if
if (size(frac_dst) /= ns_o) then
write(6,*) subname//' ERROR: incorrect size of frac_dst'
@@ -195,7 +195,7 @@ subroutine output_diagnostics_continuous(data_i, data_o, gridmap, name, units, n
write(6,*) 'ns_i = ', ns_i
write(6,*) 'size(data_o) = ', size(data_o)
write(6,*) 'ns_o = ', ns_o
- stop
+ call abort()
end if
if (size(frac_dst) /= ns_o) then
write(6,*) subname//' ERROR: incorrect size of frac_dst'
@@ -300,7 +300,7 @@ subroutine output_diagnostics_continuous_outonly(data_o, gridmap, name, units, n
write(6,*) subname//' ERROR: array size inconsistencies for ', trim(name)
write(6,*) 'size(data_o) = ', size(data_o)
write(6,*) 'ns_o = ', ns_o
- stop
+ call abort()
end if
! Sums on output grid
@@ -380,7 +380,7 @@ subroutine output_diagnostics_index(data_i, data_o, gridmap, name, &
write(6,*) 'ns_i = ', ns_i
write(6,*) 'size(data_o) = ', size(data_o)
write(6,*) 'ns_o = ', ns_o
- stop
+ call abort()
end if
if (size(frac_dst) /= ns_o) then
write(6,*) subname//' ERROR: incorrect size of frac_dst'
diff --git a/tools/mksurfdata_map/src/mkgdpMod.F90 b/tools/mksurfdata_map/src/mkgdpMod.F90
index 6a560e61b5..138ddf1805 100644
--- a/tools/mksurfdata_map/src/mkgdpMod.F90
+++ b/tools/mksurfdata_map/src/mkgdpMod.F90
@@ -122,7 +122,7 @@ subroutine mkgdp(ldomain, mapfname, datfname, ndiag, gdp_o)
! Check validity of output data
if (min_bad(gdp_o, min_valid, 'gdp')) then
- stop
+ call abort()
end if
call output_diagnostics_continuous(data_i, gdp_o, tgridmap, "GDP", "x1000 US$ per capita", ndiag, tdomain%mask, frac_dst)
diff --git a/tools/mksurfdata_map/src/mkglacierregionMod.F90 b/tools/mksurfdata_map/src/mkglacierregionMod.F90
index beae6a8d97..e644129ed3 100644
--- a/tools/mksurfdata_map/src/mkglacierregionMod.F90
+++ b/tools/mksurfdata_map/src/mkglacierregionMod.F90
@@ -105,7 +105,7 @@ subroutine mkglacierregion(ldomain, mapfname, datfname, ndiag, &
call check_ret(nf_inq_varid(ncid, 'GLACIER_REGION', varid), subname)
call check_ret(nf_get_var_int(ncid, varid, glacier_region_i), subname)
if (min_bad(glacier_region_i, 0, 'GLACIER_REGION')) then
- stop
+ call abort()
end if
call get_max_indices( &
diff --git a/tools/mksurfdata_map/src/mkglcmecMod.F90 b/tools/mksurfdata_map/src/mkglcmecMod.F90
index 2ac4d94e4f..9fbad66689 100644
--- a/tools/mksurfdata_map/src/mkglcmecMod.F90
+++ b/tools/mksurfdata_map/src/mkglcmecMod.F90
@@ -606,7 +606,7 @@ subroutine mkglacier(ldomain, mapfname, datfname, ndiag, zero_out, glac_o)
write (6,*) 'MKGLACIER error: glacier = ',glac_o(no), &
' greater than 100.000001 for column, row = ',no
call shr_sys_flush(6)
- stop
+ call abort()
end if
enddo
diff --git a/tools/mksurfdata_map/src/mkgridmapMod.F90 b/tools/mksurfdata_map/src/mkgridmapMod.F90
index 21ca23f4d6..eeb5afdbb8 100644
--- a/tools/mksurfdata_map/src/mkgridmapMod.F90
+++ b/tools/mksurfdata_map/src/mkgridmapMod.F90
@@ -505,7 +505,7 @@ subroutine gridmap_check(gridmap, mask_src, frac_dst, caller)
write (6,*) subname//' ERROR from '//trim(caller)//': mapping areas not conserved'
write (6,'(a30,e20.10)') 'global sum output field = ',sum_area_o
write (6,'(a30,e20.10)') 'global sum input field = ',sum_area_i
- stop
+ call abort()
end if
end if
diff --git a/tools/mksurfdata_map/src/mklaiMod.F90 b/tools/mksurfdata_map/src/mklaiMod.F90
index aef33f3463..e4b6d9bfa1 100644
--- a/tools/mksurfdata_map/src/mklaiMod.F90
+++ b/tools/mksurfdata_map/src/mklaiMod.F90
@@ -140,7 +140,7 @@ subroutine mklai(ldomain, mapfname, datfname, ndiag, ncido)
! invalid, all the loop bounds over output data in this
! routine will need to be double checked!
write(6, *) "ERROR:" // trim(subname) // "(): input numpft must be less than or equal to output numpft+1."
- stop
+ call abort()
end if
endif
if (ntim /= 12) then
@@ -434,7 +434,7 @@ subroutine pft_laicheck( ni_s, pctpft_i, laimask )
write (6,*) subName//' :: pft/LAI+SAI inconsistency over more than 25% land-cover'
write (6,*) '# inconsistent points, total PFT pts, total LAI+SAI pts = ', &
n, nc, sum(laimask(:,l))
- stop
+ call abort()
end if
end do
diff --git a/tools/mksurfdata_map/src/mklanwatMod.F90 b/tools/mksurfdata_map/src/mklanwatMod.F90
index 49a1485fa7..4e1c590803 100644
--- a/tools/mksurfdata_map/src/mklanwatMod.F90
+++ b/tools/mksurfdata_map/src/mklanwatMod.F90
@@ -478,7 +478,7 @@ subroutine mklakparams(ldomain, mapfname, datfname, ndiag, &
! Check validity of output data
if (min_bad(lakedepth_o, min_valid_lakedepth, 'lakedepth')) then
- stop
+ call abort()
end if
call output_diagnostics_continuous(data_i, lakedepth_o, tgridmap, "Lake Depth", "m", ndiag, tdomain%mask, frac_dst)
diff --git a/tools/mksurfdata_map/src/mkpeatMod.F90 b/tools/mksurfdata_map/src/mkpeatMod.F90
index 974566a056..8e47f5032d 100644
--- a/tools/mksurfdata_map/src/mkpeatMod.F90
+++ b/tools/mksurfdata_map/src/mkpeatMod.F90
@@ -123,7 +123,7 @@ subroutine mkpeat(ldomain, mapfname, datfname, ndiag, peat_o)
! Check validity of output data
if (min_bad(peat_o, min_valid, 'peat') .or. &
max_bad(peat_o, max_valid, 'peat')) then
- stop
+ call abort()
end if
call output_diagnostics_area(data_i, peat_o, tgridmap, "Peat", percent=.false., ndiag=ndiag, mask_src=tdomain%mask, frac_dst=frac_dst)
diff --git a/tools/mksurfdata_map/src/mkpftMod.F90 b/tools/mksurfdata_map/src/mkpftMod.F90
index 3a12c38cdf..2eae1ae381 100644
--- a/tools/mksurfdata_map/src/mkpftMod.F90
+++ b/tools/mksurfdata_map/src/mkpftMod.F90
@@ -644,7 +644,7 @@ subroutine mkpft(ldomain, mapfname, fpft, ndiag, &
write (6,*) subname//'error: nat pft = ', &
(pct_nat_pft_o(no,m), m = 0, num_natpft), &
' do not sum to 100. at no = ',no,' but to ', wst_sum
- stop
+ call abort()
end if
! Correct sum so that if it differs slightly from 100, it is corrected to equal
@@ -661,7 +661,7 @@ subroutine mkpft(ldomain, mapfname, fpft, ndiag, &
write (6,*) subname//'error: crop cft = ', &
(pct_cft_o(no,m), m = 1, num_cft), &
' do not sum to 100. at no = ',no,' but to ', wst_sum
- stop
+ call abort()
end if
! Correct sum so that if it differs slightly from 100, it is corrected to equal
diff --git a/tools/mksurfdata_map/src/mksoilMod.F90 b/tools/mksurfdata_map/src/mksoilMod.F90
index 959749ca1a..d7cad23e0d 100644
--- a/tools/mksurfdata_map/src/mksoilMod.F90
+++ b/tools/mksurfdata_map/src/mksoilMod.F90
@@ -319,7 +319,7 @@ subroutine mksoiltex(ldomain, mapfname, datfname, ndiag, sand_o, clay_o)
write(6,*)'kmax is > kmap_max= ',kmax(no), 'kmap_max = ', &
kmap_max,' for no = ',no
write(6,*)'reset kmap_max in mksoilMod to a greater value'
- stop
+ call abort()
end if
kmap(kmax(no),no) = k
kwgt(kmax(no),no) = wt
@@ -841,7 +841,7 @@ subroutine mkorganic(ldomain, mapfname, datfname, ndiag, organic_o)
if (nlay /= nlevsoi) then
write(6,*)'nlay, nlevsoi= ',nlay,nlevsoi,' do not match'
- stop
+ call abort()
end if
call check_ret(nf_inq_varid (ncid, 'ORGANIC', varid), subname)
@@ -873,7 +873,7 @@ subroutine mkorganic(ldomain, mapfname, datfname, ndiag, organic_o)
write (6,*) 'MKORGANIC error: organic = ',organic_o(no,lev), &
' greater than 130.000001 for column, row = ',no
call shr_sys_flush(6)
- stop
+ call abort()
end if
enddo
@@ -942,7 +942,7 @@ subroutine mksoilfmaxInit( )
if ( soil_fmax /= unset )then
if ( soil_fmax < 0.0 .or. soil_fmax > 1.0 )then
write(6,*)'soil_fmax is out of range = ', soil_fmax
- stop
+ call abort()
end if
write(6,*) 'Replace soil fmax for all points with: ', soil_fmax
end if
@@ -1053,7 +1053,7 @@ subroutine mkfmax(ldomain, mapfname, datfname, ndiag, fmax_o)
write (6,*) 'MKFMAX error: fmax = ',fmax_o(no), &
' greater than 1.000001 for column, row = ',no
call shr_sys_flush(6)
- stop
+ call abort()
end if
enddo
@@ -1086,7 +1086,7 @@ subroutine mkfmax(ldomain, mapfname, datfname, ndiag, fmax_o)
frac_dst(no)*re**2
if ((frac_dst(no) < 0.0) .or. (frac_dst(no) > 1.0001)) then
write(6,*) "ERROR:: frac_dst out of range: ", frac_dst(no),no
- stop
+ call abort()
end if
end do
diff --git a/tools/mksurfdata_map/src/mksoildepthMod.F90 b/tools/mksurfdata_map/src/mksoildepthMod.F90
index 521ac2c6f0..c69cf375a4 100644
--- a/tools/mksurfdata_map/src/mksoildepthMod.F90
+++ b/tools/mksurfdata_map/src/mksoildepthMod.F90
@@ -146,7 +146,7 @@ subroutine mksoildepth(ldomain, mapfname, datfname, ndiag, soildepth_o)
! Check validity of output data
if (min_bad(soildepth_o, min_valid, 'soildepth') .or. &
max_bad(soildepth_o, max_valid, 'soildepth')) then
- stop
+ call abort()
end if
call output_diagnostics_area(data_i, soildepth_o, tgridmap, "Soildepth", percent=.false., ndiag=ndiag, mask_src=tdomain%mask, frac_dst=frac_dst)
diff --git a/tools/mksurfdata_map/src/mksurfdat.F90 b/tools/mksurfdata_map/src/mksurfdat.F90
index 9051c57707..aa965f097d 100644
--- a/tools/mksurfdata_map/src/mksurfdat.F90
+++ b/tools/mksurfdata_map/src/mksurfdat.F90
@@ -425,7 +425,7 @@ program mksurfdat
! Make sure ldomain is on a 0 to 360 grid as that's a requirement for CESM
if ( .not. is_domain_0to360_longs( ldomain ) )then
write(6,*)' Output domain must be on a 0 to 360 longitude grid rather than a -180 to 180 grid as it is required for CESM'
- stop
+ call abort()
end if
! ----------------------------------------------------------------------
! Allocate and initialize dynamic memory
@@ -493,7 +493,7 @@ program mksurfdat
if (fsurlog == ' ') then
write(6,*)' must specify fsurlog in namelist'
- stop
+ call abort()
else
ndiag = getavu(); call opnfil (fsurlog, ndiag, 'f')
end if
@@ -1091,7 +1091,7 @@ program mksurfdat
if (fdyndat == ' ') then
write(6,*)' must specify fdyndat in namelist if mksrf_fdynuse is not blank'
- stop
+ call abort()
end if
! Define dimensions and global attributes
diff --git a/tools/mksurfdata_map/src/mktopostatsMod.F90 b/tools/mksurfdata_map/src/mktopostatsMod.F90
index 2ecd705f4c..7e102d9bcf 100644
--- a/tools/mksurfdata_map/src/mktopostatsMod.F90
+++ b/tools/mksurfdata_map/src/mktopostatsMod.F90
@@ -131,7 +131,7 @@ subroutine mktopostats(ldomain, mapfname, datfname, ndiag, topo_stddev_o, slope_
! Check validity of output data
if (min_bad(topo_stddev_o, min_valid_topo_stddev, 'topo_stddev')) then
- stop
+ call abort()
end if
@@ -158,7 +158,7 @@ subroutine mktopostats(ldomain, mapfname, datfname, ndiag, topo_stddev_o, slope_
! Check validity of output data
if (min_bad(slope_o, min_valid_slope, 'slope') .or. &
max_bad(slope_o, max_valid_slope, 'slope')) then
- stop
+ call abort()
end if
diff --git a/tools/mksurfdata_map/src/mkurbanparCommonMod.F90 b/tools/mksurfdata_map/src/mkurbanparCommonMod.F90
index 5db84e8351..ab738ea03c 100644
--- a/tools/mksurfdata_map/src/mkurbanparCommonMod.F90
+++ b/tools/mksurfdata_map/src/mkurbanparCommonMod.F90
@@ -93,13 +93,13 @@ subroutine mkurban_pct(ldomain, tdomain, tgridmap, urbn_i, urbn_o, frac_dst)
write(6,*) 'tdomain%ns = ', tdomain%ns
write(6,*) 'size(urbn_o) = ', size(urbn_o)
write(6,*) 'ldomain%ns = ', ldomain%ns
- stop
+ call abort()
end if
if (size(frac_dst) /= ldomain%ns) then
write(6,*) subname//' ERROR: array size inconsistencies'
write(6,*) 'size(frac_dst) = ', size(frac_dst)
write(6,*) 'ldomain%ns = ', ldomain%ns
- stop
+ call abort()
end if
! Error checks for domain and map consistencies
@@ -119,7 +119,7 @@ subroutine mkurban_pct(ldomain, tdomain, tgridmap, urbn_i, urbn_o, frac_dst)
if ((urbn_o(no)) > 100.000001_r8) then
write (6,*) 'MKURBAN error: urban = ',urbn_o(no), &
' greater than 100.000001 for column, row = ',no
- stop
+ call abort()
end if
enddo
@@ -191,7 +191,7 @@ subroutine mkurban_pct_diagnostics(ldomain, tdomain, tgridmap, urbn_i, urbn_o, n
write(6,*) subname//' ERROR: array size inconsistencies'
write(6,*) 'size(frac_dst) = ', size(frac_dst)
write(6,*) 'ldomain%ns = ', ldomain%ns
- stop
+ call abort()
end if
! -----------------------------------------------------------------
diff --git a/tools/mksurfdata_map/src/mkurbanparMod.F90 b/tools/mksurfdata_map/src/mkurbanparMod.F90
index 07319b1f27..49ce95dd07 100644
--- a/tools/mksurfdata_map/src/mkurbanparMod.F90
+++ b/tools/mksurfdata_map/src/mkurbanparMod.F90
@@ -548,17 +548,17 @@ subroutine mkurbanpar(datfname, ncido, region_o, urbn_classes_gcell_o, urban_ski
if (nlevurb_i /= nlevurb) then
write(6,*)'MKURBANPAR: parameter nlevurb= ',nlevurb, &
'does not equal input dataset nlevurb= ',nlevurb_i
- stop
+ call abort()
endif
if (numsolar_i /= numsolar) then
write(6,*)'MKURBANPAR: parameter numsolar= ',numsolar, &
'does not equal input dataset numsolar= ',numsolar_i
- stop
+ call abort()
endif
if (numrad_i /= numrad) then
write(6,*)'MKURBANPAR: parameter numrad= ',numrad, &
'does not equal input dataset numrad= ',numrad_i
- stop
+ call abort()
endif
! Create an array that will hold the density indices
diff --git a/tools/site_and_regional/modify_singlept_site_neon.py b/tools/site_and_regional/modify_singlept_site_neon.py
index 7b469641f1..d3db55126b 100755
--- a/tools/site_and_regional/modify_singlept_site_neon.py
+++ b/tools/site_and_regional/modify_singlept_site_neon.py
@@ -12,8 +12,8 @@
This script will do the following:
- Download neon data for the specified site if it does not exist
- in the specified directory.
-- Modify surface dataset with downloaded data (neon-specific).
+ in the specified directory : (i.e. ../../../neon_surffiles).
+- Modify surface dataset with downloaded data.
-------------------------------------------------------------------
Instructions for running on Cheyenne/Casper:
@@ -28,13 +28,14 @@
To see the available options:
./modify_singlept_site_neon.py --help
-------------------------------------------------------------------
+Example:
+ ./modify_singlept_site_neon.py --neon_site PUUM --debug
+-------------------------------------------------------------------
"""
# TODO (NS)
-#--[] If file not found run subset_data.py
-#--[] Clean up imports for both codes...
-#--[] Check against a list of valid names.
-#--[] List of valid neon sites for all scripts come from one place.
-#--[] zbedrock
+# --[] If subset file not found run subset_data.py
+# --[] List of valid neon sites for all scripts come from one place.
+# --[] Download files only when available.
# Import libraries
from __future__ import print_function
@@ -45,6 +46,7 @@
import argparse
import requests
+import logging
import numpy as np
import pandas as pd
import xarray as xr
@@ -56,72 +58,118 @@
myname = getuser()
-#-- valid neon sites
-valid_neon_sites = ['ABBY','BARR','BART','BLAN',
- 'BONA','CLBJ','CPER','DCFS',
- 'DEJU','DELA','DSNY','GRSM',
- 'GUAN','HARV','HEAL','JERC',
- 'JORN','KONA','KONZ','LAJA',
- 'LENO','MLBS','MOAB','NIWO',
- 'NOGP','OAES','ONAQ','ORNL',
- 'OSBS','PUUM','RMNP','SCBI',
- 'SERC','SJER','SOAP','SRER',
- 'STEI','STER','TALL','TEAK',
- 'TOOL','TREE','UKFS','UNDE',
- 'WOOD','WREF','YELL'
- ]
-
-
-def get_parser():
+# -- valid neon sites
+valid_neon_sites = [
+ "ABBY",
+ "BARR",
+ "BART",
+ "BLAN",
+ "BONA",
+ "CLBJ",
+ "CPER",
+ "DCFS",
+ "DEJU",
+ "DELA",
+ "DSNY",
+ "GRSM",
+ "GUAN",
+ "HARV",
+ "HEAL",
+ "JERC",
+ "JORN",
+ "KONA",
+ "KONZ",
+ "LAJA",
+ "LENO",
+ "MLBS",
+ "MOAB",
+ "NIWO",
+ "NOGP",
+ "OAES",
+ "ONAQ",
+ "ORNL",
+ "OSBS",
+ "PUUM",
+ "RMNP",
+ "SCBI",
+ "SERC",
+ "SJER",
+ "SOAP",
+ "SRER",
+ "STEI",
+ "STER",
+ "TALL",
+ "TEAK",
+ "TOOL",
+ "TREE",
+ "UKFS",
+ "UNDE",
+ "WOOD",
+ "WREF",
+ "YELL",
+]
+
+
+def get_parser():
"""
Get parser object for this script.
"""
- parser = argparse.ArgumentParser(description=__doc__,
- formatter_class=argparse.RawDescriptionHelpFormatter)
+ parser = argparse.ArgumentParser(
+ description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter
+ )
parser.print_usage = parser.print_help
- parser.add_argument('--neon_site',
- help='4-letter neon site code.',
- action="store",
- dest="site_name",
- choices=valid_neon_sites,
- required=True)
- parser.add_argument('--surf_dir',
- help='''
+ parser.add_argument(
+ "--neon_site",
+ help="4-letter neon site code.",
+ action="store",
+ dest="site_name",
+ choices=valid_neon_sites,
+ required=True,
+ )
+ parser.add_argument(
+ "--surf_dir",
+ help="""
Directory of single point surface dataset.
[default: %(default)s]
- ''',
- action="store",
- dest="surf_dir",
- type =str,
- required=False,
- default="/glade/scratch/"+myname+"/single_point/")
- parser.add_argument('--out_dir',
- help='''
+ """,
+ action="store",
+ dest="surf_dir",
+ type=str,
+ required=False,
+ default="/glade/scratch/" + myname + "/single_point/",
+ )
+ parser.add_argument(
+ "--out_dir",
+ help="""
Directory to write updated single point surface dataset.
[default: %(default)s]
- ''',
- action="store",
- dest="out_dir",
- type =str,
- required=False,
- default="/glade/scratch/"+myname+"/single_point_neon_updated/")
- parser.add_argument('-d','--debug',
- help='''
+ """,
+ action="store",
+ dest="out_dir",
+ type=str,
+ required=False,
+ default="/glade/scratch/" + myname + "/single_point_neon_updated/",
+ )
+ parser.add_argument(
+ "-d",
+ "--debug",
+ help="""
Debug mode will print more information.
[default: %(default)s]
- ''',
- action="store_true",
- dest="debug",
- default=False)
+ """,
+ action="store_true",
+ dest="debug",
+ default=False,
+ )
return parser
def get_neon(neon_dir, site_name):
"""
- Function for finding neon data file
+ Function for finding neon data files
and download from neon server if the
file does not exits.
@@ -132,53 +180,62 @@ def get_neon(neon_dir, site_name):
Raises:
Error if the download was not successful (exit code:404).
In case the data does not exist in the neon server or if
- neon server is down.
+ neon server is down.
Returns:
neon_file (str) : complete file name of the downloaded data
"""
- #-- create directory if not exists
+ # -- create directory if not exists
if not os.path.exists(neon_dir):
os.makedirs(neon_dir)
neon_file = os.path.join(neon_dir, site_name + "_surfaceData.csv")
- #-- Download the file if it does not exits
+ # -- Download the file if it does not exits
if os.path.isfile(neon_file):
- print('neon file for', site_name, 'already exists! ')
- print('Skipping download from neon for', site_name,'...')
+ print("neon file for", site_name, "already exists! ")
+ print("Skipping download from neon for", site_name, "...")
else:
- print('------------------------------------------------')
- print('Beginning download from neon server for', site_name,'...')
-
- url = ('https://s3.data.neonscience.org/neon-ncar/NEON/surf_files/v1/'
- +site_name+'_surfaceData.csv')
+ print("------------------------------------------------")
+ print("Beginning download from neon server for", site_name, "...")
+
+ url = (
+ "https://s3.data.neonscience.org/neon-ncar/NEON/surf_files/v1/"
+ + site_name
+ + "_surfaceData.csv"
+ )
response = requests.get(url)
- with open(neon_file, 'wb') as f:
+ with open(neon_file, "wb") as f:
f.write(response.content)
- #-- Check if download status_code
+ # -- Check if download status_code
if response.status_code == 200:
- print('Download finished successfully for', site_name)
+ print("Download finished successfully for", site_name)
elif response.status_code == 404:
- sys.exit('Data for this site '+site_name+
- ' was not available on the neon server:'+ url)
-
- print('Download exit status code: ',response.status_code)
- print('Downloaded file type : ',response.headers['content-type'])
- print('Downloaded file encoding : ',response.encoding)
- print('------------------------------------------------')
+ sys.exit(
+ "Data for this site "
+ + site_name
+ + " was not available on the neon server:"
+ + url
+ )
+
+ print("Download exit status code: ", response.status_code)
+ print("Downloaded file type : ", response.headers["content-type"])
+ print("Downloaded file encoding : ", response.encoding)
+ print("------------------------------------------------")
response.close()
return neon_file
-def find_surffile (surf_dir, site_name):
+
+def find_surffile(surf_dir, site_name):
"""
Function for finding and choosing surface file for
a neon site.
+ These files are created using ./subset_data.py script.
In case multiple files exist for the neon site, it
will choose the file created the latest.
@@ -187,32 +244,36 @@ def find_surffile (surf_dir, site_name):
site_name (str): 4 letter neon site name
Raises:
- Error if the surface data for the site is not created
+ Error if the surface data for the site is not created
Returns:
surf_file (str): name of the surface dataset file
"""
- #sf_name = "surfdata_hist_16pfts_Irrig_CMIP6_simyr2000_"+site_name+"*.nc"
- sf_name = "surfdata_hist_78pfts_CMIP6_simyr2000_"+site_name+"*.nc"
- #surf_file = glob.glob(os.path.join(surf_dir,sf_name))
- surf_file = glob.glob(surf_dir+"/"+sf_name)
+ # sf_name = "surfdata_hist_16pfts_Irrig_CMIP6_simyr2000_"+site_name+"*.nc"
+ sf_name = "surfdata_hist_78pfts_CMIP6_simyr2000_" + site_name + "*.nc"
+ # surf_file = glob.glob(os.path.join(surf_dir,sf_name))
+ surf_file = glob.glob(surf_dir + "/" + sf_name)
- if len(surf_file)>1:
- print ("The following files found :", *surf_file, sep='\n- ')
- print ("The latest file is chosen :", surf_file[-1])
+ if len(surf_file) > 1:
+ print("The following files found :", *surf_file, sep="\n- ")
+ print("The latest file is chosen :", surf_file[-1])
surf_file = surf_file[-1]
- elif len(surf_file)==1:
- print ("File found : ")
- print (surf_file)
+ elif len(surf_file) == 1:
+ print("File found : ")
+ print(surf_file)
surf_file = surf_file[0]
else:
- sys.exit('Surface data for this site '+site_name+
- 'was not found:'+ surf_file,'.',
- '\n','Please run ./subset_data.py for this site.')
+ sys.exit(
+ "Surface data for this site " + site_name + "was not found:" + surf_file,
+ ".",
+ "\n",
+ "Please run ./subset_data.py for this site.",
+ )
return surf_file
-def find_soil_structure (surf_file):
+
+def find_soil_structure(surf_file):
"""
Function for finding surface dataset soil
strucutre using surface data metadata.
@@ -227,42 +288,49 @@ def find_soil_structure (surf_file):
surf_file (str): single point surface data filename
Raises:
- error if the soil layer strucutre file does not exist
+ error if the soil layer strucutre file does not exist
Returns:
soil_bot : array of soil layers top depths
soil_top : array of soil layers bottom depths
"""
- #TODO: What if not cheyenne? Self-contained depth info.
+ # TODO: What if not cheyenne? Self-contained depth info.
- print ('------------')
- print (surf_file)
- print (type(surf_file))
+ print("------------")
+ print("surf_file : ", surf_file)
f1 = xr.open_dataset(surf_file)
- print ('------------')
- #print (f1.attrs["Soil_texture_raw_data_file_name"])
+ print("------------")
+ # print (f1.attrs["Soil_texture_raw_data_file_name"])
clm_input_dir = "/glade/p/cesmdata/cseg/inputdata/lnd/clm2/rawdata/"
- surf_soildepth_file = os.path.join(clm_input_dir,
- f1.attrs["Soil_texture_raw_data_file_name"])
-
- if os.path.exists (surf_soildepth_file):
- print ("\n\n Reading", surf_soildepth_file,
- "for surface data soil structure information:")
+ surf_soildepth_file = os.path.join(
+ clm_input_dir, f1.attrs["Soil_texture_raw_data_file_name"]
+ )
+
+ if os.path.exists(surf_soildepth_file):
+ print(
+ "\n\n Reading",
+ surf_soildepth_file,
+ "for surface data soil structure information:",
+ )
f1_soildepth = xr.open_dataset(surf_soildepth_file)
- print (f1_soildepth['DZSOI'])
- soil_bot = f1_soildepth['DZSOI'].values
+ print(f1_soildepth["DZSOI"])
+ soil_bot = f1_soildepth["DZSOI"].values
- #-- soil layer top
+ # -- soil layer top
soil_top = soil_bot[:-1]
- soil_top = np.insert(soil_top,0, 0)
+ soil_top = np.insert(soil_top, 0, 0)
else:
- sys.exit('Cannot find soil structure file : '+surf_soildepth_file+
- 'for the surface dataset.')
+ sys.exit(
+ "Cannot find soil structure file : "
+ + surf_soildepth_file
+ + "for the surface dataset."
+ )
return soil_bot, soil_top
+
def update_metadata(nc, surf_file, neon_file, zb_flag):
"""
Function for updating modified surface dataset
@@ -279,21 +347,20 @@ def update_metadata(nc, surf_file, neon_file, zb_flag):
today = date.today()
today_string = today.strftime("%Y-%m-%d")
- nc.attrs['Updated_on'] = today_string
- nc.attrs['Updated_by'] = myname
- nc.attrs['Updated_with'] = os.path.abspath(__file__)
- nc.attrs['Updated_from'] = surf_file
- nc.attrs['Updated_using'] = neon_file
+ nc.attrs["Updated_on"] = today_string
+ nc.attrs["Updated_by"] = myname
+ nc.attrs["Updated_with"] = os.path.abspath(__file__)
+ nc.attrs["Updated_from"] = surf_file
+ nc.attrs["Updated_using"] = neon_file
if zb_flag:
- nc.attrs['Updated_fields'] = "PCT_CLAY, PCT_SAND, ORGANIC, zbedrock"
- #nc.attrs['Updated_fields'] = ['PCT_CLAY','PCT_SAND','ORGANIC','zbedrock']
+ nc.attrs["Updated_fields"] = "PCT_CLAY, PCT_SAND, ORGANIC, zbedrock"
else:
- nc.attrs['Updated_fields'] = "PCT_CLAY, PCT_SAND, ORGANIC"
- # nc.attrs['Updated_fields'] = ['PCT_CLAY','PCT_SAND','ORGANIC']
+ nc.attrs["Updated_fields"] = "PCT_CLAY, PCT_SAND, ORGANIC"
return nc
-def update_time_tag (fname_in):
+
+def update_time_tag(fname_in):
"""
Function for updating time tag on surface dataset
files.
@@ -315,13 +382,14 @@ def update_time_tag (fname_in):
basename = os.path.basename(fname_in)
cend = -10
- if ( basename[cend] == "c" ):
- cend = cend - 1
- if ( (basename[cend] != ".") and (basename[cend] != "_") ):
- sys.exit( "Trouble figuring out where to add tag to filename:"+fname_in )
+ if basename[cend] == "c":
+ cend = cend - 1
+ if (basename[cend] != ".") and (basename[cend] != "_"):
+ sys.exit("Trouble figuring out where to add tag to filename:" + fname_in)
+
+ fname_out = basename[:cend] + "_" + "c" + today_string + ".nc"
+ return fname_out
- fname_out = basename[:cend]+"_"+"c"+today_string+".nc"
- return(fname_out)
def sort_print_soil_layers(obs_bot, soil_bot):
"""
@@ -329,58 +397,54 @@ def sort_print_soil_layers(obs_bot, soil_bot):
original surface dataset and neon dataset.
Args:
- obs_bot : array of neon soil layers bottom depths
+ obs_bot : array of neon soil layers bottom depths
soil_bot : array of soil layers bottom depths
"""
- obs_bot_df = pd.DataFrame({'depth':obs_bot,'type':"obs"})
- soil_bot_df = pd.DataFrame({'depth':soil_bot,'type':"sfc"})
- depth_df = pd.concat([obs_bot_df,soil_bot_df])
+ obs_bot_df = pd.DataFrame({"depth": obs_bot, "type": "obs"})
+ soil_bot_df = pd.DataFrame({"depth": soil_bot, "type": "sfc"})
+ depth_df = pd.concat([obs_bot_df, soil_bot_df])
- depth_df = depth_df.sort_values('depth')
+ depth_df = depth_df.sort_values("depth")
- space = ' '
- print ("================================",
- "================================")
+ space = " "
+ print("================================", "================================")
- print (" Neon data soil structure: " ,
- " Surface data soil structure: ")
+ print(" Neon data soil structure: ", " Surface data soil structure: ")
- print ("================================",
- "================================")
+ print("================================", "================================")
for index, row in depth_df.iterrows():
- if row['type']=="obs":
- print ("-------------",
- "{0:.3f}".format(row['depth']),
- "------------")
+ if row["type"] == "obs":
+ print("-------------", "{0:.3f}".format(row["depth"]), "------------")
else:
- print (33*space+
- "-------------",
- "{0:.3f}".format(row['depth']),
- "-----------")
+ print(
+ 33 * space + "-------------",
+ "{0:.3f}".format(row["depth"]),
+ "-----------",
+ )
+
+ print("--------------------------------" + "--------------------------------")
- print ("--------------------------------"+
- "--------------------------------")
def check_neon_time():
"""
A function to download and parse neon listing file.
+
+ Returns:
+ dict_out (str) :
+ dictionary of *_surfaceData.csv files with the last modified
"""
- listing_file = 'listing.csv'
- url = 'https://neon-ncar.s3.data.neonscience.org/listing.csv'
+ listing_file = "listing.csv"
+ url = "https://neon-ncar.s3.data.neonscience.org/listing.csv"
download_file(url, listing_file)
df = pd.read_csv(listing_file)
- df = df[df['object'].str.contains("_surfaceData.csv")]
- #df=df.join(df['object'].str.split("/", expand=True))
- dict_out = dict(zip(df['object'],df['last_modified']))
- print (dict_out)
- #df_out = df[['object','6','last_modified']]
- #print (df['last_modified'])
- #print (df_out)
- #print (df['last_modified'].to_datetime())
+ df = df[df["object"].str.contains("_surfaceData.csv")]
+ # df=df.join(df['object'].str.split("/", expand=True))
+ dict_out = dict(zip(df["object"], df["last_modified"]))
+ print(dict_out)
return dict_out
@@ -388,88 +452,114 @@ def download_file(url, fname):
"""
Function to download a file.
Args:
- url (str):
+ url (str):
url of the file for downloading
- fname (str) :
+ fname (str) :
file name to save the downloaded file.
"""
response = requests.get(url)
- with open(fname, 'wb') as f:
+ with open(fname, "wb") as f:
f.write(response.content)
- #-- Check if download status_code
+ # -- Check if download status_code
if response.status_code == 200:
- print('Download finished successfully for', fname,'.')
+ print("Download finished successfully for", fname, ".")
elif response.status_code == 404:
- print('File '+fname+'was not available on the neon server:'+ url)
+ print("File " + fname + "was not available on the neon server:" + url)
+
+
+def fill_interpolate(f2, var, method):
+ """
+ Function to interpolate a variable in a
+ xarray dataset a specific method
+ """
+ print("=====================================")
+ print("Filling in ", var, "with interpolation (method =" + method + ").")
+ print("Variable before filling : ")
+ print(f2[var])
+
+ tmp_df = pd.DataFrame(f2[var].values.ravel())
+
+ tmp_df = tmp_df.interpolate(method=method, limit_direction="both")
+ # tmp_df = tmp_df.interpolate(method ='spline',order = 2, limit_direction ='both')
+ # tmp_df = tmp_df.interpolate(method="pad", limit=5, limit_direction = 'forward')
+
+ tmp = tmp_df.to_numpy()
+
+ soil_levels = f2[var].size
+ for soil_lev in range(soil_levels):
+ f2[var][soil_lev] = tmp[soil_lev].reshape(1, 1)
+
+ print("Variable after filling : ")
+ print(f2[var])
+ print("=====================================")
def main():
args = get_parser().parse_args()
- #-- debugging option
+ # -- debugging option
if args.debug:
logging.basicConfig(level=logging.DEBUG)
file_time = check_neon_time()
- #-- specify site from which to extract data
- site_name=args.site_name
+ # -- specify site from which to extract data
+ site_name = args.site_name
- #-- Look for surface data
+ # -- Look for surface data
surf_dir = args.surf_dir
- surf_file = find_surffile (surf_dir, site_name)
+ surf_file = find_surffile(surf_dir, site_name)
- #-- directory structure
+ # -- directory structure
current_dir = os.getcwd()
- parent_dir = os.path.dirname(current_dir)
- clone_dir = os.path.abspath(os.path.join(__file__ ,"../../.."))
- neon_dir = os.path.join(clone_dir,"neon_surffiles")
- print("Present Directory", current_dir)
+ parent_dir = os.path.dirname(current_dir)
+ clone_dir = os.path.abspath(os.path.join(__file__, "../../.."))
+ neon_dir = os.path.join(clone_dir, "neon_surffiles")
+
+ print("Present Directory", current_dir)
- #-- download neon data if needed
+ # -- download neon data if needed
neon_file = get_neon(neon_dir, site_name)
- #-- Read neon data
- df = pd.read_csv (neon_file)
+ # -- Read neon data
+ df = pd.read_csv(neon_file)
# -- Read surface dataset files
- print (type(surf_file))
- print ("surf_file:", surf_file)
+ print("surf_file:", surf_file)
f1 = xr.open_dataset(surf_file)
# -- Find surface dataset soil depth information
- soil_bot, soil_top = find_soil_structure (surf_file)
+ soil_bot, soil_top = find_soil_structure(surf_file)
# -- Find surface dataset soil levels
- # TODO: how? NS uses metadata on file to find
+ # TODO: how? NS uses metadata on file to find
# soil strucure
# better suggestion by WW to write dzsoi to neon surface dataset
# This todo needs to go to the subset_data
# TODO Will: if I sum them up , are they 3.5? (m) YES
- print ("soil_top:", soil_top)
- print ("soil_bot:", soil_bot)
- print ("Sum of soil top depths :", sum(soil_top))
- print ("Sum of soil bottom depths :",sum(soil_bot))
+ print("soil_top:", soil_top)
+ print("soil_bot:", soil_bot)
+ print("Sum of soil top depths :", sum(soil_top))
+ print("Sum of soil bottom depths :", sum(soil_bot))
soil_top = np.cumsum(soil_top)
soil_bot = np.cumsum(soil_bot)
- soil_mid = 0.5*(soil_bot - soil_top)+soil_top
- #print ("Cumulative sum of soil bottom depths :", sum(soil_bot))
+ soil_mid = 0.5 * (soil_bot - soil_top) + soil_top
+ # print ("Cumulative sum of soil bottom depths :", sum(soil_bot))
- obs_top = df['biogeoTopDepth']/100
- obs_bot = df['biogeoBottomDepth']/100
+ obs_top = df["biogeoTopDepth"] / 100
+ obs_bot = df["biogeoBottomDepth"] / 100
# -- Mapping surface dataset and neon soil levels
- bins = df['biogeoTopDepth']/100
- bin_index = np.digitize(soil_mid, bins)-1
+ bins = df["biogeoTopDepth"] / 100
+ bin_index = np.digitize(soil_mid, bins) - 1
-
- '''
+ """
print ("================================")
print (" Neon data soil structure: ")
print ("================================")
@@ -491,66 +581,126 @@ def main():
print ("-------------",
"{0:.2f}".format(soil_bot[b]),
"-------------")
- '''
- #-- update fields with neon
- f2= f1
- soil_levels = f2['PCT_CLAY'].size
- for soil_lev in range(soil_levels):
- print ("--------------------------")
- print ("soil_lev:",soil_lev)
- print (df['clayTotal'][bin_index[soil_lev]])
- f2['PCT_CLAY'][soil_lev] = df['clayTotal'][bin_index[soil_lev]]
- f2['PCT_SAND'][soil_lev] = df['sandTotal'][bin_index[soil_lev]]
- bulk_den = df['bulkDensExclCoarseFrag'][bin_index[soil_lev]]
- carbon_tot = df['carbonTot'][bin_index[soil_lev]]
- #print ("carbon_tot:", carbon_tot)
- layer_depth = df['biogeoBottomDepth'][bin_index[soil_lev]] - df['biogeoTopDepth'][bin_index[soil_lev]]
- f2['ORGANIC'][soil_lev] = carbon_tot * bulk_den * 0.1 / layer_depth * 100 / 0.58
- print ("bin_index:", bin_index[soil_lev])
- print ("layer_depth:", layer_depth)
- print ("carbon_tot:",carbon_tot)
- print ("bulk_den:",bulk_den)
- print ("organic=carbon_tot*bulk_den*0.1/layer_depth * 100/0.58 ")
- print ("organic:", f2['ORGANIC'][soil_lev].values)
- print ("--------------------------")
-
- #TODO : max depth for neon sites from WW (zbedrock)
- # Update zbedrock if neon observation don't make it down to 2m depth
- # zbedrock = neon depth if neon does not make to 2m depth
+ """
+ # -- update fields with neon
+ f2 = f1
+ soil_levels = f2["PCT_CLAY"].size
+ for soil_lev in range(soil_levels):
+ print("--------------------------")
+ print("soil_lev:", soil_lev)
+ print(df["clayTotal"][bin_index[soil_lev]])
+ f2["PCT_CLAY"][soil_lev] = df["clayTotal"][bin_index[soil_lev]]
+ f2["PCT_SAND"][soil_lev] = df["sandTotal"][bin_index[soil_lev]]
+
+ bulk_den = df["bulkDensExclCoarseFrag"][bin_index[soil_lev]]
+ carbon_tot = df["carbonTot"][bin_index[soil_lev]]
+ estimated_oc = df["estimatedOC"][bin_index[soil_lev]]
+
+ # -- estimated_oc in neon data is rounded to the nearest integer.
+ # -- Check to make sure the rounded oc is not higher than carbon_tot.
+ # -- Use carbon_tot if estimated_oc is bigger than carbon_tot.
+
+ if estimated_oc > carbon_tot:
+ estimated_oc = carbon_tot
+
+ layer_depth = (
+ df["biogeoBottomDepth"][bin_index[soil_lev]]
+ - df["biogeoTopDepth"][bin_index[soil_lev]]
+ )
+
+ # f2["ORGANIC"][soil_lev] = estimated_oc * bulk_den / 0.58
+
+ # -- after adding caco3 by NEON:
+ # -- if caco3 exists:
+ # -- inorganic = caco3/100.0869*12.0107
+ # -- organic = carbon_tot - inorganic
+ # -- else:
+ # -- oranigc = estimated_oc * bulk_den /0.58
+
+ caco3 = df["caco3Conc"][bin_index[soil_lev]]
+ inorganic = caco3 / 100.0869 * 12.0107
+ print("inorganic:", inorganic)
+
+ if not np.isnan(inorganic):
+ actual_oc = carbon_tot - inorganic
+ else:
+ actual_oc = estimated_oc
+
+ f2["ORGANIC"][soil_lev] = actual_oc * bulk_den / 0.58
+
+ print("~~~~~~~~~~~~~~~~~~~~~~~~")
+ print("inorganic:")
+ print("~~~~~~~~~~~~~~~~~~~~~~~~")
+ print(inorganic)
+ print("~~~~~~~~~~~~~~~~~~~~~~~~")
+
+ print("bin_index : ", bin_index[soil_lev])
+ print("layer_depth : ", layer_depth)
+ print("carbon_tot : ", carbon_tot)
+ print("estimated_oc : ", estimated_oc)
+ print("bulk_den : ", bulk_den)
+ print("organic :", f2["ORGANIC"][soil_lev].values)
+ print("--------------------------")
+
+ # -- Interpolate missing values
+ method = "linear"
+ fill_interpolate(f2, "PCT_CLAY", method)
+ fill_interpolate(f2, "PCT_SAND", method)
+ fill_interpolate(f2, "ORGANIC", method)
+
+ # -- Update zbedrock if neon observation does not make it down to 2m depth
rock_thresh = 2
zb_flag = False
- if (obs_bot.iloc[-1]> ',*command,'\n')
+
+ try:
+ subprocess.check_call(command, stdout=open(os.devnull, "w"), stderr=subprocess.STDOUT)
+
+ except subprocess.CalledProcessError as e:
+ #raise RuntimeError("command '{}' return with error (code {}): {}".format(e.cmd, e.returncode, e.output))
+ #print (e.ouput)
+ print (e)
+
+
+
+
+
+
+def main():
+
+ args = get_parser().parse_args()
+
+ if args.verbose:
+ logging.basicConfig(level=logging.DEBUG)
+
+
+ neon_sites = pd.read_csv('neon_sites_dompft.csv')
+
+
+ for i, row in tqdm.tqdm(neon_sites.iterrows()):
+ lat = row['Lat']
+ lon = row['Lon']
+ site = row['Site']
+ pft = row['pft']
+ print ("Now processing site :", site)
+ command = ['./subset_data.py','point','--lat',str(lat),'--lon',str(lon),'--site',site,'--dompft',str(pft),'--crop']
+ execute(command)
+
+ command = ['./modify_singlept_site_neon.py','--neon_site',site]
+ execute(command)
+
+if __name__ == "__main__":
+ main()
+
diff --git a/tools/site_and_regional/run_neon.py b/tools/site_and_regional/run_neon.py
index 17c457b61e..020bc2e8ee 100755
--- a/tools/site_and_regional/run_neon.py
+++ b/tools/site_and_regional/run_neon.py
@@ -19,7 +19,7 @@
2) Make the case for the specific neon site(s).
3) Make changes to the case, for:
a. AD spinup
- b. post-AD spinup
+ b. post-AD spinup
c. transient
#---------------
d. SASU or Matrix spinup
@@ -67,7 +67,7 @@
import subprocess
import pandas as pd
import glob
-from datetime import datetime
+import datetime
from getpass import getuser
# Get the ctsm util tools and then the cime tools.
@@ -106,18 +106,6 @@ def get_parser(args, description, valid_neon_sites):
default=["OSBS"],
nargs='+')
-# not used
-# parser.add_argument('--surf-dir',
-# help='''
-# Directory of single point surface dataset.
-# [default: %(default)s]
-# ''',
-# action="store",
-# dest="surf_dir",
-# type =str,
-# required=False,
-# default="/glade/scratch/"+myname+"/single_point/")
-
parser.add_argument('--base-case',
help='''
Root Directory of base case build
@@ -129,13 +117,13 @@ def get_parser(args, description, valid_neon_sites):
required=False,
default=None)
- parser.add_argument('--case-root',
+ parser.add_argument('--output-root',
help='''
- Root Directory of cases
+ Root output directory of cases
[default: %(default)s]
''',
action="store",
- dest="case_root",
+ dest="output_root",
type =str,
required=False,
default="CIME_OUTPUT_ROOT as defined in cime")
@@ -150,99 +138,83 @@ def get_parser(args, description, valid_neon_sites):
required = False,
default = False)
- subparsers = parser.add_subparsers (
- dest='run_type',
- help='Four different ways to run this script.')
-
- ad_parser = subparsers.add_parser ('ad',
- help=''' AD spin-up options ''')
-
- pad_parser = subparsers.add_parser ('postad',
- help=''' Post-AD spin-up options ''')
-
- tr_parser = subparsers.add_parser ('transient',
- help=''' Transient spin-up options ''')
+ parser.add_argument('--setup-only',
+ help='''
+ Only setup the requested cases, do not build or run
+ [default: %(default)s]
+ ''',
+ action="store_true",
+ dest="setup_only",
+ required = False,
+ default = False)
- sasu_parser = subparsers.add_parser ('sasu',
- help=''' Sasu spin-up options --not in CTSM yet''')
+ parser.add_argument('--rerun',
+ help='''
+ If the case exists but does not appear to be complete, restart it.
+ [default: %(default)s]
+ ''',
+ action="store_true",
+ dest="rerun",
+ required = False,
+ default = False)
- ad_parser.add_argument ('--ad-length',
+ parser.add_argument('--no-batch',
help='''
- How many years to run AD spin-up
+ Run locally, do not use batch queueing system (if defined for Machine)
[default: %(default)s]
''',
+ action="store_true",
+ dest="no_batch",
required = False,
- type = int,
- default = 200)
+ default = False)
- pad_parser.add_argument ('--postad-length',
+ parser.add_argument('--run-type',
+ help='''
+ Type of run to do
+ [default: %(default)s]
+ ''',
+ choices = ["ad", "postad", "transient", "sasu"],
+ default = "transient")
+
+ parser.add_argument ('--run-length',
help='''
- How many years to run in post-AD mode
+ How long to run (modified ISO 8601 duration)
[default: %(default)s]
''',
required = False,
- type = int,
- default = 100)
+ type = str,
+ default = '0Y')
- tr_parser.add_argument('--start-year',
+ parser.add_argument('--start-date',
help='''
- Start year for running CTSM simulation.
+ Start date for running CTSM simulation in ISO format.
[default: %(default)s]
''',
action="store",
- dest="start_year",
+ dest="start_date",
required = False,
- type = int,
- default = 2018)
+ type = datetime.date.fromisoformat,
+ default = datetime.datetime.strptime("2018-01-01",'%Y-%m-%d'))
- tr_parser.add_argument('--end-year',
+ parser.add_argument('--end-date',
help='''
- End year for running CTSM simulation.
+ End date for running CTSM simulation in ISO format.
[default: %(default)s]
''',
action="store",
- dest="end_year",
+ dest="end_date",
required = False,
- type = int,
- default = 2020)
-
- #parser.add_argument('--spinup',
- # help='''
- # AD spin-up
- # [default: %(default)s]
- # ''',
- # action="store_true",
- # dest="ad_flag",
- # required = False,
- # default = True)
- #parser.add_argument('--postad',
- # help='''
- # Post-AD spin-up
- # [default: %(default)s]
- # ''',
- # action="store_true",
- # dest="postad_flag",
- # required = False,
- # default = True)
- #parser.add_argument('--transient',
- # help='''
- # Transient
- # [default: %(default)s]
- # ''',
- # action="store_true",
- # dest="transient_flag",
- # required = False,
- # default = True)
-
- #parser.add_argument('--sasu','--matrix',
- # help='''
- # Matrix (SASU) spin-up
- # [default: %(default)s]
- # ''',
- # action="store_true",
- # dest="sasu_flag",
- # required = False,
- # default = False)
+ type = datetime.date.fromisoformat,
+ default = datetime.datetime.strptime("2021-01-01",'%Y-%m-%d'))
+
+ parser.add_argument('--run-from-postad',
+ help='''
+ For transient runs only - should we start from the postad spinup or finidat?
+ By default start from finidat, if this flag is used the postad run must be available.
+ ''',
+ action="store_true",
+ required = False,
+ default = False)
args = CIME.utils.parse_args_and_handle_standard_logging_options(args, parser)
@@ -254,20 +226,53 @@ def get_parser(args, description, valid_neon_sites):
if site not in valid_neon_sites:
raise ValueError("Invalid site name {}".format(site))
- if "CIME_OUTPUT_ROOT" in args.case_root:
- args.case_root = None
- if args.run_type:
- run_type = args.run_type
- else:
- run_type = "ad"
- if run_type == "ad":
- run_length = int(args.ad_length)
- elif run_type == "postad":
- run_length = int(args.postad_length)
- else:
- run_length = 0
+ if "CIME_OUTPUT_ROOT" in args.output_root:
+ args.output_root = None
- return neon_sites, args.case_root, run_type, args.overwrite, run_length, args.base_case_root
+ if args.run_length == '0Y':
+ if args.run_type == 'ad':
+ run_length = '200Y'
+ elif args.run_type == 'postad':
+ run_length = '50Y'
+ else:
+ # The transient run length is set by cdeps atm buildnml to the last date of the available tower data
+ # this value is not used
+ run_length = '4Y'
+
+ run_length = parse_isoduration(run_length)
+ base_case_root = None
+ if args.base_case_root:
+ base_case_root = os.path.abspath(args.base_case_root)
+
+ # Reduce output level for this script unless --debug or --verbose is provided on the command line
+ if not args.debug and not args.verbose:
+ root_logger = logging.getLogger()
+ root_logger.setLevel(logging.WARN)
+
+ return neon_sites, args.output_root, args.run_type, args.overwrite, run_length, base_case_root, args.run_from_postad, args.setup_only, args.no_batch, args.rerun
+
+def get_isosplit(s, split):
+ if split in s:
+ n, s = s.split(split)
+ else:
+ n = 0
+ return n, s
+
+def parse_isoduration(s):
+ '''
+ simple ISO 8601 duration parser, does not account for leap years and assumes 30 day months
+ '''
+ # Remove prefix
+ s = s.split('P')[-1]
+
+ # Step through letter dividers
+ years, s = get_isosplit(s, 'Y')
+ months, s = get_isosplit(s, 'M')
+ days, s = get_isosplit(s, 'D')
+
+ # Convert all to timedelta
+ dt = datetime.timedelta(days=int(days)+365*int(years)+30*int(months))
+ return int(dt.total_seconds()/86400)
class NeonSite :
"""
@@ -281,19 +286,20 @@ class NeonSite :
Methods
-------
"""
- def __init__(self, name, start_year, end_year, start_month, end_month):
+ def __init__(self, name, start_year, end_year, start_month, end_month, finidat):
self.name = name
self.start_year= int(start_year)
self.end_year = int(end_year)
self.start_month = int(start_month)
self.end_month = int(end_month)
self.cesmroot = path_to_ctsm_root()
+ self.finidat = finidat
def __str__(self):
return str(self.__class__) + '\n' + '\n'.join((str(item) + ' = '
for item in (self.__dict__)))
- def build_base_case(self, cesmroot, case_root, res, compset, overwrite):
+ def build_base_case(self, cesmroot, output_root, res, compset, overwrite=False, setup_only=False):
"""
Function for building a base_case to clone.
To spend less time on building ctsm for the neon cases,
@@ -311,77 +317,99 @@ def build_base_case(self, cesmroot, case_root, res, compset, overwrite):
overwrite (bool) :
Flag to overwrite the case if exists
"""
- logger.info("---- building a base case -------")
- self.base_case_root = case_root
+ print("---- building a base case -------")
+ self.base_case_root = output_root
user_mods_dirs = [os.path.join(cesmroot,"cime_config","usermods_dirs","NEON",self.name)]
- if case_root:
- case_path = os.path.join(case_root,self.name)
- logger.info ('case_root : {}'.format(case_root))
- else:
- case_path = self.name
+ if not output_root:
+ output_root = os.getcwd()
+ case_path = os.path.join(output_root,self.name)
logger.info ('base_case_name : {}'.format(self.name))
logger.info ('user_mods_dir : {}'.format(user_mods_dirs[0]))
if overwrite and os.path.isdir(case_path):
- logger.info ("Removing the existing case at: {}".format(case_path))
+ print ("Removing the existing case at: {}".format(case_path))
shutil.rmtree(case_path)
-
+
with Case(case_path, read_only=False) as case:
if not os.path.isdir(case_path):
- logger.info("---- creating a base case -------")
+ print("---- creating a base case -------")
- case.create(case_path, cesmroot, compset, res, mpilib="mpi-serial",
- run_unsupported=True, answer="r",
+ case.create(case_path, cesmroot, compset, res,
+ run_unsupported=True, answer="r",output_root=output_root,
user_mods_dirs = user_mods_dirs, driver="nuopc")
- logger.info("---- base case created ------")
+ print("---- base case created ------")
#--change any config for base_case:
#case.set_value("RUN_TYPE","startup")
- logger.info("---- base case setup ------")
+ print("---- base case setup ------")
case.case_setup()
else:
case.case_setup(reset=True)
+ case_path = case.get_value("CASEROOT")
+
+ if setup_only:
+ return case_path
- logger.info("---- base case build ------")
+ print("---- base case build ------")
# always walk through the build process to make sure it's up to date.
t0 = time.time()
build.case_build(case_path, case=case)
t1 = time.time()
total = t1-t0
- logger.info ("Time required to building the base case: {} s.".format(total))
+ print ("Time required to building the base case: {} s.".format(total))
# update case_path to be the full path to the base case
- case_path = case.get_value("CASEROOT")
return case_path
def diff_month(self):
- d1 = datetime(self.end_year,self.end_month, 1)
- d2 = datetime(self.start_year, self.start_month, 1)
+ d1 = datetime.datetime(self.end_year,self.end_month, 1)
+ d2 = datetime.datetime(self.start_year, self.start_month, 1)
return (d1.year - d2.year) * 12 + d1.month - d2.month
- def run_case(self, base_case_root, run_type, run_length, overwrite=False):
+ def run_case(self, base_case_root, run_type, run_length, overwrite=False, setup_only=False, no_batch=False, rerun=False):
user_mods_dirs = [os.path.join(self.cesmroot,"cime_config","usermods_dirs","NEON",self.name)]
expect(os.path.isdir(base_case_root), "Error base case does not exist in {}".format(base_case_root))
case_root = os.path.abspath(os.path.join(base_case_root,"..", self.name+"."+run_type))
- if os.path.isdir(case_root) and overwrite:
- logger.info("---- removing the existing case -------")
- shutil.rmtree(case_root)
+ rundir = None
+ if os.path.isdir(case_root):
+ if overwrite:
+ print("---- removing the existing case -------")
+ shutil.rmtree(case_root)
+ elif rerun:
+ with Case(case_root, read_only=False) as case:
+ rundir = case.get_value("RUNDIR")
+ if os.path.isfile(os.path.join(rundir,"ESMF_Profile.summary")):
+ print("Case {} appears to be complete, not rerunning.".format(case_root))
+ elif not setup_only:
+ print("Resubmitting case {}".format(case_root))
+ case.submit(no_batch=no_batch)
+ return
+ else:
+ logger.warning("Case already exists in {}, not overwritting.".format(case_root))
+ return
+
+ if run_type == "postad":
+ adcase_root = case_root.replace('.postad','.ad')
+ if not os.path.isdir(adcase_root):
+ logger.warning("postad requested but no ad case found in {}".format(adcase_root))
+ return
+
if not os.path.isdir(case_root):
# read_only = False should not be required here
with Case(base_case_root, read_only=False) as basecase:
- logger.info("---- cloning the base case in {}".format(case_root))
+ print("---- cloning the base case in {}".format(case_root))
basecase.create_clone(case_root, keepexe=True, user_mods_dirs=user_mods_dirs)
-
with Case(case_root, read_only=False) as case:
- case.set_value("STOP_OPTION", "nyears")
+ # in order to avoid the complication of leap years we always set the run_length in units of days.
+ case.set_value("STOP_OPTION", "ndays")
case.set_value("STOP_N", run_length)
- case.set_value("REST_N", 100)
+ case.set_value("REST_OPTION","end")
case.set_value("CONTINUE_RUN", False)
if run_type == "ad":
@@ -398,13 +426,17 @@ def run_case(self, base_case_root, run_type, run_length, overwrite=False):
self.set_ref_case(case)
if run_type == "transient":
- self.set_ref_case(case)
+ if self.finidat:
+ case.set_value("RUN_TYPE","startup")
+ else:
+ if not self.set_ref_case(case):
+ return
case.set_value("STOP_OPTION","nmonths")
case.set_value("STOP_N", self.diff_month())
- case.set_value("REST_N", "12")
case.set_value("DATM_YR_ALIGN",self.start_year)
case.set_value("DATM_YR_START",self.start_year)
case.set_value("DATM_YR_END",self.end_year)
+ case.set_value("CALENDAR","GREGORIAN")
else:
# for the spinup we want the start and end on year boundaries
if self.start_month == 1:
@@ -418,11 +450,16 @@ def run_case(self, base_case_root, run_type, run_length, overwrite=False):
else:
case.set_value("DATM_YR_END",self.end_year-1)
-
- self.modify_user_nl(case_root, run_type)
+ if not rundir:
+ rundir = case.get_value("RUNDIR")
+
+ self.modify_user_nl(case_root, run_type, rundir)
case.create_namelists()
- case.submit()
+ # explicitly run check_input_data
+ case.check_all_input_data()
+ if not setup_only:
+ case.submit(no_batch=no_batch)
def set_ref_case(self, case):
rundir = case.get_value("RUNDIR")
@@ -433,31 +470,45 @@ def set_ref_case(self, case):
else:
ref_case_root = case_root.replace(".transient",".postad")
root = ".postad"
- expect(os.path.isdir(ref_case_root), "ERROR: spinup must be completed first, could not find directory {}".format(ref_case_root))
+ if not os.path.isdir(ref_case_root):
+ logger.warning("ERROR: spinup must be completed first, could not find directory {}".format(ref_case_root))
+ return False
+
with Case(ref_case_root) as refcase:
refrundir = refcase.get_value("RUNDIR")
case.set_value("RUN_REFDIR", refrundir)
case.set_value("RUN_REFCASE", os.path.basename(ref_case_root))
- for reffile in glob.iglob(refrundir + "/{}{}.*.nc".format(self.name, root)):
+ refdate = None
+ for reffile in glob.iglob(refrundir + "/{}{}.clm2.r.*.nc".format(self.name, root)):
m = re.search("(\d\d\d\d-\d\d-\d\d)-\d\d\d\d\d.nc", reffile)
if m:
refdate = m.group(1)
symlink_force(reffile, os.path.join(rundir,os.path.basename(reffile)))
+ logger.info("Found refdate of {}".format(refdate))
+ if not refdate:
+ logger.warning("Could not find refcase for {}".format(case_root))
+ return False
+
for rpfile in glob.iglob(refrundir + "/rpointer*"):
safe_copy(rpfile, rundir)
if not os.path.isdir(os.path.join(rundir, "inputdata")) and os.path.isdir(os.path.join(refrundir,"inputdata")):
symlink_force(os.path.join(refrundir,"inputdata"),os.path.join(rundir,"inputdata"))
+
+
case.set_value("RUN_REFDATE", refdate)
if case_root.endswith(".postad"):
case.set_value("RUN_STARTDATE", refdate)
else:
case.set_value("RUN_STARTDATE", "{yr:04d}-{mo:02d}-01".format(yr=self.start_year, mo=self.start_month))
-
+ return True
- def modify_user_nl(self, case_root, run_type):
+ def modify_user_nl(self, case_root, run_type, rundir):
user_nl_fname = os.path.join(case_root, "user_nl_clm")
-
- if run_type != "transient":
+ user_nl_lines = None
+ if run_type == "transient":
+ if self.finidat:
+ user_nl_lines = ["finidat = '{}/inputdata/lnd/ctsm/initdata/{}'".format(rundir,self.finidat)]
+ else:
user_nl_lines = [
"hist_fincl2 = ''",
"hist_mfilt = 20",
@@ -465,9 +516,10 @@ def modify_user_nl(self, case_root, run_type):
"hist_empty_htapes = .true.",
"hist_fincl1 = 'TOTECOSYSC', 'TOTECOSYSN', 'TOTSOMC', 'TOTSOMN', 'TOTVEGC', 'TOTVEGN', 'TLAI', 'GPP', 'CPOOL', 'NPP', 'TWS', 'H2OSNO'"]
- with open(user_nl_fname, "a") as fd:
- for line in user_nl_lines:
- fd.write("{}\n".format(line))
+ if user_nl_lines:
+ with open(user_nl_fname, "a") as fd:
+ for line in user_nl_lines:
+ fd.write("{}\n".format(line))
@@ -503,8 +555,9 @@ def parse_neon_listing(listing_file, valid_neon_sites):
df = pd.read_csv(listing_file)
- #-- TODO: do we want to check for v2 in future?
-
+ # check for finidat files for transient run
+ finidatlist = df[df['object'].str.contains("lnd/ctsm")]
+
#-- filter lines with atm/cdep
df = df[df['object'].str.contains("atm/cdeps/")]
@@ -544,8 +597,12 @@ def parse_neon_listing(listing_file, valid_neon_sites):
logger.debug ('end_year={}'.format(end_year))
logger.debug ('start_month={}'.format(start_month))
logger.debug ('end_month={}'.format(end_month))
-
- neon_site = NeonSite(site_name, start_year, end_year, start_month, end_month)
+ finidat = None
+ for line in finidatlist['object']:
+ if site_name in line:
+ finidat = line.split(',')[0].split('/')[-1]
+
+ neon_site = NeonSite(site_name, start_year, end_year, start_month, end_month, finidat)
logger.debug (neon_site)
available_list.append(neon_site)
@@ -577,14 +634,14 @@ def main(description):
cesmroot = path_to_ctsm_root()
# Get the list of supported neon sites from usermods
valid_neon_sites = glob.glob(os.path.join(cesmroot,"cime_config","usermods_dirs","NEON","[!d]*"))
- valid_neon_sites = [v.split('/')[-1] for v in valid_neon_sites]
-
- site_list, case_root, run_type, overwrite, run_length, base_case_root = get_parser(sys.argv, description, valid_neon_sites)
+ valid_neon_sites = sorted([v.split('/')[-1] for v in valid_neon_sites])
- logger.debug ("case_root : "+ case_root)
+ site_list, output_root, run_type, overwrite, run_length, base_case_root, run_from_postad, setup_only, no_batch, rerun = get_parser(sys.argv, description, valid_neon_sites)
- if not os.path.exists(case_root):
- os.makedirs(case_root)
+ if output_root:
+ logger.debug ("output_root : "+ output_root)
+ if not os.path.exists(output_root):
+ os.makedirs(output_root)
#-- check neon listing file for available data:
available_list = check_neon_listing(valid_neon_sites)
@@ -600,12 +657,14 @@ def main(description):
for neon_site in available_list:
if neon_site.name in site_list:
+ if run_from_postad:
+ neon_site.finidat = None
if not base_case_root:
- base_case_root = neon_site.build_base_case(cesmroot, case_root, res,
- compset, overwrite)
+ base_case_root = neon_site.build_base_case(cesmroot, output_root, res,
+ compset, overwrite, setup_only)
logger.info ("-----------------------------------")
logger.info ("Running CTSM for neon site : {}".format(neon_site.name))
- neon_site.run_case(base_case_root, run_type, run_length, overwrite)
+ neon_site.run_case(base_case_root, run_type, run_length, overwrite, setup_only, no_batch, rerun)
if __name__ == "__main__":
main(__doc__)
diff --git a/tools/site_and_regional/subset_data.py b/tools/site_and_regional/subset_data.py
index 61c3fe0cf2..a649c657b9 100755
--- a/tools/site_and_regional/subset_data.py
+++ b/tools/site_and_regional/subset_data.py
@@ -1,4 +1,4 @@
-#! /usr/bin/env python3
+#! /usr/bin/env python
"""
|------------------------------------------------------------------|
|--------------------- Instructions -----------------------------|
@@ -178,6 +178,33 @@ def get_parser():
required = False,
type = int,
default = 2014)
+ pt_parser.add_argument('--datm_from_tower',
+ help='Flag for creating DATM forcing data at single point for a tower data. [default: %(default)s]',
+ action="store",
+ dest="datm_tower",
+ type = str2bool,
+ nargs = '?',
+ const = True,
+ required = False,
+ default = False)
+ pt_parser.add_argument('--create_user_mods',
+ help='Flag for creating user mods directory . [default: %(default)s]',
+ action="store",
+ dest="datm_tower",
+ type = str2bool,
+ nargs = '?',
+ const = True,
+ required = False,
+ default = False)
+ pt_parser.add_argument('--user_mods_dir',
+ help='Flag for creating user mods directory . [default: %(default)s]',
+ action="store",
+ dest="user_mod_dir",
+ type = str,
+ nargs = '?',
+ const = True,
+ required = False,
+ default = False)
pt_parser.add_argument('--crop',
help='Create datasets using the extensive list of prognostic crop types. [default: %(default)s]',
action="store_true",
@@ -369,9 +396,9 @@ def plon_type(x):
"""
x = float(x)
if (-180 < x) and (x < 0):
- print ("lon is :", lon)
+ print ("lon is :", x)
x= x%360
- print ("after modulo lon is :", lon)
+ print ("after modulo lon is :", x)
if (x < 0) or (x > 360):
raise argparse.ArgumentTypeError("ERROR: Latitude of single point should be between 0 and 360 or -180 and 180.")
return x
@@ -380,11 +407,7 @@ def get_git_sha():
"""
Returns Git short SHA for the currect directory.
"""
- try:
- sha = subprocess.check_output(['git', 'rev-parse', '--short', 'HEAD']).strip().decode()
- except subprocess.CalledProcessError:
- sha = "NOT-A-GIT-REPOSITORY"
- return sha
+ return subprocess.check_output(['git', 'rev-parse', '--short', 'HEAD']).strip().decode()
class BaseCase :
"""
@@ -552,8 +575,11 @@ def create_fileout_name( filename,tag):
items = basename.split('_')
today = date.today()
today_string = today.strftime("%y%m%d")
+ print (items[-1])
+ #new_string = items[0]+"_"+items[2]+"_"+items[3]+"_"+ items[4] \
+ # +"_"+items[5]+"_"+items[6]+"_"+tag+"_c"+today_string+".nc"
new_string = items[0]+"_"+items[2]+"_"+items[3]+"_"+ items[4] \
- +"_"+items[5]+"_"+items[6]+"_"+tag+"_c"+today_string+".nc"
+ +"_"+items[5]+"_"+tag+"_c"+today_string+".nc"
return new_string
def create_domain_at_point (self):
@@ -622,7 +648,9 @@ def create_surfdata_at_point(self):
# modify surface data properties
if self.overwrite_single_pft:
f3['PCT_NAT_PFT'][:,:,:] = 0
- f3['PCT_NAT_PFT'][:,:,self.dominant_pft] = 100
+ if (self.dominant_pft <16):
+ f3['PCT_NAT_PFT'][:,:,self.dominant_pft] = 100
+ #else:@@@
if self.zero_nonveg_landunits:
f3['PCT_NATVEG'][:,:] = 100
f3['PCT_CROP'][:,:] = 0
@@ -638,6 +666,14 @@ def create_surfdata_at_point(self):
# specify dimension order
#f3 = f3.transpose(u'time', u'cft', u'natpft', u'lsmlat', u'lsmlon')
f3 = f3.transpose(u'time', u'cft', u'lsmpft', u'natpft', u'nglcec', u'nglcecp1', u'nlevsoi', u'nlevurb', u'numrad', u'numurbl', 'lsmlat', 'lsmlon')
+
+ #update lsmlat and lsmlon to match site specific instead of the nearest point
+ #f3['lon']= self.plon
+ #f3['lat']= self.plat
+ f3['lsmlon']= np.atleast_1d(self.plon)
+ f3['lsmlat']= np.atleast_1d(self.plat)
+ f3['LATIXY'][:,:]= self.plat
+ f3['LONGXY'][:,:]= self.plon
#update attributes
self.update_metadata(f3)
@@ -789,7 +825,7 @@ def create_surfdata_at_reg(self):
f3.attrs['Created_from'] = self.fsurf_in
# mode 'w' overwrites file
- f3.to_netcdf(path=self.fsurf_out, mode='w')
+ f3.to_netcdf(path=self.fsurf_out, mode='w', format='NETCDF3_64BIT')
print('created file (fsurf_out)'+self.fsurf_out)
#f1.close();
f2.close(); f3.close()