diff --git a/Externals.cfg b/Externals.cfg index d3ee5df893..10963cfd20 100644 --- a/Externals.cfg +++ b/Externals.cfg @@ -41,7 +41,7 @@ tag = cime6.0.11 required = True [cmeps] -tag = cmeps0.13.40 +tag = cmeps0.13.43 protocol = git repo_url = https://github.com/ESCOMP/CMEPS.git local_path = components/cmeps @@ -56,14 +56,14 @@ externals = Externals_CDEPS.cfg required = True [cpl7] -tag = cpl7.0.5 +tag = cpl7.0.7 protocol = git repo_url = https://github.com/ESCOMP/CESM_CPL7andDataComps local_path = components/cpl7 required = True [share] -tag = share1.0.8 +tag = share1.0.10 protocol = git repo_url = https://github.com/ESCOMP/CESM_share local_path = share diff --git a/bld/CLMBuildNamelist.pm b/bld/CLMBuildNamelist.pm index d9417ca8b7..f105491d18 100755 --- a/bld/CLMBuildNamelist.pm +++ b/bld/CLMBuildNamelist.pm @@ -4363,7 +4363,7 @@ sub check_input_files { my $pathname = $nl->get_variable_value($group, $var); # Need to strip the quotes $pathname =~ s/['"]//g; - + next if ($pathname =~ /UNSET$/); if ($input_pathname_type eq 'abs') { if ($inputdata_rootdir) { if ( $pathname !~ /^\s*$/ ) { # If pathname isn't blank or null diff --git a/bld/namelist_files/namelist_defaults_ctsm.xml b/bld/namelist_files/namelist_defaults_ctsm.xml index cd0da15c89..a61d66360b 100644 --- a/bld/namelist_files/namelist_defaults_ctsm.xml +++ b/bld/namelist_files/namelist_defaults_ctsm.xml @@ -1812,21 +1812,21 @@ lnd/clm2/surfdata_map/release-clm5.0.30/surfdata_ne0np4.CONUS.ne30x8_hist_78pfts lnd/clm2/mappingdata/maps/1x1_brazil/map_0.125x0.125_nomask_to_1x1_brazil_nomask_aave_da_c200206.nc +>lnd/clm2/mappingdata/maps/1x1_brazil/map_0.125x0.125_nomask_to_1x1_brazil_nomask_aave_da_c211212.nc lnd/clm2/mappingdata/maps/1x1_brazil/map_0.5x0.5_nomask_to_1x1_brazil_nomask_aave_da_c200206.nc +>lnd/clm2/mappingdata/maps/1x1_brazil/map_0.5x0.5_nomask_to_1x1_brazil_nomask_aave_da_c211212.nc lnd/clm2/mappingdata/maps/1x1_brazil/map_0.25x0.25_nomask_to_1x1_brazil_nomask_aave_da_c200309.nc +>lnd/clm2/mappingdata/maps/1x1_brazil/map_0.25x0.25_nomask_to_1x1_brazil_nomask_aave_da_c211212.nc lnd/clm2/mappingdata/maps/1x1_brazil/map_3x3min_nomask_to_1x1_brazil_nomask_aave_da_c200309.nc +>lnd/clm2/mappingdata/maps/1x1_brazil/map_3x3min_nomask_to_1x1_brazil_nomask_aave_da_c211212.nc lnd/clm2/mappingdata/maps/1x1_brazil/map_10x10min_nomask_to_1x1_brazil_nomask_aave_da_c200206.nc +>lnd/clm2/mappingdata/maps/1x1_brazil/map_10x10min_nomask_to_1x1_brazil_nomask_aave_da_c211212.nc lnd/clm2/mappingdata/maps/1x1_brazil/map_5x5min_nomask_to_1x1_brazil_nomask_aave_da_c200309.nc +>lnd/clm2/mappingdata/maps/1x1_brazil/map_5x5min_nomask_to_1x1_brazil_nomask_aave_da_c211212.nc lnd/clm2/mappingdata/maps/1x1_brazil/map_0.9x1.25_nomask_to_1x1_brazil_nomask_aave_da_c200206.nc +>lnd/clm2/mappingdata/maps/1x1_brazil/map_0.9x1.25_nomask_to_1x1_brazil_nomask_aave_da_c211212.nc lnd/clm2/mappingdata/maps/1x1_brazil/map_1km-merge-10min_HYDRO1K-merge-nomask_to_1x1_brazil_nomask_aave_da_c130403.nc +>lnd/clm2/mappingdata/maps/1x1_brazil/map_1km-merge-10min_HYDRO1K-merge-nomask_to_1x1_brazil_nomask_aave_da_c211212.nc diff --git a/bld/namelist_files/namelist_defaults_ctsm_tools.xml b/bld/namelist_files/namelist_defaults_ctsm_tools.xml index 1166e80c9c..ff309c6fc9 100644 --- a/bld/namelist_files/namelist_defaults_ctsm_tools.xml +++ b/bld/namelist_files/namelist_defaults_ctsm_tools.xml @@ -95,7 +95,7 @@ attributes from the config_cache.xml file (with keys converted to upper-case). lnd/clm2/mappingdata/grids/SCRIPgrid_0.125nldas2_nomask_c190328.nc -lnd/clm2/mappingdata/grids/SCRIPgrid_1x1pt_brazil_nomask_c110308.nc +lnd/clm2/mappingdata/grids/SCRIPgrid_1x1pt_brazil_nomask_c20211211.nc lnd/clm2/mappingdata/grids/SCRIPgrid_1x1pt_mexicocityMEX_nomask_c110308.nc lnd/clm2/mappingdata/grids/SCRIPgrid_1x1pt_numaIA_nomask_c110308.nc lnd/clm2/mappingdata/grids/SCRIPgrid_1x1pt_smallvilleIA_nomask_c110308.nc diff --git a/cime_config/usermods_dirs/NEON/GUAN/shell_commands b/cime_config/usermods_dirs/NEON/GUAN/shell_commands index 4d750b77f8..b2d1b32dbf 100644 --- a/cime_config/usermods_dirs/NEON/GUAN/shell_commands +++ b/cime_config/usermods_dirs/NEON/GUAN/shell_commands @@ -1,3 +1,4 @@ ./xmlchange NEONSITE=GUAN ./xmlchange PTS_LON=293.13112 ./xmlchange PTS_LAT=17.96882 +./xmlchange RUN_STARTDATE=2018-06-01 \ No newline at end of file diff --git a/cime_config/usermods_dirs/NEON/LAJA/shell_commands b/cime_config/usermods_dirs/NEON/LAJA/shell_commands index 330690c330..36f01cff81 100644 --- a/cime_config/usermods_dirs/NEON/LAJA/shell_commands +++ b/cime_config/usermods_dirs/NEON/LAJA/shell_commands @@ -1,3 +1,4 @@ ./xmlchange NEONSITE=LAJA ./xmlchange PTS_LON=292.92392 ./xmlchange PTS_LAT=18.02184 +./xmlchange RUN_STARTDATE=2018-05-01 \ No newline at end of file diff --git a/cime_config/usermods_dirs/NEON/SJER/shell_commands b/cime_config/usermods_dirs/NEON/SJER/shell_commands index 45de246989..3683443ec0 100644 --- a/cime_config/usermods_dirs/NEON/SJER/shell_commands +++ b/cime_config/usermods_dirs/NEON/SJER/shell_commands @@ -1,3 +1,4 @@ ./xmlchange NEONSITE=SJER ./xmlchange PTS_LON=240.267 ./xmlchange PTS_LAT=37.107117 +./xmlchange RUN_STARTDATE=2018-09-01 \ No newline at end of file diff --git a/cime_config/usermods_dirs/NEON/TEAK/shell_commands b/cime_config/usermods_dirs/NEON/TEAK/shell_commands index 53ebedc664..f3a9fd75ef 100644 --- a/cime_config/usermods_dirs/NEON/TEAK/shell_commands +++ b/cime_config/usermods_dirs/NEON/TEAK/shell_commands @@ -1,3 +1,5 @@ ./xmlchange NEONSITE=TEAK ./xmlchange PTS_LON=240.99424199999999 ./xmlchange PTS_LAT=37.006472 +# This site is missing data for first half of 2018 +./xmlchange RUN_STARTDATE=2018-06-01 \ No newline at end of file diff --git a/cime_config/usermods_dirs/NEON/YELL/shell_commands b/cime_config/usermods_dirs/NEON/YELL/shell_commands index a40ef81477..3924dff420 100644 --- a/cime_config/usermods_dirs/NEON/YELL/shell_commands +++ b/cime_config/usermods_dirs/NEON/YELL/shell_commands @@ -1,3 +1,4 @@ ./xmlchange NEONSITE=YELL ./xmlchange PTS_LON=249.45803999999998 ./xmlchange PTS_LAT=44.95597 +./xmlchange RUN_STARTDATE=2018-08-01 \ No newline at end of file diff --git a/cime_config/usermods_dirs/NEON/defaults/shell_commands b/cime_config/usermods_dirs/NEON/defaults/shell_commands index 2ebe1b4f86..f82278e4b5 100644 --- a/cime_config/usermods_dirs/NEON/defaults/shell_commands +++ b/cime_config/usermods_dirs/NEON/defaults/shell_commands @@ -3,4 +3,4 @@ ./xmlchange CLM_NML_USE_CASE=1850-2100_SSP3-7.0_transient ./xmlchange CCSM_CO2_PPMV=408.83 ./xmlchange DATM_PRESAERO=SSP3-7.0 -./xmlchange DATM_YR_ALIGN=2018,DATM_YR_END=2019,DATM_YR_START=2018 +./xmlchange DATM_YR_ALIGN=2018,DATM_YR_END=2020,DATM_YR_START=2018 diff --git a/cime_config/usermods_dirs/NEON/defaults/user_nl_clm b/cime_config/usermods_dirs/NEON/defaults/user_nl_clm index 742cd6f65e..eff648d4e5 100644 --- a/cime_config/usermods_dirs/NEON/defaults/user_nl_clm +++ b/cime_config/usermods_dirs/NEON/defaults/user_nl_clm @@ -19,17 +19,18 @@ !---------------------------------------------------------------------------------- flanduse_timeseries = ' ' ! This isn't needed for a non transient case, but will be once we start using transient compsets -fsurdat = "$DIN_LOC_ROOT/lnd/clm2/surfdata_map/NEON/surfdata_hist_78pfts_CMIP6_simyr2000_${NEONSITE}_c210720.nc" +fsurdat = "$DIN_LOC_ROOT/lnd/clm2/surfdata_map/NEON/surfdata_hist_78pfts_CMIP6_simyr2000_${NEONSITE}_c211102.nc" model_year_align_urbantv = 2018 stream_year_first_urbantv = 2018 -stream_year_last_urbantv = 2019 +stream_year_last_urbantv = 2020 stream_year_first_ndep = 2018 model_year_align_ndep = 2018 -stream_year_last_ndep = 2019 +stream_year_last_ndep = 2020 model_year_align_popdens = 2018 stream_year_first_popdens = 2018 -stream_year_last_popdens = 2019 +stream_year_last_popdens = 2020 stream_fldfilename_lightng = '$DIN_LOC_ROOT/atm/datm7/NASA_LIS/clmforc.Li_2016_climo1995-2013.360x720.lnfm_Total_NEONarea_c210625.nc' +stream_fldfilename_ndep = '$DIN_LOC_ROOT/lnd/clm2/ndepdata/fndep_clm_f09_g17.CMIP6-SSP3-7.0-WACCM_2018-2030_monthly_c210826.nc' ! h1 output stream hist_fincl2 = 'AR','ELAI','FCEV','FCTR','FGEV','FIRA','FSA','FSH','GPP','H2OSOI', 'HR','SNOW_DEPTH','TBOT','TSOI','SOILC_vr','FV','NET_NMIN_vr' diff --git a/cime_config/usermods_dirs/NEON/defaults/user_nl_datm_streams b/cime_config/usermods_dirs/NEON/defaults/user_nl_datm_streams new file mode 100644 index 0000000000..41ddbfa611 --- /dev/null +++ b/cime_config/usermods_dirs/NEON/defaults/user_nl_datm_streams @@ -0,0 +1,6 @@ +presaero.SSP3-7.0:datafiles = $DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/aero/aerodep_clm_SSP370_b.e21.BWSSP370cmip6.f09_g17.CMIP6-SSP3-7.0-WACCM.001_2018-2030_monthly_0.9x1.25_c210826.nc +presaero.SSP3-7.0:year_first=2018 +presaero.SSP3-7.0:year_last=2030 +presaero.SSP3-7.0:year_align=2018 +presaero.SSP3-7.0:dtlimit=30 + diff --git a/doc/ChangeLog b/doc/ChangeLog index 4a8050bfe1..8c17383973 100644 --- a/doc/ChangeLog +++ b/doc/ChangeLog @@ -1,4 +1,157 @@ =============================================================== +Tag name: ctsm5.1.dev067 +Originator(s): jedwards4b/negin513/erik (Erik Kluzek,UCAR/TSS,303-497-1326) +Date: Mon Dec 13 00:50:01 MST 2021 +One-line Summary: NEON UI update, externals updates, small miscellanouse fixes + +Purpose and description of changes +---------------------------------- + +Redo options list to remove positional arguments that were difficult to input correctly. +Transient runs now use run_type startup and get finidat from s3 server unless --run-from-postad option is used (or finidat is not +available). Use mpi instead of mpi-serial, this mod was recommended for container use. Add a new script neon_finidat_upload which +allows authorized users to upload finidat files to the s3 server. + +This includes the following changes to the script for updating the surface dataset at neon sites using available neon data (i.e. +modify_singlept_site_neon.py) to address ctsm/issues #1353, #1429, and neon/issue #44: + +Update Organic calculation to use the following equation based on discussions in +https://github.com/ESCOMP/CTSM/pull/1375/files#r669590971 : +ORGANIC = estimatedOC * bulkDensity / 0.58 + +Because estimatedOC is rounded to the nearest integer in neon data, it is sometimes bigger than carbonTot. Therefore, in cases where +estimatedOC > carbonTot, we use carbonTot instead of estimatedOC in the above equation. + +Previously, we had missing data on neon files for some neon soil layers (see Modified NEON surface datasets have errors #1429 +(comment)). Therefore, it caused some missing values in the updated dataset. Here, we interpolate to fill in the missing data using +different interpolation techniques. Finally, we chose to use linear interpolation to fill in missing data. + +This includes the scripts for modification of the surface dataset for neon sites to address #1429. +Specifically, the following has been addressed in this PR: + +Update the calculation of ORGANIC to use the new field (CaCO3) from NEON data. +For this calculation if CaCO3 data is available, we first calculate inorganic carbon by: +inorganic carbon = (caco3 /100.0869)*12.0107 +Next, we calculate organic carbon by subtracting inorganic carbon from the total carbon: +[organic carbon = carbon_tot - inorganic carbon] +If the CaCO3 is not available then the code uses carbonTot and estimatedOC by NEON. + +Discussed here (Modified NEON surface datasets have errors #1429 (comment)) +For the Ag sites (KONA and STER), it changes the PCT_NATVEG, PCT_CROP, and PCT_NAT_PFT to avoid the error that we previously had in +spin-up: surfrd_veg_all ERROR: sum of wt_nat_patch not 1.00000000000000 at nl= 1 sum is: 0.000000000000000E+000 + +Discussed here (Modified NEON surface datasets have errors #1429 (comment)) +There was a typo previously in the NEON data for ABBY sites caused by mix of sample measurements. Please note that this was updated +by hand once data was downloaded from NEON site. + +With recent versions of CIME, the LILAC build with a user-defined machine was broken for a couple of reasons. This fixes it. + +Fix mksurfdata_map for 1x1_brazil. Get tools testing working again. Increase skip_steps by 1, which is needed for a change in CAM +where balance checks need to occur after the radiation update now rather than before. glob changed for bsd_glob in perl MkDepends +for mksurfdata_map. + + +Significant changes to scientifically-supported configurations +-------------------------------------------------------------- + +Does this tag change answers significantly for any of the following physics configurations? +(Details of any changes will be given in the "Answer changes" section below.) + + [Put an [X] in the box for any configuration with significant answer changes.] + +[ ] clm5_1 + +[ ] clm5_0 + +[ ] ctsm5_0-nwp + +[ ] clm4_5 + + +Bugs fixed or introduced +------------------------ + +Issues fixed (include CTSM Issue #): + Fixes #1563 increase skip_steps for balance checks by one to permit new CAM physics re-ordering + Fixes #1550 In perl code replace glob with bsd_glob + Fixes #1574 Trouble with 1x1_brazil for mksurfdata_map because of negative longitude in SCRIP grid file + Fixes #1429 Modified NEON surface datasets have errors + Fixes #1353 Modify NEON surface data + Fixes #1492 Need to update LILAC build process to use cmake macros instead of config_compilers.xml + +Known bugs found since the previous tag (include issue #): + #1575 -- Build problem for mksurfdata tools testers + +Notes of particular relevance for users +--------------------------------------- + +Changes made to namelist defaults (e.g., changed parameter values): New 1x1_brazil SCRIP grid file and maps + Some NEON namelist settings changed. Last year is now 2020 + +Changes to the datasets (e.g., parameter, surface or initial files): New NEON surface datasets + +Notes of particular relevance for developers: +--------------------------------------------- + +Caveats for developers (e.g., code that is duplicated that requires double maintenance): + pyproject.toml file added to configure for black python formatter in python directory + +Changes to tests or testing: + Got tools testing working again. + + +Testing summary: regular tools +---------------- + [PASS means all tests PASS; OK means tests PASS other than expected fails.] + + build-namelist tests (if CLMBuildNamelist.pm has changed): + + cheyenne - PASS (47 compare tests fail because of changes to NEON sites) + + tools-tests (test/tools) (if tools have been changed): + + cheyenne - OK (1x1_brazil mksurfdata changes, run_neon and modify_subset fail as expected) + + python testing (if python code has changed; see instructions in python/README.md; document testing done): + + cheyenne - OK (new black checks do NOT pass as expected) + + regular tests (aux_clm: https://github.com/ESCOMP/CTSM/wiki/System-Testing-Guide#pre-merge-system-testing): + + cheyenne ---- PASS + izumi ------- OK + +If the tag used for baseline comparisons was NOT the previous tag, note that here: + + +Answer changes +-------------- + +Changes answers relative to baseline: No bit-for-bit (other than NEON tests because of updated namelists and surface dataset) + + Summarize any changes to answers, i.e., + - what code configurations: Only NEON sites + - what platforms/compilers: all + - nature of change: new surface datasets and settings + +Other details +------------- +List any externals directories updated (cime, rtm, mosart, cism, fates, etc.): + Update most externals to version in cesm2_3_alpha07c + cmeps to cmeps0.13.43 (version with channel depths) + cpl7 to cpl7.0.7 + share to share1.0.10 + +Pull Requests that document the changes (include PR ids): +(https://github.com/ESCOMP/ctsm/pull) + + PR #1467 -- Improve UI for NEON script + PR #1474 -- Script for modifying neon surface dataset -- updated (negin513) + PR #1539 -- Neon modify surfurface dataset (negin513) + PR #1571 -- Fix LILAC build with user-defined machine with latest CIME (billsacks) + +=============================================================== +=============================================================== Tag name: ctsm5.1.dev066 Originator(s): rgknox (Ryan Knox,,,) Date: Sat Dec 4 01:58:42 MST 2021 diff --git a/doc/ChangeSum b/doc/ChangeSum index 73281e5a9c..7742b1f18c 100644 --- a/doc/ChangeSum +++ b/doc/ChangeSum @@ -1,5 +1,6 @@ Tag Who Date Summary ============================================================================================================================ + ctsm5.1.dev067 jedwards 12/13/2021 NEON UI update, externals updates, small miscellanouse fixes ctsm5.1.dev066 rgknox 12/04/2021 API change with FATES to enable running means inside fates, includes passing in of model timestep ctsm5.1.dev065 glemieux 12/02/2021 Refactor static fire data input by moving variables into fire_base_type from cnveg_state_type ctsm5.1.dev064 afoster 11/29/2021 Updates to facilitate FATES history variable overhaul diff --git a/doc/source/lilac/obtaining-building-and-running/obtaining-and-building-ctsm.rst b/doc/source/lilac/obtaining-building-and-running/obtaining-and-building-ctsm.rst index a51dcfcf25..99cb908d28 100644 --- a/doc/source/lilac/obtaining-building-and-running/obtaining-and-building-ctsm.rst +++ b/doc/source/lilac/obtaining-building-and-running/obtaining-and-building-ctsm.rst @@ -41,7 +41,7 @@ On a machine that has *not* been ported to CIME, you will need to provide some a information. Run ``./lilac/build_ctsm -h`` for details, but the basic command will look like this:: - ./lilac/build_ctsm ~/ctsm_build_dir --os Darwin --compiler gnu --netcdf-path /usr/local --esmf-lib-path /Users/sacks/ESMF/esmf8.0.0/lib/libO/Darwin.gfortranclang.64.mpich3.default --max-mpitasks-per-node 4 --no-pnetcdf + ./lilac/build_ctsm ~/ctsm_build_dir --os Darwin --compiler gnu --netcdf-path /usr/local --esmf-mkfile-path /Users/sacks/ESMF/esmf8.0.0/lib/libO/Darwin.gfortranclang.64.mpich3.default/esmf.mk --max-mpitasks-per-node 4 --no-pnetcdf In both cases, you will then need to include the necessary information in the include and link lines of the atmosphere model's build. For a Makefile-based build, this can be done @@ -205,7 +205,7 @@ above`. The minimal amount of information needed is given by the following:: - ./lilac/build_ctsm /PATH/TO/CTSM/BUILD --os OS --compiler COMPILER --netcdf-path NETCDF_PATH --esmf-lib-path ESMF_LIB_PATH --max-mpitasks-per-node MAX_MPITASKS_PER_NODE --pnetcdf-path PNETCDF_PATH + ./lilac/build_ctsm /PATH/TO/CTSM/BUILD --os OS --compiler COMPILER --netcdf-path NETCDF_PATH --esmf-mkfile-path ESMF_MKFILE_PATH --max-mpitasks-per-node MAX_MPITASKS_PER_NODE --pnetcdf-path PNETCDF_PATH where you should fill in the capitalized arguments with appropriate values for your machine. Run ``./lilac/build_ctsm -h`` for details on these arguments, as well as documentation @@ -229,17 +229,16 @@ model performance. Example usage for a Mac (a simple case) is:: - ./lilac/build_ctsm ~/ctsm_build_dir --os Darwin --compiler gnu --netcdf-path /usr/local --esmf-lib-path /Users/sacks/ESMF/esmf8.0.0/lib/libO/Darwin.gfortranclang.64.mpich3.default --max-mpitasks-per-node 4 --no-pnetcdf + ./lilac/build_ctsm ~/ctsm_build_dir --os Darwin --compiler gnu --netcdf-path /usr/local --esmf-mkfile-path /Users/sacks/ESMF/esmf8.0.0/lib/libO/Darwin.gfortranclang.64.mpich3.default/esmf.mk --max-mpitasks-per-node 4 --no-pnetcdf Example usage for NCAR's ``cheyenne`` machine (a more complex case) is:: module purge - module load ncarenv/1.3 intel/19.0.5 esmf_libs mkl - module use /glade/work/himanshu/PROGS/modulefiles/esmfpkgs/intel/19.0.5 - module load esmf-8.1.0b14-ncdfio-mpt-O mpt/2.21 netcdf/4.7.3 pnetcdf/1.12.1 ncarcompilers/0.5.0 - module load python + module load ncarenv/1.3 python/3.7.9 cmake intel/19.1.1 esmf_libs mkl + module use /glade/p/cesmdata/cseg/PROGS/modulefiles/esmfpkgs/intel/19.1.1/ + module load esmf-8.2.0b23-ncdfio-mpt-O mpt/2.22 netcdf-mpi/4.8.0 pnetcdf/1.12.2 ncarcompilers/0.5.0 - ./lilac/build_ctsm /glade/scratch/$USER/ctsm_build_dir --os linux --compiler intel --netcdf-path '$ENV{NETCDF}' --pio-filesystem-hints gpfs --pnetcdf-path '$ENV{PNETCDF}' --esmf-lib-path '$ENV{ESMF_LIBDIR}' --max-mpitasks-per-node 36 --extra-cflags '-xCORE_AVX2 -no-fma' --extra-fflags '-xCORE_AVX2 -no-fma' + ./lilac/build_ctsm /glade/scratch/$USER/ctsm_build_dir --os linux --compiler intel --netcdf-path '$ENV{NETCDF}' --pio-filesystem-hints gpfs --pnetcdf-path '$ENV{PNETCDF}' --esmf-mkfile-path '$ENV{ESMFMKFILE}' --max-mpitasks-per-node 36 --extra-cflags '-xCORE_AVX2 -no-fma' --extra-fflags '-xCORE_AVX2 -no-fma' (It's better to use the :ref:`alternative process for a CIME-supported machine` in this case, but the above illustrates diff --git a/lilac/bld_templates/config_compilers_template.xml b/lilac/bld_templates/config_compilers_template.xml deleted file mode 100644 index 9fc3358408..0000000000 --- a/lilac/bld_templates/config_compilers_template.xml +++ /dev/null @@ -1,44 +0,0 @@ - - - - - - - - - - $GPTL_CPPDEFS - - - $NETCDF_PATH - - - $PIO_FILESYSTEM_HINTS - - - $PNETCDF_PATH - - $ESMF_LIBDIR - - - $EXTRA_CFLAGS - - - - $EXTRA_FFLAGS - - - - - diff --git a/lilac/bld_templates/config_machines_template.xml b/lilac/bld_templates/config_machines_template.xml index a197e02dfa..ccde849607 100644 --- a/lilac/bld_templates/config_machines_template.xml +++ b/lilac/bld_templates/config_machines_template.xml @@ -12,7 +12,7 @@ --> - + Temporary build information for a CTSM build @@ -111,5 +111,9 @@ --> + + $ESMF_MKFILE_PATH + + diff --git a/lilac/bld_templates/ctsm-build_template.cmake b/lilac/bld_templates/ctsm-build_template.cmake new file mode 100644 index 0000000000..a4dae78533 --- /dev/null +++ b/lilac/bld_templates/ctsm-build_template.cmake @@ -0,0 +1,23 @@ +# This, together with config_machines_template.xml, provides a machine port for building +# CTSM. +# +# If you are looking at the template file: Variable names prefixed with a dollar sign will +# be replaced with machine-specific values. A double dollar sign gets replaced with a +# single dollar sign, so something like $$MYVAR refers to the MYVAR cime variable. + +if (COMP_NAME STREQUAL gptl) + string(APPEND CPPDEFS " $GPTL_CPPDEFS") +endif() + +set(NETCDF_PATH "$NETCDF_PATH") + +# If PIO_FILESYSTEM_HINTS is provided, this will set a PIO_FILESYSTEM_HINTS variable; if +# not provided, this will just be a blank line. +$PIO_FILESYSTEM_HINTS + +# If PNETCDF_PATH is provided, this will set a PNETCDF_PATH variable; if not provided, +# this will just be a blank line. +$PNETCDF_PATH + +string(APPEND CFLAGS " $EXTRA_CFLAGS") +string(APPEND FFLAGS " $EXTRA_FFLAGS") diff --git a/python/Makefile b/python/Makefile index 6c7e1ab32c..4ea5fba85d 100644 --- a/python/Makefile +++ b/python/Makefile @@ -23,7 +23,7 @@ PYLINT_ARGS=-j 4 --rcfile=ctsm/.pylintrc PYLINT_SRC = \ ctsm -all: test lint +all: test lint black test: utest stest .PHONY: utest @@ -38,6 +38,12 @@ stest: FORCE lint: FORCE $(PYLINT) $(PYLINT_ARGS) $(PYLINT_SRC) +.PHONY: black +# Run black on all of the python files here and undeneath. +# Use the black configure file to explicitly set a few things and specifiy the exact files. +black: FORCE + black --check --config pyproject.toml . + .PHONY: clean clean: FORCE find . -name '*.pyc' -exec rm {} \; diff --git a/python/ctsm/.pylintrc b/python/ctsm/.pylintrc index 46c4837b6c..bc7ae54dd2 100644 --- a/python/ctsm/.pylintrc +++ b/python/ctsm/.pylintrc @@ -140,6 +140,7 @@ disable=print-statement, deprecated-sys-function, exception-escape, comprehension-escape, + C0330, # This is an option that the formatter "black" requires us to disable # --- default list is above here, our own list is below here --- # While pylint's recommendations to keep the number of arguments, local # variables and branches low is generally a good one, I don't want it to diff --git a/python/ctsm/lilac_build_ctsm.py b/python/ctsm/lilac_build_ctsm.py index 888008897a..be1fbf7350 100644 --- a/python/ctsm/lilac_build_ctsm.py +++ b/python/ctsm/lilac_build_ctsm.py @@ -18,7 +18,7 @@ # ======================================================================== # this matches the machine name in config_machines_template.xml -_MACH_NAME = 'ctsm_build' +_MACH_NAME = 'ctsm-build' # these are arbitrary, since we only use the case for its build, not any of the runtime # settings; they just need to be valid @@ -71,7 +71,7 @@ def main(cime_path): machine=args.machine, os_type=args.os, netcdf_path=args.netcdf_path, - esmf_lib_path=args.esmf_lib_path, + esmf_mkfile_path=args.esmf_mkfile_path, max_mpitasks_per_node=args.max_mpitasks_per_node, gmake=args.gmake, gmake_j=args.gmake_j, @@ -92,7 +92,7 @@ def build_ctsm(cime_path, machine=None, os_type=None, netcdf_path=None, - esmf_lib_path=None, + esmf_mkfile_path=None, max_mpitasks_per_node=None, gmake=None, gmake_j=None, @@ -117,7 +117,7 @@ def build_ctsm(cime_path, Must be given if machine isn't given; ignored if machine is given netcdf_path (str or None): path to NetCDF installation Must be given if machine isn't given; ignored if machine is given - esmf_lib_path (str or None): path to ESMF library directory + esmf_mkfile_path (str or None): path to esmf.mk file (typically within ESMF library directory) Must be given if machine isn't given; ignored if machine is given max_mpitasks_per_node (int or None): number of physical processors per shared-memory node Must be given if machine isn't given; ignored if machine is given @@ -153,7 +153,7 @@ def build_ctsm(cime_path, if machine is None: assert os_type is not None, 'with machine absent, os_type must be given' assert netcdf_path is not None, 'with machine absent, netcdf_path must be given' - assert esmf_lib_path is not None, 'with machine absent, esmf_lib_path must be given' + assert esmf_mkfile_path is not None, 'with machine absent, esmf_mkfile_path must be given' assert max_mpitasks_per_node is not None, ('with machine absent ' 'max_mpitasks_per_node must be given') os_type = _check_and_transform_os(os_type) @@ -161,7 +161,7 @@ def build_ctsm(cime_path, os_type=os_type, compiler=compiler, netcdf_path=netcdf_path, - esmf_lib_path=esmf_lib_path, + esmf_mkfile_path=esmf_mkfile_path, max_mpitasks_per_node=max_mpitasks_per_node, gmake=gmake, gmake_j=gmake_j, @@ -242,7 +242,7 @@ def _commandline_args(args_to_parse=None): For a fresh build with a machine that has NOT been ported to cime: - build_ctsm /path/to/nonexistent/directory --os OS --compiler COMPILER --netcdf-path NETCDF_PATH --esmf-lib-path ESMF_LIB_PATH --max-mpitasks-per-node MAX_MPITASKS_PER_NODE --pnetcdf-path PNETCDF_PATH + build_ctsm /path/to/nonexistent/directory --os OS --compiler COMPILER --netcdf-path NETCDF_PATH --esmf-mkfile-path ESMF_MKFILE_PATH --max-mpitasks-per-node MAX_MPITASKS_PER_NODE --pnetcdf-path PNETCDF_PATH If PNetCDF is not available, set --no-pnetcdf instead of --pnetcdf-path. @@ -346,10 +346,10 @@ def _commandline_args(args_to_parse=None): 'named lib, include, etc.)') new_machine_required_list.append('netcdf-path') - new_machine_required.add_argument('--esmf-lib-path', - help='Path to ESMF library directory\n' - 'This directory should include an esmf.mk file') - new_machine_required_list.append('esmf-lib-path') + new_machine_required.add_argument('--esmf-mkfile-path', + help='Path to esmf.mk file\n' + '(typically within ESMF library directory)') + new_machine_required_list.append('esmf-mkfile-path') new_machine_required.add_argument('--max-mpitasks-per-node', type=int, help='Number of physical processors per shared-memory node\n' @@ -499,7 +499,7 @@ def _fill_out_machine_files(build_dir, os_type, compiler, netcdf_path, - esmf_lib_path, + esmf_mkfile_path, max_mpitasks_per_node, gmake, gmake_j, @@ -512,7 +512,7 @@ def _fill_out_machine_files(build_dir, For documentation of args, see the documentation in the build_ctsm function """ - os.makedirs(os.path.join(build_dir, _MACHINE_CONFIG_DIRNAME)) + os.makedirs(os.path.join(build_dir, _MACHINE_CONFIG_DIRNAME, "cmake_macros")) # ------------------------------------------------------------------------ # Fill in config_machines.xml @@ -526,10 +526,11 @@ def _fill_out_machine_files(build_dir, 'CIME_OUTPUT_ROOT':build_dir, 'GMAKE':gmake, 'GMAKE_J':gmake_j, - 'MAX_MPITASKS_PER_NODE':max_mpitasks_per_node}) + 'MAX_MPITASKS_PER_NODE':max_mpitasks_per_node, + 'ESMF_MKFILE_PATH':esmf_mkfile_path}) # ------------------------------------------------------------------------ - # Fill in config_compilers.xml + # Fill in ctsm-build_template.cmake # ------------------------------------------------------------------------ if gptl_nano_timers: @@ -538,27 +539,26 @@ def _fill_out_machine_files(build_dir, gptl_cppdefs = '' if pio_filesystem_hints: - pio_filesystem_hints_tag = '{}'.format( + pio_filesystem_hints_addition = 'set(PIO_FILESYSTEM_HINTS "{}")'.format( pio_filesystem_hints) else: - pio_filesystem_hints_tag = '' + pio_filesystem_hints_addition = '' if pnetcdf_path: - pnetcdf_path_tag = '{}'.format( + pnetcdf_path_addition = 'set(PNETCDF_PATH "{}")'.format( pnetcdf_path) else: - pnetcdf_path_tag = '' + pnetcdf_path_addition = '' fill_template_file( path_to_template=os.path.join(_PATH_TO_TEMPLATES, - 'config_compilers_template.xml'), - path_to_final=os.path.join(build_dir, _MACHINE_CONFIG_DIRNAME, 'config_compilers.xml'), - substitutions={'COMPILER':compiler, - 'GPTL_CPPDEFS':gptl_cppdefs, + 'ctsm-build_template.cmake'), + path_to_final=os.path.join(build_dir, _MACHINE_CONFIG_DIRNAME, "cmake_macros", + '{}_{}.cmake'.format(compiler, _MACH_NAME)), + substitutions={'GPTL_CPPDEFS':gptl_cppdefs, 'NETCDF_PATH':netcdf_path, - 'PIO_FILESYSTEM_HINTS':pio_filesystem_hints_tag, - 'PNETCDF_PATH':pnetcdf_path_tag, - 'ESMF_LIBDIR':esmf_lib_path, + 'PIO_FILESYSTEM_HINTS':pio_filesystem_hints_addition, + 'PNETCDF_PATH':pnetcdf_path_addition, 'EXTRA_CFLAGS':extra_cflags, 'EXTRA_FFLAGS':extra_fflags}) diff --git a/python/ctsm/test/test_sys_lilac_build_ctsm.py b/python/ctsm/test/test_sys_lilac_build_ctsm.py index 5a44688171..3c4117fd45 100755 --- a/python/ctsm/test/test_sys_lilac_build_ctsm.py +++ b/python/ctsm/test/test_sys_lilac_build_ctsm.py @@ -46,7 +46,7 @@ def test_buildSetup_userDefinedMachine_minimalInfo(self): no_build=True, os_type='linux', netcdf_path='/path/to/netcdf', - esmf_lib_path='/path/to/esmf/lib', + esmf_mkfile_path='/path/to/esmf/lib/esmf.mk', max_mpitasks_per_node=16, gmake='gmake', gmake_j=8, @@ -76,7 +76,7 @@ def test_buildSetup_userDefinedMachine_allInfo(self): no_build=True, os_type='linux', netcdf_path='/path/to/netcdf', - esmf_lib_path='/path/to/esmf/lib', + esmf_mkfile_path='/path/to/esmf/lib/esmf.mk', max_mpitasks_per_node=16, gmake='gmake', gmake_j=8, diff --git a/python/ctsm/test/test_unit_lilac_build_ctsm.py b/python/ctsm/test/test_unit_lilac_build_ctsm.py index 3c1a600326..96f79f3765 100755 --- a/python/ctsm/test/test_unit_lilac_build_ctsm.py +++ b/python/ctsm/test/test_unit_lilac_build_ctsm.py @@ -105,7 +105,7 @@ def test_commandlineArgs_noRebuild_valid(self): '--os', 'linux', '--compiler', 'intel', '--netcdf-path', '/path/to/netcdf', - '--esmf-lib-path', '/path/to/esmf/lib', + '--esmf-mkfile-path', '/path/to/esmf/lib/esmf.mk', '--max-mpitasks-per-node', '16', '--no-pnetcdf']) @@ -120,7 +120,7 @@ def test_commandlineArgs_noRebuild_invalid1(self, mock_stderr): _ = _commandline_args(args_to_parse=['build/directory', '--os', 'linux', '--netcdf-path', '/path/to/netcdf', - '--esmf-lib-path', '/path/to/esmf/lib', + '--esmf-mkfile-path', '/path/to/esmf/lib/esmf.mk', '--max-mpitasks-per-node', '16', '--no-pnetcdf']) self.assertRegex(mock_stderr.getvalue(), expected_re) @@ -136,7 +136,7 @@ def test_commandlineArgs_noRebuild_invalid2(self, mock_stderr): _ = _commandline_args(args_to_parse=['build/directory', '--compiler', 'intel', '--netcdf-path', '/path/to/netcdf', - '--esmf-lib-path', '/path/to/esmf/lib', + '--esmf-mkfile-path', '/path/to/esmf/lib/esmf.mk', '--max-mpitasks-per-node', '16', '--no-pnetcdf']) self.assertRegex(mock_stderr.getvalue(), expected_re) @@ -150,7 +150,7 @@ def test_commandlineArgs_noRebuild_invalidNeedToDictatePnetcdf(self, mock_stderr '--os', 'linux', '--compiler', 'intel', '--netcdf-path', '/path/to/netcdf', - '--esmf-lib-path', '/path/to/esmf/lib', + '--esmf-mkfile-path', '/path/to/esmf/lib/esmf.mk', '--max-mpitasks-per-node', '16']) self.assertRegex(mock_stderr.getvalue(), expected_re) @@ -163,7 +163,7 @@ def test_commandlineArgs_noRebuild_invalidConflictingPnetcdf(self, mock_stderr): '--os', 'linux', '--compiler', 'intel', '--netcdf-path', '/path/to/netcdf', - '--esmf-lib-path', '/path/to/esmf/lib', + '--esmf-mkfile-path', '/path/to/esmf/lib/esmf.mk', '--max-mpitasks-per-node', '16', '--no-pnetcdf', '--pnetcdf-path', '/path/to/pnetcdf']) diff --git a/python/pyproject.toml b/python/pyproject.toml new file mode 100644 index 0000000000..fd8d8ac03c --- /dev/null +++ b/python/pyproject.toml @@ -0,0 +1,13 @@ +# +# This is a configuration file for python projects. +# Sepcifically covering build system requirements. +# +# Here we are just using a couple options to specify the operation +# of the python formatter "black". +# +[tool.black] + + line-length = 88 # This is the black default + target-version = ['py37'] + include = '(run_ctsm_py_tests|\.py$)' # Files to include + exclude = '(\.pylintrc|\.pyc)' # Files to explicitly exclude pylint file and compiled python diff --git a/src/biogeophys/BalanceCheckMod.F90 b/src/biogeophys/BalanceCheckMod.F90 index 508e118b66..ec928f8645 100644 --- a/src/biogeophys/BalanceCheckMod.F90 +++ b/src/biogeophys/BalanceCheckMod.F90 @@ -82,7 +82,8 @@ subroutine BalanceCheckInit( ) !----------------------------------------------------------------------- dtime = get_step_size_real() ! Skip a minimum of two time steps, but otherwise skip the number of time-steps in the skip_size rounded to the nearest integer - skip_steps = max(2, nint( (skip_size / dtime) ) ) + ! Add an additional step as now required to be after the hourly radiation time-step see github issue #1563 + skip_steps = max(2, nint( (skip_size / dtime) ) ) + 1 if ( masterproc ) write(iulog,*) ' Skip balance checking for the first ', skip_steps, ' time steps' diff --git a/src/biogeophys/test/Balance_test/test_Balance.pf b/src/biogeophys/test/Balance_test/test_Balance.pf index 3d07385ffb..824eebb78c 100644 --- a/src/biogeophys/test/Balance_test/test_Balance.pf +++ b/src/biogeophys/test/Balance_test/test_Balance.pf @@ -46,7 +46,7 @@ contains call unittest_timemgr_setup(dtime=dtime) call BalanceCheckInit() nskip = GetBalanceCheckSkipSteps() - @assertEqual( 2, nskip, message="Ensure standard balance check is 2 time-steps" ) + @assertEqual( 3, nskip, message="Ensure standard balance check is 3 time-steps" ) end subroutine test_balance_init @Test @@ -59,7 +59,7 @@ contains call unittest_timemgr_setup(dtime=dtime) call BalanceCheckInit() nskip = GetBalanceCheckSkipSteps() - @assertEqual( 2, nskip, message="Ensure even with a long time-step skip is 2 time-steps" ) + @assertEqual( 3, nskip, message="Ensure even with a long time-step skip is 3 time-steps" ) end subroutine test_balance_longstep @Test @@ -72,7 +72,7 @@ contains call unittest_timemgr_setup(dtime=dtime) call BalanceCheckInit() nskip = GetBalanceCheckSkipSteps() - @assertEqual( 12, nskip, message="Check skip length for 300 sec time-step" ) + @assertEqual( 13, nskip, message="Check skip length for 300 sec time-step" ) end subroutine test_balance_300sec @Test @@ -101,7 +101,7 @@ contains call unittest_timemgr_setup(dtime=dtime) call BalanceCheckInit() nskip = GetBalanceCheckSkipSteps() - @assertEqual( 100, nskip, message="Ensure with a short step correct number of skip steps is done" ) + @assertEqual( 101, nskip, message="Ensure with a short step correct number of skip steps is done" ) end subroutine test_balance_shortstep end module test_balance diff --git a/test/tools/README b/test/tools/README index 186f1dfe5d..cb5dcdec34 100644 --- a/test/tools/README +++ b/test/tools/README @@ -15,6 +15,11 @@ on cheyenne qcmd -l walltime=08:00:00 -- ./test_driver.sh -i >& run.out & +And to for example to compare to another baseline code (in this case ctsm5.1.dev066, which would need to be cloned at the given +path) ... + +qcmd -l walltime=08:00:00 -- env BL_ROOT=/glade/scratch/erik/ctsm5.1.dev066 ./test_driver.sh -i >& run.out & + on izumi nohup ./test_driver.sh -i >& run.out & diff --git a/test/tools/tests_posttag_hobart_nompi b/test/tools/tests_posttag_hobart_nompi index 4655f29853..9f07863e4d 100644 --- a/test/tools/tests_posttag_hobart_nompi +++ b/test/tools/tests_posttag_hobart_nompi @@ -2,5 +2,3 @@ smc#4 blc#4 smi54 bli54 smi57 bli57 smiT4 bliT4 -smf84 blf84 -smfc4 blfc4 diff --git a/test/tools/tests_posttag_izumi_nompi b/test/tools/tests_posttag_izumi_nompi index 90be9522dc..62687a7e3d 100644 --- a/test/tools/tests_posttag_izumi_nompi +++ b/test/tools/tests_posttag_izumi_nompi @@ -1,5 +1,3 @@ smi54 bli54 smi57 bli57 smiT4 bliT4 -smf84 blf84 -smfc4 blfc4 diff --git a/test/tools/tests_posttag_nompi_regression b/test/tools/tests_posttag_nompi_regression index 1785b5da47..5b5d76fd60 100644 --- a/test/tools/tests_posttag_nompi_regression +++ b/test/tools/tests_posttag_nompi_regression @@ -1,5 +1,4 @@ smc#4 blc#4 -sme14 ble14 smg54 blg54 smi24 bli24 smi53 bli53 @@ -10,9 +9,6 @@ smi74 bli74 smi78 bli78 smiT4 bliT4 smiT2 bliT2 -smf84 blf84 -smfc4 blfc4 -smfg4 blfg4 smiS4 bliS4 smiS8 bliS8 smiS9 bliS9 diff --git a/test/tools/tests_pretag_cheyenne_nompi b/test/tools/tests_pretag_cheyenne_nompi index f99ab6b691..fec9d08448 100644 --- a/test/tools/tests_pretag_cheyenne_nompi +++ b/test/tools/tests_pretag_cheyenne_nompi @@ -1,7 +1,5 @@ smi79 bli79 smc#4 blc#4 -sme14 ble14 -sme@4 ble@4 smg54 blg54 sm0a1 bl0a1 smaa2 blaa2 @@ -17,6 +15,3 @@ smiS4 bliS4 smi74 bli74 smiT4 bliT4 smiT2 bliT2 -smf84 blf84 -smfc4 blfc4 -smfg4 blfg4 diff --git a/tools/mksurfdata_map/src/Mkdepends b/tools/mksurfdata_map/src/Mkdepends index a75e8fdde0..ddb1682da4 100755 --- a/tools/mksurfdata_map/src/Mkdepends +++ b/tools/mksurfdata_map/src/Mkdepends @@ -35,6 +35,7 @@ use Getopt::Std; use File::Basename; +use File::Glob ':bsd_glob'; # Check for usage request. @ARGV >= 2 or usage(); @@ -61,7 +62,7 @@ chomp @file_paths; unshift(@file_paths,'.'); foreach $dir (@file_paths) { # (could check that directories exist here) $dir =~ s!/?\s*$!!; # remove / and any whitespace at end of directory name - ($dir) = glob $dir; # Expand tildes in path names. + ($dir) = bsd_glob $dir; # Expand tildes in path names. } # Make list of files containing source code. @@ -91,7 +92,7 @@ my ($dir); my ($f, $name, $path, $suffix, $mod); my @suffixes = ('\.mod' ); foreach $dir (@file_paths) { - @filenames = (glob("$dir/*.mod")); + @filenames = (bsd_glob("$dir/*.mod")); foreach $f (@filenames) { ($name, $path, $suffix) = fileparse($f, @suffixes); ($mod = $name) =~ tr/a-z/A-Z/; diff --git a/tools/mksurfdata_map/src/mkVICparamsMod.F90 b/tools/mksurfdata_map/src/mkVICparamsMod.F90 index f7cb4946c6..431e43cb28 100644 --- a/tools/mksurfdata_map/src/mkVICparamsMod.F90 +++ b/tools/mksurfdata_map/src/mkVICparamsMod.F90 @@ -129,7 +129,7 @@ subroutine mkVICparams(ldomain, mapfname, datfname, ndiag, & ! Check validity of output data if (min_bad(binfl_o, min_valid_binfl, 'binfl')) then - stop + call abort() end if call output_diagnostics_continuous(data_i, binfl_o, tgridmap, "VIC b parameter", "unitless", ndiag, tdomain%mask, frac_dst) @@ -144,7 +144,7 @@ subroutine mkVICparams(ldomain, mapfname, datfname, ndiag, & ! Check validity of output data if (min_bad(ws_o, min_valid_ws, 'Ws')) then - stop + call abort() end if call output_diagnostics_continuous(data_i, ws_o, tgridmap, "VIC Ws parameter", "unitless", ndiag, tdomain%mask, frac_dst) @@ -159,7 +159,7 @@ subroutine mkVICparams(ldomain, mapfname, datfname, ndiag, & ! Check validity of output data if (min_bad(dsmax_o, min_valid_dsmax, 'Dsmax')) then - stop + call abort() end if call output_diagnostics_continuous(data_i, dsmax_o, tgridmap, "VIC Dsmax parameter", "mm/day", ndiag, tdomain%mask, frac_dst) @@ -174,7 +174,7 @@ subroutine mkVICparams(ldomain, mapfname, datfname, ndiag, & ! Check validity of output data if (min_bad(ds_o, min_valid_ds, 'Ds')) then - stop + call abort() end if call output_diagnostics_continuous(data_i, ds_o, tgridmap, "VIC Ds parameter", "unitless", ndiag, tdomain%mask, frac_dst) diff --git a/tools/mksurfdata_map/src/mkagfirepkmonthMod.F90 b/tools/mksurfdata_map/src/mkagfirepkmonthMod.F90 index af8001263f..7b58ddffad 100644 --- a/tools/mksurfdata_map/src/mkagfirepkmonthMod.F90 +++ b/tools/mksurfdata_map/src/mkagfirepkmonthMod.F90 @@ -145,7 +145,7 @@ subroutine mkagfirepkmon(ldomain, mapfname, datfname, ndiag, & ! Check validity of output data if (min_bad(agfirepkmon_o, min_valid, 'agfirepkmon') .or. & max_bad(agfirepkmon_o, max_valid, 'agfirepkmon')) then - stop + call abort() end if diff --git a/tools/mksurfdata_map/src/mkdiagnosticsMod.F90 b/tools/mksurfdata_map/src/mkdiagnosticsMod.F90 index a53d9ca4d2..91769a5823 100644 --- a/tools/mksurfdata_map/src/mkdiagnosticsMod.F90 +++ b/tools/mksurfdata_map/src/mkdiagnosticsMod.F90 @@ -81,7 +81,7 @@ subroutine output_diagnostics_area(data_i, data_o, gridmap, name, percent, ndiag write(6,*) 'ns_i = ', ns_i write(6,*) 'size(data_o) = ', size(data_o) write(6,*) 'ns_o = ', ns_o - stop + call abort() end if if (size(frac_dst) /= ns_o) then write(6,*) subname//' ERROR: incorrect size of frac_dst' @@ -195,7 +195,7 @@ subroutine output_diagnostics_continuous(data_i, data_o, gridmap, name, units, n write(6,*) 'ns_i = ', ns_i write(6,*) 'size(data_o) = ', size(data_o) write(6,*) 'ns_o = ', ns_o - stop + call abort() end if if (size(frac_dst) /= ns_o) then write(6,*) subname//' ERROR: incorrect size of frac_dst' @@ -300,7 +300,7 @@ subroutine output_diagnostics_continuous_outonly(data_o, gridmap, name, units, n write(6,*) subname//' ERROR: array size inconsistencies for ', trim(name) write(6,*) 'size(data_o) = ', size(data_o) write(6,*) 'ns_o = ', ns_o - stop + call abort() end if ! Sums on output grid @@ -380,7 +380,7 @@ subroutine output_diagnostics_index(data_i, data_o, gridmap, name, & write(6,*) 'ns_i = ', ns_i write(6,*) 'size(data_o) = ', size(data_o) write(6,*) 'ns_o = ', ns_o - stop + call abort() end if if (size(frac_dst) /= ns_o) then write(6,*) subname//' ERROR: incorrect size of frac_dst' diff --git a/tools/mksurfdata_map/src/mkgdpMod.F90 b/tools/mksurfdata_map/src/mkgdpMod.F90 index 6a560e61b5..138ddf1805 100644 --- a/tools/mksurfdata_map/src/mkgdpMod.F90 +++ b/tools/mksurfdata_map/src/mkgdpMod.F90 @@ -122,7 +122,7 @@ subroutine mkgdp(ldomain, mapfname, datfname, ndiag, gdp_o) ! Check validity of output data if (min_bad(gdp_o, min_valid, 'gdp')) then - stop + call abort() end if call output_diagnostics_continuous(data_i, gdp_o, tgridmap, "GDP", "x1000 US$ per capita", ndiag, tdomain%mask, frac_dst) diff --git a/tools/mksurfdata_map/src/mkglacierregionMod.F90 b/tools/mksurfdata_map/src/mkglacierregionMod.F90 index beae6a8d97..e644129ed3 100644 --- a/tools/mksurfdata_map/src/mkglacierregionMod.F90 +++ b/tools/mksurfdata_map/src/mkglacierregionMod.F90 @@ -105,7 +105,7 @@ subroutine mkglacierregion(ldomain, mapfname, datfname, ndiag, & call check_ret(nf_inq_varid(ncid, 'GLACIER_REGION', varid), subname) call check_ret(nf_get_var_int(ncid, varid, glacier_region_i), subname) if (min_bad(glacier_region_i, 0, 'GLACIER_REGION')) then - stop + call abort() end if call get_max_indices( & diff --git a/tools/mksurfdata_map/src/mkglcmecMod.F90 b/tools/mksurfdata_map/src/mkglcmecMod.F90 index 2ac4d94e4f..9fbad66689 100644 --- a/tools/mksurfdata_map/src/mkglcmecMod.F90 +++ b/tools/mksurfdata_map/src/mkglcmecMod.F90 @@ -606,7 +606,7 @@ subroutine mkglacier(ldomain, mapfname, datfname, ndiag, zero_out, glac_o) write (6,*) 'MKGLACIER error: glacier = ',glac_o(no), & ' greater than 100.000001 for column, row = ',no call shr_sys_flush(6) - stop + call abort() end if enddo diff --git a/tools/mksurfdata_map/src/mkgridmapMod.F90 b/tools/mksurfdata_map/src/mkgridmapMod.F90 index 21ca23f4d6..eeb5afdbb8 100644 --- a/tools/mksurfdata_map/src/mkgridmapMod.F90 +++ b/tools/mksurfdata_map/src/mkgridmapMod.F90 @@ -505,7 +505,7 @@ subroutine gridmap_check(gridmap, mask_src, frac_dst, caller) write (6,*) subname//' ERROR from '//trim(caller)//': mapping areas not conserved' write (6,'(a30,e20.10)') 'global sum output field = ',sum_area_o write (6,'(a30,e20.10)') 'global sum input field = ',sum_area_i - stop + call abort() end if end if diff --git a/tools/mksurfdata_map/src/mklaiMod.F90 b/tools/mksurfdata_map/src/mklaiMod.F90 index aef33f3463..e4b6d9bfa1 100644 --- a/tools/mksurfdata_map/src/mklaiMod.F90 +++ b/tools/mksurfdata_map/src/mklaiMod.F90 @@ -140,7 +140,7 @@ subroutine mklai(ldomain, mapfname, datfname, ndiag, ncido) ! invalid, all the loop bounds over output data in this ! routine will need to be double checked! write(6, *) "ERROR:" // trim(subname) // "(): input numpft must be less than or equal to output numpft+1." - stop + call abort() end if endif if (ntim /= 12) then @@ -434,7 +434,7 @@ subroutine pft_laicheck( ni_s, pctpft_i, laimask ) write (6,*) subName//' :: pft/LAI+SAI inconsistency over more than 25% land-cover' write (6,*) '# inconsistent points, total PFT pts, total LAI+SAI pts = ', & n, nc, sum(laimask(:,l)) - stop + call abort() end if end do diff --git a/tools/mksurfdata_map/src/mklanwatMod.F90 b/tools/mksurfdata_map/src/mklanwatMod.F90 index 49a1485fa7..4e1c590803 100644 --- a/tools/mksurfdata_map/src/mklanwatMod.F90 +++ b/tools/mksurfdata_map/src/mklanwatMod.F90 @@ -478,7 +478,7 @@ subroutine mklakparams(ldomain, mapfname, datfname, ndiag, & ! Check validity of output data if (min_bad(lakedepth_o, min_valid_lakedepth, 'lakedepth')) then - stop + call abort() end if call output_diagnostics_continuous(data_i, lakedepth_o, tgridmap, "Lake Depth", "m", ndiag, tdomain%mask, frac_dst) diff --git a/tools/mksurfdata_map/src/mkpeatMod.F90 b/tools/mksurfdata_map/src/mkpeatMod.F90 index 974566a056..8e47f5032d 100644 --- a/tools/mksurfdata_map/src/mkpeatMod.F90 +++ b/tools/mksurfdata_map/src/mkpeatMod.F90 @@ -123,7 +123,7 @@ subroutine mkpeat(ldomain, mapfname, datfname, ndiag, peat_o) ! Check validity of output data if (min_bad(peat_o, min_valid, 'peat') .or. & max_bad(peat_o, max_valid, 'peat')) then - stop + call abort() end if call output_diagnostics_area(data_i, peat_o, tgridmap, "Peat", percent=.false., ndiag=ndiag, mask_src=tdomain%mask, frac_dst=frac_dst) diff --git a/tools/mksurfdata_map/src/mkpftMod.F90 b/tools/mksurfdata_map/src/mkpftMod.F90 index 3a12c38cdf..2eae1ae381 100644 --- a/tools/mksurfdata_map/src/mkpftMod.F90 +++ b/tools/mksurfdata_map/src/mkpftMod.F90 @@ -644,7 +644,7 @@ subroutine mkpft(ldomain, mapfname, fpft, ndiag, & write (6,*) subname//'error: nat pft = ', & (pct_nat_pft_o(no,m), m = 0, num_natpft), & ' do not sum to 100. at no = ',no,' but to ', wst_sum - stop + call abort() end if ! Correct sum so that if it differs slightly from 100, it is corrected to equal @@ -661,7 +661,7 @@ subroutine mkpft(ldomain, mapfname, fpft, ndiag, & write (6,*) subname//'error: crop cft = ', & (pct_cft_o(no,m), m = 1, num_cft), & ' do not sum to 100. at no = ',no,' but to ', wst_sum - stop + call abort() end if ! Correct sum so that if it differs slightly from 100, it is corrected to equal diff --git a/tools/mksurfdata_map/src/mksoilMod.F90 b/tools/mksurfdata_map/src/mksoilMod.F90 index 959749ca1a..d7cad23e0d 100644 --- a/tools/mksurfdata_map/src/mksoilMod.F90 +++ b/tools/mksurfdata_map/src/mksoilMod.F90 @@ -319,7 +319,7 @@ subroutine mksoiltex(ldomain, mapfname, datfname, ndiag, sand_o, clay_o) write(6,*)'kmax is > kmap_max= ',kmax(no), 'kmap_max = ', & kmap_max,' for no = ',no write(6,*)'reset kmap_max in mksoilMod to a greater value' - stop + call abort() end if kmap(kmax(no),no) = k kwgt(kmax(no),no) = wt @@ -841,7 +841,7 @@ subroutine mkorganic(ldomain, mapfname, datfname, ndiag, organic_o) if (nlay /= nlevsoi) then write(6,*)'nlay, nlevsoi= ',nlay,nlevsoi,' do not match' - stop + call abort() end if call check_ret(nf_inq_varid (ncid, 'ORGANIC', varid), subname) @@ -873,7 +873,7 @@ subroutine mkorganic(ldomain, mapfname, datfname, ndiag, organic_o) write (6,*) 'MKORGANIC error: organic = ',organic_o(no,lev), & ' greater than 130.000001 for column, row = ',no call shr_sys_flush(6) - stop + call abort() end if enddo @@ -942,7 +942,7 @@ subroutine mksoilfmaxInit( ) if ( soil_fmax /= unset )then if ( soil_fmax < 0.0 .or. soil_fmax > 1.0 )then write(6,*)'soil_fmax is out of range = ', soil_fmax - stop + call abort() end if write(6,*) 'Replace soil fmax for all points with: ', soil_fmax end if @@ -1053,7 +1053,7 @@ subroutine mkfmax(ldomain, mapfname, datfname, ndiag, fmax_o) write (6,*) 'MKFMAX error: fmax = ',fmax_o(no), & ' greater than 1.000001 for column, row = ',no call shr_sys_flush(6) - stop + call abort() end if enddo @@ -1086,7 +1086,7 @@ subroutine mkfmax(ldomain, mapfname, datfname, ndiag, fmax_o) frac_dst(no)*re**2 if ((frac_dst(no) < 0.0) .or. (frac_dst(no) > 1.0001)) then write(6,*) "ERROR:: frac_dst out of range: ", frac_dst(no),no - stop + call abort() end if end do diff --git a/tools/mksurfdata_map/src/mksoildepthMod.F90 b/tools/mksurfdata_map/src/mksoildepthMod.F90 index 521ac2c6f0..c69cf375a4 100644 --- a/tools/mksurfdata_map/src/mksoildepthMod.F90 +++ b/tools/mksurfdata_map/src/mksoildepthMod.F90 @@ -146,7 +146,7 @@ subroutine mksoildepth(ldomain, mapfname, datfname, ndiag, soildepth_o) ! Check validity of output data if (min_bad(soildepth_o, min_valid, 'soildepth') .or. & max_bad(soildepth_o, max_valid, 'soildepth')) then - stop + call abort() end if call output_diagnostics_area(data_i, soildepth_o, tgridmap, "Soildepth", percent=.false., ndiag=ndiag, mask_src=tdomain%mask, frac_dst=frac_dst) diff --git a/tools/mksurfdata_map/src/mksurfdat.F90 b/tools/mksurfdata_map/src/mksurfdat.F90 index 9051c57707..aa965f097d 100644 --- a/tools/mksurfdata_map/src/mksurfdat.F90 +++ b/tools/mksurfdata_map/src/mksurfdat.F90 @@ -425,7 +425,7 @@ program mksurfdat ! Make sure ldomain is on a 0 to 360 grid as that's a requirement for CESM if ( .not. is_domain_0to360_longs( ldomain ) )then write(6,*)' Output domain must be on a 0 to 360 longitude grid rather than a -180 to 180 grid as it is required for CESM' - stop + call abort() end if ! ---------------------------------------------------------------------- ! Allocate and initialize dynamic memory @@ -493,7 +493,7 @@ program mksurfdat if (fsurlog == ' ') then write(6,*)' must specify fsurlog in namelist' - stop + call abort() else ndiag = getavu(); call opnfil (fsurlog, ndiag, 'f') end if @@ -1091,7 +1091,7 @@ program mksurfdat if (fdyndat == ' ') then write(6,*)' must specify fdyndat in namelist if mksrf_fdynuse is not blank' - stop + call abort() end if ! Define dimensions and global attributes diff --git a/tools/mksurfdata_map/src/mktopostatsMod.F90 b/tools/mksurfdata_map/src/mktopostatsMod.F90 index 2ecd705f4c..7e102d9bcf 100644 --- a/tools/mksurfdata_map/src/mktopostatsMod.F90 +++ b/tools/mksurfdata_map/src/mktopostatsMod.F90 @@ -131,7 +131,7 @@ subroutine mktopostats(ldomain, mapfname, datfname, ndiag, topo_stddev_o, slope_ ! Check validity of output data if (min_bad(topo_stddev_o, min_valid_topo_stddev, 'topo_stddev')) then - stop + call abort() end if @@ -158,7 +158,7 @@ subroutine mktopostats(ldomain, mapfname, datfname, ndiag, topo_stddev_o, slope_ ! Check validity of output data if (min_bad(slope_o, min_valid_slope, 'slope') .or. & max_bad(slope_o, max_valid_slope, 'slope')) then - stop + call abort() end if diff --git a/tools/mksurfdata_map/src/mkurbanparCommonMod.F90 b/tools/mksurfdata_map/src/mkurbanparCommonMod.F90 index 5db84e8351..ab738ea03c 100644 --- a/tools/mksurfdata_map/src/mkurbanparCommonMod.F90 +++ b/tools/mksurfdata_map/src/mkurbanparCommonMod.F90 @@ -93,13 +93,13 @@ subroutine mkurban_pct(ldomain, tdomain, tgridmap, urbn_i, urbn_o, frac_dst) write(6,*) 'tdomain%ns = ', tdomain%ns write(6,*) 'size(urbn_o) = ', size(urbn_o) write(6,*) 'ldomain%ns = ', ldomain%ns - stop + call abort() end if if (size(frac_dst) /= ldomain%ns) then write(6,*) subname//' ERROR: array size inconsistencies' write(6,*) 'size(frac_dst) = ', size(frac_dst) write(6,*) 'ldomain%ns = ', ldomain%ns - stop + call abort() end if ! Error checks for domain and map consistencies @@ -119,7 +119,7 @@ subroutine mkurban_pct(ldomain, tdomain, tgridmap, urbn_i, urbn_o, frac_dst) if ((urbn_o(no)) > 100.000001_r8) then write (6,*) 'MKURBAN error: urban = ',urbn_o(no), & ' greater than 100.000001 for column, row = ',no - stop + call abort() end if enddo @@ -191,7 +191,7 @@ subroutine mkurban_pct_diagnostics(ldomain, tdomain, tgridmap, urbn_i, urbn_o, n write(6,*) subname//' ERROR: array size inconsistencies' write(6,*) 'size(frac_dst) = ', size(frac_dst) write(6,*) 'ldomain%ns = ', ldomain%ns - stop + call abort() end if ! ----------------------------------------------------------------- diff --git a/tools/mksurfdata_map/src/mkurbanparMod.F90 b/tools/mksurfdata_map/src/mkurbanparMod.F90 index 07319b1f27..49ce95dd07 100644 --- a/tools/mksurfdata_map/src/mkurbanparMod.F90 +++ b/tools/mksurfdata_map/src/mkurbanparMod.F90 @@ -548,17 +548,17 @@ subroutine mkurbanpar(datfname, ncido, region_o, urbn_classes_gcell_o, urban_ski if (nlevurb_i /= nlevurb) then write(6,*)'MKURBANPAR: parameter nlevurb= ',nlevurb, & 'does not equal input dataset nlevurb= ',nlevurb_i - stop + call abort() endif if (numsolar_i /= numsolar) then write(6,*)'MKURBANPAR: parameter numsolar= ',numsolar, & 'does not equal input dataset numsolar= ',numsolar_i - stop + call abort() endif if (numrad_i /= numrad) then write(6,*)'MKURBANPAR: parameter numrad= ',numrad, & 'does not equal input dataset numrad= ',numrad_i - stop + call abort() endif ! Create an array that will hold the density indices diff --git a/tools/site_and_regional/modify_singlept_site_neon.py b/tools/site_and_regional/modify_singlept_site_neon.py index 7b469641f1..d3db55126b 100755 --- a/tools/site_and_regional/modify_singlept_site_neon.py +++ b/tools/site_and_regional/modify_singlept_site_neon.py @@ -12,8 +12,8 @@ This script will do the following: - Download neon data for the specified site if it does not exist - in the specified directory. -- Modify surface dataset with downloaded data (neon-specific). + in the specified directory : (i.e. ../../../neon_surffiles). +- Modify surface dataset with downloaded data. ------------------------------------------------------------------- Instructions for running on Cheyenne/Casper: @@ -28,13 +28,14 @@ To see the available options: ./modify_singlept_site_neon.py --help ------------------------------------------------------------------- +Example: + ./modify_singlept_site_neon.py --neon_site PUUM --debug +------------------------------------------------------------------- """ # TODO (NS) -#--[] If file not found run subset_data.py -#--[] Clean up imports for both codes... -#--[] Check against a list of valid names. -#--[] List of valid neon sites for all scripts come from one place. -#--[] zbedrock +# --[] If subset file not found run subset_data.py +# --[] List of valid neon sites for all scripts come from one place. +# --[] Download files only when available. # Import libraries from __future__ import print_function @@ -45,6 +46,7 @@ import argparse import requests +import logging import numpy as np import pandas as pd import xarray as xr @@ -56,72 +58,118 @@ myname = getuser() -#-- valid neon sites -valid_neon_sites = ['ABBY','BARR','BART','BLAN', - 'BONA','CLBJ','CPER','DCFS', - 'DEJU','DELA','DSNY','GRSM', - 'GUAN','HARV','HEAL','JERC', - 'JORN','KONA','KONZ','LAJA', - 'LENO','MLBS','MOAB','NIWO', - 'NOGP','OAES','ONAQ','ORNL', - 'OSBS','PUUM','RMNP','SCBI', - 'SERC','SJER','SOAP','SRER', - 'STEI','STER','TALL','TEAK', - 'TOOL','TREE','UKFS','UNDE', - 'WOOD','WREF','YELL' - ] - - -def get_parser(): +# -- valid neon sites +valid_neon_sites = [ + "ABBY", + "BARR", + "BART", + "BLAN", + "BONA", + "CLBJ", + "CPER", + "DCFS", + "DEJU", + "DELA", + "DSNY", + "GRSM", + "GUAN", + "HARV", + "HEAL", + "JERC", + "JORN", + "KONA", + "KONZ", + "LAJA", + "LENO", + "MLBS", + "MOAB", + "NIWO", + "NOGP", + "OAES", + "ONAQ", + "ORNL", + "OSBS", + "PUUM", + "RMNP", + "SCBI", + "SERC", + "SJER", + "SOAP", + "SRER", + "STEI", + "STER", + "TALL", + "TEAK", + "TOOL", + "TREE", + "UKFS", + "UNDE", + "WOOD", + "WREF", + "YELL", +] + + +def get_parser(): """ Get parser object for this script. """ - parser = argparse.ArgumentParser(description=__doc__, - formatter_class=argparse.RawDescriptionHelpFormatter) + parser = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + ) parser.print_usage = parser.print_help - parser.add_argument('--neon_site', - help='4-letter neon site code.', - action="store", - dest="site_name", - choices=valid_neon_sites, - required=True) - parser.add_argument('--surf_dir', - help=''' + parser.add_argument( + "--neon_site", + help="4-letter neon site code.", + action="store", + dest="site_name", + choices=valid_neon_sites, + required=True, + ) + parser.add_argument( + "--surf_dir", + help=""" Directory of single point surface dataset. [default: %(default)s] - ''', - action="store", - dest="surf_dir", - type =str, - required=False, - default="/glade/scratch/"+myname+"/single_point/") - parser.add_argument('--out_dir', - help=''' + """, + action="store", + dest="surf_dir", + type=str, + required=False, + default="/glade/scratch/" + myname + "/single_point/", + ) + parser.add_argument( + "--out_dir", + help=""" Directory to write updated single point surface dataset. [default: %(default)s] - ''', - action="store", - dest="out_dir", - type =str, - required=False, - default="/glade/scratch/"+myname+"/single_point_neon_updated/") - parser.add_argument('-d','--debug', - help=''' + """, + action="store", + dest="out_dir", + type=str, + required=False, + default="/glade/scratch/" + myname + "/single_point_neon_updated/", + ) + parser.add_argument( + "-d", + "--debug", + help=""" Debug mode will print more information. [default: %(default)s] - ''', - action="store_true", - dest="debug", - default=False) + """, + action="store_true", + dest="debug", + default=False, + ) return parser def get_neon(neon_dir, site_name): """ - Function for finding neon data file + Function for finding neon data files and download from neon server if the file does not exits. @@ -132,53 +180,62 @@ def get_neon(neon_dir, site_name): Raises: Error if the download was not successful (exit code:404). In case the data does not exist in the neon server or if - neon server is down. + neon server is down. Returns: neon_file (str) : complete file name of the downloaded data """ - #-- create directory if not exists + # -- create directory if not exists if not os.path.exists(neon_dir): os.makedirs(neon_dir) neon_file = os.path.join(neon_dir, site_name + "_surfaceData.csv") - #-- Download the file if it does not exits + # -- Download the file if it does not exits if os.path.isfile(neon_file): - print('neon file for', site_name, 'already exists! ') - print('Skipping download from neon for', site_name,'...') + print("neon file for", site_name, "already exists! ") + print("Skipping download from neon for", site_name, "...") else: - print('------------------------------------------------') - print('Beginning download from neon server for', site_name,'...') - - url = ('https://s3.data.neonscience.org/neon-ncar/NEON/surf_files/v1/' - +site_name+'_surfaceData.csv') + print("------------------------------------------------") + print("Beginning download from neon server for", site_name, "...") + + url = ( + "https://s3.data.neonscience.org/neon-ncar/NEON/surf_files/v1/" + + site_name + + "_surfaceData.csv" + ) response = requests.get(url) - with open(neon_file, 'wb') as f: + with open(neon_file, "wb") as f: f.write(response.content) - #-- Check if download status_code + # -- Check if download status_code if response.status_code == 200: - print('Download finished successfully for', site_name) + print("Download finished successfully for", site_name) elif response.status_code == 404: - sys.exit('Data for this site '+site_name+ - ' was not available on the neon server:'+ url) - - print('Download exit status code: ',response.status_code) - print('Downloaded file type : ',response.headers['content-type']) - print('Downloaded file encoding : ',response.encoding) - print('------------------------------------------------') + sys.exit( + "Data for this site " + + site_name + + " was not available on the neon server:" + + url + ) + + print("Download exit status code: ", response.status_code) + print("Downloaded file type : ", response.headers["content-type"]) + print("Downloaded file encoding : ", response.encoding) + print("------------------------------------------------") response.close() return neon_file -def find_surffile (surf_dir, site_name): + +def find_surffile(surf_dir, site_name): """ Function for finding and choosing surface file for a neon site. + These files are created using ./subset_data.py script. In case multiple files exist for the neon site, it will choose the file created the latest. @@ -187,32 +244,36 @@ def find_surffile (surf_dir, site_name): site_name (str): 4 letter neon site name Raises: - Error if the surface data for the site is not created + Error if the surface data for the site is not created Returns: surf_file (str): name of the surface dataset file """ - #sf_name = "surfdata_hist_16pfts_Irrig_CMIP6_simyr2000_"+site_name+"*.nc" - sf_name = "surfdata_hist_78pfts_CMIP6_simyr2000_"+site_name+"*.nc" - #surf_file = glob.glob(os.path.join(surf_dir,sf_name)) - surf_file = glob.glob(surf_dir+"/"+sf_name) + # sf_name = "surfdata_hist_16pfts_Irrig_CMIP6_simyr2000_"+site_name+"*.nc" + sf_name = "surfdata_hist_78pfts_CMIP6_simyr2000_" + site_name + "*.nc" + # surf_file = glob.glob(os.path.join(surf_dir,sf_name)) + surf_file = glob.glob(surf_dir + "/" + sf_name) - if len(surf_file)>1: - print ("The following files found :", *surf_file, sep='\n- ') - print ("The latest file is chosen :", surf_file[-1]) + if len(surf_file) > 1: + print("The following files found :", *surf_file, sep="\n- ") + print("The latest file is chosen :", surf_file[-1]) surf_file = surf_file[-1] - elif len(surf_file)==1: - print ("File found : ") - print (surf_file) + elif len(surf_file) == 1: + print("File found : ") + print(surf_file) surf_file = surf_file[0] else: - sys.exit('Surface data for this site '+site_name+ - 'was not found:'+ surf_file,'.', - '\n','Please run ./subset_data.py for this site.') + sys.exit( + "Surface data for this site " + site_name + "was not found:" + surf_file, + ".", + "\n", + "Please run ./subset_data.py for this site.", + ) return surf_file -def find_soil_structure (surf_file): + +def find_soil_structure(surf_file): """ Function for finding surface dataset soil strucutre using surface data metadata. @@ -227,42 +288,49 @@ def find_soil_structure (surf_file): surf_file (str): single point surface data filename Raises: - error if the soil layer strucutre file does not exist + error if the soil layer strucutre file does not exist Returns: soil_bot : array of soil layers top depths soil_top : array of soil layers bottom depths """ - #TODO: What if not cheyenne? Self-contained depth info. + # TODO: What if not cheyenne? Self-contained depth info. - print ('------------') - print (surf_file) - print (type(surf_file)) + print("------------") + print("surf_file : ", surf_file) f1 = xr.open_dataset(surf_file) - print ('------------') - #print (f1.attrs["Soil_texture_raw_data_file_name"]) + print("------------") + # print (f1.attrs["Soil_texture_raw_data_file_name"]) clm_input_dir = "/glade/p/cesmdata/cseg/inputdata/lnd/clm2/rawdata/" - surf_soildepth_file = os.path.join(clm_input_dir, - f1.attrs["Soil_texture_raw_data_file_name"]) - - if os.path.exists (surf_soildepth_file): - print ("\n\n Reading", surf_soildepth_file, - "for surface data soil structure information:") + surf_soildepth_file = os.path.join( + clm_input_dir, f1.attrs["Soil_texture_raw_data_file_name"] + ) + + if os.path.exists(surf_soildepth_file): + print( + "\n\n Reading", + surf_soildepth_file, + "for surface data soil structure information:", + ) f1_soildepth = xr.open_dataset(surf_soildepth_file) - print (f1_soildepth['DZSOI']) - soil_bot = f1_soildepth['DZSOI'].values + print(f1_soildepth["DZSOI"]) + soil_bot = f1_soildepth["DZSOI"].values - #-- soil layer top + # -- soil layer top soil_top = soil_bot[:-1] - soil_top = np.insert(soil_top,0, 0) + soil_top = np.insert(soil_top, 0, 0) else: - sys.exit('Cannot find soil structure file : '+surf_soildepth_file+ - 'for the surface dataset.') + sys.exit( + "Cannot find soil structure file : " + + surf_soildepth_file + + "for the surface dataset." + ) return soil_bot, soil_top + def update_metadata(nc, surf_file, neon_file, zb_flag): """ Function for updating modified surface dataset @@ -279,21 +347,20 @@ def update_metadata(nc, surf_file, neon_file, zb_flag): today = date.today() today_string = today.strftime("%Y-%m-%d") - nc.attrs['Updated_on'] = today_string - nc.attrs['Updated_by'] = myname - nc.attrs['Updated_with'] = os.path.abspath(__file__) - nc.attrs['Updated_from'] = surf_file - nc.attrs['Updated_using'] = neon_file + nc.attrs["Updated_on"] = today_string + nc.attrs["Updated_by"] = myname + nc.attrs["Updated_with"] = os.path.abspath(__file__) + nc.attrs["Updated_from"] = surf_file + nc.attrs["Updated_using"] = neon_file if zb_flag: - nc.attrs['Updated_fields'] = "PCT_CLAY, PCT_SAND, ORGANIC, zbedrock" - #nc.attrs['Updated_fields'] = ['PCT_CLAY','PCT_SAND','ORGANIC','zbedrock'] + nc.attrs["Updated_fields"] = "PCT_CLAY, PCT_SAND, ORGANIC, zbedrock" else: - nc.attrs['Updated_fields'] = "PCT_CLAY, PCT_SAND, ORGANIC" - # nc.attrs['Updated_fields'] = ['PCT_CLAY','PCT_SAND','ORGANIC'] + nc.attrs["Updated_fields"] = "PCT_CLAY, PCT_SAND, ORGANIC" return nc -def update_time_tag (fname_in): + +def update_time_tag(fname_in): """ Function for updating time tag on surface dataset files. @@ -315,13 +382,14 @@ def update_time_tag (fname_in): basename = os.path.basename(fname_in) cend = -10 - if ( basename[cend] == "c" ): - cend = cend - 1 - if ( (basename[cend] != ".") and (basename[cend] != "_") ): - sys.exit( "Trouble figuring out where to add tag to filename:"+fname_in ) + if basename[cend] == "c": + cend = cend - 1 + if (basename[cend] != ".") and (basename[cend] != "_"): + sys.exit("Trouble figuring out where to add tag to filename:" + fname_in) + + fname_out = basename[:cend] + "_" + "c" + today_string + ".nc" + return fname_out - fname_out = basename[:cend]+"_"+"c"+today_string+".nc" - return(fname_out) def sort_print_soil_layers(obs_bot, soil_bot): """ @@ -329,58 +397,54 @@ def sort_print_soil_layers(obs_bot, soil_bot): original surface dataset and neon dataset. Args: - obs_bot : array of neon soil layers bottom depths + obs_bot : array of neon soil layers bottom depths soil_bot : array of soil layers bottom depths """ - obs_bot_df = pd.DataFrame({'depth':obs_bot,'type':"obs"}) - soil_bot_df = pd.DataFrame({'depth':soil_bot,'type':"sfc"}) - depth_df = pd.concat([obs_bot_df,soil_bot_df]) + obs_bot_df = pd.DataFrame({"depth": obs_bot, "type": "obs"}) + soil_bot_df = pd.DataFrame({"depth": soil_bot, "type": "sfc"}) + depth_df = pd.concat([obs_bot_df, soil_bot_df]) - depth_df = depth_df.sort_values('depth') + depth_df = depth_df.sort_values("depth") - space = ' ' - print ("================================", - "================================") + space = " " + print("================================", "================================") - print (" Neon data soil structure: " , - " Surface data soil structure: ") + print(" Neon data soil structure: ", " Surface data soil structure: ") - print ("================================", - "================================") + print("================================", "================================") for index, row in depth_df.iterrows(): - if row['type']=="obs": - print ("-------------", - "{0:.3f}".format(row['depth']), - "------------") + if row["type"] == "obs": + print("-------------", "{0:.3f}".format(row["depth"]), "------------") else: - print (33*space+ - "-------------", - "{0:.3f}".format(row['depth']), - "-----------") + print( + 33 * space + "-------------", + "{0:.3f}".format(row["depth"]), + "-----------", + ) + + print("--------------------------------" + "--------------------------------") - print ("--------------------------------"+ - "--------------------------------") def check_neon_time(): """ A function to download and parse neon listing file. + + Returns: + dict_out (str) : + dictionary of *_surfaceData.csv files with the last modified """ - listing_file = 'listing.csv' - url = 'https://neon-ncar.s3.data.neonscience.org/listing.csv' + listing_file = "listing.csv" + url = "https://neon-ncar.s3.data.neonscience.org/listing.csv" download_file(url, listing_file) df = pd.read_csv(listing_file) - df = df[df['object'].str.contains("_surfaceData.csv")] - #df=df.join(df['object'].str.split("/", expand=True)) - dict_out = dict(zip(df['object'],df['last_modified'])) - print (dict_out) - #df_out = df[['object','6','last_modified']] - #print (df['last_modified']) - #print (df_out) - #print (df['last_modified'].to_datetime()) + df = df[df["object"].str.contains("_surfaceData.csv")] + # df=df.join(df['object'].str.split("/", expand=True)) + dict_out = dict(zip(df["object"], df["last_modified"])) + print(dict_out) return dict_out @@ -388,88 +452,114 @@ def download_file(url, fname): """ Function to download a file. Args: - url (str): + url (str): url of the file for downloading - fname (str) : + fname (str) : file name to save the downloaded file. """ response = requests.get(url) - with open(fname, 'wb') as f: + with open(fname, "wb") as f: f.write(response.content) - #-- Check if download status_code + # -- Check if download status_code if response.status_code == 200: - print('Download finished successfully for', fname,'.') + print("Download finished successfully for", fname, ".") elif response.status_code == 404: - print('File '+fname+'was not available on the neon server:'+ url) + print("File " + fname + "was not available on the neon server:" + url) + + +def fill_interpolate(f2, var, method): + """ + Function to interpolate a variable in a + xarray dataset a specific method + """ + print("=====================================") + print("Filling in ", var, "with interpolation (method =" + method + ").") + print("Variable before filling : ") + print(f2[var]) + + tmp_df = pd.DataFrame(f2[var].values.ravel()) + + tmp_df = tmp_df.interpolate(method=method, limit_direction="both") + # tmp_df = tmp_df.interpolate(method ='spline',order = 2, limit_direction ='both') + # tmp_df = tmp_df.interpolate(method="pad", limit=5, limit_direction = 'forward') + + tmp = tmp_df.to_numpy() + + soil_levels = f2[var].size + for soil_lev in range(soil_levels): + f2[var][soil_lev] = tmp[soil_lev].reshape(1, 1) + + print("Variable after filling : ") + print(f2[var]) + print("=====================================") def main(): args = get_parser().parse_args() - #-- debugging option + # -- debugging option if args.debug: logging.basicConfig(level=logging.DEBUG) file_time = check_neon_time() - #-- specify site from which to extract data - site_name=args.site_name + # -- specify site from which to extract data + site_name = args.site_name - #-- Look for surface data + # -- Look for surface data surf_dir = args.surf_dir - surf_file = find_surffile (surf_dir, site_name) + surf_file = find_surffile(surf_dir, site_name) - #-- directory structure + # -- directory structure current_dir = os.getcwd() - parent_dir = os.path.dirname(current_dir) - clone_dir = os.path.abspath(os.path.join(__file__ ,"../../..")) - neon_dir = os.path.join(clone_dir,"neon_surffiles") - print("Present Directory", current_dir) + parent_dir = os.path.dirname(current_dir) + clone_dir = os.path.abspath(os.path.join(__file__, "../../..")) + neon_dir = os.path.join(clone_dir, "neon_surffiles") + + print("Present Directory", current_dir) - #-- download neon data if needed + # -- download neon data if needed neon_file = get_neon(neon_dir, site_name) - #-- Read neon data - df = pd.read_csv (neon_file) + # -- Read neon data + df = pd.read_csv(neon_file) # -- Read surface dataset files - print (type(surf_file)) - print ("surf_file:", surf_file) + print("surf_file:", surf_file) f1 = xr.open_dataset(surf_file) # -- Find surface dataset soil depth information - soil_bot, soil_top = find_soil_structure (surf_file) + soil_bot, soil_top = find_soil_structure(surf_file) # -- Find surface dataset soil levels - # TODO: how? NS uses metadata on file to find + # TODO: how? NS uses metadata on file to find # soil strucure # better suggestion by WW to write dzsoi to neon surface dataset # This todo needs to go to the subset_data # TODO Will: if I sum them up , are they 3.5? (m) YES - print ("soil_top:", soil_top) - print ("soil_bot:", soil_bot) - print ("Sum of soil top depths :", sum(soil_top)) - print ("Sum of soil bottom depths :",sum(soil_bot)) + print("soil_top:", soil_top) + print("soil_bot:", soil_bot) + print("Sum of soil top depths :", sum(soil_top)) + print("Sum of soil bottom depths :", sum(soil_bot)) soil_top = np.cumsum(soil_top) soil_bot = np.cumsum(soil_bot) - soil_mid = 0.5*(soil_bot - soil_top)+soil_top - #print ("Cumulative sum of soil bottom depths :", sum(soil_bot)) + soil_mid = 0.5 * (soil_bot - soil_top) + soil_top + # print ("Cumulative sum of soil bottom depths :", sum(soil_bot)) - obs_top = df['biogeoTopDepth']/100 - obs_bot = df['biogeoBottomDepth']/100 + obs_top = df["biogeoTopDepth"] / 100 + obs_bot = df["biogeoBottomDepth"] / 100 # -- Mapping surface dataset and neon soil levels - bins = df['biogeoTopDepth']/100 - bin_index = np.digitize(soil_mid, bins)-1 + bins = df["biogeoTopDepth"] / 100 + bin_index = np.digitize(soil_mid, bins) - 1 - - ''' + """ print ("================================") print (" Neon data soil structure: ") print ("================================") @@ -491,66 +581,126 @@ def main(): print ("-------------", "{0:.2f}".format(soil_bot[b]), "-------------") - ''' - #-- update fields with neon - f2= f1 - soil_levels = f2['PCT_CLAY'].size - for soil_lev in range(soil_levels): - print ("--------------------------") - print ("soil_lev:",soil_lev) - print (df['clayTotal'][bin_index[soil_lev]]) - f2['PCT_CLAY'][soil_lev] = df['clayTotal'][bin_index[soil_lev]] - f2['PCT_SAND'][soil_lev] = df['sandTotal'][bin_index[soil_lev]] - bulk_den = df['bulkDensExclCoarseFrag'][bin_index[soil_lev]] - carbon_tot = df['carbonTot'][bin_index[soil_lev]] - #print ("carbon_tot:", carbon_tot) - layer_depth = df['biogeoBottomDepth'][bin_index[soil_lev]] - df['biogeoTopDepth'][bin_index[soil_lev]] - f2['ORGANIC'][soil_lev] = carbon_tot * bulk_den * 0.1 / layer_depth * 100 / 0.58 - print ("bin_index:", bin_index[soil_lev]) - print ("layer_depth:", layer_depth) - print ("carbon_tot:",carbon_tot) - print ("bulk_den:",bulk_den) - print ("organic=carbon_tot*bulk_den*0.1/layer_depth * 100/0.58 ") - print ("organic:", f2['ORGANIC'][soil_lev].values) - print ("--------------------------") - - #TODO : max depth for neon sites from WW (zbedrock) - # Update zbedrock if neon observation don't make it down to 2m depth - # zbedrock = neon depth if neon does not make to 2m depth + """ + # -- update fields with neon + f2 = f1 + soil_levels = f2["PCT_CLAY"].size + for soil_lev in range(soil_levels): + print("--------------------------") + print("soil_lev:", soil_lev) + print(df["clayTotal"][bin_index[soil_lev]]) + f2["PCT_CLAY"][soil_lev] = df["clayTotal"][bin_index[soil_lev]] + f2["PCT_SAND"][soil_lev] = df["sandTotal"][bin_index[soil_lev]] + + bulk_den = df["bulkDensExclCoarseFrag"][bin_index[soil_lev]] + carbon_tot = df["carbonTot"][bin_index[soil_lev]] + estimated_oc = df["estimatedOC"][bin_index[soil_lev]] + + # -- estimated_oc in neon data is rounded to the nearest integer. + # -- Check to make sure the rounded oc is not higher than carbon_tot. + # -- Use carbon_tot if estimated_oc is bigger than carbon_tot. + + if estimated_oc > carbon_tot: + estimated_oc = carbon_tot + + layer_depth = ( + df["biogeoBottomDepth"][bin_index[soil_lev]] + - df["biogeoTopDepth"][bin_index[soil_lev]] + ) + + # f2["ORGANIC"][soil_lev] = estimated_oc * bulk_den / 0.58 + + # -- after adding caco3 by NEON: + # -- if caco3 exists: + # -- inorganic = caco3/100.0869*12.0107 + # -- organic = carbon_tot - inorganic + # -- else: + # -- oranigc = estimated_oc * bulk_den /0.58 + + caco3 = df["caco3Conc"][bin_index[soil_lev]] + inorganic = caco3 / 100.0869 * 12.0107 + print("inorganic:", inorganic) + + if not np.isnan(inorganic): + actual_oc = carbon_tot - inorganic + else: + actual_oc = estimated_oc + + f2["ORGANIC"][soil_lev] = actual_oc * bulk_den / 0.58 + + print("~~~~~~~~~~~~~~~~~~~~~~~~") + print("inorganic:") + print("~~~~~~~~~~~~~~~~~~~~~~~~") + print(inorganic) + print("~~~~~~~~~~~~~~~~~~~~~~~~") + + print("bin_index : ", bin_index[soil_lev]) + print("layer_depth : ", layer_depth) + print("carbon_tot : ", carbon_tot) + print("estimated_oc : ", estimated_oc) + print("bulk_den : ", bulk_den) + print("organic :", f2["ORGANIC"][soil_lev].values) + print("--------------------------") + + # -- Interpolate missing values + method = "linear" + fill_interpolate(f2, "PCT_CLAY", method) + fill_interpolate(f2, "PCT_SAND", method) + fill_interpolate(f2, "ORGANIC", method) + + # -- Update zbedrock if neon observation does not make it down to 2m depth rock_thresh = 2 zb_flag = False - if (obs_bot.iloc[-1]> ',*command,'\n') + + try: + subprocess.check_call(command, stdout=open(os.devnull, "w"), stderr=subprocess.STDOUT) + + except subprocess.CalledProcessError as e: + #raise RuntimeError("command '{}' return with error (code {}): {}".format(e.cmd, e.returncode, e.output)) + #print (e.ouput) + print (e) + + + + + + +def main(): + + args = get_parser().parse_args() + + if args.verbose: + logging.basicConfig(level=logging.DEBUG) + + + neon_sites = pd.read_csv('neon_sites_dompft.csv') + + + for i, row in tqdm.tqdm(neon_sites.iterrows()): + lat = row['Lat'] + lon = row['Lon'] + site = row['Site'] + pft = row['pft'] + print ("Now processing site :", site) + command = ['./subset_data.py','point','--lat',str(lat),'--lon',str(lon),'--site',site,'--dompft',str(pft),'--crop'] + execute(command) + + command = ['./modify_singlept_site_neon.py','--neon_site',site] + execute(command) + +if __name__ == "__main__": + main() + diff --git a/tools/site_and_regional/run_neon.py b/tools/site_and_regional/run_neon.py index 17c457b61e..020bc2e8ee 100755 --- a/tools/site_and_regional/run_neon.py +++ b/tools/site_and_regional/run_neon.py @@ -19,7 +19,7 @@ 2) Make the case for the specific neon site(s). 3) Make changes to the case, for: a. AD spinup - b. post-AD spinup + b. post-AD spinup c. transient #--------------- d. SASU or Matrix spinup @@ -67,7 +67,7 @@ import subprocess import pandas as pd import glob -from datetime import datetime +import datetime from getpass import getuser # Get the ctsm util tools and then the cime tools. @@ -106,18 +106,6 @@ def get_parser(args, description, valid_neon_sites): default=["OSBS"], nargs='+') -# not used -# parser.add_argument('--surf-dir', -# help=''' -# Directory of single point surface dataset. -# [default: %(default)s] -# ''', -# action="store", -# dest="surf_dir", -# type =str, -# required=False, -# default="/glade/scratch/"+myname+"/single_point/") - parser.add_argument('--base-case', help=''' Root Directory of base case build @@ -129,13 +117,13 @@ def get_parser(args, description, valid_neon_sites): required=False, default=None) - parser.add_argument('--case-root', + parser.add_argument('--output-root', help=''' - Root Directory of cases + Root output directory of cases [default: %(default)s] ''', action="store", - dest="case_root", + dest="output_root", type =str, required=False, default="CIME_OUTPUT_ROOT as defined in cime") @@ -150,99 +138,83 @@ def get_parser(args, description, valid_neon_sites): required = False, default = False) - subparsers = parser.add_subparsers ( - dest='run_type', - help='Four different ways to run this script.') - - ad_parser = subparsers.add_parser ('ad', - help=''' AD spin-up options ''') - - pad_parser = subparsers.add_parser ('postad', - help=''' Post-AD spin-up options ''') - - tr_parser = subparsers.add_parser ('transient', - help=''' Transient spin-up options ''') + parser.add_argument('--setup-only', + help=''' + Only setup the requested cases, do not build or run + [default: %(default)s] + ''', + action="store_true", + dest="setup_only", + required = False, + default = False) - sasu_parser = subparsers.add_parser ('sasu', - help=''' Sasu spin-up options --not in CTSM yet''') + parser.add_argument('--rerun', + help=''' + If the case exists but does not appear to be complete, restart it. + [default: %(default)s] + ''', + action="store_true", + dest="rerun", + required = False, + default = False) - ad_parser.add_argument ('--ad-length', + parser.add_argument('--no-batch', help=''' - How many years to run AD spin-up + Run locally, do not use batch queueing system (if defined for Machine) [default: %(default)s] ''', + action="store_true", + dest="no_batch", required = False, - type = int, - default = 200) + default = False) - pad_parser.add_argument ('--postad-length', + parser.add_argument('--run-type', + help=''' + Type of run to do + [default: %(default)s] + ''', + choices = ["ad", "postad", "transient", "sasu"], + default = "transient") + + parser.add_argument ('--run-length', help=''' - How many years to run in post-AD mode + How long to run (modified ISO 8601 duration) [default: %(default)s] ''', required = False, - type = int, - default = 100) + type = str, + default = '0Y') - tr_parser.add_argument('--start-year', + parser.add_argument('--start-date', help=''' - Start year for running CTSM simulation. + Start date for running CTSM simulation in ISO format. [default: %(default)s] ''', action="store", - dest="start_year", + dest="start_date", required = False, - type = int, - default = 2018) + type = datetime.date.fromisoformat, + default = datetime.datetime.strptime("2018-01-01",'%Y-%m-%d')) - tr_parser.add_argument('--end-year', + parser.add_argument('--end-date', help=''' - End year for running CTSM simulation. + End date for running CTSM simulation in ISO format. [default: %(default)s] ''', action="store", - dest="end_year", + dest="end_date", required = False, - type = int, - default = 2020) - - #parser.add_argument('--spinup', - # help=''' - # AD spin-up - # [default: %(default)s] - # ''', - # action="store_true", - # dest="ad_flag", - # required = False, - # default = True) - #parser.add_argument('--postad', - # help=''' - # Post-AD spin-up - # [default: %(default)s] - # ''', - # action="store_true", - # dest="postad_flag", - # required = False, - # default = True) - #parser.add_argument('--transient', - # help=''' - # Transient - # [default: %(default)s] - # ''', - # action="store_true", - # dest="transient_flag", - # required = False, - # default = True) - - #parser.add_argument('--sasu','--matrix', - # help=''' - # Matrix (SASU) spin-up - # [default: %(default)s] - # ''', - # action="store_true", - # dest="sasu_flag", - # required = False, - # default = False) + type = datetime.date.fromisoformat, + default = datetime.datetime.strptime("2021-01-01",'%Y-%m-%d')) + + parser.add_argument('--run-from-postad', + help=''' + For transient runs only - should we start from the postad spinup or finidat? + By default start from finidat, if this flag is used the postad run must be available. + ''', + action="store_true", + required = False, + default = False) args = CIME.utils.parse_args_and_handle_standard_logging_options(args, parser) @@ -254,20 +226,53 @@ def get_parser(args, description, valid_neon_sites): if site not in valid_neon_sites: raise ValueError("Invalid site name {}".format(site)) - if "CIME_OUTPUT_ROOT" in args.case_root: - args.case_root = None - if args.run_type: - run_type = args.run_type - else: - run_type = "ad" - if run_type == "ad": - run_length = int(args.ad_length) - elif run_type == "postad": - run_length = int(args.postad_length) - else: - run_length = 0 + if "CIME_OUTPUT_ROOT" in args.output_root: + args.output_root = None - return neon_sites, args.case_root, run_type, args.overwrite, run_length, args.base_case_root + if args.run_length == '0Y': + if args.run_type == 'ad': + run_length = '200Y' + elif args.run_type == 'postad': + run_length = '50Y' + else: + # The transient run length is set by cdeps atm buildnml to the last date of the available tower data + # this value is not used + run_length = '4Y' + + run_length = parse_isoduration(run_length) + base_case_root = None + if args.base_case_root: + base_case_root = os.path.abspath(args.base_case_root) + + # Reduce output level for this script unless --debug or --verbose is provided on the command line + if not args.debug and not args.verbose: + root_logger = logging.getLogger() + root_logger.setLevel(logging.WARN) + + return neon_sites, args.output_root, args.run_type, args.overwrite, run_length, base_case_root, args.run_from_postad, args.setup_only, args.no_batch, args.rerun + +def get_isosplit(s, split): + if split in s: + n, s = s.split(split) + else: + n = 0 + return n, s + +def parse_isoduration(s): + ''' + simple ISO 8601 duration parser, does not account for leap years and assumes 30 day months + ''' + # Remove prefix + s = s.split('P')[-1] + + # Step through letter dividers + years, s = get_isosplit(s, 'Y') + months, s = get_isosplit(s, 'M') + days, s = get_isosplit(s, 'D') + + # Convert all to timedelta + dt = datetime.timedelta(days=int(days)+365*int(years)+30*int(months)) + return int(dt.total_seconds()/86400) class NeonSite : """ @@ -281,19 +286,20 @@ class NeonSite : Methods ------- """ - def __init__(self, name, start_year, end_year, start_month, end_month): + def __init__(self, name, start_year, end_year, start_month, end_month, finidat): self.name = name self.start_year= int(start_year) self.end_year = int(end_year) self.start_month = int(start_month) self.end_month = int(end_month) self.cesmroot = path_to_ctsm_root() + self.finidat = finidat def __str__(self): return str(self.__class__) + '\n' + '\n'.join((str(item) + ' = ' for item in (self.__dict__))) - def build_base_case(self, cesmroot, case_root, res, compset, overwrite): + def build_base_case(self, cesmroot, output_root, res, compset, overwrite=False, setup_only=False): """ Function for building a base_case to clone. To spend less time on building ctsm for the neon cases, @@ -311,77 +317,99 @@ def build_base_case(self, cesmroot, case_root, res, compset, overwrite): overwrite (bool) : Flag to overwrite the case if exists """ - logger.info("---- building a base case -------") - self.base_case_root = case_root + print("---- building a base case -------") + self.base_case_root = output_root user_mods_dirs = [os.path.join(cesmroot,"cime_config","usermods_dirs","NEON",self.name)] - if case_root: - case_path = os.path.join(case_root,self.name) - logger.info ('case_root : {}'.format(case_root)) - else: - case_path = self.name + if not output_root: + output_root = os.getcwd() + case_path = os.path.join(output_root,self.name) logger.info ('base_case_name : {}'.format(self.name)) logger.info ('user_mods_dir : {}'.format(user_mods_dirs[0])) if overwrite and os.path.isdir(case_path): - logger.info ("Removing the existing case at: {}".format(case_path)) + print ("Removing the existing case at: {}".format(case_path)) shutil.rmtree(case_path) - + with Case(case_path, read_only=False) as case: if not os.path.isdir(case_path): - logger.info("---- creating a base case -------") + print("---- creating a base case -------") - case.create(case_path, cesmroot, compset, res, mpilib="mpi-serial", - run_unsupported=True, answer="r", + case.create(case_path, cesmroot, compset, res, + run_unsupported=True, answer="r",output_root=output_root, user_mods_dirs = user_mods_dirs, driver="nuopc") - logger.info("---- base case created ------") + print("---- base case created ------") #--change any config for base_case: #case.set_value("RUN_TYPE","startup") - logger.info("---- base case setup ------") + print("---- base case setup ------") case.case_setup() else: case.case_setup(reset=True) + case_path = case.get_value("CASEROOT") + + if setup_only: + return case_path - logger.info("---- base case build ------") + print("---- base case build ------") # always walk through the build process to make sure it's up to date. t0 = time.time() build.case_build(case_path, case=case) t1 = time.time() total = t1-t0 - logger.info ("Time required to building the base case: {} s.".format(total)) + print ("Time required to building the base case: {} s.".format(total)) # update case_path to be the full path to the base case - case_path = case.get_value("CASEROOT") return case_path def diff_month(self): - d1 = datetime(self.end_year,self.end_month, 1) - d2 = datetime(self.start_year, self.start_month, 1) + d1 = datetime.datetime(self.end_year,self.end_month, 1) + d2 = datetime.datetime(self.start_year, self.start_month, 1) return (d1.year - d2.year) * 12 + d1.month - d2.month - def run_case(self, base_case_root, run_type, run_length, overwrite=False): + def run_case(self, base_case_root, run_type, run_length, overwrite=False, setup_only=False, no_batch=False, rerun=False): user_mods_dirs = [os.path.join(self.cesmroot,"cime_config","usermods_dirs","NEON",self.name)] expect(os.path.isdir(base_case_root), "Error base case does not exist in {}".format(base_case_root)) case_root = os.path.abspath(os.path.join(base_case_root,"..", self.name+"."+run_type)) - if os.path.isdir(case_root) and overwrite: - logger.info("---- removing the existing case -------") - shutil.rmtree(case_root) + rundir = None + if os.path.isdir(case_root): + if overwrite: + print("---- removing the existing case -------") + shutil.rmtree(case_root) + elif rerun: + with Case(case_root, read_only=False) as case: + rundir = case.get_value("RUNDIR") + if os.path.isfile(os.path.join(rundir,"ESMF_Profile.summary")): + print("Case {} appears to be complete, not rerunning.".format(case_root)) + elif not setup_only: + print("Resubmitting case {}".format(case_root)) + case.submit(no_batch=no_batch) + return + else: + logger.warning("Case already exists in {}, not overwritting.".format(case_root)) + return + + if run_type == "postad": + adcase_root = case_root.replace('.postad','.ad') + if not os.path.isdir(adcase_root): + logger.warning("postad requested but no ad case found in {}".format(adcase_root)) + return + if not os.path.isdir(case_root): # read_only = False should not be required here with Case(base_case_root, read_only=False) as basecase: - logger.info("---- cloning the base case in {}".format(case_root)) + print("---- cloning the base case in {}".format(case_root)) basecase.create_clone(case_root, keepexe=True, user_mods_dirs=user_mods_dirs) - with Case(case_root, read_only=False) as case: - case.set_value("STOP_OPTION", "nyears") + # in order to avoid the complication of leap years we always set the run_length in units of days. + case.set_value("STOP_OPTION", "ndays") case.set_value("STOP_N", run_length) - case.set_value("REST_N", 100) + case.set_value("REST_OPTION","end") case.set_value("CONTINUE_RUN", False) if run_type == "ad": @@ -398,13 +426,17 @@ def run_case(self, base_case_root, run_type, run_length, overwrite=False): self.set_ref_case(case) if run_type == "transient": - self.set_ref_case(case) + if self.finidat: + case.set_value("RUN_TYPE","startup") + else: + if not self.set_ref_case(case): + return case.set_value("STOP_OPTION","nmonths") case.set_value("STOP_N", self.diff_month()) - case.set_value("REST_N", "12") case.set_value("DATM_YR_ALIGN",self.start_year) case.set_value("DATM_YR_START",self.start_year) case.set_value("DATM_YR_END",self.end_year) + case.set_value("CALENDAR","GREGORIAN") else: # for the spinup we want the start and end on year boundaries if self.start_month == 1: @@ -418,11 +450,16 @@ def run_case(self, base_case_root, run_type, run_length, overwrite=False): else: case.set_value("DATM_YR_END",self.end_year-1) - - self.modify_user_nl(case_root, run_type) + if not rundir: + rundir = case.get_value("RUNDIR") + + self.modify_user_nl(case_root, run_type, rundir) case.create_namelists() - case.submit() + # explicitly run check_input_data + case.check_all_input_data() + if not setup_only: + case.submit(no_batch=no_batch) def set_ref_case(self, case): rundir = case.get_value("RUNDIR") @@ -433,31 +470,45 @@ def set_ref_case(self, case): else: ref_case_root = case_root.replace(".transient",".postad") root = ".postad" - expect(os.path.isdir(ref_case_root), "ERROR: spinup must be completed first, could not find directory {}".format(ref_case_root)) + if not os.path.isdir(ref_case_root): + logger.warning("ERROR: spinup must be completed first, could not find directory {}".format(ref_case_root)) + return False + with Case(ref_case_root) as refcase: refrundir = refcase.get_value("RUNDIR") case.set_value("RUN_REFDIR", refrundir) case.set_value("RUN_REFCASE", os.path.basename(ref_case_root)) - for reffile in glob.iglob(refrundir + "/{}{}.*.nc".format(self.name, root)): + refdate = None + for reffile in glob.iglob(refrundir + "/{}{}.clm2.r.*.nc".format(self.name, root)): m = re.search("(\d\d\d\d-\d\d-\d\d)-\d\d\d\d\d.nc", reffile) if m: refdate = m.group(1) symlink_force(reffile, os.path.join(rundir,os.path.basename(reffile))) + logger.info("Found refdate of {}".format(refdate)) + if not refdate: + logger.warning("Could not find refcase for {}".format(case_root)) + return False + for rpfile in glob.iglob(refrundir + "/rpointer*"): safe_copy(rpfile, rundir) if not os.path.isdir(os.path.join(rundir, "inputdata")) and os.path.isdir(os.path.join(refrundir,"inputdata")): symlink_force(os.path.join(refrundir,"inputdata"),os.path.join(rundir,"inputdata")) + + case.set_value("RUN_REFDATE", refdate) if case_root.endswith(".postad"): case.set_value("RUN_STARTDATE", refdate) else: case.set_value("RUN_STARTDATE", "{yr:04d}-{mo:02d}-01".format(yr=self.start_year, mo=self.start_month)) - + return True - def modify_user_nl(self, case_root, run_type): + def modify_user_nl(self, case_root, run_type, rundir): user_nl_fname = os.path.join(case_root, "user_nl_clm") - - if run_type != "transient": + user_nl_lines = None + if run_type == "transient": + if self.finidat: + user_nl_lines = ["finidat = '{}/inputdata/lnd/ctsm/initdata/{}'".format(rundir,self.finidat)] + else: user_nl_lines = [ "hist_fincl2 = ''", "hist_mfilt = 20", @@ -465,9 +516,10 @@ def modify_user_nl(self, case_root, run_type): "hist_empty_htapes = .true.", "hist_fincl1 = 'TOTECOSYSC', 'TOTECOSYSN', 'TOTSOMC', 'TOTSOMN', 'TOTVEGC', 'TOTVEGN', 'TLAI', 'GPP', 'CPOOL', 'NPP', 'TWS', 'H2OSNO'"] - with open(user_nl_fname, "a") as fd: - for line in user_nl_lines: - fd.write("{}\n".format(line)) + if user_nl_lines: + with open(user_nl_fname, "a") as fd: + for line in user_nl_lines: + fd.write("{}\n".format(line)) @@ -503,8 +555,9 @@ def parse_neon_listing(listing_file, valid_neon_sites): df = pd.read_csv(listing_file) - #-- TODO: do we want to check for v2 in future? - + # check for finidat files for transient run + finidatlist = df[df['object'].str.contains("lnd/ctsm")] + #-- filter lines with atm/cdep df = df[df['object'].str.contains("atm/cdeps/")] @@ -544,8 +597,12 @@ def parse_neon_listing(listing_file, valid_neon_sites): logger.debug ('end_year={}'.format(end_year)) logger.debug ('start_month={}'.format(start_month)) logger.debug ('end_month={}'.format(end_month)) - - neon_site = NeonSite(site_name, start_year, end_year, start_month, end_month) + finidat = None + for line in finidatlist['object']: + if site_name in line: + finidat = line.split(',')[0].split('/')[-1] + + neon_site = NeonSite(site_name, start_year, end_year, start_month, end_month, finidat) logger.debug (neon_site) available_list.append(neon_site) @@ -577,14 +634,14 @@ def main(description): cesmroot = path_to_ctsm_root() # Get the list of supported neon sites from usermods valid_neon_sites = glob.glob(os.path.join(cesmroot,"cime_config","usermods_dirs","NEON","[!d]*")) - valid_neon_sites = [v.split('/')[-1] for v in valid_neon_sites] - - site_list, case_root, run_type, overwrite, run_length, base_case_root = get_parser(sys.argv, description, valid_neon_sites) + valid_neon_sites = sorted([v.split('/')[-1] for v in valid_neon_sites]) - logger.debug ("case_root : "+ case_root) + site_list, output_root, run_type, overwrite, run_length, base_case_root, run_from_postad, setup_only, no_batch, rerun = get_parser(sys.argv, description, valid_neon_sites) - if not os.path.exists(case_root): - os.makedirs(case_root) + if output_root: + logger.debug ("output_root : "+ output_root) + if not os.path.exists(output_root): + os.makedirs(output_root) #-- check neon listing file for available data: available_list = check_neon_listing(valid_neon_sites) @@ -600,12 +657,14 @@ def main(description): for neon_site in available_list: if neon_site.name in site_list: + if run_from_postad: + neon_site.finidat = None if not base_case_root: - base_case_root = neon_site.build_base_case(cesmroot, case_root, res, - compset, overwrite) + base_case_root = neon_site.build_base_case(cesmroot, output_root, res, + compset, overwrite, setup_only) logger.info ("-----------------------------------") logger.info ("Running CTSM for neon site : {}".format(neon_site.name)) - neon_site.run_case(base_case_root, run_type, run_length, overwrite) + neon_site.run_case(base_case_root, run_type, run_length, overwrite, setup_only, no_batch, rerun) if __name__ == "__main__": main(__doc__) diff --git a/tools/site_and_regional/subset_data.py b/tools/site_and_regional/subset_data.py index 61c3fe0cf2..a649c657b9 100755 --- a/tools/site_and_regional/subset_data.py +++ b/tools/site_and_regional/subset_data.py @@ -1,4 +1,4 @@ -#! /usr/bin/env python3 +#! /usr/bin/env python """ |------------------------------------------------------------------| |--------------------- Instructions -----------------------------| @@ -178,6 +178,33 @@ def get_parser(): required = False, type = int, default = 2014) + pt_parser.add_argument('--datm_from_tower', + help='Flag for creating DATM forcing data at single point for a tower data. [default: %(default)s]', + action="store", + dest="datm_tower", + type = str2bool, + nargs = '?', + const = True, + required = False, + default = False) + pt_parser.add_argument('--create_user_mods', + help='Flag for creating user mods directory . [default: %(default)s]', + action="store", + dest="datm_tower", + type = str2bool, + nargs = '?', + const = True, + required = False, + default = False) + pt_parser.add_argument('--user_mods_dir', + help='Flag for creating user mods directory . [default: %(default)s]', + action="store", + dest="user_mod_dir", + type = str, + nargs = '?', + const = True, + required = False, + default = False) pt_parser.add_argument('--crop', help='Create datasets using the extensive list of prognostic crop types. [default: %(default)s]', action="store_true", @@ -369,9 +396,9 @@ def plon_type(x): """ x = float(x) if (-180 < x) and (x < 0): - print ("lon is :", lon) + print ("lon is :", x) x= x%360 - print ("after modulo lon is :", lon) + print ("after modulo lon is :", x) if (x < 0) or (x > 360): raise argparse.ArgumentTypeError("ERROR: Latitude of single point should be between 0 and 360 or -180 and 180.") return x @@ -380,11 +407,7 @@ def get_git_sha(): """ Returns Git short SHA for the currect directory. """ - try: - sha = subprocess.check_output(['git', 'rev-parse', '--short', 'HEAD']).strip().decode() - except subprocess.CalledProcessError: - sha = "NOT-A-GIT-REPOSITORY" - return sha + return subprocess.check_output(['git', 'rev-parse', '--short', 'HEAD']).strip().decode() class BaseCase : """ @@ -552,8 +575,11 @@ def create_fileout_name( filename,tag): items = basename.split('_') today = date.today() today_string = today.strftime("%y%m%d") + print (items[-1]) + #new_string = items[0]+"_"+items[2]+"_"+items[3]+"_"+ items[4] \ + # +"_"+items[5]+"_"+items[6]+"_"+tag+"_c"+today_string+".nc" new_string = items[0]+"_"+items[2]+"_"+items[3]+"_"+ items[4] \ - +"_"+items[5]+"_"+items[6]+"_"+tag+"_c"+today_string+".nc" + +"_"+items[5]+"_"+tag+"_c"+today_string+".nc" return new_string def create_domain_at_point (self): @@ -622,7 +648,9 @@ def create_surfdata_at_point(self): # modify surface data properties if self.overwrite_single_pft: f3['PCT_NAT_PFT'][:,:,:] = 0 - f3['PCT_NAT_PFT'][:,:,self.dominant_pft] = 100 + if (self.dominant_pft <16): + f3['PCT_NAT_PFT'][:,:,self.dominant_pft] = 100 + #else:@@@ if self.zero_nonveg_landunits: f3['PCT_NATVEG'][:,:] = 100 f3['PCT_CROP'][:,:] = 0 @@ -638,6 +666,14 @@ def create_surfdata_at_point(self): # specify dimension order #f3 = f3.transpose(u'time', u'cft', u'natpft', u'lsmlat', u'lsmlon') f3 = f3.transpose(u'time', u'cft', u'lsmpft', u'natpft', u'nglcec', u'nglcecp1', u'nlevsoi', u'nlevurb', u'numrad', u'numurbl', 'lsmlat', 'lsmlon') + + #update lsmlat and lsmlon to match site specific instead of the nearest point + #f3['lon']= self.plon + #f3['lat']= self.plat + f3['lsmlon']= np.atleast_1d(self.plon) + f3['lsmlat']= np.atleast_1d(self.plat) + f3['LATIXY'][:,:]= self.plat + f3['LONGXY'][:,:]= self.plon #update attributes self.update_metadata(f3) @@ -789,7 +825,7 @@ def create_surfdata_at_reg(self): f3.attrs['Created_from'] = self.fsurf_in # mode 'w' overwrites file - f3.to_netcdf(path=self.fsurf_out, mode='w') + f3.to_netcdf(path=self.fsurf_out, mode='w', format='NETCDF3_64BIT') print('created file (fsurf_out)'+self.fsurf_out) #f1.close(); f2.close(); f3.close()