From 85f72cc1c59359ee1e7a907cd1ca9879acb54716 Mon Sep 17 00:00:00 2001 From: kaveh Date: Fri, 15 Jul 2022 05:17:10 -0600 Subject: [PATCH 01/95] Fix albgrd_col value in SurfaceAlbedoType --- src/biogeophys/SurfaceAlbedoType.F90 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/biogeophys/SurfaceAlbedoType.F90 b/src/biogeophys/SurfaceAlbedoType.F90 index cf6b0a518a..dba2938d98 100644 --- a/src/biogeophys/SurfaceAlbedoType.F90 +++ b/src/biogeophys/SurfaceAlbedoType.F90 @@ -188,7 +188,7 @@ subroutine InitHistory(this, bounds) avgflag='A', long_name='cosine of solar zenith angle', & ptr_col=this%coszen_col, default='inactive') - this%albgri_col(begc:endc,:) = spval + this%albgrd_col(begc:endc,:) = spval call hist_addfld2d (fname='ALBGRD', units='proportion', type2d='numrad', & avgflag='A', long_name='ground albedo (direct)', & ptr_col=this%albgrd_col, default='inactive') From 8606f39997bc8825f66ed528405fa2d9f7e6b892 Mon Sep 17 00:00:00 2001 From: kaveh Date: Tue, 19 Jul 2022 02:43:31 -0600 Subject: [PATCH 02/95] Add error checking in ParitionWoodFluxes --- src/biogeochem/CNProductsMod.F90 | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/src/biogeochem/CNProductsMod.F90 b/src/biogeochem/CNProductsMod.F90 index 9744b04aed..176ff77c27 100644 --- a/src/biogeochem/CNProductsMod.F90 +++ b/src/biogeochem/CNProductsMod.F90 @@ -615,19 +615,12 @@ subroutine PartitionWoodFluxes(this, bounds, & if (pprod_tot > 0) then pprod10_frac = pprod10 / pprod_tot pprod100_frac = pprod100 / pprod_tot - else - ! Avoid divide by 0 - pprod10_frac = 0._r8 - pprod100_frac = 0._r8 + else if (dwt_wood_product_gain_patch(p) > 0) then + call endrun(& + msg='ERROR: dwt_wood_product_gain_patch(p) > 0' // & + errMsg(sourcefile, __LINE__)) end if - ! Note that the patch-level fluxes are expressed per unit gridcell area. So, to go - ! from patch-level fluxes to gridcell-level fluxes, we simply add up the various - ! patch contributions, without having to multiply by any area weightings. - this%dwt_prod10_gain_grc(g) = this%dwt_prod10_gain_grc(g) + & - dwt_wood_product_gain_patch(p) * pprod10_frac - this%dwt_prod100_gain_grc(g) = this%dwt_prod100_gain_grc(g) + & - dwt_wood_product_gain_patch(p) * pprod100_frac end do end subroutine PartitionWoodFluxes From f0f28e711a3d33a184a312560475e401c73e50c7 Mon Sep 17 00:00:00 2001 From: wwieder Date: Fri, 14 Oct 2022 14:38:08 -0600 Subject: [PATCH 03/95] python updates for #1871 --- .../modify_singlept_site_neon.py | 2 +- tools/site_and_regional/neon_sites_dompft.csv | 88 +++++++++---------- tools/site_and_regional/neon_surf_wrapper.py | 2 +- 3 files changed, 46 insertions(+), 46 deletions(-) diff --git a/tools/site_and_regional/modify_singlept_site_neon.py b/tools/site_and_regional/modify_singlept_site_neon.py index 3610b0bb5c..336366a49b 100755 --- a/tools/site_and_regional/modify_singlept_site_neon.py +++ b/tools/site_and_regional/modify_singlept_site_neon.py @@ -682,7 +682,7 @@ def main(): print("Updating PCT_NAT_PFT") #print (f2.PCT_NAT_PFT) print(f2.PCT_NAT_PFT.values[0]) - f2.PCT_NAT_PFT.values[0] = [[100.0]] + #f2.PCT_NAT_PFT.values[0] = [[100.0]] print(f2.PCT_NAT_PFT[0].values) out_dir = args.out_dir diff --git a/tools/site_and_regional/neon_sites_dompft.csv b/tools/site_and_regional/neon_sites_dompft.csv index ad5d14e53d..e67d67375b 100644 --- a/tools/site_and_regional/neon_sites_dompft.csv +++ b/tools/site_and_regional/neon_sites_dompft.csv @@ -1,48 +1,48 @@ ,Site,Domain,Lat,Lon,pft,start_year,end_year -1,BART,1,44.06516,-71.28834,7,2018,2021 -2,HARV,1,42.53562,-72.17562,7,2018,2021 -3,BLAN,2,39.06044,-78.07115,7,2018,2021 -4,SCBI,2,38.89209,-78.13764,7,2018,2021 -5,SERC,2,38.89124,-76.55884,7,2018,2021 -6,DSNY,3,28.12919,-81.43394,14,2018,2021 -7,JERC,3,31.19608,-84.46647,1,2018,2021 -8,OSBS,3,29.68819,-81.99345,1,2018,2021 -9,GUAN,4,17.96882,-66.86888,6,2019,2021 -10,LAJA,4,18.02184,-67.07608,14,2019,2021 -11,STEI,5,45.5076,-89.5888,7,2018,2021 -12,TREE,5,45.49266,-89.58748,7,2018,2021 -13,UNDE,5,46.14103,-89.3221,7,2018,2021 -14,KONA,6,39.10828,-96.61044,18,2018,2021 -15,KONZ,6,39.1007,-96.56227,14,2018,2021 -16,UKFS,6,39.04168,-95.20495,7,2018,2021 -17,GRSM,7,35.68839,-83.50185,7,2018,2021 -18,MLBS,7,37.37783,-80.52425,7,2018,2019 -19,ORNL,7,35.57525,-84.16581,7,2018,2021 -20,DELA,8,32.54092,-87.80341,7,2018,2021 -21,LENO,8,31.8531,-88.16103,7,2021,2021 -22,TALL,8,32.95106,-87.3941,1,2018,2021 -23,DCFS,9,47.15919,-99.11251,13,2018,2021 -24,NOGP,9,46.76846,-100.91832,13,2018,2021 -25,WOOD,9,47.12833,-99.23907,13,2018,2021 -26,CPER,10,40.81297,-104.74455,14,2018,2021 -27,RMNP,10,40.27707,-105.54524,1,2018,2021 -28,STER,10,40.45984,-103.03008,18,2018,2021 -29,CLBJ,11,33.40143,-97.56725,7,2018,2021 -30,OAES,11,35.41062,-99.06044,14,2018,2021 -31,YELL,12,44.95597,-110.54196,1,2019,2021 -32,MOAB,13,38.25136,-109.38882,14,2018,2020 -33,NIWO,13,40.05236,-105.58324,12,2018,2021 -34,JORN,14,32.59052,-106.84377,14,2018,2021 -35,SRER,14,31.91068,-110.83549,9,2018,2021 -36,ONAQ,15,35.68839,-83.50185,9,2018,2019 +1,BART, 1, 44.06516, -71.28834,7,2018,2021 +2,HARV, 1, 42.53562, -72.17562,7,2018,2021 +3,BLAN, 2, 39.06044, -78.07115,7,2018,2021 +4,SCBI, 2, 38.89209, -78.13764,7,2018,2021 +5,SERC, 2, 38.89124, -76.55884,7,2018,2021 +6,DSNY, 3, 28.12919, -81.43394,14,2018,2021 +7,JERC, 3, 31.19608, -84.46647,1,2018,2021 +8,OSBS, 3, 29.68819, -81.99345,1,2018,2021 +9,GUAN, 4, 17.96882, -66.86888,6,2019,2021 +10,LAJA, 4, 18.02184, -67.07608,14,2019,2021 +11,STEI, 5, 45.50760, -89.58880,7,2018,2021 +12,TREE, 5, 45.49266, -89.58748,7,2018,2021 +13,UNDE, 5, 46.14103, -89.32210,7,2018,2021 +14,KONA, 6, 39.10828, -96.61044,19,2018,2021 +15,KONZ, 6, 39.10070, -96.56227,14,2018,2021 +16,UKFS, 6, 39.04168, -95.20495,7,2018,2021 +17,GRSM, 7, 35.68839, -83.50185,7,2018,2021 +18,MLBS, 7, 37.37783, -80.52425,7,2018,2019 +19,ORNL, 7, 35.57525, -84.16581,7,2018,2021 +20,DELA, 8, 32.54092, -87.80341,7,2018,2021 +21,LENO, 8, 31.85310, -88.16103,7,2021,2021 +22,TALL, 8, 32.95106, -87.39410,1,2018,2021 +23,DCFS, 9, 47.15919, -99.11251,13,2018,2021 +24,NOGP, 9, 46.76846, -100.91832,13,2018,2021 +25,WOOD, 9, 47.12833, -99.23907,13,2018,2021 +26,CPER,10, 40.81297, -104.74455,14,2018,2021 +27,RMNP,10, 40.27707, -105.54524,1,2018,2021 +28,STER,10, 40.45984, -103.03008,19,2018,2021 +29,CLBJ,11, 33.40143, -97.56725,7,2018,2021 +30,OAES,11, 35.41062, -99.06044,14,2018,2021 +31,YELL,12, 44.95597, -110.54196,1,2019,2021 +32,MOAB,13, 38.25136, -109.38882,14,2018,2020 +33,NIWO,13, 40.05236, -105.58324,12,2018,2021 +34,JORN,14, 32.59052, -106.84377,14,2018,2021 +35,SRER,14, 31.91068, -110.83549,9,2018,2021 +36,ONAQ,15, 35.68839, -83.50185,9,2018,2019 37,ABBY,16,45.762378,-122.329672,1,2018,2021 -38,WREF,16,45.81637,-121.95838,1,2019,2021 -39,SJER,17,37.107117,-119.733,13,2019,2021 -40,SOAP,17,37.03269,-119.2621,1,2018,2021 +38,WREF,16, 45.81637, -121.95838,1,2019,2021 +39,SJER,17,37.107117, -119.73300,13,2019,2021 +40,SOAP,17,37.032690, -119.26210,1,2018,2021 41,TEAK,17,37.006472,-119.005758,1,2019,2021 -42,TOOL,17,68.66045,-149.370128,1,2020,2021 +42,TOOL,17, 68.66045,-149.370128,1,2020,2021 43,BARR,18,71.281711,-156.650219,12,2019,2021 -44,BONA,19,65.15333,-147.50194,2,2018,2021 -45,DEJU,19,63.87983,-145.74765,2,2018,2021 -46,HEAL,19,63.8798,-149.21539,12,2018,2021 -47,PUUM,20,19.55309,-155.31731,4,2020,2020 +44,BONA,19, 65.15333, -147.50194,2,2018,2021 +45,DEJU,19, 63.87983, -145.74765,2,2018,2021 +46,HEAL,19, 63.87980, -149.21539,12,2018,2021 +47,PUUM,20, 19.55309, -155.31731,4,2020,2020 diff --git a/tools/site_and_regional/neon_surf_wrapper.py b/tools/site_and_regional/neon_surf_wrapper.py index df58d3ab36..2f8a00500c 100755 --- a/tools/site_and_regional/neon_surf_wrapper.py +++ b/tools/site_and_regional/neon_surf_wrapper.py @@ -98,7 +98,7 @@ def main(): pft = row['pft'] print ("Now processing site :", site) command = ['./subset_data','point','--lat',str(lat),'--lon',str(lon),'--site',site,'--dompft',str(pft),'--crop', - '--create-surface','--uniform-snowpack','--cap-saturation','--verbose'] + '--create-surface','--uniform-snowpack','--cap-saturation','--verbose','--overwrite'] execute(command) command = ['./modify_singlept_site_neon.py','--neon_site',site, '--surf_dir', From c616dff513c65194e43f6c108a7f0099631e49cc Mon Sep 17 00:00:00 2001 From: wwieder Date: Fri, 14 Oct 2022 14:41:49 -0600 Subject: [PATCH 04/95] usermod changes for #1871 --- cime_config/usermods_dirs/NEON/WREF/shell_commands | 4 ++++ cime_config/usermods_dirs/NEON/defaults/user_nl_clm | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/cime_config/usermods_dirs/NEON/WREF/shell_commands b/cime_config/usermods_dirs/NEON/WREF/shell_commands index 77a0b750cd..bbb0e14312 100644 --- a/cime_config/usermods_dirs/NEON/WREF/shell_commands +++ b/cime_config/usermods_dirs/NEON/WREF/shell_commands @@ -1,3 +1,7 @@ ./xmlchange NEONSITE=WREF ./xmlchange PTS_LON=238.04162 ./xmlchange PTS_LAT=45.81637 +./xmlchange RUN_STARTDATE=2019-01-01 +./xmlchange DATM_YR_ALIGN=2019,DATM_YR_START=2019 +./xmlchange STOP_N=39 + diff --git a/cime_config/usermods_dirs/NEON/defaults/user_nl_clm b/cime_config/usermods_dirs/NEON/defaults/user_nl_clm index 332060dd99..49528a524e 100644 --- a/cime_config/usermods_dirs/NEON/defaults/user_nl_clm +++ b/cime_config/usermods_dirs/NEON/defaults/user_nl_clm @@ -19,7 +19,7 @@ !---------------------------------------------------------------------------------- flanduse_timeseries = ' ' ! This isn't needed for a non transient case, but will be once we start using transient compsets -fsurdat = "$DIN_LOC_ROOT/lnd/clm2/surfdata_map/NEON/surfdata_hist_78pfts_CMIP6_simyr2000_${NEONSITE}_c211102.nc" +fsurdat = "$DIN_LOC_ROOT/lnd/clm2/surfdata_map/NEON/surfdata_0.9x1.25_hist_78pfts_CMIP6_simyr2000_${NEONSITE}_c221014.nc" model_year_align_urbantv = 2018 stream_year_first_urbantv = 2018 stream_year_last_urbantv = 2021 From b9736ebc91eea7b59c4349cfdbb19010ca196268 Mon Sep 17 00:00:00 2001 From: wwieder Date: Fri, 14 Oct 2022 16:34:15 -0600 Subject: [PATCH 05/95] local gcs modifications --- tools/site_and_regional/neon_gcs_upload | 171 ++++++++++++++++++++++++ 1 file changed, 171 insertions(+) create mode 100755 tools/site_and_regional/neon_gcs_upload diff --git a/tools/site_and_regional/neon_gcs_upload b/tools/site_and_regional/neon_gcs_upload new file mode 100755 index 0000000000..4854ac8b99 --- /dev/null +++ b/tools/site_and_regional/neon_gcs_upload @@ -0,0 +1,171 @@ +#! /usr/bin/env python3 +""" +Script to rename and upload NEON site finidat files for use in transient startup cases + +""" + +import os, sys +# Get the ctsm util tools and then the cime tools. +_CTSM_PYTHON = os.path.abspath(os.path.join(os.path.dirname(__file__), "..","..",'python')) +sys.path.insert(1, _CTSM_PYTHON) + +from google.cloud import storage +import glob +import datetime +from ctsm import add_cime_to_path +from ctsm.path_utils import path_to_ctsm_root +from standard_script_setup import * +from CIME.case import Case +from CIME.utils import expect, safe_copy + +logger = logging.getLogger(__name__) + +def get_parser(args, description, valid_neon_sites): + """ + Get Parser object for this script + """ + parser = argparse.ArgumentParser(description=description, + formatter_class=argparse.RawDescriptionHelpFormatter) + + CIME.utils.setup_standard_logging_options(parser) + + parser.print_usage = parser.print_help + + parser.add_argument('--neon-sites', + help='4-letter neon site code.', + action="store", + required=False, + choices=valid_neon_sites + ['all'], + dest="neon_sites", + default=["OSBS"], + nargs='+') + + parser.add_argument('--output-root', + help=''' + Root Directory of case results + [default: %(default)s] + ''', + action="store", + dest="output_root", + type =str, + required=False, + default=os.getcwd()) + + parser.add_argument('--file-date', + help=''' + Date of ctsm restart file(s) to upload + ''', + action="store", + dest="file_date", + required = False, + type = datetime.date.fromisoformat, + default = datetime.datetime.strptime("0318-01-01",'%Y-%m-%d')) + + + parser.add_argument('--upload-finidat', + help=''' + Upload the final restart files from the end of the postad run for each site. + ''', + action="store_true", + dest="upload_finidat", + required = False, + default = False) + + parser.add_argument('--upload-history', + help=''' + Upload the transient run h1 history files for each site. + ''', + action="store_true", + dest="upload_history", + required = False, + default = False) + + + + args = CIME.utils.parse_args_and_handle_standard_logging_options(args, parser) + + if 'all' in args.neon_sites: + neon_sites = valid_neon_sites + else: + neon_sites = args.neon_sites + for site in neon_sites: + if site not in valid_neon_sites: + raise ValueError("Invalid site name {}".format(site)) + + expect(args.upload_finidat or args.upload_history,"Must specify at least one of --upload-finidat or --upload-history") + + return neon_sites, args.output_root, args.file_date, args.upload_finidat, args.upload_history + +def upload_blob(bucket_name, source_file_name, destination_blob_name): + """Uploads a file to the bucket.""" + # The ID of your GCS bucket + # bucket_name = "your-bucket-name" + # The path to your file to upload + # source_file_name = "local/path/to/file" + # The ID of your GCS object + # destination_blob_name = "storage-object-name" + + storage_client = storage.Client() + bucket = storage_client.bucket(bucket_name) + blob = bucket.blob(destination_blob_name) + + blob.upload_from_filename(source_file_name) + + print( + f"File {source_file_name} uploaded to {destination_blob_name}." + ) + +def main(description): + """ + For each site in the site_list find the site.postad run directory and grab the latest clm restart file + from there, + """ + + #if not os.path.isfile(os.path.join(os.getenv("HOME"),".aws","credentials")): + # raise FileNotFoundError("User account must have valid aws credentials to run this script.") + + cesmroot = path_to_ctsm_root() + os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = os.path.join(os.environ["HOME"],'.uploadGCSkey.json') + #os.path.join(os.environ["HOME"],"gcwriter") + + # Get the list of supported neon sites from usermods + valid_neon_sites = glob.glob(os.path.join(cesmroot,"cime_config","usermods_dirs","NEON","[!d]*")) + valid_neon_sites = [v.split('/')[-1] for v in valid_neon_sites] + filedatestamp = datetime.datetime.now().date() + site_list, output_root, file_date, upload_finidat, upload_history = get_parser(sys.argv, description, valid_neon_sites) + for site in site_list: + rundir = None + if upload_finidat: + logger.info("Upload finidat for {}".format(site)) + case_path = os.path.join(output_root, site+".postad") + if os.path.isdir(case_path): + with Case(case_path) as case: + rundir = case.get_value("RUNDIR") + basefile = site+".postad.clm2.r.{}-00000.nc".format(file_date.strftime("%4Y-%m-%d")) + finidat_file = os.path.join(rundir,basefile) + if not os.path.isfile(finidat_file): + logger.warning("Could not find file {}".format(finidat_file)) + continue + newfile = basefile.replace(".postad.",".{}.".format(filedatestamp)) + upload_blob("neon-prod-ncar", finidat_file, newfile) + + if upload_history: + logger.info("Upload history for {}".format(site)) + case_path = os.path.join(output_root, site+".transient") + if not os.path.isdir(case_path): + logger.warning("No case found in {}".format(case_path)) + continue + with Case(case_path) as case: + archive_dir = os.path.join(case.get_value("DOUT_S_ROOT"),"lnd","hist") + for histfile in glob.iglob(archive_dir + "/*.h1.*"): + newfile = os.path.basename(histfile) + upload_blob("neon-prod-ncar", histfile, os.path.join("NEON","archive",site,"lnd","hist",newfile)) + + + + + + +if __name__ == "__main__": + main(__doc__) + From ae3d8b3a6f020a03a29d238499c0f7775c0d4bc3 Mon Sep 17 00:00:00 2001 From: wwieder Date: Sun, 16 Oct 2022 06:19:39 -0600 Subject: [PATCH 06/95] modified path for uploading to GCS --- tools/site_and_regional/neon_gcs_upload | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/site_and_regional/neon_gcs_upload b/tools/site_and_regional/neon_gcs_upload index 4854ac8b99..d69534db19 100755 --- a/tools/site_and_regional/neon_gcs_upload +++ b/tools/site_and_regional/neon_gcs_upload @@ -147,7 +147,7 @@ def main(description): logger.warning("Could not find file {}".format(finidat_file)) continue newfile = basefile.replace(".postad.",".{}.".format(filedatestamp)) - upload_blob("neon-prod-ncar", finidat_file, newfile) + upload_blob("neon-ncar-artifacts", finidat_file, newfile) if upload_history: logger.info("Upload history for {}".format(site)) @@ -159,7 +159,7 @@ def main(description): archive_dir = os.path.join(case.get_value("DOUT_S_ROOT"),"lnd","hist") for histfile in glob.iglob(archive_dir + "/*.h1.*"): newfile = os.path.basename(histfile) - upload_blob("neon-prod-ncar", histfile, os.path.join("NEON","archive",site,"lnd","hist",newfile)) + upload_blob("neon-ncar-artifacts", histfile, os.path.join("NEON","archive",site,"lnd","hist",newfile)) From 3c9615ba7a4b493d1b6940bb4854bd919e398f5e Mon Sep 17 00:00:00 2001 From: Kaveh Karimi <4112646+ka7eh@users.noreply.github.com> Date: Tue, 18 Oct 2022 08:59:47 +0200 Subject: [PATCH 07/95] Fix per comment https://github.com/ESCOMP/CTSM/pull/1814/files#r997597612 --- src/biogeochem/CNProductsMod.F90 | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/biogeochem/CNProductsMod.F90 b/src/biogeochem/CNProductsMod.F90 index 176ff77c27..b6e5d6dad3 100644 --- a/src/biogeochem/CNProductsMod.F90 +++ b/src/biogeochem/CNProductsMod.F90 @@ -615,6 +615,13 @@ subroutine PartitionWoodFluxes(this, bounds, & if (pprod_tot > 0) then pprod10_frac = pprod10 / pprod_tot pprod100_frac = pprod100 / pprod_tot + ! Note that the patch-level fluxes are expressed per unit gridcell area. So, to go + ! from patch-level fluxes to gridcell-level fluxes, we simply add up the various + ! patch contributions, without having to multiply by any area weightings. + this%dwt_prod10_gain_grc(g) = this%dwt_prod10_gain_grc(g) + & + dwt_wood_product_gain_patch(p) * pprod10_frac + this%dwt_prod100_gain_grc(g) = this%dwt_prod100_gain_grc(g) + & + dwt_wood_product_gain_patch(p) * pprod100_frac else if (dwt_wood_product_gain_patch(p) > 0) then call endrun(& msg='ERROR: dwt_wood_product_gain_patch(p) > 0' // & From 8a7777cf3717e33e7bb6f800ddc0f6f0cda4d4c5 Mon Sep 17 00:00:00 2001 From: will wieder Date: Wed, 19 Oct 2022 14:44:48 -0600 Subject: [PATCH 08/95] Update neon_gcs_upload Correct path for initial conditions --- tools/site_and_regional/neon_gcs_upload | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/site_and_regional/neon_gcs_upload b/tools/site_and_regional/neon_gcs_upload index d69534db19..9853c96849 100755 --- a/tools/site_and_regional/neon_gcs_upload +++ b/tools/site_and_regional/neon_gcs_upload @@ -147,7 +147,7 @@ def main(description): logger.warning("Could not find file {}".format(finidat_file)) continue newfile = basefile.replace(".postad.",".{}.".format(filedatestamp)) - upload_blob("neon-ncar-artifacts", finidat_file, newfile) + upload_blob("neon-ncar-artifacts", finidat_file, os.path.join("NEON","lnd","ctsm","initdata",newfile) ) if upload_history: logger.info("Upload history for {}".format(site)) From 1cf1d17ae749e4da3a8a80e7d2b76d3b332b489f Mon Sep 17 00:00:00 2001 From: will wieder Date: Wed, 19 Oct 2022 16:40:21 -0600 Subject: [PATCH 09/95] Update shell_commands BARR --- cime_config/usermods_dirs/NEON/BARR/shell_commands | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/cime_config/usermods_dirs/NEON/BARR/shell_commands b/cime_config/usermods_dirs/NEON/BARR/shell_commands index 55037c6b37..29ddc20f52 100644 --- a/cime_config/usermods_dirs/NEON/BARR/shell_commands +++ b/cime_config/usermods_dirs/NEON/BARR/shell_commands @@ -1,7 +1,10 @@ +#!/bin/bash + ./xmlchange NEONSITE=BARR ./xmlchange PTS_LON=203.349781 ./xmlchange PTS_LAT=71.281711 -./xmlchange RUN_STARTDATE=2019-01-01 -./xmlchange DATM_YR_ALIGN=2019,DATM_YR_START=2019,DATM_YR_END=2022 -./xmlchange STOP_N=39 - +./xmlchange DATM_YR_ALIGN=2019,DATM_YR_START=2019 +if [ "`./xmlquery CLM_NML_USE_CASE --value`" =~ "*_transient" ]; then + ./xmlchange RUN_STARTDATE=2019-01-01 + ./xmlchange STOP_N=39 +fi From c2d185ea593254b9a1bc5934a9278fcfe9cf1086 Mon Sep 17 00:00:00 2001 From: will wieder Date: Wed, 19 Oct 2022 16:43:15 -0600 Subject: [PATCH 10/95] Update shell_commands GUAN --- cime_config/usermods_dirs/NEON/GUAN/shell_commands | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/cime_config/usermods_dirs/NEON/GUAN/shell_commands b/cime_config/usermods_dirs/NEON/GUAN/shell_commands index ee2eca82d9..f886245a1a 100644 --- a/cime_config/usermods_dirs/NEON/GUAN/shell_commands +++ b/cime_config/usermods_dirs/NEON/GUAN/shell_commands @@ -1,6 +1,10 @@ +#!/bin/bash + ./xmlchange NEONSITE=GUAN ./xmlchange PTS_LON=293.13112 ./xmlchange PTS_LAT=17.96882 -./xmlchange RUN_STARTDATE=2019-01-01 ./xmlchange DATM_YR_ALIGN=2019,DATM_YR_START=2019 -./xmlchange STOP_N=39 +if [ "`./xmlquery CLM_NML_USE_CASE --value`" =~ "*_transient" ]; then + ./xmlchange RUN_STARTDATE=2019-01-01 + ./xmlchange STOP_N=39 +fi From 5f1d190c391c130d44b4c18147e70e8ab96025a4 Mon Sep 17 00:00:00 2001 From: will wieder Date: Wed, 19 Oct 2022 16:46:26 -0600 Subject: [PATCH 11/95] Update shell_commands LAJA --- cime_config/usermods_dirs/NEON/LAJA/shell_commands | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/cime_config/usermods_dirs/NEON/LAJA/shell_commands b/cime_config/usermods_dirs/NEON/LAJA/shell_commands index 522818a697..7e946bf9b4 100644 --- a/cime_config/usermods_dirs/NEON/LAJA/shell_commands +++ b/cime_config/usermods_dirs/NEON/LAJA/shell_commands @@ -1,8 +1,11 @@ +#!/bin/bash ./xmlchange NEONSITE=LAJA ./xmlchange PTS_LON=292.92392 ./xmlchange PTS_LAT=18.02184 -./xmlchange RUN_STARTDATE=2019-01-01 ./xmlchange DATM_YR_ALIGN=2019,DATM_YR_START=2019 -./xmlchange STOP_N=39 +if [ "`./xmlquery CLM_NML_USE_CASE --value`" =~ "*_transient" ]; then + ./xmlchange RUN_STARTDATE=2019-01-01 + ./xmlchange STOP_N=39 +fi From 63dd7795419a09d85e409ce8e6f57fe8fa842e4a Mon Sep 17 00:00:00 2001 From: will wieder Date: Wed, 19 Oct 2022 16:47:51 -0600 Subject: [PATCH 12/95] Update shell_commands LENO --- cime_config/usermods_dirs/NEON/LENO/shell_commands | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/cime_config/usermods_dirs/NEON/LENO/shell_commands b/cime_config/usermods_dirs/NEON/LENO/shell_commands index 89583ed158..3af172e8f0 100644 --- a/cime_config/usermods_dirs/NEON/LENO/shell_commands +++ b/cime_config/usermods_dirs/NEON/LENO/shell_commands @@ -1,7 +1,11 @@ +#!/bin/bash + ./xmlchange NEONSITE=LENO ./xmlchange PTS_LON=271.83897 ./xmlchange PTS_LAT=31.8531 -./xmlchange RUN_STARTDATE=2021-01-01 ./xmlchange DATM_YR_ALIGN=2021,DATM_YR_START=2021 -./xmlchange STOP_N=15 +if [ "`./xmlquery CLM_NML_USE_CASE --value`" =~ "*_transient" ]; then + ./xmlchange RUN_STARTDATE=2021-01-01 + ./xmlchange STOP_N=15 +fi From b9acc522c886296d3206ec46ee0cae8b13d555eb Mon Sep 17 00:00:00 2001 From: will wieder Date: Wed, 19 Oct 2022 16:49:07 -0600 Subject: [PATCH 13/95] Update shell_commands MLBS --- cime_config/usermods_dirs/NEON/MLBS/shell_commands | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/cime_config/usermods_dirs/NEON/MLBS/shell_commands b/cime_config/usermods_dirs/NEON/MLBS/shell_commands index 9f70ecd662..6467bad1cc 100644 --- a/cime_config/usermods_dirs/NEON/MLBS/shell_commands +++ b/cime_config/usermods_dirs/NEON/MLBS/shell_commands @@ -1,6 +1,8 @@ +#!/bin/bash ./xmlchange NEONSITE=MLBS ./xmlchange PTS_LON=279.47575 ./xmlchange PTS_LAT=37.37783 -./xmlchange STOP_N=24 ./xmlchange DATM_YR_END=2019 - +if [ "`./xmlquery CLM_NML_USE_CASE --value`" =~ "*_transient" ]; then + ./xmlchange STOP_N=24 +fi From d2f502a335863ddce85ca1882dd6fc1f9d758099 Mon Sep 17 00:00:00 2001 From: will wieder Date: Wed, 19 Oct 2022 16:50:23 -0600 Subject: [PATCH 14/95] Update shell_commands MOAB --- cime_config/usermods_dirs/NEON/MOAB/shell_commands | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/cime_config/usermods_dirs/NEON/MOAB/shell_commands b/cime_config/usermods_dirs/NEON/MOAB/shell_commands index d91513a92c..33fbf970c1 100644 --- a/cime_config/usermods_dirs/NEON/MOAB/shell_commands +++ b/cime_config/usermods_dirs/NEON/MOAB/shell_commands @@ -1,8 +1,8 @@ +#!/bin/bash ./xmlchange NEONSITE=MOAB ./xmlchange PTS_LON=250.61118 ./xmlchange PTS_LAT=38.25136 -./xmlchange RUN_STARTDATE=2018-01-01 -./xmlchange DATM_YR_ALIGN=2018,DATM_YR_START=2018,DATM_YR_END=2020 -./xmlchange STOP_N=36 - - +./xmlchange DATM_YR_END=2020 +if [ "`./xmlquery CLM_NML_USE_CASE --value`" =~ "*_transient" ]; then + ./xmlchange STOP_N=36 +fi From eb288b30412f6fcb760955102d18264e8ee7a43b Mon Sep 17 00:00:00 2001 From: will wieder Date: Wed, 19 Oct 2022 17:02:43 -0600 Subject: [PATCH 15/95] Update shell_commands ONAQ --- cime_config/usermods_dirs/NEON/ONAQ/shell_commands | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/cime_config/usermods_dirs/NEON/ONAQ/shell_commands b/cime_config/usermods_dirs/NEON/ONAQ/shell_commands index f2e1640725..7399d95701 100644 --- a/cime_config/usermods_dirs/NEON/ONAQ/shell_commands +++ b/cime_config/usermods_dirs/NEON/ONAQ/shell_commands @@ -1,8 +1,8 @@ +#!/bin/bash ./xmlchange NEONSITE=ONAQ ./xmlchange PTS_LON=276.49815 ./xmlchange PTS_LAT=35.68839 -./xmlchange RUN_STARTDATE=2018-01-01 -./xmlchange DATM_YR_ALIGN=2018,DATM_YR_START=2018,DATM_YR_END=2019 -./xmlchange STOP_N=24 - - +./xmlchange DATM_YR_END=2019 +if [ "`./xmlquery CLM_NML_USE_CASE --value`" =~ "*_transient" ]; then + ./xmlchange STOP_N=24 +fi From 973044218ae6be89e1539a3e73f861e09a323a67 Mon Sep 17 00:00:00 2001 From: will wieder Date: Wed, 19 Oct 2022 17:04:38 -0600 Subject: [PATCH 16/95] Update shell_commands TEAK --- cime_config/usermods_dirs/NEON/TEAK/shell_commands | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/cime_config/usermods_dirs/NEON/TEAK/shell_commands b/cime_config/usermods_dirs/NEON/TEAK/shell_commands index 5309888a12..0392a36dce 100644 --- a/cime_config/usermods_dirs/NEON/TEAK/shell_commands +++ b/cime_config/usermods_dirs/NEON/TEAK/shell_commands @@ -1,7 +1,9 @@ +#!/bin/bash ./xmlchange NEONSITE=TEAK ./xmlchange PTS_LON=240.99424199999999 ./xmlchange PTS_LAT=37.006472 -./xmlchange RUN_STARTDATE=2019-01-01 -./xmlchange DATM_YR_ALIGN=2019,DATM_YR_START=2019 -./xmlchange STOP_N=39 - +./xmlchange DATM_YR_ALIGN=2019,DATM_YR_START=2019 +if [ "`./xmlquery CLM_NML_USE_CASE --value`" =~ "*_transient" ]; then + ./xmlchange RUN_STARTDATE=2019-01-01 + ./xmlchange STOP_N=39 +fi From d8b1d887e837ce44ed5358baf14267abf1189a87 Mon Sep 17 00:00:00 2001 From: will wieder Date: Wed, 19 Oct 2022 17:05:38 -0600 Subject: [PATCH 17/95] Update shell_commands TOOL --- cime_config/usermods_dirs/NEON/TOOL/shell_commands | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/cime_config/usermods_dirs/NEON/TOOL/shell_commands b/cime_config/usermods_dirs/NEON/TOOL/shell_commands index fc2551390b..9f6403a9c6 100644 --- a/cime_config/usermods_dirs/NEON/TOOL/shell_commands +++ b/cime_config/usermods_dirs/NEON/TOOL/shell_commands @@ -1,8 +1,9 @@ +#!/bin/bash ./xmlchange NEONSITE=TOOL ./xmlchange PTS_LON=210.629872 ./xmlchange PTS_LAT=68.66045 -./xmlchange RUN_STARTDATE=2020-01-01 ./xmlchange DATM_YR_ALIGN=2020,DATM_YR_START=2020 -./xmlchange STOP_N=27 - - +if [ "`./xmlquery CLM_NML_USE_CASE --value`" =~ "*_transient" ]; then + ./xmlchange RUN_STARTDATE=2020-01-01 + ./xmlchange STOP_N=27 +fi From 279a54d68feb7cef9b6a72743a63222e5bd7617f Mon Sep 17 00:00:00 2001 From: will wieder Date: Wed, 19 Oct 2022 17:06:49 -0600 Subject: [PATCH 18/95] Update shell_commands WREF --- cime_config/usermods_dirs/NEON/WREF/shell_commands | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/cime_config/usermods_dirs/NEON/WREF/shell_commands b/cime_config/usermods_dirs/NEON/WREF/shell_commands index bbb0e14312..ccf6c75138 100644 --- a/cime_config/usermods_dirs/NEON/WREF/shell_commands +++ b/cime_config/usermods_dirs/NEON/WREF/shell_commands @@ -1,7 +1,9 @@ +#!/bin/bash ./xmlchange NEONSITE=WREF ./xmlchange PTS_LON=238.04162 ./xmlchange PTS_LAT=45.81637 -./xmlchange RUN_STARTDATE=2019-01-01 -./xmlchange DATM_YR_ALIGN=2019,DATM_YR_START=2019 -./xmlchange STOP_N=39 - +./xmlchange DATM_YR_ALIGN=2019,DATM_YR_START=2019 +if [ "`./xmlquery CLM_NML_USE_CASE --value`" =~ "*_transient" ]; then + ./xmlchange RUN_STARTDATE=2019-01-01 + ./xmlchange STOP_N=39 +fi From 06b4059e7ad0941abf9765350e4f3fd2dc738428 Mon Sep 17 00:00:00 2001 From: will wieder Date: Wed, 19 Oct 2022 17:07:30 -0600 Subject: [PATCH 19/95] Update shell_commands YELL --- cime_config/usermods_dirs/NEON/YELL/shell_commands | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/cime_config/usermods_dirs/NEON/YELL/shell_commands b/cime_config/usermods_dirs/NEON/YELL/shell_commands index c32b11ef7d..0207e56265 100644 --- a/cime_config/usermods_dirs/NEON/YELL/shell_commands +++ b/cime_config/usermods_dirs/NEON/YELL/shell_commands @@ -1,7 +1,9 @@ +#!/bin/bash ./xmlchange NEONSITE=YELL ./xmlchange PTS_LON=249.45803999999998 ./xmlchange PTS_LAT=44.95597 -./xmlchange RUN_STARTDATE=2019-01-01 -./xmlchange DATM_YR_ALIGN=2019,DATM_YR_START=2019 -./xmlchange STOP_N=39 -# ./xmlchange RUN_STARTDATE=2018-08-01 +./xmlchange DATM_YR_ALIGN=2019,DATM_YR_START=2019 +if [ "`./xmlquery CLM_NML_USE_CASE --value`" =~ "*_transient" ]; then + ./xmlchange RUN_STARTDATE=2019-01-01 + ./xmlchange STOP_N=39 +fi From f9831ac7f6d79ba87927ea75971805a2f137649b Mon Sep 17 00:00:00 2001 From: will wieder Date: Wed, 19 Oct 2022 17:10:21 -0600 Subject: [PATCH 20/95] Update neon_gcs_upload removed unused code, per @ekluzek 's suggestion --- tools/site_and_regional/neon_gcs_upload | 3 --- 1 file changed, 3 deletions(-) diff --git a/tools/site_and_regional/neon_gcs_upload b/tools/site_and_regional/neon_gcs_upload index 9853c96849..40afef8e74 100755 --- a/tools/site_and_regional/neon_gcs_upload +++ b/tools/site_and_regional/neon_gcs_upload @@ -121,9 +121,6 @@ def main(description): from there, """ - #if not os.path.isfile(os.path.join(os.getenv("HOME"),".aws","credentials")): - # raise FileNotFoundError("User account must have valid aws credentials to run this script.") - cesmroot = path_to_ctsm_root() os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = os.path.join(os.environ["HOME"],'.uploadGCSkey.json') #os.path.join(os.environ["HOME"],"gcwriter") From 730bdbed4b135e50753ee1178a543f3c17089067 Mon Sep 17 00:00:00 2001 From: will wieder Date: Fri, 21 Oct 2022 08:53:15 -0600 Subject: [PATCH 21/95] Update shell_commands SJER --- cime_config/usermods_dirs/NEON/SJER/shell_commands | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/cime_config/usermods_dirs/NEON/SJER/shell_commands b/cime_config/usermods_dirs/NEON/SJER/shell_commands index 9d3ee15a81..01cf50d337 100644 --- a/cime_config/usermods_dirs/NEON/SJER/shell_commands +++ b/cime_config/usermods_dirs/NEON/SJER/shell_commands @@ -1,8 +1,9 @@ +#!/bin/bash ./xmlchange NEONSITE=SJER ./xmlchange PTS_LON=240.267 ./xmlchange PTS_LAT=37.107117 -./xmlchange RUN_STARTDATE=2019-01-01 ./xmlchange DATM_YR_ALIGN=2019,DATM_YR_START=2019 -./xmlchange STOP_N=39 - -#./xmlchange RUN_STARTDATE=2018-09-01 +if [ "`./xmlquery CLM_NML_USE_CASE --value`" =~ "*_transient" ]; then + ./xmlchange RUN_STARTDATE=2019-01-01 + ./xmlchange STOP_N=39 +fi From 78cb00e504482739990b1182852d613a3b5d5769 Mon Sep 17 00:00:00 2001 From: wwieder Date: Sat, 22 Oct 2022 08:19:54 -0600 Subject: [PATCH 22/95] trying to address cases with non-standard start/end years --- cime_config/usermods_dirs/NEON/SJER/shell_commands | 8 +++++--- .../usermods_dirs/NEON/defaults/shell_commands | 7 ++++++- tools/site_and_regional/run_neon.py | 12 +++++++----- 3 files changed, 18 insertions(+), 9 deletions(-) diff --git a/cime_config/usermods_dirs/NEON/SJER/shell_commands b/cime_config/usermods_dirs/NEON/SJER/shell_commands index 9d3ee15a81..cd7bf2fe2d 100644 --- a/cime_config/usermods_dirs/NEON/SJER/shell_commands +++ b/cime_config/usermods_dirs/NEON/SJER/shell_commands @@ -1,8 +1,10 @@ +#!/bin/bash ./xmlchange NEONSITE=SJER ./xmlchange PTS_LON=240.267 ./xmlchange PTS_LAT=37.107117 -./xmlchange RUN_STARTDATE=2019-01-01 ./xmlchange DATM_YR_ALIGN=2019,DATM_YR_START=2019 -./xmlchange STOP_N=39 +if [ "`./xmlquery CASE --value`" =~ "*transient" ]; then + ./xmlchange RUN_STARTDATE=2019-01-01 + ./xmlchange STOP_N=39 +fi -#./xmlchange RUN_STARTDATE=2018-09-01 diff --git a/cime_config/usermods_dirs/NEON/defaults/shell_commands b/cime_config/usermods_dirs/NEON/defaults/shell_commands index 53e445e06a..8957af3f9b 100644 --- a/cime_config/usermods_dirs/NEON/defaults/shell_commands +++ b/cime_config/usermods_dirs/NEON/defaults/shell_commands @@ -1,11 +1,16 @@ +#!/bin/bash ./xmlchange CLM_USRDAT_NAME=NEON -./xmlchange RUN_STARTDATE=2018-01-01 ./xmlchange CLM_NML_USE_CASE=1850-2100_SSP3-7.0_transient ./xmlchange CCSM_CO2_PPMV=408.83 ./xmlchange DATM_PRESAERO=SSP3-7.0 ./xmlchange DATM_PRESNDEP=SSP3-7.0 ./xmlchange DATM_PRESO3=SSP3-7.0 ./xmlchange DATM_YR_ALIGN=2018,DATM_YR_END=2021,DATM_YR_START=2018 +if [ "`./xmlquery CASE --value`" =~ "*transient" ]; then + ./xmlchange DATM_YR_END=2022 + ./xmlchange RUN_STARTDATE=2018-01-01 + ./xmlchange STOP_N=51 +fi # Explicitly set PIO Type to NETCDF since this is a single processor case (should already be set this way) ./xmlchange PIO_TYPENAME=netcdf diff --git a/tools/site_and_regional/run_neon.py b/tools/site_and_regional/run_neon.py index e0aae36274..fe93014d76 100755 --- a/tools/site_and_regional/run_neon.py +++ b/tools/site_and_regional/run_neon.py @@ -539,10 +539,12 @@ def run_case( if not self.set_ref_case(case): return case.set_value("STOP_OPTION", "nmonths") - case.set_value("STOP_N", self.diff_month()) - case.set_value("DATM_YR_ALIGN", self.start_year) - case.set_value("DATM_YR_START", self.start_year) - case.set_value("DATM_YR_END", self.end_year) + # Hard wire the with usermods for now + # TODO this wil have to be fixed when we get new NEON data + #case.set_value("STOP_N", self.diff_month()) + #case.set_value("DATM_YR_ALIGN", self.start_year) + #case.set_value("DATM_YR_START", self.start_year) + #case.set_value("DATM_YR_END", self.end_year) case.set_value("CALENDAR", "GREGORIAN") case.set_value("RESUBMIT", 0) else: @@ -558,7 +560,7 @@ def run_case( else: case.set_value("DATM_YR_END", self.end_year - 1) - # Let's no be so clevar with start / end dates + # Let's no be so clever with start / end dates #case.set_value("DATM_YR_ALIGN", int(args.start_date[0:4])) #case.set_value("DATM_YR_START", int(args.start_date[0:4])) #case.set_value("DATM_YR_END", int(args.end_date[0:4])) From 69d25d9bf202c726c9e3bc806473975ac57a9ea5 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Sat, 22 Oct 2022 13:30:15 -0600 Subject: [PATCH 23/95] Correct the syntax so the bash if statements work --- cime_config/usermods_dirs/NEON/BARR/shell_commands | 2 +- cime_config/usermods_dirs/NEON/GUAN/shell_commands | 2 +- cime_config/usermods_dirs/NEON/LAJA/shell_commands | 2 +- cime_config/usermods_dirs/NEON/LENO/shell_commands | 2 +- cime_config/usermods_dirs/NEON/MLBS/shell_commands | 2 +- cime_config/usermods_dirs/NEON/MOAB/shell_commands | 2 +- cime_config/usermods_dirs/NEON/ONAQ/shell_commands | 2 +- cime_config/usermods_dirs/NEON/SJER/shell_commands | 2 +- cime_config/usermods_dirs/NEON/TEAK/shell_commands | 2 +- cime_config/usermods_dirs/NEON/TOOL/shell_commands | 2 +- cime_config/usermods_dirs/NEON/WREF/shell_commands | 2 +- cime_config/usermods_dirs/NEON/YELL/shell_commands | 2 +- cime_config/usermods_dirs/NEON/defaults/shell_commands | 2 +- 13 files changed, 13 insertions(+), 13 deletions(-) diff --git a/cime_config/usermods_dirs/NEON/BARR/shell_commands b/cime_config/usermods_dirs/NEON/BARR/shell_commands index 29ddc20f52..321d7e7daf 100644 --- a/cime_config/usermods_dirs/NEON/BARR/shell_commands +++ b/cime_config/usermods_dirs/NEON/BARR/shell_commands @@ -4,7 +4,7 @@ ./xmlchange PTS_LON=203.349781 ./xmlchange PTS_LAT=71.281711 ./xmlchange DATM_YR_ALIGN=2019,DATM_YR_START=2019 -if [ "`./xmlquery CLM_NML_USE_CASE --value`" =~ "*_transient" ]; then +if [[ `./xmlquery CLM_NML_USE_CASE --value` =~ .+_transient ]]; then ./xmlchange RUN_STARTDATE=2019-01-01 ./xmlchange STOP_N=39 fi diff --git a/cime_config/usermods_dirs/NEON/GUAN/shell_commands b/cime_config/usermods_dirs/NEON/GUAN/shell_commands index f886245a1a..3ed68c34e5 100644 --- a/cime_config/usermods_dirs/NEON/GUAN/shell_commands +++ b/cime_config/usermods_dirs/NEON/GUAN/shell_commands @@ -4,7 +4,7 @@ ./xmlchange PTS_LON=293.13112 ./xmlchange PTS_LAT=17.96882 ./xmlchange DATM_YR_ALIGN=2019,DATM_YR_START=2019 -if [ "`./xmlquery CLM_NML_USE_CASE --value`" =~ "*_transient" ]; then +if [[ `./xmlquery CLM_NML_USE_CASE --value` =~ .+_transient ]]; then ./xmlchange RUN_STARTDATE=2019-01-01 ./xmlchange STOP_N=39 fi diff --git a/cime_config/usermods_dirs/NEON/LAJA/shell_commands b/cime_config/usermods_dirs/NEON/LAJA/shell_commands index 7e946bf9b4..45bd0f06b0 100644 --- a/cime_config/usermods_dirs/NEON/LAJA/shell_commands +++ b/cime_config/usermods_dirs/NEON/LAJA/shell_commands @@ -3,7 +3,7 @@ ./xmlchange PTS_LON=292.92392 ./xmlchange PTS_LAT=18.02184 ./xmlchange DATM_YR_ALIGN=2019,DATM_YR_START=2019 -if [ "`./xmlquery CLM_NML_USE_CASE --value`" =~ "*_transient" ]; then +if [[ `./xmlquery CLM_NML_USE_CASE --value` =~ .+_transient ]]; then ./xmlchange RUN_STARTDATE=2019-01-01 ./xmlchange STOP_N=39 fi diff --git a/cime_config/usermods_dirs/NEON/LENO/shell_commands b/cime_config/usermods_dirs/NEON/LENO/shell_commands index 3af172e8f0..a8c85a0cd1 100644 --- a/cime_config/usermods_dirs/NEON/LENO/shell_commands +++ b/cime_config/usermods_dirs/NEON/LENO/shell_commands @@ -4,7 +4,7 @@ ./xmlchange PTS_LON=271.83897 ./xmlchange PTS_LAT=31.8531 ./xmlchange DATM_YR_ALIGN=2021,DATM_YR_START=2021 -if [ "`./xmlquery CLM_NML_USE_CASE --value`" =~ "*_transient" ]; then +if [[ `./xmlquery CLM_NML_USE_CASE --value` =~ .+_transient ]]; then ./xmlchange RUN_STARTDATE=2021-01-01 ./xmlchange STOP_N=15 fi diff --git a/cime_config/usermods_dirs/NEON/MLBS/shell_commands b/cime_config/usermods_dirs/NEON/MLBS/shell_commands index 6467bad1cc..ba64a253eb 100644 --- a/cime_config/usermods_dirs/NEON/MLBS/shell_commands +++ b/cime_config/usermods_dirs/NEON/MLBS/shell_commands @@ -3,6 +3,6 @@ ./xmlchange PTS_LON=279.47575 ./xmlchange PTS_LAT=37.37783 ./xmlchange DATM_YR_END=2019 -if [ "`./xmlquery CLM_NML_USE_CASE --value`" =~ "*_transient" ]; then +if [[ `./xmlquery CLM_NML_USE_CASE --value` =~ .+_transient ]]; then ./xmlchange STOP_N=24 fi diff --git a/cime_config/usermods_dirs/NEON/MOAB/shell_commands b/cime_config/usermods_dirs/NEON/MOAB/shell_commands index 33fbf970c1..558dfe6d34 100644 --- a/cime_config/usermods_dirs/NEON/MOAB/shell_commands +++ b/cime_config/usermods_dirs/NEON/MOAB/shell_commands @@ -3,6 +3,6 @@ ./xmlchange PTS_LON=250.61118 ./xmlchange PTS_LAT=38.25136 ./xmlchange DATM_YR_END=2020 -if [ "`./xmlquery CLM_NML_USE_CASE --value`" =~ "*_transient" ]; then +if [[ `./xmlquery CLM_NML_USE_CASE --value` =~ .+_transient ]]; then ./xmlchange STOP_N=36 fi diff --git a/cime_config/usermods_dirs/NEON/ONAQ/shell_commands b/cime_config/usermods_dirs/NEON/ONAQ/shell_commands index 7399d95701..2fb2b67f14 100644 --- a/cime_config/usermods_dirs/NEON/ONAQ/shell_commands +++ b/cime_config/usermods_dirs/NEON/ONAQ/shell_commands @@ -3,6 +3,6 @@ ./xmlchange PTS_LON=276.49815 ./xmlchange PTS_LAT=35.68839 ./xmlchange DATM_YR_END=2019 -if [ "`./xmlquery CLM_NML_USE_CASE --value`" =~ "*_transient" ]; then +if [[ `./xmlquery CLM_NML_USE_CASE --value` =~ .+_transient ]]; then ./xmlchange STOP_N=24 fi diff --git a/cime_config/usermods_dirs/NEON/SJER/shell_commands b/cime_config/usermods_dirs/NEON/SJER/shell_commands index cd7bf2fe2d..61207df4ad 100644 --- a/cime_config/usermods_dirs/NEON/SJER/shell_commands +++ b/cime_config/usermods_dirs/NEON/SJER/shell_commands @@ -3,7 +3,7 @@ ./xmlchange PTS_LON=240.267 ./xmlchange PTS_LAT=37.107117 ./xmlchange DATM_YR_ALIGN=2019,DATM_YR_START=2019 -if [ "`./xmlquery CASE --value`" =~ "*transient" ]; then +if [[ `./xmlquery CLM_NML_USE_CASE --value` =~ .+_transient ]]; then ./xmlchange RUN_STARTDATE=2019-01-01 ./xmlchange STOP_N=39 fi diff --git a/cime_config/usermods_dirs/NEON/TEAK/shell_commands b/cime_config/usermods_dirs/NEON/TEAK/shell_commands index 0392a36dce..40cee5dacd 100644 --- a/cime_config/usermods_dirs/NEON/TEAK/shell_commands +++ b/cime_config/usermods_dirs/NEON/TEAK/shell_commands @@ -3,7 +3,7 @@ ./xmlchange PTS_LON=240.99424199999999 ./xmlchange PTS_LAT=37.006472 ./xmlchange DATM_YR_ALIGN=2019,DATM_YR_START=2019 -if [ "`./xmlquery CLM_NML_USE_CASE --value`" =~ "*_transient" ]; then +if [[ `./xmlquery CLM_NML_USE_CASE --value` =~ .+_transient ]]; then ./xmlchange RUN_STARTDATE=2019-01-01 ./xmlchange STOP_N=39 fi diff --git a/cime_config/usermods_dirs/NEON/TOOL/shell_commands b/cime_config/usermods_dirs/NEON/TOOL/shell_commands index 9f6403a9c6..480ad833f0 100644 --- a/cime_config/usermods_dirs/NEON/TOOL/shell_commands +++ b/cime_config/usermods_dirs/NEON/TOOL/shell_commands @@ -3,7 +3,7 @@ ./xmlchange PTS_LON=210.629872 ./xmlchange PTS_LAT=68.66045 ./xmlchange DATM_YR_ALIGN=2020,DATM_YR_START=2020 -if [ "`./xmlquery CLM_NML_USE_CASE --value`" =~ "*_transient" ]; then +if [[ `./xmlquery CLM_NML_USE_CASE --value` =~ .+_transient ]]; then ./xmlchange RUN_STARTDATE=2020-01-01 ./xmlchange STOP_N=27 fi diff --git a/cime_config/usermods_dirs/NEON/WREF/shell_commands b/cime_config/usermods_dirs/NEON/WREF/shell_commands index ccf6c75138..e38a22a502 100644 --- a/cime_config/usermods_dirs/NEON/WREF/shell_commands +++ b/cime_config/usermods_dirs/NEON/WREF/shell_commands @@ -3,7 +3,7 @@ ./xmlchange PTS_LON=238.04162 ./xmlchange PTS_LAT=45.81637 ./xmlchange DATM_YR_ALIGN=2019,DATM_YR_START=2019 -if [ "`./xmlquery CLM_NML_USE_CASE --value`" =~ "*_transient" ]; then +if [[ `./xmlquery CLM_NML_USE_CASE --value` =~ .+_transient ]]; then ./xmlchange RUN_STARTDATE=2019-01-01 ./xmlchange STOP_N=39 fi diff --git a/cime_config/usermods_dirs/NEON/YELL/shell_commands b/cime_config/usermods_dirs/NEON/YELL/shell_commands index 0207e56265..0d87c40b0c 100644 --- a/cime_config/usermods_dirs/NEON/YELL/shell_commands +++ b/cime_config/usermods_dirs/NEON/YELL/shell_commands @@ -3,7 +3,7 @@ ./xmlchange PTS_LON=249.45803999999998 ./xmlchange PTS_LAT=44.95597 ./xmlchange DATM_YR_ALIGN=2019,DATM_YR_START=2019 -if [ "`./xmlquery CLM_NML_USE_CASE --value`" =~ "*_transient" ]; then +if [[ `./xmlquery CLM_NML_USE_CASE --value` =~ .+_transient ]]; then ./xmlchange RUN_STARTDATE=2019-01-01 ./xmlchange STOP_N=39 fi diff --git a/cime_config/usermods_dirs/NEON/defaults/shell_commands b/cime_config/usermods_dirs/NEON/defaults/shell_commands index 8957af3f9b..f8288abe60 100644 --- a/cime_config/usermods_dirs/NEON/defaults/shell_commands +++ b/cime_config/usermods_dirs/NEON/defaults/shell_commands @@ -6,7 +6,7 @@ ./xmlchange DATM_PRESNDEP=SSP3-7.0 ./xmlchange DATM_PRESO3=SSP3-7.0 ./xmlchange DATM_YR_ALIGN=2018,DATM_YR_END=2021,DATM_YR_START=2018 -if [ "`./xmlquery CASE --value`" =~ "*transient" ]; then +if [[ `./xmlquery CASE --value` =~ .+_transient ]]; then ./xmlchange DATM_YR_END=2022 ./xmlchange RUN_STARTDATE=2018-01-01 ./xmlchange STOP_N=51 From 7f839cc1bdbfa2d394f451c140e6ea14edc78892 Mon Sep 17 00:00:00 2001 From: wwieder Date: Sun, 23 Oct 2022 08:53:12 -0600 Subject: [PATCH 24/95] additional modifications needed for usermods to work as intended --- .../NEON/defaults/shell_commands | 2 +- tools/site_and_regional/run_neon.py | 27 ++++++++++--------- 2 files changed, 16 insertions(+), 13 deletions(-) diff --git a/cime_config/usermods_dirs/NEON/defaults/shell_commands b/cime_config/usermods_dirs/NEON/defaults/shell_commands index f8288abe60..743880366b 100644 --- a/cime_config/usermods_dirs/NEON/defaults/shell_commands +++ b/cime_config/usermods_dirs/NEON/defaults/shell_commands @@ -6,7 +6,7 @@ ./xmlchange DATM_PRESNDEP=SSP3-7.0 ./xmlchange DATM_PRESO3=SSP3-7.0 ./xmlchange DATM_YR_ALIGN=2018,DATM_YR_END=2021,DATM_YR_START=2018 -if [[ `./xmlquery CASE --value` =~ .+_transient ]]; then +if [[ `./xmlquery CLM_NML_USE_CASE --value` =~ .+_transient ]]; then ./xmlchange DATM_YR_END=2022 ./xmlchange RUN_STARTDATE=2018-01-01 ./xmlchange STOP_N=51 diff --git a/tools/site_and_regional/run_neon.py b/tools/site_and_regional/run_neon.py index fe93014d76..a226a42e1e 100755 --- a/tools/site_and_regional/run_neon.py +++ b/tools/site_and_regional/run_neon.py @@ -514,7 +514,6 @@ def run_case( with Case(case_root, read_only=False) as case: # in order to avoid the complication of leap years we always set the run_length in units of days. case.set_value("STOP_OPTION", "ndays") - case.set_value("STOP_N", run_length) case.set_value("REST_OPTION", "end") case.set_value("CONTINUE_RUN", False) case.set_value("NEONVERSION", version) @@ -524,6 +523,8 @@ def run_case( case.set_value("RUN_REFDATE", "0018-01-01") case.set_value("RUN_STARTDATE", "0018-01-01") case.set_value("RESUBMIT", 1) + case.set_value("STOP_N", run_length) + else: case.set_value("CLM_FORCE_COLDSTART", "off") case.set_value("CLM_ACCELERATED_SPINUP", "off") @@ -531,6 +532,7 @@ def run_case( if run_type == "postad": self.set_ref_case(case) + case.set_value("STOP_N", run_length) if run_type == "transient": if self.finidat: @@ -547,18 +549,19 @@ def run_case( #case.set_value("DATM_YR_END", self.end_year) case.set_value("CALENDAR", "GREGORIAN") case.set_value("RESUBMIT", 0) - else: + # Turning all this off for usermod controsl over start, align, and end year + #else: # for the spinup we want the start and end on year boundaries - if self.start_month == 1: - case.set_value("DATM_YR_ALIGN", self.start_year) - case.set_value("DATM_YR_START", self.start_year) - elif self.start_year + 1 <= self.end_year: - case.set_value("DATM_YR_ALIGN", self.start_year + 1) - case.set_value("DATM_YR_START", self.start_year + 1) - if self.end_month == 12: - case.set_value("DATM_YR_END", self.end_year) - else: - case.set_value("DATM_YR_END", self.end_year - 1) + #if self.start_month == 1: + # case.set_value("DATM_YR_ALIGN", self.start_year) + # case.set_value("DATM_YR_START", self.start_year) + #elif self.start_year + 1 <= self.end_year: + # case.set_value("DATM_YR_ALIGN", self.start_year + 1) + # case.set_value("DATM_YR_START", self.start_year + 1) + #if self.end_month == 12: + # case.set_value("DATM_YR_END", self.end_year) + #else: + # case.set_value("DATM_YR_END", self.end_year - 1) # Let's no be so clever with start / end dates #case.set_value("DATM_YR_ALIGN", int(args.start_date[0:4])) From bcbc456381df4e96925ca76df0488cdabf2a861d Mon Sep 17 00:00:00 2001 From: wwieder Date: Sun, 23 Oct 2022 09:13:56 -0600 Subject: [PATCH 25/95] correction for DATM_YR_END in ad and postAD --- tools/site_and_regional/run_neon.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tools/site_and_regional/run_neon.py b/tools/site_and_regional/run_neon.py index a226a42e1e..957c313df0 100755 --- a/tools/site_and_regional/run_neon.py +++ b/tools/site_and_regional/run_neon.py @@ -550,7 +550,7 @@ def run_case( case.set_value("CALENDAR", "GREGORIAN") case.set_value("RESUBMIT", 0) # Turning all this off for usermod controsl over start, align, and end year - #else: + else: # for the spinup we want the start and end on year boundaries #if self.start_month == 1: # case.set_value("DATM_YR_ALIGN", self.start_year) @@ -558,10 +558,10 @@ def run_case( #elif self.start_year + 1 <= self.end_year: # case.set_value("DATM_YR_ALIGN", self.start_year + 1) # case.set_value("DATM_YR_START", self.start_year + 1) - #if self.end_month == 12: - # case.set_value("DATM_YR_END", self.end_year) - #else: - # case.set_value("DATM_YR_END", self.end_year - 1) + if self.end_month == 12: + case.set_value("DATM_YR_END", self.end_year) + else: + case.set_value("DATM_YR_END", self.end_year - 1) # Let's no be so clever with start / end dates #case.set_value("DATM_YR_ALIGN", int(args.start_date[0:4])) From 8e7efb03ebcaeeee969c349b1eea14ad56a87aac Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Tue, 25 Oct 2022 15:40:51 -0600 Subject: [PATCH 26/95] Working version of run_neon.py, I tried running shell_commands after the case is setup, but that does not work as the read of env_run.xml would need to be done again --- tools/site_and_regional/run_neon.py | 41 +++++++++++++---------------- 1 file changed, 19 insertions(+), 22 deletions(-) diff --git a/tools/site_and_regional/run_neon.py b/tools/site_and_regional/run_neon.py index 957c313df0..5c605b2c6d 100755 --- a/tools/site_and_regional/run_neon.py +++ b/tools/site_and_regional/run_neon.py @@ -83,7 +83,7 @@ import CIME.build as build from standard_script_setup import * from CIME.case import Case -from CIME.utils import safe_copy, expect, symlink_force +from CIME.utils import safe_copy, expect, symlink_force, run_cmd_no_fail from argparse import RawTextHelpFormatter from CIME.locked_files import lock_file, unlock_file @@ -511,10 +511,12 @@ def run_case( case_root, keepexe=True, user_mods_dirs=user_mods_dirs ) + print("---- Before Case----") with Case(case_root, read_only=False) as case: - # in order to avoid the complication of leap years we always set the run_length in units of days. - case.set_value("STOP_OPTION", "ndays") - case.set_value("REST_OPTION", "end") + if run_type is not"transient": + # in order to avoid the complication of leap years we always set the run_length in units of days. + case.set_value("STOP_OPTION", "ndays") + case.set_value("REST_OPTION", "end") case.set_value("CONTINUE_RUN", False) case.set_value("NEONVERSION", version) if run_type == "ad": @@ -534,49 +536,44 @@ def run_case( self.set_ref_case(case) case.set_value("STOP_N", run_length) + # For transient cases STOP will be set in the user_mod_directory if run_type == "transient": if self.finidat: case.set_value("RUN_TYPE", "startup") else: if not self.set_ref_case(case): return - case.set_value("STOP_OPTION", "nmonths") - # Hard wire the with usermods for now - # TODO this wil have to be fixed when we get new NEON data - #case.set_value("STOP_N", self.diff_month()) - #case.set_value("DATM_YR_ALIGN", self.start_year) - #case.set_value("DATM_YR_START", self.start_year) - #case.set_value("DATM_YR_END", self.end_year) case.set_value("CALENDAR", "GREGORIAN") case.set_value("RESUBMIT", 0) # Turning all this off for usermod controsl over start, align, and end year else: # for the spinup we want the start and end on year boundaries - #if self.start_month == 1: - # case.set_value("DATM_YR_ALIGN", self.start_year) - # case.set_value("DATM_YR_START", self.start_year) - #elif self.start_year + 1 <= self.end_year: - # case.set_value("DATM_YR_ALIGN", self.start_year + 1) - # case.set_value("DATM_YR_START", self.start_year + 1) if self.end_month == 12: case.set_value("DATM_YR_END", self.end_year) else: case.set_value("DATM_YR_END", self.end_year - 1) - # Let's no be so clever with start / end dates - #case.set_value("DATM_YR_ALIGN", int(args.start_date[0:4])) - #case.set_value("DATM_YR_START", int(args.start_date[0:4])) - #case.set_value("DATM_YR_END", int(args.end_date[0:4])) - if not rundir: rundir = case.get_value("RUNDIR") + print("---- modify_user_nl----") self.modify_user_nl(case_root, run_type, rundir) + # Run the shell_commands again, now that more settings are there + #print("---- shell_commands ---") + #run_cmd_no_fail( "./shell_commands" ) + #casedir = case.get_value("CASEROOT") + #envrunfilename = os.path.join( casedir, "env_run.xml" ) + #print( envrunfilename ) + #case.invalidate( envrunfilename ) + + print("---- create_namelists ---") case.create_namelists() # explicitly run check_input_data + print("---- check_input ---") case.check_all_input_data() if not setup_only: + print("---- submit ---") case.submit(no_batch=no_batch) def set_ref_case(self, case): From 1c5e9002a3b3297f5565270d6fbeed5c81039bc7 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Tue, 25 Oct 2022 20:45:06 -0600 Subject: [PATCH 27/95] Remove some of the prints I added, and the shell_commands part I commented out, and most importantly use a transient compset for the transient case --- tools/site_and_regional/run_neon.py | 18 ++++-------------- 1 file changed, 4 insertions(+), 14 deletions(-) diff --git a/tools/site_and_regional/run_neon.py b/tools/site_and_regional/run_neon.py index 5c605b2c6d..1805ed7ebd 100755 --- a/tools/site_and_regional/run_neon.py +++ b/tools/site_and_regional/run_neon.py @@ -511,7 +511,6 @@ def run_case( case_root, keepexe=True, user_mods_dirs=user_mods_dirs ) - print("---- Before Case----") with Case(case_root, read_only=False) as case: if run_type is not"transient": # in order to avoid the complication of leap years we always set the run_length in units of days. @@ -556,24 +555,12 @@ def run_case( if not rundir: rundir = case.get_value("RUNDIR") - print("---- modify_user_nl----") self.modify_user_nl(case_root, run_type, rundir) - # Run the shell_commands again, now that more settings are there - #print("---- shell_commands ---") - #run_cmd_no_fail( "./shell_commands" ) - #casedir = case.get_value("CASEROOT") - #envrunfilename = os.path.join( casedir, "env_run.xml" ) - #print( envrunfilename ) - #case.invalidate( envrunfilename ) - - print("---- create_namelists ---") case.create_namelists() # explicitly run check_input_data - print("---- check_input ---") case.check_all_input_data() if not setup_only: - print("---- submit ---") case.submit(no_batch=no_batch) def set_ref_case(self, case): @@ -786,7 +773,10 @@ def main(description): # -- so no need to define a base_case for every site. res = "CLM_USRDAT" - compset = "I1PtClm51Bgc" + if run_type == "transient": + compset = "IHist1PtClm51Bgc" + else: + compset = "I1PtClm51Bgc" # -- Looping over neon sites From 13efae5d2bcb0077c17c8acdd4013339e8fa4af1 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Tue, 25 Oct 2022 20:46:30 -0600 Subject: [PATCH 28/95] Use the compset to determine if this is a transient case or not --- cime_config/usermods_dirs/NEON/BARR/shell_commands | 3 ++- cime_config/usermods_dirs/NEON/GUAN/shell_commands | 3 ++- cime_config/usermods_dirs/NEON/LAJA/shell_commands | 3 ++- cime_config/usermods_dirs/NEON/LENO/shell_commands | 3 ++- cime_config/usermods_dirs/NEON/MLBS/shell_commands | 3 ++- cime_config/usermods_dirs/NEON/MOAB/shell_commands | 3 ++- cime_config/usermods_dirs/NEON/ONAQ/shell_commands | 3 ++- cime_config/usermods_dirs/NEON/SJER/shell_commands | 3 ++- cime_config/usermods_dirs/NEON/TEAK/shell_commands | 3 ++- cime_config/usermods_dirs/NEON/TOOL/shell_commands | 3 ++- cime_config/usermods_dirs/NEON/WREF/shell_commands | 3 ++- cime_config/usermods_dirs/NEON/YELL/shell_commands | 3 ++- cime_config/usermods_dirs/NEON/defaults/shell_commands | 7 ++++++- 13 files changed, 30 insertions(+), 13 deletions(-) diff --git a/cime_config/usermods_dirs/NEON/BARR/shell_commands b/cime_config/usermods_dirs/NEON/BARR/shell_commands index 321d7e7daf..713331c0ee 100644 --- a/cime_config/usermods_dirs/NEON/BARR/shell_commands +++ b/cime_config/usermods_dirs/NEON/BARR/shell_commands @@ -4,7 +4,8 @@ ./xmlchange PTS_LON=203.349781 ./xmlchange PTS_LAT=71.281711 ./xmlchange DATM_YR_ALIGN=2019,DATM_YR_START=2019 -if [[ `./xmlquery CLM_NML_USE_CASE --value` =~ .+_transient ]]; then +# Different default start date and number of months to run for transient case +if [[ $compset =~ ^HIST ]]; then ./xmlchange RUN_STARTDATE=2019-01-01 ./xmlchange STOP_N=39 fi diff --git a/cime_config/usermods_dirs/NEON/GUAN/shell_commands b/cime_config/usermods_dirs/NEON/GUAN/shell_commands index 3ed68c34e5..4214a35e3c 100644 --- a/cime_config/usermods_dirs/NEON/GUAN/shell_commands +++ b/cime_config/usermods_dirs/NEON/GUAN/shell_commands @@ -4,7 +4,8 @@ ./xmlchange PTS_LON=293.13112 ./xmlchange PTS_LAT=17.96882 ./xmlchange DATM_YR_ALIGN=2019,DATM_YR_START=2019 -if [[ `./xmlquery CLM_NML_USE_CASE --value` =~ .+_transient ]]; then +# Different default start date and number of months to run for transient case +if [[ $compset =~ ^HIST ]]; then ./xmlchange RUN_STARTDATE=2019-01-01 ./xmlchange STOP_N=39 fi diff --git a/cime_config/usermods_dirs/NEON/LAJA/shell_commands b/cime_config/usermods_dirs/NEON/LAJA/shell_commands index 45bd0f06b0..a7bda447e6 100644 --- a/cime_config/usermods_dirs/NEON/LAJA/shell_commands +++ b/cime_config/usermods_dirs/NEON/LAJA/shell_commands @@ -3,7 +3,8 @@ ./xmlchange PTS_LON=292.92392 ./xmlchange PTS_LAT=18.02184 ./xmlchange DATM_YR_ALIGN=2019,DATM_YR_START=2019 -if [[ `./xmlquery CLM_NML_USE_CASE --value` =~ .+_transient ]]; then +# Different default start date and number of months to run for transient case +if [[ $compset =~ ^HIST ]]; then ./xmlchange RUN_STARTDATE=2019-01-01 ./xmlchange STOP_N=39 fi diff --git a/cime_config/usermods_dirs/NEON/LENO/shell_commands b/cime_config/usermods_dirs/NEON/LENO/shell_commands index a8c85a0cd1..c5ae590186 100644 --- a/cime_config/usermods_dirs/NEON/LENO/shell_commands +++ b/cime_config/usermods_dirs/NEON/LENO/shell_commands @@ -4,7 +4,8 @@ ./xmlchange PTS_LON=271.83897 ./xmlchange PTS_LAT=31.8531 ./xmlchange DATM_YR_ALIGN=2021,DATM_YR_START=2021 -if [[ `./xmlquery CLM_NML_USE_CASE --value` =~ .+_transient ]]; then +# Different default start date and number of months to run for transient case +if [[ $compset =~ ^HIST ]]; then ./xmlchange RUN_STARTDATE=2021-01-01 ./xmlchange STOP_N=15 fi diff --git a/cime_config/usermods_dirs/NEON/MLBS/shell_commands b/cime_config/usermods_dirs/NEON/MLBS/shell_commands index ba64a253eb..e0a2ea5443 100644 --- a/cime_config/usermods_dirs/NEON/MLBS/shell_commands +++ b/cime_config/usermods_dirs/NEON/MLBS/shell_commands @@ -3,6 +3,7 @@ ./xmlchange PTS_LON=279.47575 ./xmlchange PTS_LAT=37.37783 ./xmlchange DATM_YR_END=2019 -if [[ `./xmlquery CLM_NML_USE_CASE --value` =~ .+_transient ]]; then +# Different default number of months to run for transient case +if [[ $compset =~ ^HIST ]]; then ./xmlchange STOP_N=24 fi diff --git a/cime_config/usermods_dirs/NEON/MOAB/shell_commands b/cime_config/usermods_dirs/NEON/MOAB/shell_commands index 558dfe6d34..c90aaa30d4 100644 --- a/cime_config/usermods_dirs/NEON/MOAB/shell_commands +++ b/cime_config/usermods_dirs/NEON/MOAB/shell_commands @@ -3,6 +3,7 @@ ./xmlchange PTS_LON=250.61118 ./xmlchange PTS_LAT=38.25136 ./xmlchange DATM_YR_END=2020 -if [[ `./xmlquery CLM_NML_USE_CASE --value` =~ .+_transient ]]; then +# Different default number of months to run for transient case +if [[ $compset =~ ^HIST ]]; then ./xmlchange STOP_N=36 fi diff --git a/cime_config/usermods_dirs/NEON/ONAQ/shell_commands b/cime_config/usermods_dirs/NEON/ONAQ/shell_commands index 2fb2b67f14..df2ca55dad 100644 --- a/cime_config/usermods_dirs/NEON/ONAQ/shell_commands +++ b/cime_config/usermods_dirs/NEON/ONAQ/shell_commands @@ -3,6 +3,7 @@ ./xmlchange PTS_LON=276.49815 ./xmlchange PTS_LAT=35.68839 ./xmlchange DATM_YR_END=2019 -if [[ `./xmlquery CLM_NML_USE_CASE --value` =~ .+_transient ]]; then +# Different default number of months to run for transient case +if [[ $compset =~ ^HIST ]]; then ./xmlchange STOP_N=24 fi diff --git a/cime_config/usermods_dirs/NEON/SJER/shell_commands b/cime_config/usermods_dirs/NEON/SJER/shell_commands index 61207df4ad..6e05d23792 100644 --- a/cime_config/usermods_dirs/NEON/SJER/shell_commands +++ b/cime_config/usermods_dirs/NEON/SJER/shell_commands @@ -3,7 +3,8 @@ ./xmlchange PTS_LON=240.267 ./xmlchange PTS_LAT=37.107117 ./xmlchange DATM_YR_ALIGN=2019,DATM_YR_START=2019 -if [[ `./xmlquery CLM_NML_USE_CASE --value` =~ .+_transient ]]; then +# Different default start date and number of months to run for transient case +if [[ $compset =~ ^HIST ]]; then ./xmlchange RUN_STARTDATE=2019-01-01 ./xmlchange STOP_N=39 fi diff --git a/cime_config/usermods_dirs/NEON/TEAK/shell_commands b/cime_config/usermods_dirs/NEON/TEAK/shell_commands index 40cee5dacd..9058eda022 100644 --- a/cime_config/usermods_dirs/NEON/TEAK/shell_commands +++ b/cime_config/usermods_dirs/NEON/TEAK/shell_commands @@ -3,7 +3,8 @@ ./xmlchange PTS_LON=240.99424199999999 ./xmlchange PTS_LAT=37.006472 ./xmlchange DATM_YR_ALIGN=2019,DATM_YR_START=2019 -if [[ `./xmlquery CLM_NML_USE_CASE --value` =~ .+_transient ]]; then +# Different default start date and number of months to run for transient case +if [[ $compset =~ ^HIST ]]; then ./xmlchange RUN_STARTDATE=2019-01-01 ./xmlchange STOP_N=39 fi diff --git a/cime_config/usermods_dirs/NEON/TOOL/shell_commands b/cime_config/usermods_dirs/NEON/TOOL/shell_commands index 480ad833f0..f1f2e1771a 100644 --- a/cime_config/usermods_dirs/NEON/TOOL/shell_commands +++ b/cime_config/usermods_dirs/NEON/TOOL/shell_commands @@ -3,7 +3,8 @@ ./xmlchange PTS_LON=210.629872 ./xmlchange PTS_LAT=68.66045 ./xmlchange DATM_YR_ALIGN=2020,DATM_YR_START=2020 -if [[ `./xmlquery CLM_NML_USE_CASE --value` =~ .+_transient ]]; then +# Different default start date and number of months to run for transient case +if [[ $compset =~ ^HIST ]]; then ./xmlchange RUN_STARTDATE=2020-01-01 ./xmlchange STOP_N=27 fi diff --git a/cime_config/usermods_dirs/NEON/WREF/shell_commands b/cime_config/usermods_dirs/NEON/WREF/shell_commands index e38a22a502..807055ae6e 100644 --- a/cime_config/usermods_dirs/NEON/WREF/shell_commands +++ b/cime_config/usermods_dirs/NEON/WREF/shell_commands @@ -3,7 +3,8 @@ ./xmlchange PTS_LON=238.04162 ./xmlchange PTS_LAT=45.81637 ./xmlchange DATM_YR_ALIGN=2019,DATM_YR_START=2019 -if [[ `./xmlquery CLM_NML_USE_CASE --value` =~ .+_transient ]]; then +# Different default start date and number of months to run for transient case +if [[ $compset =~ ^HIST ]]; then ./xmlchange RUN_STARTDATE=2019-01-01 ./xmlchange STOP_N=39 fi diff --git a/cime_config/usermods_dirs/NEON/YELL/shell_commands b/cime_config/usermods_dirs/NEON/YELL/shell_commands index 0d87c40b0c..800d7d01ad 100644 --- a/cime_config/usermods_dirs/NEON/YELL/shell_commands +++ b/cime_config/usermods_dirs/NEON/YELL/shell_commands @@ -3,7 +3,8 @@ ./xmlchange PTS_LON=249.45803999999998 ./xmlchange PTS_LAT=44.95597 ./xmlchange DATM_YR_ALIGN=2019,DATM_YR_START=2019 -if [[ `./xmlquery CLM_NML_USE_CASE --value` =~ .+_transient ]]; then +# Different default start date and number of months to run for transient case +if [[ $compset =~ ^HIST ]]; then ./xmlchange RUN_STARTDATE=2019-01-01 ./xmlchange STOP_N=39 fi diff --git a/cime_config/usermods_dirs/NEON/defaults/shell_commands b/cime_config/usermods_dirs/NEON/defaults/shell_commands index 743880366b..e34df28191 100644 --- a/cime_config/usermods_dirs/NEON/defaults/shell_commands +++ b/cime_config/usermods_dirs/NEON/defaults/shell_commands @@ -6,9 +6,14 @@ ./xmlchange DATM_PRESNDEP=SSP3-7.0 ./xmlchange DATM_PRESO3=SSP3-7.0 ./xmlchange DATM_YR_ALIGN=2018,DATM_YR_END=2021,DATM_YR_START=2018 -if [[ `./xmlquery CLM_NML_USE_CASE --value` =~ .+_transient ]]; then +use_case=`./xmlquery CLM_NML_USE_CASE --value` +compset=`./xmlquery COMPSET --value` +# For a transient case run the whole length and don't cycle +if [[ $compset =~ ^HIST ]]; then ./xmlchange DATM_YR_END=2022 ./xmlchange RUN_STARTDATE=2018-01-01 + # Number of months that can be run for the full transient case + ./xmlchange STOP_OPTION="nmonths" ./xmlchange STOP_N=51 fi From 6950b1f2e7020929ec6e41ecc5ffce9efa4f622a Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Tue, 25 Oct 2022 20:56:54 -0600 Subject: [PATCH 29/95] Have the last year for the datm data be 2022, so that it will work for all cases: ad, postad, and transient. This does mean the ad and postad cases won't have this data align with the meterological forcing data, but that's probably fine --- .../usermods_dirs/NEON/defaults/user_nl_datm_streams | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cime_config/usermods_dirs/NEON/defaults/user_nl_datm_streams b/cime_config/usermods_dirs/NEON/defaults/user_nl_datm_streams index 6244eed2fa..f1af044c31 100644 --- a/cime_config/usermods_dirs/NEON/defaults/user_nl_datm_streams +++ b/cime_config/usermods_dirs/NEON/defaults/user_nl_datm_streams @@ -1,16 +1,16 @@ presaero.SSP3-7.0:datafiles = $DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/aero/aerodep_clm_SSP370_b.e21.BWSSP370cmip6.f09_g17.CMIP6-SSP3-7.0-WACCM.001_2018-2030_monthly_0.9x1.25_c210826.nc presaero.SSP3-7.0:year_first=2018 -presaero.SSP3-7.0:year_last=2030 +presaero.SSP3-7.0:year_last=2022 presaero.SSP3-7.0:year_align=2018 presaero.SSP3-7.0:dtlimit=30 presndep.SSP3-7.0:datafiles = $DIN_LOC_ROOT/lnd/clm2/ndepdata/fndep_clm_f09_g17.CMIP6-SSP3-7.0-WACCM_2018-2030_monthly_c210826.nc presndep.SSP3-7.0:year_first=2018 -presndep.SSP3-7.0:year_last=2030 +presndep.SSP3-7.0:year_last=2022 presndep.SSP3-7.0:year_align=2018 presndep.SSP3-7.0:dtlimit=30 preso3.SSP3-7.0:year_first=2018 -preso3.SSP3-7.0:year_last=2030 +preso3.SSP3-7.0:year_last=2022 preso3.SSP3-7.0:year_align=2018 preso3.SSP3-7.0:dtlimit=30 From a5780c0bad6a5fae314b467af9ae6bc0bc8b200f Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Wed, 26 Oct 2022 16:15:04 -0600 Subject: [PATCH 30/95] Set CLM streams to go until 2022 as well --- cime_config/usermods_dirs/NEON/defaults/user_nl_clm | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cime_config/usermods_dirs/NEON/defaults/user_nl_clm b/cime_config/usermods_dirs/NEON/defaults/user_nl_clm index 49528a524e..10880bd619 100644 --- a/cime_config/usermods_dirs/NEON/defaults/user_nl_clm +++ b/cime_config/usermods_dirs/NEON/defaults/user_nl_clm @@ -22,13 +22,13 @@ flanduse_timeseries = ' ' ! This isn't needed for a non transient case, but wi fsurdat = "$DIN_LOC_ROOT/lnd/clm2/surfdata_map/NEON/surfdata_0.9x1.25_hist_78pfts_CMIP6_simyr2000_${NEONSITE}_c221014.nc" model_year_align_urbantv = 2018 stream_year_first_urbantv = 2018 -stream_year_last_urbantv = 2021 +stream_year_last_urbantv = 2022 stream_year_first_ndep = 2018 model_year_align_ndep = 2018 -stream_year_last_ndep = 2021 +stream_year_last_ndep = 2022 model_year_align_popdens = 2018 stream_year_first_popdens = 2018 -stream_year_last_popdens = 2021 +stream_year_last_popdens = 2022 stream_fldfilename_lightng = '$DIN_LOC_ROOT/atm/datm7/NASA_LIS/clmforc.Li_2016_climo1995-2013.360x720.lnfm_Total_NEONarea_c210625.nc' !stream_fldfilename_ndep = '$DIN_LOC_ROOT/lnd/clm2/ndepdata/fndep_clm_f09_g17.CMIP6-SSP3-7.0-WACCM_2018-2030_monthly_c210826.nc' From e51d3a456bf77f502c2db4dbec4fee2599f7aba1 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Wed, 26 Oct 2022 16:16:51 -0600 Subject: [PATCH 31/95] Set the use-case by the type of compset --- cime_config/usermods_dirs/NEON/defaults/shell_commands | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/cime_config/usermods_dirs/NEON/defaults/shell_commands b/cime_config/usermods_dirs/NEON/defaults/shell_commands index e34df28191..1ab3fdca14 100644 --- a/cime_config/usermods_dirs/NEON/defaults/shell_commands +++ b/cime_config/usermods_dirs/NEON/defaults/shell_commands @@ -1,12 +1,10 @@ #!/bin/bash ./xmlchange CLM_USRDAT_NAME=NEON -./xmlchange CLM_NML_USE_CASE=1850-2100_SSP3-7.0_transient ./xmlchange CCSM_CO2_PPMV=408.83 ./xmlchange DATM_PRESAERO=SSP3-7.0 ./xmlchange DATM_PRESNDEP=SSP3-7.0 ./xmlchange DATM_PRESO3=SSP3-7.0 ./xmlchange DATM_YR_ALIGN=2018,DATM_YR_END=2021,DATM_YR_START=2018 -use_case=`./xmlquery CLM_NML_USE_CASE --value` compset=`./xmlquery COMPSET --value` # For a transient case run the whole length and don't cycle if [[ $compset =~ ^HIST ]]; then @@ -15,6 +13,9 @@ if [[ $compset =~ ^HIST ]]; then # Number of months that can be run for the full transient case ./xmlchange STOP_OPTION="nmonths" ./xmlchange STOP_N=51 + ./xmlchange CLM_NML_USE_CASE="2018-PD_transient" +else + ./xmlchange CLM_NML_USE_CASE="2018_control" fi # Explicitly set PIO Type to NETCDF since this is a single processor case (should already be set this way) From 3d452541c84c715bd5e5922387b2eb2d0f526f3a Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Thu, 27 Oct 2022 14:52:17 -0600 Subject: [PATCH 32/95] Add run_neon output directories to the git ignore --- .gitignore | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.gitignore b/.gitignore index 4b998f4dcb..7a22cf4b9b 100644 --- a/.gitignore +++ b/.gitignore @@ -105,6 +105,12 @@ unit_test_build /tools/mkmapdata/regrid.o* /tools/mkmapdata/map*.nc +# run_neon output directories +/tools/site_and_regional/????/ +/tools/site_and_regional/????.ad/ +/tools/site_and_regional/????.postad/ +/tools/site_and_regional/????.transient/ + # build output *.o *.mod From 6f646346dea67a1a28483c29a3f46e30f548d5ee Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Thu, 27 Oct 2022 15:16:47 -0600 Subject: [PATCH 33/95] Add the neon listing to git ignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 7a22cf4b9b..ca701132a7 100644 --- a/.gitignore +++ b/.gitignore @@ -106,6 +106,7 @@ unit_test_build /tools/mkmapdata/map*.nc # run_neon output directories +/tools/site_and_regional/listing.csv /tools/site_and_regional/????/ /tools/site_and_regional/????.ad/ /tools/site_and_regional/????.postad/ From ae750e14b927a44f640e014cc937565ff8a5dde2 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Thu, 27 Oct 2022 15:17:26 -0600 Subject: [PATCH 34/95] Add in 2018 and 2018-PD transient use cases --- bld/CLMBuildNamelist.pm | 2 ++ bld/namelist_files/namelist_defaults_ctsm.xml | 10 +++++++ .../namelist_definition_ctsm.xml | 2 +- .../use_cases/2018-PD_transient.xml | 29 +++++++++++++++++++ bld/namelist_files/use_cases/2018_control.xml | 11 +++++++ bld/namelist_files/use_cases/README | 5 ++++ bld/unit_testers/build-namelist_test.pl | 2 +- 7 files changed, 59 insertions(+), 2 deletions(-) create mode 100644 bld/namelist_files/use_cases/2018-PD_transient.xml create mode 100644 bld/namelist_files/use_cases/2018_control.xml diff --git a/bld/CLMBuildNamelist.pm b/bld/CLMBuildNamelist.pm index d9eeb1e5a0..c2a32da4ac 100755 --- a/bld/CLMBuildNamelist.pm +++ b/bld/CLMBuildNamelist.pm @@ -4560,6 +4560,8 @@ sub check_use_case_name { } else { $log->fatal_error($diestring); } + } elsif ( $use_case =~ /^([0-9]+|PI)-PD_*($desc)_transient$/ ) { + # valid name } elsif ( $use_case =~ /^([0-9]+)_*($desc)_control$/ ) { # valid name } elsif ( $use_case =~ /^($desc)_pd$/ ) { diff --git a/bld/namelist_files/namelist_defaults_ctsm.xml b/bld/namelist_files/namelist_defaults_ctsm.xml index 2686d62b9a..4bd9a6ee0b 100644 --- a/bld/namelist_files/namelist_defaults_ctsm.xml +++ b/bld/namelist_files/namelist_defaults_ctsm.xml @@ -25,6 +25,7 @@ attributes from the config_cache.xml file (with keys converted to upper-case). 379.0 388.8 397.5 +408.83 284.7 284.7 @@ -1449,6 +1450,9 @@ lnd/clm2/surfdata_map/release-clm5.0.30/surfdata_ne0np4.CONUS.ne30x8_hist_78pfts 2101 2015 +2018 +2018 + 2010 2010 @@ -1601,6 +1605,9 @@ lnd/clm2/surfdata_map/release-clm5.0.30/surfdata_ne0np4.CONUS.ne30x8_hist_78pfts 2100 2015 +2018 +2018 + 2010 2010 @@ -1619,6 +1626,9 @@ lnd/clm2/surfdata_map/release-clm5.0.30/surfdata_ne0np4.CONUS.ne30x8_hist_78pfts 2000 2000 +2018 +2018 + 2010 2010 diff --git a/bld/namelist_files/namelist_definition_ctsm.xml b/bld/namelist_files/namelist_definition_ctsm.xml index a08795dd1f..bc1dd034aa 100644 --- a/bld/namelist_files/namelist_definition_ctsm.xml +++ b/bld/namelist_files/namelist_definition_ctsm.xml @@ -2057,7 +2057,7 @@ If 1, turn on the MEGAN model for BVOC's (Biogenic Volitile Organic Compounds) +"PtVg,1000,850,1100,1350,1600,1850,1855,1865,1875,1885,1895,1905,1915,1925,1935,1945,1955,1965,1975,1979,1980,1982,1985,1995,2000,2005,2010,2013,2015,2018,2025,2035,2045,2055,2065,2075,2085,2095,2105"> Year to simulate and to provide datasets for (such as surface datasets, initial conditions, aerosol-deposition, Nitrogen deposition rates etc.) A sim_year of 1000 corresponds to data used for testing only, NOT corresponding to any real datasets. A sim_year greater than 2015 corresponds to ssp_rcp scenario data diff --git a/bld/namelist_files/use_cases/2018-PD_transient.xml b/bld/namelist_files/use_cases/2018-PD_transient.xml new file mode 100644 index 0000000000..d838efbd00 --- /dev/null +++ b/bld/namelist_files/use_cases/2018-PD_transient.xml @@ -0,0 +1,29 @@ + + + + +Simulate transient land-use, and aerosol deposition changes from 2018 to current day with a mix of historical data, and future scenario data + + + +2018 + +1850-2100 + + +SSP3-7.0 + +2018 +2022 +2018 + +2018 +2022 +2018 + +2018 +2022 +2018 + + diff --git a/bld/namelist_files/use_cases/2018_control.xml b/bld/namelist_files/use_cases/2018_control.xml new file mode 100644 index 0000000000..d5b745a4c7 --- /dev/null +++ b/bld/namelist_files/use_cases/2018_control.xml @@ -0,0 +1,11 @@ + + + + +Conditions to simulate 2018 land-use + +2018 + +constant + + diff --git a/bld/namelist_files/use_cases/README b/bld/namelist_files/use_cases/README index 4ccaf00bdc..e55fd9285b 100644 --- a/bld/namelist_files/use_cases/README +++ b/bld/namelist_files/use_cases/README @@ -17,6 +17,10 @@ Transient cases: 20thC$desc_transient (means nominal 1850-2000 although some datasets are 1850-2005) + or + + yyyy-PD_$desc_transient (means nominal year yyyy through present day (PD) (with the year for PD advancing) + Control cases: yyyy$desc_control @@ -30,6 +34,7 @@ Where yyyy = Simulation year (such as 1850 or 2000). yyyy-yyyy = Range of simulation years to run over (i.e.. 1850-2000). +yyyy-PD = Range of simulation years to run over until present day (i.e.. 2018-2024). $ssp_rcp = Shared Socieconomic Pathway (SSP) Representative concentration pathway (RCP) description string for future scenarios: SSP#-#.# (for example: SSP5-8.5, SSP1-2.6, SSP4-6.0 diff --git a/bld/unit_testers/build-namelist_test.pl b/bld/unit_testers/build-namelist_test.pl index f6c8d75937..784d3b5e0c 100755 --- a/bld/unit_testers/build-namelist_test.pl +++ b/bld/unit_testers/build-namelist_test.pl @@ -393,7 +393,7 @@ sub cat_and_create_namelistinfile { # # Now run the site # - my $options = "-res CLM_USRDAT -clm_usr_name NEON -no-megan -bgc bgc -sim_year 2000 -infile $namelistfile"; + my $options = "-res CLM_USRDAT -clm_usr_name NEON -no-megan -bgc bgc -sim_year 2018 -infile $namelistfile"; eval{ system( "$bldnml -envxml_dir . $options > $tempfile 2>&1 " ); }; is( $@, '', "options: $options" ); $cfiles->checkfilesexist( "$options", $mode ); From 8cc2405c618ca063b04d9a481f108ba1b1816f88 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Thu, 27 Oct 2022 15:36:17 -0600 Subject: [PATCH 35/95] CLM stream settings are now in the 2018 use cases --- cime_config/usermods_dirs/NEON/defaults/user_nl_clm | 9 --------- 1 file changed, 9 deletions(-) diff --git a/cime_config/usermods_dirs/NEON/defaults/user_nl_clm b/cime_config/usermods_dirs/NEON/defaults/user_nl_clm index 10880bd619..7495857ae7 100644 --- a/cime_config/usermods_dirs/NEON/defaults/user_nl_clm +++ b/cime_config/usermods_dirs/NEON/defaults/user_nl_clm @@ -20,15 +20,6 @@ flanduse_timeseries = ' ' ! This isn't needed for a non transient case, but will be once we start using transient compsets fsurdat = "$DIN_LOC_ROOT/lnd/clm2/surfdata_map/NEON/surfdata_0.9x1.25_hist_78pfts_CMIP6_simyr2000_${NEONSITE}_c221014.nc" -model_year_align_urbantv = 2018 -stream_year_first_urbantv = 2018 -stream_year_last_urbantv = 2022 -stream_year_first_ndep = 2018 -model_year_align_ndep = 2018 -stream_year_last_ndep = 2022 -model_year_align_popdens = 2018 -stream_year_first_popdens = 2018 -stream_year_last_popdens = 2022 stream_fldfilename_lightng = '$DIN_LOC_ROOT/atm/datm7/NASA_LIS/clmforc.Li_2016_climo1995-2013.360x720.lnfm_Total_NEONarea_c210625.nc' !stream_fldfilename_ndep = '$DIN_LOC_ROOT/lnd/clm2/ndepdata/fndep_clm_f09_g17.CMIP6-SSP3-7.0-WACCM_2018-2030_monthly_c210826.nc' From fcacc1cb77a9df2ff8a1453491e5e10069fc9845 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Wed, 2 Nov 2022 11:57:51 -0600 Subject: [PATCH 36/95] Set urbantv start and end for 2018 --- bld/namelist_files/namelist_defaults_ctsm.xml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/bld/namelist_files/namelist_defaults_ctsm.xml b/bld/namelist_files/namelist_defaults_ctsm.xml index 4bd9a6ee0b..c859fc13fc 100644 --- a/bld/namelist_files/namelist_defaults_ctsm.xml +++ b/bld/namelist_files/namelist_defaults_ctsm.xml @@ -1716,6 +1716,12 @@ lnd/clm2/surfdata_map/release-clm5.0.30/surfdata_ne0np4.CONUS.ne30x8_hist_78pfts 2106 2015 +2018 +2018 + +2010 +2010 + 2000 2000 From 6d6a43d76635e53e81289dccb17723a9b852d1ab Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Wed, 2 Nov 2022 12:43:32 -0600 Subject: [PATCH 37/95] 2018 control needs to use future scenario data --- bld/namelist_files/use_cases/2018_control.xml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/bld/namelist_files/use_cases/2018_control.xml b/bld/namelist_files/use_cases/2018_control.xml index d5b745a4c7..e5e572d749 100644 --- a/bld/namelist_files/use_cases/2018_control.xml +++ b/bld/namelist_files/use_cases/2018_control.xml @@ -8,4 +8,7 @@ constant + +SSP3-7.0 + From 78ad4ae21432711f61b34b720e87c47166fdbeac Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Wed, 2 Nov 2022 17:25:14 -0600 Subject: [PATCH 38/95] Explicitly turn crop on for the two NEON AG sites fixing #1889 --- cime_config/usermods_dirs/NEON/KONA/shell_commands | 4 ++++ cime_config/usermods_dirs/NEON/STER/shell_commands | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/cime_config/usermods_dirs/NEON/KONA/shell_commands b/cime_config/usermods_dirs/NEON/KONA/shell_commands index c00e220e77..66f274dd36 100644 --- a/cime_config/usermods_dirs/NEON/KONA/shell_commands +++ b/cime_config/usermods_dirs/NEON/KONA/shell_commands @@ -1,3 +1,7 @@ ./xmlchange NEONSITE=KONA ./xmlchange PTS_LON=263.38956 ./xmlchange PTS_LAT=39.10828 +# Setup to run with prognostic crops for this site +# If you want to explicitly run in SP mode or add other +# options you'll need to add that after this... +./xmlchange CLM_BLDNML_OPTS="--bgc bgc --crop" diff --git a/cime_config/usermods_dirs/NEON/STER/shell_commands b/cime_config/usermods_dirs/NEON/STER/shell_commands index 2c1699fc9c..38b173c309 100644 --- a/cime_config/usermods_dirs/NEON/STER/shell_commands +++ b/cime_config/usermods_dirs/NEON/STER/shell_commands @@ -1,3 +1,7 @@ ./xmlchange NEONSITE=STER ./xmlchange PTS_LON=256.96992 ./xmlchange PTS_LAT=40.45984 +# Setup to run with prognostic crops for this site +# If you want to explicitly run in SP mode or add other +# # options you'll need to add that after this... +./xmlchange CLM_BLDNML_OPTS="--bgc bgc --crop" From 0e09412a4bea1062ba1d9d6de7890a5f605c1cee Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Wed, 2 Nov 2022 19:26:59 -0600 Subject: [PATCH 39/95] Fix a little glitch where run-length couldn't be set --- tools/site_and_regional/run_neon.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tools/site_and_regional/run_neon.py b/tools/site_and_regional/run_neon.py index 1805ed7ebd..4b1407a2c1 100755 --- a/tools/site_and_regional/run_neon.py +++ b/tools/site_and_regional/run_neon.py @@ -280,6 +280,8 @@ def get_parser(args, description, valid_neon_sites): # The transient run length is set by cdeps atm buildnml to the last date of the available tower data # this value is not used run_length = "4Y" + else: + run_length = args.run_length run_length = parse_isoduration(run_length) base_case_root = None From b2040f46ca357abfdfe23334052ad61e2904e7da Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Fri, 4 Nov 2022 00:53:56 -0600 Subject: [PATCH 40/95] Make sure the compset type is correct for both basecase and case cloning from that --- tools/site_and_regional/run_neon.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/tools/site_and_regional/run_neon.py b/tools/site_and_regional/run_neon.py index 4b1407a2c1..c7418b786c 100755 --- a/tools/site_and_regional/run_neon.py +++ b/tools/site_and_regional/run_neon.py @@ -421,6 +421,14 @@ def build_base_case( print("---- base case setup ------") case.case_setup() else: + # For existing case check that the compset name is correct + existingcompname = case.get_value("COMPSET") + match = re.search("^HIST", existingcompname, flags=re.IGNORECASE) + if re.search("^HIST", compset, flags=re.IGNORECASE) is None: + expect( match == None, "Existing base case is a historical type and should not be -- rerun with the --orverwrite option" ) + else: + expect( match != None, "Existing base case should be a historical type and is not -- rerun with the --orverwrite option" ) + # reset the case case.case_setup(reset=True) case_path = case.get_value("CASEROOT") @@ -481,6 +489,13 @@ def run_case( elif rerun: with Case(case_root, read_only=False) as case: rundir = case.get_value("RUNDIR") + # For existing case check that the compset name is correct + existingcompname = case.get_value("COMPSET") + match = re.search("^HIST", existingcompname, flags=re.IGNORECASE) + if re.search("^HIST", compset, flags=re.IGNORECASE) is None: + expect( match == None, "Existing base case is a historical type and should not be -- rerun with the --orverwrite option" ) + else: + expect( match != None, "Existing base case should be a historical type and is not -- rerun with the --orverwrite option" ) if os.path.isfile(os.path.join(rundir, "ESMF_Profile.summary")): print( "Case {} appears to be complete, not rerunning.".format( From b824c780c8db20a39fd6f8bb01ce78404341451c Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Fri, 4 Nov 2022 10:25:11 -0600 Subject: [PATCH 41/95] Always ensure to build/run single point NEON sites with mpi-serial rather than a full MPI library to save complexity --- cime_config/usermods_dirs/NEON/defaults/shell_commands | 1 + 1 file changed, 1 insertion(+) diff --git a/cime_config/usermods_dirs/NEON/defaults/shell_commands b/cime_config/usermods_dirs/NEON/defaults/shell_commands index 1ab3fdca14..5a20d6ac38 100644 --- a/cime_config/usermods_dirs/NEON/defaults/shell_commands +++ b/cime_config/usermods_dirs/NEON/defaults/shell_commands @@ -4,6 +4,7 @@ ./xmlchange DATM_PRESAERO=SSP3-7.0 ./xmlchange DATM_PRESNDEP=SSP3-7.0 ./xmlchange DATM_PRESO3=SSP3-7.0 +./xmlchange MPILIB=mpi-serial ./xmlchange DATM_YR_ALIGN=2018,DATM_YR_END=2021,DATM_YR_START=2018 compset=`./xmlquery COMPSET --value` # For a transient case run the whole length and don't cycle From 69b8e3204f941977070581d6efaf104694e13208 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Fri, 4 Nov 2022 10:25:38 -0600 Subject: [PATCH 42/95] Add the header for user_nl_datm_streams, so that there are instructions on usage --- .../NEON/defaults/user_nl_datm_streams | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/cime_config/usermods_dirs/NEON/defaults/user_nl_datm_streams b/cime_config/usermods_dirs/NEON/defaults/user_nl_datm_streams index f1af044c31..36f1e72b3a 100644 --- a/cime_config/usermods_dirs/NEON/defaults/user_nl_datm_streams +++ b/cime_config/usermods_dirs/NEON/defaults/user_nl_datm_streams @@ -1,3 +1,26 @@ +!------------------------------------------------------------------------ +! This file is used to modify datm.streams.xml generated in $RUNDIR +! Entries should have the form +! :<= new stream_value> +! The following are accepted values for an assume streamname of foo +! foo:meshfile = character string +! foo:datafiles = comma separated string of full pathnames (e.g. file1,file2,file3...) +! foo:datavars = comma separated string of field pairs (e.g. foo foobar,foo2 foobar2...) +! foo:taxmode = one of [cycle, extend, limit] +! foo:tintalgo = one of [lower,upper,nearest,linear,coszen] +! foo:readmode = single (only suported mode right now) +! foo:mapalgo = one of [bilinear,redist,nn,consf,consd,none] +! foo:dtlimit = real (1.5 is default) +! foo:year_first = integer +! foo:year_last = integer +! foo:year_align = integer +! foo:vectors = one of [none,u:v] +! foo:lev_dimname: = one of [null,name of level dimenion name] +! foo:offset = integer +! As an example: +! foo:year_first = 1950 +! would change the stream year_first stream_entry to 1950 for the foo stream block +!------------------------------------------------------------------------ presaero.SSP3-7.0:datafiles = $DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/aero/aerodep_clm_SSP370_b.e21.BWSSP370cmip6.f09_g17.CMIP6-SSP3-7.0-WACCM.001_2018-2030_monthly_0.9x1.25_c210826.nc presaero.SSP3-7.0:year_first=2018 presaero.SSP3-7.0:year_last=2022 From dbe02ede634bdbd3222d8ec9cdfbd75d8f33ed05 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Fri, 4 Nov 2022 10:27:11 -0600 Subject: [PATCH 43/95] Don't reapply user-mod in run_neon clone, and don't set DATM_YR_END The base case already has the NEON user-mods applied in it, so we don't need to reapply in the cloned case. Don't set DATM_YR_END, let it be set in the default user-mods and the specific user-mods for the site. The only advantage of setting DATM_YR_END in run_neon is if your run-length is shorter than what's in the user-mods, and it only saves downloading the extra files and a little bit of time to read them at initialization. So there's no need to have logic to handle that here. --- tools/site_and_regional/run_neon.py | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/tools/site_and_regional/run_neon.py b/tools/site_and_regional/run_neon.py index c7418b786c..879b41119d 100755 --- a/tools/site_and_regional/run_neon.py +++ b/tools/site_and_regional/run_neon.py @@ -525,7 +525,7 @@ def run_case( with Case(base_case_root, read_only=False) as basecase: print("---- cloning the base case in {}".format(case_root)) basecase.create_clone( - case_root, keepexe=True, user_mods_dirs=user_mods_dirs + case_root, keepexe=True ) with Case(case_root, read_only=False) as case: @@ -561,13 +561,6 @@ def run_case( return case.set_value("CALENDAR", "GREGORIAN") case.set_value("RESUBMIT", 0) - # Turning all this off for usermod controsl over start, align, and end year - else: - # for the spinup we want the start and end on year boundaries - if self.end_month == 12: - case.set_value("DATM_YR_END", self.end_year) - else: - case.set_value("DATM_YR_END", self.end_year - 1) if not rundir: rundir = case.get_value("RUNDIR") From 222ba813d1ae566ce3bcbe9a93bcc52b21a457eb Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Fri, 4 Nov 2022 15:03:59 -0600 Subject: [PATCH 44/95] Add mesh file for reduced Lightning dataset --- cime_config/usermods_dirs/NEON/defaults/user_nl_clm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cime_config/usermods_dirs/NEON/defaults/user_nl_clm b/cime_config/usermods_dirs/NEON/defaults/user_nl_clm index 7495857ae7..7dd1ce7f32 100644 --- a/cime_config/usermods_dirs/NEON/defaults/user_nl_clm +++ b/cime_config/usermods_dirs/NEON/defaults/user_nl_clm @@ -22,7 +22,7 @@ flanduse_timeseries = ' ' ! This isn't needed for a non transient case, but wi fsurdat = "$DIN_LOC_ROOT/lnd/clm2/surfdata_map/NEON/surfdata_0.9x1.25_hist_78pfts_CMIP6_simyr2000_${NEONSITE}_c221014.nc" stream_fldfilename_lightng = '$DIN_LOC_ROOT/atm/datm7/NASA_LIS/clmforc.Li_2016_climo1995-2013.360x720.lnfm_Total_NEONarea_c210625.nc' -!stream_fldfilename_ndep = '$DIN_LOC_ROOT/lnd/clm2/ndepdata/fndep_clm_f09_g17.CMIP6-SSP3-7.0-WACCM_2018-2030_monthly_c210826.nc' +stream_meshfile_lightng = '$DIN_LOC_ROOT/atm/datm7/NASA_LIS/ESMF_MESH.Li_2016.360x720.NEONarea_cdf5_c221104.nc' ! h1 output stream hist_fincl2 = 'AR','ELAI','FCEV','FCTR','FGEV','FIRA','FSA','FSH','GPP','H2OSOI', From e855ee41a58a5033fc97adc531865635abbcace0 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Fri, 4 Nov 2022 15:05:36 -0600 Subject: [PATCH 45/95] Remove the PIO_REARRANGER setting, which is hopefully not needed with the ESMF mesh file for lightning, and use of mpi-serial --- cime_config/usermods_dirs/NEON/defaults/shell_commands | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/cime_config/usermods_dirs/NEON/defaults/shell_commands b/cime_config/usermods_dirs/NEON/defaults/shell_commands index 5a20d6ac38..7095e1def7 100644 --- a/cime_config/usermods_dirs/NEON/defaults/shell_commands +++ b/cime_config/usermods_dirs/NEON/defaults/shell_commands @@ -1,10 +1,13 @@ #!/bin/bash ./xmlchange CLM_USRDAT_NAME=NEON ./xmlchange CCSM_CO2_PPMV=408.83 +# Set data forcing data to future scenario so will have data from 2018 to present-day ./xmlchange DATM_PRESAERO=SSP3-7.0 ./xmlchange DATM_PRESNDEP=SSP3-7.0 ./xmlchange DATM_PRESO3=SSP3-7.0 +# Explicitly set the MPI library to mpi-serial so won't have the build/run complexity of a full MPI library ./xmlchange MPILIB=mpi-serial +# Set years to run forcing data over ./xmlchange DATM_YR_ALIGN=2018,DATM_YR_END=2021,DATM_YR_START=2018 compset=`./xmlquery COMPSET --value` # For a transient case run the whole length and don't cycle @@ -21,8 +24,3 @@ fi # Explicitly set PIO Type to NETCDF since this is a single processor case (should already be set this way) ./xmlchange PIO_TYPENAME=netcdf -# BD:05/06/2022 - The PIO_REARRANGER_LND value - for global runs, PIO_REARRANGER_LND = 1 is ideal -# and a value of 2 results in slow I/O. For point runs like these, a value of 1 results in a crash (PIO bug, probably), -# so we explicitly set a value of 2. -./xmlchange PIO_REARRANGER_LND=2 - From 4dd4a702562d9efbda1f7e073d88eb71adc860c7 Mon Sep 17 00:00:00 2001 From: wwieder Date: Sat, 5 Nov 2022 07:43:04 -0600 Subject: [PATCH 46/95] needed so cloned cases point to correct datm input directory --- tools/site_and_regional/run_neon.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/site_and_regional/run_neon.py b/tools/site_and_regional/run_neon.py index 879b41119d..982495f611 100755 --- a/tools/site_and_regional/run_neon.py +++ b/tools/site_and_regional/run_neon.py @@ -525,7 +525,7 @@ def run_case( with Case(base_case_root, read_only=False) as basecase: print("---- cloning the base case in {}".format(case_root)) basecase.create_clone( - case_root, keepexe=True + case_root, keepexe=True, user_mods_dirs=user_mods_dirs ) with Case(case_root, read_only=False) as case: From b1096d71576e825687ae9728a8241c8b3d71f406 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Sat, 5 Nov 2022 13:49:50 -0600 Subject: [PATCH 47/95] Add comment about needing to keep the user_mod_dirs argument in the clone case --- tools/site_and_regional/run_neon.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tools/site_and_regional/run_neon.py b/tools/site_and_regional/run_neon.py index 982495f611..6892dd3d00 100755 --- a/tools/site_and_regional/run_neon.py +++ b/tools/site_and_regional/run_neon.py @@ -524,6 +524,12 @@ def run_case( # read_only = False should not be required here with Case(base_case_root, read_only=False) as basecase: print("---- cloning the base case in {}".format(case_root)) + # + # EBK: 11/05/2022 -- Note keeping the user_mods_dirs argument is important. Although + # it causes some of the user_nl_* files to have duplicated inputs. It also ensures + # that the shell_commands file is copied, as well as taking care of the DATM inputs. + # See https://github.com/ESCOMP/CTSM/pull/1872#pullrequestreview-1169407493 + # basecase.create_clone( case_root, keepexe=True, user_mods_dirs=user_mods_dirs ) From 1d6bb47fbc40cfca325c64f64592e6bfa049858e Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Sat, 5 Nov 2022 20:11:42 -0600 Subject: [PATCH 48/95] Add run_black target to not just check with black but to run it, add beginning of a new test for add_tag_to_filename, is currently hardwired to only work on todays date --- python/Makefile | 8 +++++- python/ctsm/test/test_unit_utils_add_tag.py | 30 +++++++++++++++++++++ 2 files changed, 37 insertions(+), 1 deletion(-) create mode 100755 python/ctsm/test/test_unit_utils_add_tag.py diff --git a/python/Makefile b/python/Makefile index 4ea5fba85d..bf219f16b6 100644 --- a/python/Makefile +++ b/python/Makefile @@ -39,11 +39,17 @@ lint: FORCE $(PYLINT) $(PYLINT_ARGS) $(PYLINT_SRC) .PHONY: black -# Run black on all of the python files here and undeneath. +# Run the black check on all of the python files here and undeneath. # Use the black configure file to explicitly set a few things and specifiy the exact files. black: FORCE black --check --config pyproject.toml . +.PHONY: run_black +# Run black on all of the python files here and undeneath. +# Use the black configure file to explicitly set a few things and specifiy the exact files. +run_black: FORCE + black --config pyproject.toml . + .PHONY: clean clean: FORCE find . -name '*.pyc' -exec rm {} \; diff --git a/python/ctsm/test/test_unit_utils_add_tag.py b/python/ctsm/test/test_unit_utils_add_tag.py new file mode 100755 index 0000000000..f1564fc1f6 --- /dev/null +++ b/python/ctsm/test/test_unit_utils_add_tag.py @@ -0,0 +1,30 @@ +#!/usr/bin/env python3 + +"""Unit tests for add_tag_to_filename +""" + +import unittest + +from ctsm import unit_testing +from ctsm.utils import add_tag_to_filename + +# Allow names that pylint doesn't like, because otherwise I find it hard +# to make readable unit test names +# pylint: disable=invalid-name + + +class TestUtilsAddTag(unittest.TestCase): + """Tests of utils: add_tag_to_filename""" + + def testSimple(self): + """Simple test of surface dataset name""" + + fsurf_in = "surfdata_0.9x1.25_hist_16pfts_Irrig_CMIP6_simyr2000_c221105.nc" + fsurf_out = add_tag_to_filename(fsurf_in, "tag") + expect_fsurf = "surfdata_0.9x1.25_hist_16pfts_Irrig_CMIP6_simyr2000_tag_c221105.nc" + self.assertEqual(expect_fsurf, fsurf_out, "Expect filenames to be as expected") + + +if __name__ == "__main__": + unit_testing.setup_for_tests() + unittest.main() From 4c7e79383a37c8eea47048f53e7440c5b91b1154 Mon Sep 17 00:00:00 2001 From: wwieder Date: Sun, 6 Nov 2022 06:04:41 -0700 Subject: [PATCH 49/95] force STOP_OPTION in transient run to nmonths --- tools/site_and_regional/run_neon.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/site_and_regional/run_neon.py b/tools/site_and_regional/run_neon.py index 6892dd3d00..1b314bd469 100755 --- a/tools/site_and_regional/run_neon.py +++ b/tools/site_and_regional/run_neon.py @@ -535,7 +535,7 @@ def run_case( ) with Case(case_root, read_only=False) as case: - if run_type is not"transient": + if run_type is not "transient": # in order to avoid the complication of leap years we always set the run_length in units of days. case.set_value("STOP_OPTION", "ndays") case.set_value("REST_OPTION", "end") @@ -567,6 +567,7 @@ def run_case( return case.set_value("CALENDAR", "GREGORIAN") case.set_value("RESUBMIT", 0) + case.set_value("STOP_OPTION", "nmonths") if not rundir: rundir = case.get_value("RUNDIR") From 83f3304cfea30af58bf225455a451142605301e8 Mon Sep 17 00:00:00 2001 From: wwieder Date: Sun, 6 Nov 2022 07:09:55 -0700 Subject: [PATCH 50/95] modified for last month of DATM data at CPER --- cime_config/usermods_dirs/NEON/CPER/shell_commands | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cime_config/usermods_dirs/NEON/CPER/shell_commands b/cime_config/usermods_dirs/NEON/CPER/shell_commands index 169b358a40..b6ccbcdf57 100644 --- a/cime_config/usermods_dirs/NEON/CPER/shell_commands +++ b/cime_config/usermods_dirs/NEON/CPER/shell_commands @@ -1,3 +1,6 @@ ./xmlchange NEONSITE=CPER ./xmlchange PTS_LON=255.25545 ./xmlchange PTS_LAT=40.81297 +if [[ $compset =~ ^HIST ]]; then + ./xmlchange STOP_N=50 +fi From ba48af8962a795784b7b414b1b99d9618bfa2f8b Mon Sep 17 00:00:00 2001 From: wwieder Date: Sun, 6 Nov 2022 07:22:55 -0700 Subject: [PATCH 51/95] let usermods set start date for transient runs --- tools/site_and_regional/run_neon.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tools/site_and_regional/run_neon.py b/tools/site_and_regional/run_neon.py index 1b314bd469..2879a9cf9e 100755 --- a/tools/site_and_regional/run_neon.py +++ b/tools/site_and_regional/run_neon.py @@ -626,11 +626,11 @@ def set_ref_case(self, case): case.set_value("RUN_REFDATE", refdate) if case_root.endswith(".postad"): case.set_value("RUN_STARTDATE", refdate) - else: - case.set_value( - "RUN_STARTDATE", - "{yr:04d}-{mo:02d}-01".format(yr=self.start_year, mo=self.start_month), - ) + #else: + #case.set_value( + # "RUN_STARTDATE", + # "{yr:04d}-{mo:02d}-01".format(yr=self.start_year, mo=self.start_month), + #) return True def modify_user_nl(self, case_root, run_type, rundir): From 457f5085b6647c5f93908a903b88f6d6ae77ce64 Mon Sep 17 00:00:00 2001 From: wwieder Date: Sun, 6 Nov 2022 10:58:45 -0700 Subject: [PATCH 52/95] modify shell comands to avoid cycling climte data for last month in transient case --- cime_config/usermods_dirs/NEON/MLBS/shell_commands | 1 + cime_config/usermods_dirs/NEON/MOAB/shell_commands | 1 + cime_config/usermods_dirs/NEON/ONAQ/shell_commands | 1 + 3 files changed, 3 insertions(+) diff --git a/cime_config/usermods_dirs/NEON/MLBS/shell_commands b/cime_config/usermods_dirs/NEON/MLBS/shell_commands index e0a2ea5443..d5de0f64eb 100644 --- a/cime_config/usermods_dirs/NEON/MLBS/shell_commands +++ b/cime_config/usermods_dirs/NEON/MLBS/shell_commands @@ -5,5 +5,6 @@ ./xmlchange DATM_YR_END=2019 # Different default number of months to run for transient case if [[ $compset =~ ^HIST ]]; then + ./xmlchange DATM_YR_END=2020 ./xmlchange STOP_N=24 fi diff --git a/cime_config/usermods_dirs/NEON/MOAB/shell_commands b/cime_config/usermods_dirs/NEON/MOAB/shell_commands index c90aaa30d4..96d0bcbe68 100644 --- a/cime_config/usermods_dirs/NEON/MOAB/shell_commands +++ b/cime_config/usermods_dirs/NEON/MOAB/shell_commands @@ -5,5 +5,6 @@ ./xmlchange DATM_YR_END=2020 # Different default number of months to run for transient case if [[ $compset =~ ^HIST ]]; then + ./xmlchange DATM_YR_END=2021 ./xmlchange STOP_N=36 fi diff --git a/cime_config/usermods_dirs/NEON/ONAQ/shell_commands b/cime_config/usermods_dirs/NEON/ONAQ/shell_commands index df2ca55dad..43dab69998 100644 --- a/cime_config/usermods_dirs/NEON/ONAQ/shell_commands +++ b/cime_config/usermods_dirs/NEON/ONAQ/shell_commands @@ -5,5 +5,6 @@ ./xmlchange DATM_YR_END=2019 # Different default number of months to run for transient case if [[ $compset =~ ^HIST ]]; then + ./xmlchange DATM_YR_END=2020 ./xmlchange STOP_N=24 fi From a262232a1d942408dac3fa8e9cc45c0dac21c4fd Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Tue, 8 Nov 2022 12:49:47 -0700 Subject: [PATCH 53/95] Get mock working for date, and use the way date was used in utils --- python/ctsm/test/test_unit_utils_add_tag.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/python/ctsm/test/test_unit_utils_add_tag.py b/python/ctsm/test/test_unit_utils_add_tag.py index f1564fc1f6..176f1a1b08 100755 --- a/python/ctsm/test/test_unit_utils_add_tag.py +++ b/python/ctsm/test/test_unit_utils_add_tag.py @@ -6,7 +6,10 @@ import unittest from ctsm import unit_testing -from ctsm.utils import add_tag_to_filename +from unittest.mock import patch +from datetime import date + +from ctsm import utils # Allow names that pylint doesn't like, because otherwise I find it hard # to make readable unit test names @@ -16,12 +19,20 @@ class TestUtilsAddTag(unittest.TestCase): """Tests of utils: add_tag_to_filename""" + @staticmethod + def _fake_today(): + return date(year=2022, month=10, day=31) + def testSimple(self): """Simple test of surface dataset name""" fsurf_in = "surfdata_0.9x1.25_hist_16pfts_Irrig_CMIP6_simyr2000_c221105.nc" - fsurf_out = add_tag_to_filename(fsurf_in, "tag") - expect_fsurf = "surfdata_0.9x1.25_hist_16pfts_Irrig_CMIP6_simyr2000_tag_c221105.nc" + with patch("ctsm.utils.date") as mock_date: + mock_date.today.side_effect = self._fake_today + + fsurf_out = utils.add_tag_to_filename(fsurf_in, "tag") + + expect_fsurf = "surfdata_0.9x1.25_hist_16pfts_Irrig_CMIP6_simyr2000_tag_c221031.nc" self.assertEqual(expect_fsurf, fsurf_out, "Expect filenames to be as expected") From 8958f12bf2cdb2ba4afeef223267e8f68c935cfd Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Tue, 8 Nov 2022 15:29:58 -0700 Subject: [PATCH 54/95] Add some more tests --- python/ctsm/test/test_unit_utils_add_tag.py | 24 +++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/python/ctsm/test/test_unit_utils_add_tag.py b/python/ctsm/test/test_unit_utils_add_tag.py index 176f1a1b08..04e0f02306 100755 --- a/python/ctsm/test/test_unit_utils_add_tag.py +++ b/python/ctsm/test/test_unit_utils_add_tag.py @@ -21,6 +21,7 @@ class TestUtilsAddTag(unittest.TestCase): @staticmethod def _fake_today(): + """Set the fake date to Halloween""" return date(year=2022, month=10, day=31) def testSimple(self): @@ -35,6 +36,29 @@ def testSimple(self): expect_fsurf = "surfdata_0.9x1.25_hist_16pfts_Irrig_CMIP6_simyr2000_tag_c221031.nc" self.assertEqual(expect_fsurf, fsurf_out, "Expect filenames to be as expected") + def testSimpleLanduse(self): + """Simple test of landuse dataset name""" + + landuse_in = "landuse.timeseries_0.9x1.25_hist_78pfts_CMIP6_simyr1850-2015_c190214.nc" + with patch("ctsm.utils.date") as mock_date: + mock_date.today.side_effect = self._fake_today + + landuse_out = utils.add_tag_to_filename(landuse_in, "tag") + + expect_landuse = "landuse.timeseries_0.9x1.25_hist_78pfts_CMIP6_simyr1850-2015_tag_c221031.nc" + self.assertEqual(expect_landuse, landuse_out, "Expect filenames to be as expected") + + def testSimpleDatmDomain(self): + """Simple test of datm domain dataset name""" + + file_in = "domain.lnd.360x720_gswp3.0v1.c170606.nc" + with patch("ctsm.utils.date") as mock_date: + mock_date.today.side_effect = self._fake_today + + file_out = utils.add_tag_to_filename(file_in, "tag") + + expect_filename = "domain.lnd.360x720_gswp3.0v1_tag_c221031.nc" + self.assertEqual(expect_filename, file_out, "Expect filenames to be as expected") if __name__ == "__main__": unit_testing.setup_for_tests() From cf5841ef8c7341a190dd527194b8902fc0745cc7 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Tue, 8 Nov 2022 15:32:14 -0700 Subject: [PATCH 55/95] Add test for domain file as well --- python/ctsm/test/test_unit_utils_add_tag.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/python/ctsm/test/test_unit_utils_add_tag.py b/python/ctsm/test/test_unit_utils_add_tag.py index 04e0f02306..b63a76eba9 100755 --- a/python/ctsm/test/test_unit_utils_add_tag.py +++ b/python/ctsm/test/test_unit_utils_add_tag.py @@ -60,6 +60,18 @@ def testSimpleDatmDomain(self): expect_filename = "domain.lnd.360x720_gswp3.0v1_tag_c221031.nc" self.assertEqual(expect_filename, file_out, "Expect filenames to be as expected") + def testSimpleDomain(self): + """Simple test of domain dataset name""" + + file_in = "domain.lnd.fv0.9x1.25_gx1v7.151020.nc" + with patch("ctsm.utils.date") as mock_date: + mock_date.today.side_effect = self._fake_today + + file_out = utils.add_tag_to_filename(file_in, "tag") + + expect_filename = "domain.lnd.fv0.9x1.25_gx1v7_tag_c221031.nc" + self.assertEqual(expect_filename, file_out, "Expect filenames to be as expected") + if __name__ == "__main__": unit_testing.setup_for_tests() unittest.main() From 52a0179d98d5dbd9292aeba38e4d14c952752e13 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Tue, 8 Nov 2022 23:04:01 -0700 Subject: [PATCH 56/95] Add ability of add_tag_to_filename to replace resolution for the surface dataset, and add a unit test for it --- python/ctsm/test/test_unit_utils_add_tag.py | 10 +++++++-- python/ctsm/utils.py | 23 +++++++++++++++++++-- 2 files changed, 29 insertions(+), 4 deletions(-) diff --git a/python/ctsm/test/test_unit_utils_add_tag.py b/python/ctsm/test/test_unit_utils_add_tag.py index b63a76eba9..491dfc32c9 100755 --- a/python/ctsm/test/test_unit_utils_add_tag.py +++ b/python/ctsm/test/test_unit_utils_add_tag.py @@ -5,9 +5,9 @@ import unittest -from ctsm import unit_testing from unittest.mock import patch from datetime import date +from ctsm import unit_testing from ctsm import utils @@ -32,9 +32,12 @@ def testSimple(self): mock_date.today.side_effect = self._fake_today fsurf_out = utils.add_tag_to_filename(fsurf_in, "tag") + fsurf_out2 = utils.add_tag_to_filename(fsurf_in, "tag", replace_res=True) expect_fsurf = "surfdata_0.9x1.25_hist_16pfts_Irrig_CMIP6_simyr2000_tag_c221031.nc" self.assertEqual(expect_fsurf, fsurf_out, "Expect filenames to be as expected") + expect_fsurf2 = "surfdata_tag_hist_16pfts_Irrig_CMIP6_simyr2000_c221031.nc" + self.assertEqual(expect_fsurf2, fsurf_out2, "Expect filenames to be as expected") def testSimpleLanduse(self): """Simple test of landuse dataset name""" @@ -45,7 +48,9 @@ def testSimpleLanduse(self): landuse_out = utils.add_tag_to_filename(landuse_in, "tag") - expect_landuse = "landuse.timeseries_0.9x1.25_hist_78pfts_CMIP6_simyr1850-2015_tag_c221031.nc" + expect_landuse = ( + "landuse.timeseries_0.9x1.25_hist_78pfts_CMIP6_simyr1850-2015_tag_c221031.nc" + ) self.assertEqual(expect_landuse, landuse_out, "Expect filenames to be as expected") def testSimpleDatmDomain(self): @@ -72,6 +77,7 @@ def testSimpleDomain(self): expect_filename = "domain.lnd.fv0.9x1.25_gx1v7_tag_c221031.nc" self.assertEqual(expect_filename, file_out, "Expect filenames to be as expected") + if __name__ == "__main__": unit_testing.setup_for_tests() unittest.main() diff --git a/python/ctsm/utils.py b/python/ctsm/utils.py index 2851a3b619..2ef40945ed 100644 --- a/python/ctsm/utils.py +++ b/python/ctsm/utils.py @@ -4,6 +4,7 @@ import os import sys import string +import re import pdb from datetime import date @@ -43,21 +44,28 @@ def fill_template_file(path_to_template, path_to_final, substitutions): final_file.write(final_file_contents) -def add_tag_to_filename(filename, tag): +def add_tag_to_filename(filename, tag, replace_res=False): """ Add a tag and replace timetag of a filename Expects file to end with [._]cYYMMDD.nc or [._]YYMMDD.nc Add the tag to just before that ending part and change the ending part to the current time tag. + if replace_res is True, then replace the resolution + part of the filename. Expects the file to start with + [a-z.]_ and then the resolution. + Parameters ---------- filename (str) : file name tag (str) : string of a tag to be added to the end of filename + (or to replace the resolution part of the filename) Raises ------ Error: When it cannot find . and _ in the filename. + Error: When it's asked to replace the resolution and + can't figure out where that is in the filename. Returns ------ @@ -73,7 +81,18 @@ def add_tag_to_filename(filename, tag): abort(err_msg) today = date.today() today_string = today.strftime("%y%m%d") - fname_out = basename[:cend] + "_" + tag + "_c" + today_string + ".nc" + if not replace_res: + fname_out = basename[:cend] + "_" + tag + "_c" + today_string + ".nc" + else: + match = re.fullmatch(r"([a-z.]+)_([0-9x.]+)_(.+?)", basename[:cend]) + if match is not None: + fname_out = ( + match.group(1) + "_" + tag + "_" + match.group(3) + "_c" + today_string + ".nc" + ) + else: + abort( + "Trouble figuring out where to replace the resolution in the filename: " + filename + ) return fname_out From c40fcd529c43aece073daa09576c63c1ab1e445f Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Tue, 8 Nov 2022 23:12:54 -0700 Subject: [PATCH 57/95] Allow more characters for resolution in filename --- python/ctsm/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/ctsm/utils.py b/python/ctsm/utils.py index 2ef40945ed..7294f09d4c 100644 --- a/python/ctsm/utils.py +++ b/python/ctsm/utils.py @@ -84,7 +84,7 @@ def add_tag_to_filename(filename, tag, replace_res=False): if not replace_res: fname_out = basename[:cend] + "_" + tag + "_c" + today_string + ".nc" else: - match = re.fullmatch(r"([a-z.]+)_([0-9x.]+)_(.+?)", basename[:cend]) + match = re.fullmatch(r"([a-z.]+)_([fv0-9x.crunldas]+)_(.+?)", basename[:cend]) if match is not None: fname_out = ( match.group(1) + "_" + tag + "_" + match.group(3) + "_c" + today_string + ".nc" From 9dd28857ec4565a44785b9b88ec04ca7c9ef80b7 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Tue, 8 Nov 2022 23:33:38 -0700 Subject: [PATCH 58/95] Add a landunit test for the replace_res option --- python/ctsm/test/test_unit_utils_add_tag.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/python/ctsm/test/test_unit_utils_add_tag.py b/python/ctsm/test/test_unit_utils_add_tag.py index 491dfc32c9..4c9534b1d3 100755 --- a/python/ctsm/test/test_unit_utils_add_tag.py +++ b/python/ctsm/test/test_unit_utils_add_tag.py @@ -47,11 +47,16 @@ def testSimpleLanduse(self): mock_date.today.side_effect = self._fake_today landuse_out = utils.add_tag_to_filename(landuse_in, "tag") + landuse_out2 = utils.add_tag_to_filename(landuse_in, "tag", replace_res=True) expect_landuse = ( "landuse.timeseries_0.9x1.25_hist_78pfts_CMIP6_simyr1850-2015_tag_c221031.nc" ) self.assertEqual(expect_landuse, landuse_out, "Expect filenames to be as expected") + expect_landuse2 = ( + "landuse.timeseries_tag_hist_78pfts_CMIP6_simyr1850-2015_c221031.nc" + ) + self.assertEqual(expect_landuse2, landuse_out2, "Expect filenames to be as expected") def testSimpleDatmDomain(self): """Simple test of datm domain dataset name""" From 3781d6b6d8aace20f6268cfca12712743e9c5649 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Wed, 9 Nov 2022 11:23:02 -0700 Subject: [PATCH 59/95] Add a test to loop over a list of surface dataset names --- python/ctsm/test/test_unit_utils_add_tag.py | 68 ++++++++++++++++++++- 1 file changed, 65 insertions(+), 3 deletions(-) diff --git a/python/ctsm/test/test_unit_utils_add_tag.py b/python/ctsm/test/test_unit_utils_add_tag.py index 4c9534b1d3..005fb787dc 100755 --- a/python/ctsm/test/test_unit_utils_add_tag.py +++ b/python/ctsm/test/test_unit_utils_add_tag.py @@ -53,9 +53,7 @@ def testSimpleLanduse(self): "landuse.timeseries_0.9x1.25_hist_78pfts_CMIP6_simyr1850-2015_tag_c221031.nc" ) self.assertEqual(expect_landuse, landuse_out, "Expect filenames to be as expected") - expect_landuse2 = ( - "landuse.timeseries_tag_hist_78pfts_CMIP6_simyr1850-2015_c221031.nc" - ) + expect_landuse2 = "landuse.timeseries_tag_hist_78pfts_CMIP6_simyr1850-2015_c221031.nc" self.assertEqual(expect_landuse2, landuse_out2, "Expect filenames to be as expected") def testSimpleDatmDomain(self): @@ -82,6 +80,70 @@ def testSimpleDomain(self): expect_filename = "domain.lnd.fv0.9x1.25_gx1v7_tag_c221031.nc" self.assertEqual(expect_filename, file_out, "Expect filenames to be as expected") + def testSurfReplaceListDomain(self): + """Simple test of list of surface dataset name with replace_res option""" + + files_in = [ + "surfdata_48x96_hist_16pfts_Irrig_CMIP6_simyr2000_c190214.nc", + "surfdata_0.9x1.25_hist_16pfts_Irrig_CMIP6_simyr2000_c190214.nc", + "surfdata_0.9x1.25_hist_16pfts_Irrig_CMIP6_simyr2000_c190214.nc", + "surfdata_1.9x2.5_hist_16pfts_Irrig_CMIP6_simyr2000_c190304.nc", + "surfdata_1.9x2.5_hist_16pfts_Irrig_CMIP6_simyr2000_c190304.nc", + "surfdata_4x5_hist_16pfts_Irrig_CMIP6_simyr2000_c190214.nc", + "surfdata_10x15_hist_16pfts_Irrig_CMIP6_simyr2000_c190214.nc", + "surfdata_10x15_hist_16pfts_Irrig_CMIP6_simyr2000_c190214.nc", + "surfdata_0.125nldas2_hist_16pfts_Irrig_CMIP6_simyr2005_c190412.nc", + "surfdata_64x128_hist_16pfts_Irrig_CMIP6_simyr2000_c190214.nc", + "surfdata_0.9x1.25_hist_78pfts_CMIP6_simyr2000_c190214.nc", + "surfdata_1.9x2.5_hist_78pfts_CMIP6_simyr2000_c190304.nc", + "surfdata_0.125x0.125_hist_78pfts_CMIP6_simyr2005_c190624.nc", + "surfdata_10x15_hist_78pfts_CMIP6_simyr2000_c190214.nc", + "surfdata_4x5_hist_78pfts_CMIP6_simyr2000_c190214.nc", + "surfdata_1.9x2.5_hist_16pfts_Irrig_CMIP6_simyr1850_c190304.nc", + "surfdata_10x15_hist_16pfts_Irrig_CMIP6_simyr1850_c190214.nc", + "surfdata_4x5_hist_16pfts_Irrig_CMIP6_simyr1850_c190214.nc", + "surfdata_48x96_hist_78pfts_CMIP6_simyr1850_c190214.nc", + "surfdata_0.9x1.25_hist_78pfts_CMIP6_simyr1850_c190214.nc", + "surfdata_1.9x2.5_hist_78pfts_CMIP6_simyr1850_c190304.nc", + "surfdata_10x15_hist_78pfts_CMIP6_simyr1850_c190214.nc", + "surfdata_4x5_hist_78pfts_CMIP6_simyr1850_c190214.nc", + "surfdata_0.9x1.25_hist_16pfts_nourb_CMIP6_simyrPtVg_c181114.nc", + ] + expect_filenames = [ + "surfdata_tag_hist_16pfts_Irrig_CMIP6_simyr2000_c221031.nc", + "surfdata_tag_hist_16pfts_Irrig_CMIP6_simyr2000_c221031.nc", + "surfdata_tag_hist_16pfts_Irrig_CMIP6_simyr2000_c221031.nc", + "surfdata_tag_hist_16pfts_Irrig_CMIP6_simyr2000_c221031.nc", + "surfdata_tag_hist_16pfts_Irrig_CMIP6_simyr2000_c221031.nc", + "surfdata_tag_hist_16pfts_Irrig_CMIP6_simyr2000_c221031.nc", + "surfdata_tag_hist_16pfts_Irrig_CMIP6_simyr2000_c221031.nc", + "surfdata_tag_hist_16pfts_Irrig_CMIP6_simyr2000_c221031.nc", + "surfdata_tag_hist_16pfts_Irrig_CMIP6_simyr2005_c221031.nc", + "surfdata_tag_hist_16pfts_Irrig_CMIP6_simyr2000_c221031.nc", + "surfdata_tag_hist_78pfts_CMIP6_simyr2000_c221031.nc", + "surfdata_tag_hist_78pfts_CMIP6_simyr2000_c221031.nc", + "surfdata_tag_hist_78pfts_CMIP6_simyr2005_c221031.nc", + "surfdata_tag_hist_78pfts_CMIP6_simyr2000_c221031.nc", + "surfdata_tag_hist_78pfts_CMIP6_simyr2000_c221031.nc", + "surfdata_tag_hist_16pfts_Irrig_CMIP6_simyr1850_c221031.nc", + "surfdata_tag_hist_16pfts_Irrig_CMIP6_simyr1850_c221031.nc", + "surfdata_tag_hist_16pfts_Irrig_CMIP6_simyr1850_c221031.nc", + "surfdata_tag_hist_78pfts_CMIP6_simyr1850_c221031.nc", + "surfdata_tag_hist_78pfts_CMIP6_simyr1850_c221031.nc", + "surfdata_tag_hist_78pfts_CMIP6_simyr1850_c221031.nc", + "surfdata_tag_hist_78pfts_CMIP6_simyr1850_c221031.nc", + "surfdata_tag_hist_78pfts_CMIP6_simyr1850_c221031.nc", + "surfdata_tag_hist_16pfts_nourb_CMIP6_simyrPtVg_c221031.nc", + ] + for i, file_in in enumerate(files_in): + + with patch("ctsm.utils.date") as mock_date: + mock_date.today.side_effect = self._fake_today + + file_out = utils.add_tag_to_filename(file_in, "tag", replace_res=True) + + self.assertEqual(expect_filenames[i], file_out, "Expect filenames to be as expected") + if __name__ == "__main__": unit_testing.setup_for_tests() From 363154cb807f468a64b46a16b7f02215cfd478e3 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Wed, 9 Nov 2022 11:38:39 -0700 Subject: [PATCH 60/95] Add more complex surface dataset names and make sure works --- python/ctsm/test/test_unit_utils_add_tag.py | 7 +++++++ python/ctsm/utils.py | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/python/ctsm/test/test_unit_utils_add_tag.py b/python/ctsm/test/test_unit_utils_add_tag.py index 005fb787dc..d416a2909f 100755 --- a/python/ctsm/test/test_unit_utils_add_tag.py +++ b/python/ctsm/test/test_unit_utils_add_tag.py @@ -107,6 +107,8 @@ def testSurfReplaceListDomain(self): "surfdata_1.9x2.5_hist_78pfts_CMIP6_simyr1850_c190304.nc", "surfdata_10x15_hist_78pfts_CMIP6_simyr1850_c190214.nc", "surfdata_4x5_hist_78pfts_CMIP6_simyr1850_c190214.nc", + "surfdata_ne0np4.ARCTICGRIS.ne30x8_hist_78pfts_CMIP6_simyr2000_c200426.nc", + "surfdata_C96_hist_78pfts_CMIP6_simyr1850_c203171.nc", "surfdata_0.9x1.25_hist_16pfts_nourb_CMIP6_simyrPtVg_c181114.nc", ] expect_filenames = [ @@ -133,8 +135,13 @@ def testSurfReplaceListDomain(self): "surfdata_tag_hist_78pfts_CMIP6_simyr1850_c221031.nc", "surfdata_tag_hist_78pfts_CMIP6_simyr1850_c221031.nc", "surfdata_tag_hist_78pfts_CMIP6_simyr1850_c221031.nc", + "surfdata_tag_hist_78pfts_CMIP6_simyr2000_c221031.nc", + "surfdata_tag_hist_78pfts_CMIP6_simyr1850_c221031.nc", "surfdata_tag_hist_16pfts_nourb_CMIP6_simyrPtVg_c221031.nc", ] + self.assertEqual( + len(files_in), len(expect_filenames), "length of arrays does not match as expected" + ) for i, file_in in enumerate(files_in): with patch("ctsm.utils.date") as mock_date: diff --git a/python/ctsm/utils.py b/python/ctsm/utils.py index 7294f09d4c..9690782652 100644 --- a/python/ctsm/utils.py +++ b/python/ctsm/utils.py @@ -84,7 +84,7 @@ def add_tag_to_filename(filename, tag, replace_res=False): if not replace_res: fname_out = basename[:cend] + "_" + tag + "_c" + today_string + ".nc" else: - match = re.fullmatch(r"([a-z.]+)_([fv0-9x.crunldas]+)_(.+?)", basename[:cend]) + match = re.fullmatch(r"([a-z.]+)_([Cfvnenp0-9x.crunldasA-Z]+)_(.+?)", basename[:cend]) if match is not None: fname_out = ( match.group(1) + "_" + tag + "_" + match.group(3) + "_c" + today_string + ".nc" From 5a4f4151e59146ae6b867281b8719fe99e2329d9 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Wed, 9 Nov 2022 12:17:10 -0700 Subject: [PATCH 61/95] Make sure works for a 4-digit year date string as well --- python/ctsm/test/test_unit_utils_add_tag.py | 4 +++- python/ctsm/utils.py | 9 +++++++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/python/ctsm/test/test_unit_utils_add_tag.py b/python/ctsm/test/test_unit_utils_add_tag.py index d416a2909f..bef69f6154 100755 --- a/python/ctsm/test/test_unit_utils_add_tag.py +++ b/python/ctsm/test/test_unit_utils_add_tag.py @@ -108,7 +108,8 @@ def testSurfReplaceListDomain(self): "surfdata_10x15_hist_78pfts_CMIP6_simyr1850_c190214.nc", "surfdata_4x5_hist_78pfts_CMIP6_simyr1850_c190214.nc", "surfdata_ne0np4.ARCTICGRIS.ne30x8_hist_78pfts_CMIP6_simyr2000_c200426.nc", - "surfdata_C96_hist_78pfts_CMIP6_simyr1850_c203171.nc", + "surfdata_C96_hist_78pfts_CMIP6_simyr1850_c200317.nc", + "surfdata_C96_hist_78pfts_CMIP6_simyr1850_c20221108.nc", "surfdata_0.9x1.25_hist_16pfts_nourb_CMIP6_simyrPtVg_c181114.nc", ] expect_filenames = [ @@ -137,6 +138,7 @@ def testSurfReplaceListDomain(self): "surfdata_tag_hist_78pfts_CMIP6_simyr1850_c221031.nc", "surfdata_tag_hist_78pfts_CMIP6_simyr2000_c221031.nc", "surfdata_tag_hist_78pfts_CMIP6_simyr1850_c221031.nc", + "surfdata_tag_hist_78pfts_CMIP6_simyr1850_c221031.nc", "surfdata_tag_hist_16pfts_nourb_CMIP6_simyrPtVg_c221031.nc", ] self.assertEqual( diff --git a/python/ctsm/utils.py b/python/ctsm/utils.py index 9690782652..8b2043593e 100644 --- a/python/ctsm/utils.py +++ b/python/ctsm/utils.py @@ -77,8 +77,13 @@ def add_tag_to_filename(filename, tag, replace_res=False): if basename[cend] == "c": cend = cend - 1 if (basename[cend] != ".") and (basename[cend] != "_"): - err_msg = "Trouble figuring out where to add tag to filename: " + filename - abort(err_msg) + # Check if date stirng at end includes a 4 digit year + cend = -12 + if basename[cend] == "c": + cend = cend - 1 + if (basename[cend] != ".") and (basename[cend] != "_"): + err_msg = "Trouble figuring out where to add tag to filename: " + filename + abort(err_msg) today = date.today() today_string = today.strftime("%y%m%d") if not replace_res: From 65e0c9a60b3b3b2b9762ad9e99cbfe27d7cb4dc3 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Wed, 9 Nov 2022 12:32:48 -0700 Subject: [PATCH 62/95] Add note that 4-digit years are allowed and test for it --- python/ctsm/utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/python/ctsm/utils.py b/python/ctsm/utils.py index 8b2043593e..bf48762a43 100644 --- a/python/ctsm/utils.py +++ b/python/ctsm/utils.py @@ -48,6 +48,7 @@ def add_tag_to_filename(filename, tag, replace_res=False): """ Add a tag and replace timetag of a filename Expects file to end with [._]cYYMMDD.nc or [._]YYMMDD.nc + or with 4-digit years YYYYMMDD. Add the tag to just before that ending part and change the ending part to the current time tag. From 96209ceb39ddae054dbcea45120adfc1c1026d4b Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Wed, 9 Nov 2022 12:40:30 -0700 Subject: [PATCH 63/95] Replace the resolution in the surface and landuse.timeseries filenames --- python/ctsm/site_and_regional/regional_case.py | 4 ++-- python/ctsm/site_and_regional/single_point_case.py | 4 ++-- python/ctsm/utils.py | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/python/ctsm/site_and_regional/regional_case.py b/python/ctsm/site_and_regional/regional_case.py index 1dc6a522cc..8d5d81e014 100644 --- a/python/ctsm/site_and_regional/regional_case.py +++ b/python/ctsm/site_and_regional/regional_case.py @@ -154,7 +154,7 @@ def create_surfdata_at_reg(self, indir, file, user_mods_dir): # specify files fsurf_in = os.path.join(indir, file) - fsurf_out = add_tag_to_filename(fsurf_in, self.tag) + fsurf_out = add_tag_to_filename(fsurf_in, self.tag, replace_res=True) logger.info("fsurf_in: %s", fsurf_in) logger.info("fsurf_out: %s", os.path.join(self.out_dir, fsurf_out)) @@ -194,7 +194,7 @@ def create_landuse_at_reg(self, indir, file, user_mods_dir): # specify files fluse_in = os.path.join(indir, file) - fluse_out = add_tag_to_filename(fluse_in, self.tag) + fluse_out = add_tag_to_filename(fluse_in, self.tag, replace_res=True) logger.info("fluse_in: %s", fluse_in) logger.info("fluse_out: %s", os.path.join(self.out_dir, fluse_out)) diff --git a/python/ctsm/site_and_regional/single_point_case.py b/python/ctsm/site_and_regional/single_point_case.py index 31ab158706..05b918d316 100644 --- a/python/ctsm/site_and_regional/single_point_case.py +++ b/python/ctsm/site_and_regional/single_point_case.py @@ -342,7 +342,7 @@ def create_landuse_at_point(self, indir, file, user_mods_dir): # specify files fluse_in = os.path.join(indir, file) - fluse_out = add_tag_to_filename(fluse_in, self.tag) + fluse_out = add_tag_to_filename(fluse_in, self.tag, replace_res=True) logger.info("fluse_in: %s", fluse_in) logger.info("fluse_out: %s", os.path.join(self.out_dir, fluse_out)) @@ -464,7 +464,7 @@ def create_surfdata_at_point(self, indir, file, user_mods_dir): # specify file fsurf_in = os.path.join(indir, file) - fsurf_out = add_tag_to_filename(fsurf_in, self.tag) + fsurf_out = add_tag_to_filename(fsurf_in, self.tag, replace_res=True) logger.info("fsurf_in: %s", fsurf_in) logger.info("fsurf_out: %s", os.path.join(self.out_dir, fsurf_out)) diff --git a/python/ctsm/utils.py b/python/ctsm/utils.py index bf48762a43..42444e32c5 100644 --- a/python/ctsm/utils.py +++ b/python/ctsm/utils.py @@ -83,8 +83,8 @@ def add_tag_to_filename(filename, tag, replace_res=False): if basename[cend] == "c": cend = cend - 1 if (basename[cend] != ".") and (basename[cend] != "_"): - err_msg = "Trouble figuring out where to add tag to filename: " + filename - abort(err_msg) + err_msg = "Trouble figuring out where to add tag to filename: " + filename + abort(err_msg) today = date.today() today_string = today.strftime("%y%m%d") if not replace_res: From a5dc3f40b650a47a5b06b2542a5546fbef19785d Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Wed, 9 Nov 2022 12:53:35 -0700 Subject: [PATCH 64/95] Change NEON surface dataset name --- tools/site_and_regional/modify_singlept_site_neon.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tools/site_and_regional/modify_singlept_site_neon.py b/tools/site_and_regional/modify_singlept_site_neon.py index 336366a49b..35d8e3f4e6 100755 --- a/tools/site_and_regional/modify_singlept_site_neon.py +++ b/tools/site_and_regional/modify_singlept_site_neon.py @@ -250,8 +250,7 @@ def find_surffile(surf_dir, site_name): surf_file (str): name of the surface dataset file """ - # sf_name = "surfdata_hist_16pfts_Irrig_CMIP6_simyr2000_"+site_name+"*.nc" - sf_name = "surfdata_*hist_78pfts_CMIP6_simyr2000_" + site_name + "*.nc" + sf_name = "surfdata_1x1_NEON_"+site_name+"*hist_78pfts_CMIP6_simyr2000_*.nc" print (os.path.join(surf_dir , sf_name)) surf_file = sorted(glob.glob(os.path.join(surf_dir , sf_name))) From cc3770c211c192885e16f838faba96122cdc7ad5 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Wed, 9 Nov 2022 13:26:56 -0700 Subject: [PATCH 65/95] Add modules needed for neon_surf_wrapper --- python/conda_env_ctsm_py.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/python/conda_env_ctsm_py.txt b/python/conda_env_ctsm_py.txt index 8c110e8f3f..d757ae1782 100644 --- a/python/conda_env_ctsm_py.txt +++ b/python/conda_env_ctsm_py.txt @@ -11,6 +11,8 @@ # conda activate ctsm_py # Do this anytime you want to run a CTSM python script # python=3.7.9 +pandas +tqdm scipy netcdf4 requests From beb002ee53232c6d1484e6a12b6e4aab5a378d4d Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Wed, 9 Nov 2022 13:27:20 -0700 Subject: [PATCH 66/95] Update surface dataset names for NEON --- tools/site_and_regional/neon_surf_wrapper.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/site_and_regional/neon_surf_wrapper.py b/tools/site_and_regional/neon_surf_wrapper.py index 2f8a00500c..edba722f39 100755 --- a/tools/site_and_regional/neon_surf_wrapper.py +++ b/tools/site_and_regional/neon_surf_wrapper.py @@ -96,8 +96,9 @@ def main(): lon = row['Lon'] site = row['Site'] pft = row['pft'] + clmsite = "1x1_NEON_"+site print ("Now processing site :", site) - command = ['./subset_data','point','--lat',str(lat),'--lon',str(lon),'--site',site,'--dompft',str(pft),'--crop', + command = ['./subset_data','point','--lat',str(lat),'--lon',str(lon),'--site',clmsite,'--dompft',str(pft),'--crop', '--create-surface','--uniform-snowpack','--cap-saturation','--verbose','--overwrite'] execute(command) From e1459d40aa5cd39730fcf944998da74ba6c7be82 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Wed, 9 Nov 2022 20:26:09 -0700 Subject: [PATCH 67/95] Add python modules needed for neon_surf_wrapper to all conda requirements files --- python/conda_env_ctsm_py_cgd.txt | 2 ++ python/conda_env_ctsm_py_latest.txt | 2 ++ 2 files changed, 4 insertions(+) diff --git a/python/conda_env_ctsm_py_cgd.txt b/python/conda_env_ctsm_py_cgd.txt index 45025506a8..e7ee4af8ab 100644 --- a/python/conda_env_ctsm_py_cgd.txt +++ b/python/conda_env_ctsm_py_cgd.txt @@ -13,6 +13,8 @@ # conda activate ctsm_py # Do this anytime you want to run a CTSM python script # python=3.7.0 # The python version MUST match the python version available on CGD systems through modules exactly +pandas +tqdm scipy netcdf4 requests diff --git a/python/conda_env_ctsm_py_latest.txt b/python/conda_env_ctsm_py_latest.txt index a7a28c9fc3..2dc2ed518d 100644 --- a/python/conda_env_ctsm_py_latest.txt +++ b/python/conda_env_ctsm_py_latest.txt @@ -1,5 +1,7 @@ # This is a test python environment intended to represent the latest environment that can be built python>=3.9.13,<3.10 # Moving to 3.10 runs into conflicts +pandas>=1.5.1 +tqdm>=4.64.1 scipy netcdf4 requests From 134ec4e1e10a7ebc6abaf3c293eb8ee0705333ef Mon Sep 17 00:00:00 2001 From: wwieder Date: Fri, 11 Nov 2022 13:50:33 -0700 Subject: [PATCH 68/95] updated path to surface datasets --- cime_config/usermods_dirs/NEON/defaults/user_nl_clm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cime_config/usermods_dirs/NEON/defaults/user_nl_clm b/cime_config/usermods_dirs/NEON/defaults/user_nl_clm index 7dd1ce7f32..29e50431ce 100644 --- a/cime_config/usermods_dirs/NEON/defaults/user_nl_clm +++ b/cime_config/usermods_dirs/NEON/defaults/user_nl_clm @@ -19,7 +19,7 @@ !---------------------------------------------------------------------------------- flanduse_timeseries = ' ' ! This isn't needed for a non transient case, but will be once we start using transient compsets -fsurdat = "$DIN_LOC_ROOT/lnd/clm2/surfdata_map/NEON/surfdata_0.9x1.25_hist_78pfts_CMIP6_simyr2000_${NEONSITE}_c221014.nc" +fsurdat = "$DIN_LOC_ROOT/lnd/clm2/surfdata_map/NEON/surfdata_1x1_NEON_${NEONSITE}_hist_78pfts_CMIP6_simyr2000_c221111.nc" stream_fldfilename_lightng = '$DIN_LOC_ROOT/atm/datm7/NASA_LIS/clmforc.Li_2016_climo1995-2013.360x720.lnfm_Total_NEONarea_c210625.nc' stream_meshfile_lightng = '$DIN_LOC_ROOT/atm/datm7/NASA_LIS/ESMF_MESH.Li_2016.360x720.NEONarea_cdf5_c221104.nc' From 74b9cd3c3a4c59511e2991caf877fff27243c961 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Fri, 11 Nov 2022 15:55:52 -0700 Subject: [PATCH 69/95] Replace notes about using ncar_pylib with running py_env_create --- python/ctsm/subset_data.py | 8 ++++---- tools/site_and_regional/modify_singlept_site_neon.py | 9 +++------ tools/site_and_regional/neon_surf_wrapper.py | 8 ++++---- tools/site_and_regional/run_neon.py | 12 ++++-------- tools/site_and_regional/subset_data | 9 +++------ 5 files changed, 18 insertions(+), 28 deletions(-) diff --git a/python/ctsm/subset_data.py b/python/ctsm/subset_data.py index 7a33d9c2fa..4832e98f7f 100644 --- a/python/ctsm/subset_data.py +++ b/python/ctsm/subset_data.py @@ -2,10 +2,10 @@ |------------------------------------------------------------------| |--------------------- Instructions -----------------------------| |------------------------------------------------------------------| -Instructions for running on Cheyenne/Casper: -load the following into your local environment - module load python - ncar_pylib +Instructions for running using conda python environments: + +../../py_env_create +conda activate ctsm_py ------------------------------------------------------------------- To see the available options for single point or regional cases: ./subset_data.py --help diff --git a/tools/site_and_regional/modify_singlept_site_neon.py b/tools/site_and_regional/modify_singlept_site_neon.py index 35d8e3f4e6..334ab7d6aa 100755 --- a/tools/site_and_regional/modify_singlept_site_neon.py +++ b/tools/site_and_regional/modify_singlept_site_neon.py @@ -16,14 +16,11 @@ - Modify surface dataset with downloaded data. ------------------------------------------------------------------- -Instructions for running on Cheyenne/Casper: +Instructions for running using conda python environments: -load the following into your local environment - module load python - ncar_pylib +../../py_env_create +conda activate ctsm_py -To remove NPL from your environment on Cheyenne/Casper: - deactivate ------------------------------------------------------------------- To see the available options: ./modify_singlept_site_neon.py --help diff --git a/tools/site_and_regional/neon_surf_wrapper.py b/tools/site_and_regional/neon_surf_wrapper.py index edba722f39..e02ee2bd53 100755 --- a/tools/site_and_regional/neon_surf_wrapper.py +++ b/tools/site_and_regional/neon_surf_wrapper.py @@ -11,10 +11,10 @@ based on the downloaded neon data. (i.e. modify_singlept_site_neon.py) -Instructions for running on Cheyenne/Casper: -load the following into your local environment - module load python - ncar_pylib +Instructions for running using conda python environments: + +../../py_env_create +conda activate ctsm_py """ # TODO diff --git a/tools/site_and_regional/run_neon.py b/tools/site_and_regional/run_neon.py index 2879a9cf9e..b8bc79a845 100755 --- a/tools/site_and_regional/run_neon.py +++ b/tools/site_and_regional/run_neon.py @@ -26,14 +26,10 @@ 4) Build and submit the case. ------------------------------------------------------------------- -Instructions for running on Cheyenne/Casper: - -load the following into your local environment - module load python - ncar_pylib - -To remove NPL from your environment on Cheyenne/Casper: - deactivate +Instructions for running using conda python environments: + +../../py_env_create +conda activate ctsm_py ------------------------------------------------------------------- To see the available options: diff --git a/tools/site_and_regional/subset_data b/tools/site_and_regional/subset_data index bb582b21f8..162dcf1d4e 100755 --- a/tools/site_and_regional/subset_data +++ b/tools/site_and_regional/subset_data @@ -18,13 +18,10 @@ To see all available options for single-point/regional subsetting: ./subset_data --help ---------------------------------------------------------------- -Instructions for running on Cheyenne/Casper: - load the following into your local environment - module load python - ncar_pylib +Instructions for running using conda python environments: -To remove from your environment on Cheyenne/Casper: - deactivate +../../py_env_create +conda activate ctsm_py """ import os From 4588f0c8e13d78c1ed8f5848f1046d1300a44d2a Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Fri, 11 Nov 2022 16:01:35 -0700 Subject: [PATCH 70/95] Remove the toolchain files, this work was moved over to the ctsm5.2 branch, which changed the process for how surface datasets would be created --- python/ctsm/gen_mksurf_namelist.py | 377 ---------------------------- tools/toolchain/gen_mksurf_namelist | 77 ------ 2 files changed, 454 deletions(-) delete mode 100644 python/ctsm/gen_mksurf_namelist.py delete mode 100755 tools/toolchain/gen_mksurf_namelist diff --git a/python/ctsm/gen_mksurf_namelist.py b/python/ctsm/gen_mksurf_namelist.py deleted file mode 100644 index 735ae0493f..0000000000 --- a/python/ctsm/gen_mksurf_namelist.py +++ /dev/null @@ -1,377 +0,0 @@ -# 2020-11-08 Negin Sobhani - -""" -|------------------------------------------------------------------| -|--------------------- Instructions -----------------------------| -|------------------------------------------------------------------| -This Python script is part of the simplified toolchain for creating -the surface dataset for ctsm cases. -This script should be used as the first step of the new toolchain. -It will automatically create namelist (control file) that is -needed for creating surface dataset and requisite intermediate files for -running CTSM cases. -For transient cases, it will also create a txt file that includes the -landuse files for every year. - -------------------------------------------------------------------- -Instructions for running on Cheyenne/Casper: - -load the following into your local environment: - - module load python - ncar_pylib -------------------------------------------------------------------- -To see the available options: - ./gen_mksurf_namelist.py --help - -To run the script: - ./gen_mksurf_namelist.py - -To remove NPL(ncar_pylib) from your environment on Cheyenne/Casper: - deactivate -------------------------------------------------------------------- -""" - -# TODO (NS) - -# -[x] Add default values in the help page. -# -[x] Add info for help page note for end_year -- by default is start_year -# -[x] Possibly remove year --years and range options -# Currently comment them out. - -# -[x] maybe a verbose option and removing debug -# -[x] --debug mode is not working... - -# -[ ] add error check for hi-res and years if they are 1850 and 2005. -# -[ ] hirespft data only for 2005? add error-check - -# -[x] different path for each range of years for transient cases. -# default should be picked based on the year. 1850 - 2015 --> -# /glade/p/cesm/cseg/inputdata/lnd/clm2/rawdata/ -# pftcftdynharv.0.25x0.25.LUH2.histsimyr1850-2015.c170629/ -# 850-1850 --> -# pftcftdynharv.0.25x0.25.LUH2.histsimyr0850-1849.c171012 - - -# Import libraries -from __future__ import print_function - -import os -import sys -import logging -import argparse - -# -- import local classes for this script -from ctsm.toolchain.ctsm_case import CtsmCase - -# -- import ctsm logging flags -from ctsm.ctsm_logging import ( - setup_logging_pre_config, - add_logging_args, - process_logging_args, -) - -logger = logging.getLogger(__name__) - -## valid options for resolution and SSP scenarios: -VALID_OPTS = { - "res": [ - "512x1024", - "360x720cru", - "128x256", - "64x128", - "48x96", - "94x192", - "0.23x0.31", - "0.47x0.63", - "0.9x1.25", - "1.9x2.5", - "2.5x3.33", - "4x5", - "10x15", - "0.125nldas2", - "5x5_amazon", - "1x1_camdenNJ", - "1x1_vancouverCAN", - "1x1_mexicocityMEX", - "1x1_asphaltjungleNJ", - "1x1_brazil,1x1_urbanc_alpha", - "1x1_numaIA,1x1_smallvilleIA", - "0.1x0.1", - "0.25x0.25", - "0.5x0.5", - "3x3min", - "5x5min", - "10x10min", - "0.33x0.33", - "0.125x0.125", - "ne4np4,ne16np4", - "ne30np4.pg2", - "ne30np4.pg3", - "ne30np4", - "ne60np4", - "ne120np4", - ], - "ssp_rcp": [ - "hist", - "SSP1-2.6", - "SSP3-7.0", - "SSP5-3.4", - "SSP2-4.5", - "SSP1-1.9", - "SSP4-3.4", - "SSP4-6.0", - "SSP5-8.5", - ], -} - - -def get_parser(): - """ - Get parser object for this script. - """ - parser = argparse.ArgumentParser( - description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter - ) - - parser.print_usage = parser.print_help - - parser.add_argument( - "--sy", - "--start-year", - help="Simulation start year. [default: %(default)s] ", - action="store", - dest="start_year", - required=False, - type=start_year_type, - default=2000, - ) - parser.add_argument( - "--ey", - "--end-year", - help="Simulation end year. [default: start_year] ", - action="store", - dest="end_year", - required=False, - type=int, - ) - parser.add_argument( - "--glc-nec", - help=""" - Number of glacier elevation classes to use. - [default: %(default)s] - """, - action="store", - dest="glc_nec", - type=glc_nec_type, - default="10", - ) - parser.add_argument( - "--rundir", - help=""" - Directory to run in. - [default: %(default)s] - """, - action="store", - dest="run_dir", - required=False, - default=os.getcwd(), - ) - parser.add_argument( - "--ssp-rcp", - help=""" - Shared Socioeconomic Pathway and Representative - Concentration Pathway Scenario name(s). - [default: %(default)s] - """, - action="store", - dest="ssp_rcp", - required=False, - choices=VALID_OPTS["ssp_rcp"], - default="hist", - ) - - ############################################## - # In mksurfdata.pl these options are -l --dinlc - # But the group decided --raw_dir is more descriptive. - # If everyone agrees, the commented out line should be removed. - # parser.add_argument('-l','--dinlc', #--raw-dir or --rawdata-dir - - parser.add_argument( - "--raw-dir", - "--rawdata-dir", - help=""" - /path/of/root/of/input/data', - [default: %(default)s] - """, - action="store", - dest="input_path", - default="/glade/p/cesm/cseg/inputdata/lnd/clm2/rawdata/", - ) - parser.add_argument( - "--vic", - help=""" - Flag for adding the fields required for the VIC model. - """, - action="store_true", - dest="vic_flag", - default=False, - ) - parser.add_argument( - "--glc", - help=""" - Flag for adding the optional 3D glacier fields for verification of the glacier model. - """, - action="store_true", - dest="glc_flag", - default=False, - ) - parser.add_argument( - "--hirespft", - help=""" - If you want to use the high-resolution pft dataset rather - than the default lower resolution dataset. - (Low resolution is at quarter-degree, high resolution at 3-minute) - [Note: hires only available for 1850 and 2005.] - """, - action="store_true", - dest="hres_flag", - default=False, - ) - parser.add_argument( - "--nocrop", - help=""" - Create datasets with the extensive list of prognostic crop types. - """, - action="store_false", - dest="crop_flag", - default=True, - ) - parser.add_argument( - "-f", - "--fast", - help="Toggle fast mode which does not user the large mapping file", - action="store_true", - dest="fast_flag", - default=False, - ) - parser.add_argument( - "-r", - "--res", - help=""" - Resolution is the supported resolution(s) to use for files. - [default: %(default)s] - """, - action="store", - dest="res", - choices=VALID_OPTS["res"], - required=False, - default="4x5", - ) - return parser - - -# -- types for this parser - - -def glc_nec_type(glc): - """ - Function for defining acceptable glc_nec input. - - Args: - x (str) : glc_nec value from command line args. - - Raises: - Error if value of glc_nec is not in the range - of 1-99. - - Returns: - x (int) : Acceptable glc_nec value. - """ - glc = int(glc) - if (glc <= 0) or (glc >= 100): - raise argparse.ArgumentTypeError("ERROR: glc_nec must be between 1 and 99.") - return glc.__str__() - - -def start_year_type(year): - """ - Function for defining acceptable start_year input. - - Args: - year (str) : start_year string from command line args. - - Raises: - Error if value of start_year is not in the range - of 850-2105. - - Returns: - year (int) : Acceptable start_year value. - """ - year = int(year) - if (year < 850) or (year > 2105): - raise argparse.ArgumentTypeError( - "ERROR: Simulation start year should be between 850 and 2105." - ) - return year - - -def main(): - """ - Main function for gen_mksurf_namelist. - """ - # -- add logging flags from ctsm_logging - setup_logging_pre_config() - parser = get_parser() - add_logging_args(parser) - - args = parser.parse_args() - process_logging_args(args) - - res = args.res - glc_nec = args.glc_nec - input_path = args.input_path - ssp_rcp = args.ssp_rcp - crop_flag = args.crop_flag - vic_flag = args.vic_flag - glc_flag = args.glc_flag - hres_flag = args.hres_flag - - start_year = args.start_year - end_year = args.end_year - - # -- determine end_year if not given as an argument: - if not end_year: - end_year = start_year - - # -- check if the input path exist - if not os.path.exists(input_path): - sys.exit( - "ERROR: \n" - + "\t raw_dir does not exist on this machine. \n" - + "\t Please point to the correct raw_dir using --raw-dir" - + "or --rawdata-dir flags." - ) - - ctsm_case = CtsmCase( - res, - glc_nec, - ssp_rcp, - crop_flag, - input_path, - vic_flag, - glc_flag, - start_year, - end_year, - hres_flag, - ) - - logger.info("--------------------------") - logger.info(" ctsm case : %s", ctsm_case) - logger.info("--------------------------") - - ctsm_case.create_namelist_file() - - -if __name__ == "__main__": - main() diff --git a/tools/toolchain/gen_mksurf_namelist b/tools/toolchain/gen_mksurf_namelist deleted file mode 100755 index ecd225dd19..0000000000 --- a/tools/toolchain/gen_mksurf_namelist +++ /dev/null @@ -1,77 +0,0 @@ -#!/usr/bin/env python - -# 2020-11-08 Negin Sobhani - -""" -|------------------------------------------------------------------| -|--------------------- Instructions -----------------------------| -|------------------------------------------------------------------| -This is a just top-level skeleton script that calls -gen_mksurf_namelist.py. -The original code (./gen_mksurf_namelist.py) is located under -python/ctsm folder. - -This Python script is part of the simplified toolchain for creating -the surface dataset for ctsm cases. -This script should be used as the first step of the new toolchain. -It will automatically create namelist (control file) that is -needed for creating surface dataset and requisite intermediate files for -running CTSM cases. -For transient cases, it will also create a txt file that includes the -landuse files for every year. - -------------------------------------------------------------------- -Instructions for running on Cheyenne/Casper: - -load the following into your local environment: - - module load python - ncar_pylib -------------------------------------------------------------------- -To see the available options: - ./gen_mksurf_namelist.py --help - -To run the script: - ./gen_mksurf_namelist.py - -To remove NPL(ncar_pylib) from your environment on Cheyenne/Casper: - deactivate -------------------------------------------------------------------- -""" - -#TODO (NS) - -# -[x] Add default values in the help page. -# -[x] Add info for help page note for end_year -- by default is start_year -# -[ ] Possibly remove year --years and range options -# Currently comment them out. - -# -[ ] maybe a verbose option and removing debug -# -[x] --debug mode is not working... - -# -[ ] add error check for hi-res and years if they are 1850 and 2005. - -# -[x] different path for each range of years for transient cases. -# default should be picked based on the year. 1850 - 2015 --> -# /glade/p/cesm/cseg/inputdata/lnd/clm2/rawdata/pftcftdynharv.0.25x0.25.LUH2.histsimyr1850-2015.c170629/ -# 850-1850 --> -# pftcftdynharv.0.25x0.25.LUH2.histsimyr0850-1849.c171012 - -# -[ ] hirespft data only for 2005? - -# -- Import libraries -import os -import sys - -# -- add python/ctsm to path -_CTSM_PYTHON = os.path.join( - os.path.dirname(os.path.realpath(__file__)), os.pardir, os.pardir, "python" - ) -sys.path.insert(1, _CTSM_PYTHON) - -from ctsm.gen_mksurf_namelist import main - -if __name__ == "__main__": - main() - - From 5bf84996213c0ed083fc65d51fbe65eeb7bc4fca Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Fri, 11 Nov 2022 17:30:30 -0700 Subject: [PATCH 71/95] Add note that start option is not functional and remove the code about it, but add a comment for it --- tools/site_and_regional/run_neon.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/tools/site_and_regional/run_neon.py b/tools/site_and_regional/run_neon.py index b8bc79a845..50f0640d03 100755 --- a/tools/site_and_regional/run_neon.py +++ b/tools/site_and_regional/run_neon.py @@ -209,6 +209,7 @@ def get_parser(args, description, valid_neon_sites): help=""" Start date for running CTSM simulation in ISO format. [default: %(default)s] + (currently non-functional) """, action="store", dest="start_date", @@ -622,11 +623,7 @@ def set_ref_case(self, case): case.set_value("RUN_REFDATE", refdate) if case_root.endswith(".postad"): case.set_value("RUN_STARTDATE", refdate) - #else: - #case.set_value( - # "RUN_STARTDATE", - # "{yr:04d}-{mo:02d}-01".format(yr=self.start_year, mo=self.start_month), - #) + # NOTE: if start options are set, RUN_STARTDATE should be modified here return True def modify_user_nl(self, case_root, run_type, rundir): From f40d8fa566b8baf1ed0e74a566fb5d40d7ac4d33 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Sun, 13 Nov 2022 19:39:53 -0700 Subject: [PATCH 72/95] Update mosart to bring in direct_to_outlet fixes --- Externals.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Externals.cfg b/Externals.cfg index 491405a33b..2df854055e 100644 --- a/Externals.cfg +++ b/Externals.cfg @@ -23,7 +23,7 @@ required = True local_path = components/mosart protocol = git repo_url = https://github.com/ESCOMP/MOSART -tag = mosart1_0_45 +tag = mosart1_0_47 required = True [mizuRoute] From 4e508585c025bfda089b4016f79c36a1d6f6ad5e Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Sun, 13 Nov 2022 20:14:00 -0700 Subject: [PATCH 73/95] Add a direction for the check_for_nans so that the direction the fields are going will be written out fixing #1896 --- src/cpl/lilac/lnd_import_export.F90 | 4 ++-- src/cpl/mct/lnd_import_export.F90 | 4 ++-- src/cpl/nuopc/lnd_import_export.F90 | 8 ++++---- src/cpl/utils/lnd_import_export_utils.F90 | 5 +++-- 4 files changed, 11 insertions(+), 10 deletions(-) diff --git a/src/cpl/lilac/lnd_import_export.F90 b/src/cpl/lilac/lnd_import_export.F90 index 2da277dcee..281666c3e7 100644 --- a/src/cpl/lilac/lnd_import_export.F90 +++ b/src/cpl/lilac/lnd_import_export.F90 @@ -571,7 +571,7 @@ subroutine state_getimport(state, fb, fldname, bounds, output, ungridded_index, end if ! Check for nans - call check_for_nans(output, trim(fldname), bounds%begg) + call check_for_nans(output, trim(fldname), bounds%begg, "output") end subroutine state_getimport @@ -657,7 +657,7 @@ subroutine state_setexport(state, fb, fldname, bounds, input, minus, ungridded_i end if ! check for nans - call check_for_nans(input, trim(fldname), bounds%begg) + call check_for_nans(input, trim(fldname), bounds%begg, "input") end subroutine state_setexport diff --git a/src/cpl/mct/lnd_import_export.F90 b/src/cpl/mct/lnd_import_export.F90 index 2c84d2e471..3f7e67af68 100644 --- a/src/cpl/mct/lnd_import_export.F90 +++ b/src/cpl/mct/lnd_import_export.F90 @@ -136,7 +136,7 @@ subroutine lnd_import( bounds, x2l, glc_present, atm2lnd_inst, glc2lnd_inst, wat ! Check for nans from coupler !-------------------------- - call check_for_nans(x2l(:,i), fname, begg) + call check_for_nans(x2l(:,i), fname, begg, "x2l") end do @@ -344,7 +344,7 @@ subroutine lnd_export( bounds, waterlnd2atmbulk_inst, lnd2atm_inst, lnd2glc_inst ! Check for nans to coupler !-------------------------- - call check_for_nans(l2x(:,i), fname, begg) + call check_for_nans(l2x(:,i), fname, begg, "l2x") end do diff --git a/src/cpl/nuopc/lnd_import_export.F90 b/src/cpl/nuopc/lnd_import_export.F90 index 0e7a5e2eef..340009ccb3 100644 --- a/src/cpl/nuopc/lnd_import_export.F90 +++ b/src/cpl/nuopc/lnd_import_export.F90 @@ -1080,7 +1080,7 @@ subroutine state_getimport_1d(state, fldname, ctsmdata, rc) do g = 1,size(ctsmdata) ctsmdata(g) = fldptr1d(g) end do - call check_for_nans(ctsmdata, trim(fldname), 1) + call check_for_nans(ctsmdata, trim(fldname), 1, "import_1D") end subroutine state_getimport_1d @@ -1114,7 +1114,7 @@ subroutine state_getimport_2d(state, fldname, ctsmdata, rc) do g = 1,size(ctsmdata,dim=1) ctsmdata(g,n) = fldptr2d(n,g) end do - call check_for_nans(ctsmdata(:,n), trim(fldname)//trim(cnum), 1) + call check_for_nans(ctsmdata(:,n), trim(fldname)//trim(cnum), 1, "import_2D") end do end subroutine state_getimport_2d @@ -1167,7 +1167,7 @@ subroutine state_setexport_1d(state, fldname, ctsmdata, init_spval, minus, rc) fldptr1d(g) = ctsmdata(g) end do end if - call check_for_nans(ctsmdata, trim(fldname), 1) + call check_for_nans(ctsmdata, trim(fldname), 1, "export_1D") end subroutine state_setexport_1d @@ -1222,7 +1222,7 @@ subroutine state_setexport_2d(state, fldname, ctsmdata, init_spval, minus, rc) fldptr2d(n,g) = ctsmdata(g,n) end do end if - call check_for_nans(ctsmdata(:,n), trim(fldname)//trim(cnum), 1) + call check_for_nans(ctsmdata(:,n), trim(fldname)//trim(cnum), 1, "export_2D") end do end subroutine state_setexport_2d diff --git a/src/cpl/utils/lnd_import_export_utils.F90 b/src/cpl/utils/lnd_import_export_utils.F90 index 032cb19b6f..4b7941da5b 100644 --- a/src/cpl/utils/lnd_import_export_utils.F90 +++ b/src/cpl/utils/lnd_import_export_utils.F90 @@ -140,12 +140,13 @@ end subroutine check_for_errors !============================================================================= - subroutine check_for_nans(array, fname, begg) + subroutine check_for_nans(array, fname, begg, direction) ! input/output variables real(r8) , intent(in) :: array(:) character(len=*) , intent(in) :: fname integer , intent(in) :: begg + character(len=*) , intent(in) :: direction ! local variables integer :: i @@ -161,7 +162,7 @@ subroutine check_for_nans(array, fname, begg) write(iulog,*) "NaN found in field ", trim(fname), ' at gridcell index ',begg+i-1 end if end do - call shr_sys_abort(' ERROR: One or more of the output from CLM to the coupler are NaN ' ) + call shr_sys_abort(' ERROR: One or more of the CTSM cap '//direction//' fields are NaN ' ) end if end subroutine check_for_nans From 5769dd1d4e0e12a19c4bb588babe1270d3abe17d Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Mon, 14 Nov 2022 14:22:16 -0700 Subject: [PATCH 74/95] Update change files --- doc/ChangeLog | 127 ++++++++++++++++++++++++++++++++++++++++++++++++++ doc/ChangeSum | 1 + 2 files changed, 128 insertions(+) diff --git a/doc/ChangeLog b/doc/ChangeLog index 68aea97f7e..b58f7f92b9 100644 --- a/doc/ChangeLog +++ b/doc/ChangeLog @@ -1,4 +1,131 @@ =============================================================== +Tag name: ctsm5.1.dev114 +Originator(s): erik (Erik Kluzek,UCAR/TSS,303-497-1326)/@wwieder/@olyson/@ka7eh +Date: Mon Nov 14 14:22:05 MST 2022 +One-line Summary: Some NEON updates fixing AG sites, update MOSART, small fixes + +Purpose and description of changes +---------------------------------- + +Minor changes to python scripts and usermod_dirs for NEON cases. Also update the lightning mesh file so that it goes with the +smaller lightning file. Have NEON use new use-cases for 2018 and 2018-PD conditions for CLM. Have NEON +Agricultural sites run with prognostic crop. Simple fix for warning about NaN's in import/export data from/to coupler. + +Also update MOSART with fixed for direct_to_outlet option. + +Add error checking in ParitionWoodFluxes. Fix value of albgrd_col in SurfaceAlbefdoType.F90. +Previously, the wrong value (albgri_col) was being set in InitHistory. + +Significant changes to scientifically-supported configurations +-------------------------------------------------------------- + +Does this tag change answers significantly for any of the following physics configurations? +(Details of any changes will be given in the "Answer changes" section below.) + + [Put an [X] in the box for any configuration with significant answer changes.] + +[ ] clm5_1 + +[ ] clm5_0 + +[ ] ctsm5_0-nwp + +[ ] clm4_5 + + +Bugs fixed or introduced +------------------------ + +CTSM issues fixed (include CTSM Issue #): + Fixes #1871 -- Error in NEON surface datasets for AG sites + Fixes #1876 -- NEON data in container + Fixes #1889 -- NEON AG sites are running without prognostic crop + Fixes #1363 -- 2018_control and 2018-PD_transient use-cases for NEON + Fixes #1896 -- Improve misleading error message in check_for_nans + Fixes #1263 -- Fix partitionWood fluxes + Fixes #1788 -- Fix albgrd_col + +Notes of particular relevance for users +--------------------------------------- + +Caveats for users (e.g., need to interpolate initial conditions): + NEON users: use neon_gcs_upload now. Filenames for NEON surface + datasets are changed. Start and end of simulations is different + for some sites, and managed by the user-mod-directories. The NEON + user-mod assumes transient cases will run with a transient compset + and the settings are slightly different for transient vs control + including pointing to 2018_control or 2018-PD_transient use-cases. + +Changes to CTSM's user interface (e.g., new/renamed XML or namelist variables): + Add notes to python tools to run using conda environment setup in py_env_create + +Changes made to namelist defaults (e.g., changed parameter values): + New use cases: 2018_control and 2018-PD_transient + +Changes to the datasets (e.g., parameter, surface or initial files): + New updated NEON surface datasets + +Notes of particular relevance for developers: +--------------------------------------------- + +Caveats for developers (e.g., code that is duplicated that requires double maintenance): + Remove toolchain python scripts as this work was moved over to the ctsm5.2 development + +Changes to tests or testing: + Add a run_black target to the python directory Makefile to run black and not just do a black check + Add python modules needed for neon scripts to conda py_create_env conda environment + +Testing summary: regular, tools +---------------- + [PASS means all tests PASS; OK means tests PASS other than expected fails.] + + build-namelist tests (if CLMBuildNamelist.pm has changed): + + cheyenne - OK (NEON tests are different than baseline) + + tools-tests (test/tools) (if tools have been changed): + + cheyenne - + + python testing (if python code has changed; see instructions in python/README.md; document testing done): + + Acheyenne -- PASS + + regular tests (aux_clm: https://github.com/ESCOMP/CTSM/wiki/System-Testing-Guide#pre-merge-system-testing): + + cheyenne ---- OK + izumi ------- OK + + any other testing (give details below): + +Answer changes +-------------- + +Changes answers relative to baseline: No (other than NEON sites, and if direct_to_outlet turned on in MOSART) + + Summarize any changes to answers, i.e., + - what code configurations: NEON or if bypass_routing_option==direct_to_outlet in MOSART + - what platforms/compilers: all + - nature of change: + NEON AG sites are significantly different + + NEON sites reran and reevaluated + MOSART direct_to_outlet option evaluated by @swensosc and @olyson + +Other details +------------- + +List any externals directories updated (cime, rtm, mosart, cism, fates, etc.): mosart + mosart updated from mosart1_0_45 to to mosart1_0_47 (asynchronous changes, and direct_to_outlet fixes) + +Pull Requests that document the changes (include PR ids): +(https://github.com/ESCOMP/ctsm/pull) + #1872 -- NEON updates + #1814 -- Add error checking in partitionWoodFluxes + #1810 -- Fix albdgrd_col value + +=============================================================== +=============================================================== Tag name: ctsm5.1.dev113 Originator(s): sacks (Bill Sacks), ekluzek (Erik Kluzek), jedwards (Jim Edwards) Date: Fri Oct 28 11:00:26 MDT 2022 diff --git a/doc/ChangeSum b/doc/ChangeSum index 3008f25f69..fb0f96fc06 100644 --- a/doc/ChangeSum +++ b/doc/ChangeSum @@ -1,5 +1,6 @@ Tag Who Date Summary ============================================================================================================================ + ctsm5.1.dev114 multiple 11/14/2022 Some NEON updates fixing AG sites, update MOSART, small fixes ctsm5.1.dev113 multiple 10/28/2022 Fix some compsets; add only clauses for ESMF use statements ctsm5.1.dev112 multiple 10/15/2022 Rework fates test definitions and add new fates tests ctsm5.1.dev111 multiple 10/05/2022 Fixes for NEON cases From 92b6899b6c10bf54f95a7891985fcb481bb36fe3 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Mon, 14 Nov 2022 15:22:57 -0700 Subject: [PATCH 75/95] change file updates, rm old neon_s3_upload as replaced by neon_gcs_upload version --- doc/ChangeLog | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/doc/ChangeLog b/doc/ChangeLog index b58f7f92b9..766e4fe186 100644 --- a/doc/ChangeLog +++ b/doc/ChangeLog @@ -1,7 +1,7 @@ =============================================================== Tag name: ctsm5.1.dev114 Originator(s): erik (Erik Kluzek,UCAR/TSS,303-497-1326)/@wwieder/@olyson/@ka7eh -Date: Mon Nov 14 14:22:05 MST 2022 +Date: Mon Nov 14 15:21:50 MST 2022 One-line Summary: Some NEON updates fixing AG sites, update MOSART, small fixes Purpose and description of changes @@ -81,11 +81,12 @@ Testing summary: regular, tools build-namelist tests (if CLMBuildNamelist.pm has changed): - cheyenne - OK (NEON tests are different than baseline) + cheyenne - OK (141 NEON tests are different than baseline) tools-tests (test/tools) (if tools have been changed): - cheyenne - + cheyenne - OK + izumi (NEON) -- OK python testing (if python code has changed; see instructions in python/README.md; document testing done): @@ -97,6 +98,7 @@ Testing summary: regular, tools izumi ------- OK any other testing (give details below): + run_neon.py ran for all NEON sites ad, post-ad, and transient Answer changes -------------- From ea4d150f9a1f5e82c72fb4db5dc7bc957e1270bb Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Mon, 14 Nov 2022 15:23:24 -0700 Subject: [PATCH 76/95] rm old neon_s3_upload as replaced by neon_gcs_upload version --- tools/site_and_regional/neon_s3_upload | 173 ------------------------- 1 file changed, 173 deletions(-) delete mode 100755 tools/site_and_regional/neon_s3_upload diff --git a/tools/site_and_regional/neon_s3_upload b/tools/site_and_regional/neon_s3_upload deleted file mode 100755 index 447886e936..0000000000 --- a/tools/site_and_regional/neon_s3_upload +++ /dev/null @@ -1,173 +0,0 @@ -#! /usr/bin/env python3 -""" -Script to rename and upload NEON site finidat files for use in transient startup cases - -""" - -import os, sys -# Get the ctsm util tools and then the cime tools. -_CTSM_PYTHON = os.path.abspath(os.path.join(os.path.dirname(__file__), "..","..",'python')) -sys.path.insert(1, _CTSM_PYTHON) - -import boto3 -from botocore.exceptions import ClientError -import glob -import datetime -from ctsm import add_cime_to_path -from ctsm.path_utils import path_to_ctsm_root -from standard_script_setup import * -from CIME.case import Case -from CIME.utils import expect, safe_copy - -logger = logging.getLogger(__name__) - -def get_parser(args, description, valid_neon_sites): - """ - Get Parser object for this script - """ - parser = argparse.ArgumentParser(description=description, - formatter_class=argparse.RawDescriptionHelpFormatter) - - CIME.utils.setup_standard_logging_options(parser) - - parser.print_usage = parser.print_help - - parser.add_argument('--neon-sites', - help='4-letter neon site code.', - action="store", - required=False, - choices=valid_neon_sites + ['all'], - dest="neon_sites", - default=["OSBS"], - nargs='+') - - parser.add_argument('--output-root', - help=''' - Root Directory of case results - [default: %(default)s] - ''', - action="store", - dest="output_root", - type =str, - required=False, - default=os.getcwd()) - - parser.add_argument('--file-date', - help=''' - Date of ctsm restart file(s) to upload - ''', - action="store", - dest="file_date", - required = False, - type = datetime.date.fromisoformat, - default = datetime.datetime.strptime("0268-01-01",'%Y-%m-%d')) - - - parser.add_argument('--upload-finidat', - help=''' - Upload the final restart files from the end of the postad run for each site. - ''', - action="store_true", - dest="upload_finidat", - required = False, - default = False) - - parser.add_argument('--upload-history', - help=''' - Upload the transient run h1 history files for each site. - ''', - action="store_true", - dest="upload_history", - required = False, - default = False) - - - - args = CIME.utils.parse_args_and_handle_standard_logging_options(args, parser) - - if 'all' in args.neon_sites: - neon_sites = valid_neon_sites - else: - neon_sites = args.neon_sites - for site in neon_sites: - if site not in valid_neon_sites: - raise ValueError("Invalid site name {}".format(site)) - - expect(args.upload_finidat or args.upload_history,"Must specify at least one of --upload-finidat or --upload-history") - - return neon_sites, args.output_root, args.file_date, args.upload_finidat, args.upload_history - -def upload_file(file_name, bucket, object_name=None): - """Upload a file to an S3 bucket - - :param file_name: File to upload - :param bucket: Bucket to upload to - :param object_name: S3 object name. If not specified then file_name is used - :return: True if file was uploaded, else False - """ - - # If S3 object_name was not specified, use file_name - if object_name is None: - object_name = os.path.basename(file_name) - - # Upload the file - s3_client = boto3.client('s3') - try: - logger.info("Uploading file {} to {}".format(file_name, object_name)) - response = s3_client.upload_file(file_name, bucket, object_name) - except ClientError as e: - logger.error(e) - return False - return True - -def main(description): - """ - For each site in the site_list find the site.postad run directory and grab the latest clm restart file - from there, - """ - - if not os.path.isfile(os.path.join(os.getenv("HOME"),".aws","credentials")): - raise FileNotFoundError("User account must have valid aws credentials to run this script.") - - cesmroot = path_to_ctsm_root() - # Get the list of supported neon sites from usermods - valid_neon_sites = glob.glob(os.path.join(cesmroot,"cime_config","usermods_dirs","NEON","[!d]*")) - valid_neon_sites = [v.split('/')[-1] for v in valid_neon_sites] - filedatestamp = datetime.datetime.now().date() - site_list, output_root, file_date, upload_finidat, upload_history = get_parser(sys.argv, description, valid_neon_sites) - for site in site_list: - rundir = None - if upload_finidat: - logger.info("Upload finidat for {}".format(site)) - case_path = os.path.join(output_root, site+".postad") - if os.path.isdir(case_path): - with Case(case_path) as case: - rundir = case.get_value("RUNDIR") - basefile = site+".postad.clm2.r.{}-00000.nc".format(file_date.strftime("%4Y-%m-%d")) - finidat_file = os.path.join(rundir,basefile) - if not os.path.isfile(finidat_file): - logger.warning("Could not find file {}".format(finidat_file)) - continue - newfile = basefile.replace(".postad.",".{}.".format(filedatestamp)) - - upload_file(finidat_file, 'neon-ncar-transfer', os.path.join("NEON","lnd","ctsm","initdata",newfile)) - if upload_history: - logger.info("Upload history for {}".format(site)) - case_path = os.path.join(output_root, site+".transient") - if not os.path.isdir(case_path): - logger.warning("No case found in {}".format(case_path)) - continue - with Case(case_path) as case: - archive_dir = os.path.join(case.get_value("DOUT_S_ROOT"),"lnd","hist") - for histfile in glob.iglob(archive_dir + "/*.h1.*"): - newfile = os.path.basename(histfile) - upload_file(histfile, 'neon-ncar-transfer', os.path.join("NEON","archive",site,"lnd","hist",newfile)) - - - - - - -if __name__ == "__main__": - main(__doc__) - From deedb0494fb70b3352a790172ac083f73a59ebe3 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Mon, 14 Nov 2022 16:28:30 -0700 Subject: [PATCH 77/95] Fix for #1901 --- cime_config/config_component.xml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cime_config/config_component.xml b/cime_config/config_component.xml index d3c2eda0a7..94e610440a 100644 --- a/cime_config/config_component.xml +++ b/cime_config/config_component.xml @@ -290,10 +290,11 @@ char + <-- NOTE: Be sure to have a comma at the last site-name -- otherwise there will be issues see CTSM github issue #1901 --> ABBY,BLAN,CPER,DEJU,GRSM,HEAL,KONA,LENO,NIWO,ONAQ,PUUM,SERC,SRER,TALL,TREE,WOOD, BARR,BONA,DCFS,DELA,GUAN,JERC,KONZ,MLBS,NOGP,ORNL,RMNP,SJER,STEI,TEAK,UKFS,WREF, - BART,CLBJ,DSNY,HARV,JORN,LAJA,MOAB,OAES,OSBS,SCBI,SOAP,STER,TOOL,UNDE,YELL + BART,CLBJ,DSNY,HARV,JORN,LAJA,MOAB,OAES,OSBS,SCBI,SOAP,STER,TOOL,UNDE,YELL, run_component_ctsm From bfb840d5559c17a4828b32f988ace1260171c834 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Wed, 16 Nov 2022 16:27:51 -0700 Subject: [PATCH 78/95] Explicitly set the NEON site to run --- test/tools/nl_files/run_neon_OSBS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/tools/nl_files/run_neon_OSBS b/test/tools/nl_files/run_neon_OSBS index c49fb77783..0c274b13ad 100644 --- a/test/tools/nl_files/run_neon_OSBS +++ b/test/tools/nl_files/run_neon_OSBS @@ -1 +1 @@ ---verbose --run-type ad --setup-only +--verbose --run-type ad --setup-only --neon-site OSBS From 1ed3774f0f0f575f398af8ffbaa09ae1244b7da7 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Wed, 16 Nov 2022 16:28:48 -0700 Subject: [PATCH 79/95] Move run of py_env_create to after CLM_ROOT is set and call it for all machines --- test/tools/test_driver.sh | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/test/tools/test_driver.sh b/test/tools/test_driver.sh index 1b3c141d79..ce501f980a 100755 --- a/test/tools/test_driver.sh +++ b/test/tools/test_driver.sh @@ -54,8 +54,6 @@ module load nco module load ncl module load conda -$CESMDATAROOT/py_env_create -conda activate ctsm_py ##omp threads @@ -119,8 +117,6 @@ module load openmpi module load nco module load conda module load ncl -$CESMDATAROOT/py_env_create -conda activate ctsm_py ##omp threads @@ -220,8 +216,6 @@ module load compiler/intel module load tool/nco module load tool/netcdf module load lang/python -$CESMDATAROOT/py_env_create -conda activate ctsm_py export NETCDF_DIR=\$NETCDF_PATH export INC_NETCDF=\${NETCDF_PATH}/include @@ -303,8 +297,6 @@ module load compiler/intel module load tool/nco module load tool/netcdf module load lang/python -$CESMDATAROOT/py_env_create -conda activate ctsm_py export NETCDF_DIR=\$NETCDF_PATH export INC_NETCDF=\${NETCDF_PATH}/include @@ -380,6 +372,10 @@ else fi fi +# Setup conda environement +\$CLM_ROOT/py_env_create +conda activate ctsm_py + ##output files clm_log=\${initdir}/td.\${JOBID}.log if [ -f \$clm_log ]; then From c31e832c72d6149bb726f0fb932240054d8fdd11 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Wed, 16 Nov 2022 16:31:36 -0700 Subject: [PATCH 80/95] Update date for change files --- doc/ChangeLog | 3 ++- doc/ChangeSum | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/doc/ChangeLog b/doc/ChangeLog index 766e4fe186..b4d2da11c6 100644 --- a/doc/ChangeLog +++ b/doc/ChangeLog @@ -1,7 +1,7 @@ =============================================================== Tag name: ctsm5.1.dev114 Originator(s): erik (Erik Kluzek,UCAR/TSS,303-497-1326)/@wwieder/@olyson/@ka7eh -Date: Mon Nov 14 15:21:50 MST 2022 +Date: Wed Nov 16 16:31:14 MST 2022 One-line Summary: Some NEON updates fixing AG sites, update MOSART, small fixes Purpose and description of changes @@ -44,6 +44,7 @@ CTSM issues fixed (include CTSM Issue #): Fixes #1896 -- Improve misleading error message in check_for_nans Fixes #1263 -- Fix partitionWood fluxes Fixes #1788 -- Fix albgrd_col + Fixes #1901 -- Fix NEONSITE YELL Notes of particular relevance for users --------------------------------------- diff --git a/doc/ChangeSum b/doc/ChangeSum index fb0f96fc06..a90365740a 100644 --- a/doc/ChangeSum +++ b/doc/ChangeSum @@ -1,6 +1,6 @@ Tag Who Date Summary ============================================================================================================================ - ctsm5.1.dev114 multiple 11/14/2022 Some NEON updates fixing AG sites, update MOSART, small fixes + ctsm5.1.dev114 multiple 11/16/2022 Some NEON updates fixing AG sites, update MOSART, small fixes ctsm5.1.dev113 multiple 10/28/2022 Fix some compsets; add only clauses for ESMF use statements ctsm5.1.dev112 multiple 10/15/2022 Rework fates test definitions and add new fates tests ctsm5.1.dev111 multiple 10/05/2022 Fixes for NEON cases From 6450b8ce08811fdd80b4da97f9d6b6e27aa2f3f3 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Wed, 16 Nov 2022 19:41:51 -0700 Subject: [PATCH 81/95] Fix syntax error --- cime_config/config_component.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cime_config/config_component.xml b/cime_config/config_component.xml index 94e610440a..60d8311b95 100644 --- a/cime_config/config_component.xml +++ b/cime_config/config_component.xml @@ -290,7 +290,7 @@ char - <-- NOTE: Be sure to have a comma at the last site-name -- otherwise there will be issues see CTSM github issue #1901 --> + ABBY,BLAN,CPER,DEJU,GRSM,HEAL,KONA,LENO,NIWO,ONAQ,PUUM,SERC,SRER,TALL,TREE,WOOD, BARR,BONA,DCFS,DELA,GUAN,JERC,KONZ,MLBS,NOGP,ORNL,RMNP,SJER,STEI,TEAK,UKFS,WREF, From be3a100a81cb8293cf444b2cd5614f4d1cac64ea Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Wed, 16 Nov 2022 19:43:57 -0700 Subject: [PATCH 82/95] OK really fix the syntax for the comment --- cime_config/config_component.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cime_config/config_component.xml b/cime_config/config_component.xml index 60d8311b95..555a0ce9b5 100644 --- a/cime_config/config_component.xml +++ b/cime_config/config_component.xml @@ -290,7 +290,7 @@ char - + ABBY,BLAN,CPER,DEJU,GRSM,HEAL,KONA,LENO,NIWO,ONAQ,PUUM,SERC,SRER,TALL,TREE,WOOD, BARR,BONA,DCFS,DELA,GUAN,JERC,KONZ,MLBS,NOGP,ORNL,RMNP,SJER,STEI,TEAK,UKFS,WREF, From a8630ad6ac0af399ff35906523c8a0def28462b8 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Wed, 16 Nov 2022 19:48:41 -0700 Subject: [PATCH 83/95] Add some checks that directories are there as expected --- python/ctsm/test/test_sys_lilac_build_ctsm.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/python/ctsm/test/test_sys_lilac_build_ctsm.py b/python/ctsm/test/test_sys_lilac_build_ctsm.py index f1c5e22f8f..d773749bf7 100755 --- a/python/ctsm/test/test_sys_lilac_build_ctsm.py +++ b/python/ctsm/test/test_sys_lilac_build_ctsm.py @@ -27,6 +27,7 @@ class TestSysBuildCtsm(unittest.TestCase): def setUp(self): self._tempdir = tempfile.mkdtemp() + self.assertTrue(os.path.isdir(self._tempdir)) # Hack around a check in CIME: As of https://github.com/ESMCI/cime/pull/4228, If # NCAR_HOST is in the environment, CIME checks if the machine you're running on is @@ -68,6 +69,7 @@ def test_buildSetup_userDefinedMachine_minimalInfo(self): gmake_j=8, no_pnetcdf=True, ) + self.assertTrue(os.path.isdir(build_dir)) # the critical piece of this test is that the above command doesn't generate any # errors; however we also do some assertions below @@ -87,6 +89,7 @@ def test_buildSetup_userDefinedMachine_allInfo(self): build_dir = os.path.join(self._tempdir, "ctsm_build") inputdata_path = os.path.realpath(os.path.join(self._tempdir, "my_inputdata")) os.makedirs(inputdata_path) + self.assertTrue(os.path.isdir(inputdata_path)) build_ctsm( cime_path=_CIME_PATH, build_dir=build_dir, @@ -107,6 +110,7 @@ def test_buildSetup_userDefinedMachine_allInfo(self): build_with_openmp=True, inputdata_path=os.path.join(self._tempdir, "my_inputdata"), ) + self.assertTrue(os.path.isdir(build_dir)) # the critical piece of this test is that the above command doesn't generate any # errors; however we also do some assertions below From 57e9d4e63d6b935830d8eb6356fee4c680c7fc1e Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Wed, 16 Nov 2022 19:53:44 -0700 Subject: [PATCH 84/95] Add some more checking that create_newcase exists as expected and that the inputdata_path directory exists if provided --- python/ctsm/lilac_build_ctsm.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/python/ctsm/lilac_build_ctsm.py b/python/ctsm/lilac_build_ctsm.py index 20231c0df9..b189cb56ea 100644 --- a/python/ctsm/lilac_build_ctsm.py +++ b/python/ctsm/lilac_build_ctsm.py @@ -718,8 +718,13 @@ def _create_case( else: machine_args = ["--machine", machine] + cmd = os.path.join(cime_path, "scripts", "create_newcase") + if not os.path.exists(cmd): + abort( + "The create_newcase command doesn't exist as expected <{}> does not exist)".format(cmd) + ) create_newcase_cmd = [ - os.path.join(cime_path, "scripts", "create_newcase"), + cmd, "--output-root", build_dir, "--case", @@ -741,7 +746,12 @@ def _create_case( create_newcase_cmd.extend(machine_args) if inputdata_path: create_newcase_cmd.extend(["--input-dir", inputdata_path]) - run_cmd_output_on_error(create_newcase_cmd, errmsg="Problem creating CTSM case directory") + if not os.path.isdir(inputdata_path): + abort("inputdata_path directory (<{}> does not exist)".format(inputdata_path)) + run_cmd_output_on_error( + create_newcase_cmd, + errmsg="Problem running create_newcase to create the CTSM case directory", + ) subprocess.check_call([xmlchange, "LILAC_MODE=on"], cwd=case_dir) if build_debug: From 8b36987253ade815c734a3f6fde48c2f1a162e0b Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Thu, 17 Nov 2022 15:25:47 -0700 Subject: [PATCH 85/95] Add a check that the main inputdata directory exists or not, and add a unit test for it --- python/ctsm/subset_data.py | 11 +++-- python/ctsm/test/test_unit_subset_data.py | 57 +++++++++++++++++++++++ 2 files changed, 65 insertions(+), 3 deletions(-) create mode 100755 python/ctsm/test/test_unit_subset_data.py diff --git a/python/ctsm/subset_data.py b/python/ctsm/subset_data.py index 4832e98f7f..2d23aba12f 100644 --- a/python/ctsm/subset_data.py +++ b/python/ctsm/subset_data.py @@ -67,6 +67,7 @@ from ctsm.site_and_regional.regional_case import RegionalCase from ctsm.args_utils import plon_type, plat_type from ctsm.path_utils import path_to_ctsm_root +from ctsm.utils import abort # -- import ctsm logging flags from ctsm.ctsm_logging import ( @@ -396,16 +397,20 @@ def setup_files(args, defaults, cesmroot): fsurf_in = defaults.get("surfdat", "surfdat_" + num_pft + "pft") fluse_in = defaults.get("landuse", "landuse_" + num_pft + "pft") + clmforcingindir = defaults.get("main", "clmforcingindir") + if not os.path.isdir(clmforcingindir): + logger.info("clmforcingindir does not exist: %s", clmforcingindir) + abort("inputdata directory does not exist") file_dict = { - "main_dir": defaults.get("main", "clmforcingindir"), + "main_dir": clmforcingindir, "fdomain_in": defaults.get("domain", "file"), "fsurf_dir": os.path.join( - defaults.get("main", "clmforcingindir"), + clmforcingindir, os.path.join(defaults.get("surfdat", "dir")), ), "fluse_dir": os.path.join( - defaults.get("main", "clmforcingindir"), + clmforcingindir, os.path.join(defaults.get("landuse", "dir")), ), "fsurf_in": fsurf_in, diff --git a/python/ctsm/test/test_unit_subset_data.py b/python/ctsm/test/test_unit_subset_data.py new file mode 100755 index 0000000000..8245b4a1ba --- /dev/null +++ b/python/ctsm/test/test_unit_subset_data.py @@ -0,0 +1,57 @@ +#!/usr/bin/env python3 +""" +Unit tests for subset_data + +You can run this by: + python -m unittest test_unit_subset_data.py +""" + +import unittest +import configparser +import argparse +import os +import sys + +# -- add python/ctsm to path (needed if we want to run the test stand-alone) +_CTSM_PYTHON = os.path.join(os.path.dirname(os.path.realpath(__file__)), os.pardir, os.pardir) +sys.path.insert(1, _CTSM_PYTHON) + +# pylint: disable=wrong-import-position +from ctsm import unit_testing +from ctsm.subset_data import get_parser, setup_files +from ctsm.path_utils import path_to_ctsm_root + +# pylint: disable=invalid-name + + +class TestSubsetData(unittest.TestCase): + """ + Basic class for testing SubsetData class in subset_data.py. + """ + + def setUp(self): + sys.argv = ["subset_data", "point"] + DEFAULTS_FILE = "default_data.cfg" + parser = get_parser() + self.args = parser.parse_args() + self.cesmroot = path_to_ctsm_root() + self.defaults = configparser.ConfigParser() + self.defaults.read(os.path.join(self.cesmroot, "tools/site_and_regional", DEFAULTS_FILE)) + + def test_inputdata_setup_files_basic(self): + """ + Test + """ + setup_files(self.args, self.defaults, self.cesmroot) + + def test_inputdata_setup_files_inputdata_dne(self): + """ + Test that inputdata directory does not exist + """ + self.defaults.set("main", "clmforcingindir", "/zztop") + with self.assertRaisesRegex(SystemExit, "inputdata directory does not exist"): + setup_files(self.args, self.defaults, self.cesmroot) + +if __name__ == "__main__": + unit_testing.setup_for_tests() + unittest.main() From 5381b911e2a0b281a92d565de22e501e0240a190 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Thu, 17 Nov 2022 15:39:51 -0700 Subject: [PATCH 86/95] Add inputdata-dir option --- python/ctsm/subset_data.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/python/ctsm/subset_data.py b/python/ctsm/subset_data.py index 2d23aba12f..513f75ec59 100644 --- a/python/ctsm/subset_data.py +++ b/python/ctsm/subset_data.py @@ -315,6 +315,14 @@ def get_parser(): action="store_true", dest="overwrite", ) + subparser.add_argument( + "--inputdata-dir", + help="Top level path to the CESM inputdata directory.", + action="store", + dest="inputdatadir", + type=str, + default="defaults.cfg", + ) add_logging_args(subparser) # -- print help for both subparsers @@ -397,7 +405,11 @@ def setup_files(args, defaults, cesmroot): fsurf_in = defaults.get("surfdat", "surfdat_" + num_pft + "pft") fluse_in = defaults.get("landuse", "landuse_" + num_pft + "pft") - clmforcingindir = defaults.get("main", "clmforcingindir") + if args.inputdatadir == "defaults.cfg": + clmforcingindir = defaults.get("main", "clmforcingindir") + else: + clmforcingindir = args.inputdatadir + if not os.path.isdir(clmforcingindir): logger.info("clmforcingindir does not exist: %s", clmforcingindir) abort("inputdata directory does not exist") From a4ea6e85192466ef39a072b0cf462b4966c8f805 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Thu, 17 Nov 2022 15:48:31 -0700 Subject: [PATCH 87/95] Add a check that using command line argument to something that doesn't exist aborts as expected --- python/ctsm/test/test_unit_subset_data.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/python/ctsm/test/test_unit_subset_data.py b/python/ctsm/test/test_unit_subset_data.py index 8245b4a1ba..6a72e5e8ee 100755 --- a/python/ctsm/test/test_unit_subset_data.py +++ b/python/ctsm/test/test_unit_subset_data.py @@ -52,6 +52,14 @@ def test_inputdata_setup_files_inputdata_dne(self): with self.assertRaisesRegex(SystemExit, "inputdata directory does not exist"): setup_files(self.args, self.defaults, self.cesmroot) + def test_inputdata_setup_files_bad_inputdata_arg(self): + """ + Test that inputdata directory provided on command line does not exist if it's bad + """ + self.args.inputdatadir = "/zztop" + with self.assertRaisesRegex(SystemExit, "inputdata directory does not exist"): + setup_files(self.args, self.defaults, self.cesmroot) + if __name__ == "__main__": unit_testing.setup_for_tests() unittest.main() From c5c054819f9d8da9b4e21cd296398bcde27c8151 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Thu, 17 Nov 2022 15:52:28 -0700 Subject: [PATCH 88/95] Run black --- python/ctsm/subset_data.py | 4 ++-- python/ctsm/test/test_unit_subset_data.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/python/ctsm/subset_data.py b/python/ctsm/subset_data.py index 513f75ec59..59b512723c 100644 --- a/python/ctsm/subset_data.py +++ b/python/ctsm/subset_data.py @@ -406,9 +406,9 @@ def setup_files(args, defaults, cesmroot): fsurf_in = defaults.get("surfdat", "surfdat_" + num_pft + "pft") fluse_in = defaults.get("landuse", "landuse_" + num_pft + "pft") if args.inputdatadir == "defaults.cfg": - clmforcingindir = defaults.get("main", "clmforcingindir") + clmforcingindir = defaults.get("main", "clmforcingindir") else: - clmforcingindir = args.inputdatadir + clmforcingindir = args.inputdatadir if not os.path.isdir(clmforcingindir): logger.info("clmforcingindir does not exist: %s", clmforcingindir) diff --git a/python/ctsm/test/test_unit_subset_data.py b/python/ctsm/test/test_unit_subset_data.py index 6a72e5e8ee..aa4c412bbb 100755 --- a/python/ctsm/test/test_unit_subset_data.py +++ b/python/ctsm/test/test_unit_subset_data.py @@ -8,7 +8,6 @@ import unittest import configparser -import argparse import os import sys @@ -40,7 +39,7 @@ def setUp(self): def test_inputdata_setup_files_basic(self): """ - Test + Test """ setup_files(self.args, self.defaults, self.cesmroot) @@ -60,6 +59,7 @@ def test_inputdata_setup_files_bad_inputdata_arg(self): with self.assertRaisesRegex(SystemExit, "inputdata directory does not exist"): setup_files(self.args, self.defaults, self.cesmroot) + if __name__ == "__main__": unit_testing.setup_for_tests() unittest.main() From 00d117aec9cb3517da8f53f0b0aafee54b0aafac Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Thu, 17 Nov 2022 16:24:33 -0700 Subject: [PATCH 89/95] Set inputdata option so testing can run on both cheyenne and izumi --- test/tools/nl_files/subset_data_KONA | 2 +- test/tools/nl_files/subset_data_US-UMB | 2 +- test/tools/nl_files/subset_data_YELL | 2 +- test/tools/nl_files/subset_data_f09_US_pt | 2 +- test/tools/nl_files/subset_data_region1 | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/test/tools/nl_files/subset_data_KONA b/test/tools/nl_files/subset_data_KONA index cb743f2b45..c3be007869 100644 --- a/test/tools/nl_files/subset_data_KONA +++ b/test/tools/nl_files/subset_data_KONA @@ -1 +1 @@ -point --lon 263.38956 --lat 39.1082 --site KONA --dompft 17 19 23 45 --pctpft 28 12 32 28 --crop --create-domain --create-surface --outdir EXEDIR/KONA_user-mod_and_data --user-mods-dir EXEDIR/KONA_user-mod_and_data --verbose +point --lon 263.38956 --lat 39.1082 --site KONA --dompft 17 19 23 45 --pctpft 28 12 32 28 --crop --create-domain --create-surface --outdir EXEDIR/KONA_user-mod_and_data --user-mods-dir EXEDIR/KONA_user-mod_and_data --verbose --inputdata-dir CSMDATA diff --git a/test/tools/nl_files/subset_data_US-UMB b/test/tools/nl_files/subset_data_US-UMB index 499b5f53fd..935b0dc99d 100644 --- a/test/tools/nl_files/subset_data_US-UMB +++ b/test/tools/nl_files/subset_data_US-UMB @@ -1 +1 @@ -point --lon 275.28626 --lat 45.5598 --site 1x1_US-UMB --dompft 7 --cap-saturation --uniform-snowpack --create-surface --outdir EXEDIR/US-UMB_user-mod_and_data --user-mods-dir EXEDIR/US-UMB_user-mod_and_data --verbose +point --lon 275.28626 --lat 45.5598 --site 1x1_US-UMB --dompft 7 --cap-saturation --uniform-snowpack --create-surface --outdir EXEDIR/US-UMB_user-mod_and_data --user-mods-dir EXEDIR/US-UMB_user-mod_and_data --verbose --inputdata-dir CSMDATA diff --git a/test/tools/nl_files/subset_data_YELL b/test/tools/nl_files/subset_data_YELL index 5e142713df..8295830c25 100644 --- a/test/tools/nl_files/subset_data_YELL +++ b/test/tools/nl_files/subset_data_YELL @@ -1 +1 @@ -point --lon 250.45804 --lat 44.95597 --site YELL --dompft 1 --crop --create-domain --create-surface --outdir EXEDIR/YELL_user-mod_and_data --user-mods-dir EXEDIR/YELL_user-mod_and_data --verbose +point --lon 250.45804 --lat 44.95597 --site YELL --dompft 1 --crop --create-domain --create-surface --outdir EXEDIR/YELL_user-mod_and_data --user-mods-dir EXEDIR/YELL_user-mod_and_data --verbose --inputdata-dir CSMDATA diff --git a/test/tools/nl_files/subset_data_f09_US_pt b/test/tools/nl_files/subset_data_f09_US_pt index 4acdfeabd4..bf6d5e2861 100644 --- a/test/tools/nl_files/subset_data_f09_US_pt +++ b/test/tools/nl_files/subset_data_f09_US_pt @@ -1 +1 @@ -point --lon 257.5 --lat 43.822 --site 1x1_ --include-nonveg --crop --create-landuse --create-datm --create-user-mods --datm-syr 2000 --datm-eyr 2000 --create-surface --outdir EXEDIR/f09_US_pt_user-mod_and_data --user-mods-dir EXEDIR/f09_US_pt_user-mod_and_data --verbose +point --lon 257.5 --lat 43.822 --site 1x1_ --include-nonveg --crop --create-landuse --create-datm --create-user-mods --datm-syr 2000 --datm-eyr 2000 --create-surface --outdir EXEDIR/f09_US_pt_user-mod_and_data --user-mods-dir EXEDIR/f09_US_pt_user-mod_and_data --verbose --inputdata-dir CSMDATA diff --git a/test/tools/nl_files/subset_data_region1 b/test/tools/nl_files/subset_data_region1 index c1c5607239..fce83f0e2e 100644 --- a/test/tools/nl_files/subset_data_region1 +++ b/test/tools/nl_files/subset_data_region1 @@ -1 +1 @@ -region --lat1 -40 --lat2 15 --lon1 275 --lon2 330 --create-domain --create-surface --create-landuse --verbose --overwrite --reg test1 +region --lat1 -40 --lat2 15 --lon1 275 --lon2 330 --create-domain --create-surface --create-landuse --verbose --overwrite --reg test1 --inputdata-dir CSMDATA From b5f153fbe4bdaa6e018b695949a2c7c5cfd50103 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Thu, 17 Nov 2022 18:54:59 -0700 Subject: [PATCH 90/95] Add argument for inputdata-directory --- .../modify_singlept_site_neon.py | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/tools/site_and_regional/modify_singlept_site_neon.py b/tools/site_and_regional/modify_singlept_site_neon.py index 334ab7d6aa..e842a84f07 100755 --- a/tools/site_and_regional/modify_singlept_site_neon.py +++ b/tools/site_and_regional/modify_singlept_site_neon.py @@ -149,6 +149,18 @@ def get_parser(): required=False, default="/glade/scratch/" + myname + "/single_point_neon_updated/", ) + parser.add_argument( + "--inputdata-dir", + help=""" + Directory to write updated single point surface dataset. + [default: %(default)s] + """, + action="store", + dest="inputdatadir", + type=str, + required=False, + default="/glade/p/cesmdata/cseg/inputdata" + ) parser.add_argument( "-d", "--debug", @@ -269,7 +281,7 @@ def find_surffile(surf_dir, site_name): return surf_file -def find_soil_structure(surf_file): +def find_soil_structure(args, surf_file): """ Function for finding surface dataset soil strucutre using surface data metadata. @@ -298,7 +310,7 @@ def find_soil_structure(surf_file): print("------------") # print (f1.attrs["Soil_texture_raw_data_file_name"]) - clm_input_dir = "/glade/p/cesmdata/cseg/inputdata/lnd/clm2/rawdata/" + clm_input_dir = os.path.join( args.inputdatadir, "lnd/clm2/rawdata/" ) surf_soildepth_file = os.path.join( clm_input_dir, f1.attrs["Soil_texture_raw_data_file_name"] ) @@ -534,7 +546,7 @@ def main(): f1 = xr.open_dataset(surf_file) # -- Find surface dataset soil depth information - soil_bot, soil_top = find_soil_structure(surf_file) + soil_bot, soil_top = find_soil_structure(args, surf_file) # -- Find surface dataset soil levels # TODO: how? NS uses metadata on file to find From 2c523aafaa334a5bf54ca5cb7ee263c76132f29a Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Thu, 17 Nov 2022 18:57:17 -0700 Subject: [PATCH 91/95] Set inputdata directory so can work on izumi --- test/tools/nl_files/modify_data_YELL | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/tools/nl_files/modify_data_YELL b/test/tools/nl_files/modify_data_YELL index e76322cdeb..159c92ae63 100644 --- a/test/tools/nl_files/modify_data_YELL +++ b/test/tools/nl_files/modify_data_YELL @@ -1 +1 @@ ---neon_site YELL --surf_dir CSMDATA/lnd/clm2/surfdata_map/NEON --out_dir EXEDIR +--neon_site YELL --surf_dir CSMDATA/lnd/clm2/surfdata_map/NEON --out_dir EXEDIR --inputdata-dir CSMDATA From 2e122e0de3ef899eb162cea97cea9a74ee9fab2a Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Thu, 17 Nov 2022 22:11:43 -0700 Subject: [PATCH 92/95] Make datm forcing data under the inputdata-dir so doesn't need a seperate path and can work on izumi as well --- python/ctsm/subset_data.py | 19 ++++++++++--------- tools/site_and_regional/default_data.cfg | 2 +- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/python/ctsm/subset_data.py b/python/ctsm/subset_data.py index 59b512723c..34a9e583d0 100644 --- a/python/ctsm/subset_data.py +++ b/python/ctsm/subset_data.py @@ -390,10 +390,19 @@ def setup_files(args, defaults, cesmroot): if args.create_user_mods: setup_user_mods(args.user_mods_dir, cesmroot) + if args.inputdatadir == "defaults.cfg": + clmforcingindir = defaults.get("main", "clmforcingindir") + else: + clmforcingindir = args.inputdatadir + + if not os.path.isdir(clmforcingindir): + logger.info("clmforcingindir does not exist: %s", clmforcingindir) + abort("inputdata directory does not exist") + # DATM data datm_type = "datm_gswp3" dir_output_datm = "datmdata" - dir_input_datm = defaults.get(datm_type, "dir") + dir_input_datm = os.path.join(clmforcingindir, defaults.get(datm_type, "dir")) if args.create_datm: if not os.path.isdir(os.path.join(args.out_dir, dir_output_datm)): os.mkdir(os.path.join(args.out_dir, dir_output_datm)) @@ -405,14 +414,6 @@ def setup_files(args, defaults, cesmroot): fsurf_in = defaults.get("surfdat", "surfdat_" + num_pft + "pft") fluse_in = defaults.get("landuse", "landuse_" + num_pft + "pft") - if args.inputdatadir == "defaults.cfg": - clmforcingindir = defaults.get("main", "clmforcingindir") - else: - clmforcingindir = args.inputdatadir - - if not os.path.isdir(clmforcingindir): - logger.info("clmforcingindir does not exist: %s", clmforcingindir) - abort("inputdata directory does not exist") file_dict = { "main_dir": clmforcingindir, diff --git a/tools/site_and_regional/default_data.cfg b/tools/site_and_regional/default_data.cfg index f689c99044..7e841dca54 100644 --- a/tools/site_and_regional/default_data.cfg +++ b/tools/site_and_regional/default_data.cfg @@ -2,7 +2,7 @@ clmforcingindir = /glade/p/cesmdata/inputdata [datm_gswp3] -dir = /glade/p/cgd/tss/CTSM_datm_forcing_data/atm_forcing.datm7.GSWP3.0.5d.v1.c170516 +dir = atm/datm7/atm_forcing.datm7.GSWP3.0.5d.v1.c170516 domain = domain.lnd.360x720_gswp3.0v1.c170606.nc solardir = Solar precdir = Precip From 6b017ae5e7fad159afb368282b9d9c3c3ed38a93 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Fri, 18 Nov 2022 17:30:33 -0700 Subject: [PATCH 93/95] Check if pandas is recent enough --- tools/site_and_regional/modify_singlept_site_neon.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tools/site_and_regional/modify_singlept_site_neon.py b/tools/site_and_regional/modify_singlept_site_neon.py index e842a84f07..f9f7188b42 100755 --- a/tools/site_and_regional/modify_singlept_site_neon.py +++ b/tools/site_and_regional/modify_singlept_site_neon.py @@ -47,6 +47,7 @@ import numpy as np import pandas as pd import xarray as xr +from packaging import version from datetime import date from getpass import getuser @@ -518,6 +519,12 @@ def main(): if args.debug: logging.basicConfig(level=logging.DEBUG) + # Check if pandas is a recent enough version + pdvers = pd.__version__ + if version.parse(pdvers) < version.parse("1.1.0"): + sys.exit("The pandas version in your python environment is too old, update to a newer version of pandas (>=1.1.0): version=%s", pdvers ) + + file_time = check_neon_time() # -- specify site from which to extract data From b8753eb6259fc9471a34e2430f433b8caa500a00 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Fri, 18 Nov 2022 18:58:20 -0700 Subject: [PATCH 94/95] Fix the error, which was failing in writing an empty list --- tools/site_and_regional/modify_singlept_site_neon.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tools/site_and_regional/modify_singlept_site_neon.py b/tools/site_and_regional/modify_singlept_site_neon.py index f9f7188b42..2798f463ec 100755 --- a/tools/site_and_regional/modify_singlept_site_neon.py +++ b/tools/site_and_regional/modify_singlept_site_neon.py @@ -274,10 +274,10 @@ def find_surffile(surf_dir, site_name): surf_file = surf_file[0] else: sys.exit( - "Surface data for this site " + site_name + "was not found:" + surf_file, - ".", - "\n", - "Please run ./subset_data.py for this site.", + "Surface data for this site " + str(site_name) + " was not found:" + str(surf_dir) + str(sf_name) + + "." + + "\n" + + "Please run ./subset_data.py for this site." ) return surf_file From 01b411857fdef867dd177075d0a4f602a220f765 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Sat, 19 Nov 2022 18:11:31 -0700 Subject: [PATCH 95/95] Update changelog --- doc/ChangeLog | 14 ++++++++++++-- doc/ChangeSum | 2 +- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/doc/ChangeLog b/doc/ChangeLog index b4d2da11c6..5f16bf5634 100644 --- a/doc/ChangeLog +++ b/doc/ChangeLog @@ -1,7 +1,7 @@ =============================================================== Tag name: ctsm5.1.dev114 Originator(s): erik (Erik Kluzek,UCAR/TSS,303-497-1326)/@wwieder/@olyson/@ka7eh -Date: Wed Nov 16 16:31:14 MST 2022 +Date: Sat Nov 19 18:11:15 MST 2022 One-line Summary: Some NEON updates fixing AG sites, update MOSART, small fixes Purpose and description of changes @@ -11,6 +11,9 @@ Minor changes to python scripts and usermod_dirs for NEON cases. Also update the smaller lightning file. Have NEON use new use-cases for 2018 and 2018-PD conditions for CLM. Have NEON Agricultural sites run with prognostic crop. Simple fix for warning about NaN's in import/export data from/to coupler. +Get NEON tests working on izumi, add --inputdata-dir to subset_data and modify_singlept_site_neon.py so they aren't tied +to only running on cheyenne. + Also update MOSART with fixed for direct_to_outlet option. Add error checking in ParitionWoodFluxes. Fix value of albgrd_col in SurfaceAlbefdoType.F90. @@ -45,6 +48,11 @@ CTSM issues fixed (include CTSM Issue #): Fixes #1263 -- Fix partitionWood fluxes Fixes #1788 -- Fix albgrd_col Fixes #1901 -- Fix NEONSITE YELL + + Some on #1910 -- add pandas version check to modify_singlept_site_neon.py so will abort cleanly if version not updated + + Known bugs found since the previous tag (include issue #): + #1910 -- modify_singlept_site_neon.py has trouble on izumi Notes of particular relevance for users --------------------------------------- @@ -87,7 +95,9 @@ Testing summary: regular, tools tools-tests (test/tools) (if tools have been changed): cheyenne - OK - izumi (NEON) -- OK + cheyenne (NEON) - PASS + izumi (NEON) -- OK (modify_singlept_site_neon.py test fails due to #1910) + izumi -- OK python testing (if python code has changed; see instructions in python/README.md; document testing done): diff --git a/doc/ChangeSum b/doc/ChangeSum index a90365740a..772545d00c 100644 --- a/doc/ChangeSum +++ b/doc/ChangeSum @@ -1,6 +1,6 @@ Tag Who Date Summary ============================================================================================================================ - ctsm5.1.dev114 multiple 11/16/2022 Some NEON updates fixing AG sites, update MOSART, small fixes + ctsm5.1.dev114 multiple 11/19/2022 Some NEON updates fixing AG sites, update MOSART, small fixes ctsm5.1.dev113 multiple 10/28/2022 Fix some compsets; add only clauses for ESMF use statements ctsm5.1.dev112 multiple 10/15/2022 Rework fates test definitions and add new fates tests ctsm5.1.dev111 multiple 10/05/2022 Fixes for NEON cases