From 6b3599850d95b13cb3027d82faa1b9cd7302e245 Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Wed, 18 Aug 2021 11:08:14 -0600 Subject: [PATCH 01/55] improve ui for neon script --- tools/site_and_regional/run_neon.py | 171 ++++++++++------------------ 1 file changed, 59 insertions(+), 112 deletions(-) diff --git a/tools/site_and_regional/run_neon.py b/tools/site_and_regional/run_neon.py index 17c457b61e..41a6aca5c8 100755 --- a/tools/site_and_regional/run_neon.py +++ b/tools/site_and_regional/run_neon.py @@ -67,7 +67,7 @@ import subprocess import pandas as pd import glob -from datetime import datetime +import datetime from getpass import getuser # Get the ctsm util tools and then the cime tools. @@ -106,18 +106,6 @@ def get_parser(args, description, valid_neon_sites): default=["OSBS"], nargs='+') -# not used -# parser.add_argument('--surf-dir', -# help=''' -# Directory of single point surface dataset. -# [default: %(default)s] -# ''', -# action="store", -# dest="surf_dir", -# type =str, -# required=False, -# default="/glade/scratch/"+myname+"/single_point/") - parser.add_argument('--base-case', help=''' Root Directory of base case build @@ -150,99 +138,44 @@ def get_parser(args, description, valid_neon_sites): required = False, default = False) - subparsers = parser.add_subparsers ( - dest='run_type', - help='Four different ways to run this script.') - - ad_parser = subparsers.add_parser ('ad', - help=''' AD spin-up options ''') - - pad_parser = subparsers.add_parser ('postad', - help=''' Post-AD spin-up options ''') + parser.add_argument('--run-type', + help=''' + Type of run to do + [default: %(default)s] + ''', + choices = ["ad", "postad", "transient", "sasu"], + default = "transient") - tr_parser = subparsers.add_parser ('transient', - help=''' Transient spin-up options ''') - - sasu_parser = subparsers.add_parser ('sasu', - help=''' Sasu spin-up options --not in CTSM yet''') - - ad_parser.add_argument ('--ad-length', + parser.add_argument ('--run-length', help=''' - How many years to run AD spin-up + How long to run (modified ISO 8601 duration) [default: %(default)s] ''', required = False, - type = int, - default = 200) + type = str, + default = '200Y') - pad_parser.add_argument ('--postad-length', - help=''' - How many years to run in post-AD mode - [default: %(default)s] - ''', - required = False, - type = int, - default = 100) - - tr_parser.add_argument('--start-year', + parser.add_argument('--start-date', help=''' - Start year for running CTSM simulation. + Start date for running CTSM simulation in ISO format. [default: %(default)s] ''', action="store", - dest="start_year", + dest="start_date", required = False, - type = int, - default = 2018) + type = datetime.date.fromisoformat, + default = datetime.datetime.strptime("2018-01-01",'%Y-%m-%d')) - tr_parser.add_argument('--end-year', + parser.add_argument('--end-date', help=''' - End year for running CTSM simulation. + End date for running CTSM simulation in ISO format. [default: %(default)s] ''', action="store", - dest="end_year", + dest="end_date", required = False, - type = int, - default = 2020) - - #parser.add_argument('--spinup', - # help=''' - # AD spin-up - # [default: %(default)s] - # ''', - # action="store_true", - # dest="ad_flag", - # required = False, - # default = True) - #parser.add_argument('--postad', - # help=''' - # Post-AD spin-up - # [default: %(default)s] - # ''', - # action="store_true", - # dest="postad_flag", - # required = False, - # default = True) - #parser.add_argument('--transient', - # help=''' - # Transient - # [default: %(default)s] - # ''', - # action="store_true", - # dest="transient_flag", - # required = False, - # default = True) - - #parser.add_argument('--sasu','--matrix', - # help=''' - # Matrix (SASU) spin-up - # [default: %(default)s] - # ''', - # action="store_true", - # dest="sasu_flag", - # required = False, - # default = False) + type = datetime.date.fromisoformat, + default = datetime.datetime.strptime("2020-01-01",'%Y-%m-%d')) args = CIME.utils.parse_args_and_handle_standard_logging_options(args, parser) @@ -256,18 +189,34 @@ def get_parser(args, description, valid_neon_sites): if "CIME_OUTPUT_ROOT" in args.case_root: args.case_root = None - if args.run_type: - run_type = args.run_type - else: - run_type = "ad" - if run_type == "ad": - run_length = int(args.ad_length) - elif run_type == "postad": - run_length = int(args.postad_length) - else: - run_length = 0 - return neon_sites, args.case_root, run_type, args.overwrite, run_length, args.base_case_root + + run_length = parse_isoduration(args.run_length) + + return neon_sites, args.case_root, args.run_type, args.overwrite, run_length, args.base_case_root + +def get_isosplit(s, split): + if split in s: + n, s = s.split(split) + else: + n = 0 + return n, s + +def parse_isoduration(s): + ''' + simple ISO 8601 duration parser, does not account for leap years and assumes 30 day months + ''' + # Remove prefix + s = s.split('P')[-1] + + # Step through letter dividers + years, s = get_isosplit(s, 'Y') + months, s = get_isosplit(s, 'M') + days, s = get_isosplit(s, 'D') + + # Convert all to timedelta + dt = datetime.timedelta(days=int(days)+365*int(years)+30*int(months)) + return int(dt.total_seconds()/86400) class NeonSite : """ @@ -314,11 +263,9 @@ def build_base_case(self, cesmroot, case_root, res, compset, overwrite): logger.info("---- building a base case -------") self.base_case_root = case_root user_mods_dirs = [os.path.join(cesmroot,"cime_config","usermods_dirs","NEON",self.name)] - if case_root: - case_path = os.path.join(case_root,self.name) - logger.info ('case_root : {}'.format(case_root)) - else: - case_path = self.name + if not case_root: + case_root = os.getcwd() + case_path = os.path.join(case_root,self.name) logger.info ('base_case_name : {}'.format(self.name)) logger.info ('user_mods_dir : {}'.format(user_mods_dirs[0])) @@ -332,7 +279,7 @@ def build_base_case(self, cesmroot, case_root, res, compset, overwrite): logger.info("---- creating a base case -------") case.create(case_path, cesmroot, compset, res, mpilib="mpi-serial", - run_unsupported=True, answer="r", + run_unsupported=True, answer="r",output_root=case_root, user_mods_dirs = user_mods_dirs, driver="nuopc") logger.info("---- base case created ------") @@ -379,7 +326,7 @@ def run_case(self, base_case_root, run_type, run_length, overwrite=False): with Case(case_root, read_only=False) as case: - case.set_value("STOP_OPTION", "nyears") + case.set_value("STOP_OPTION", "ndays") case.set_value("STOP_N", run_length) case.set_value("REST_N", 100) case.set_value("CONTINUE_RUN", False) @@ -389,6 +336,7 @@ def run_case(self, base_case_root, run_type, run_length, overwrite=False): case.set_value("CLM_ACCELERATED_SPINUP","on") case.set_value("RUN_REFDATE", "0018-01-01") case.set_value("RUN_STARTDATE", "0018-01-01") + case.set_value("JOB_WALLCLOCK_TIME", "12:00:00", subgroup="case.run") else: case.set_value("CLM_FORCE_COLDSTART","off") case.set_value("CLM_ACCELERATED_SPINUP","off") @@ -580,11 +528,10 @@ def main(description): valid_neon_sites = [v.split('/')[-1] for v in valid_neon_sites] site_list, case_root, run_type, overwrite, run_length, base_case_root = get_parser(sys.argv, description, valid_neon_sites) - - logger.debug ("case_root : "+ case_root) - - if not os.path.exists(case_root): - os.makedirs(case_root) + if case_root: + logger.debug ("case_root : "+ case_root) + if not os.path.exists(case_root): + os.makedirs(case_root) #-- check neon listing file for available data: available_list = check_neon_listing(valid_neon_sites) From 73a41649c91dd8eafce589009b83650259653c15 Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Thu, 19 Aug 2021 08:32:07 -0600 Subject: [PATCH 02/55] fix run_length issues --- tools/site_and_regional/run_neon.py | 37 ++++++++++++++++++++++++----- 1 file changed, 31 insertions(+), 6 deletions(-) diff --git a/tools/site_and_regional/run_neon.py b/tools/site_and_regional/run_neon.py index 41a6aca5c8..93bffb7dec 100755 --- a/tools/site_and_regional/run_neon.py +++ b/tools/site_and_regional/run_neon.py @@ -153,7 +153,7 @@ def get_parser(args, description, valid_neon_sites): ''', required = False, type = str, - default = '200Y') + default = '0Y') parser.add_argument('--start-date', help=''' @@ -190,8 +190,15 @@ def get_parser(args, description, valid_neon_sites): if "CIME_OUTPUT_ROOT" in args.case_root: args.case_root = None + if args.run_length == '0Y': + if args.run_type == 'ad': + run_length = '200Y' + elif args.run_type == 'postad': + run_length = '50Y' + else: + run_length = '4Y' - run_length = parse_isoduration(args.run_length) + run_length = parse_isoduration(run_length) return neon_sites, args.case_root, args.run_type, args.overwrite, run_length, args.base_case_root @@ -315,16 +322,26 @@ def run_case(self, base_case_root, run_type, run_length, overwrite=False): user_mods_dirs = [os.path.join(self.cesmroot,"cime_config","usermods_dirs","NEON",self.name)] expect(os.path.isdir(base_case_root), "Error base case does not exist in {}".format(base_case_root)) case_root = os.path.abspath(os.path.join(base_case_root,"..", self.name+"."+run_type)) - if os.path.isdir(case_root) and overwrite: - logger.info("---- removing the existing case -------") - shutil.rmtree(case_root) + if os.path.isdir(case_root): + if overwrite: + logger.info("---- removing the existing case -------") + shutil.rmtree(case_root) + else: + logger.warning("Case already exists in {}, not overwritting.".format(case_root)) + return + + if run_type == "postad": + adcase_root = case_root.replace('.postad','.ad') + if not os.path.isdir(adcase_root): + logger.warning("postad requested but no ad case found in {}".format(adcase_root)) + return + if not os.path.isdir(case_root): # read_only = False should not be required here with Case(base_case_root, read_only=False) as basecase: logger.info("---- cloning the base case in {}".format(case_root)) basecase.create_clone(case_root, keepexe=True, user_mods_dirs=user_mods_dirs) - with Case(case_root, read_only=False) as case: case.set_value("STOP_OPTION", "ndays") case.set_value("STOP_N", run_length) @@ -386,15 +403,23 @@ def set_ref_case(self, case): refrundir = refcase.get_value("RUNDIR") case.set_value("RUN_REFDIR", refrundir) case.set_value("RUN_REFCASE", os.path.basename(ref_case_root)) + refdate = None for reffile in glob.iglob(refrundir + "/{}{}.*.nc".format(self.name, root)): m = re.search("(\d\d\d\d-\d\d-\d\d)-\d\d\d\d\d.nc", reffile) if m: refdate = m.group(1) symlink_force(reffile, os.path.join(rundir,os.path.basename(reffile))) + + if not refdate: + logger.warning("Could not find refcase for {}".format(case_root)) + return + for rpfile in glob.iglob(refrundir + "/rpointer*"): safe_copy(rpfile, rundir) if not os.path.isdir(os.path.join(rundir, "inputdata")) and os.path.isdir(os.path.join(refrundir,"inputdata")): symlink_force(os.path.join(refrundir,"inputdata"),os.path.join(rundir,"inputdata")) + + case.set_value("RUN_REFDATE", refdate) if case_root.endswith(".postad"): case.set_value("RUN_STARTDATE", refdate) From eaa0e0eb179b834dace42ea8703e7fc64019d32e Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Thu, 19 Aug 2021 08:34:51 -0600 Subject: [PATCH 03/55] do not use mpi-serial --- tools/site_and_regional/run_neon.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/site_and_regional/run_neon.py b/tools/site_and_regional/run_neon.py index 93bffb7dec..9d94c2bb23 100755 --- a/tools/site_and_regional/run_neon.py +++ b/tools/site_and_regional/run_neon.py @@ -285,7 +285,7 @@ def build_base_case(self, cesmroot, case_root, res, compset, overwrite): if not os.path.isdir(case_path): logger.info("---- creating a base case -------") - case.create(case_path, cesmroot, compset, res, mpilib="mpi-serial", + case.create(case_path, cesmroot, compset, res, run_unsupported=True, answer="r",output_root=case_root, user_mods_dirs = user_mods_dirs, driver="nuopc") From 231658f6875aa3c50a27911fe5b107e7e9260ee3 Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Fri, 20 Aug 2021 08:23:55 -0600 Subject: [PATCH 04/55] update surface files --- .../usermods_dirs/NEON/defaults/user_nl_clm | 2 +- tools/site_and_regional/neon_finidat_upload | 108 ++++++++++++++++++ tools/site_and_regional/run_neon.py | 54 +++++---- 3 files changed, 138 insertions(+), 26 deletions(-) create mode 100755 tools/site_and_regional/neon_finidat_upload diff --git a/cime_config/usermods_dirs/NEON/defaults/user_nl_clm b/cime_config/usermods_dirs/NEON/defaults/user_nl_clm index 742cd6f65e..4bf88213a6 100644 --- a/cime_config/usermods_dirs/NEON/defaults/user_nl_clm +++ b/cime_config/usermods_dirs/NEON/defaults/user_nl_clm @@ -19,7 +19,7 @@ !---------------------------------------------------------------------------------- flanduse_timeseries = ' ' ! This isn't needed for a non transient case, but will be once we start using transient compsets -fsurdat = "$DIN_LOC_ROOT/lnd/clm2/surfdata_map/NEON/surfdata_hist_78pfts_CMIP6_simyr2000_${NEONSITE}_c210720.nc" +fsurdat = "$DIN_LOC_ROOT/lnd/clm2/surfdata_map/NEON/surfdata_hist_78pfts_CMIP6_simyr2000_${NEONSITE}_c210805.nc" model_year_align_urbantv = 2018 stream_year_first_urbantv = 2018 stream_year_last_urbantv = 2019 diff --git a/tools/site_and_regional/neon_finidat_upload b/tools/site_and_regional/neon_finidat_upload new file mode 100755 index 0000000000..7def28645b --- /dev/null +++ b/tools/site_and_regional/neon_finidat_upload @@ -0,0 +1,108 @@ +#! /usr/bin/env python3 +""" +Script to rename and upload NEON site finidat files for use in transient startup cases + +""" + +import os, sys +# Get the ctsm util tools and then the cime tools. +_CTSM_PYTHON = os.path.abspath(os.path.join(os.path.dirname(__file__), "..","..",'python')) +sys.path.insert(1, _CTSM_PYTHON) + +import boto3 +import glob +import datetime +from ctsm import add_cime_to_path +from ctsm.path_utils import path_to_ctsm_root +from standard_script_setup import * +from CIME.case import Case + + +logger = logging.getLogger(__name__) + +def get_parser(args, description, valid_neon_sites): + """ + Get Parser object for this script + """ + parser = argparse.ArgumentParser(description=description, + formatter_class=argparse.RawDescriptionHelpFormatter) + + CIME.utils.setup_standard_logging_options(parser) + + parser.print_usage = parser.print_help + + parser.add_argument('--neon-sites', + help='4-letter neon site code.', + action="store", + required=False, + choices=valid_neon_sites + ['all'], + dest="neon_sites", + default=["OSBS"], + nargs='+') + + parser.add_argument('--output-root', + help=''' + Root Directory of case results + [default: %(default)s] + ''', + action="store", + dest="output_root", + type =str, + required=False, + default=os.getcwd()) + + parser.add_argument('--file-date', + help=''' + Date of ctsm restart file(s) to upload + ''', + action="store", + dest="file_date", + required = False, + type = datetime.date.fromisoformat, + default = datetime.datetime.strptime("0268-01-01",'%Y-%m-%d')) + + + + args = CIME.utils.parse_args_and_handle_standard_logging_options(args, parser) + + if 'all' in args.neon_sites: + neon_sites = valid_neon_sites + else: + neon_sites = args.neon_sites + for site in neon_sites: + if site not in valid_neon_sites: + raise ValueError("Invalid site name {}".format(site)) + + return neon_sites, args.output_root, args.file_date + +def main(description): + """ + For each site in the site_list find the site.postad run directory and grab the latest clm restart file + from there, + """ + + if not os.path.isfile(os.path.join(os.getenv("HOME"),".aws","credentials")): + raise FileNotFoundError("User account must have valid aws credentials to run this script.") + + cesmroot = path_to_ctsm_root() + # Get the list of supported neon sites from usermods + valid_neon_sites = glob.glob(os.path.join(cesmroot,"cime_config","usermods_dirs","NEON","[!d]*")) + valid_neon_sites = [v.split('/')[-1] for v in valid_neon_sites] + + site_list, output_root, file_date = get_parser(sys.argv, description, valid_neon_sites) + for site in site_list: + rundir = None + case_path = os.path.join(output_root, site+".postad") + if os.path.isdir(case_path): + with Case(case_path) as case: + rundir = case.get_value("RUNDIR") + finidat_file = os.path.join(rundir,site+".postad.clm2.r.{}-00000.nc".format(file_date.strftime("%4Y-%m-%d"))) + if not os.path.isfile(finidat_file): + logger.warning("Could not find file {}".format(finidat_file)) + + + + +if __name__ == "__main__": + main(__doc__) + diff --git a/tools/site_and_regional/run_neon.py b/tools/site_and_regional/run_neon.py index 9d94c2bb23..7bd27329d2 100755 --- a/tools/site_and_regional/run_neon.py +++ b/tools/site_and_regional/run_neon.py @@ -19,7 +19,7 @@ 2) Make the case for the specific neon site(s). 3) Make changes to the case, for: a. AD spinup - b. post-AD spinup + b. post-AD spinup c. transient #--------------- d. SASU or Matrix spinup @@ -117,13 +117,13 @@ def get_parser(args, description, valid_neon_sites): required=False, default=None) - parser.add_argument('--case-root', + parser.add_argument('--output-root', help=''' - Root Directory of cases + Root output directory of cases [default: %(default)s] ''', action="store", - dest="case_root", + dest="output_root", type =str, required=False, default="CIME_OUTPUT_ROOT as defined in cime") @@ -187,8 +187,8 @@ def get_parser(args, description, valid_neon_sites): if site not in valid_neon_sites: raise ValueError("Invalid site name {}".format(site)) - if "CIME_OUTPUT_ROOT" in args.case_root: - args.case_root = None + if "CIME_OUTPUT_ROOT" in args.output_root: + args.output_root = None if args.run_length == '0Y': if args.run_type == 'ad': @@ -199,8 +199,10 @@ def get_parser(args, description, valid_neon_sites): run_length = '4Y' run_length = parse_isoduration(run_length) - - return neon_sites, args.case_root, args.run_type, args.overwrite, run_length, args.base_case_root + if args.base_case_root: + base_case_root = os.path.abspath(args.base_case_root) + + return neon_sites, args.output_root, args.run_type, args.overwrite, run_length, base_case_root def get_isosplit(s, split): if split in s: @@ -249,7 +251,7 @@ def __str__(self): return str(self.__class__) + '\n' + '\n'.join((str(item) + ' = ' for item in (self.__dict__))) - def build_base_case(self, cesmroot, case_root, res, compset, overwrite): + def build_base_case(self, cesmroot, output_root, res, compset, overwrite): """ Function for building a base_case to clone. To spend less time on building ctsm for the neon cases, @@ -268,11 +270,11 @@ def build_base_case(self, cesmroot, case_root, res, compset, overwrite): Flag to overwrite the case if exists """ logger.info("---- building a base case -------") - self.base_case_root = case_root + self.base_case_root = output_root user_mods_dirs = [os.path.join(cesmroot,"cime_config","usermods_dirs","NEON",self.name)] - if not case_root: - case_root = os.getcwd() - case_path = os.path.join(case_root,self.name) + if not output_root: + output_root = os.getcwd() + case_path = os.path.join(output_root,self.name) logger.info ('base_case_name : {}'.format(self.name)) logger.info ('user_mods_dir : {}'.format(user_mods_dirs[0])) @@ -286,7 +288,7 @@ def build_base_case(self, cesmroot, case_root, res, compset, overwrite): logger.info("---- creating a base case -------") case.create(case_path, cesmroot, compset, res, - run_unsupported=True, answer="r",output_root=case_root, + run_unsupported=True, answer="r",output_root=output_root, user_mods_dirs = user_mods_dirs, driver="nuopc") logger.info("---- base case created ------") @@ -312,8 +314,8 @@ def build_base_case(self, cesmroot, case_root, res, compset, overwrite): return case_path def diff_month(self): - d1 = datetime(self.end_year,self.end_month, 1) - d2 = datetime(self.start_year, self.start_month, 1) + d1 = datetime.datetime(self.end_year,self.end_month, 1) + d2 = datetime.datetime(self.start_year, self.start_month, 1) return (d1.year - d2.year) * 12 + d1.month - d2.month @@ -361,12 +363,14 @@ def run_case(self, base_case_root, run_type, run_length, overwrite=False): if run_type == "postad": self.set_ref_case(case) + case.set_value("REST_OPTION","nmonths") + case.set_value("REST_N", 12) if run_type == "transient": self.set_ref_case(case) case.set_value("STOP_OPTION","nmonths") case.set_value("STOP_N", self.diff_month()) - case.set_value("REST_N", "12") + case.set_value("REST_OPTION", "end") case.set_value("DATM_YR_ALIGN",self.start_year) case.set_value("DATM_YR_START",self.start_year) case.set_value("DATM_YR_END",self.end_year) @@ -404,12 +408,12 @@ def set_ref_case(self, case): case.set_value("RUN_REFDIR", refrundir) case.set_value("RUN_REFCASE", os.path.basename(ref_case_root)) refdate = None - for reffile in glob.iglob(refrundir + "/{}{}.*.nc".format(self.name, root)): + for reffile in glob.iglob(refrundir + "/{}{}.clm2.r.*.nc".format(self.name, root)): m = re.search("(\d\d\d\d-\d\d-\d\d)-\d\d\d\d\d.nc", reffile) if m: refdate = m.group(1) symlink_force(reffile, os.path.join(rundir,os.path.basename(reffile))) - + print("Found refdate of {}".format(refdate)) if not refdate: logger.warning("Could not find refcase for {}".format(case_root)) return @@ -552,11 +556,11 @@ def main(description): valid_neon_sites = glob.glob(os.path.join(cesmroot,"cime_config","usermods_dirs","NEON","[!d]*")) valid_neon_sites = [v.split('/')[-1] for v in valid_neon_sites] - site_list, case_root, run_type, overwrite, run_length, base_case_root = get_parser(sys.argv, description, valid_neon_sites) - if case_root: - logger.debug ("case_root : "+ case_root) - if not os.path.exists(case_root): - os.makedirs(case_root) + site_list, output_root, run_type, overwrite, run_length, base_case_root = get_parser(sys.argv, description, valid_neon_sites) + if output_root: + logger.debug ("output_root : "+ output_root) + if not os.path.exists(output_root): + os.makedirs(output_root) #-- check neon listing file for available data: available_list = check_neon_listing(valid_neon_sites) @@ -573,7 +577,7 @@ def main(description): for neon_site in available_list: if neon_site.name in site_list: if not base_case_root: - base_case_root = neon_site.build_base_case(cesmroot, case_root, res, + base_case_root = neon_site.build_base_case(cesmroot, output_root, res, compset, overwrite) logger.info ("-----------------------------------") logger.info ("Running CTSM for neon site : {}".format(neon_site.name)) From 00102738817262e92a408919281e1a2c43f735ac Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Mon, 23 Aug 2021 11:00:13 -0600 Subject: [PATCH 05/55] more run_neon improvements --- tools/site_and_regional/neon_finidat_upload | 35 ++++++++++++-- tools/site_and_regional/run_neon.py | 53 +++++++++++++++------ 2 files changed, 69 insertions(+), 19 deletions(-) diff --git a/tools/site_and_regional/neon_finidat_upload b/tools/site_and_regional/neon_finidat_upload index 7def28645b..f1598c6194 100755 --- a/tools/site_and_regional/neon_finidat_upload +++ b/tools/site_and_regional/neon_finidat_upload @@ -10,13 +10,14 @@ _CTSM_PYTHON = os.path.abspath(os.path.join(os.path.dirname(__file__), "..",".." sys.path.insert(1, _CTSM_PYTHON) import boto3 +from botocore.exceptions import ClientError import glob import datetime from ctsm import add_cime_to_path from ctsm.path_utils import path_to_ctsm_root from standard_script_setup import * from CIME.case import Case - +from CIME.utils import expect, safe_copy logger = logging.getLogger(__name__) @@ -75,6 +76,28 @@ def get_parser(args, description, valid_neon_sites): return neon_sites, args.output_root, args.file_date +def upload_file(file_name, bucket, object_name=None): + """Upload a file to an S3 bucket + + :param file_name: File to upload + :param bucket: Bucket to upload to + :param object_name: S3 object name. If not specified then file_name is used + :return: True if file was uploaded, else False + """ + + # If S3 object_name was not specified, use file_name + if object_name is None: + object_name = os.path.basename(file_name) + + # Upload the file + s3_client = boto3.client('s3') + try: + response = s3_client.upload_file(file_name, bucket, object_name) + except ClientError as e: + logger.error(e) + return False + return True + def main(description): """ For each site in the site_list find the site.postad run directory and grab the latest clm restart file @@ -88,7 +111,7 @@ def main(description): # Get the list of supported neon sites from usermods valid_neon_sites = glob.glob(os.path.join(cesmroot,"cime_config","usermods_dirs","NEON","[!d]*")) valid_neon_sites = [v.split('/')[-1] for v in valid_neon_sites] - + filedatestamp = datetime.datetime.now().date() site_list, output_root, file_date = get_parser(sys.argv, description, valid_neon_sites) for site in site_list: rundir = None @@ -96,10 +119,14 @@ def main(description): if os.path.isdir(case_path): with Case(case_path) as case: rundir = case.get_value("RUNDIR") - finidat_file = os.path.join(rundir,site+".postad.clm2.r.{}-00000.nc".format(file_date.strftime("%4Y-%m-%d"))) + basefile = site+".postad.clm2.r.{}-00000.nc".format(file_date.strftime("%4Y-%m-%d")) + finidat_file = os.path.join(rundir,basefile) if not os.path.isfile(finidat_file): logger.warning("Could not find file {}".format(finidat_file)) - + continue + newfile = basefile.replace(".postad.",".{}.".format(filedatestamp)) + + upload_file(finidat_file, 'neon-ncar-transfer', newfile) diff --git a/tools/site_and_regional/run_neon.py b/tools/site_and_regional/run_neon.py index 7bd27329d2..2b89ef6738 100755 --- a/tools/site_and_regional/run_neon.py +++ b/tools/site_and_regional/run_neon.py @@ -177,6 +177,14 @@ def get_parser(args, description, valid_neon_sites): type = datetime.date.fromisoformat, default = datetime.datetime.strptime("2020-01-01",'%Y-%m-%d')) + parser.add_argument('--run-from-postad', + help=''' + For transient runs only - should we start from the postad spinup or finidat? + ''', + action="store_true", + required = False, + default = False) + args = CIME.utils.parse_args_and_handle_standard_logging_options(args, parser) if 'all' in args.neon_sites: @@ -202,7 +210,7 @@ def get_parser(args, description, valid_neon_sites): if args.base_case_root: base_case_root = os.path.abspath(args.base_case_root) - return neon_sites, args.output_root, args.run_type, args.overwrite, run_length, base_case_root + return neon_sites, args.output_root, args.run_type, args.overwrite, run_length, base_case_root, args.run_from_postad def get_isosplit(s, split): if split in s: @@ -239,13 +247,14 @@ class NeonSite : Methods ------- """ - def __init__(self, name, start_year, end_year, start_month, end_month): + def __init__(self, name, start_year, end_year, start_month, end_month, finidat): self.name = name self.start_year= int(start_year) self.end_year = int(end_year) self.start_month = int(start_month) self.end_month = int(end_month) self.cesmroot = path_to_ctsm_root() + self.finidat = finidat def __str__(self): return str(self.__class__) + '\n' + '\n'.join((str(item) + ' = ' @@ -367,7 +376,10 @@ def run_case(self, base_case_root, run_type, run_length, overwrite=False): case.set_value("REST_N", 12) if run_type == "transient": - self.set_ref_case(case) + if self.finidat: + case.set_value("RUN_TYPE","startup") + else: + self.set_ref_case(case) case.set_value("STOP_OPTION","nmonths") case.set_value("STOP_N", self.diff_month()) case.set_value("REST_OPTION", "end") @@ -388,7 +400,7 @@ def run_case(self, base_case_root, run_type, run_length, overwrite=False): case.set_value("DATM_YR_END",self.end_year-1) - self.modify_user_nl(case_root, run_type) + self.modify_user_nl(case_root, run_type, case.get_value("RUNDIR")) case.create_namelists() case.submit() @@ -431,10 +443,13 @@ def set_ref_case(self, case): case.set_value("RUN_STARTDATE", "{yr:04d}-{mo:02d}-01".format(yr=self.start_year, mo=self.start_month)) - def modify_user_nl(self, case_root, run_type): + def modify_user_nl(self, case_root, run_type, rundir): user_nl_fname = os.path.join(case_root, "user_nl_clm") - - if run_type != "transient": + user_nl_lines = None + if run_type == "transient": + if self.finidat: + user_nl_lines = ["finidat = '{}/inputdata/lnd/ctsm/initdata/{}'".format(rundir,self.finidat)] + else: user_nl_lines = [ "hist_fincl2 = ''", "hist_mfilt = 20", @@ -442,9 +457,10 @@ def modify_user_nl(self, case_root, run_type): "hist_empty_htapes = .true.", "hist_fincl1 = 'TOTECOSYSC', 'TOTECOSYSN', 'TOTSOMC', 'TOTSOMN', 'TOTVEGC', 'TOTVEGN', 'TLAI', 'GPP', 'CPOOL', 'NPP', 'TWS', 'H2OSNO'"] - with open(user_nl_fname, "a") as fd: - for line in user_nl_lines: - fd.write("{}\n".format(line)) + if user_nl_lines: + with open(user_nl_fname, "a") as fd: + for line in user_nl_lines: + fd.write("{}\n".format(line)) @@ -480,8 +496,9 @@ def parse_neon_listing(listing_file, valid_neon_sites): df = pd.read_csv(listing_file) - #-- TODO: do we want to check for v2 in future? - + # check for finidat files for transient run + finidatlist = df[df['object'].str.contains("lnd/ctsm")] + #-- filter lines with atm/cdep df = df[df['object'].str.contains("atm/cdeps/")] @@ -521,8 +538,12 @@ def parse_neon_listing(listing_file, valid_neon_sites): logger.debug ('end_year={}'.format(end_year)) logger.debug ('start_month={}'.format(start_month)) logger.debug ('end_month={}'.format(end_month)) - - neon_site = NeonSite(site_name, start_year, end_year, start_month, end_month) + finidat = None + for line in finidatlist['object']: + if site_name in line: + finidat = line.split(',')[0].split('/')[-1] + + neon_site = NeonSite(site_name, start_year, end_year, start_month, end_month, finidat) logger.debug (neon_site) available_list.append(neon_site) @@ -556,7 +577,7 @@ def main(description): valid_neon_sites = glob.glob(os.path.join(cesmroot,"cime_config","usermods_dirs","NEON","[!d]*")) valid_neon_sites = [v.split('/')[-1] for v in valid_neon_sites] - site_list, output_root, run_type, overwrite, run_length, base_case_root = get_parser(sys.argv, description, valid_neon_sites) + site_list, output_root, run_type, overwrite, run_length, base_case_root, run_from_postad = get_parser(sys.argv, description, valid_neon_sites) if output_root: logger.debug ("output_root : "+ output_root) if not os.path.exists(output_root): @@ -576,6 +597,8 @@ def main(description): for neon_site in available_list: if neon_site.name in site_list: + if run_from_postad: + neon_site.finidat = None if not base_case_root: base_case_root = neon_site.build_base_case(cesmroot, output_root, res, compset, overwrite) From 9c67f3045a8d23b121d04a63a592e91720887027 Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Tue, 24 Aug 2021 07:37:50 -0600 Subject: [PATCH 06/55] changes prompeted by code review --- tools/site_and_regional/run_neon.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/tools/site_and_regional/run_neon.py b/tools/site_and_regional/run_neon.py index 2b89ef6738..efaf108c7f 100755 --- a/tools/site_and_regional/run_neon.py +++ b/tools/site_and_regional/run_neon.py @@ -175,7 +175,7 @@ def get_parser(args, description, valid_neon_sites): dest="end_date", required = False, type = datetime.date.fromisoformat, - default = datetime.datetime.strptime("2020-01-01",'%Y-%m-%d')) + default = datetime.datetime.strptime("2021-01-01",'%Y-%m-%d')) parser.add_argument('--run-from-postad', help=''' @@ -356,7 +356,7 @@ def run_case(self, base_case_root, run_type, run_length, overwrite=False): with Case(case_root, read_only=False) as case: case.set_value("STOP_OPTION", "ndays") case.set_value("STOP_N", run_length) - case.set_value("REST_N", 100) + case.set_value("REST_OPTION","end") case.set_value("CONTINUE_RUN", False) if run_type == "ad": @@ -372,8 +372,6 @@ def run_case(self, base_case_root, run_type, run_length, overwrite=False): if run_type == "postad": self.set_ref_case(case) - case.set_value("REST_OPTION","nmonths") - case.set_value("REST_N", 12) if run_type == "transient": if self.finidat: @@ -382,7 +380,6 @@ def run_case(self, base_case_root, run_type, run_length, overwrite=False): self.set_ref_case(case) case.set_value("STOP_OPTION","nmonths") case.set_value("STOP_N", self.diff_month()) - case.set_value("REST_OPTION", "end") case.set_value("DATM_YR_ALIGN",self.start_year) case.set_value("DATM_YR_START",self.start_year) case.set_value("DATM_YR_END",self.end_year) From 68b00593ba1fd8a573341cc14771b46ef3908c9d Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Tue, 24 Aug 2021 08:50:13 -0600 Subject: [PATCH 07/55] initialize base_case_root --- tools/site_and_regional/run_neon.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/site_and_regional/run_neon.py b/tools/site_and_regional/run_neon.py index efaf108c7f..51ae642420 100755 --- a/tools/site_and_regional/run_neon.py +++ b/tools/site_and_regional/run_neon.py @@ -207,6 +207,7 @@ def get_parser(args, description, valid_neon_sites): run_length = '4Y' run_length = parse_isoduration(run_length) + base_case_root = None if args.base_case_root: base_case_root = os.path.abspath(args.base_case_root) From e5ecb4628531f0bd2a653c0b843d543cfeb242ab Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Tue, 24 Aug 2021 11:51:15 -0600 Subject: [PATCH 08/55] add --setup-only and --no-batch options --- tools/site_and_regional/run_neon.py | 42 +++++++++++++++++++++++------ 1 file changed, 34 insertions(+), 8 deletions(-) diff --git a/tools/site_and_regional/run_neon.py b/tools/site_and_regional/run_neon.py index 51ae642420..8afb09b80c 100755 --- a/tools/site_and_regional/run_neon.py +++ b/tools/site_and_regional/run_neon.py @@ -138,6 +138,26 @@ def get_parser(args, description, valid_neon_sites): required = False, default = False) + parser.add_argument('--setup-only', + help=''' + Only setup the requested cases, do not build or run + [default: %(default)s] + ''', + action="store_true", + dest="setup_only", + required = False, + default = False) + + parser.add_argument('--no-batch', + help=''' + Run locally, do not use batch queueing system (if defined for Machine) + [default: %(default)s] + ''', + action="store_true", + dest="no_batch", + required = False, + default = False) + parser.add_argument('--run-type', help=''' Type of run to do @@ -204,6 +224,8 @@ def get_parser(args, description, valid_neon_sites): elif args.run_type == 'postad': run_length = '50Y' else: + # The transient run length is set by cdeps atm buildnml to the last date of the available tower data + # this value is not used run_length = '4Y' run_length = parse_isoduration(run_length) @@ -211,7 +233,7 @@ def get_parser(args, description, valid_neon_sites): if args.base_case_root: base_case_root = os.path.abspath(args.base_case_root) - return neon_sites, args.output_root, args.run_type, args.overwrite, run_length, base_case_root, args.run_from_postad + return neon_sites, args.output_root, args.run_type, args.overwrite, run_length, base_case_root, args.run_from_postad, args.setup_only, args.no_batch def get_isosplit(s, split): if split in s: @@ -261,7 +283,7 @@ def __str__(self): return str(self.__class__) + '\n' + '\n'.join((str(item) + ' = ' for item in (self.__dict__))) - def build_base_case(self, cesmroot, output_root, res, compset, overwrite): + def build_base_case(self, cesmroot, output_root, res, compset, overwrite=False, setup_only=False): """ Function for building a base_case to clone. To spend less time on building ctsm for the neon cases, @@ -311,6 +333,10 @@ def build_base_case(self, cesmroot, output_root, res, compset, overwrite): case.case_setup() else: case.case_setup(reset=True) + case_path = case.get_value("CASEROOT") + + if setup_only: + return case_path logger.info("---- base case build ------") # always walk through the build process to make sure it's up to date. @@ -320,7 +346,6 @@ def build_base_case(self, cesmroot, output_root, res, compset, overwrite): total = t1-t0 logger.info ("Time required to building the base case: {} s.".format(total)) # update case_path to be the full path to the base case - case_path = case.get_value("CASEROOT") return case_path def diff_month(self): @@ -330,7 +355,7 @@ def diff_month(self): - def run_case(self, base_case_root, run_type, run_length, overwrite=False): + def run_case(self, base_case_root, run_type, run_length, overwrite=False, setup_only=False, no_batch=False): user_mods_dirs = [os.path.join(self.cesmroot,"cime_config","usermods_dirs","NEON",self.name)] expect(os.path.isdir(base_case_root), "Error base case does not exist in {}".format(base_case_root)) case_root = os.path.abspath(os.path.join(base_case_root,"..", self.name+"."+run_type)) @@ -401,7 +426,8 @@ def run_case(self, base_case_root, run_type, run_length, overwrite=False): self.modify_user_nl(case_root, run_type, case.get_value("RUNDIR")) case.create_namelists() - case.submit() + if not setup_only: + case.submit(no_batch=no_batch) def set_ref_case(self, case): rundir = case.get_value("RUNDIR") @@ -575,7 +601,7 @@ def main(description): valid_neon_sites = glob.glob(os.path.join(cesmroot,"cime_config","usermods_dirs","NEON","[!d]*")) valid_neon_sites = [v.split('/')[-1] for v in valid_neon_sites] - site_list, output_root, run_type, overwrite, run_length, base_case_root, run_from_postad = get_parser(sys.argv, description, valid_neon_sites) + site_list, output_root, run_type, overwrite, run_length, base_case_root, run_from_postad, setup_only, no_batch = get_parser(sys.argv, description, valid_neon_sites) if output_root: logger.debug ("output_root : "+ output_root) if not os.path.exists(output_root): @@ -599,10 +625,10 @@ def main(description): neon_site.finidat = None if not base_case_root: base_case_root = neon_site.build_base_case(cesmroot, output_root, res, - compset, overwrite) + compset, overwrite, setup_only) logger.info ("-----------------------------------") logger.info ("Running CTSM for neon site : {}".format(neon_site.name)) - neon_site.run_case(base_case_root, run_type, run_length, overwrite) + neon_site.run_case(base_case_root, run_type, run_length, overwrite, setup_only, no_batch) if __name__ == "__main__": main(__doc__) From 39d46e7a125fb2dafa078d39f9e3d54302224156 Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Tue, 24 Aug 2021 16:11:56 -0600 Subject: [PATCH 09/55] rename neon upload script and add history file upload to it --- .../{neon_finidat_upload => neon_s3_upload} | 62 +++++++++++++++---- tools/site_and_regional/run_neon.py | 1 + 2 files changed, 51 insertions(+), 12 deletions(-) rename tools/site_and_regional/{neon_finidat_upload => neon_s3_upload} (59%) diff --git a/tools/site_and_regional/neon_finidat_upload b/tools/site_and_regional/neon_s3_upload similarity index 59% rename from tools/site_and_regional/neon_finidat_upload rename to tools/site_and_regional/neon_s3_upload index f1598c6194..447886e936 100755 --- a/tools/site_and_regional/neon_finidat_upload +++ b/tools/site_and_regional/neon_s3_upload @@ -63,6 +63,25 @@ def get_parser(args, description, valid_neon_sites): default = datetime.datetime.strptime("0268-01-01",'%Y-%m-%d')) + parser.add_argument('--upload-finidat', + help=''' + Upload the final restart files from the end of the postad run for each site. + ''', + action="store_true", + dest="upload_finidat", + required = False, + default = False) + + parser.add_argument('--upload-history', + help=''' + Upload the transient run h1 history files for each site. + ''', + action="store_true", + dest="upload_history", + required = False, + default = False) + + args = CIME.utils.parse_args_and_handle_standard_logging_options(args, parser) @@ -74,7 +93,9 @@ def get_parser(args, description, valid_neon_sites): if site not in valid_neon_sites: raise ValueError("Invalid site name {}".format(site)) - return neon_sites, args.output_root, args.file_date + expect(args.upload_finidat or args.upload_history,"Must specify at least one of --upload-finidat or --upload-history") + + return neon_sites, args.output_root, args.file_date, args.upload_finidat, args.upload_history def upload_file(file_name, bucket, object_name=None): """Upload a file to an S3 bucket @@ -92,6 +113,7 @@ def upload_file(file_name, bucket, object_name=None): # Upload the file s3_client = boto3.client('s3') try: + logger.info("Uploading file {} to {}".format(file_name, object_name)) response = s3_client.upload_file(file_name, bucket, object_name) except ClientError as e: logger.error(e) @@ -112,21 +134,37 @@ def main(description): valid_neon_sites = glob.glob(os.path.join(cesmroot,"cime_config","usermods_dirs","NEON","[!d]*")) valid_neon_sites = [v.split('/')[-1] for v in valid_neon_sites] filedatestamp = datetime.datetime.now().date() - site_list, output_root, file_date = get_parser(sys.argv, description, valid_neon_sites) + site_list, output_root, file_date, upload_finidat, upload_history = get_parser(sys.argv, description, valid_neon_sites) for site in site_list: rundir = None - case_path = os.path.join(output_root, site+".postad") - if os.path.isdir(case_path): + if upload_finidat: + logger.info("Upload finidat for {}".format(site)) + case_path = os.path.join(output_root, site+".postad") + if os.path.isdir(case_path): + with Case(case_path) as case: + rundir = case.get_value("RUNDIR") + basefile = site+".postad.clm2.r.{}-00000.nc".format(file_date.strftime("%4Y-%m-%d")) + finidat_file = os.path.join(rundir,basefile) + if not os.path.isfile(finidat_file): + logger.warning("Could not find file {}".format(finidat_file)) + continue + newfile = basefile.replace(".postad.",".{}.".format(filedatestamp)) + + upload_file(finidat_file, 'neon-ncar-transfer', os.path.join("NEON","lnd","ctsm","initdata",newfile)) + if upload_history: + logger.info("Upload history for {}".format(site)) + case_path = os.path.join(output_root, site+".transient") + if not os.path.isdir(case_path): + logger.warning("No case found in {}".format(case_path)) + continue with Case(case_path) as case: - rundir = case.get_value("RUNDIR") - basefile = site+".postad.clm2.r.{}-00000.nc".format(file_date.strftime("%4Y-%m-%d")) - finidat_file = os.path.join(rundir,basefile) - if not os.path.isfile(finidat_file): - logger.warning("Could not find file {}".format(finidat_file)) - continue - newfile = basefile.replace(".postad.",".{}.".format(filedatestamp)) + archive_dir = os.path.join(case.get_value("DOUT_S_ROOT"),"lnd","hist") + for histfile in glob.iglob(archive_dir + "/*.h1.*"): + newfile = os.path.basename(histfile) + upload_file(histfile, 'neon-ncar-transfer', os.path.join("NEON","archive",site,"lnd","hist",newfile)) + - upload_file(finidat_file, 'neon-ncar-transfer', newfile) + diff --git a/tools/site_and_regional/run_neon.py b/tools/site_and_regional/run_neon.py index 8afb09b80c..c985b8fa7d 100755 --- a/tools/site_and_regional/run_neon.py +++ b/tools/site_and_regional/run_neon.py @@ -409,6 +409,7 @@ def run_case(self, base_case_root, run_type, run_length, overwrite=False, setup_ case.set_value("DATM_YR_ALIGN",self.start_year) case.set_value("DATM_YR_START",self.start_year) case.set_value("DATM_YR_END",self.end_year) + case.set_value("CALENDAR","GREGORIAN") else: # for the spinup we want the start and end on year boundaries if self.start_month == 1: From 4d0b046992bec12ffdc2b266cd619fdf86958901 Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Wed, 25 Aug 2021 14:47:50 -0600 Subject: [PATCH 10/55] sort available site list --- tools/site_and_regional/run_neon.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/site_and_regional/run_neon.py b/tools/site_and_regional/run_neon.py index c985b8fa7d..13888314c1 100755 --- a/tools/site_and_regional/run_neon.py +++ b/tools/site_and_regional/run_neon.py @@ -600,7 +600,7 @@ def main(description): cesmroot = path_to_ctsm_root() # Get the list of supported neon sites from usermods valid_neon_sites = glob.glob(os.path.join(cesmroot,"cime_config","usermods_dirs","NEON","[!d]*")) - valid_neon_sites = [v.split('/')[-1] for v in valid_neon_sites] + valid_neon_sites = sorted([v.split('/')[-1] for v in valid_neon_sites]) site_list, output_root, run_type, overwrite, run_length, base_case_root, run_from_postad, setup_only, no_batch = get_parser(sys.argv, description, valid_neon_sites) if output_root: From 8e145cddf3f768c03209583312e35139712e8c6e Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Wed, 25 Aug 2021 17:09:00 -0600 Subject: [PATCH 11/55] dont use UNSET for data downloads --- bld/CLMBuildNamelist.pm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bld/CLMBuildNamelist.pm b/bld/CLMBuildNamelist.pm index 126683a5c8..62bc4ce6e4 100755 --- a/bld/CLMBuildNamelist.pm +++ b/bld/CLMBuildNamelist.pm @@ -4254,7 +4254,7 @@ sub check_input_files { my $pathname = $nl->get_variable_value($group, $var); # Need to strip the quotes $pathname =~ s/['"]//g; - + next if ($pathname =~ /UNSET$/); if ($input_pathname_type eq 'abs') { if ($inputdata_rootdir) { #MV $pathname =~ s:$inputdata_rootdir::; From 234bf616251e38786834ce8c4691c95d3a2323bf Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Thu, 26 Aug 2021 11:33:32 -0600 Subject: [PATCH 12/55] update with corrected start dates and new surface files --- cime_config/usermods_dirs/NEON/GUAN/shell_commands | 1 + cime_config/usermods_dirs/NEON/LAJA/shell_commands | 1 + cime_config/usermods_dirs/NEON/SJER/shell_commands | 1 + cime_config/usermods_dirs/NEON/TEAK/shell_commands | 2 ++ cime_config/usermods_dirs/NEON/YELL/shell_commands | 1 + cime_config/usermods_dirs/NEON/defaults/user_nl_clm | 2 +- 6 files changed, 7 insertions(+), 1 deletion(-) diff --git a/cime_config/usermods_dirs/NEON/GUAN/shell_commands b/cime_config/usermods_dirs/NEON/GUAN/shell_commands index 4d750b77f8..b2d1b32dbf 100644 --- a/cime_config/usermods_dirs/NEON/GUAN/shell_commands +++ b/cime_config/usermods_dirs/NEON/GUAN/shell_commands @@ -1,3 +1,4 @@ ./xmlchange NEONSITE=GUAN ./xmlchange PTS_LON=293.13112 ./xmlchange PTS_LAT=17.96882 +./xmlchange RUN_STARTDATE=2018-06-01 \ No newline at end of file diff --git a/cime_config/usermods_dirs/NEON/LAJA/shell_commands b/cime_config/usermods_dirs/NEON/LAJA/shell_commands index 330690c330..36f01cff81 100644 --- a/cime_config/usermods_dirs/NEON/LAJA/shell_commands +++ b/cime_config/usermods_dirs/NEON/LAJA/shell_commands @@ -1,3 +1,4 @@ ./xmlchange NEONSITE=LAJA ./xmlchange PTS_LON=292.92392 ./xmlchange PTS_LAT=18.02184 +./xmlchange RUN_STARTDATE=2018-05-01 \ No newline at end of file diff --git a/cime_config/usermods_dirs/NEON/SJER/shell_commands b/cime_config/usermods_dirs/NEON/SJER/shell_commands index 45de246989..3683443ec0 100644 --- a/cime_config/usermods_dirs/NEON/SJER/shell_commands +++ b/cime_config/usermods_dirs/NEON/SJER/shell_commands @@ -1,3 +1,4 @@ ./xmlchange NEONSITE=SJER ./xmlchange PTS_LON=240.267 ./xmlchange PTS_LAT=37.107117 +./xmlchange RUN_STARTDATE=2018-09-01 \ No newline at end of file diff --git a/cime_config/usermods_dirs/NEON/TEAK/shell_commands b/cime_config/usermods_dirs/NEON/TEAK/shell_commands index 53ebedc664..f3a9fd75ef 100644 --- a/cime_config/usermods_dirs/NEON/TEAK/shell_commands +++ b/cime_config/usermods_dirs/NEON/TEAK/shell_commands @@ -1,3 +1,5 @@ ./xmlchange NEONSITE=TEAK ./xmlchange PTS_LON=240.99424199999999 ./xmlchange PTS_LAT=37.006472 +# This site is missing data for first half of 2018 +./xmlchange RUN_STARTDATE=2018-06-01 \ No newline at end of file diff --git a/cime_config/usermods_dirs/NEON/YELL/shell_commands b/cime_config/usermods_dirs/NEON/YELL/shell_commands index a40ef81477..3924dff420 100644 --- a/cime_config/usermods_dirs/NEON/YELL/shell_commands +++ b/cime_config/usermods_dirs/NEON/YELL/shell_commands @@ -1,3 +1,4 @@ ./xmlchange NEONSITE=YELL ./xmlchange PTS_LON=249.45803999999998 ./xmlchange PTS_LAT=44.95597 +./xmlchange RUN_STARTDATE=2018-08-01 \ No newline at end of file diff --git a/cime_config/usermods_dirs/NEON/defaults/user_nl_clm b/cime_config/usermods_dirs/NEON/defaults/user_nl_clm index 4bf88213a6..062683cc54 100644 --- a/cime_config/usermods_dirs/NEON/defaults/user_nl_clm +++ b/cime_config/usermods_dirs/NEON/defaults/user_nl_clm @@ -19,7 +19,7 @@ !---------------------------------------------------------------------------------- flanduse_timeseries = ' ' ! This isn't needed for a non transient case, but will be once we start using transient compsets -fsurdat = "$DIN_LOC_ROOT/lnd/clm2/surfdata_map/NEON/surfdata_hist_78pfts_CMIP6_simyr2000_${NEONSITE}_c210805.nc" +fsurdat = "$DIN_LOC_ROOT/lnd/clm2/surfdata_map/NEON/surfdata_hist_78pfts_CMIP6_simyr2000_${NEONSITE}_c210824.nc" model_year_align_urbantv = 2018 stream_year_first_urbantv = 2018 stream_year_last_urbantv = 2019 From d28115c15fd01fa14ebfe8c6b5d4dd3afac0dcba Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Thu, 26 Aug 2021 14:57:53 -0600 Subject: [PATCH 13/55] update cdeps and comments --- Externals.cfg | 2 +- tools/site_and_regional/run_neon.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/Externals.cfg b/Externals.cfg index 0c149001f1..eab26763bc 100644 --- a/Externals.cfg +++ b/Externals.cfg @@ -48,7 +48,7 @@ local_path = components/cmeps required = True [cdeps] -tag = cdeps0.12.19 +tag = cdeps0.12.22 protocol = git repo_url = https://github.com/ESCOMP/CDEPS.git local_path = components/cdeps diff --git a/tools/site_and_regional/run_neon.py b/tools/site_and_regional/run_neon.py index 13888314c1..dd8f10167e 100755 --- a/tools/site_and_regional/run_neon.py +++ b/tools/site_and_regional/run_neon.py @@ -200,6 +200,7 @@ def get_parser(args, description, valid_neon_sites): parser.add_argument('--run-from-postad', help=''' For transient runs only - should we start from the postad spinup or finidat? + By default start from finidat, if this flag is used the postad run must be available. ''', action="store_true", required = False, @@ -380,6 +381,7 @@ def run_case(self, base_case_root, run_type, run_length, overwrite=False, setup_ basecase.create_clone(case_root, keepexe=True, user_mods_dirs=user_mods_dirs) with Case(case_root, read_only=False) as case: + # in order to avoid the complication of leap years we always set the run_length in units of days. case.set_value("STOP_OPTION", "ndays") case.set_value("STOP_N", run_length) case.set_value("REST_OPTION","end") From 098501d618594ff1a033a7a38ef82da403051703 Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Tue, 31 Aug 2021 07:28:26 -0600 Subject: [PATCH 14/55] add user_nl_datm_streams to neon defaults --- .../usermods_dirs/NEON/defaults/user_nl_datm_streams | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 cime_config/usermods_dirs/NEON/defaults/user_nl_datm_streams diff --git a/cime_config/usermods_dirs/NEON/defaults/user_nl_datm_streams b/cime_config/usermods_dirs/NEON/defaults/user_nl_datm_streams new file mode 100644 index 0000000000..41ddbfa611 --- /dev/null +++ b/cime_config/usermods_dirs/NEON/defaults/user_nl_datm_streams @@ -0,0 +1,6 @@ +presaero.SSP3-7.0:datafiles = $DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/aero/aerodep_clm_SSP370_b.e21.BWSSP370cmip6.f09_g17.CMIP6-SSP3-7.0-WACCM.001_2018-2030_monthly_0.9x1.25_c210826.nc +presaero.SSP3-7.0:year_first=2018 +presaero.SSP3-7.0:year_last=2030 +presaero.SSP3-7.0:year_align=2018 +presaero.SSP3-7.0:dtlimit=30 + From 8d9074345dfa63b1f5ed518fe4f76ce0a51a26bd Mon Sep 17 00:00:00 2001 From: Negin Sobhani Date: Tue, 31 Aug 2021 13:01:31 -0600 Subject: [PATCH 15/55] interpolate missing values, fix organic calculations. --- .../modify_singlept_site_neon.py | 88 ++++++++++++++++++- 1 file changed, 87 insertions(+), 1 deletion(-) diff --git a/tools/site_and_regional/modify_singlept_site_neon.py b/tools/site_and_regional/modify_singlept_site_neon.py index 7b469641f1..16509ee6b8 100755 --- a/tools/site_and_regional/modify_singlept_site_neon.py +++ b/tools/site_and_regional/modify_singlept_site_neon.py @@ -45,6 +45,7 @@ import argparse import requests +import logging import numpy as np import pandas as pd import xarray as xr @@ -405,6 +406,34 @@ def download_file(url, fname): print('File '+fname+'was not available on the neon server:'+ url) +def fill_interpolate (f2, var, method): + """ + Function to interpolate a variable in a + xarray dataset a specific method + """ + print ("=====================================") + print ("Filling in ", var, "with interpolation (method ="+method+").") + print ("Variable before filling : ") + print (f2[var]) + + tmp_df = pd.DataFrame (f2[var].values.ravel()) + + # -- tested a few of these and these two gave the best answers: + tmp_df = tmp_df.interpolate(method = method, limit_direction ='both') + #tmp_df = tmp_df.interpolate(method ='spline',order = 5, limit_direction ='both') + #-- + ###tmp_df = tmp_df.interpolate(method="pad", limit=5, limit_direction = 'forward') + ###tmp_df = tmp_df.interpolate(method ='spline',order=2, limit_direction ='backward') + + tmp = tmp_df.to_numpy() + + soil_levels = f2[var].size + for soil_lev in range(soil_levels): + f2[var][soil_lev] = tmp[soil_lev].reshape(1,1) + + print ("Variable after filling : ") + print (f2[var]) + print ("=====================================") def main(): @@ -503,17 +532,74 @@ def main(): f2['PCT_SAND'][soil_lev] = df['sandTotal'][bin_index[soil_lev]] bulk_den = df['bulkDensExclCoarseFrag'][bin_index[soil_lev]] carbon_tot = df['carbonTot'][bin_index[soil_lev]] + estimated_oc = df['estimatedOC'][bin_index[soil_lev]] + + if (estimated_oc > carbon_tot): + estimated_oc = carbon_tot + #print ("carbon_tot:", carbon_tot) layer_depth = df['biogeoBottomDepth'][bin_index[soil_lev]] - df['biogeoTopDepth'][bin_index[soil_lev]] - f2['ORGANIC'][soil_lev] = carbon_tot * bulk_den * 0.1 / layer_depth * 100 / 0.58 + #f2['ORGANIC'][soil_lev] = carbon_tot * bulk_den * 0.1 / layer_depth * 100 / 0.58 + f2['ORGANIC'][soil_lev] = estimated_oc * bulk_den / 0.58 print ("bin_index:", bin_index[soil_lev]) print ("layer_depth:", layer_depth) print ("carbon_tot:",carbon_tot) + print ("organic carbon :", estimated_oc) print ("bulk_den:",bulk_den) print ("organic=carbon_tot*bulk_den*0.1/layer_depth * 100/0.58 ") print ("organic:", f2['ORGANIC'][soil_lev].values) + #if f2['ORGANIC'][soil_lev].values == nan: + # f2['ORGANIC'][soil_lev].values == + print ("--------------------------") + + method = 'linear' + fill_interpolate (f2, 'PCT_CLAY', method) + fill_interpolate (f2, 'PCT_SAND', method) + fill_interpolate (f2, 'ORGANIC', method) + + + # -- Interpolate? + + + """ + print ("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~") + print (f2['ORGANIC'].values) + temp = pd.DataFrame (f2['ORGANIC'].values.ravel()) + print ("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~") + #temp = temp.interpolate(method ='linear', limit_direction ='backward') + temp = temp.interpolate(method ='spline', order = 2, limit_direction ='both') + print (temp.shape) + #print (temp.bfill(inplace='True')) + + temp = temp.to_numpy() + print ("temp:") + print (temp) + #temp = pd.DataFrame (f2['ORGANIC'].values.ravel()) + #temp = temp.interpolate(method='spline', order=2) + #print ("temp:") + #print (temp) + print ("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~") + + for soil_lev in range(soil_levels): + print ("......") + print ("soil_lev:", soil_lev) + print (temp[soil_lev]) + + f2['ORGANIC'][soil_lev] = temp[soil_lev].reshape(1,1) + print (temp[soil_lev].reshape(1,1)) + print (f2['ORGANIC'][soil_lev].values) + + print (f2['ORGANIC'].values) + print ("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~") + print ("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~") + print ("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~") + print (f2['PCT_CLAY']) + print (f2['PCT_SAND']) + + """ + #TODO : max depth for neon sites from WW (zbedrock) # Update zbedrock if neon observation don't make it down to 2m depth # zbedrock = neon depth if neon does not make to 2m depth From cdc3a5e3adb18fe74541052d493d401486f5523e Mon Sep 17 00:00:00 2001 From: Negin Sobhani Date: Tue, 31 Aug 2021 13:14:33 -0600 Subject: [PATCH 16/55] clean ups and adding comments. --- .../modify_singlept_site_neon.py | 72 ++++--------------- 1 file changed, 15 insertions(+), 57 deletions(-) diff --git a/tools/site_and_regional/modify_singlept_site_neon.py b/tools/site_and_regional/modify_singlept_site_neon.py index 16509ee6b8..1be7ccc722 100755 --- a/tools/site_and_regional/modify_singlept_site_neon.py +++ b/tools/site_and_regional/modify_singlept_site_neon.py @@ -237,8 +237,7 @@ def find_soil_structure (surf_file): #TODO: What if not cheyenne? Self-contained depth info. print ('------------') - print (surf_file) - print (type(surf_file)) + print ("surf_file : ", surf_file) f1 = xr.open_dataset(surf_file) print ('------------') #print (f1.attrs["Soil_texture_raw_data_file_name"]) @@ -457,6 +456,7 @@ def main(): parent_dir = os.path.dirname(current_dir) clone_dir = os.path.abspath(os.path.join(__file__ ,"../../..")) neon_dir = os.path.join(clone_dir,"neon_surffiles") + print("Present Directory", current_dir) #-- download neon data if needed @@ -466,7 +466,6 @@ def main(): df = pd.read_csv (neon_file) # -- Read surface dataset files - print (type(surf_file)) print ("surf_file:", surf_file) f1 = xr.open_dataset(surf_file) @@ -521,6 +520,7 @@ def main(): "{0:.2f}".format(soil_bot[b]), "-------------") ''' + #-- update fields with neon f2= f1 soil_levels = f2['PCT_CLAY'].size @@ -530,27 +530,27 @@ def main(): print (df['clayTotal'][bin_index[soil_lev]]) f2['PCT_CLAY'][soil_lev] = df['clayTotal'][bin_index[soil_lev]] f2['PCT_SAND'][soil_lev] = df['sandTotal'][bin_index[soil_lev]] + bulk_den = df['bulkDensExclCoarseFrag'][bin_index[soil_lev]] carbon_tot = df['carbonTot'][bin_index[soil_lev]] estimated_oc = df['estimatedOC'][bin_index[soil_lev]] + # -- estimated_oc in neon data is rounded to the nearest integer. + # -- Check to make sure the rounded oc is not higher than carbon_tot. + # -- Use carbon_tot if estimated_oc is bigger than carbon_tot. if (estimated_oc > carbon_tot): estimated_oc = carbon_tot - - #print ("carbon_tot:", carbon_tot) + layer_depth = df['biogeoBottomDepth'][bin_index[soil_lev]] - df['biogeoTopDepth'][bin_index[soil_lev]] - #f2['ORGANIC'][soil_lev] = carbon_tot * bulk_den * 0.1 / layer_depth * 100 / 0.58 + f2['ORGANIC'][soil_lev] = estimated_oc * bulk_den / 0.58 - print ("bin_index:", bin_index[soil_lev]) - print ("layer_depth:", layer_depth) - print ("carbon_tot:",carbon_tot) - print ("organic carbon :", estimated_oc) - print ("bulk_den:",bulk_den) - print ("organic=carbon_tot*bulk_den*0.1/layer_depth * 100/0.58 ") - print ("organic:", f2['ORGANIC'][soil_lev].values) - #if f2['ORGANIC'][soil_lev].values == nan: - # f2['ORGANIC'][soil_lev].values == + print ("bin_index : ", bin_index[soil_lev]) + print ("layer_depth : ", layer_depth) + print ("carbon_tot : ",carbon_tot) + print ("estimated_oc : ", estimated_oc) + print ("bulk_den : ",bulk_den) + print ("organic :", f2['ORGANIC'][soil_lev].values) print ("--------------------------") @@ -562,44 +562,6 @@ def main(): # -- Interpolate? - - """ - print ("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~") - print (f2['ORGANIC'].values) - temp = pd.DataFrame (f2['ORGANIC'].values.ravel()) - print ("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~") - #temp = temp.interpolate(method ='linear', limit_direction ='backward') - temp = temp.interpolate(method ='spline', order = 2, limit_direction ='both') - print (temp.shape) - #print (temp.bfill(inplace='True')) - - temp = temp.to_numpy() - print ("temp:") - print (temp) - #temp = pd.DataFrame (f2['ORGANIC'].values.ravel()) - #temp = temp.interpolate(method='spline', order=2) - #print ("temp:") - #print (temp) - print ("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~") - - for soil_lev in range(soil_levels): - print ("......") - print ("soil_lev:", soil_lev) - print (temp[soil_lev]) - - f2['ORGANIC'][soil_lev] = temp[soil_lev].reshape(1,1) - print (temp[soil_lev].reshape(1,1)) - print (f2['ORGANIC'][soil_lev].values) - - print (f2['ORGANIC'].values) - print ("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~") - print ("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~") - print ("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~") - print (f2['PCT_CLAY']) - print (f2['PCT_SAND']) - - """ - #TODO : max depth for neon sites from WW (zbedrock) # Update zbedrock if neon observation don't make it down to 2m depth # zbedrock = neon depth if neon does not make to 2m depth @@ -613,10 +575,6 @@ def main(): f2['zbedrock'].values[:,:]=obs_bot.iloc[-1] zb_flag = True - - #if (f2['zbedrock'] Date: Tue, 31 Aug 2021 13:18:01 -0600 Subject: [PATCH 17/55] clean up formattings. --- .../modify_singlept_site_neon.py | 555 ++++++++++-------- 1 file changed, 312 insertions(+), 243 deletions(-) diff --git a/tools/site_and_regional/modify_singlept_site_neon.py b/tools/site_and_regional/modify_singlept_site_neon.py index 1be7ccc722..2333b92ce6 100755 --- a/tools/site_and_regional/modify_singlept_site_neon.py +++ b/tools/site_and_regional/modify_singlept_site_neon.py @@ -30,11 +30,11 @@ ------------------------------------------------------------------- """ # TODO (NS) -#--[] If file not found run subset_data.py -#--[] Clean up imports for both codes... -#--[] Check against a list of valid names. -#--[] List of valid neon sites for all scripts come from one place. -#--[] zbedrock +# --[] If file not found run subset_data.py +# --[] Clean up imports for both codes... +# --[] Check against a list of valid names. +# --[] List of valid neon sites for all scripts come from one place. +# --[] zbedrock # Import libraries from __future__ import print_function @@ -57,72 +57,118 @@ myname = getuser() -#-- valid neon sites -valid_neon_sites = ['ABBY','BARR','BART','BLAN', - 'BONA','CLBJ','CPER','DCFS', - 'DEJU','DELA','DSNY','GRSM', - 'GUAN','HARV','HEAL','JERC', - 'JORN','KONA','KONZ','LAJA', - 'LENO','MLBS','MOAB','NIWO', - 'NOGP','OAES','ONAQ','ORNL', - 'OSBS','PUUM','RMNP','SCBI', - 'SERC','SJER','SOAP','SRER', - 'STEI','STER','TALL','TEAK', - 'TOOL','TREE','UKFS','UNDE', - 'WOOD','WREF','YELL' - ] - - -def get_parser(): +# -- valid neon sites +valid_neon_sites = [ + "ABBY", + "BARR", + "BART", + "BLAN", + "BONA", + "CLBJ", + "CPER", + "DCFS", + "DEJU", + "DELA", + "DSNY", + "GRSM", + "GUAN", + "HARV", + "HEAL", + "JERC", + "JORN", + "KONA", + "KONZ", + "LAJA", + "LENO", + "MLBS", + "MOAB", + "NIWO", + "NOGP", + "OAES", + "ONAQ", + "ORNL", + "OSBS", + "PUUM", + "RMNP", + "SCBI", + "SERC", + "SJER", + "SOAP", + "SRER", + "STEI", + "STER", + "TALL", + "TEAK", + "TOOL", + "TREE", + "UKFS", + "UNDE", + "WOOD", + "WREF", + "YELL", +] + + +def get_parser(): """ Get parser object for this script. """ - parser = argparse.ArgumentParser(description=__doc__, - formatter_class=argparse.RawDescriptionHelpFormatter) + parser = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + ) parser.print_usage = parser.print_help - parser.add_argument('--neon_site', - help='4-letter neon site code.', - action="store", - dest="site_name", - choices=valid_neon_sites, - required=True) - parser.add_argument('--surf_dir', - help=''' + parser.add_argument( + "--neon_site", + help="4-letter neon site code.", + action="store", + dest="site_name", + choices=valid_neon_sites, + required=True, + ) + parser.add_argument( + "--surf_dir", + help=""" Directory of single point surface dataset. [default: %(default)s] - ''', - action="store", - dest="surf_dir", - type =str, - required=False, - default="/glade/scratch/"+myname+"/single_point/") - parser.add_argument('--out_dir', - help=''' + """, + action="store", + dest="surf_dir", + type=str, + required=False, + default="/glade/scratch/" + myname + "/single_point/", + ) + parser.add_argument( + "--out_dir", + help=""" Directory to write updated single point surface dataset. [default: %(default)s] - ''', - action="store", - dest="out_dir", - type =str, - required=False, - default="/glade/scratch/"+myname+"/single_point_neon_updated/") - parser.add_argument('-d','--debug', - help=''' + """, + action="store", + dest="out_dir", + type=str, + required=False, + default="/glade/scratch/" + myname + "/single_point_neon_updated/", + ) + parser.add_argument( + "-d", + "--debug", + help=""" Debug mode will print more information. [default: %(default)s] - ''', - action="store_true", - dest="debug", - default=False) + """, + action="store_true", + dest="debug", + default=False, + ) return parser def get_neon(neon_dir, site_name): """ - Function for finding neon data file + Function for finding neon data file and download from neon server if the file does not exits. @@ -133,50 +179,58 @@ def get_neon(neon_dir, site_name): Raises: Error if the download was not successful (exit code:404). In case the data does not exist in the neon server or if - neon server is down. + neon server is down. Returns: neon_file (str) : complete file name of the downloaded data """ - #-- create directory if not exists + # -- create directory if not exists if not os.path.exists(neon_dir): os.makedirs(neon_dir) neon_file = os.path.join(neon_dir, site_name + "_surfaceData.csv") - #-- Download the file if it does not exits + # -- Download the file if it does not exits if os.path.isfile(neon_file): - print('neon file for', site_name, 'already exists! ') - print('Skipping download from neon for', site_name,'...') + print("neon file for", site_name, "already exists! ") + print("Skipping download from neon for", site_name, "...") else: - print('------------------------------------------------') - print('Beginning download from neon server for', site_name,'...') - - url = ('https://s3.data.neonscience.org/neon-ncar/NEON/surf_files/v1/' - +site_name+'_surfaceData.csv') + print("------------------------------------------------") + print("Beginning download from neon server for", site_name, "...") + + url = ( + "https://s3.data.neonscience.org/neon-ncar/NEON/surf_files/v1/" + + site_name + + "_surfaceData.csv" + ) response = requests.get(url) - with open(neon_file, 'wb') as f: + with open(neon_file, "wb") as f: f.write(response.content) - #-- Check if download status_code + # -- Check if download status_code if response.status_code == 200: - print('Download finished successfully for', site_name) + print("Download finished successfully for", site_name) elif response.status_code == 404: - sys.exit('Data for this site '+site_name+ - ' was not available on the neon server:'+ url) - - print('Download exit status code: ',response.status_code) - print('Downloaded file type : ',response.headers['content-type']) - print('Downloaded file encoding : ',response.encoding) - print('------------------------------------------------') + sys.exit( + "Data for this site " + + site_name + + " was not available on the neon server:" + + url + ) + + print("Download exit status code: ", response.status_code) + print("Downloaded file type : ", response.headers["content-type"]) + print("Downloaded file encoding : ", response.encoding) + print("------------------------------------------------") response.close() return neon_file -def find_surffile (surf_dir, site_name): + +def find_surffile(surf_dir, site_name): """ Function for finding and choosing surface file for a neon site. @@ -188,32 +242,36 @@ def find_surffile (surf_dir, site_name): site_name (str): 4 letter neon site name Raises: - Error if the surface data for the site is not created + Error if the surface data for the site is not created Returns: surf_file (str): name of the surface dataset file """ - #sf_name = "surfdata_hist_16pfts_Irrig_CMIP6_simyr2000_"+site_name+"*.nc" - sf_name = "surfdata_hist_78pfts_CMIP6_simyr2000_"+site_name+"*.nc" - #surf_file = glob.glob(os.path.join(surf_dir,sf_name)) - surf_file = glob.glob(surf_dir+"/"+sf_name) + # sf_name = "surfdata_hist_16pfts_Irrig_CMIP6_simyr2000_"+site_name+"*.nc" + sf_name = "surfdata_hist_78pfts_CMIP6_simyr2000_" + site_name + "*.nc" + # surf_file = glob.glob(os.path.join(surf_dir,sf_name)) + surf_file = glob.glob(surf_dir + "/" + sf_name) - if len(surf_file)>1: - print ("The following files found :", *surf_file, sep='\n- ') - print ("The latest file is chosen :", surf_file[-1]) + if len(surf_file) > 1: + print("The following files found :", *surf_file, sep="\n- ") + print("The latest file is chosen :", surf_file[-1]) surf_file = surf_file[-1] - elif len(surf_file)==1: - print ("File found : ") - print (surf_file) + elif len(surf_file) == 1: + print("File found : ") + print(surf_file) surf_file = surf_file[0] else: - sys.exit('Surface data for this site '+site_name+ - 'was not found:'+ surf_file,'.', - '\n','Please run ./subset_data.py for this site.') + sys.exit( + "Surface data for this site " + site_name + "was not found:" + surf_file, + ".", + "\n", + "Please run ./subset_data.py for this site.", + ) return surf_file -def find_soil_structure (surf_file): + +def find_soil_structure(surf_file): """ Function for finding surface dataset soil strucutre using surface data metadata. @@ -228,41 +286,49 @@ def find_soil_structure (surf_file): surf_file (str): single point surface data filename Raises: - error if the soil layer strucutre file does not exist + error if the soil layer strucutre file does not exist Returns: soil_bot : array of soil layers top depths soil_top : array of soil layers bottom depths """ - #TODO: What if not cheyenne? Self-contained depth info. + # TODO: What if not cheyenne? Self-contained depth info. - print ('------------') - print ("surf_file : ", surf_file) + print("------------") + print("surf_file : ", surf_file) f1 = xr.open_dataset(surf_file) - print ('------------') - #print (f1.attrs["Soil_texture_raw_data_file_name"]) + print("------------") + # print (f1.attrs["Soil_texture_raw_data_file_name"]) clm_input_dir = "/glade/p/cesmdata/cseg/inputdata/lnd/clm2/rawdata/" - surf_soildepth_file = os.path.join(clm_input_dir, - f1.attrs["Soil_texture_raw_data_file_name"]) - - if os.path.exists (surf_soildepth_file): - print ("\n\n Reading", surf_soildepth_file, - "for surface data soil structure information:") + surf_soildepth_file = os.path.join( + clm_input_dir, f1.attrs["Soil_texture_raw_data_file_name"] + ) + + if os.path.exists(surf_soildepth_file): + print( + "\n\n Reading", + surf_soildepth_file, + "for surface data soil structure information:", + ) f1_soildepth = xr.open_dataset(surf_soildepth_file) - print (f1_soildepth['DZSOI']) - soil_bot = f1_soildepth['DZSOI'].values + print(f1_soildepth["DZSOI"]) + soil_bot = f1_soildepth["DZSOI"].values - #-- soil layer top + # -- soil layer top soil_top = soil_bot[:-1] - soil_top = np.insert(soil_top,0, 0) + soil_top = np.insert(soil_top, 0, 0) else: - sys.exit('Cannot find soil structure file : '+surf_soildepth_file+ - 'for the surface dataset.') + sys.exit( + "Cannot find soil structure file : " + + surf_soildepth_file + + "for the surface dataset." + ) return soil_bot, soil_top + def update_metadata(nc, surf_file, neon_file, zb_flag): """ Function for updating modified surface dataset @@ -279,21 +345,22 @@ def update_metadata(nc, surf_file, neon_file, zb_flag): today = date.today() today_string = today.strftime("%Y-%m-%d") - nc.attrs['Updated_on'] = today_string - nc.attrs['Updated_by'] = myname - nc.attrs['Updated_with'] = os.path.abspath(__file__) - nc.attrs['Updated_from'] = surf_file - nc.attrs['Updated_using'] = neon_file + nc.attrs["Updated_on"] = today_string + nc.attrs["Updated_by"] = myname + nc.attrs["Updated_with"] = os.path.abspath(__file__) + nc.attrs["Updated_from"] = surf_file + nc.attrs["Updated_using"] = neon_file if zb_flag: - nc.attrs['Updated_fields'] = "PCT_CLAY, PCT_SAND, ORGANIC, zbedrock" - #nc.attrs['Updated_fields'] = ['PCT_CLAY','PCT_SAND','ORGANIC','zbedrock'] + nc.attrs["Updated_fields"] = "PCT_CLAY, PCT_SAND, ORGANIC, zbedrock" + # nc.attrs['Updated_fields'] = ['PCT_CLAY','PCT_SAND','ORGANIC','zbedrock'] else: - nc.attrs['Updated_fields'] = "PCT_CLAY, PCT_SAND, ORGANIC" + nc.attrs["Updated_fields"] = "PCT_CLAY, PCT_SAND, ORGANIC" # nc.attrs['Updated_fields'] = ['PCT_CLAY','PCT_SAND','ORGANIC'] return nc -def update_time_tag (fname_in): + +def update_time_tag(fname_in): """ Function for updating time tag on surface dataset files. @@ -315,13 +382,14 @@ def update_time_tag (fname_in): basename = os.path.basename(fname_in) cend = -10 - if ( basename[cend] == "c" ): - cend = cend - 1 - if ( (basename[cend] != ".") and (basename[cend] != "_") ): - sys.exit( "Trouble figuring out where to add tag to filename:"+fname_in ) + if basename[cend] == "c": + cend = cend - 1 + if (basename[cend] != ".") and (basename[cend] != "_"): + sys.exit("Trouble figuring out where to add tag to filename:" + fname_in) + + fname_out = basename[:cend] + "_" + "c" + today_string + ".nc" + return fname_out - fname_out = basename[:cend]+"_"+"c"+today_string+".nc" - return(fname_out) def sort_print_soil_layers(obs_bot, soil_bot): """ @@ -329,58 +397,54 @@ def sort_print_soil_layers(obs_bot, soil_bot): original surface dataset and neon dataset. Args: - obs_bot : array of neon soil layers bottom depths + obs_bot : array of neon soil layers bottom depths soil_bot : array of soil layers bottom depths """ - obs_bot_df = pd.DataFrame({'depth':obs_bot,'type':"obs"}) - soil_bot_df = pd.DataFrame({'depth':soil_bot,'type':"sfc"}) - depth_df = pd.concat([obs_bot_df,soil_bot_df]) + obs_bot_df = pd.DataFrame({"depth": obs_bot, "type": "obs"}) + soil_bot_df = pd.DataFrame({"depth": soil_bot, "type": "sfc"}) + depth_df = pd.concat([obs_bot_df, soil_bot_df]) - depth_df = depth_df.sort_values('depth') + depth_df = depth_df.sort_values("depth") - space = ' ' - print ("================================", - "================================") + space = " " + print("================================", "================================") - print (" Neon data soil structure: " , - " Surface data soil structure: ") + print(" Neon data soil structure: ", " Surface data soil structure: ") - print ("================================", - "================================") + print("================================", "================================") for index, row in depth_df.iterrows(): - if row['type']=="obs": - print ("-------------", - "{0:.3f}".format(row['depth']), - "------------") + if row["type"] == "obs": + print("-------------", "{0:.3f}".format(row["depth"]), "------------") else: - print (33*space+ - "-------------", - "{0:.3f}".format(row['depth']), - "-----------") + print( + 33 * space + "-------------", + "{0:.3f}".format(row["depth"]), + "-----------", + ) + + print("--------------------------------" + "--------------------------------") - print ("--------------------------------"+ - "--------------------------------") def check_neon_time(): """ A function to download and parse neon listing file. """ - listing_file = 'listing.csv' - url = 'https://neon-ncar.s3.data.neonscience.org/listing.csv' + listing_file = "listing.csv" + url = "https://neon-ncar.s3.data.neonscience.org/listing.csv" download_file(url, listing_file) df = pd.read_csv(listing_file) - df = df[df['object'].str.contains("_surfaceData.csv")] - #df=df.join(df['object'].str.split("/", expand=True)) - dict_out = dict(zip(df['object'],df['last_modified'])) - print (dict_out) - #df_out = df[['object','6','last_modified']] - #print (df['last_modified']) - #print (df_out) - #print (df['last_modified'].to_datetime()) + df = df[df["object"].str.contains("_surfaceData.csv")] + # df=df.join(df['object'].str.split("/", expand=True)) + dict_out = dict(zip(df["object"], df["last_modified"])) + print(dict_out) + # df_out = df[['object','6','last_modified']] + # print (df['last_modified']) + # print (df_out) + # print (df['last_modified'].to_datetime()) return dict_out @@ -388,39 +452,39 @@ def download_file(url, fname): """ Function to download a file. Args: - url (str): + url (str): url of the file for downloading - fname (str) : + fname (str) : file name to save the downloaded file. """ response = requests.get(url) - with open(fname, 'wb') as f: + with open(fname, "wb") as f: f.write(response.content) - #-- Check if download status_code + # -- Check if download status_code if response.status_code == 200: - print('Download finished successfully for', fname,'.') + print("Download finished successfully for", fname, ".") elif response.status_code == 404: - print('File '+fname+'was not available on the neon server:'+ url) + print("File " + fname + "was not available on the neon server:" + url) -def fill_interpolate (f2, var, method): +def fill_interpolate(f2, var, method): """ Function to interpolate a variable in a xarray dataset a specific method """ - print ("=====================================") - print ("Filling in ", var, "with interpolation (method ="+method+").") - print ("Variable before filling : ") - print (f2[var]) + print("=====================================") + print("Filling in ", var, "with interpolation (method =" + method + ").") + print("Variable before filling : ") + print(f2[var]) - tmp_df = pd.DataFrame (f2[var].values.ravel()) + tmp_df = pd.DataFrame(f2[var].values.ravel()) # -- tested a few of these and these two gave the best answers: - tmp_df = tmp_df.interpolate(method = method, limit_direction ='both') - #tmp_df = tmp_df.interpolate(method ='spline',order = 5, limit_direction ='both') - #-- + tmp_df = tmp_df.interpolate(method=method, limit_direction="both") + # tmp_df = tmp_df.interpolate(method ='spline',order = 5, limit_direction ='both') + # -- ###tmp_df = tmp_df.interpolate(method="pad", limit=5, limit_direction = 'forward') ###tmp_df = tmp_df.interpolate(method ='spline',order=2, limit_direction ='backward') @@ -428,76 +492,76 @@ def fill_interpolate (f2, var, method): soil_levels = f2[var].size for soil_lev in range(soil_levels): - f2[var][soil_lev] = tmp[soil_lev].reshape(1,1) + f2[var][soil_lev] = tmp[soil_lev].reshape(1, 1) + + print("Variable after filling : ") + print(f2[var]) + print("=====================================") - print ("Variable after filling : ") - print (f2[var]) - print ("=====================================") def main(): args = get_parser().parse_args() - #-- debugging option + # -- debugging option if args.debug: logging.basicConfig(level=logging.DEBUG) file_time = check_neon_time() - #-- specify site from which to extract data - site_name=args.site_name + # -- specify site from which to extract data + site_name = args.site_name - #-- Look for surface data + # -- Look for surface data surf_dir = args.surf_dir - surf_file = find_surffile (surf_dir, site_name) + surf_file = find_surffile(surf_dir, site_name) - #-- directory structure + # -- directory structure current_dir = os.getcwd() - parent_dir = os.path.dirname(current_dir) - clone_dir = os.path.abspath(os.path.join(__file__ ,"../../..")) - neon_dir = os.path.join(clone_dir,"neon_surffiles") + parent_dir = os.path.dirname(current_dir) + clone_dir = os.path.abspath(os.path.join(__file__, "../../..")) + neon_dir = os.path.join(clone_dir, "neon_surffiles") - print("Present Directory", current_dir) + print("Present Directory", current_dir) - #-- download neon data if needed + # -- download neon data if needed neon_file = get_neon(neon_dir, site_name) - #-- Read neon data - df = pd.read_csv (neon_file) + # -- Read neon data + df = pd.read_csv(neon_file) # -- Read surface dataset files - print ("surf_file:", surf_file) + print("surf_file:", surf_file) f1 = xr.open_dataset(surf_file) # -- Find surface dataset soil depth information - soil_bot, soil_top = find_soil_structure (surf_file) + soil_bot, soil_top = find_soil_structure(surf_file) # -- Find surface dataset soil levels - # TODO: how? NS uses metadata on file to find + # TODO: how? NS uses metadata on file to find # soil strucure # better suggestion by WW to write dzsoi to neon surface dataset # This todo needs to go to the subset_data # TODO Will: if I sum them up , are they 3.5? (m) YES - print ("soil_top:", soil_top) - print ("soil_bot:", soil_bot) - print ("Sum of soil top depths :", sum(soil_top)) - print ("Sum of soil bottom depths :",sum(soil_bot)) + print("soil_top:", soil_top) + print("soil_bot:", soil_bot) + print("Sum of soil top depths :", sum(soil_top)) + print("Sum of soil bottom depths :", sum(soil_bot)) soil_top = np.cumsum(soil_top) soil_bot = np.cumsum(soil_bot) - soil_mid = 0.5*(soil_bot - soil_top)+soil_top - #print ("Cumulative sum of soil bottom depths :", sum(soil_bot)) + soil_mid = 0.5 * (soil_bot - soil_top) + soil_top + # print ("Cumulative sum of soil bottom depths :", sum(soil_bot)) - obs_top = df['biogeoTopDepth']/100 - obs_bot = df['biogeoBottomDepth']/100 + obs_top = df["biogeoTopDepth"] / 100 + obs_bot = df["biogeoBottomDepth"] / 100 # -- Mapping surface dataset and neon soil levels - bins = df['biogeoTopDepth']/100 - bin_index = np.digitize(soil_mid, bins)-1 + bins = df["biogeoTopDepth"] / 100 + bin_index = np.digitize(soil_mid, bins) - 1 - - ''' + """ print ("================================") print (" Neon data soil structure: ") print ("================================") @@ -519,50 +583,51 @@ def main(): print ("-------------", "{0:.2f}".format(soil_bot[b]), "-------------") - ''' + """ - #-- update fields with neon - f2= f1 - soil_levels = f2['PCT_CLAY'].size + # -- update fields with neon + f2 = f1 + soil_levels = f2["PCT_CLAY"].size for soil_lev in range(soil_levels): - print ("--------------------------") - print ("soil_lev:",soil_lev) - print (df['clayTotal'][bin_index[soil_lev]]) - f2['PCT_CLAY'][soil_lev] = df['clayTotal'][bin_index[soil_lev]] - f2['PCT_SAND'][soil_lev] = df['sandTotal'][bin_index[soil_lev]] + print("--------------------------") + print("soil_lev:", soil_lev) + print(df["clayTotal"][bin_index[soil_lev]]) + f2["PCT_CLAY"][soil_lev] = df["clayTotal"][bin_index[soil_lev]] + f2["PCT_SAND"][soil_lev] = df["sandTotal"][bin_index[soil_lev]] - bulk_den = df['bulkDensExclCoarseFrag'][bin_index[soil_lev]] - carbon_tot = df['carbonTot'][bin_index[soil_lev]] - estimated_oc = df['estimatedOC'][bin_index[soil_lev]] + bulk_den = df["bulkDensExclCoarseFrag"][bin_index[soil_lev]] + carbon_tot = df["carbonTot"][bin_index[soil_lev]] + estimated_oc = df["estimatedOC"][bin_index[soil_lev]] # -- estimated_oc in neon data is rounded to the nearest integer. # -- Check to make sure the rounded oc is not higher than carbon_tot. # -- Use carbon_tot if estimated_oc is bigger than carbon_tot. - if (estimated_oc > carbon_tot): + if estimated_oc > carbon_tot: estimated_oc = carbon_tot - layer_depth = df['biogeoBottomDepth'][bin_index[soil_lev]] - df['biogeoTopDepth'][bin_index[soil_lev]] - - f2['ORGANIC'][soil_lev] = estimated_oc * bulk_den / 0.58 + layer_depth = ( + df["biogeoBottomDepth"][bin_index[soil_lev]] + - df["biogeoTopDepth"][bin_index[soil_lev]] + ) - print ("bin_index : ", bin_index[soil_lev]) - print ("layer_depth : ", layer_depth) - print ("carbon_tot : ",carbon_tot) - print ("estimated_oc : ", estimated_oc) - print ("bulk_den : ",bulk_den) - print ("organic :", f2['ORGANIC'][soil_lev].values) - print ("--------------------------") + f2["ORGANIC"][soil_lev] = estimated_oc * bulk_den / 0.58 + print("bin_index : ", bin_index[soil_lev]) + print("layer_depth : ", layer_depth) + print("carbon_tot : ", carbon_tot) + print("estimated_oc : ", estimated_oc) + print("bulk_den : ", bulk_den) + print("organic :", f2["ORGANIC"][soil_lev].values) + print("--------------------------") - method = 'linear' - fill_interpolate (f2, 'PCT_CLAY', method) - fill_interpolate (f2, 'PCT_SAND', method) - fill_interpolate (f2, 'ORGANIC', method) - + method = "linear" + fill_interpolate(f2, "PCT_CLAY", method) + fill_interpolate(f2, "PCT_SAND", method) + fill_interpolate(f2, "ORGANIC", method) # -- Interpolate? - #TODO : max depth for neon sites from WW (zbedrock) + # TODO : max depth for neon sites from WW (zbedrock) # Update zbedrock if neon observation don't make it down to 2m depth # zbedrock = neon depth if neon does not make to 2m depth @@ -570,31 +635,35 @@ def main(): zb_flag = False - if (obs_bot.iloc[-1] Date: Tue, 31 Aug 2021 13:34:08 -0600 Subject: [PATCH 18/55] more cleanups --- .../modify_singlept_site_neon.py | 38 +++++++++---------- 1 file changed, 18 insertions(+), 20 deletions(-) diff --git a/tools/site_and_regional/modify_singlept_site_neon.py b/tools/site_and_regional/modify_singlept_site_neon.py index 2333b92ce6..883427b642 100755 --- a/tools/site_and_regional/modify_singlept_site_neon.py +++ b/tools/site_and_regional/modify_singlept_site_neon.py @@ -12,8 +12,8 @@ This script will do the following: - Download neon data for the specified site if it does not exist - in the specified directory. -- Modify surface dataset with downloaded data (neon-specific). + in the specified directory : (i.e. ../../../neon_surffiles). +- Modify surface dataset with downloaded data. ------------------------------------------------------------------- Instructions for running on Cheyenne/Casper: @@ -28,13 +28,14 @@ To see the available options: ./modify_singlept_site_neon.py --help ------------------------------------------------------------------- +Example: + ./modify_singlept_site_neon.py --neon_site PUUM --debug +------------------------------------------------------------------- """ # TODO (NS) -# --[] If file not found run subset_data.py -# --[] Clean up imports for both codes... +# --[] If subset file not found run subset_data.py # --[] Check against a list of valid names. # --[] List of valid neon sites for all scripts come from one place. -# --[] zbedrock # Import libraries from __future__ import print_function @@ -168,7 +169,7 @@ def get_parser(): def get_neon(neon_dir, site_name): """ - Function for finding neon data file + Function for finding neon data files and download from neon server if the file does not exits. @@ -234,6 +235,7 @@ def find_surffile(surf_dir, site_name): """ Function for finding and choosing surface file for a neon site. + These files are created using ./subset_data.py script. In case multiple files exist for the neon site, it will choose the file created the latest. @@ -352,10 +354,8 @@ def update_metadata(nc, surf_file, neon_file, zb_flag): nc.attrs["Updated_using"] = neon_file if zb_flag: nc.attrs["Updated_fields"] = "PCT_CLAY, PCT_SAND, ORGANIC, zbedrock" - # nc.attrs['Updated_fields'] = ['PCT_CLAY','PCT_SAND','ORGANIC','zbedrock'] else: nc.attrs["Updated_fields"] = "PCT_CLAY, PCT_SAND, ORGANIC" - # nc.attrs['Updated_fields'] = ['PCT_CLAY','PCT_SAND','ORGANIC'] return nc @@ -410,7 +410,7 @@ def sort_print_soil_layers(obs_bot, soil_bot): space = " " print("================================", "================================") - print(" Neon data soil structure: ", " Surface data soil structure: ") + print(" Neon data soil structure: ", " Surface data soil structure: ") print("================================", "================================") @@ -430,6 +430,10 @@ def sort_print_soil_layers(obs_bot, soil_bot): def check_neon_time(): """ A function to download and parse neon listing file. + + Returns: + dict_out (str) : + dictionary of *_surfaceData.csv files with the last modified """ listing_file = "listing.csv" url = "https://neon-ncar.s3.data.neonscience.org/listing.csv" @@ -443,7 +447,6 @@ def check_neon_time(): print(dict_out) # df_out = df[['object','6','last_modified']] # print (df['last_modified']) - # print (df_out) # print (df['last_modified'].to_datetime()) return dict_out @@ -476,17 +479,15 @@ def fill_interpolate(f2, var, method): """ print("=====================================") print("Filling in ", var, "with interpolation (method =" + method + ").") + print("Variable before filling : ") print(f2[var]) tmp_df = pd.DataFrame(f2[var].values.ravel()) - # -- tested a few of these and these two gave the best answers: tmp_df = tmp_df.interpolate(method=method, limit_direction="both") - # tmp_df = tmp_df.interpolate(method ='spline',order = 5, limit_direction ='both') - # -- - ###tmp_df = tmp_df.interpolate(method="pad", limit=5, limit_direction = 'forward') - ###tmp_df = tmp_df.interpolate(method ='spline',order=2, limit_direction ='backward') + # tmp_df = tmp_df.interpolate(method ='spline',order = 2, limit_direction ='both') + # tmp_df = tmp_df.interpolate(method="pad", limit=5, limit_direction = 'forward') tmp = tmp_df.to_numpy() @@ -620,17 +621,14 @@ def main(): print("organic :", f2["ORGANIC"][soil_lev].values) print("--------------------------") + # -- Interpolate missing values method = "linear" fill_interpolate(f2, "PCT_CLAY", method) fill_interpolate(f2, "PCT_SAND", method) fill_interpolate(f2, "ORGANIC", method) - # -- Interpolate? - - # TODO : max depth for neon sites from WW (zbedrock) - # Update zbedrock if neon observation don't make it down to 2m depth - # zbedrock = neon depth if neon does not make to 2m depth + # -- Update zbedrock if neon observation does not make it down to 2m depth rock_thresh = 2 zb_flag = False From d8d9af829b6a79d10a999e0d62f277c5ef37defa Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Tue, 31 Aug 2021 17:02:51 -0600 Subject: [PATCH 19/55] update surfdata files --- cime_config/usermods_dirs/NEON/defaults/user_nl_clm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cime_config/usermods_dirs/NEON/defaults/user_nl_clm b/cime_config/usermods_dirs/NEON/defaults/user_nl_clm index 062683cc54..05e3f8095d 100644 --- a/cime_config/usermods_dirs/NEON/defaults/user_nl_clm +++ b/cime_config/usermods_dirs/NEON/defaults/user_nl_clm @@ -19,7 +19,7 @@ !---------------------------------------------------------------------------------- flanduse_timeseries = ' ' ! This isn't needed for a non transient case, but will be once we start using transient compsets -fsurdat = "$DIN_LOC_ROOT/lnd/clm2/surfdata_map/NEON/surfdata_hist_78pfts_CMIP6_simyr2000_${NEONSITE}_c210824.nc" +fsurdat = "$DIN_LOC_ROOT/lnd/clm2/surfdata_map/NEON/surfdata_hist_78pfts_CMIP6_simyr2000_${NEONSITE}_c210831.nc" model_year_align_urbantv = 2018 stream_year_first_urbantv = 2018 stream_year_last_urbantv = 2019 From 1770da539bf8a0e5727ec776095c8ededcfcd488 Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Wed, 1 Sep 2021 13:48:01 -0600 Subject: [PATCH 20/55] update cdeps external --- Externals.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Externals.cfg b/Externals.cfg index eab26763bc..f3a49b610d 100644 --- a/Externals.cfg +++ b/Externals.cfg @@ -48,7 +48,7 @@ local_path = components/cmeps required = True [cdeps] -tag = cdeps0.12.22 +tag = cdeps0.12.24 protocol = git repo_url = https://github.com/ESCOMP/CDEPS.git local_path = components/cdeps From f8bfd174931073b25115d1cc5a8723ab58df80a3 Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Thu, 2 Sep 2021 15:45:18 -0600 Subject: [PATCH 21/55] add a rerun option --- tools/site_and_regional/run_neon.py | 60 ++++++++++++++++++++--------- 1 file changed, 42 insertions(+), 18 deletions(-) diff --git a/tools/site_and_regional/run_neon.py b/tools/site_and_regional/run_neon.py index dd8f10167e..2e52b27a2a 100755 --- a/tools/site_and_regional/run_neon.py +++ b/tools/site_and_regional/run_neon.py @@ -148,6 +148,16 @@ def get_parser(args, description, valid_neon_sites): required = False, default = False) + parser.add_argument('--rerun', + help=''' + If the case exists but does not appear to be complete, restart it. + [default: %(default)s] + ''', + action="store_true", + dest="rerun", + required = False, + default = False) + parser.add_argument('--no-batch', help=''' Run locally, do not use batch queueing system (if defined for Machine) @@ -234,7 +244,7 @@ def get_parser(args, description, valid_neon_sites): if args.base_case_root: base_case_root = os.path.abspath(args.base_case_root) - return neon_sites, args.output_root, args.run_type, args.overwrite, run_length, base_case_root, args.run_from_postad, args.setup_only, args.no_batch + return neon_sites, args.output_root, args.run_type, args.overwrite, run_length, base_case_root, args.run_from_postad, args.setup_only, args.no_batch, args.rerun def get_isosplit(s, split): if split in s: @@ -356,17 +366,26 @@ def diff_month(self): - def run_case(self, base_case_root, run_type, run_length, overwrite=False, setup_only=False, no_batch=False): + def run_case(self, base_case_root, run_type, run_length, overwrite=False, setup_only=False, no_batch=False, rerun=False): user_mods_dirs = [os.path.join(self.cesmroot,"cime_config","usermods_dirs","NEON",self.name)] expect(os.path.isdir(base_case_root), "Error base case does not exist in {}".format(base_case_root)) case_root = os.path.abspath(os.path.join(base_case_root,"..", self.name+"."+run_type)) + rundir = None if os.path.isdir(case_root): - if overwrite: - logger.info("---- removing the existing case -------") - shutil.rmtree(case_root) - else: - logger.warning("Case already exists in {}, not overwritting.".format(case_root)) - return + with Case(case_root, read_only=False) as case: + rundir = case.get_value("RUNDIR") + if overwrite: + logger.info("---- removing the existing case -------") + shutil.rmtree(case_root) + elif rerun: + if os.path.isfile(os.path.join(rundir,"ESMF_Profile.summary")): + logger.info("Case {} appears to be complete, not rerunning.".format(case_root)) + elif not setup_only: + logger.info("Resubmitting case {}".format(case_root)) + case.submit(no_batch=no_batch) + else: + logger.warning("Case already exists in {}, not overwritting.".format(case_root)) + return if run_type == "postad": adcase_root = case_root.replace('.postad','.ad') @@ -392,7 +411,6 @@ def run_case(self, base_case_root, run_type, run_length, overwrite=False, setup_ case.set_value("CLM_ACCELERATED_SPINUP","on") case.set_value("RUN_REFDATE", "0018-01-01") case.set_value("RUN_STARTDATE", "0018-01-01") - case.set_value("JOB_WALLCLOCK_TIME", "12:00:00", subgroup="case.run") else: case.set_value("CLM_FORCE_COLDSTART","off") case.set_value("CLM_ACCELERATED_SPINUP","off") @@ -405,7 +423,8 @@ def run_case(self, base_case_root, run_type, run_length, overwrite=False, setup_ if self.finidat: case.set_value("RUN_TYPE","startup") else: - self.set_ref_case(case) + if not self.set_ref_case(case): + return case.set_value("STOP_OPTION","nmonths") case.set_value("STOP_N", self.diff_month()) case.set_value("DATM_YR_ALIGN",self.start_year) @@ -425,8 +444,10 @@ def run_case(self, base_case_root, run_type, run_length, overwrite=False, setup_ else: case.set_value("DATM_YR_END",self.end_year-1) - - self.modify_user_nl(case_root, run_type, case.get_value("RUNDIR")) + if not rundir: + rundir = case.get_value("RUNDIR") + + self.modify_user_nl(case_root, run_type, rundir) case.create_namelists() if not setup_only: @@ -441,7 +462,10 @@ def set_ref_case(self, case): else: ref_case_root = case_root.replace(".transient",".postad") root = ".postad" - expect(os.path.isdir(ref_case_root), "ERROR: spinup must be completed first, could not find directory {}".format(ref_case_root)) + if not os.path.isdir(ref_case_root): + logger.warning("ERROR: spinup must be completed first, could not find directory {}".format(ref_case_root)) + return False + with Case(ref_case_root) as refcase: refrundir = refcase.get_value("RUNDIR") case.set_value("RUN_REFDIR", refrundir) @@ -452,10 +476,10 @@ def set_ref_case(self, case): if m: refdate = m.group(1) symlink_force(reffile, os.path.join(rundir,os.path.basename(reffile))) - print("Found refdate of {}".format(refdate)) + logger.info("Found refdate of {}".format(refdate)) if not refdate: logger.warning("Could not find refcase for {}".format(case_root)) - return + return False for rpfile in glob.iglob(refrundir + "/rpointer*"): safe_copy(rpfile, rundir) @@ -468,7 +492,7 @@ def set_ref_case(self, case): case.set_value("RUN_STARTDATE", refdate) else: case.set_value("RUN_STARTDATE", "{yr:04d}-{mo:02d}-01".format(yr=self.start_year, mo=self.start_month)) - + return True def modify_user_nl(self, case_root, run_type, rundir): user_nl_fname = os.path.join(case_root, "user_nl_clm") @@ -604,7 +628,7 @@ def main(description): valid_neon_sites = glob.glob(os.path.join(cesmroot,"cime_config","usermods_dirs","NEON","[!d]*")) valid_neon_sites = sorted([v.split('/')[-1] for v in valid_neon_sites]) - site_list, output_root, run_type, overwrite, run_length, base_case_root, run_from_postad, setup_only, no_batch = get_parser(sys.argv, description, valid_neon_sites) + site_list, output_root, run_type, overwrite, run_length, base_case_root, run_from_postad, setup_only, no_batch, rerun = get_parser(sys.argv, description, valid_neon_sites) if output_root: logger.debug ("output_root : "+ output_root) if not os.path.exists(output_root): @@ -631,7 +655,7 @@ def main(description): compset, overwrite, setup_only) logger.info ("-----------------------------------") logger.info ("Running CTSM for neon site : {}".format(neon_site.name)) - neon_site.run_case(base_case_root, run_type, run_length, overwrite, setup_only, no_batch) + neon_site.run_case(base_case_root, run_type, run_length, overwrite, setup_only, no_batch, rerun) if __name__ == "__main__": main(__doc__) From 235f524afe931152a64e202d5f67844d2630f43a Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Tue, 7 Sep 2021 09:33:33 -0600 Subject: [PATCH 22/55] explicitly check_input_data in script --- tools/site_and_regional/run_neon.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tools/site_and_regional/run_neon.py b/tools/site_and_regional/run_neon.py index 2e52b27a2a..2df04b71d2 100755 --- a/tools/site_and_regional/run_neon.py +++ b/tools/site_and_regional/run_neon.py @@ -450,6 +450,8 @@ def run_case(self, base_case_root, run_type, run_length, overwrite=False, setup_ self.modify_user_nl(case_root, run_type, rundir) case.create_namelists() + # explicitly run check_input_data + case.check_all_input_data() if not setup_only: case.submit(no_batch=no_batch) From 4a0373ff5ff33c77066a4b428fc8d33f8a194125 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Tue, 7 Sep 2021 14:59:18 -0600 Subject: [PATCH 23/55] Update cime to a hash on Jim's branch --- Externals.cfg | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Externals.cfg b/Externals.cfg index f3a49b610d..33dcf87f82 100644 --- a/Externals.cfg +++ b/Externals.cfg @@ -36,8 +36,10 @@ required = True [cime] local_path = cime protocol = git -repo_url = https://github.com/ESMCI/cime -tag = cime6.0.4 +repo_url = https://github.com/jedwards4b/cime +#tag = cime6.0.8 +#branch = better_neon_data_handling +hash = cb00de44c1ccab4e5e8d8769b474a44675d4fd82 required = True [cmeps] From b5661afb0afdeeb5af9e0f656afc76ba1d7ed319 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Thu, 9 Sep 2021 14:27:58 -0600 Subject: [PATCH 24/55] Update cime to latest tag with the needed update in it --- Externals.cfg | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/Externals.cfg b/Externals.cfg index 33dcf87f82..dd0d2b4446 100644 --- a/Externals.cfg +++ b/Externals.cfg @@ -36,10 +36,8 @@ required = True [cime] local_path = cime protocol = git -repo_url = https://github.com/jedwards4b/cime -#tag = cime6.0.8 -#branch = better_neon_data_handling -hash = cb00de44c1ccab4e5e8d8769b474a44675d4fd82 +repo_url = https://github.com/ESCOMP/cime +tag = cime6.0.9 required = True [cmeps] From a82688c992fb2b63af8b097ab5f17286fe6fab10 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Mon, 13 Sep 2021 13:28:09 -0600 Subject: [PATCH 25/55] Correct CIME org --- Externals.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Externals.cfg b/Externals.cfg index dd0d2b4446..497241dfc3 100644 --- a/Externals.cfg +++ b/Externals.cfg @@ -36,7 +36,7 @@ required = True [cime] local_path = cime protocol = git -repo_url = https://github.com/ESCOMP/cime +repo_url = https://github.com/ESMCI/cime tag = cime6.0.9 required = True From 25ab14f6597e6f2779a9ab2fe91583a0f6701aa1 Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Thu, 30 Sep 2021 14:59:34 -0600 Subject: [PATCH 26/55] update ndep file for NEON --- cime_config/usermods_dirs/NEON/defaults/user_nl_clm | 1 + 1 file changed, 1 insertion(+) diff --git a/cime_config/usermods_dirs/NEON/defaults/user_nl_clm b/cime_config/usermods_dirs/NEON/defaults/user_nl_clm index 05e3f8095d..818b013b9c 100644 --- a/cime_config/usermods_dirs/NEON/defaults/user_nl_clm +++ b/cime_config/usermods_dirs/NEON/defaults/user_nl_clm @@ -30,6 +30,7 @@ model_year_align_popdens = 2018 stream_year_first_popdens = 2018 stream_year_last_popdens = 2019 stream_fldfilename_lightng = '$DIN_LOC_ROOT/atm/datm7/NASA_LIS/clmforc.Li_2016_climo1995-2013.360x720.lnfm_Total_NEONarea_c210625.nc' +stream_fldfilename_ndep = '$DIN_LOC_ROOT/lnd/clm2/ndepdata/fndep_clm_f09_g17.CMIP6-SSP3-7.0-WACCM_2018-2030_monthly_c210826.nc' ! h1 output stream hist_fincl2 = 'AR','ELAI','FCEV','FCTR','FGEV','FIRA','FSA','FSH','GPP','H2OSOI', 'HR','SNOW_DEPTH','TBOT','TSOI','SOILC_vr','FV','NET_NMIN_vr' From 99f7812528fc50eda992d9e1c2a1116878a391a4 Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Tue, 5 Oct 2021 14:48:44 -0600 Subject: [PATCH 27/55] reduce stdout volume by changing default to --silent --- tools/site_and_regional/run_neon.py | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/tools/site_and_regional/run_neon.py b/tools/site_and_regional/run_neon.py index 2df04b71d2..bedd79801a 100755 --- a/tools/site_and_regional/run_neon.py +++ b/tools/site_and_regional/run_neon.py @@ -244,6 +244,11 @@ def get_parser(args, description, valid_neon_sites): if args.base_case_root: base_case_root = os.path.abspath(args.base_case_root) + # Reduce output level for this script unless --debug or --verbose is provided on the command line + if not args.debug and not args.verbose: + root_logger = logging.getLogger() + root_logger.setLevel(logging.WARN) + return neon_sites, args.output_root, args.run_type, args.overwrite, run_length, base_case_root, args.run_from_postad, args.setup_only, args.no_batch, args.rerun def get_isosplit(s, split): @@ -312,7 +317,7 @@ def build_base_case(self, cesmroot, output_root, res, compset, overwrite=False, overwrite (bool) : Flag to overwrite the case if exists """ - logger.info("---- building a base case -------") + print("---- building a base case -------") self.base_case_root = output_root user_mods_dirs = [os.path.join(cesmroot,"cime_config","usermods_dirs","NEON",self.name)] if not output_root: @@ -328,19 +333,19 @@ def build_base_case(self, cesmroot, output_root, res, compset, overwrite=False, with Case(case_path, read_only=False) as case: if not os.path.isdir(case_path): - logger.info("---- creating a base case -------") + print("---- creating a base case -------") case.create(case_path, cesmroot, compset, res, run_unsupported=True, answer="r",output_root=output_root, user_mods_dirs = user_mods_dirs, driver="nuopc") - logger.info("---- base case created ------") + print("---- base case created ------") #--change any config for base_case: #case.set_value("RUN_TYPE","startup") - logger.info("---- base case setup ------") + print("---- base case setup ------") case.case_setup() else: case.case_setup(reset=True) @@ -349,13 +354,13 @@ def build_base_case(self, cesmroot, output_root, res, compset, overwrite=False, if setup_only: return case_path - logger.info("---- base case build ------") + print("---- base case build ------") # always walk through the build process to make sure it's up to date. t0 = time.time() build.case_build(case_path, case=case) t1 = time.time() total = t1-t0 - logger.info ("Time required to building the base case: {} s.".format(total)) + print ("Time required to building the base case: {} s.".format(total)) # update case_path to be the full path to the base case return case_path @@ -375,13 +380,13 @@ def run_case(self, base_case_root, run_type, run_length, overwrite=False, setup_ with Case(case_root, read_only=False) as case: rundir = case.get_value("RUNDIR") if overwrite: - logger.info("---- removing the existing case -------") + print("---- removing the existing case -------") shutil.rmtree(case_root) elif rerun: if os.path.isfile(os.path.join(rundir,"ESMF_Profile.summary")): - logger.info("Case {} appears to be complete, not rerunning.".format(case_root)) + print("Case {} appears to be complete, not rerunning.".format(case_root)) elif not setup_only: - logger.info("Resubmitting case {}".format(case_root)) + print("Resubmitting case {}".format(case_root)) case.submit(no_batch=no_batch) else: logger.warning("Case already exists in {}, not overwritting.".format(case_root)) @@ -396,7 +401,7 @@ def run_case(self, base_case_root, run_type, run_length, overwrite=False, setup_ if not os.path.isdir(case_root): # read_only = False should not be required here with Case(base_case_root, read_only=False) as basecase: - logger.info("---- cloning the base case in {}".format(case_root)) + print("---- cloning the base case in {}".format(case_root)) basecase.create_clone(case_root, keepexe=True, user_mods_dirs=user_mods_dirs) with Case(case_root, read_only=False) as case: @@ -631,6 +636,7 @@ def main(description): valid_neon_sites = sorted([v.split('/')[-1] for v in valid_neon_sites]) site_list, output_root, run_type, overwrite, run_length, base_case_root, run_from_postad, setup_only, no_batch, rerun = get_parser(sys.argv, description, valid_neon_sites) + if output_root: logger.debug ("output_root : "+ output_root) if not os.path.exists(output_root): From 7b352c60b5b9971de11c0c7cefd201e27780ce0e Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Tue, 12 Oct 2021 08:18:30 -0600 Subject: [PATCH 28/55] improve overwrite logic --- tools/site_and_regional/run_neon.py | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/tools/site_and_regional/run_neon.py b/tools/site_and_regional/run_neon.py index bedd79801a..db9dfea6a3 100755 --- a/tools/site_and_regional/run_neon.py +++ b/tools/site_and_regional/run_neon.py @@ -326,11 +326,13 @@ def build_base_case(self, cesmroot, output_root, res, compset, overwrite=False, logger.info ('base_case_name : {}'.format(self.name)) logger.info ('user_mods_dir : {}'.format(user_mods_dirs[0])) - + import pdb + pdb.set_trace() if overwrite and os.path.isdir(case_path): - logger.info ("Removing the existing case at: {}".format(case_path)) + print ("Removing the existing case at: {}".format(case_path)) shutil.rmtree(case_path) - + sleep(1) + with Case(case_path, read_only=False) as case: if not os.path.isdir(case_path): print("---- creating a base case -------") @@ -377,20 +379,21 @@ def run_case(self, base_case_root, run_type, run_length, overwrite=False, setup_ case_root = os.path.abspath(os.path.join(base_case_root,"..", self.name+"."+run_type)) rundir = None if os.path.isdir(case_root): - with Case(case_root, read_only=False) as case: - rundir = case.get_value("RUNDIR") - if overwrite: - print("---- removing the existing case -------") - shutil.rmtree(case_root) - elif rerun: + if overwrite: + print("---- removing the existing case -------") + shutil.rmtree(case_root) + elif rerun: + with Case(case_root, read_only=False) as case: + rundir = case.get_value("RUNDIR") if os.path.isfile(os.path.join(rundir,"ESMF_Profile.summary")): print("Case {} appears to be complete, not rerunning.".format(case_root)) elif not setup_only: print("Resubmitting case {}".format(case_root)) case.submit(no_batch=no_batch) - else: - logger.warning("Case already exists in {}, not overwritting.".format(case_root)) - return + return + else: + logger.warning("Case already exists in {}, not overwritting.".format(case_root)) + return if run_type == "postad": adcase_root = case_root.replace('.postad','.ad') From 157a2efd7a9e13e84c48030526a8510dcdef0ef3 Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Tue, 12 Oct 2021 08:23:18 -0600 Subject: [PATCH 29/55] clean up debug statements --- tools/site_and_regional/run_neon.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tools/site_and_regional/run_neon.py b/tools/site_and_regional/run_neon.py index db9dfea6a3..020bc2e8ee 100755 --- a/tools/site_and_regional/run_neon.py +++ b/tools/site_and_regional/run_neon.py @@ -326,12 +326,10 @@ def build_base_case(self, cesmroot, output_root, res, compset, overwrite=False, logger.info ('base_case_name : {}'.format(self.name)) logger.info ('user_mods_dir : {}'.format(user_mods_dirs[0])) - import pdb - pdb.set_trace() + if overwrite and os.path.isdir(case_path): print ("Removing the existing case at: {}".format(case_path)) shutil.rmtree(case_path) - sleep(1) with Case(case_path, read_only=False) as case: if not os.path.isdir(case_path): From 5afa96ce37f17083a7f172c079d5c46e495b226d Mon Sep 17 00:00:00 2001 From: Negin Sobhani Date: Tue, 2 Nov 2021 05:04:32 -0600 Subject: [PATCH 30/55] updates regarding organic and other changes --- .../modify_singlept_site_neon.py | 604 +++++++++++------- 1 file changed, 369 insertions(+), 235 deletions(-) diff --git a/tools/site_and_regional/modify_singlept_site_neon.py b/tools/site_and_regional/modify_singlept_site_neon.py index 7b469641f1..075fbe8a1b 100755 --- a/tools/site_and_regional/modify_singlept_site_neon.py +++ b/tools/site_and_regional/modify_singlept_site_neon.py @@ -12,8 +12,8 @@ This script will do the following: - Download neon data for the specified site if it does not exist - in the specified directory. -- Modify surface dataset with downloaded data (neon-specific). + in the specified directory : (i.e. ../../../neon_surffiles). +- Modify surface dataset with downloaded data. ------------------------------------------------------------------- Instructions for running on Cheyenne/Casper: @@ -28,13 +28,14 @@ To see the available options: ./modify_singlept_site_neon.py --help ------------------------------------------------------------------- +Example: + ./modify_singlept_site_neon.py --neon_site PUUM --debug +------------------------------------------------------------------- """ # TODO (NS) -#--[] If file not found run subset_data.py -#--[] Clean up imports for both codes... -#--[] Check against a list of valid names. -#--[] List of valid neon sites for all scripts come from one place. -#--[] zbedrock +# --[] If subset file not found run subset_data.py +# --[] List of valid neon sites for all scripts come from one place. +# --[] Download files only when available. # Import libraries from __future__ import print_function @@ -45,6 +46,7 @@ import argparse import requests +import logging import numpy as np import pandas as pd import xarray as xr @@ -56,72 +58,118 @@ myname = getuser() -#-- valid neon sites -valid_neon_sites = ['ABBY','BARR','BART','BLAN', - 'BONA','CLBJ','CPER','DCFS', - 'DEJU','DELA','DSNY','GRSM', - 'GUAN','HARV','HEAL','JERC', - 'JORN','KONA','KONZ','LAJA', - 'LENO','MLBS','MOAB','NIWO', - 'NOGP','OAES','ONAQ','ORNL', - 'OSBS','PUUM','RMNP','SCBI', - 'SERC','SJER','SOAP','SRER', - 'STEI','STER','TALL','TEAK', - 'TOOL','TREE','UKFS','UNDE', - 'WOOD','WREF','YELL' - ] - - -def get_parser(): +# -- valid neon sites +valid_neon_sites = [ + "ABBY", + "BARR", + "BART", + "BLAN", + "BONA", + "CLBJ", + "CPER", + "DCFS", + "DEJU", + "DELA", + "DSNY", + "GRSM", + "GUAN", + "HARV", + "HEAL", + "JERC", + "JORN", + "KONA", + "KONZ", + "LAJA", + "LENO", + "MLBS", + "MOAB", + "NIWO", + "NOGP", + "OAES", + "ONAQ", + "ORNL", + "OSBS", + "PUUM", + "RMNP", + "SCBI", + "SERC", + "SJER", + "SOAP", + "SRER", + "STEI", + "STER", + "TALL", + "TEAK", + "TOOL", + "TREE", + "UKFS", + "UNDE", + "WOOD", + "WREF", + "YELL", +] + + +def get_parser(): """ Get parser object for this script. """ - parser = argparse.ArgumentParser(description=__doc__, - formatter_class=argparse.RawDescriptionHelpFormatter) + parser = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + ) parser.print_usage = parser.print_help - parser.add_argument('--neon_site', - help='4-letter neon site code.', - action="store", - dest="site_name", - choices=valid_neon_sites, - required=True) - parser.add_argument('--surf_dir', - help=''' + parser.add_argument( + "--neon_site", + help="4-letter neon site code.", + action="store", + dest="site_name", + choices=valid_neon_sites, + required=True, + ) + parser.add_argument( + "--surf_dir", + help=""" Directory of single point surface dataset. [default: %(default)s] - ''', - action="store", - dest="surf_dir", - type =str, - required=False, - default="/glade/scratch/"+myname+"/single_point/") - parser.add_argument('--out_dir', - help=''' + """, + action="store", + dest="surf_dir", + type=str, + required=False, + default="/glade/scratch/" + myname + "/single_point/", + ) + parser.add_argument( + "--out_dir", + help=""" Directory to write updated single point surface dataset. [default: %(default)s] - ''', - action="store", - dest="out_dir", - type =str, - required=False, - default="/glade/scratch/"+myname+"/single_point_neon_updated/") - parser.add_argument('-d','--debug', - help=''' + """, + action="store", + dest="out_dir", + type=str, + required=False, + default="/glade/scratch/" + myname + "/single_point_neon_updated/", + ) + parser.add_argument( + "-d", + "--debug", + help=""" Debug mode will print more information. [default: %(default)s] - ''', - action="store_true", - dest="debug", - default=False) + """, + action="store_true", + dest="debug", + default=False, + ) return parser def get_neon(neon_dir, site_name): """ - Function for finding neon data file + Function for finding neon data files and download from neon server if the file does not exits. @@ -132,53 +180,62 @@ def get_neon(neon_dir, site_name): Raises: Error if the download was not successful (exit code:404). In case the data does not exist in the neon server or if - neon server is down. + neon server is down. Returns: neon_file (str) : complete file name of the downloaded data """ - #-- create directory if not exists + # -- create directory if not exists if not os.path.exists(neon_dir): os.makedirs(neon_dir) neon_file = os.path.join(neon_dir, site_name + "_surfaceData.csv") - #-- Download the file if it does not exits + # -- Download the file if it does not exits if os.path.isfile(neon_file): - print('neon file for', site_name, 'already exists! ') - print('Skipping download from neon for', site_name,'...') + print("neon file for", site_name, "already exists! ") + print("Skipping download from neon for", site_name, "...") else: - print('------------------------------------------------') - print('Beginning download from neon server for', site_name,'...') - - url = ('https://s3.data.neonscience.org/neon-ncar/NEON/surf_files/v1/' - +site_name+'_surfaceData.csv') + print("------------------------------------------------") + print("Beginning download from neon server for", site_name, "...") + + url = ( + "https://s3.data.neonscience.org/neon-ncar/NEON/surf_files/v1/" + + site_name + + "_surfaceData.csv" + ) response = requests.get(url) - with open(neon_file, 'wb') as f: + with open(neon_file, "wb") as f: f.write(response.content) - #-- Check if download status_code + # -- Check if download status_code if response.status_code == 200: - print('Download finished successfully for', site_name) + print("Download finished successfully for", site_name) elif response.status_code == 404: - sys.exit('Data for this site '+site_name+ - ' was not available on the neon server:'+ url) - - print('Download exit status code: ',response.status_code) - print('Downloaded file type : ',response.headers['content-type']) - print('Downloaded file encoding : ',response.encoding) - print('------------------------------------------------') + sys.exit( + "Data for this site " + + site_name + + " was not available on the neon server:" + + url + ) + + print("Download exit status code: ", response.status_code) + print("Downloaded file type : ", response.headers["content-type"]) + print("Downloaded file encoding : ", response.encoding) + print("------------------------------------------------") response.close() return neon_file -def find_surffile (surf_dir, site_name): + +def find_surffile(surf_dir, site_name): """ Function for finding and choosing surface file for a neon site. + These files are created using ./subset_data.py script. In case multiple files exist for the neon site, it will choose the file created the latest. @@ -187,32 +244,36 @@ def find_surffile (surf_dir, site_name): site_name (str): 4 letter neon site name Raises: - Error if the surface data for the site is not created + Error if the surface data for the site is not created Returns: surf_file (str): name of the surface dataset file """ - #sf_name = "surfdata_hist_16pfts_Irrig_CMIP6_simyr2000_"+site_name+"*.nc" - sf_name = "surfdata_hist_78pfts_CMIP6_simyr2000_"+site_name+"*.nc" - #surf_file = glob.glob(os.path.join(surf_dir,sf_name)) - surf_file = glob.glob(surf_dir+"/"+sf_name) + # sf_name = "surfdata_hist_16pfts_Irrig_CMIP6_simyr2000_"+site_name+"*.nc" + sf_name = "surfdata_hist_78pfts_CMIP6_simyr2000_" + site_name + "*.nc" + # surf_file = glob.glob(os.path.join(surf_dir,sf_name)) + surf_file = glob.glob(surf_dir + "/" + sf_name) - if len(surf_file)>1: - print ("The following files found :", *surf_file, sep='\n- ') - print ("The latest file is chosen :", surf_file[-1]) + if len(surf_file) > 1: + print("The following files found :", *surf_file, sep="\n- ") + print("The latest file is chosen :", surf_file[-1]) surf_file = surf_file[-1] - elif len(surf_file)==1: - print ("File found : ") - print (surf_file) + elif len(surf_file) == 1: + print("File found : ") + print(surf_file) surf_file = surf_file[0] else: - sys.exit('Surface data for this site '+site_name+ - 'was not found:'+ surf_file,'.', - '\n','Please run ./subset_data.py for this site.') + sys.exit( + "Surface data for this site " + site_name + "was not found:" + surf_file, + ".", + "\n", + "Please run ./subset_data.py for this site.", + ) return surf_file -def find_soil_structure (surf_file): + +def find_soil_structure(surf_file): """ Function for finding surface dataset soil strucutre using surface data metadata. @@ -227,42 +288,49 @@ def find_soil_structure (surf_file): surf_file (str): single point surface data filename Raises: - error if the soil layer strucutre file does not exist + error if the soil layer strucutre file does not exist Returns: soil_bot : array of soil layers top depths soil_top : array of soil layers bottom depths """ - #TODO: What if not cheyenne? Self-contained depth info. + # TODO: What if not cheyenne? Self-contained depth info. - print ('------------') - print (surf_file) - print (type(surf_file)) + print("------------") + print("surf_file : ", surf_file) f1 = xr.open_dataset(surf_file) - print ('------------') - #print (f1.attrs["Soil_texture_raw_data_file_name"]) + print("------------") + # print (f1.attrs["Soil_texture_raw_data_file_name"]) clm_input_dir = "/glade/p/cesmdata/cseg/inputdata/lnd/clm2/rawdata/" - surf_soildepth_file = os.path.join(clm_input_dir, - f1.attrs["Soil_texture_raw_data_file_name"]) - - if os.path.exists (surf_soildepth_file): - print ("\n\n Reading", surf_soildepth_file, - "for surface data soil structure information:") + surf_soildepth_file = os.path.join( + clm_input_dir, f1.attrs["Soil_texture_raw_data_file_name"] + ) + + if os.path.exists(surf_soildepth_file): + print( + "\n\n Reading", + surf_soildepth_file, + "for surface data soil structure information:", + ) f1_soildepth = xr.open_dataset(surf_soildepth_file) - print (f1_soildepth['DZSOI']) - soil_bot = f1_soildepth['DZSOI'].values + print(f1_soildepth["DZSOI"]) + soil_bot = f1_soildepth["DZSOI"].values - #-- soil layer top + # -- soil layer top soil_top = soil_bot[:-1] - soil_top = np.insert(soil_top,0, 0) + soil_top = np.insert(soil_top, 0, 0) else: - sys.exit('Cannot find soil structure file : '+surf_soildepth_file+ - 'for the surface dataset.') + sys.exit( + "Cannot find soil structure file : " + + surf_soildepth_file + + "for the surface dataset." + ) return soil_bot, soil_top + def update_metadata(nc, surf_file, neon_file, zb_flag): """ Function for updating modified surface dataset @@ -279,21 +347,20 @@ def update_metadata(nc, surf_file, neon_file, zb_flag): today = date.today() today_string = today.strftime("%Y-%m-%d") - nc.attrs['Updated_on'] = today_string - nc.attrs['Updated_by'] = myname - nc.attrs['Updated_with'] = os.path.abspath(__file__) - nc.attrs['Updated_from'] = surf_file - nc.attrs['Updated_using'] = neon_file + nc.attrs["Updated_on"] = today_string + nc.attrs["Updated_by"] = myname + nc.attrs["Updated_with"] = os.path.abspath(__file__) + nc.attrs["Updated_from"] = surf_file + nc.attrs["Updated_using"] = neon_file if zb_flag: - nc.attrs['Updated_fields'] = "PCT_CLAY, PCT_SAND, ORGANIC, zbedrock" - #nc.attrs['Updated_fields'] = ['PCT_CLAY','PCT_SAND','ORGANIC','zbedrock'] + nc.attrs["Updated_fields"] = "PCT_CLAY, PCT_SAND, ORGANIC, zbedrock" else: - nc.attrs['Updated_fields'] = "PCT_CLAY, PCT_SAND, ORGANIC" - # nc.attrs['Updated_fields'] = ['PCT_CLAY','PCT_SAND','ORGANIC'] + nc.attrs["Updated_fields"] = "PCT_CLAY, PCT_SAND, ORGANIC" return nc -def update_time_tag (fname_in): + +def update_time_tag(fname_in): """ Function for updating time tag on surface dataset files. @@ -315,13 +382,14 @@ def update_time_tag (fname_in): basename = os.path.basename(fname_in) cend = -10 - if ( basename[cend] == "c" ): - cend = cend - 1 - if ( (basename[cend] != ".") and (basename[cend] != "_") ): - sys.exit( "Trouble figuring out where to add tag to filename:"+fname_in ) + if basename[cend] == "c": + cend = cend - 1 + if (basename[cend] != ".") and (basename[cend] != "_"): + sys.exit("Trouble figuring out where to add tag to filename:" + fname_in) + + fname_out = basename[:cend] + "_" + "c" + today_string + ".nc" + return fname_out - fname_out = basename[:cend]+"_"+"c"+today_string+".nc" - return(fname_out) def sort_print_soil_layers(obs_bot, soil_bot): """ @@ -329,58 +397,54 @@ def sort_print_soil_layers(obs_bot, soil_bot): original surface dataset and neon dataset. Args: - obs_bot : array of neon soil layers bottom depths + obs_bot : array of neon soil layers bottom depths soil_bot : array of soil layers bottom depths """ - obs_bot_df = pd.DataFrame({'depth':obs_bot,'type':"obs"}) - soil_bot_df = pd.DataFrame({'depth':soil_bot,'type':"sfc"}) - depth_df = pd.concat([obs_bot_df,soil_bot_df]) + obs_bot_df = pd.DataFrame({"depth": obs_bot, "type": "obs"}) + soil_bot_df = pd.DataFrame({"depth": soil_bot, "type": "sfc"}) + depth_df = pd.concat([obs_bot_df, soil_bot_df]) - depth_df = depth_df.sort_values('depth') + depth_df = depth_df.sort_values("depth") - space = ' ' - print ("================================", - "================================") + space = " " + print("================================", "================================") - print (" Neon data soil structure: " , - " Surface data soil structure: ") + print(" Neon data soil structure: ", " Surface data soil structure: ") - print ("================================", - "================================") + print("================================", "================================") for index, row in depth_df.iterrows(): - if row['type']=="obs": - print ("-------------", - "{0:.3f}".format(row['depth']), - "------------") + if row["type"] == "obs": + print("-------------", "{0:.3f}".format(row["depth"]), "------------") else: - print (33*space+ - "-------------", - "{0:.3f}".format(row['depth']), - "-----------") + print( + 33 * space + "-------------", + "{0:.3f}".format(row["depth"]), + "-----------", + ) + + print("--------------------------------" + "--------------------------------") - print ("--------------------------------"+ - "--------------------------------") def check_neon_time(): """ A function to download and parse neon listing file. + + Returns: + dict_out (str) : + dictionary of *_surfaceData.csv files with the last modified """ - listing_file = 'listing.csv' - url = 'https://neon-ncar.s3.data.neonscience.org/listing.csv' + listing_file = "listing.csv" + url = "https://neon-ncar.s3.data.neonscience.org/listing.csv" download_file(url, listing_file) df = pd.read_csv(listing_file) - df = df[df['object'].str.contains("_surfaceData.csv")] - #df=df.join(df['object'].str.split("/", expand=True)) - dict_out = dict(zip(df['object'],df['last_modified'])) - print (dict_out) - #df_out = df[['object','6','last_modified']] - #print (df['last_modified']) - #print (df_out) - #print (df['last_modified'].to_datetime()) + df = df[df["object"].str.contains("_surfaceData.csv")] + # df=df.join(df['object'].str.split("/", expand=True)) + dict_out = dict(zip(df["object"], df["last_modified"])) + print(dict_out) return dict_out @@ -388,88 +452,114 @@ def download_file(url, fname): """ Function to download a file. Args: - url (str): + url (str): url of the file for downloading - fname (str) : + fname (str) : file name to save the downloaded file. """ response = requests.get(url) - with open(fname, 'wb') as f: + with open(fname, "wb") as f: f.write(response.content) - #-- Check if download status_code + # -- Check if download status_code if response.status_code == 200: - print('Download finished successfully for', fname,'.') + print("Download finished successfully for", fname, ".") elif response.status_code == 404: - print('File '+fname+'was not available on the neon server:'+ url) + print("File " + fname + "was not available on the neon server:" + url) + + +def fill_interpolate(f2, var, method): + """ + Function to interpolate a variable in a + xarray dataset a specific method + """ + print("=====================================") + print("Filling in ", var, "with interpolation (method =" + method + ").") + + print("Variable before filling : ") + print(f2[var]) + + tmp_df = pd.DataFrame(f2[var].values.ravel()) + + tmp_df = tmp_df.interpolate(method=method, limit_direction="both") + # tmp_df = tmp_df.interpolate(method ='spline',order = 2, limit_direction ='both') + # tmp_df = tmp_df.interpolate(method="pad", limit=5, limit_direction = 'forward') + + tmp = tmp_df.to_numpy() + + soil_levels = f2[var].size + for soil_lev in range(soil_levels): + f2[var][soil_lev] = tmp[soil_lev].reshape(1, 1) + print("Variable after filling : ") + print(f2[var]) + print("=====================================") def main(): args = get_parser().parse_args() - #-- debugging option + # -- debugging option if args.debug: logging.basicConfig(level=logging.DEBUG) file_time = check_neon_time() - #-- specify site from which to extract data - site_name=args.site_name + # -- specify site from which to extract data + site_name = args.site_name - #-- Look for surface data + # -- Look for surface data surf_dir = args.surf_dir - surf_file = find_surffile (surf_dir, site_name) + surf_file = find_surffile(surf_dir, site_name) - #-- directory structure + # -- directory structure current_dir = os.getcwd() - parent_dir = os.path.dirname(current_dir) - clone_dir = os.path.abspath(os.path.join(__file__ ,"../../..")) - neon_dir = os.path.join(clone_dir,"neon_surffiles") - print("Present Directory", current_dir) + parent_dir = os.path.dirname(current_dir) + clone_dir = os.path.abspath(os.path.join(__file__, "../../..")) + neon_dir = os.path.join(clone_dir, "neon_surffiles_v3") - #-- download neon data if needed + print("Present Directory", current_dir) + + # -- download neon data if needed neon_file = get_neon(neon_dir, site_name) - #-- Read neon data - df = pd.read_csv (neon_file) + # -- Read neon data + df = pd.read_csv(neon_file) # -- Read surface dataset files - print (type(surf_file)) - print ("surf_file:", surf_file) + print("surf_file:", surf_file) f1 = xr.open_dataset(surf_file) # -- Find surface dataset soil depth information - soil_bot, soil_top = find_soil_structure (surf_file) + soil_bot, soil_top = find_soil_structure(surf_file) # -- Find surface dataset soil levels - # TODO: how? NS uses metadata on file to find + # TODO: how? NS uses metadata on file to find # soil strucure # better suggestion by WW to write dzsoi to neon surface dataset # This todo needs to go to the subset_data # TODO Will: if I sum them up , are they 3.5? (m) YES - print ("soil_top:", soil_top) - print ("soil_bot:", soil_bot) - print ("Sum of soil top depths :", sum(soil_top)) - print ("Sum of soil bottom depths :",sum(soil_bot)) + print("soil_top:", soil_top) + print("soil_bot:", soil_bot) + print("Sum of soil top depths :", sum(soil_top)) + print("Sum of soil bottom depths :", sum(soil_bot)) soil_top = np.cumsum(soil_top) soil_bot = np.cumsum(soil_bot) - soil_mid = 0.5*(soil_bot - soil_top)+soil_top - #print ("Cumulative sum of soil bottom depths :", sum(soil_bot)) + soil_mid = 0.5 * (soil_bot - soil_top) + soil_top + # print ("Cumulative sum of soil bottom depths :", sum(soil_bot)) - obs_top = df['biogeoTopDepth']/100 - obs_bot = df['biogeoBottomDepth']/100 + obs_top = df["biogeoTopDepth"] / 100 + obs_bot = df["biogeoBottomDepth"] / 100 # -- Mapping surface dataset and neon soil levels - bins = df['biogeoTopDepth']/100 - bin_index = np.digitize(soil_mid, bins)-1 - + bins = df["biogeoTopDepth"] / 100 + bin_index = np.digitize(soil_mid, bins) - 1 - ''' + """ print ("================================") print (" Neon data soil structure: ") print ("================================") @@ -491,66 +581,110 @@ def main(): print ("-------------", "{0:.2f}".format(soil_bot[b]), "-------------") - ''' - #-- update fields with neon - f2= f1 - soil_levels = f2['PCT_CLAY'].size + """ + + # -- update fields with neon + f2 = f1 + soil_levels = f2["PCT_CLAY"].size for soil_lev in range(soil_levels): - print ("--------------------------") - print ("soil_lev:",soil_lev) - print (df['clayTotal'][bin_index[soil_lev]]) - f2['PCT_CLAY'][soil_lev] = df['clayTotal'][bin_index[soil_lev]] - f2['PCT_SAND'][soil_lev] = df['sandTotal'][bin_index[soil_lev]] - bulk_den = df['bulkDensExclCoarseFrag'][bin_index[soil_lev]] - carbon_tot = df['carbonTot'][bin_index[soil_lev]] - #print ("carbon_tot:", carbon_tot) - layer_depth = df['biogeoBottomDepth'][bin_index[soil_lev]] - df['biogeoTopDepth'][bin_index[soil_lev]] - f2['ORGANIC'][soil_lev] = carbon_tot * bulk_den * 0.1 / layer_depth * 100 / 0.58 - print ("bin_index:", bin_index[soil_lev]) - print ("layer_depth:", layer_depth) - print ("carbon_tot:",carbon_tot) - print ("bulk_den:",bulk_den) - print ("organic=carbon_tot*bulk_den*0.1/layer_depth * 100/0.58 ") - print ("organic:", f2['ORGANIC'][soil_lev].values) - print ("--------------------------") - - #TODO : max depth for neon sites from WW (zbedrock) - # Update zbedrock if neon observation don't make it down to 2m depth - # zbedrock = neon depth if neon does not make to 2m depth + print("--------------------------") + print("soil_lev:", soil_lev) + print(df["clayTotal"][bin_index[soil_lev]]) + f2["PCT_CLAY"][soil_lev] = df["clayTotal"][bin_index[soil_lev]] + f2["PCT_SAND"][soil_lev] = df["sandTotal"][bin_index[soil_lev]] + + bulk_den = df["bulkDensExclCoarseFrag"][bin_index[soil_lev]] + carbon_tot = df["carbonTot"][bin_index[soil_lev]] + estimated_oc = df["estimatedOC"][bin_index[soil_lev]] + + # -- estimated_oc in neon data is rounded to the nearest integer. + # -- Check to make sure the rounded oc is not higher than carbon_tot. + # -- Use carbon_tot if estimated_oc is bigger than carbon_tot. + + + #if estimated_oc > carbon_tot: + # estimated_oc = carbon_tot + + layer_depth = ( + df["biogeoBottomDepth"][bin_index[soil_lev]] + - df["biogeoTopDepth"][bin_index[soil_lev]] + ) + + #f2["ORGANIC"][soil_lev] = estimated_oc * bulk_den / 0.58 + + # -- after adding caco3 by NEON: + # -- if caco3 exists: + # -- inorganic = caco3/100.0869*12.0107 + # -- organic = carbon_tot - inorganic + # -- else: + # -- oranigc = estimated_oc * bulk_den /0.58 + caco3 = df["caco3Conc"][bin_index[soil_lev]] + inorganic = caco3 /100.0869*12.0107 + print ("inorganic:", inorganic) + + if not np.isnan(inorganic): + actual_oc = carbon_tot -inorganic + else: + actual_oc = estimated_oc + + + f2["ORGANIC"][soil_lev] = actual_oc * bulk_den / 0.58 + + print ("~~~~~~~~~~~~~~~~~~~~~~~~") + print ("inorganic:") + print ("~~~~~~~~~~~~~~~~~~~~~~~~") + print (inorganic) + print ("~~~~~~~~~~~~~~~~~~~~~~~~") + + print("bin_index : ", bin_index[soil_lev]) + print("layer_depth : ", layer_depth) + print("carbon_tot : ", carbon_tot) + print("estimated_oc : ", estimated_oc) + print("bulk_den : ", bulk_den) + print("organic :", f2["ORGANIC"][soil_lev].values) + print("--------------------------") + + # -- Interpolate missing values + method = "linear" + fill_interpolate(f2, "PCT_CLAY", method) + fill_interpolate(f2, "PCT_SAND", method) + fill_interpolate(f2, "ORGANIC", method) + + # -- Update zbedrock if neon observation does not make it down to 2m depth rock_thresh = 2 zb_flag = False - if (obs_bot.iloc[-1] Date: Tue, 2 Nov 2021 05:27:54 -0600 Subject: [PATCH 31/55] updates for the ag sites --- .../modify_singlept_site_neon.py | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/tools/site_and_regional/modify_singlept_site_neon.py b/tools/site_and_regional/modify_singlept_site_neon.py index 075fbe8a1b..cd767d2471 100755 --- a/tools/site_and_regional/modify_singlept_site_neon.py +++ b/tools/site_and_regional/modify_singlept_site_neon.py @@ -663,6 +663,26 @@ def main(): sort_print_soil_layers(obs_bot, soil_bot) + + # -- updates for ag sites : KONA and STER + ag_sites = ['KONA', 'STER'] + if site_name in ag_sites: + print ("Updating PCT_NATVEG") + print ("Original : ", f2.PCT_NATVEG.values) + f2.PCT_NATVEG.values = [[0.]] + print ("Updated : ", f2.PCT_NATVEG.values) + + print ("Updating PCT_CROP") + print ("Original : ",f2.PCT_CROP.values) + f2.PCT_CROP.values =[[100.]] + print ("Updated : ",f2.PCT_CROP.values) + + print ("Updating PCT_NAT_PFT") + print (f2.PCT_NAT_PFT.values[0]) + f2.PCT_NAT_PFT.values[0] =[[100.]] + print (f2.PCT_NAT_PFT[0].values) + + out_dir = args.out_dir # -- make out_dir if it does not exist From 99a85fe53cf169615294c46c8c35da3a89b1545b Mon Sep 17 00:00:00 2001 From: Negin Sobhani Date: Tue, 2 Nov 2021 05:33:52 -0600 Subject: [PATCH 32/55] quick changes that should be done for this PR to work. --- tools/site_and_regional/subset_data.py | 58 +++++++++++++++++++++----- 1 file changed, 47 insertions(+), 11 deletions(-) diff --git a/tools/site_and_regional/subset_data.py b/tools/site_and_regional/subset_data.py index 61c3fe0cf2..a649c657b9 100755 --- a/tools/site_and_regional/subset_data.py +++ b/tools/site_and_regional/subset_data.py @@ -1,4 +1,4 @@ -#! /usr/bin/env python3 +#! /usr/bin/env python """ |------------------------------------------------------------------| |--------------------- Instructions -----------------------------| @@ -178,6 +178,33 @@ def get_parser(): required = False, type = int, default = 2014) + pt_parser.add_argument('--datm_from_tower', + help='Flag for creating DATM forcing data at single point for a tower data. [default: %(default)s]', + action="store", + dest="datm_tower", + type = str2bool, + nargs = '?', + const = True, + required = False, + default = False) + pt_parser.add_argument('--create_user_mods', + help='Flag for creating user mods directory . [default: %(default)s]', + action="store", + dest="datm_tower", + type = str2bool, + nargs = '?', + const = True, + required = False, + default = False) + pt_parser.add_argument('--user_mods_dir', + help='Flag for creating user mods directory . [default: %(default)s]', + action="store", + dest="user_mod_dir", + type = str, + nargs = '?', + const = True, + required = False, + default = False) pt_parser.add_argument('--crop', help='Create datasets using the extensive list of prognostic crop types. [default: %(default)s]', action="store_true", @@ -369,9 +396,9 @@ def plon_type(x): """ x = float(x) if (-180 < x) and (x < 0): - print ("lon is :", lon) + print ("lon is :", x) x= x%360 - print ("after modulo lon is :", lon) + print ("after modulo lon is :", x) if (x < 0) or (x > 360): raise argparse.ArgumentTypeError("ERROR: Latitude of single point should be between 0 and 360 or -180 and 180.") return x @@ -380,11 +407,7 @@ def get_git_sha(): """ Returns Git short SHA for the currect directory. """ - try: - sha = subprocess.check_output(['git', 'rev-parse', '--short', 'HEAD']).strip().decode() - except subprocess.CalledProcessError: - sha = "NOT-A-GIT-REPOSITORY" - return sha + return subprocess.check_output(['git', 'rev-parse', '--short', 'HEAD']).strip().decode() class BaseCase : """ @@ -552,8 +575,11 @@ def create_fileout_name( filename,tag): items = basename.split('_') today = date.today() today_string = today.strftime("%y%m%d") + print (items[-1]) + #new_string = items[0]+"_"+items[2]+"_"+items[3]+"_"+ items[4] \ + # +"_"+items[5]+"_"+items[6]+"_"+tag+"_c"+today_string+".nc" new_string = items[0]+"_"+items[2]+"_"+items[3]+"_"+ items[4] \ - +"_"+items[5]+"_"+items[6]+"_"+tag+"_c"+today_string+".nc" + +"_"+items[5]+"_"+tag+"_c"+today_string+".nc" return new_string def create_domain_at_point (self): @@ -622,7 +648,9 @@ def create_surfdata_at_point(self): # modify surface data properties if self.overwrite_single_pft: f3['PCT_NAT_PFT'][:,:,:] = 0 - f3['PCT_NAT_PFT'][:,:,self.dominant_pft] = 100 + if (self.dominant_pft <16): + f3['PCT_NAT_PFT'][:,:,self.dominant_pft] = 100 + #else:@@@ if self.zero_nonveg_landunits: f3['PCT_NATVEG'][:,:] = 100 f3['PCT_CROP'][:,:] = 0 @@ -638,6 +666,14 @@ def create_surfdata_at_point(self): # specify dimension order #f3 = f3.transpose(u'time', u'cft', u'natpft', u'lsmlat', u'lsmlon') f3 = f3.transpose(u'time', u'cft', u'lsmpft', u'natpft', u'nglcec', u'nglcecp1', u'nlevsoi', u'nlevurb', u'numrad', u'numurbl', 'lsmlat', 'lsmlon') + + #update lsmlat and lsmlon to match site specific instead of the nearest point + #f3['lon']= self.plon + #f3['lat']= self.plat + f3['lsmlon']= np.atleast_1d(self.plon) + f3['lsmlat']= np.atleast_1d(self.plat) + f3['LATIXY'][:,:]= self.plat + f3['LONGXY'][:,:]= self.plon #update attributes self.update_metadata(f3) @@ -789,7 +825,7 @@ def create_surfdata_at_reg(self): f3.attrs['Created_from'] = self.fsurf_in # mode 'w' overwrites file - f3.to_netcdf(path=self.fsurf_out, mode='w') + f3.to_netcdf(path=self.fsurf_out, mode='w', format='NETCDF3_64BIT') print('created file (fsurf_out)'+self.fsurf_out) #f1.close(); f2.close(); f3.close() From b807e8e95d36fefe537688bf740d5ed42a274d4d Mon Sep 17 00:00:00 2001 From: Negin Sobhani Date: Tue, 2 Nov 2021 05:58:05 -0600 Subject: [PATCH 33/55] very simple wrapper for creating and modifying all neon sites at once. --- tools/site_and_regional/neon_sites_dompft.csv | 48 ++++++++ tools/site_and_regional/neon_surf_wrapper.py | 108 ++++++++++++++++++ 2 files changed, 156 insertions(+) create mode 100644 tools/site_and_regional/neon_sites_dompft.csv create mode 100755 tools/site_and_regional/neon_surf_wrapper.py diff --git a/tools/site_and_regional/neon_sites_dompft.csv b/tools/site_and_regional/neon_sites_dompft.csv new file mode 100644 index 0000000000..6182114ac1 --- /dev/null +++ b/tools/site_and_regional/neon_sites_dompft.csv @@ -0,0 +1,48 @@ +Site,Domain,Lat,Lon,pft +BART,1,44.06516,-71.28834,7 +HARV,1,42.53562,-72.17562,7 +BLAN,2,39.06044,-78.07115,7 +SCBI,2,38.89209,-78.13764,7 +SERC,2,38.89124,-76.55884,7 +DSNY,3,28.12919,-81.43394,14 +JERC,3,31.19608,-84.46647,1 +OSBS,3,29.68819,-81.99345,1 +GUAN,4,17.96882,-66.86888,6 +LAJA,4,18.02184,-67.07608,14 +STEI,5,45.5076,-89.5888,7 +TREE,5,45.49266,-89.58748,7 +UNDE,5,46.14103,-89.3221,7 +KONA,6,39.10828,-96.61044,18 +KONZ,6,39.1007,-96.56227,14 +UKFS,6,39.04168,-95.20495,7 +GRSM,7,35.68839,-83.50185,7 +MLBS,7,37.37783,-80.52425,7 +ORNL,7,35.57525,-84.16581,7 +DELA,8,32.54092,-87.80341,7 +LENO,8,31.8531,-88.16103,7 +TALL,8,32.95106,-87.3941,1 +DCFS,9,47.15919,-99.11251,13 +NOGP,9,46.76846,-100.91832,13 +WOOD,9,47.12833,-99.23907,13 +CPER,10,40.81297,-104.74455,14 +RMNP,10,40.27707,-105.54524,1 +STER,10,40.45984,-103.03008,18 +CLBJ,11,33.40143,-97.56725,7 +OAES,11,35.41062,-99.06044,14 +YELL,12,44.95597,-110.54196,1 +MOAB,13,38.25136,-109.38882,14 +NIWO,13,40.05236,-105.58324,12 +JORN,14,32.59052,-106.84377,14 +SRER,14,31.91068,-110.83549,9 +ONAQ,15,35.68839,-83.50185,9 +ABBY,16,45.762378,-122.329672,1 +WREF,16,45.81637,-121.95838,1 +SJER,17,37.107117,-119.733,13 +SOAP,17,37.03269,-119.2621,1 +TEAK,17,37.006472,-119.005758,1 +TOOL,17,68.66045,-149.370128,1 +BARR,18,71.281711,-156.650219,12 +BONA,19,65.15333,-147.50194,2 +DEJU,19,63.87983,-145.74765,2 +HEAL,19,63.8798,-149.21539,12 +PUUM,20,19.55309,-155.31731,4 diff --git a/tools/site_and_regional/neon_surf_wrapper.py b/tools/site_and_regional/neon_surf_wrapper.py new file mode 100755 index 0000000000..e5b37a74e3 --- /dev/null +++ b/tools/site_and_regional/neon_surf_wrapper.py @@ -0,0 +1,108 @@ +#! /usr/bin/env python3 +""" +|------------------------------------------------------------------| +|--------------------- Instructions -----------------------------| +|------------------------------------------------------------------| +This script is a simple wrapper for neon sites that performs the +following: + 1) For neon sites, subset surface dataset from global dataset + (i.e. ./subset_data.py ) + 2) Download neon and update the created surface dataset + based on the downloaded neon data. + (i.e. modify_singlept_site_neon.py) + +Instructions for running on Cheyenne/Casper: +load the following into your local environment + module load python + ncar_pylib + +""" +# TODO +# Automatic downloading of missing files if they are missing +#-[ ] Download neon sites and dom pft file +#-[ ] Make sure verbose works for printing out commands running + +# Import libraries +from __future__ import print_function + +import os +import sys +import tqdm +import logging +import argparse +import subprocess + +import pandas as pd +#import tqdm as tqdm + + + +def get_parser(): + """ + Get parser object for this script. + """ + parser = argparse.ArgumentParser(description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + + parser.print_usage = parser.print_help + + parser.add_argument('-v','--verbose', + help='Verbose mode will print more information. ', + action="store_true", + dest="verbose", + default=False) + + + return parser + + +def execute(command): + """ + Function for running a command on shell. + Args: + command (str): + command that we want to run. + Raises: + Error with the return code from shell. + """ + print ('\n',' >> ',*command,'\n') + + try: + subprocess.check_call(command, stdout=open(os.devnull, "w"), stderr=subprocess.STDOUT) + + except subprocess.CalledProcessError as e: + #raise RuntimeError("command '{}' return with error (code {}): {}".format(e.cmd, e.returncode, e.output)) + #print (e.ouput) + print (e) + + + + + + +def main(): + + args = get_parser().parse_args() + + if args.verbose: + logging.basicConfig(level=logging.DEBUG) + + + neon_sites = pd.read_csv('neon_sites_dompft.csv') + + + for i, row in tqdm.tqdm(neon_sites.iterrows()): + lat = row['Lat'] + lon = row['Lon'] + site = row['Site'] + pft = row['pft'] + print ("Now processing site :", site) + command = ['./subset_data.py','point','--lat',str(lat),'--lon',str(lon),'--site',site,'--dompft',str(pft),'--crop'] + execute(command) + + command = ['./modify_singlept_site_neon.py','--neon_site',site] + execute(command) + +if __name__ == "__main__": + main() + From 91edfeba8f99cf2bb1bbb026dd2dbecd195a59cc Mon Sep 17 00:00:00 2001 From: Negin Sobhani Date: Tue, 2 Nov 2021 05:58:25 -0600 Subject: [PATCH 34/55] just a quick change for estimated_oc round up issue. --- tools/site_and_regional/modify_singlept_site_neon.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/site_and_regional/modify_singlept_site_neon.py b/tools/site_and_regional/modify_singlept_site_neon.py index cd767d2471..d604bc59ef 100755 --- a/tools/site_and_regional/modify_singlept_site_neon.py +++ b/tools/site_and_regional/modify_singlept_site_neon.py @@ -602,8 +602,8 @@ def main(): # -- Use carbon_tot if estimated_oc is bigger than carbon_tot. - #if estimated_oc > carbon_tot: - # estimated_oc = carbon_tot + if estimated_oc > carbon_tot: + estimated_oc = carbon_tot layer_depth = ( df["biogeoBottomDepth"][bin_index[soil_lev]] From 0d69139adbffa4c06886ab9fc5aab3d70757f616 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Thu, 2 Dec 2021 17:02:03 -0700 Subject: [PATCH 35/55] Small update with running through black --- tools/site_and_regional/modify_singlept_site_neon.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tools/site_and_regional/modify_singlept_site_neon.py b/tools/site_and_regional/modify_singlept_site_neon.py index 883427b642..ee52d475af 100755 --- a/tools/site_and_regional/modify_singlept_site_neon.py +++ b/tools/site_and_regional/modify_singlept_site_neon.py @@ -432,7 +432,7 @@ def check_neon_time(): A function to download and parse neon listing file. Returns: - dict_out (str) : + dict_out (str) : dictionary of *_surfaceData.csv files with the last modified """ listing_file = "listing.csv" @@ -627,7 +627,6 @@ def main(): fill_interpolate(f2, "PCT_SAND", method) fill_interpolate(f2, "ORGANIC", method) - # -- Update zbedrock if neon observation does not make it down to 2m depth rock_thresh = 2 From a1062750281af1058d3821ef2b5e681bf82fbf9c Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Thu, 2 Dec 2021 22:48:55 -0700 Subject: [PATCH 36/55] Run modify_singlept_site_neon.py through black --- .../modify_singlept_site_neon.py | 51 +++++++++---------- 1 file changed, 24 insertions(+), 27 deletions(-) diff --git a/tools/site_and_regional/modify_singlept_site_neon.py b/tools/site_and_regional/modify_singlept_site_neon.py index d99c2eeac2..d3db55126b 100755 --- a/tools/site_and_regional/modify_singlept_site_neon.py +++ b/tools/site_and_regional/modify_singlept_site_neon.py @@ -601,7 +601,6 @@ def main(): # -- Check to make sure the rounded oc is not higher than carbon_tot. # -- Use carbon_tot if estimated_oc is bigger than carbon_tot. - if estimated_oc > carbon_tot: estimated_oc = carbon_tot @@ -610,7 +609,7 @@ def main(): - df["biogeoTopDepth"][bin_index[soil_lev]] ) - #f2["ORGANIC"][soil_lev] = estimated_oc * bulk_den / 0.58 + # f2["ORGANIC"][soil_lev] = estimated_oc * bulk_den / 0.58 # -- after adding caco3 by NEON: # -- if caco3 exists: @@ -620,22 +619,21 @@ def main(): # -- oranigc = estimated_oc * bulk_den /0.58 caco3 = df["caco3Conc"][bin_index[soil_lev]] - inorganic = caco3 /100.0869*12.0107 - print ("inorganic:", inorganic) + inorganic = caco3 / 100.0869 * 12.0107 + print("inorganic:", inorganic) if not np.isnan(inorganic): - actual_oc = carbon_tot -inorganic + actual_oc = carbon_tot - inorganic else: actual_oc = estimated_oc - f2["ORGANIC"][soil_lev] = actual_oc * bulk_den / 0.58 - print ("~~~~~~~~~~~~~~~~~~~~~~~~") - print ("inorganic:") - print ("~~~~~~~~~~~~~~~~~~~~~~~~") - print (inorganic) - print ("~~~~~~~~~~~~~~~~~~~~~~~~") + print("~~~~~~~~~~~~~~~~~~~~~~~~") + print("inorganic:") + print("~~~~~~~~~~~~~~~~~~~~~~~~") + print(inorganic) + print("~~~~~~~~~~~~~~~~~~~~~~~~") print("bin_index : ", bin_index[soil_lev]) print("layer_depth : ", layer_depth) @@ -663,24 +661,23 @@ def main(): sort_print_soil_layers(obs_bot, soil_bot) - # -- updates for ag sites : KONA and STER - ag_sites = ['KONA', 'STER'] + ag_sites = ["KONA", "STER"] if site_name in ag_sites: - print ("Updating PCT_NATVEG") - print ("Original : ", f2.PCT_NATVEG.values) - f2.PCT_NATVEG.values = [[0.]] - print ("Updated : ", f2.PCT_NATVEG.values) - - print ("Updating PCT_CROP") - print ("Original : ",f2.PCT_CROP.values) - f2.PCT_CROP.values =[[100.]] - print ("Updated : ",f2.PCT_CROP.values) - - print ("Updating PCT_NAT_PFT") - print (f2.PCT_NAT_PFT.values[0]) - f2.PCT_NAT_PFT.values[0] =[[100.]] - print (f2.PCT_NAT_PFT[0].values) + print("Updating PCT_NATVEG") + print("Original : ", f2.PCT_NATVEG.values) + f2.PCT_NATVEG.values = [[0.0]] + print("Updated : ", f2.PCT_NATVEG.values) + + print("Updating PCT_CROP") + print("Original : ", f2.PCT_CROP.values) + f2.PCT_CROP.values = [[100.0]] + print("Updated : ", f2.PCT_CROP.values) + + print("Updating PCT_NAT_PFT") + print(f2.PCT_NAT_PFT.values[0]) + f2.PCT_NAT_PFT.values[0] = [[100.0]] + print(f2.PCT_NAT_PFT[0].values) out_dir = args.out_dir From 2b5bc4c588a2a1b3179c2b02833c8a2258ec3980 Mon Sep 17 00:00:00 2001 From: Bill Sacks Date: Mon, 6 Dec 2021 12:53:05 -0700 Subject: [PATCH 37/55] LILAC build: change --esmf-lib-path to --esmf-mkfile-path ESMF_LIBDIR is deprecated; the preferred method for specifying the path to ESMF is now to use ESMFMKFILE. And the latter is needed for a check in the CMEPS buildnml; see also https://github.com/ESCOMP/CTSM/issues/1498 --- .../obtaining-and-building-ctsm.rst | 15 +++++------ .../config_compilers_template.xml | 2 -- .../config_machines_template.xml | 4 +++ python/ctsm/lilac_build_ctsm.py | 26 +++++++++---------- python/ctsm/test/test_sys_lilac_build_ctsm.py | 4 +-- .../ctsm/test/test_unit_lilac_build_ctsm.py | 10 +++---- 6 files changed, 31 insertions(+), 30 deletions(-) diff --git a/doc/source/lilac/obtaining-building-and-running/obtaining-and-building-ctsm.rst b/doc/source/lilac/obtaining-building-and-running/obtaining-and-building-ctsm.rst index a51dcfcf25..99cb908d28 100644 --- a/doc/source/lilac/obtaining-building-and-running/obtaining-and-building-ctsm.rst +++ b/doc/source/lilac/obtaining-building-and-running/obtaining-and-building-ctsm.rst @@ -41,7 +41,7 @@ On a machine that has *not* been ported to CIME, you will need to provide some a information. Run ``./lilac/build_ctsm -h`` for details, but the basic command will look like this:: - ./lilac/build_ctsm ~/ctsm_build_dir --os Darwin --compiler gnu --netcdf-path /usr/local --esmf-lib-path /Users/sacks/ESMF/esmf8.0.0/lib/libO/Darwin.gfortranclang.64.mpich3.default --max-mpitasks-per-node 4 --no-pnetcdf + ./lilac/build_ctsm ~/ctsm_build_dir --os Darwin --compiler gnu --netcdf-path /usr/local --esmf-mkfile-path /Users/sacks/ESMF/esmf8.0.0/lib/libO/Darwin.gfortranclang.64.mpich3.default/esmf.mk --max-mpitasks-per-node 4 --no-pnetcdf In both cases, you will then need to include the necessary information in the include and link lines of the atmosphere model's build. For a Makefile-based build, this can be done @@ -205,7 +205,7 @@ above`. The minimal amount of information needed is given by the following:: - ./lilac/build_ctsm /PATH/TO/CTSM/BUILD --os OS --compiler COMPILER --netcdf-path NETCDF_PATH --esmf-lib-path ESMF_LIB_PATH --max-mpitasks-per-node MAX_MPITASKS_PER_NODE --pnetcdf-path PNETCDF_PATH + ./lilac/build_ctsm /PATH/TO/CTSM/BUILD --os OS --compiler COMPILER --netcdf-path NETCDF_PATH --esmf-mkfile-path ESMF_MKFILE_PATH --max-mpitasks-per-node MAX_MPITASKS_PER_NODE --pnetcdf-path PNETCDF_PATH where you should fill in the capitalized arguments with appropriate values for your machine. Run ``./lilac/build_ctsm -h`` for details on these arguments, as well as documentation @@ -229,17 +229,16 @@ model performance. Example usage for a Mac (a simple case) is:: - ./lilac/build_ctsm ~/ctsm_build_dir --os Darwin --compiler gnu --netcdf-path /usr/local --esmf-lib-path /Users/sacks/ESMF/esmf8.0.0/lib/libO/Darwin.gfortranclang.64.mpich3.default --max-mpitasks-per-node 4 --no-pnetcdf + ./lilac/build_ctsm ~/ctsm_build_dir --os Darwin --compiler gnu --netcdf-path /usr/local --esmf-mkfile-path /Users/sacks/ESMF/esmf8.0.0/lib/libO/Darwin.gfortranclang.64.mpich3.default/esmf.mk --max-mpitasks-per-node 4 --no-pnetcdf Example usage for NCAR's ``cheyenne`` machine (a more complex case) is:: module purge - module load ncarenv/1.3 intel/19.0.5 esmf_libs mkl - module use /glade/work/himanshu/PROGS/modulefiles/esmfpkgs/intel/19.0.5 - module load esmf-8.1.0b14-ncdfio-mpt-O mpt/2.21 netcdf/4.7.3 pnetcdf/1.12.1 ncarcompilers/0.5.0 - module load python + module load ncarenv/1.3 python/3.7.9 cmake intel/19.1.1 esmf_libs mkl + module use /glade/p/cesmdata/cseg/PROGS/modulefiles/esmfpkgs/intel/19.1.1/ + module load esmf-8.2.0b23-ncdfio-mpt-O mpt/2.22 netcdf-mpi/4.8.0 pnetcdf/1.12.2 ncarcompilers/0.5.0 - ./lilac/build_ctsm /glade/scratch/$USER/ctsm_build_dir --os linux --compiler intel --netcdf-path '$ENV{NETCDF}' --pio-filesystem-hints gpfs --pnetcdf-path '$ENV{PNETCDF}' --esmf-lib-path '$ENV{ESMF_LIBDIR}' --max-mpitasks-per-node 36 --extra-cflags '-xCORE_AVX2 -no-fma' --extra-fflags '-xCORE_AVX2 -no-fma' + ./lilac/build_ctsm /glade/scratch/$USER/ctsm_build_dir --os linux --compiler intel --netcdf-path '$ENV{NETCDF}' --pio-filesystem-hints gpfs --pnetcdf-path '$ENV{PNETCDF}' --esmf-mkfile-path '$ENV{ESMFMKFILE}' --max-mpitasks-per-node 36 --extra-cflags '-xCORE_AVX2 -no-fma' --extra-fflags '-xCORE_AVX2 -no-fma' (It's better to use the :ref:`alternative process for a CIME-supported machine` in this case, but the above illustrates diff --git a/lilac/bld_templates/config_compilers_template.xml b/lilac/bld_templates/config_compilers_template.xml index 9fc3358408..143e088776 100644 --- a/lilac/bld_templates/config_compilers_template.xml +++ b/lilac/bld_templates/config_compilers_template.xml @@ -29,8 +29,6 @@ $$MYVAR refers to the MYVAR cime variable. blank line. --> $PNETCDF_PATH - $ESMF_LIBDIR - $EXTRA_CFLAGS diff --git a/lilac/bld_templates/config_machines_template.xml b/lilac/bld_templates/config_machines_template.xml index a197e02dfa..6eff902242 100644 --- a/lilac/bld_templates/config_machines_template.xml +++ b/lilac/bld_templates/config_machines_template.xml @@ -111,5 +111,9 @@ --> + + $ESMF_MKFILE_PATH + + diff --git a/python/ctsm/lilac_build_ctsm.py b/python/ctsm/lilac_build_ctsm.py index 888008897a..0d2ae4d1b9 100644 --- a/python/ctsm/lilac_build_ctsm.py +++ b/python/ctsm/lilac_build_ctsm.py @@ -71,7 +71,7 @@ def main(cime_path): machine=args.machine, os_type=args.os, netcdf_path=args.netcdf_path, - esmf_lib_path=args.esmf_lib_path, + esmf_mkfile_path=args.esmf_mkfile_path, max_mpitasks_per_node=args.max_mpitasks_per_node, gmake=args.gmake, gmake_j=args.gmake_j, @@ -92,7 +92,7 @@ def build_ctsm(cime_path, machine=None, os_type=None, netcdf_path=None, - esmf_lib_path=None, + esmf_mkfile_path=None, max_mpitasks_per_node=None, gmake=None, gmake_j=None, @@ -117,7 +117,7 @@ def build_ctsm(cime_path, Must be given if machine isn't given; ignored if machine is given netcdf_path (str or None): path to NetCDF installation Must be given if machine isn't given; ignored if machine is given - esmf_lib_path (str or None): path to ESMF library directory + esmf_mkfile_path (str or None): path to esmf.mk file (typically within ESMF library directory) Must be given if machine isn't given; ignored if machine is given max_mpitasks_per_node (int or None): number of physical processors per shared-memory node Must be given if machine isn't given; ignored if machine is given @@ -153,7 +153,7 @@ def build_ctsm(cime_path, if machine is None: assert os_type is not None, 'with machine absent, os_type must be given' assert netcdf_path is not None, 'with machine absent, netcdf_path must be given' - assert esmf_lib_path is not None, 'with machine absent, esmf_lib_path must be given' + assert esmf_mkfile_path is not None, 'with machine absent, esmf_mkfile_path must be given' assert max_mpitasks_per_node is not None, ('with machine absent ' 'max_mpitasks_per_node must be given') os_type = _check_and_transform_os(os_type) @@ -161,7 +161,7 @@ def build_ctsm(cime_path, os_type=os_type, compiler=compiler, netcdf_path=netcdf_path, - esmf_lib_path=esmf_lib_path, + esmf_mkfile_path=esmf_mkfile_path, max_mpitasks_per_node=max_mpitasks_per_node, gmake=gmake, gmake_j=gmake_j, @@ -242,7 +242,7 @@ def _commandline_args(args_to_parse=None): For a fresh build with a machine that has NOT been ported to cime: - build_ctsm /path/to/nonexistent/directory --os OS --compiler COMPILER --netcdf-path NETCDF_PATH --esmf-lib-path ESMF_LIB_PATH --max-mpitasks-per-node MAX_MPITASKS_PER_NODE --pnetcdf-path PNETCDF_PATH + build_ctsm /path/to/nonexistent/directory --os OS --compiler COMPILER --netcdf-path NETCDF_PATH --esmf-mkfile-path ESMF_MKFILE_PATH --max-mpitasks-per-node MAX_MPITASKS_PER_NODE --pnetcdf-path PNETCDF_PATH If PNetCDF is not available, set --no-pnetcdf instead of --pnetcdf-path. @@ -346,10 +346,10 @@ def _commandline_args(args_to_parse=None): 'named lib, include, etc.)') new_machine_required_list.append('netcdf-path') - new_machine_required.add_argument('--esmf-lib-path', - help='Path to ESMF library directory\n' - 'This directory should include an esmf.mk file') - new_machine_required_list.append('esmf-lib-path') + new_machine_required.add_argument('--esmf-mkfile-path', + help='Path to esmf.mk file\n' + '(typically within ESMF library directory)') + new_machine_required_list.append('esmf-mkfile-path') new_machine_required.add_argument('--max-mpitasks-per-node', type=int, help='Number of physical processors per shared-memory node\n' @@ -499,7 +499,7 @@ def _fill_out_machine_files(build_dir, os_type, compiler, netcdf_path, - esmf_lib_path, + esmf_mkfile_path, max_mpitasks_per_node, gmake, gmake_j, @@ -526,7 +526,8 @@ def _fill_out_machine_files(build_dir, 'CIME_OUTPUT_ROOT':build_dir, 'GMAKE':gmake, 'GMAKE_J':gmake_j, - 'MAX_MPITASKS_PER_NODE':max_mpitasks_per_node}) + 'MAX_MPITASKS_PER_NODE':max_mpitasks_per_node, + 'ESMF_MKFILE_PATH':esmf_mkfile_path}) # ------------------------------------------------------------------------ # Fill in config_compilers.xml @@ -558,7 +559,6 @@ def _fill_out_machine_files(build_dir, 'NETCDF_PATH':netcdf_path, 'PIO_FILESYSTEM_HINTS':pio_filesystem_hints_tag, 'PNETCDF_PATH':pnetcdf_path_tag, - 'ESMF_LIBDIR':esmf_lib_path, 'EXTRA_CFLAGS':extra_cflags, 'EXTRA_FFLAGS':extra_fflags}) diff --git a/python/ctsm/test/test_sys_lilac_build_ctsm.py b/python/ctsm/test/test_sys_lilac_build_ctsm.py index 5a44688171..3c4117fd45 100755 --- a/python/ctsm/test/test_sys_lilac_build_ctsm.py +++ b/python/ctsm/test/test_sys_lilac_build_ctsm.py @@ -46,7 +46,7 @@ def test_buildSetup_userDefinedMachine_minimalInfo(self): no_build=True, os_type='linux', netcdf_path='/path/to/netcdf', - esmf_lib_path='/path/to/esmf/lib', + esmf_mkfile_path='/path/to/esmf/lib/esmf.mk', max_mpitasks_per_node=16, gmake='gmake', gmake_j=8, @@ -76,7 +76,7 @@ def test_buildSetup_userDefinedMachine_allInfo(self): no_build=True, os_type='linux', netcdf_path='/path/to/netcdf', - esmf_lib_path='/path/to/esmf/lib', + esmf_mkfile_path='/path/to/esmf/lib/esmf.mk', max_mpitasks_per_node=16, gmake='gmake', gmake_j=8, diff --git a/python/ctsm/test/test_unit_lilac_build_ctsm.py b/python/ctsm/test/test_unit_lilac_build_ctsm.py index 3c1a600326..96f79f3765 100755 --- a/python/ctsm/test/test_unit_lilac_build_ctsm.py +++ b/python/ctsm/test/test_unit_lilac_build_ctsm.py @@ -105,7 +105,7 @@ def test_commandlineArgs_noRebuild_valid(self): '--os', 'linux', '--compiler', 'intel', '--netcdf-path', '/path/to/netcdf', - '--esmf-lib-path', '/path/to/esmf/lib', + '--esmf-mkfile-path', '/path/to/esmf/lib/esmf.mk', '--max-mpitasks-per-node', '16', '--no-pnetcdf']) @@ -120,7 +120,7 @@ def test_commandlineArgs_noRebuild_invalid1(self, mock_stderr): _ = _commandline_args(args_to_parse=['build/directory', '--os', 'linux', '--netcdf-path', '/path/to/netcdf', - '--esmf-lib-path', '/path/to/esmf/lib', + '--esmf-mkfile-path', '/path/to/esmf/lib/esmf.mk', '--max-mpitasks-per-node', '16', '--no-pnetcdf']) self.assertRegex(mock_stderr.getvalue(), expected_re) @@ -136,7 +136,7 @@ def test_commandlineArgs_noRebuild_invalid2(self, mock_stderr): _ = _commandline_args(args_to_parse=['build/directory', '--compiler', 'intel', '--netcdf-path', '/path/to/netcdf', - '--esmf-lib-path', '/path/to/esmf/lib', + '--esmf-mkfile-path', '/path/to/esmf/lib/esmf.mk', '--max-mpitasks-per-node', '16', '--no-pnetcdf']) self.assertRegex(mock_stderr.getvalue(), expected_re) @@ -150,7 +150,7 @@ def test_commandlineArgs_noRebuild_invalidNeedToDictatePnetcdf(self, mock_stderr '--os', 'linux', '--compiler', 'intel', '--netcdf-path', '/path/to/netcdf', - '--esmf-lib-path', '/path/to/esmf/lib', + '--esmf-mkfile-path', '/path/to/esmf/lib/esmf.mk', '--max-mpitasks-per-node', '16']) self.assertRegex(mock_stderr.getvalue(), expected_re) @@ -163,7 +163,7 @@ def test_commandlineArgs_noRebuild_invalidConflictingPnetcdf(self, mock_stderr): '--os', 'linux', '--compiler', 'intel', '--netcdf-path', '/path/to/netcdf', - '--esmf-lib-path', '/path/to/esmf/lib', + '--esmf-mkfile-path', '/path/to/esmf/lib/esmf.mk', '--max-mpitasks-per-node', '16', '--no-pnetcdf', '--pnetcdf-path', '/path/to/pnetcdf']) From 06c99c6c9eed74bfec53c4c2ee6b66f490f0a284 Mon Sep 17 00:00:00 2001 From: Bill Sacks Date: Tue, 7 Dec 2021 10:31:52 -0700 Subject: [PATCH 38/55] Get lilac user-defined machine build working with latest cime config_compilers.xml is deprecated; need a cmake macros file instead. Resolves ESCOMP/CTSM#1492 --- .../config_compilers_template.xml | 42 ------------------- .../config_machines_template.xml | 2 +- lilac/bld_templates/ctsm-build_template.cmake | 23 ++++++++++ python/ctsm/lilac_build_ctsm.py | 26 ++++++------ 4 files changed, 37 insertions(+), 56 deletions(-) delete mode 100644 lilac/bld_templates/config_compilers_template.xml create mode 100644 lilac/bld_templates/ctsm-build_template.cmake diff --git a/lilac/bld_templates/config_compilers_template.xml b/lilac/bld_templates/config_compilers_template.xml deleted file mode 100644 index 143e088776..0000000000 --- a/lilac/bld_templates/config_compilers_template.xml +++ /dev/null @@ -1,42 +0,0 @@ - - - - - - - - - - $GPTL_CPPDEFS - - - $NETCDF_PATH - - - $PIO_FILESYSTEM_HINTS - - - $PNETCDF_PATH - - - $EXTRA_CFLAGS - - - - $EXTRA_FFLAGS - - - - - diff --git a/lilac/bld_templates/config_machines_template.xml b/lilac/bld_templates/config_machines_template.xml index 6eff902242..ccde849607 100644 --- a/lilac/bld_templates/config_machines_template.xml +++ b/lilac/bld_templates/config_machines_template.xml @@ -12,7 +12,7 @@ --> - + Temporary build information for a CTSM build diff --git a/lilac/bld_templates/ctsm-build_template.cmake b/lilac/bld_templates/ctsm-build_template.cmake new file mode 100644 index 0000000000..a4dae78533 --- /dev/null +++ b/lilac/bld_templates/ctsm-build_template.cmake @@ -0,0 +1,23 @@ +# This, together with config_machines_template.xml, provides a machine port for building +# CTSM. +# +# If you are looking at the template file: Variable names prefixed with a dollar sign will +# be replaced with machine-specific values. A double dollar sign gets replaced with a +# single dollar sign, so something like $$MYVAR refers to the MYVAR cime variable. + +if (COMP_NAME STREQUAL gptl) + string(APPEND CPPDEFS " $GPTL_CPPDEFS") +endif() + +set(NETCDF_PATH "$NETCDF_PATH") + +# If PIO_FILESYSTEM_HINTS is provided, this will set a PIO_FILESYSTEM_HINTS variable; if +# not provided, this will just be a blank line. +$PIO_FILESYSTEM_HINTS + +# If PNETCDF_PATH is provided, this will set a PNETCDF_PATH variable; if not provided, +# this will just be a blank line. +$PNETCDF_PATH + +string(APPEND CFLAGS " $EXTRA_CFLAGS") +string(APPEND FFLAGS " $EXTRA_FFLAGS") diff --git a/python/ctsm/lilac_build_ctsm.py b/python/ctsm/lilac_build_ctsm.py index 0d2ae4d1b9..be1fbf7350 100644 --- a/python/ctsm/lilac_build_ctsm.py +++ b/python/ctsm/lilac_build_ctsm.py @@ -18,7 +18,7 @@ # ======================================================================== # this matches the machine name in config_machines_template.xml -_MACH_NAME = 'ctsm_build' +_MACH_NAME = 'ctsm-build' # these are arbitrary, since we only use the case for its build, not any of the runtime # settings; they just need to be valid @@ -512,7 +512,7 @@ def _fill_out_machine_files(build_dir, For documentation of args, see the documentation in the build_ctsm function """ - os.makedirs(os.path.join(build_dir, _MACHINE_CONFIG_DIRNAME)) + os.makedirs(os.path.join(build_dir, _MACHINE_CONFIG_DIRNAME, "cmake_macros")) # ------------------------------------------------------------------------ # Fill in config_machines.xml @@ -530,7 +530,7 @@ def _fill_out_machine_files(build_dir, 'ESMF_MKFILE_PATH':esmf_mkfile_path}) # ------------------------------------------------------------------------ - # Fill in config_compilers.xml + # Fill in ctsm-build_template.cmake # ------------------------------------------------------------------------ if gptl_nano_timers: @@ -539,26 +539,26 @@ def _fill_out_machine_files(build_dir, gptl_cppdefs = '' if pio_filesystem_hints: - pio_filesystem_hints_tag = '{}'.format( + pio_filesystem_hints_addition = 'set(PIO_FILESYSTEM_HINTS "{}")'.format( pio_filesystem_hints) else: - pio_filesystem_hints_tag = '' + pio_filesystem_hints_addition = '' if pnetcdf_path: - pnetcdf_path_tag = '{}'.format( + pnetcdf_path_addition = 'set(PNETCDF_PATH "{}")'.format( pnetcdf_path) else: - pnetcdf_path_tag = '' + pnetcdf_path_addition = '' fill_template_file( path_to_template=os.path.join(_PATH_TO_TEMPLATES, - 'config_compilers_template.xml'), - path_to_final=os.path.join(build_dir, _MACHINE_CONFIG_DIRNAME, 'config_compilers.xml'), - substitutions={'COMPILER':compiler, - 'GPTL_CPPDEFS':gptl_cppdefs, + 'ctsm-build_template.cmake'), + path_to_final=os.path.join(build_dir, _MACHINE_CONFIG_DIRNAME, "cmake_macros", + '{}_{}.cmake'.format(compiler, _MACH_NAME)), + substitutions={'GPTL_CPPDEFS':gptl_cppdefs, 'NETCDF_PATH':netcdf_path, - 'PIO_FILESYSTEM_HINTS':pio_filesystem_hints_tag, - 'PNETCDF_PATH':pnetcdf_path_tag, + 'PIO_FILESYSTEM_HINTS':pio_filesystem_hints_addition, + 'PNETCDF_PATH':pnetcdf_path_addition, 'EXTRA_CFLAGS':extra_cflags, 'EXTRA_FFLAGS':extra_fflags}) From 9963e463c53a1f7f5cdea15fc60336a3c9d29b60 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Thu, 9 Dec 2021 11:04:57 -0700 Subject: [PATCH 39/55] Update to cesm2_3_alpha07c externals --- Externals.cfg | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Externals.cfg b/Externals.cfg index d3ee5df893..b664641b74 100644 --- a/Externals.cfg +++ b/Externals.cfg @@ -41,7 +41,7 @@ tag = cime6.0.11 required = True [cmeps] -tag = cmeps0.13.40 +tag = cmeps0.13.42 protocol = git repo_url = https://github.com/ESCOMP/CMEPS.git local_path = components/cmeps @@ -56,14 +56,14 @@ externals = Externals_CDEPS.cfg required = True [cpl7] -tag = cpl7.0.5 +tag = cpl7.0.7 protocol = git repo_url = https://github.com/ESCOMP/CESM_CPL7andDataComps local_path = components/cpl7 required = True [share] -tag = share1.0.8 +tag = share1.0.10 protocol = git repo_url = https://github.com/ESCOMP/CESM_share local_path = share From 4c1b3555477f1ca1536b5dcb692b7baba20dbda6 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Thu, 9 Dec 2021 12:48:55 -0700 Subject: [PATCH 40/55] Switch out glob for bsd_glog because it's being removed in newer perl versions this fixes #1550 --- tools/mksurfdata_map/src/Mkdepends | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tools/mksurfdata_map/src/Mkdepends b/tools/mksurfdata_map/src/Mkdepends index a75e8fdde0..ddb1682da4 100755 --- a/tools/mksurfdata_map/src/Mkdepends +++ b/tools/mksurfdata_map/src/Mkdepends @@ -35,6 +35,7 @@ use Getopt::Std; use File::Basename; +use File::Glob ':bsd_glob'; # Check for usage request. @ARGV >= 2 or usage(); @@ -61,7 +62,7 @@ chomp @file_paths; unshift(@file_paths,'.'); foreach $dir (@file_paths) { # (could check that directories exist here) $dir =~ s!/?\s*$!!; # remove / and any whitespace at end of directory name - ($dir) = glob $dir; # Expand tildes in path names. + ($dir) = bsd_glob $dir; # Expand tildes in path names. } # Make list of files containing source code. @@ -91,7 +92,7 @@ my ($dir); my ($f, $name, $path, $suffix, $mod); my @suffixes = ('\.mod' ); foreach $dir (@file_paths) { - @filenames = (glob("$dir/*.mod")); + @filenames = (bsd_glob("$dir/*.mod")); foreach $f (@filenames) { ($name, $path, $suffix) = fileparse($f, @suffixes); ($mod = $name) =~ tr/a-z/A-Z/; From 9f767fac0c96d41f068c774aa8e85b1018b9fc7d Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Thu, 9 Dec 2021 13:24:44 -0700 Subject: [PATCH 41/55] Add one to the number of skip steps as required by the new CAM configuration, now needs to be after the radiation time-step this fixes #1563 --- src/biogeophys/BalanceCheckMod.F90 | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/biogeophys/BalanceCheckMod.F90 b/src/biogeophys/BalanceCheckMod.F90 index 508e118b66..ec928f8645 100644 --- a/src/biogeophys/BalanceCheckMod.F90 +++ b/src/biogeophys/BalanceCheckMod.F90 @@ -82,7 +82,8 @@ subroutine BalanceCheckInit( ) !----------------------------------------------------------------------- dtime = get_step_size_real() ! Skip a minimum of two time steps, but otherwise skip the number of time-steps in the skip_size rounded to the nearest integer - skip_steps = max(2, nint( (skip_size / dtime) ) ) + ! Add an additional step as now required to be after the hourly radiation time-step see github issue #1563 + skip_steps = max(2, nint( (skip_size / dtime) ) ) + 1 if ( masterproc ) write(iulog,*) ' Skip balance checking for the first ', skip_steps, ' time steps' From 3d16070c40a65977a897888371b0f523bb2a977d Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Thu, 9 Dec 2021 13:39:17 -0700 Subject: [PATCH 42/55] Add a target to the makefile to run the python code through black --- python/Makefile | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/python/Makefile b/python/Makefile index 6c7e1ab32c..d3b80e5a89 100644 --- a/python/Makefile +++ b/python/Makefile @@ -23,7 +23,7 @@ PYLINT_ARGS=-j 4 --rcfile=ctsm/.pylintrc PYLINT_SRC = \ ctsm -all: test lint +all: test lint black test: utest stest .PHONY: utest @@ -38,6 +38,10 @@ stest: FORCE lint: FORCE $(PYLINT) $(PYLINT_ARGS) $(PYLINT_SRC) +.PHONY: black +black: FORCE + black $(PYLINT_SRC) + .PHONY: clean clean: FORCE find . -name '*.pyc' -exec rm {} \; From 5f00876908aa4d78fb5219b07366806aa5b9088e Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Thu, 9 Dec 2021 14:10:02 -0700 Subject: [PATCH 43/55] Update cmeps to version which allows channel depths to be passed --- Externals.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Externals.cfg b/Externals.cfg index b664641b74..10963cfd20 100644 --- a/Externals.cfg +++ b/Externals.cfg @@ -41,7 +41,7 @@ tag = cime6.0.11 required = True [cmeps] -tag = cmeps0.13.42 +tag = cmeps0.13.43 protocol = git repo_url = https://github.com/ESCOMP/CMEPS.git local_path = components/cmeps From a06a059c1ea2b6ebd9b2dfbf8ce8ce8486783a22 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Fri, 10 Dec 2021 11:43:38 -0700 Subject: [PATCH 44/55] Get balance check unit test on skip steps working with change in skip_steps --- src/biogeophys/test/Balance_test/test_Balance.pf | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/biogeophys/test/Balance_test/test_Balance.pf b/src/biogeophys/test/Balance_test/test_Balance.pf index 3d07385ffb..824eebb78c 100644 --- a/src/biogeophys/test/Balance_test/test_Balance.pf +++ b/src/biogeophys/test/Balance_test/test_Balance.pf @@ -46,7 +46,7 @@ contains call unittest_timemgr_setup(dtime=dtime) call BalanceCheckInit() nskip = GetBalanceCheckSkipSteps() - @assertEqual( 2, nskip, message="Ensure standard balance check is 2 time-steps" ) + @assertEqual( 3, nskip, message="Ensure standard balance check is 3 time-steps" ) end subroutine test_balance_init @Test @@ -59,7 +59,7 @@ contains call unittest_timemgr_setup(dtime=dtime) call BalanceCheckInit() nskip = GetBalanceCheckSkipSteps() - @assertEqual( 2, nskip, message="Ensure even with a long time-step skip is 2 time-steps" ) + @assertEqual( 3, nskip, message="Ensure even with a long time-step skip is 3 time-steps" ) end subroutine test_balance_longstep @Test @@ -72,7 +72,7 @@ contains call unittest_timemgr_setup(dtime=dtime) call BalanceCheckInit() nskip = GetBalanceCheckSkipSteps() - @assertEqual( 12, nskip, message="Check skip length for 300 sec time-step" ) + @assertEqual( 13, nskip, message="Check skip length for 300 sec time-step" ) end subroutine test_balance_300sec @Test @@ -101,7 +101,7 @@ contains call unittest_timemgr_setup(dtime=dtime) call BalanceCheckInit() nskip = GetBalanceCheckSkipSteps() - @assertEqual( 100, nskip, message="Ensure with a short step correct number of skip steps is done" ) + @assertEqual( 101, nskip, message="Ensure with a short step correct number of skip steps is done" ) end subroutine test_balance_shortstep end module test_balance From abb3b30069a23c591bfdbd7423d81d9a7ba5c926 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Fri, 10 Dec 2021 14:57:36 -0700 Subject: [PATCH 45/55] Change the call of black from a reformat to a check, incldue the main script and exclude the pytlint file, also have pylint ignore an option that black uses so they are compatable --- python/Makefile | 4 +++- python/ctsm/.pylintrc | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/python/Makefile b/python/Makefile index d3b80e5a89..b484dc2b02 100644 --- a/python/Makefile +++ b/python/Makefile @@ -39,8 +39,10 @@ lint: FORCE $(PYLINT) $(PYLINT_ARGS) $(PYLINT_SRC) .PHONY: black +# Run black on all of the .py* python modules under python as well as the main top level script +# Exclude the pylint file black: FORCE - black $(PYLINT_SRC) + black --check --include '(run_ctsm_py_tests|\.py)' --exclude '(\.pylintrc)' . .PHONY: clean clean: FORCE diff --git a/python/ctsm/.pylintrc b/python/ctsm/.pylintrc index 46c4837b6c..bc7ae54dd2 100644 --- a/python/ctsm/.pylintrc +++ b/python/ctsm/.pylintrc @@ -140,6 +140,7 @@ disable=print-statement, deprecated-sys-function, exception-escape, comprehension-escape, + C0330, # This is an option that the formatter "black" requires us to disable # --- default list is above here, our own list is below here --- # While pylint's recommendations to keep the number of arguments, local # variables and branches low is generally a good one, I don't want it to From b407a371430041dfbaf0a876709032a69f14c6c7 Mon Sep 17 00:00:00 2001 From: Negin Sobhani Date: Fri, 10 Dec 2021 17:08:45 -0700 Subject: [PATCH 46/55] updated neon surface dataset --- cime_config/usermods_dirs/NEON/defaults/user_nl_clm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cime_config/usermods_dirs/NEON/defaults/user_nl_clm b/cime_config/usermods_dirs/NEON/defaults/user_nl_clm index 818b013b9c..83513908de 100644 --- a/cime_config/usermods_dirs/NEON/defaults/user_nl_clm +++ b/cime_config/usermods_dirs/NEON/defaults/user_nl_clm @@ -19,7 +19,7 @@ !---------------------------------------------------------------------------------- flanduse_timeseries = ' ' ! This isn't needed for a non transient case, but will be once we start using transient compsets -fsurdat = "$DIN_LOC_ROOT/lnd/clm2/surfdata_map/NEON/surfdata_hist_78pfts_CMIP6_simyr2000_${NEONSITE}_c210831.nc" +fsurdat = "$DIN_LOC_ROOT/lnd/clm2/surfdata_map/NEON/surfdata_hist_78pfts_CMIP6_simyr2000_${NEONSITE}_c211102.nc" model_year_align_urbantv = 2018 stream_year_first_urbantv = 2018 stream_year_last_urbantv = 2019 From 411c321adc78741ef632c875acfa46cb5575c33b Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Fri, 10 Dec 2021 17:41:18 -0700 Subject: [PATCH 47/55] Update last year to 2020 as we have data through the end of 2020 now --- cime_config/usermods_dirs/NEON/defaults/shell_commands | 2 +- cime_config/usermods_dirs/NEON/defaults/user_nl_clm | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cime_config/usermods_dirs/NEON/defaults/shell_commands b/cime_config/usermods_dirs/NEON/defaults/shell_commands index 2ebe1b4f86..f82278e4b5 100644 --- a/cime_config/usermods_dirs/NEON/defaults/shell_commands +++ b/cime_config/usermods_dirs/NEON/defaults/shell_commands @@ -3,4 +3,4 @@ ./xmlchange CLM_NML_USE_CASE=1850-2100_SSP3-7.0_transient ./xmlchange CCSM_CO2_PPMV=408.83 ./xmlchange DATM_PRESAERO=SSP3-7.0 -./xmlchange DATM_YR_ALIGN=2018,DATM_YR_END=2019,DATM_YR_START=2018 +./xmlchange DATM_YR_ALIGN=2018,DATM_YR_END=2020,DATM_YR_START=2018 diff --git a/cime_config/usermods_dirs/NEON/defaults/user_nl_clm b/cime_config/usermods_dirs/NEON/defaults/user_nl_clm index 818b013b9c..0f84a9cb79 100644 --- a/cime_config/usermods_dirs/NEON/defaults/user_nl_clm +++ b/cime_config/usermods_dirs/NEON/defaults/user_nl_clm @@ -22,13 +22,13 @@ flanduse_timeseries = ' ' ! This isn't needed for a non transient case, but wi fsurdat = "$DIN_LOC_ROOT/lnd/clm2/surfdata_map/NEON/surfdata_hist_78pfts_CMIP6_simyr2000_${NEONSITE}_c210831.nc" model_year_align_urbantv = 2018 stream_year_first_urbantv = 2018 -stream_year_last_urbantv = 2019 +stream_year_last_urbantv = 2020 stream_year_first_ndep = 2018 model_year_align_ndep = 2018 -stream_year_last_ndep = 2019 +stream_year_last_ndep = 2020 model_year_align_popdens = 2018 stream_year_first_popdens = 2018 -stream_year_last_popdens = 2019 +stream_year_last_popdens = 2020 stream_fldfilename_lightng = '$DIN_LOC_ROOT/atm/datm7/NASA_LIS/clmforc.Li_2016_climo1995-2013.360x720.lnfm_Total_NEONarea_c210625.nc' stream_fldfilename_ndep = '$DIN_LOC_ROOT/lnd/clm2/ndepdata/fndep_clm_f09_g17.CMIP6-SSP3-7.0-WACCM_2018-2030_monthly_c210826.nc' ! h1 output stream From abe76e95d72d81eca77ea0b30674738f908be2ec Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Sat, 11 Dec 2021 00:38:17 -0700 Subject: [PATCH 48/55] Remove tests from the test lists that were removed from the master list --- test/tools/tests_posttag_nompi_regression | 1 - test/tools/tests_pretag_cheyenne_nompi | 2 -- 2 files changed, 3 deletions(-) diff --git a/test/tools/tests_posttag_nompi_regression b/test/tools/tests_posttag_nompi_regression index 1785b5da47..bb59e4cb9b 100644 --- a/test/tools/tests_posttag_nompi_regression +++ b/test/tools/tests_posttag_nompi_regression @@ -1,5 +1,4 @@ smc#4 blc#4 -sme14 ble14 smg54 blg54 smi24 bli24 smi53 bli53 diff --git a/test/tools/tests_pretag_cheyenne_nompi b/test/tools/tests_pretag_cheyenne_nompi index f99ab6b691..856c7511eb 100644 --- a/test/tools/tests_pretag_cheyenne_nompi +++ b/test/tools/tests_pretag_cheyenne_nompi @@ -1,7 +1,5 @@ smi79 bli79 smc#4 blc#4 -sme14 ble14 -sme@4 ble@4 smg54 blg54 sm0a1 bl0a1 smaa2 blaa2 From e9953fc2103d53ba0b44a742a31234db225e0f78 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Sat, 11 Dec 2021 12:33:43 -0700 Subject: [PATCH 49/55] Remove smf tests from tests lists as they have been removed from the master list --- test/tools/tests_posttag_hobart_nompi | 2 -- test/tools/tests_posttag_izumi_nompi | 2 -- test/tools/tests_posttag_nompi_regression | 3 --- test/tools/tests_pretag_cheyenne_nompi | 3 --- 4 files changed, 10 deletions(-) diff --git a/test/tools/tests_posttag_hobart_nompi b/test/tools/tests_posttag_hobart_nompi index 4655f29853..9f07863e4d 100644 --- a/test/tools/tests_posttag_hobart_nompi +++ b/test/tools/tests_posttag_hobart_nompi @@ -2,5 +2,3 @@ smc#4 blc#4 smi54 bli54 smi57 bli57 smiT4 bliT4 -smf84 blf84 -smfc4 blfc4 diff --git a/test/tools/tests_posttag_izumi_nompi b/test/tools/tests_posttag_izumi_nompi index 90be9522dc..62687a7e3d 100644 --- a/test/tools/tests_posttag_izumi_nompi +++ b/test/tools/tests_posttag_izumi_nompi @@ -1,5 +1,3 @@ smi54 bli54 smi57 bli57 smiT4 bliT4 -smf84 blf84 -smfc4 blfc4 diff --git a/test/tools/tests_posttag_nompi_regression b/test/tools/tests_posttag_nompi_regression index bb59e4cb9b..5b5d76fd60 100644 --- a/test/tools/tests_posttag_nompi_regression +++ b/test/tools/tests_posttag_nompi_regression @@ -9,9 +9,6 @@ smi74 bli74 smi78 bli78 smiT4 bliT4 smiT2 bliT2 -smf84 blf84 -smfc4 blfc4 -smfg4 blfg4 smiS4 bliS4 smiS8 bliS8 smiS9 bliS9 diff --git a/test/tools/tests_pretag_cheyenne_nompi b/test/tools/tests_pretag_cheyenne_nompi index 856c7511eb..fec9d08448 100644 --- a/test/tools/tests_pretag_cheyenne_nompi +++ b/test/tools/tests_pretag_cheyenne_nompi @@ -15,6 +15,3 @@ smiS4 bliS4 smi74 bli74 smiT4 bliT4 smiT2 bliT2 -smf84 blf84 -smfc4 blfc4 -smfg4 blfg4 From e043f535cc9c37b779e46da7402599a760ae1d97 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Sat, 11 Dec 2021 13:25:28 -0700 Subject: [PATCH 50/55] Add note about running a baseline compare case --- test/tools/README | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/test/tools/README b/test/tools/README index 186f1dfe5d..cb5dcdec34 100644 --- a/test/tools/README +++ b/test/tools/README @@ -15,6 +15,11 @@ on cheyenne qcmd -l walltime=08:00:00 -- ./test_driver.sh -i >& run.out & +And to for example to compare to another baseline code (in this case ctsm5.1.dev066, which would need to be cloned at the given +path) ... + +qcmd -l walltime=08:00:00 -- env BL_ROOT=/glade/scratch/erik/ctsm5.1.dev066 ./test_driver.sh -i >& run.out & + on izumi nohup ./test_driver.sh -i >& run.out & From 9afbad8cb954194484108b1d33ce2b5b1b65c425 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Sat, 11 Dec 2021 13:59:05 -0700 Subject: [PATCH 51/55] Move explicit settings for black and list of includes and excludes to a config file for it --- python/Makefile | 6 +++--- python/pyproject.toml | 13 +++++++++++++ 2 files changed, 16 insertions(+), 3 deletions(-) create mode 100644 python/pyproject.toml diff --git a/python/Makefile b/python/Makefile index b484dc2b02..4ea5fba85d 100644 --- a/python/Makefile +++ b/python/Makefile @@ -39,10 +39,10 @@ lint: FORCE $(PYLINT) $(PYLINT_ARGS) $(PYLINT_SRC) .PHONY: black -# Run black on all of the .py* python modules under python as well as the main top level script -# Exclude the pylint file +# Run black on all of the python files here and undeneath. +# Use the black configure file to explicitly set a few things and specifiy the exact files. black: FORCE - black --check --include '(run_ctsm_py_tests|\.py)' --exclude '(\.pylintrc)' . + black --check --config pyproject.toml . .PHONY: clean clean: FORCE diff --git a/python/pyproject.toml b/python/pyproject.toml new file mode 100644 index 0000000000..fd8d8ac03c --- /dev/null +++ b/python/pyproject.toml @@ -0,0 +1,13 @@ +# +# This is a configuration file for python projects. +# Sepcifically covering build system requirements. +# +# Here we are just using a couple options to specify the operation +# of the python formatter "black". +# +[tool.black] + + line-length = 88 # This is the black default + target-version = ['py37'] + include = '(run_ctsm_py_tests|\.py$)' # Files to include + exclude = '(\.pylintrc|\.pyc)' # Files to explicitly exclude pylint file and compiled python From 07214b4dc5db58dcf5f57ea5a1993109f300262b Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Sat, 11 Dec 2021 23:14:23 -0700 Subject: [PATCH 52/55] Change stop statements into calls to abort() so that they will pass an error code up the chain and the code can die rather than going on as if an error didn't happen --- tools/mksurfdata_map/src/mkVICparamsMod.F90 | 8 ++++---- tools/mksurfdata_map/src/mkagfirepkmonthMod.F90 | 2 +- tools/mksurfdata_map/src/mkdiagnosticsMod.F90 | 8 ++++---- tools/mksurfdata_map/src/mkgdpMod.F90 | 2 +- tools/mksurfdata_map/src/mkglacierregionMod.F90 | 2 +- tools/mksurfdata_map/src/mkglcmecMod.F90 | 2 +- tools/mksurfdata_map/src/mkgridmapMod.F90 | 2 +- tools/mksurfdata_map/src/mklaiMod.F90 | 4 ++-- tools/mksurfdata_map/src/mklanwatMod.F90 | 2 +- tools/mksurfdata_map/src/mkpeatMod.F90 | 2 +- tools/mksurfdata_map/src/mkpftMod.F90 | 4 ++-- tools/mksurfdata_map/src/mksoilMod.F90 | 12 ++++++------ tools/mksurfdata_map/src/mksoildepthMod.F90 | 2 +- tools/mksurfdata_map/src/mksurfdat.F90 | 6 +++--- tools/mksurfdata_map/src/mktopostatsMod.F90 | 4 ++-- tools/mksurfdata_map/src/mkurbanparCommonMod.F90 | 8 ++++---- tools/mksurfdata_map/src/mkurbanparMod.F90 | 6 +++--- 17 files changed, 38 insertions(+), 38 deletions(-) diff --git a/tools/mksurfdata_map/src/mkVICparamsMod.F90 b/tools/mksurfdata_map/src/mkVICparamsMod.F90 index f7cb4946c6..431e43cb28 100644 --- a/tools/mksurfdata_map/src/mkVICparamsMod.F90 +++ b/tools/mksurfdata_map/src/mkVICparamsMod.F90 @@ -129,7 +129,7 @@ subroutine mkVICparams(ldomain, mapfname, datfname, ndiag, & ! Check validity of output data if (min_bad(binfl_o, min_valid_binfl, 'binfl')) then - stop + call abort() end if call output_diagnostics_continuous(data_i, binfl_o, tgridmap, "VIC b parameter", "unitless", ndiag, tdomain%mask, frac_dst) @@ -144,7 +144,7 @@ subroutine mkVICparams(ldomain, mapfname, datfname, ndiag, & ! Check validity of output data if (min_bad(ws_o, min_valid_ws, 'Ws')) then - stop + call abort() end if call output_diagnostics_continuous(data_i, ws_o, tgridmap, "VIC Ws parameter", "unitless", ndiag, tdomain%mask, frac_dst) @@ -159,7 +159,7 @@ subroutine mkVICparams(ldomain, mapfname, datfname, ndiag, & ! Check validity of output data if (min_bad(dsmax_o, min_valid_dsmax, 'Dsmax')) then - stop + call abort() end if call output_diagnostics_continuous(data_i, dsmax_o, tgridmap, "VIC Dsmax parameter", "mm/day", ndiag, tdomain%mask, frac_dst) @@ -174,7 +174,7 @@ subroutine mkVICparams(ldomain, mapfname, datfname, ndiag, & ! Check validity of output data if (min_bad(ds_o, min_valid_ds, 'Ds')) then - stop + call abort() end if call output_diagnostics_continuous(data_i, ds_o, tgridmap, "VIC Ds parameter", "unitless", ndiag, tdomain%mask, frac_dst) diff --git a/tools/mksurfdata_map/src/mkagfirepkmonthMod.F90 b/tools/mksurfdata_map/src/mkagfirepkmonthMod.F90 index af8001263f..7b58ddffad 100644 --- a/tools/mksurfdata_map/src/mkagfirepkmonthMod.F90 +++ b/tools/mksurfdata_map/src/mkagfirepkmonthMod.F90 @@ -145,7 +145,7 @@ subroutine mkagfirepkmon(ldomain, mapfname, datfname, ndiag, & ! Check validity of output data if (min_bad(agfirepkmon_o, min_valid, 'agfirepkmon') .or. & max_bad(agfirepkmon_o, max_valid, 'agfirepkmon')) then - stop + call abort() end if diff --git a/tools/mksurfdata_map/src/mkdiagnosticsMod.F90 b/tools/mksurfdata_map/src/mkdiagnosticsMod.F90 index a53d9ca4d2..91769a5823 100644 --- a/tools/mksurfdata_map/src/mkdiagnosticsMod.F90 +++ b/tools/mksurfdata_map/src/mkdiagnosticsMod.F90 @@ -81,7 +81,7 @@ subroutine output_diagnostics_area(data_i, data_o, gridmap, name, percent, ndiag write(6,*) 'ns_i = ', ns_i write(6,*) 'size(data_o) = ', size(data_o) write(6,*) 'ns_o = ', ns_o - stop + call abort() end if if (size(frac_dst) /= ns_o) then write(6,*) subname//' ERROR: incorrect size of frac_dst' @@ -195,7 +195,7 @@ subroutine output_diagnostics_continuous(data_i, data_o, gridmap, name, units, n write(6,*) 'ns_i = ', ns_i write(6,*) 'size(data_o) = ', size(data_o) write(6,*) 'ns_o = ', ns_o - stop + call abort() end if if (size(frac_dst) /= ns_o) then write(6,*) subname//' ERROR: incorrect size of frac_dst' @@ -300,7 +300,7 @@ subroutine output_diagnostics_continuous_outonly(data_o, gridmap, name, units, n write(6,*) subname//' ERROR: array size inconsistencies for ', trim(name) write(6,*) 'size(data_o) = ', size(data_o) write(6,*) 'ns_o = ', ns_o - stop + call abort() end if ! Sums on output grid @@ -380,7 +380,7 @@ subroutine output_diagnostics_index(data_i, data_o, gridmap, name, & write(6,*) 'ns_i = ', ns_i write(6,*) 'size(data_o) = ', size(data_o) write(6,*) 'ns_o = ', ns_o - stop + call abort() end if if (size(frac_dst) /= ns_o) then write(6,*) subname//' ERROR: incorrect size of frac_dst' diff --git a/tools/mksurfdata_map/src/mkgdpMod.F90 b/tools/mksurfdata_map/src/mkgdpMod.F90 index 6a560e61b5..138ddf1805 100644 --- a/tools/mksurfdata_map/src/mkgdpMod.F90 +++ b/tools/mksurfdata_map/src/mkgdpMod.F90 @@ -122,7 +122,7 @@ subroutine mkgdp(ldomain, mapfname, datfname, ndiag, gdp_o) ! Check validity of output data if (min_bad(gdp_o, min_valid, 'gdp')) then - stop + call abort() end if call output_diagnostics_continuous(data_i, gdp_o, tgridmap, "GDP", "x1000 US$ per capita", ndiag, tdomain%mask, frac_dst) diff --git a/tools/mksurfdata_map/src/mkglacierregionMod.F90 b/tools/mksurfdata_map/src/mkglacierregionMod.F90 index beae6a8d97..e644129ed3 100644 --- a/tools/mksurfdata_map/src/mkglacierregionMod.F90 +++ b/tools/mksurfdata_map/src/mkglacierregionMod.F90 @@ -105,7 +105,7 @@ subroutine mkglacierregion(ldomain, mapfname, datfname, ndiag, & call check_ret(nf_inq_varid(ncid, 'GLACIER_REGION', varid), subname) call check_ret(nf_get_var_int(ncid, varid, glacier_region_i), subname) if (min_bad(glacier_region_i, 0, 'GLACIER_REGION')) then - stop + call abort() end if call get_max_indices( & diff --git a/tools/mksurfdata_map/src/mkglcmecMod.F90 b/tools/mksurfdata_map/src/mkglcmecMod.F90 index 2ac4d94e4f..9fbad66689 100644 --- a/tools/mksurfdata_map/src/mkglcmecMod.F90 +++ b/tools/mksurfdata_map/src/mkglcmecMod.F90 @@ -606,7 +606,7 @@ subroutine mkglacier(ldomain, mapfname, datfname, ndiag, zero_out, glac_o) write (6,*) 'MKGLACIER error: glacier = ',glac_o(no), & ' greater than 100.000001 for column, row = ',no call shr_sys_flush(6) - stop + call abort() end if enddo diff --git a/tools/mksurfdata_map/src/mkgridmapMod.F90 b/tools/mksurfdata_map/src/mkgridmapMod.F90 index 21ca23f4d6..eeb5afdbb8 100644 --- a/tools/mksurfdata_map/src/mkgridmapMod.F90 +++ b/tools/mksurfdata_map/src/mkgridmapMod.F90 @@ -505,7 +505,7 @@ subroutine gridmap_check(gridmap, mask_src, frac_dst, caller) write (6,*) subname//' ERROR from '//trim(caller)//': mapping areas not conserved' write (6,'(a30,e20.10)') 'global sum output field = ',sum_area_o write (6,'(a30,e20.10)') 'global sum input field = ',sum_area_i - stop + call abort() end if end if diff --git a/tools/mksurfdata_map/src/mklaiMod.F90 b/tools/mksurfdata_map/src/mklaiMod.F90 index aef33f3463..e4b6d9bfa1 100644 --- a/tools/mksurfdata_map/src/mklaiMod.F90 +++ b/tools/mksurfdata_map/src/mklaiMod.F90 @@ -140,7 +140,7 @@ subroutine mklai(ldomain, mapfname, datfname, ndiag, ncido) ! invalid, all the loop bounds over output data in this ! routine will need to be double checked! write(6, *) "ERROR:" // trim(subname) // "(): input numpft must be less than or equal to output numpft+1." - stop + call abort() end if endif if (ntim /= 12) then @@ -434,7 +434,7 @@ subroutine pft_laicheck( ni_s, pctpft_i, laimask ) write (6,*) subName//' :: pft/LAI+SAI inconsistency over more than 25% land-cover' write (6,*) '# inconsistent points, total PFT pts, total LAI+SAI pts = ', & n, nc, sum(laimask(:,l)) - stop + call abort() end if end do diff --git a/tools/mksurfdata_map/src/mklanwatMod.F90 b/tools/mksurfdata_map/src/mklanwatMod.F90 index 49a1485fa7..4e1c590803 100644 --- a/tools/mksurfdata_map/src/mklanwatMod.F90 +++ b/tools/mksurfdata_map/src/mklanwatMod.F90 @@ -478,7 +478,7 @@ subroutine mklakparams(ldomain, mapfname, datfname, ndiag, & ! Check validity of output data if (min_bad(lakedepth_o, min_valid_lakedepth, 'lakedepth')) then - stop + call abort() end if call output_diagnostics_continuous(data_i, lakedepth_o, tgridmap, "Lake Depth", "m", ndiag, tdomain%mask, frac_dst) diff --git a/tools/mksurfdata_map/src/mkpeatMod.F90 b/tools/mksurfdata_map/src/mkpeatMod.F90 index 974566a056..8e47f5032d 100644 --- a/tools/mksurfdata_map/src/mkpeatMod.F90 +++ b/tools/mksurfdata_map/src/mkpeatMod.F90 @@ -123,7 +123,7 @@ subroutine mkpeat(ldomain, mapfname, datfname, ndiag, peat_o) ! Check validity of output data if (min_bad(peat_o, min_valid, 'peat') .or. & max_bad(peat_o, max_valid, 'peat')) then - stop + call abort() end if call output_diagnostics_area(data_i, peat_o, tgridmap, "Peat", percent=.false., ndiag=ndiag, mask_src=tdomain%mask, frac_dst=frac_dst) diff --git a/tools/mksurfdata_map/src/mkpftMod.F90 b/tools/mksurfdata_map/src/mkpftMod.F90 index 3a12c38cdf..2eae1ae381 100644 --- a/tools/mksurfdata_map/src/mkpftMod.F90 +++ b/tools/mksurfdata_map/src/mkpftMod.F90 @@ -644,7 +644,7 @@ subroutine mkpft(ldomain, mapfname, fpft, ndiag, & write (6,*) subname//'error: nat pft = ', & (pct_nat_pft_o(no,m), m = 0, num_natpft), & ' do not sum to 100. at no = ',no,' but to ', wst_sum - stop + call abort() end if ! Correct sum so that if it differs slightly from 100, it is corrected to equal @@ -661,7 +661,7 @@ subroutine mkpft(ldomain, mapfname, fpft, ndiag, & write (6,*) subname//'error: crop cft = ', & (pct_cft_o(no,m), m = 1, num_cft), & ' do not sum to 100. at no = ',no,' but to ', wst_sum - stop + call abort() end if ! Correct sum so that if it differs slightly from 100, it is corrected to equal diff --git a/tools/mksurfdata_map/src/mksoilMod.F90 b/tools/mksurfdata_map/src/mksoilMod.F90 index 959749ca1a..d7cad23e0d 100644 --- a/tools/mksurfdata_map/src/mksoilMod.F90 +++ b/tools/mksurfdata_map/src/mksoilMod.F90 @@ -319,7 +319,7 @@ subroutine mksoiltex(ldomain, mapfname, datfname, ndiag, sand_o, clay_o) write(6,*)'kmax is > kmap_max= ',kmax(no), 'kmap_max = ', & kmap_max,' for no = ',no write(6,*)'reset kmap_max in mksoilMod to a greater value' - stop + call abort() end if kmap(kmax(no),no) = k kwgt(kmax(no),no) = wt @@ -841,7 +841,7 @@ subroutine mkorganic(ldomain, mapfname, datfname, ndiag, organic_o) if (nlay /= nlevsoi) then write(6,*)'nlay, nlevsoi= ',nlay,nlevsoi,' do not match' - stop + call abort() end if call check_ret(nf_inq_varid (ncid, 'ORGANIC', varid), subname) @@ -873,7 +873,7 @@ subroutine mkorganic(ldomain, mapfname, datfname, ndiag, organic_o) write (6,*) 'MKORGANIC error: organic = ',organic_o(no,lev), & ' greater than 130.000001 for column, row = ',no call shr_sys_flush(6) - stop + call abort() end if enddo @@ -942,7 +942,7 @@ subroutine mksoilfmaxInit( ) if ( soil_fmax /= unset )then if ( soil_fmax < 0.0 .or. soil_fmax > 1.0 )then write(6,*)'soil_fmax is out of range = ', soil_fmax - stop + call abort() end if write(6,*) 'Replace soil fmax for all points with: ', soil_fmax end if @@ -1053,7 +1053,7 @@ subroutine mkfmax(ldomain, mapfname, datfname, ndiag, fmax_o) write (6,*) 'MKFMAX error: fmax = ',fmax_o(no), & ' greater than 1.000001 for column, row = ',no call shr_sys_flush(6) - stop + call abort() end if enddo @@ -1086,7 +1086,7 @@ subroutine mkfmax(ldomain, mapfname, datfname, ndiag, fmax_o) frac_dst(no)*re**2 if ((frac_dst(no) < 0.0) .or. (frac_dst(no) > 1.0001)) then write(6,*) "ERROR:: frac_dst out of range: ", frac_dst(no),no - stop + call abort() end if end do diff --git a/tools/mksurfdata_map/src/mksoildepthMod.F90 b/tools/mksurfdata_map/src/mksoildepthMod.F90 index 521ac2c6f0..c69cf375a4 100644 --- a/tools/mksurfdata_map/src/mksoildepthMod.F90 +++ b/tools/mksurfdata_map/src/mksoildepthMod.F90 @@ -146,7 +146,7 @@ subroutine mksoildepth(ldomain, mapfname, datfname, ndiag, soildepth_o) ! Check validity of output data if (min_bad(soildepth_o, min_valid, 'soildepth') .or. & max_bad(soildepth_o, max_valid, 'soildepth')) then - stop + call abort() end if call output_diagnostics_area(data_i, soildepth_o, tgridmap, "Soildepth", percent=.false., ndiag=ndiag, mask_src=tdomain%mask, frac_dst=frac_dst) diff --git a/tools/mksurfdata_map/src/mksurfdat.F90 b/tools/mksurfdata_map/src/mksurfdat.F90 index 9051c57707..aa965f097d 100644 --- a/tools/mksurfdata_map/src/mksurfdat.F90 +++ b/tools/mksurfdata_map/src/mksurfdat.F90 @@ -425,7 +425,7 @@ program mksurfdat ! Make sure ldomain is on a 0 to 360 grid as that's a requirement for CESM if ( .not. is_domain_0to360_longs( ldomain ) )then write(6,*)' Output domain must be on a 0 to 360 longitude grid rather than a -180 to 180 grid as it is required for CESM' - stop + call abort() end if ! ---------------------------------------------------------------------- ! Allocate and initialize dynamic memory @@ -493,7 +493,7 @@ program mksurfdat if (fsurlog == ' ') then write(6,*)' must specify fsurlog in namelist' - stop + call abort() else ndiag = getavu(); call opnfil (fsurlog, ndiag, 'f') end if @@ -1091,7 +1091,7 @@ program mksurfdat if (fdyndat == ' ') then write(6,*)' must specify fdyndat in namelist if mksrf_fdynuse is not blank' - stop + call abort() end if ! Define dimensions and global attributes diff --git a/tools/mksurfdata_map/src/mktopostatsMod.F90 b/tools/mksurfdata_map/src/mktopostatsMod.F90 index 2ecd705f4c..7e102d9bcf 100644 --- a/tools/mksurfdata_map/src/mktopostatsMod.F90 +++ b/tools/mksurfdata_map/src/mktopostatsMod.F90 @@ -131,7 +131,7 @@ subroutine mktopostats(ldomain, mapfname, datfname, ndiag, topo_stddev_o, slope_ ! Check validity of output data if (min_bad(topo_stddev_o, min_valid_topo_stddev, 'topo_stddev')) then - stop + call abort() end if @@ -158,7 +158,7 @@ subroutine mktopostats(ldomain, mapfname, datfname, ndiag, topo_stddev_o, slope_ ! Check validity of output data if (min_bad(slope_o, min_valid_slope, 'slope') .or. & max_bad(slope_o, max_valid_slope, 'slope')) then - stop + call abort() end if diff --git a/tools/mksurfdata_map/src/mkurbanparCommonMod.F90 b/tools/mksurfdata_map/src/mkurbanparCommonMod.F90 index 5db84e8351..ab738ea03c 100644 --- a/tools/mksurfdata_map/src/mkurbanparCommonMod.F90 +++ b/tools/mksurfdata_map/src/mkurbanparCommonMod.F90 @@ -93,13 +93,13 @@ subroutine mkurban_pct(ldomain, tdomain, tgridmap, urbn_i, urbn_o, frac_dst) write(6,*) 'tdomain%ns = ', tdomain%ns write(6,*) 'size(urbn_o) = ', size(urbn_o) write(6,*) 'ldomain%ns = ', ldomain%ns - stop + call abort() end if if (size(frac_dst) /= ldomain%ns) then write(6,*) subname//' ERROR: array size inconsistencies' write(6,*) 'size(frac_dst) = ', size(frac_dst) write(6,*) 'ldomain%ns = ', ldomain%ns - stop + call abort() end if ! Error checks for domain and map consistencies @@ -119,7 +119,7 @@ subroutine mkurban_pct(ldomain, tdomain, tgridmap, urbn_i, urbn_o, frac_dst) if ((urbn_o(no)) > 100.000001_r8) then write (6,*) 'MKURBAN error: urban = ',urbn_o(no), & ' greater than 100.000001 for column, row = ',no - stop + call abort() end if enddo @@ -191,7 +191,7 @@ subroutine mkurban_pct_diagnostics(ldomain, tdomain, tgridmap, urbn_i, urbn_o, n write(6,*) subname//' ERROR: array size inconsistencies' write(6,*) 'size(frac_dst) = ', size(frac_dst) write(6,*) 'ldomain%ns = ', ldomain%ns - stop + call abort() end if ! ----------------------------------------------------------------- diff --git a/tools/mksurfdata_map/src/mkurbanparMod.F90 b/tools/mksurfdata_map/src/mkurbanparMod.F90 index 07319b1f27..49ce95dd07 100644 --- a/tools/mksurfdata_map/src/mkurbanparMod.F90 +++ b/tools/mksurfdata_map/src/mkurbanparMod.F90 @@ -548,17 +548,17 @@ subroutine mkurbanpar(datfname, ncido, region_o, urbn_classes_gcell_o, urban_ski if (nlevurb_i /= nlevurb) then write(6,*)'MKURBANPAR: parameter nlevurb= ',nlevurb, & 'does not equal input dataset nlevurb= ',nlevurb_i - stop + call abort() endif if (numsolar_i /= numsolar) then write(6,*)'MKURBANPAR: parameter numsolar= ',numsolar, & 'does not equal input dataset numsolar= ',numsolar_i - stop + call abort() endif if (numrad_i /= numrad) then write(6,*)'MKURBANPAR: parameter numrad= ',numrad, & 'does not equal input dataset numrad= ',numrad_i - stop + call abort() endif ! Create an array that will hold the density indices From 6ef99990fc3a7d1362b535597d385b9ece37de8d Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Sun, 12 Dec 2021 13:48:32 -0700 Subject: [PATCH 53/55] Replace SCRIP grid file and mapping files for 1x1_brazil as they had negative longitude this fixes #1574 --- bld/namelist_files/namelist_defaults_ctsm.xml | 16 ++++++++-------- .../namelist_defaults_ctsm_tools.xml | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/bld/namelist_files/namelist_defaults_ctsm.xml b/bld/namelist_files/namelist_defaults_ctsm.xml index cd0da15c89..a61d66360b 100644 --- a/bld/namelist_files/namelist_defaults_ctsm.xml +++ b/bld/namelist_files/namelist_defaults_ctsm.xml @@ -1812,21 +1812,21 @@ lnd/clm2/surfdata_map/release-clm5.0.30/surfdata_ne0np4.CONUS.ne30x8_hist_78pfts lnd/clm2/mappingdata/maps/1x1_brazil/map_0.125x0.125_nomask_to_1x1_brazil_nomask_aave_da_c200206.nc +>lnd/clm2/mappingdata/maps/1x1_brazil/map_0.125x0.125_nomask_to_1x1_brazil_nomask_aave_da_c211212.nc lnd/clm2/mappingdata/maps/1x1_brazil/map_0.5x0.5_nomask_to_1x1_brazil_nomask_aave_da_c200206.nc +>lnd/clm2/mappingdata/maps/1x1_brazil/map_0.5x0.5_nomask_to_1x1_brazil_nomask_aave_da_c211212.nc lnd/clm2/mappingdata/maps/1x1_brazil/map_0.25x0.25_nomask_to_1x1_brazil_nomask_aave_da_c200309.nc +>lnd/clm2/mappingdata/maps/1x1_brazil/map_0.25x0.25_nomask_to_1x1_brazil_nomask_aave_da_c211212.nc lnd/clm2/mappingdata/maps/1x1_brazil/map_3x3min_nomask_to_1x1_brazil_nomask_aave_da_c200309.nc +>lnd/clm2/mappingdata/maps/1x1_brazil/map_3x3min_nomask_to_1x1_brazil_nomask_aave_da_c211212.nc lnd/clm2/mappingdata/maps/1x1_brazil/map_10x10min_nomask_to_1x1_brazil_nomask_aave_da_c200206.nc +>lnd/clm2/mappingdata/maps/1x1_brazil/map_10x10min_nomask_to_1x1_brazil_nomask_aave_da_c211212.nc lnd/clm2/mappingdata/maps/1x1_brazil/map_5x5min_nomask_to_1x1_brazil_nomask_aave_da_c200309.nc +>lnd/clm2/mappingdata/maps/1x1_brazil/map_5x5min_nomask_to_1x1_brazil_nomask_aave_da_c211212.nc lnd/clm2/mappingdata/maps/1x1_brazil/map_0.9x1.25_nomask_to_1x1_brazil_nomask_aave_da_c200206.nc +>lnd/clm2/mappingdata/maps/1x1_brazil/map_0.9x1.25_nomask_to_1x1_brazil_nomask_aave_da_c211212.nc lnd/clm2/mappingdata/maps/1x1_brazil/map_1km-merge-10min_HYDRO1K-merge-nomask_to_1x1_brazil_nomask_aave_da_c130403.nc +>lnd/clm2/mappingdata/maps/1x1_brazil/map_1km-merge-10min_HYDRO1K-merge-nomask_to_1x1_brazil_nomask_aave_da_c211212.nc diff --git a/bld/namelist_files/namelist_defaults_ctsm_tools.xml b/bld/namelist_files/namelist_defaults_ctsm_tools.xml index 1166e80c9c..ff309c6fc9 100644 --- a/bld/namelist_files/namelist_defaults_ctsm_tools.xml +++ b/bld/namelist_files/namelist_defaults_ctsm_tools.xml @@ -95,7 +95,7 @@ attributes from the config_cache.xml file (with keys converted to upper-case). lnd/clm2/mappingdata/grids/SCRIPgrid_0.125nldas2_nomask_c190328.nc -lnd/clm2/mappingdata/grids/SCRIPgrid_1x1pt_brazil_nomask_c110308.nc +lnd/clm2/mappingdata/grids/SCRIPgrid_1x1pt_brazil_nomask_c20211211.nc lnd/clm2/mappingdata/grids/SCRIPgrid_1x1pt_mexicocityMEX_nomask_c110308.nc lnd/clm2/mappingdata/grids/SCRIPgrid_1x1pt_numaIA_nomask_c110308.nc lnd/clm2/mappingdata/grids/SCRIPgrid_1x1pt_smallvilleIA_nomask_c110308.nc From a5b873a6ac38045e8e7f804a3f7f78478c1edc0b Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Sun, 12 Dec 2021 19:01:33 -0700 Subject: [PATCH 54/55] Start updating the Change files --- doc/ChangeLog | 120 ++++++++++++++++++++++++++++++++++++++++++++++++++ doc/ChangeSum | 1 + 2 files changed, 121 insertions(+) diff --git a/doc/ChangeLog b/doc/ChangeLog index 4a8050bfe1..7277103727 100644 --- a/doc/ChangeLog +++ b/doc/ChangeLog @@ -1,4 +1,124 @@ =============================================================== +Tag name: ctsm5.1.dev067 +Originator(s): jedwards4b/negin513/erik (Erik Kluzek,UCAR/TSS,303-497-1326) +Date: Sun Dec 12 17:44:44 MST 2021 +One-line Summary: NEON UI update, externals updates, small miscellanouse fixes + +Purpose and description of changes +---------------------------------- + +Redo options list to remove positional arguments that were difficult to input correctly. +Transient runs now use run_type startup and get finidat from s3 server unless --run-from-postad option is used (or finidat is not +available). Use mpi instead of mpi-serial, this mod was recommended for container use. Add a new script neon_finidat_upload which +allows authorized users to upload finidat files to the s3 server. + + +Significant changes to scientifically-supported configurations +-------------------------------------------------------------- + +Does this tag change answers significantly for any of the following physics configurations? +(Details of any changes will be given in the "Answer changes" section below.) + + [Put an [X] in the box for any configuration with significant answer changes.] + +[ ] clm5_1 + +[ ] clm5_0 + +[ ] ctsm5_0-nwp + +[ ] clm4_5 + + +Bugs fixed or introduced +------------------------ + +Issues fixed (include CTSM Issue #): + Fixes #1563 + Fixes #1550 + Fixes #1574 + +CIME Issues fixed (include issue #): + +Known bugs introduced in this tag (include issue #): + +Known bugs found since the previous tag (include issue #): + #1575 -- Build problem for mksurfdata tools testers + + +Notes of particular relevance for users +--------------------------------------- + +Caveats for users (e.g., need to interpolate initial conditions): + +Changes to CTSM's user interface (e.g., new/renamed XML or namelist variables): + +Changes made to namelist defaults (e.g., changed parameter values): + +Changes to the datasets (e.g., parameter, surface or initial files): + +Notes of particular relevance for developers: +--------------------------------------------- + +Caveats for developers (e.g., code that is duplicated that requires double maintenance): + +Changes to tests or testing: + + +Testing summary: regular tools +---------------- + [PASS means all tests PASS; OK means tests PASS other than expected fails.] + + build-namelist tests (if CLMBuildNamelist.pm has changed): + + cheyenne - PASS + + tools-tests (test/tools) (if tools have been changed): + + cheyenne - OK + + python testing (if python code has changed; see instructions in python/README.md; document testing done): + + cheyenne - OK (new black checks do NOT pass as expected) + + regular tests (aux_clm: https://github.com/ESCOMP/CTSM/wiki/System-Testing-Guide#pre-merge-system-testing): + + cheyenne ---- PASS + izumi ------- OK + + any other testing (give details below): + +If the tag used for baseline comparisons was NOT the previous tag, note that here: + + +Answer changes +-------------- + +Changes answers relative to baseline: No bit-for-bit (other than NEON tests because of updated namelists and surface dataset) + + Summarize any changes to answers, i.e., + - what code configurations: + - what platforms/compilers: + - nature of change (roundoff; larger than roundoff/same climate; new climate): + + If this tag changes climate describe the run(s) done to evaluate the new + climate (put details of the simulations in the experiment database) + - casename: + + URL for LMWG diagnostics output used to validate new climate: + + +Other details +------------- +[Remove any lines that don't apply. Remove entire section if nothing applies.] + +List any externals directories updated (cime, rtm, mosart, cism, fates, etc.): + +Pull Requests that document the changes (include PR ids): +(https://github.com/ESCOMP/ctsm/pull) + +=============================================================== +=============================================================== Tag name: ctsm5.1.dev066 Originator(s): rgknox (Ryan Knox,,,) Date: Sat Dec 4 01:58:42 MST 2021 diff --git a/doc/ChangeSum b/doc/ChangeSum index 73281e5a9c..36cc4e66cf 100644 --- a/doc/ChangeSum +++ b/doc/ChangeSum @@ -1,5 +1,6 @@ Tag Who Date Summary ============================================================================================================================ + ctsm5.1.dev067 jedwards 12/12/2021 NEON UI update, externals updates, small miscellanouse fixes ctsm5.1.dev066 rgknox 12/04/2021 API change with FATES to enable running means inside fates, includes passing in of model timestep ctsm5.1.dev065 glemieux 12/02/2021 Refactor static fire data input by moving variables into fire_base_type from cnveg_state_type ctsm5.1.dev064 afoster 11/29/2021 Updates to facilitate FATES history variable overhaul From ae1a023f921f961fbdf26ab0ea69541098aa94b2 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Mon, 13 Dec 2021 00:50:26 -0700 Subject: [PATCH 55/55] Update of Change files --- doc/ChangeLog | 95 ++++++++++++++++++++++++++++++++++----------------- doc/ChangeSum | 2 +- 2 files changed, 65 insertions(+), 32 deletions(-) diff --git a/doc/ChangeLog b/doc/ChangeLog index 7277103727..8c17383973 100644 --- a/doc/ChangeLog +++ b/doc/ChangeLog @@ -1,7 +1,7 @@ =============================================================== Tag name: ctsm5.1.dev067 Originator(s): jedwards4b/negin513/erik (Erik Kluzek,UCAR/TSS,303-497-1326) -Date: Sun Dec 12 17:44:44 MST 2021 +Date: Mon Dec 13 00:50:01 MST 2021 One-line Summary: NEON UI update, externals updates, small miscellanouse fixes Purpose and description of changes @@ -12,6 +12,44 @@ Transient runs now use run_type startup and get finidat from s3 server unless -- available). Use mpi instead of mpi-serial, this mod was recommended for container use. Add a new script neon_finidat_upload which allows authorized users to upload finidat files to the s3 server. +This includes the following changes to the script for updating the surface dataset at neon sites using available neon data (i.e. +modify_singlept_site_neon.py) to address ctsm/issues #1353, #1429, and neon/issue #44: + +Update Organic calculation to use the following equation based on discussions in +https://github.com/ESCOMP/CTSM/pull/1375/files#r669590971 : +ORGANIC = estimatedOC * bulkDensity / 0.58 + +Because estimatedOC is rounded to the nearest integer in neon data, it is sometimes bigger than carbonTot. Therefore, in cases where +estimatedOC > carbonTot, we use carbonTot instead of estimatedOC in the above equation. + +Previously, we had missing data on neon files for some neon soil layers (see Modified NEON surface datasets have errors #1429 +(comment)). Therefore, it caused some missing values in the updated dataset. Here, we interpolate to fill in the missing data using +different interpolation techniques. Finally, we chose to use linear interpolation to fill in missing data. + +This includes the scripts for modification of the surface dataset for neon sites to address #1429. +Specifically, the following has been addressed in this PR: + +Update the calculation of ORGANIC to use the new field (CaCO3) from NEON data. +For this calculation if CaCO3 data is available, we first calculate inorganic carbon by: +inorganic carbon = (caco3 /100.0869)*12.0107 +Next, we calculate organic carbon by subtracting inorganic carbon from the total carbon: +[organic carbon = carbon_tot - inorganic carbon] +If the CaCO3 is not available then the code uses carbonTot and estimatedOC by NEON. + +Discussed here (Modified NEON surface datasets have errors #1429 (comment)) +For the Ag sites (KONA and STER), it changes the PCT_NATVEG, PCT_CROP, and PCT_NAT_PFT to avoid the error that we previously had in +spin-up: surfrd_veg_all ERROR: sum of wt_nat_patch not 1.00000000000000 at nl= 1 sum is: 0.000000000000000E+000 + +Discussed here (Modified NEON surface datasets have errors #1429 (comment)) +There was a typo previously in the NEON data for ABBY sites caused by mix of sample measurements. Please note that this was updated +by hand once data was downloaded from NEON site. + +With recent versions of CIME, the LILAC build with a user-defined machine was broken for a couple of reasons. This fixes it. + +Fix mksurfdata_map for 1x1_brazil. Get tools testing working again. Increase skip_steps by 1, which is needed for a change in CAM +where balance checks need to occur after the radiation update now rather than before. glob changed for bsd_glob in perl MkDepends +for mksurfdata_map. + Significant changes to scientifically-supported configurations -------------------------------------------------------------- @@ -34,35 +72,32 @@ Bugs fixed or introduced ------------------------ Issues fixed (include CTSM Issue #): - Fixes #1563 - Fixes #1550 - Fixes #1574 - -CIME Issues fixed (include issue #): - -Known bugs introduced in this tag (include issue #): + Fixes #1563 increase skip_steps for balance checks by one to permit new CAM physics re-ordering + Fixes #1550 In perl code replace glob with bsd_glob + Fixes #1574 Trouble with 1x1_brazil for mksurfdata_map because of negative longitude in SCRIP grid file + Fixes #1429 Modified NEON surface datasets have errors + Fixes #1353 Modify NEON surface data + Fixes #1492 Need to update LILAC build process to use cmake macros instead of config_compilers.xml Known bugs found since the previous tag (include issue #): #1575 -- Build problem for mksurfdata tools testers - Notes of particular relevance for users --------------------------------------- -Caveats for users (e.g., need to interpolate initial conditions): - -Changes to CTSM's user interface (e.g., new/renamed XML or namelist variables): - -Changes made to namelist defaults (e.g., changed parameter values): +Changes made to namelist defaults (e.g., changed parameter values): New 1x1_brazil SCRIP grid file and maps + Some NEON namelist settings changed. Last year is now 2020 -Changes to the datasets (e.g., parameter, surface or initial files): +Changes to the datasets (e.g., parameter, surface or initial files): New NEON surface datasets Notes of particular relevance for developers: --------------------------------------------- Caveats for developers (e.g., code that is duplicated that requires double maintenance): + pyproject.toml file added to configure for black python formatter in python directory Changes to tests or testing: + Got tools testing working again. Testing summary: regular tools @@ -71,11 +106,11 @@ Testing summary: regular tools build-namelist tests (if CLMBuildNamelist.pm has changed): - cheyenne - PASS + cheyenne - PASS (47 compare tests fail because of changes to NEON sites) tools-tests (test/tools) (if tools have been changed): - cheyenne - OK + cheyenne - OK (1x1_brazil mksurfdata changes, run_neon and modify_subset fail as expected) python testing (if python code has changed; see instructions in python/README.md; document testing done): @@ -86,8 +121,6 @@ Testing summary: regular tools cheyenne ---- PASS izumi ------- OK - any other testing (give details below): - If the tag used for baseline comparisons was NOT the previous tag, note that here: @@ -97,26 +130,26 @@ Answer changes Changes answers relative to baseline: No bit-for-bit (other than NEON tests because of updated namelists and surface dataset) Summarize any changes to answers, i.e., - - what code configurations: - - what platforms/compilers: - - nature of change (roundoff; larger than roundoff/same climate; new climate): - - If this tag changes climate describe the run(s) done to evaluate the new - climate (put details of the simulations in the experiment database) - - casename: - - URL for LMWG diagnostics output used to validate new climate: - + - what code configurations: Only NEON sites + - what platforms/compilers: all + - nature of change: new surface datasets and settings Other details ------------- -[Remove any lines that don't apply. Remove entire section if nothing applies.] - List any externals directories updated (cime, rtm, mosart, cism, fates, etc.): + Update most externals to version in cesm2_3_alpha07c + cmeps to cmeps0.13.43 (version with channel depths) + cpl7 to cpl7.0.7 + share to share1.0.10 Pull Requests that document the changes (include PR ids): (https://github.com/ESCOMP/ctsm/pull) + PR #1467 -- Improve UI for NEON script + PR #1474 -- Script for modifying neon surface dataset -- updated (negin513) + PR #1539 -- Neon modify surfurface dataset (negin513) + PR #1571 -- Fix LILAC build with user-defined machine with latest CIME (billsacks) + =============================================================== =============================================================== Tag name: ctsm5.1.dev066 diff --git a/doc/ChangeSum b/doc/ChangeSum index 36cc4e66cf..7742b1f18c 100644 --- a/doc/ChangeSum +++ b/doc/ChangeSum @@ -1,6 +1,6 @@ Tag Who Date Summary ============================================================================================================================ - ctsm5.1.dev067 jedwards 12/12/2021 NEON UI update, externals updates, small miscellanouse fixes + ctsm5.1.dev067 jedwards 12/13/2021 NEON UI update, externals updates, small miscellanouse fixes ctsm5.1.dev066 rgknox 12/04/2021 API change with FATES to enable running means inside fates, includes passing in of model timestep ctsm5.1.dev065 glemieux 12/02/2021 Refactor static fire data input by moving variables into fire_base_type from cnveg_state_type ctsm5.1.dev064 afoster 11/29/2021 Updates to facilitate FATES history variable overhaul