improve ui for neon script

bld/CLMBuildNamelist.pm

@@ -4254,7 +4254,7 @@ sub check_input_files {
                 my $pathname = $nl->get_variable_value($group, $var);
                 # Need to strip the quotes
                 $pathname =~ s/['"]//g;
-
+                next if ($pathname =~ /UNSET$/);
                 if ($input_pathname_type eq 'abs') {
                     if ($inputdata_rootdir) {
                         #MV $pathname =~ s:$inputdata_rootdir::;

cime_config/usermods_dirs/NEON/defaults/user_nl_clm

@@ -19,7 +19,7 @@
 !----------------------------------------------------------------------------------
 
 flanduse_timeseries = ' '   ! This isn't needed for a non transient case, but will be once we start using transient compsets
-fsurdat = "$DIN_LOC_ROOT/lnd/clm2/surfdata_map/NEON/surfdata_hist_78pfts_CMIP6_simyr2000_${NEONSITE}_c210720.nc"
+fsurdat = "$DIN_LOC_ROOT/lnd/clm2/surfdata_map/NEON/surfdata_hist_78pfts_CMIP6_simyr2000_${NEONSITE}_c210805.nc"
 model_year_align_urbantv = 2018
 stream_year_first_urbantv = 2018
 stream_year_last_urbantv = 2019

tools/site_and_regional/neon_s3_upload

@@ -0,0 +1,173 @@
+#! /usr/bin/env python3
+"""
+Script to rename and upload NEON site finidat files for use in transient startup cases
+
+"""
+
+import os, sys
+# Get the ctsm util tools and then the cime tools.
+_CTSM_PYTHON = os.path.abspath(os.path.join(os.path.dirname(__file__), "..","..",'python'))
+sys.path.insert(1, _CTSM_PYTHON)
+
+import boto3
+from botocore.exceptions   import ClientError
+import glob
+import datetime
+from ctsm                  import add_cime_to_path
+from ctsm.path_utils       import path_to_ctsm_root
+from standard_script_setup import *
+from CIME.case             import Case
+from CIME.utils            import expect, safe_copy
+
+logger = logging.getLogger(__name__) 
+
+def get_parser(args, description, valid_neon_sites):
+    """
+    Get Parser object for this script
+    """
+    parser = argparse.ArgumentParser(description=description,
+                           formatter_class=argparse.RawDescriptionHelpFormatter)
+
+    CIME.utils.setup_standard_logging_options(parser)
+
+    parser.print_usage = parser.print_help
+
+    parser.add_argument('--neon-sites',
+                        help='4-letter neon site code.', 
+                        action="store",
+                        required=False,
+                        choices=valid_neon_sites + ['all'],
+                        dest="neon_sites",
+                        default=["OSBS"],
+                        nargs='+')
+
+    parser.add_argument('--output-root',
+                help='''
+                Root Directory of case results
+                [default: %(default)s] 
+                ''',
+                action="store", 
+                dest="output_root",
+                type =str,
+                required=False,
+                        default=os.getcwd())
+
+    parser.add_argument('--file-date',
+                        help='''
+                        Date of ctsm restart file(s) to upload
+                        ''',
+                        action="store", 
+                        dest="file_date",        
+                        required = False,
+                        type = datetime.date.fromisoformat,
+                        default = datetime.datetime.strptime("0268-01-01",'%Y-%m-%d'))
+
+
+    parser.add_argument('--upload-finidat',
+                        help='''
+                        Upload the final restart files from the end of the postad run for each site.
+                        ''',
+                        action="store_true",
+                        dest="upload_finidat",
+                        required = False,
+                        default = False)
+
+    parser.add_argument('--upload-history',
+                        help='''
+                        Upload the transient run h1 history files for each site.
+                        ''',
+                        action="store_true",
+                        dest="upload_history",
+                        required = False,
+                        default = False)
+
+
+
+    args = CIME.utils.parse_args_and_handle_standard_logging_options(args, parser)
+
+    if 'all' in args.neon_sites:
+        neon_sites = valid_neon_sites
+    else:
+        neon_sites = args.neon_sites
+        for site in neon_sites:
+            if site not in valid_neon_sites:
+                raise ValueError("Invalid site name {}".format(site))
+
+    expect(args.upload_finidat or args.upload_history,"Must specify at least one of --upload-finidat or --upload-history")
+
+    return neon_sites, args.output_root, args.file_date, args.upload_finidat, args.upload_history
+
+def upload_file(file_name, bucket, object_name=None):
+    """Upload a file to an S3 bucket
+
+    :param file_name: File to upload
+    :param bucket: Bucket to upload to
+    :param object_name: S3 object name. If not specified then file_name is used
+    :return: True if file was uploaded, else False
+    """
+
+    # If S3 object_name was not specified, use file_name
+    if object_name is None:
+        object_name = os.path.basename(file_name)
+
+    # Upload the file
+    s3_client = boto3.client('s3')
+    try:
+        logger.info("Uploading file {} to {}".format(file_name, object_name))
+        response = s3_client.upload_file(file_name, bucket, object_name)
+    except ClientError as e:
+        logger.error(e)
+        return False
+    return True
+
+def main(description):
+    """
+    For each site in the site_list find the site.postad run directory and grab the latest clm restart file
+    from there, 
+    """
+
+    if not os.path.isfile(os.path.join(os.getenv("HOME"),".aws","credentials")):
+        raise FileNotFoundError("User account must have valid aws credentials to run this script.")
+
+    cesmroot = path_to_ctsm_root()
+    # Get the list of supported neon sites from usermods
+    valid_neon_sites = glob.glob(os.path.join(cesmroot,"cime_config","usermods_dirs","NEON","[!d]*"))
+    valid_neon_sites = [v.split('/')[-1] for v in valid_neon_sites]
+    filedatestamp = datetime.datetime.now().date()
+    site_list, output_root, file_date, upload_finidat, upload_history = get_parser(sys.argv, description, valid_neon_sites)
+    for site in site_list:
+        rundir = None
+        if upload_finidat:
+            logger.info("Upload finidat for {}".format(site))
+            case_path = os.path.join(output_root, site+".postad")
+            if os.path.isdir(case_path):
+                with Case(case_path) as case:
+                    rundir = case.get_value("RUNDIR")
+                    basefile = site+".postad.clm2.r.{}-00000.nc".format(file_date.strftime("%4Y-%m-%d"))
+                    finidat_file = os.path.join(rundir,basefile)
+                    if not os.path.isfile(finidat_file):
+                        logger.warning("Could not find file {}".format(finidat_file))
+                        continue
+                    newfile = basefile.replace(".postad.",".{}.".format(filedatestamp))
+
+                    upload_file(finidat_file, 'neon-ncar-transfer', os.path.join("NEON","lnd","ctsm","initdata",newfile))
+        if upload_history:
+            logger.info("Upload history for {}".format(site))
+            case_path = os.path.join(output_root, site+".transient")
+            if not os.path.isdir(case_path):
+                logger.warning("No case found in {}".format(case_path))
+                continue
+            with Case(case_path) as case:
+                archive_dir = os.path.join(case.get_value("DOUT_S_ROOT"),"lnd","hist")
+                for histfile in glob.iglob(archive_dir + "/*.h1.*"):
+                    newfile = os.path.basename(histfile)
+                    upload_file(histfile, 'neon-ncar-transfer', os.path.join("NEON","archive",site,"lnd","hist",newfile))
+
+
+
+
+
+
+if __name__ == "__main__":
+    main(__doc__) 
+