From b0bfa55a76c1a595b176f7e1fee3d48845b9794a Mon Sep 17 00:00:00 2001 From: Teagan King Date: Thu, 1 Feb 2024 09:22:18 -0700 Subject: [PATCH 1/5] preliminary class refactoring of neon_site into base_site and tower_site --- .gitignore | 1 + cime_config/buildnml | 4 +- python/ctsm/site_and_regional/base_site.py | 76 +++++++ python/ctsm/site_and_regional/neon_site.py | 167 +--------------- python/ctsm/site_and_regional/tower_site.py | 207 ++++++++++++++++++++ 5 files changed, 292 insertions(+), 163 deletions(-) create mode 100644 python/ctsm/site_and_regional/base_site.py create mode 100644 python/ctsm/site_and_regional/tower_site.py diff --git a/.gitignore b/.gitignore index ca701132a7..ec2a3b17f6 100644 --- a/.gitignore +++ b/.gitignore @@ -111,6 +111,7 @@ unit_test_build /tools/site_and_regional/????.ad/ /tools/site_and_regional/????.postad/ /tools/site_and_regional/????.transient/ +/tools/site_and_regional/archive/ # build output *.o diff --git a/cime_config/buildnml b/cime_config/buildnml index 84e1581406..0521830616 100755 --- a/cime_config/buildnml +++ b/cime_config/buildnml @@ -144,8 +144,8 @@ def buildnml(case, caseroot, compname): or clm_usrdat_name is "NEON.PRISM" ): logger.warning( - "WARNING: Do you have approriprate initial conditions for this simulation?" - + " Check that the finidat file used in the lnd_in namelist is apprporiately spunup for your case" + "WARNING: Do you have appropriate initial conditions for this simulation?" + + " Check that the finidat file used in the lnd_in namelist is appropriately spunup for your case" ) if comp_atm != "datm": diff --git a/python/ctsm/site_and_regional/base_site.py b/python/ctsm/site_and_regional/base_site.py new file mode 100644 index 0000000000..27ea97c583 --- /dev/null +++ b/python/ctsm/site_and_regional/base_site.py @@ -0,0 +1,76 @@ +""" +This module includes the definition for a parent class for the TowerSite class, +which has NeonSite and Plumber2Site child classes. This class defines common structures +that are in both NeonSite and Plumber2Site classes. The TowerSite class will define common +functionalities that are in both NeonSite and Plumber2Site classes. +""" +# -- Import libraries + +# -- standard libraries +import os.path +import sys +import logging + +# Get the ctsm util tools and then the cime tools. +_CTSM_PYTHON = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "python")) +sys.path.insert(1, _CTSM_PYTHON) + +# pylint: disable=wrong-import-position, import-error, unused-import, wrong-import-order +from ctsm import add_cime_to_path +from ctsm.path_utils import path_to_ctsm_root + +logger = logging.getLogger(__name__) + + +class BaseSite: + """ + Parent class to TowerSite + ... + Attributes + ---------- + + """ + + def __init__(self, name, start_year, end_year, start_month, end_month, finidat): + """ + Initializes BaseSite with the given arguments. + + Parameters + ---------- + """ + self.name = name + self.start_year = int(start_year) + self.end_year = int(end_year) + self.start_month = int(start_month) + self.end_month = int(end_month) + self.cesmroot = path_to_ctsm_root() + self.finidat = finidat + + def __str__(self): + """ + Converts ingredients of the BaseSite to string for printing. + """ + return "{}\n{}".format( + str(self.__class__), + "\n".join( + ( + "{} = {}".format(str(key), str(self.__dict__[key])) + for key in sorted(self.__dict__) + ) + ), + ) + + # pylint: disable=no-self-use + def get_batch_query(self, case): + """ + Function for querying the batch queue query command for a case, depending on the + user's batch system. + + Args: + case: + case object + """ + + if case.get_value("BATCH_SYSTEM") == "none": + return "none" + return case.get_value("batch_query") diff --git a/python/ctsm/site_and_regional/neon_site.py b/python/ctsm/site_and_regional/neon_site.py index 31ae78f5ad..e87d58eb7c 100755 --- a/python/ctsm/site_and_regional/neon_site.py +++ b/python/ctsm/site_and_regional/neon_site.py @@ -15,6 +15,10 @@ _CTSM_PYTHON = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "python")) sys.path.insert(1, _CTSM_PYTHON) +# -- import local classes for this script +from ctsm.site_and_regional.base_site import BaseSite +from ctsm.site_and_regional.tower_site import TowerSite + # pylint: disable=wrong-import-position, import-error, unused-import, wrong-import-order from ctsm import add_cime_to_path from ctsm.path_utils import path_to_ctsm_root @@ -27,126 +31,13 @@ # pylint: disable=too-many-instance-attributes -class NeonSite: +class NeonSite(TowerSite): """ A class for encapsulating neon sites. """ def __init__(self, name, start_year, end_year, start_month, end_month, finidat): - self.name = name - self.start_year = int(start_year) - self.end_year = int(end_year) - self.start_month = int(start_month) - self.end_month = int(end_month) - self.cesmroot = path_to_ctsm_root() - self.finidat = finidat - - def build_base_case( - self, cesmroot, output_root, res, compset, overwrite=False, setup_only=False - ): - """ - Function for building a base_case to clone. - To spend less time on building ctsm for the neon cases, - all the other cases are cloned from this case - - Args: - self: - The NeonSite object - base_root (str): - root of the base_case CIME - res (str): - base_case resolution or gridname - compset (str): - base case compset - overwrite (bool) : - Flag to overwrite the case if exists - """ - print("---- building a base case -------") - # pylint: disable=attribute-defined-outside-init - self.base_case_root = output_root - # pylint: enable=attribute-defined-outside-init - user_mods_dirs = [os.path.join(cesmroot, "cime_config", "usermods_dirs", "NEON", self.name)] - if not output_root: - output_root = os.getcwd() - case_path = os.path.join(output_root, self.name) - - logger.info("base_case_name : %s", self.name) - logger.info("user_mods_dir : %s", user_mods_dirs[0]) - - if overwrite and os.path.isdir(case_path): - print("Removing the existing case at: {}".format(case_path)) - shutil.rmtree(case_path) - - with Case(case_path, read_only=False) as case: - if not os.path.isdir(case_path): - print("---- creating a base case -------") - - case.create( - case_path, - cesmroot, - compset, - res, - run_unsupported=True, - answer="r", - output_root=output_root, - user_mods_dirs=user_mods_dirs, - driver="nuopc", - ) - - print("---- base case created ------") - - # --change any config for base_case: - # case.set_value("RUN_TYPE","startup") - print("---- base case setup ------") - case.case_setup() - else: - # For existing case check that the compset name is correct - existingcompname = case.get_value("COMPSET") - match = re.search("^HIST", existingcompname, flags=re.IGNORECASE) - if re.search("^HIST", compset, flags=re.IGNORECASE) is None: - expect( - match is None, - """Existing base case is a historical type and should not be - --rerun with the --overwrite option""", - ) - else: - expect( - match is not None, - """Existing base case should be a historical type and is not - --rerun with the --overwrite option""", - ) - # reset the case - case.case_setup(reset=True) - case_path = case.get_value("CASEROOT") - - if setup_only: - return case_path - - print("---- base case build ------") - print("--- This may take a while and you may see WARNING messages ---") - # always walk through the build process to make sure it's up to date. - initial_time = time.time() - build.case_build(case_path, case=case) - end_time = time.time() - total = end_time - initial_time - print("Time required to building the base case: {} s.".format(total)) - # update case_path to be the full path to the base case - return case_path - - # pylint: disable=no-self-use - def get_batch_query(self, case): - """ - Function for querying the batch queue query command for a case, depending on the - user's batch system. - - Args: - case: - case object - """ - - if case.get_value("BATCH_SYSTEM") == "none": - return "none" - return case.get_value("batch_query") + super().__init__(name, start_year, end_year, start_month, end_month, finidat) # pylint: disable=too-many-statements def run_case( @@ -320,52 +211,6 @@ def run_case( if batch_query != "none": print(f"Use {batch_query} to check its run status") - def set_ref_case(self, case): - """ - Set an existing case as the reference case, eg for use with spinup. - """ - rundir = case.get_value("RUNDIR") - case_root = case.get_value("CASEROOT") - if case_root.endswith(".postad"): - ref_case_root = case_root.replace(".postad", ".ad") - root = ".ad" - else: - ref_case_root = case_root.replace(".transient", ".postad") - root = ".postad" - if not os.path.isdir(ref_case_root): - logger.warning( - "ERROR: spinup must be completed first, could not find directory %s", ref_case_root - ) - return False - - with Case(ref_case_root) as refcase: - refrundir = refcase.get_value("RUNDIR") - case.set_value("RUN_REFDIR", refrundir) - case.set_value("RUN_REFCASE", os.path.basename(ref_case_root)) - refdate = None - for reffile in glob.iglob(refrundir + "/{}{}.clm2.r.*.nc".format(self.name, root)): - m_searched = re.search(r"(\d\d\d\d-\d\d-\d\d)-\d\d\d\d\d.nc", reffile) - if m_searched: - refdate = m_searched.group(1) - symlink_force(reffile, os.path.join(rundir, os.path.basename(reffile))) - logger.info("Found refdate of %s", refdate) - if not refdate: - logger.warning("Could not find refcase for %s", case_root) - return False - - for rpfile in glob.iglob(refrundir + "/rpointer*"): - safe_copy(rpfile, rundir) - if not os.path.isdir(os.path.join(rundir, "inputdata")) and os.path.isdir( - os.path.join(refrundir, "inputdata") - ): - symlink_force(os.path.join(refrundir, "inputdata"), os.path.join(rundir, "inputdata")) - - case.set_value("RUN_REFDATE", refdate) - if case_root.endswith(".postad"): - case.set_value("RUN_STARTDATE", refdate) - # NOTE: if start options are set, RUN_STARTDATE should be modified here - return True - def modify_user_nl(self, case_root, run_type, rundir): """ Modify user namelist. If transient, include finidat in user_nl; diff --git a/python/ctsm/site_and_regional/tower_site.py b/python/ctsm/site_and_regional/tower_site.py new file mode 100644 index 0000000000..22d15eab86 --- /dev/null +++ b/python/ctsm/site_and_regional/tower_site.py @@ -0,0 +1,207 @@ +""" +This module includes the definition for the TowerSite class, +which has NeonSite and Plumber2Site child classes. This class defines common +functionalities that are in both NeonSite and Plumber2Site classes. +""" +# -- Import libraries + +# -- standard libraries +import os.path +import glob +import logging +import re +import shutil +import sys +import time + +# Get the ctsm util tools and then the cime tools. +_CTSM_PYTHON = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "python")) +sys.path.insert(1, _CTSM_PYTHON) + +# -- import local classes for this script +# pylint: disable=wrong-import-position +from ctsm.site_and_regional.base_site import BaseSite + +# pylint: disable=wrong-import-position, import-error, unused-import, wrong-import-order +from ctsm import add_cime_to_path +from ctsm.path_utils import path_to_ctsm_root + +from CIME import build +from CIME.case import Case +from CIME.utils import safe_copy, expect, symlink_force + +logger = logging.getLogger(__name__) + + +class TowerSite(BaseSite): + """ + Parent class to NeonSite and Plumber2Site classes. + ... + Attributes + ---------- + + Methods + ------- + + """ + + def __init__(self, name, start_year, end_year, start_month, end_month, finidat): + """ + Initializes TowerSite with the given arguments. + + Parameters + ---------- + """ + super().__init__(name, start_year, end_year, start_month, end_month, finidat) + + def __str__(self): + """ + Converts ingredients of the TowerSite to string for printing. + """ + return "{}\n{}".format( + str(self.__class__), + "\n".join( + ( + "{} = {}".format(str(key), str(self.__dict__[key])) + for key in sorted(self.__dict__) + ) + ), + ) + + def build_base_case( + self, cesmroot, output_root, res, compset, overwrite=False, setup_only=False + ): + """ + Function for building a base_case to clone. + To spend less time on building ctsm for the neon cases, + all the other cases are cloned from this case + + Args: + self: + The NeonSite object + base_root (str): + root of the base_case CIME + res (str): + base_case resolution or gridname + compset (str): + base case compset + overwrite (bool) : + Flag to overwrite the case if exists + """ + print("---- building a base case -------") + # pylint: disable=attribute-defined-outside-init + self.base_case_root = output_root + # pylint: enable=attribute-defined-outside-init + user_mods_dirs = [os.path.join(cesmroot, "cime_config", "usermods_dirs", "NEON", self.name)] + if not output_root: + output_root = os.getcwd() + case_path = os.path.join(output_root, self.name) + + logger.info("base_case_name : %s", self.name) + logger.info("user_mods_dir : %s", user_mods_dirs[0]) + + if overwrite and os.path.isdir(case_path): + print("Removing the existing case at: {}".format(case_path)) + shutil.rmtree(case_path) + + with Case(case_path, read_only=False) as case: + if not os.path.isdir(case_path): + print("---- creating a base case -------") + case.create( + case_path, + cesmroot, + compset, + res, + run_unsupported=True, + answer="r", + output_root=output_root, + user_mods_dirs=user_mods_dirs, + driver="nuopc", + ) + + print("---- base case created ------") + + # --change any config for base_case: + # case.set_value("RUN_TYPE","startup") + print("---- base case setup ------") + case.case_setup() + + else: + # For existing case check that the compset name is correct + existingcompname = case.get_value("COMPSET") + match = re.search("^HIST", existingcompname, flags=re.IGNORECASE) + if re.search("^HIST", compset, flags=re.IGNORECASE) is None: + expect( + match is None, + """Existing base case is a historical type and should not be + --rerun with the --overwrite option""", + ) + else: + expect( + match is not None, + """Existing base case should be a historical type and is not + --rerun with the --overwrite option""", + ) + # reset the case + case.case_setup(reset=True) + case_path = case.get_value("CASEROOT") + + if setup_only: + return case_path + + print("---- base case build ------") + print("--- This may take a while and you may see WARNING messages ---") + # always walk through the build process to make sure it's up to date. + initial_time = time.time() + build.case_build(case_path, case=case) + end_time = time.time() + total = end_time - initial_time + print("Time required to building the base case: {} s.".format(total)) + # update case_path to be the full path to the base case + return case_path + + def set_ref_case(self, case): + """ + Set an existing case as the reference case, eg for use with spinup. + """ + rundir = case.get_value("RUNDIR") + case_root = case.get_value("CASEROOT") + if case_root.endswith(".postad"): + ref_case_root = case_root.replace(".postad", ".ad") + root = ".ad" + else: + ref_case_root = case_root.replace(".transient", ".postad") + root = ".postad" + if not os.path.isdir(ref_case_root): + logger.warning( + "ERROR: spinup must be completed first, could not find directory %s", ref_case_root + ) + return False + + with Case(ref_case_root) as refcase: + refrundir = refcase.get_value("RUNDIR") + case.set_value("RUN_REFDIR", refrundir) + case.set_value("RUN_REFCASE", os.path.basename(ref_case_root)) + refdate = None + for reffile in glob.iglob(refrundir + "/{}{}.clm2.r.*.nc".format(self.name, root)): + m_searched = re.search(r"(\d\d\d\d-\d\d-\d\d)-\d\d\d\d\d.nc", reffile) + if m_searched: + refdate = m_searched.group(1) + symlink_force(reffile, os.path.join(rundir, os.path.basename(reffile))) + logger.info("Found refdate of %s", refdate) + if not refdate: + logger.warning("Could not find refcase for %s", case_root) + return False + + for rpfile in glob.iglob(refrundir + "/rpointer*"): + safe_copy(rpfile, rundir) + if not os.path.isdir(os.path.join(rundir, "inputdata")) and os.path.isdir( + os.path.join(refrundir, "inputdata") + ): + symlink_force(os.path.join(refrundir, "inputdata"), os.path.join(rundir, "inputdata")) + + case.set_value("RUN_REFDATE", refdate) + if case_root.endswith(".postad"): + case.set_value("RUN_STARTDATE", refdate) + # NOTE: if start options are set, RUN_STARTDATE should be modified here + return True From 013cbb085a6e4a2b60e4402e984e9fde3ed0c93e Mon Sep 17 00:00:00 2001 From: Teagan King Date: Mon, 5 Feb 2024 11:16:03 -0700 Subject: [PATCH 2/5] fix for build being called incorrectly --- python/ctsm/site_and_regional/neon_site.py | 140 ++++++++++++++++++++ python/ctsm/site_and_regional/tower_site.py | 138 ------------------- 2 files changed, 140 insertions(+), 138 deletions(-) diff --git a/python/ctsm/site_and_regional/neon_site.py b/python/ctsm/site_and_regional/neon_site.py index e87d58eb7c..122b72e146 100755 --- a/python/ctsm/site_and_regional/neon_site.py +++ b/python/ctsm/site_and_regional/neon_site.py @@ -39,6 +39,52 @@ class NeonSite(TowerSite): def __init__(self, name, start_year, end_year, start_month, end_month, finidat): super().__init__(name, start_year, end_year, start_month, end_month, finidat) + def set_ref_case(self, case): + """ + Set an existing case as the reference case, eg for use with spinup. + """ + rundir = case.get_value("RUNDIR") + case_root = case.get_value("CASEROOT") + if case_root.endswith(".postad"): + ref_case_root = case_root.replace(".postad", ".ad") + root = ".ad" + else: + ref_case_root = case_root.replace(".transient", ".postad") + root = ".postad" + if not os.path.isdir(ref_case_root): + logger.warning( + "ERROR: spinup must be completed first, could not find directory %s", ref_case_root + ) + return False + + with Case(ref_case_root) as refcase: + refrundir = refcase.get_value("RUNDIR") + case.set_value("RUN_REFDIR", refrundir) + case.set_value("RUN_REFCASE", os.path.basename(ref_case_root)) + refdate = None + for reffile in glob.iglob(refrundir + "/{}{}.clm2.r.*.nc".format(self.name, root)): + m_searched = re.search(r"(\d\d\d\d-\d\d-\d\d)-\d\d\d\d\d.nc", reffile) + if m_searched: + refdate = m_searched.group(1) + symlink_force(reffile, os.path.join(rundir, os.path.basename(reffile))) + logger.info("Found refdate of %s", refdate) + if not refdate: + logger.warning("Could not find refcase for %s", case_root) + return False + + for rpfile in glob.iglob(refrundir + "/rpointer*"): + safe_copy(rpfile, rundir) + if not os.path.isdir(os.path.join(rundir, "inputdata")) and os.path.isdir( + os.path.join(refrundir, "inputdata") + ): + symlink_force(os.path.join(refrundir, "inputdata"), os.path.join(rundir, "inputdata")) + + case.set_value("RUN_REFDATE", refdate) + if case_root.endswith(".postad"): + case.set_value("RUN_STARTDATE", refdate) + # NOTE: if start options are set, RUN_STARTDATE should be modified here + return True + # pylint: disable=too-many-statements def run_case( self, @@ -211,6 +257,100 @@ def run_case( if batch_query != "none": print(f"Use {batch_query} to check its run status") + def build_base_case( + self, cesmroot, output_root, res, compset, overwrite=False, setup_only=False + ): + """ + Function for building a base_case to clone. + To spend less time on building ctsm for the neon cases, + all the other cases are cloned from this case + + Args: + self: + The NeonSite object + base_root (str): + root of the base_case CIME + res (str): + base_case resolution or gridname + compset (str): + base case compset + overwrite (bool) : + Flag to overwrite the case if exists + """ + print("---- building a base case -------") + # pylint: disable=attribute-defined-outside-init + self.base_case_root = output_root + # pylint: enable=attribute-defined-outside-init + user_mods_dirs = [os.path.join(cesmroot, "cime_config", "usermods_dirs", "NEON", self.name)] + if not output_root: + output_root = os.getcwd() + case_path = os.path.join(output_root, self.name) + + logger.info("base_case_name : %s", self.name) + logger.info("user_mods_dir : %s", user_mods_dirs[0]) + + if overwrite and os.path.isdir(case_path): + print("Removing the existing case at: {}".format(case_path)) + shutil.rmtree(case_path) + + with Case(case_path, read_only=False) as case: + if not os.path.isdir(case_path): + print("---- creating a base case -------") + case.create( + case_path, + cesmroot, + compset, + res, + run_unsupported=True, + answer="r", + output_root=output_root, + user_mods_dirs=user_mods_dirs, + driver="nuopc", + ) + + print("---- base case created ------") + + # --change any config for base_case: + # case.set_value("RUN_TYPE","startup") + print("---- base case setup ------") + case.case_setup() + + else: + # For existing case check that the compset name is correct + existingcompname = case.get_value("COMPSET") + match = re.search("^HIST", existingcompname, flags=re.IGNORECASE) + if re.search("^HIST", compset, flags=re.IGNORECASE) is None: + expect( + match is None, + """Existing base case is a historical type and should not be + --rerun with the --overwrite option""", + ) + else: + expect( + match is not None, + """Existing base case should be a historical type and is not + --rerun with the --overwrite option""", + ) + # reset the case + case.case_setup(reset=True) + case_path = case.get_value("CASEROOT") + + if setup_only: + return case_path + + print("---- base case build ------") + print("--- This may take a while and you may see WARNING messages ---") + # always walk through the build process to make sure it's up to date. + initial_time = time.time() + build.case_build( + case_path, case=case + ) # TODO: this causes issues if run from tower_site.py + end_time = time.time() + total = end_time - initial_time + print("Time required to building the base case: {} s.".format(total)) + # update case_path to be the full path to the base case + return case_path + def modify_user_nl(self, case_root, run_type, rundir): """ Modify user namelist. If transient, include finidat in user_nl; diff --git a/python/ctsm/site_and_regional/tower_site.py b/python/ctsm/site_and_regional/tower_site.py index 22d15eab86..29488f6486 100644 --- a/python/ctsm/site_and_regional/tower_site.py +++ b/python/ctsm/site_and_regional/tower_site.py @@ -67,141 +67,3 @@ def __str__(self): ) ), ) - - def build_base_case( - self, cesmroot, output_root, res, compset, overwrite=False, setup_only=False - ): - """ - Function for building a base_case to clone. - To spend less time on building ctsm for the neon cases, - all the other cases are cloned from this case - - Args: - self: - The NeonSite object - base_root (str): - root of the base_case CIME - res (str): - base_case resolution or gridname - compset (str): - base case compset - overwrite (bool) : - Flag to overwrite the case if exists - """ - print("---- building a base case -------") - # pylint: disable=attribute-defined-outside-init - self.base_case_root = output_root - # pylint: enable=attribute-defined-outside-init - user_mods_dirs = [os.path.join(cesmroot, "cime_config", "usermods_dirs", "NEON", self.name)] - if not output_root: - output_root = os.getcwd() - case_path = os.path.join(output_root, self.name) - - logger.info("base_case_name : %s", self.name) - logger.info("user_mods_dir : %s", user_mods_dirs[0]) - - if overwrite and os.path.isdir(case_path): - print("Removing the existing case at: {}".format(case_path)) - shutil.rmtree(case_path) - - with Case(case_path, read_only=False) as case: - if not os.path.isdir(case_path): - print("---- creating a base case -------") - case.create( - case_path, - cesmroot, - compset, - res, - run_unsupported=True, - answer="r", - output_root=output_root, - user_mods_dirs=user_mods_dirs, - driver="nuopc", - ) - - print("---- base case created ------") - - # --change any config for base_case: - # case.set_value("RUN_TYPE","startup") - print("---- base case setup ------") - case.case_setup() - - else: - # For existing case check that the compset name is correct - existingcompname = case.get_value("COMPSET") - match = re.search("^HIST", existingcompname, flags=re.IGNORECASE) - if re.search("^HIST", compset, flags=re.IGNORECASE) is None: - expect( - match is None, - """Existing base case is a historical type and should not be - --rerun with the --overwrite option""", - ) - else: - expect( - match is not None, - """Existing base case should be a historical type and is not - --rerun with the --overwrite option""", - ) - # reset the case - case.case_setup(reset=True) - case_path = case.get_value("CASEROOT") - - if setup_only: - return case_path - - print("---- base case build ------") - print("--- This may take a while and you may see WARNING messages ---") - # always walk through the build process to make sure it's up to date. - initial_time = time.time() - build.case_build(case_path, case=case) - end_time = time.time() - total = end_time - initial_time - print("Time required to building the base case: {} s.".format(total)) - # update case_path to be the full path to the base case - return case_path - - def set_ref_case(self, case): - """ - Set an existing case as the reference case, eg for use with spinup. - """ - rundir = case.get_value("RUNDIR") - case_root = case.get_value("CASEROOT") - if case_root.endswith(".postad"): - ref_case_root = case_root.replace(".postad", ".ad") - root = ".ad" - else: - ref_case_root = case_root.replace(".transient", ".postad") - root = ".postad" - if not os.path.isdir(ref_case_root): - logger.warning( - "ERROR: spinup must be completed first, could not find directory %s", ref_case_root - ) - return False - - with Case(ref_case_root) as refcase: - refrundir = refcase.get_value("RUNDIR") - case.set_value("RUN_REFDIR", refrundir) - case.set_value("RUN_REFCASE", os.path.basename(ref_case_root)) - refdate = None - for reffile in glob.iglob(refrundir + "/{}{}.clm2.r.*.nc".format(self.name, root)): - m_searched = re.search(r"(\d\d\d\d-\d\d-\d\d)-\d\d\d\d\d.nc", reffile) - if m_searched: - refdate = m_searched.group(1) - symlink_force(reffile, os.path.join(rundir, os.path.basename(reffile))) - logger.info("Found refdate of %s", refdate) - if not refdate: - logger.warning("Could not find refcase for %s", case_root) - return False - - for rpfile in glob.iglob(refrundir + "/rpointer*"): - safe_copy(rpfile, rundir) - if not os.path.isdir(os.path.join(rundir, "inputdata")) and os.path.isdir( - os.path.join(refrundir, "inputdata") - ): - symlink_force(os.path.join(refrundir, "inputdata"), os.path.join(rundir, "inputdata")) - - case.set_value("RUN_REFDATE", refdate) - if case_root.endswith(".postad"): - case.set_value("RUN_STARTDATE", refdate) - # NOTE: if start options are set, RUN_STARTDATE should be modified here - return True From 2de84b5390de60f9aa18ea62e452342fc5091b01 Mon Sep 17 00:00:00 2001 From: Teagan King Date: Tue, 6 Feb 2024 10:59:24 -0700 Subject: [PATCH 3/5] general class setup before breaking down modify_user_nl and run_case --- .github/workflows/assign-to-project.yml | 2 +- python/ctsm/site_and_regional/base_site.py | 76 --------- python/ctsm/site_and_regional/neon_site.py | 144 +---------------- python/ctsm/site_and_regional/tower_site.py | 168 +++++++++++++++++++- 4 files changed, 164 insertions(+), 226 deletions(-) delete mode 100644 python/ctsm/site_and_regional/base_site.py diff --git a/.github/workflows/assign-to-project.yml b/.github/workflows/assign-to-project.yml index 8c6c259c33..74407c1bf2 100644 --- a/.github/workflows/assign-to-project.yml +++ b/.github/workflows/assign-to-project.yml @@ -13,7 +13,7 @@ jobs: runs-on: ubuntu-latest name: Assign to High Priority project steps: - - name: Assign issues and pull requests with `priority: high` label to project 25 + name: Assign issues and pull requests with `priority: high` label to project 25 uses: srggrs/assign-one-project-github-action@1.3.1 if: | contains(github.event.issue.labels.*.name, 'priority: high') || diff --git a/python/ctsm/site_and_regional/base_site.py b/python/ctsm/site_and_regional/base_site.py deleted file mode 100644 index 27ea97c583..0000000000 --- a/python/ctsm/site_and_regional/base_site.py +++ /dev/null @@ -1,76 +0,0 @@ -""" -This module includes the definition for a parent class for the TowerSite class, -which has NeonSite and Plumber2Site child classes. This class defines common structures -that are in both NeonSite and Plumber2Site classes. The TowerSite class will define common -functionalities that are in both NeonSite and Plumber2Site classes. -""" -# -- Import libraries - -# -- standard libraries -import os.path -import sys -import logging - -# Get the ctsm util tools and then the cime tools. -_CTSM_PYTHON = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "python")) -sys.path.insert(1, _CTSM_PYTHON) - -# pylint: disable=wrong-import-position, import-error, unused-import, wrong-import-order -from ctsm import add_cime_to_path -from ctsm.path_utils import path_to_ctsm_root - -logger = logging.getLogger(__name__) - - -class BaseSite: - """ - Parent class to TowerSite - ... - Attributes - ---------- - - """ - - def __init__(self, name, start_year, end_year, start_month, end_month, finidat): - """ - Initializes BaseSite with the given arguments. - - Parameters - ---------- - """ - self.name = name - self.start_year = int(start_year) - self.end_year = int(end_year) - self.start_month = int(start_month) - self.end_month = int(end_month) - self.cesmroot = path_to_ctsm_root() - self.finidat = finidat - - def __str__(self): - """ - Converts ingredients of the BaseSite to string for printing. - """ - return "{}\n{}".format( - str(self.__class__), - "\n".join( - ( - "{} = {}".format(str(key), str(self.__dict__[key])) - for key in sorted(self.__dict__) - ) - ), - ) - - # pylint: disable=no-self-use - def get_batch_query(self, case): - """ - Function for querying the batch queue query command for a case, depending on the - user's batch system. - - Args: - case: - case object - """ - - if case.get_value("BATCH_SYSTEM") == "none": - return "none" - return case.get_value("batch_query") diff --git a/python/ctsm/site_and_regional/neon_site.py b/python/ctsm/site_and_regional/neon_site.py index 122b72e146..44249f47cd 100755 --- a/python/ctsm/site_and_regional/neon_site.py +++ b/python/ctsm/site_and_regional/neon_site.py @@ -3,20 +3,18 @@ """ # Import libraries -import glob import logging import os import re import shutil import sys -import time # Get the ctsm util tools and then the cime tools. _CTSM_PYTHON = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "python")) sys.path.insert(1, _CTSM_PYTHON) # -- import local classes for this script -from ctsm.site_and_regional.base_site import BaseSite +# pylint: disable=wrong-import-position from ctsm.site_and_regional.tower_site import TowerSite # pylint: disable=wrong-import-position, import-error, unused-import, wrong-import-order @@ -39,52 +37,6 @@ class NeonSite(TowerSite): def __init__(self, name, start_year, end_year, start_month, end_month, finidat): super().__init__(name, start_year, end_year, start_month, end_month, finidat) - def set_ref_case(self, case): - """ - Set an existing case as the reference case, eg for use with spinup. - """ - rundir = case.get_value("RUNDIR") - case_root = case.get_value("CASEROOT") - if case_root.endswith(".postad"): - ref_case_root = case_root.replace(".postad", ".ad") - root = ".ad" - else: - ref_case_root = case_root.replace(".transient", ".postad") - root = ".postad" - if not os.path.isdir(ref_case_root): - logger.warning( - "ERROR: spinup must be completed first, could not find directory %s", ref_case_root - ) - return False - - with Case(ref_case_root) as refcase: - refrundir = refcase.get_value("RUNDIR") - case.set_value("RUN_REFDIR", refrundir) - case.set_value("RUN_REFCASE", os.path.basename(ref_case_root)) - refdate = None - for reffile in glob.iglob(refrundir + "/{}{}.clm2.r.*.nc".format(self.name, root)): - m_searched = re.search(r"(\d\d\d\d-\d\d-\d\d)-\d\d\d\d\d.nc", reffile) - if m_searched: - refdate = m_searched.group(1) - symlink_force(reffile, os.path.join(rundir, os.path.basename(reffile))) - logger.info("Found refdate of %s", refdate) - if not refdate: - logger.warning("Could not find refcase for %s", case_root) - return False - - for rpfile in glob.iglob(refrundir + "/rpointer*"): - safe_copy(rpfile, rundir) - if not os.path.isdir(os.path.join(rundir, "inputdata")) and os.path.isdir( - os.path.join(refrundir, "inputdata") - ): - symlink_force(os.path.join(refrundir, "inputdata"), os.path.join(rundir, "inputdata")) - - case.set_value("RUN_REFDATE", refdate) - if case_root.endswith(".postad"): - case.set_value("RUN_STARTDATE", refdate) - # NOTE: if start options are set, RUN_STARTDATE should be modified here - return True - # pylint: disable=too-many-statements def run_case( self, @@ -257,100 +209,6 @@ def run_case( if batch_query != "none": print(f"Use {batch_query} to check its run status") - def build_base_case( - self, cesmroot, output_root, res, compset, overwrite=False, setup_only=False - ): - """ - Function for building a base_case to clone. - To spend less time on building ctsm for the neon cases, - all the other cases are cloned from this case - - Args: - self: - The NeonSite object - base_root (str): - root of the base_case CIME - res (str): - base_case resolution or gridname - compset (str): - base case compset - overwrite (bool) : - Flag to overwrite the case if exists - """ - print("---- building a base case -------") - # pylint: disable=attribute-defined-outside-init - self.base_case_root = output_root - # pylint: enable=attribute-defined-outside-init - user_mods_dirs = [os.path.join(cesmroot, "cime_config", "usermods_dirs", "NEON", self.name)] - if not output_root: - output_root = os.getcwd() - case_path = os.path.join(output_root, self.name) - - logger.info("base_case_name : %s", self.name) - logger.info("user_mods_dir : %s", user_mods_dirs[0]) - - if overwrite and os.path.isdir(case_path): - print("Removing the existing case at: {}".format(case_path)) - shutil.rmtree(case_path) - - with Case(case_path, read_only=False) as case: - if not os.path.isdir(case_path): - print("---- creating a base case -------") - case.create( - case_path, - cesmroot, - compset, - res, - run_unsupported=True, - answer="r", - output_root=output_root, - user_mods_dirs=user_mods_dirs, - driver="nuopc", - ) - - print("---- base case created ------") - - # --change any config for base_case: - # case.set_value("RUN_TYPE","startup") - print("---- base case setup ------") - case.case_setup() - - else: - # For existing case check that the compset name is correct - existingcompname = case.get_value("COMPSET") - match = re.search("^HIST", existingcompname, flags=re.IGNORECASE) - if re.search("^HIST", compset, flags=re.IGNORECASE) is None: - expect( - match is None, - """Existing base case is a historical type and should not be - --rerun with the --overwrite option""", - ) - else: - expect( - match is not None, - """Existing base case should be a historical type and is not - --rerun with the --overwrite option""", - ) - # reset the case - case.case_setup(reset=True) - case_path = case.get_value("CASEROOT") - - if setup_only: - return case_path - - print("---- base case build ------") - print("--- This may take a while and you may see WARNING messages ---") - # always walk through the build process to make sure it's up to date. - initial_time = time.time() - build.case_build( - case_path, case=case - ) # TODO: this causes issues if run from tower_site.py - end_time = time.time() - total = end_time - initial_time - print("Time required to building the base case: {} s.".format(total)) - # update case_path to be the full path to the base case - return case_path - def modify_user_nl(self, case_root, run_type, rundir): """ Modify user namelist. If transient, include finidat in user_nl; diff --git a/python/ctsm/site_and_regional/tower_site.py b/python/ctsm/site_and_regional/tower_site.py index 29488f6486..5e0d9a434b 100644 --- a/python/ctsm/site_and_regional/tower_site.py +++ b/python/ctsm/site_and_regional/tower_site.py @@ -18,10 +18,6 @@ _CTSM_PYTHON = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "python")) sys.path.insert(1, _CTSM_PYTHON) -# -- import local classes for this script -# pylint: disable=wrong-import-position -from ctsm.site_and_regional.base_site import BaseSite - # pylint: disable=wrong-import-position, import-error, unused-import, wrong-import-order from ctsm import add_cime_to_path from ctsm.path_utils import path_to_ctsm_root @@ -33,7 +29,7 @@ logger = logging.getLogger(__name__) -class TowerSite(BaseSite): +class TowerSite: """ Parent class to NeonSite and Plumber2Site classes. ... @@ -52,7 +48,13 @@ def __init__(self, name, start_year, end_year, start_month, end_month, finidat): Parameters ---------- """ - super().__init__(name, start_year, end_year, start_month, end_month, finidat) + self.name = name + self.start_year = int(start_year) + self.end_year = int(end_year) + self.start_month = int(start_month) + self.end_month = int(end_month) + self.cesmroot = path_to_ctsm_root() + self.finidat = finidat def __str__(self): """ @@ -67,3 +69,157 @@ def __str__(self): ) ), ) + + def build_base_case( + self, cesmroot, output_root, res, compset, overwrite=False, setup_only=False + ): + """ + Function for building a base_case to clone. + To spend less time on building ctsm for the neon cases, + all the other cases are cloned from this case + + Args: + self: + The NeonSite object + base_root (str): + root of the base_case CIME + res (str): + base_case resolution or gridname + compset (str): + base case compset + overwrite (bool) : + Flag to overwrite the case if exists + """ + print("---- building a base case -------") + # pylint: disable=attribute-defined-outside-init + self.base_case_root = output_root + # pylint: enable=attribute-defined-outside-init + user_mods_dirs = [os.path.join(cesmroot, "cime_config", "usermods_dirs", "NEON", self.name)] + if not output_root: + output_root = os.getcwd() + case_path = os.path.join(output_root, self.name) + + logger.info("base_case_name : %s", self.name) + logger.info("user_mods_dir : %s", user_mods_dirs[0]) + + if overwrite and os.path.isdir(case_path): + print("Removing the existing case at: {}".format(case_path)) + shutil.rmtree(case_path) + + with Case(case_path, read_only=False) as case: + if not os.path.isdir(case_path): + print("---- creating a base case -------") + case.create( + case_path, + cesmroot, + compset, + res, + run_unsupported=True, + answer="r", + output_root=output_root, + user_mods_dirs=user_mods_dirs, + driver="nuopc", + ) + + print("---- base case created ------") + + # --change any config for base_case: + # case.set_value("RUN_TYPE","startup") + print("---- base case setup ------") + case.case_setup() + else: + # For existing case check that the compset name is correct + existingcompname = case.get_value("COMPSET") + match = re.search("^HIST", existingcompname, flags=re.IGNORECASE) + if re.search("^HIST", compset, flags=re.IGNORECASE) is None: + expect( + match is None, + """Existing base case is a historical type and should not be + --rerun with the --overwrite option""", + ) + else: + expect( + match is not None, + """Existing base case should be a historical type and is not + --rerun with the --overwrite option""", + ) + # reset the case + case.case_setup(reset=True) + case_path = case.get_value("CASEROOT") + + if setup_only: + return case_path + + print("---- base case build ------") + print("--- This may take a while and you may see WARNING messages ---") + # always walk through the build process to make sure it's up to date. + initial_time = time.time() + build.case_build( + case_path, case=case + ) # TODO: this causes issues if run from tower_site.py + end_time = time.time() + total = end_time - initial_time + print("Time required to building the base case: {} s.".format(total)) + # update case_path to be the full path to the base case + return case_path + + # pylint: disable=no-self-use + def get_batch_query(self, case): + """ + Function for querying the batch queue query command for a case, depending on the + user's batch system. + + Args: + case: + case object + """ + + if case.get_value("BATCH_SYSTEM") == "none": + return "none" + return case.get_value("batch_query") + + def set_ref_case(self, case): + """ + Set an existing case as the reference case, eg for use with spinup. + """ + rundir = case.get_value("RUNDIR") + case_root = case.get_value("CASEROOT") + if case_root.endswith(".postad"): + ref_case_root = case_root.replace(".postad", ".ad") + root = ".ad" + else: + ref_case_root = case_root.replace(".transient", ".postad") + root = ".postad" + if not os.path.isdir(ref_case_root): + logger.warning( + "ERROR: spinup must be completed first, could not find directory %s", ref_case_root + ) + return False + + with Case(ref_case_root) as refcase: + refrundir = refcase.get_value("RUNDIR") + case.set_value("RUN_REFDIR", refrundir) + case.set_value("RUN_REFCASE", os.path.basename(ref_case_root)) + refdate = None + for reffile in glob.iglob(refrundir + "/{}{}.clm2.r.*.nc".format(self.name, root)): + m_searched = re.search(r"(\d\d\d\d-\d\d-\d\d)-\d\d\d\d\d.nc", reffile) + if m_searched: + refdate = m_searched.group(1) + symlink_force(reffile, os.path.join(rundir, os.path.basename(reffile))) + logger.info("Found refdate of %s", refdate) + if not refdate: + logger.warning("Could not find refcase for %s", case_root) + return False + + for rpfile in glob.iglob(refrundir + "/rpointer*"): + safe_copy(rpfile, rundir) + if not os.path.isdir(os.path.join(rundir, "inputdata")) and os.path.isdir( + os.path.join(refrundir, "inputdata") + ): + symlink_force(os.path.join(refrundir, "inputdata"), os.path.join(rundir, "inputdata")) + + case.set_value("RUN_REFDATE", refdate) + if case_root.endswith(".postad"): + case.set_value("RUN_STARTDATE", refdate) + # NOTE: if start options are set, RUN_STARTDATE should be modified here + return True From 0899b77d5bbe0e8c15d84a55ff49e8244d0aba6d Mon Sep 17 00:00:00 2001 From: Teagan King Date: Tue, 6 Feb 2024 11:01:08 -0700 Subject: [PATCH 4/5] undo workflow update... --- .github/workflows/assign-to-project.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/assign-to-project.yml b/.github/workflows/assign-to-project.yml index 74407c1bf2..8c6c259c33 100644 --- a/.github/workflows/assign-to-project.yml +++ b/.github/workflows/assign-to-project.yml @@ -13,7 +13,7 @@ jobs: runs-on: ubuntu-latest name: Assign to High Priority project steps: - name: Assign issues and pull requests with `priority: high` label to project 25 + - name: Assign issues and pull requests with `priority: high` label to project 25 uses: srggrs/assign-one-project-github-action@1.3.1 if: | contains(github.event.issue.labels.*.name, 'priority: high') || From 45e39d2e35e7596c387edb53a66b212f25a14b7e Mon Sep 17 00:00:00 2001 From: Teagan King Date: Fri, 9 Feb 2024 13:24:25 -0700 Subject: [PATCH 5/5] fix fxns w/ super --- python/ctsm/site_and_regional/neon_site.py | 23 +++++++++++++++++++++ python/ctsm/site_and_regional/tower_site.py | 6 +----- 2 files changed, 24 insertions(+), 5 deletions(-) diff --git a/python/ctsm/site_and_regional/neon_site.py b/python/ctsm/site_and_regional/neon_site.py index 44249f47cd..ba6416d531 100755 --- a/python/ctsm/site_and_regional/neon_site.py +++ b/python/ctsm/site_and_regional/neon_site.py @@ -3,11 +3,13 @@ """ # Import libraries +import glob import logging import os import re import shutil import sys +import time # Get the ctsm util tools and then the cime tools. _CTSM_PYTHON = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "python")) @@ -36,6 +38,23 @@ class NeonSite(TowerSite): def __init__(self, name, start_year, end_year, start_month, end_month, finidat): super().__init__(name, start_year, end_year, start_month, end_month, finidat) + # self.name = name + # self.start_year = int(start_year) + # self.end_year = int(end_year) + # self.start_month = int(start_month) + # self.end_month = int(end_month) + # self.cesmroot = path_to_ctsm_root() + # self.finidat = finidat + + def build_base_case( + self, cesmroot, output_root, res, compset, overwrite=False, setup_only=False + ): + case_path = super().build_base_case(cesmroot, output_root, res, compset) + + return case_path + + def get_batch_query(self, case): + return super().get_batch_query(case) # pylint: disable=too-many-statements def run_case( @@ -209,6 +228,10 @@ def run_case( if batch_query != "none": print(f"Use {batch_query} to check its run status") + def set_ref_case(self, case): + super().set_ref_case(case) + return True ### Check if super returns false, if this will still return True? + def modify_user_nl(self, case_root, run_type, rundir): """ Modify user namelist. If transient, include finidat in user_nl; diff --git a/python/ctsm/site_and_regional/tower_site.py b/python/ctsm/site_and_regional/tower_site.py index 5e0d9a434b..0b4ebddb7e 100644 --- a/python/ctsm/site_and_regional/tower_site.py +++ b/python/ctsm/site_and_regional/tower_site.py @@ -29,22 +29,20 @@ logger = logging.getLogger(__name__) +#pylint: disable=too-many-instance-attributes class TowerSite: """ Parent class to NeonSite and Plumber2Site classes. ... Attributes ---------- - Methods ------- - """ def __init__(self, name, start_year, end_year, start_month, end_month, finidat): """ Initializes TowerSite with the given arguments. - Parameters ---------- """ @@ -77,7 +75,6 @@ def build_base_case( Function for building a base_case to clone. To spend less time on building ctsm for the neon cases, all the other cases are cloned from this case - Args: self: The NeonSite object @@ -168,7 +165,6 @@ def get_batch_query(self, case): """ Function for querying the batch queue query command for a case, depending on the user's batch system. - Args: case: case object