diff --git a/CIME/SystemTests/funit.py b/CIME/SystemTests/funit.py index 1ebaf720604..a7c21a06944 100644 --- a/CIME/SystemTests/funit.py +++ b/CIME/SystemTests/funit.py @@ -5,7 +5,8 @@ from CIME.XML.standard_module_setup import * from CIME.SystemTests.system_tests_common import SystemTestsCommon from CIME.build import post_build -from CIME.utils import append_testlog, get_cime_root +from CIME.status import append_testlog +from CIME.utils import get_cime_root from CIME.test_status import * logger = logging.getLogger(__name__) diff --git a/CIME/SystemTests/hommebaseclass.py b/CIME/SystemTests/hommebaseclass.py index bad27d4aa56..8ea4a03c6b9 100644 --- a/CIME/SystemTests/hommebaseclass.py +++ b/CIME/SystemTests/hommebaseclass.py @@ -4,11 +4,11 @@ from CIME.XML.standard_module_setup import * from CIME.SystemTests.system_tests_common import SystemTestsCommon from CIME.build import post_build -from CIME.utils import append_testlog, SharedArea +from CIME.status import append_testlog +from CIME.utils import SharedArea from CIME.test_status import * import shutil -from distutils import dir_util logger = logging.getLogger(__name__) @@ -97,10 +97,9 @@ def run_phase(self): shutil.rmtree(full_baseline_dir) with SharedArea(): - dir_util.copy_tree( + shutil.copytree( os.path.join(exeroot, "tests", "baseline"), full_baseline_dir, - preserve_mode=False, ) elif compare: diff --git a/CIME/SystemTests/mvk.py b/CIME/SystemTests/mvk.py index 2ab2f72cd33..854dce74472 100644 --- a/CIME/SystemTests/mvk.py +++ b/CIME/SystemTests/mvk.py @@ -10,10 +10,11 @@ import json import logging -from distutils import dir_util +from shutils import copytree import CIME.test_status import CIME.utils +from CIME.status import append_testlog from CIME.SystemTests.system_tests_common import SystemTestsCommon from CIME.case.case_setup import case_setup from CIME.XML.machines import Machines @@ -173,10 +174,9 @@ def _compare_baseline(self): urlroot = CIME.utils.get_urlroot(mach_obj) if htmlroot is not None: with CIME.utils.SharedArea(): - dir_util.copy_tree( + copytree( evv_out_dir, os.path.join(htmlroot, "evv", case_name), - preserve_mode=False, ) if urlroot is None: urlroot = "[{}_URL]".format(mach_name.capitalize()) @@ -203,4 +203,4 @@ def _compare_baseline(self): ) ) - CIME.utils.append_testlog(comments, self._orig_caseroot) + append_testlog(comments, self._orig_caseroot) diff --git a/CIME/SystemTests/pgn.py b/CIME/SystemTests/pgn.py index a8696c0afd2..83199ec674e 100644 --- a/CIME/SystemTests/pgn.py +++ b/CIME/SystemTests/pgn.py @@ -16,7 +16,7 @@ import logging from collections import OrderedDict -from distutils import dir_util +from shutils import copytree import pandas as pd import numpy as np @@ -24,6 +24,7 @@ import CIME.test_status import CIME.utils +from CIME.status import append_testlog from CIME.SystemTests.system_tests_common import SystemTestsCommon from CIME.case.case_setup import case_setup from CIME.XML.machines import Machines @@ -80,7 +81,7 @@ def build_phase(self, sharedlib_only=False, model_only=False): if not model_only: # Lay all of the components out concurrently logger.debug( - "PGN_INFO: Updating NINST for multi-instance in " "env_mach_pes.xml" + "PGN_INFO: Updating NINST for multi-instance in env_mach_pes.xml" ) for comp in ["ATM", "OCN", "WAV", "GLC", "ICE", "ROF", "LND"]: ntasks = self._case.get_value("NTASKS_{}".format(comp)) @@ -224,10 +225,9 @@ def _compare_baseline(self): urlroot = CIME.utils.get_urlroot(mach_obj) if htmlroot is not None: with CIME.utils.SharedArea(): - dir_util.copy_tree( + copytree( evv_out_dir, os.path.join(htmlroot, "evv", case_name), - preserve_mode=False, ) if urlroot is None: urlroot = "[{}_URL]".format(mach_name.capitalize()) @@ -253,7 +253,7 @@ def _compare_baseline(self): ) ) - CIME.utils.append_testlog(comments, self._orig_caseroot) + append_testlog(comments, self._orig_caseroot) def run_phase(self): logger.debug("PGN_INFO: RUN PHASE") diff --git a/CIME/SystemTests/system_tests_common.py b/CIME/SystemTests/system_tests_common.py index 13057dd52e9..93ab4cf4c60 100644 --- a/CIME/SystemTests/system_tests_common.py +++ b/CIME/SystemTests/system_tests_common.py @@ -4,8 +4,8 @@ from CIME.XML.standard_module_setup import * from CIME.XML.env_run import EnvRun from CIME.XML.env_test import EnvTest +from CIME.status import append_testlog from CIME.utils import ( - append_testlog, get_model, safe_copy, get_timestamp, diff --git a/CIME/SystemTests/tsc.py b/CIME/SystemTests/tsc.py index f50fd4c334b..be695b7f482 100644 --- a/CIME/SystemTests/tsc.py +++ b/CIME/SystemTests/tsc.py @@ -11,10 +11,11 @@ import json import logging -from distutils import dir_util +from shutil import copytree import CIME.test_status import CIME.utils +from CIME.status import append_testlog from CIME.SystemTests.system_tests_common import SystemTestsCommon from CIME.case.case_setup import case_setup from CIME.hist_utils import rename_all_hist_files @@ -213,10 +214,9 @@ def _compare_baseline(self): urlroot = CIME.utils.get_urlroot(mach_obj) if htmlroot is not None: with CIME.utils.SharedArea(): - dir_util.copy_tree( + copytree( evv_out_dir, os.path.join(htmlroot, "evv", case_name), - preserve_mode=False, ) if urlroot is None: urlroot = "[{}_URL]".format(mach_name.capitalize()) @@ -243,7 +243,7 @@ def _compare_baseline(self): ) ) - CIME.utils.append_testlog(comments, self._orig_caseroot) + append_testlog(comments, self._orig_caseroot) def _generate_baseline(self): super(TSC, self)._generate_baseline() diff --git a/CIME/Tools/xmlchange b/CIME/Tools/xmlchange index 611e3116424..a5924389f10 100755 --- a/CIME/Tools/xmlchange +++ b/CIME/Tools/xmlchange @@ -51,9 +51,10 @@ from standard_script_setup import * from CIME.utils import ( expect, convert_to_type, - append_case_status, get_batch_script_for_job, + Timeout, ) +from CIME.status import append_case_status from CIME.case import Case from CIME.locked_files import check_lockedfiles @@ -62,6 +63,7 @@ import re # Set logger logger = logging.getLogger("xmlchange") + ############################################################################### def parse_command_line(args, description): ############################################################################### @@ -241,6 +243,40 @@ def xmlchange_single_value( if xmlid == "JOB_QUEUE": case.set_value("USER_REQUESTED_QUEUE", xmlval, subgroup) + # if CASE_GIT_REPOSITORY is updated set the remote to the new repo + # note that if origin exists and is different than CASE_GIT_REPOSITORY it + # will be deleted and replaced + if xmlid == "CASE_GIT_REPOSITORY": + if not case._gitinterface: + logger.warning( + "You must run case.setup before you can set CASE_GIT_REPOSITORY" + ) + return + remote = case._gitinterface.git_operation("remote") + if remote: + if remote == xmlval: + return + else: + case._gitinterface.git_operation("remote", "rm", remote) + branch = case._gitinterface.git_operation("branch").split()[1] + remote_branches = "" + with Timeout(30): + remote_branches = case._gitinterface.git_operation( + "ls-remote", "--heads", xmlval + ) + chk = remote_branches.find("refs/heads/" + branch) + + if chk >= 0: + expect( + False, + " Could not set {} to {}, remote branch {} already exists, Rename case and try again.".format( + xmlid, xmlval, branch + ), + ) + with Timeout(30): + case._gitinterface.git_operation("remote", "add", "origin", xmlval) + case._gitinterface.git_operation("push", "--set-upstream", "origin", branch) + def xmlchange( caseroot, @@ -255,7 +291,6 @@ def xmlchange( dryrun, non_local, ): - with Case(caseroot, read_only=False, record=True, non_local=non_local) as case: comp_classes = case.get_values("COMP_CLASSES") env_test = None @@ -275,7 +310,6 @@ def xmlchange( # Change values for setting in listofsettings: - pair = setting.split("=", 1) expect( len(pair) == 2, @@ -321,7 +355,13 @@ def xmlchange( for arg in sys.argv: argstr += "%s " % arg msg = " %s " % (argstr) - append_case_status("xmlchange", "success", msg=msg, caseroot=caseroot) + append_case_status( + "xmlchange", + "success", + msg=msg, + caseroot=caseroot, + gitinterface=case._gitinterface, + ) def _main_func(description): diff --git a/CIME/XML/env_batch.py b/CIME/XML/env_batch.py index 2c705555fea..b444a29333a 100644 --- a/CIME/XML/env_batch.py +++ b/CIME/XML/env_batch.py @@ -919,7 +919,6 @@ def _submit_single_job( resubmit_immediate=False, workflow=True, ): - if not dry_run: logger.warning("Submit job {}".format(job)) batch_system = self.get_value("BATCH_SYSTEM", subgroup=None) @@ -1080,19 +1079,19 @@ def _submit_single_job( submitargs, run_args, batchredirect, - get_batch_script_for_job(job), + os.path.join(self._caseroot, get_batch_script_for_job(job)), ) else: sequence = ( batchsubmit, submitargs, batchredirect, - get_batch_script_for_job(job), + os.path.join(self._caseroot, get_batch_script_for_job(job)), run_args, ) submitcmd = " ".join(s.strip() for s in sequence if s is not None) - if submitcmd.startswith("ssh"): + if submitcmd.startswith("ssh") and "$CASEROOT" in submitcmd: # add ` before cd $CASEROOT and at end of command submitcmd = submitcmd.replace("cd $CASEROOT", "'cd $CASEROOT") + "'" @@ -1385,7 +1384,6 @@ def make_all_batch_files(self, case): template = case.get_resolved_value( env_workflow.get_value("template", subgroup=job) ) - if os.path.isabs(template): input_batch_script = template else: diff --git a/CIME/build.py b/CIME/build.py index 2faa8b720b8..2d6c6414ffa 100644 --- a/CIME/build.py +++ b/CIME/build.py @@ -4,11 +4,11 @@ import glob, shutil, time, threading, subprocess from pathlib import Path from CIME.XML.standard_module_setup import * +from CIME.status import run_and_log_case_status from CIME.utils import ( get_model, analyze_build_log, stringify_bool, - run_and_log_case_status, get_timestamp, run_sub_or_cmd, run_cmd, @@ -1318,7 +1318,9 @@ def case_build( cb = cb + " (SHAREDLIB_BUILD)" if model_only == True: cb = cb + " (MODEL_BUILD)" - return run_and_log_case_status(functor, cb, caseroot=caseroot) + return run_and_log_case_status( + functor, cb, caseroot=caseroot, gitinterface=case._gitinterface + ) ############################################################################### @@ -1326,5 +1328,8 @@ def clean(case, cleanlist=None, clean_all=False, clean_depends=None): ############################################################################### functor = lambda: _clean_impl(case, cleanlist, clean_all, clean_depends) return run_and_log_case_status( - functor, "build.clean", caseroot=case.get_value("CASEROOT") + functor, + "build.clean", + caseroot=case.get_value("CASEROOT"), + gitinterface=case._gitinterface, ) diff --git a/CIME/case/case.py b/CIME/case/case.py index 7aab433b9a1..61f48fcea53 100644 --- a/CIME/case/case.py +++ b/CIME/case/case.py @@ -13,10 +13,12 @@ # pylint: disable=import-error,redefined-builtin from CIME import utils from CIME.config import Config -from CIME.utils import expect, get_cime_root, append_status +from CIME.status import append_status +from CIME.utils import expect, get_cime_root from CIME.utils import convert_to_type, get_model, set_model from CIME.utils import get_project, get_charge_account, check_name from CIME.utils import get_current_commit, safe_copy, get_cime_default_driver +from CIME.gitinterface import GitInterface from CIME.locked_files import LOCKED_DIR, lock_file from CIME.XML.machines import Machines from CIME.XML.pes import Pes @@ -77,7 +79,7 @@ class Case(object): """ - from CIME.case.case_setup import case_setup + from CIME.case.case_setup import case_setup, _create_case_repo from CIME.case.case_clone import create_clone, _copy_user_modified_to_clone from CIME.case.case_test import case_test from CIME.case.case_submit import check_DA_settings, check_case, submit @@ -98,7 +100,6 @@ class Case(object): ) def __init__(self, case_root=None, read_only=True, record=False, non_local=False): - if case_root is None: case_root = os.getcwd() expect( @@ -164,7 +165,7 @@ def __init__(self, case_root=None, read_only=True, record=False, non_local=False self._component_description = {} self._is_env_loaded = False self._loaded_envs = None - + self._gitinterface = None # these are user_mods as defined in the compset # Command Line user_mods are handled seperately @@ -199,6 +200,10 @@ def __init__(self, case_root=None, read_only=True, record=False, non_local=False mach == probed_machine, f"Current machine {probed_machine} does not match case machine {mach}.", ) + if os.path.exists(os.path.join(self.get_value("CASEROOT"), ".git")): + self._gitinterface = GitInterface( + self.get_value("CASEROOT"), logger + ) self.initialize_derived_attributes() @@ -682,7 +687,6 @@ def _set_compset(self, compset_name, files): # Loop through all of the files listed in COMPSETS_SPEC_FILE and find the file # that has a match for either the alias or the longname in that order for component in components: - # Determine the compsets file for this component compsets_filename = files.get_value( "COMPSETS_SPEC_FILE", {"component": component} @@ -1284,7 +1288,6 @@ def configure( gpu_type=None, gpu_offload=None, ): - expect( check_name(compset_name, additional_chars="."), "Invalid compset name {}".format(compset_name), diff --git a/CIME/case/case_cmpgen_namelists.py b/CIME/case/case_cmpgen_namelists.py index c76ce826519..10ded368147 100644 --- a/CIME/case/case_cmpgen_namelists.py +++ b/CIME/case/case_cmpgen_namelists.py @@ -7,11 +7,11 @@ from CIME.compare_namelists import is_namelist_file, compare_namelist_files from CIME.simple_compare import compare_files, compare_runconfigfiles -from CIME.utils import append_status, safe_copy, SharedArea +from CIME.utils import safe_copy, SharedArea +from CIME.status import append_status from CIME.test_status import * import os, shutil, traceback, stat, glob -from distutils import dir_util logger = logging.getLogger(__name__) @@ -93,7 +93,7 @@ def _do_full_nl_gen_impl(case, test, generate_name, baseline_root=None): if os.path.isdir(baseline_casedocs): shutil.rmtree(baseline_casedocs) - dir_util.copy_tree(casedoc_dir, baseline_casedocs, preserve_mode=False) + shutil.copytree(casedoc_dir, baseline_casedocs) for item in glob.glob(os.path.join(test_dir, "user_nl*")): preexisting_baseline = os.path.join(baseline_dir, os.path.basename(item)) @@ -186,7 +186,11 @@ def case_cmpgen_namelists( NAMELIST_PHASE, TEST_PASS_STATUS if success else TEST_FAIL_STATUS ) try: - append_status(output, logfile_name, caseroot=caseroot) + append_status( + output, + logfile_name, + caseroot=caseroot, + ) except IOError: pass diff --git a/CIME/case/case_run.py b/CIME/case/case_run.py index bc132a5aee7..2c21bd4b076 100644 --- a/CIME/case/case_run.py +++ b/CIME/case/case_run.py @@ -3,9 +3,10 @@ '""" from CIME.XML.standard_module_setup import * from CIME.config import Config -from CIME.utils import gzip_existing_file, new_lid, run_and_log_case_status -from CIME.utils import run_sub_or_cmd, append_status, safe_copy, model_log, CIMEError +from CIME.utils import gzip_existing_file, new_lid +from CIME.utils import run_sub_or_cmd, safe_copy, model_log, CIMEError from CIME.utils import batch_jobid, is_comp_standalone +from CIME.status import append_status, run_and_log_case_status from CIME.get_timing import get_timing from CIME.locked_files import check_lockedfiles @@ -15,6 +16,7 @@ logger = logging.getLogger(__name__) + ############################################################################### def _pre_run_check(case, lid, skip_pnl=False, da_cycle=0): ############################################################################### @@ -180,6 +182,7 @@ def _run_model_impl(case, lid, skip_pnl=False, da_cycle=0): custom_success_msg_functor=msg_func, caseroot=case.get_value("CASEROOT"), is_batch=is_batch, + gitinterface=case._gitinterface, ) cmd_success = True except CIMEError: @@ -291,6 +294,7 @@ def _run_model(case, lid, skip_pnl=False, da_cycle=0): custom_success_msg_functor=msg_func, caseroot=case.get_value("CASEROOT"), is_batch=is_batch, + gitinterface=case._gitinterface, ) diff --git a/CIME/case/case_setup.py b/CIME/case/case_setup.py index dfbab723697..d4b510f13f4 100644 --- a/CIME/case/case_setup.py +++ b/CIME/case/case_setup.py @@ -13,16 +13,18 @@ copy_depends_files, ) from CIME.utils import ( - run_and_log_case_status, get_batch_script_for_job, safe_copy, file_contains_python_function, import_from_file, copy_local_macros_to_dir, + batch_jobid, + run_cmd_no_fail, ) -from CIME.utils import batch_jobid +from CIME.status import run_and_log_case_status, append_case_status from CIME.test_status import * from CIME.locked_files import unlock_file, lock_file, check_lockedfiles +from CIME.gitinterface import GitInterface import errno, shutil @@ -517,3 +519,41 @@ def case_setup(self, clean=False, test_mode=False, reset=False, keep=None): caseroot=caseroot, is_batch=is_batch, ) + self._create_case_repo(caseroot) + + +def _create_case_repo(self, caseroot): + version = run_cmd_no_fail("git --version") + result = re.findall(r"([0-9]+)\.([0-9]+)\.?[0-9]*", version) + major = int(result[0][0]) + minor = int(result[0][1]) + + # gitinterface needs git version 2.28 or newer + if major > 2 or (major == 2 and minor >= 28): + self._gitinterface = GitInterface( + caseroot, logger, branch=self.get_value("CASE") + ) + if not os.path.exists(os.path.join(caseroot, ".gitignore")): + safe_copy( + os.path.join( + self.get_value("CIMEROOT"), + "CIME", + "data", + "templates", + "gitignore.template", + ), + os.path.join(caseroot, ".gitignore"), + ) + append_case_status( + "", "", "local git repository created", gitinterface=self._gitinterface + ) + # add all files in caseroot to local repository + self._gitinterface._git_command("add", "*") + else: + logger.warning("git interface requires git version 2.28 or newer") + + append_case_status( + "", + "", + f"local git version too old for cime git interface {major}.{minor}", + ) diff --git a/CIME/case/case_st_archive.py b/CIME/case/case_st_archive.py index d735d6c587a..1135741228b 100644 --- a/CIME/case/case_st_archive.py +++ b/CIME/case/case_st_archive.py @@ -8,13 +8,13 @@ from CIME.XML.standard_module_setup import * from CIME.utils import ( - run_and_log_case_status, ls_sorted_by_mtime, symlink_force, safe_copy, find_files, + batch_jobid, ) -from CIME.utils import batch_jobid +from CIME.status import run_and_log_case_status from CIME.date import get_file_date from CIME.XML.archive import Archive from CIME.XML.files import Files @@ -22,6 +22,7 @@ logger = logging.getLogger(__name__) + ############################################################################### def _get_archive_fn_desc(archive_fn): ############################################################################### @@ -179,7 +180,6 @@ def _archive_rpointer_files( # Generate rpointer file(s) for interim restarts for the one datename and each # possible value of ninst_strings if save_interim_restart_files: - # parse env_archive.xml to determine the rpointer files # and contents for the given archive_entry tag rpointer_items = archive.get_rpointer_contents(archive_entry) @@ -445,7 +445,7 @@ def _archive_restarts_date( histfiles_savein_rundir_by_compname = {} - for (archive_entry, compname, compclass) in _get_component_archive_entries( + for archive_entry, compname, compclass in _get_component_archive_entries( components, archive ): if compclass: @@ -879,7 +879,7 @@ def _archive_process( # archive history files - for (_, compname, compclass) in _get_component_archive_entries(components, archive): + for _, compname, compclass in _get_component_archive_entries(components, archive): if compclass: logger.info( "Archiving history files for {} ({})".format(compname, compclass) @@ -1050,6 +1050,7 @@ def case_st_archive( custom_success_msg_functor=msg_func, caseroot=caseroot, is_batch=is_batch, + gitinterface=self._gitinterface, ) logger.info("st_archive completed") diff --git a/CIME/case/case_submit.py b/CIME/case/case_submit.py index 037a08f0615..7e0e09881e4 100644 --- a/CIME/case/case_submit.py +++ b/CIME/case/case_submit.py @@ -8,7 +8,8 @@ """ import configparser from CIME.XML.standard_module_setup import * -from CIME.utils import expect, run_and_log_case_status, CIMEError, get_time_in_seconds +from CIME.utils import expect, CIMEError, get_time_in_seconds +from CIME.status import run_and_log_case_status from CIME.locked_files import ( unlock_file, lock_file, @@ -283,6 +284,7 @@ def submit( caseroot=caseroot, custom_success_msg_functor=lambda x: x.split(":")[-1], is_batch=is_batch, + gitinterface=self._gitinterface, ) except BaseException: # Want to catch KeyboardInterrupt too # If something failed in the batch system, make sure to mark diff --git a/CIME/case/case_test.py b/CIME/case/case_test.py index 030c5f6618b..c607dd93786 100644 --- a/CIME/case/case_test.py +++ b/CIME/case/case_test.py @@ -4,8 +4,9 @@ """ from CIME.XML.standard_module_setup import * -from CIME.utils import expect, find_system_test, append_testlog, find_proc_id +from CIME.utils import expect, find_system_test, find_proc_id from CIME.SystemTests.system_tests_common import * +from CIME.status import append_testlog import sys, signal diff --git a/CIME/compare_test_results.py b/CIME/compare_test_results.py index d9220551986..a46dbe50fe7 100644 --- a/CIME/compare_test_results.py +++ b/CIME/compare_test_results.py @@ -1,5 +1,6 @@ import CIME.compare_namelists, CIME.simple_compare -from CIME.utils import append_status, EnvironmentContext, parse_test_name +from CIME.status import append_status +from CIME.utils import EnvironmentContext, parse_test_name from CIME.test_status import * from CIME.hist_utils import compare_baseline, get_ts_synopsis from CIME.case import Case diff --git a/CIME/data/templates/gitignore.template b/CIME/data/templates/gitignore.template new file mode 100644 index 00000000000..02a3c097804 --- /dev/null +++ b/CIME/data/templates/gitignore.template @@ -0,0 +1,77 @@ +# cime build and run directories +bld +run +# logger output files +*.log + +#python files +__pycache__/ +*.so + +# Swap +[._]*.s[a-v][a-z] +!*.svg # comment out if you don't need vector files +[._]*.sw[a-p] +[._]s[a-rt-v][a-z] +[._]ss[a-gi-z] +[._]sw[a-p] + +# Session +Session.vim +Sessionx.vim + +# Temporary +.netrwhist +*~ +# Auto-generated tag files +tags +# Persistent undo +[._]*.un~ +# -*- mode: gitignore; -*- +*~ +\#*\# +/.emacs.desktop +/.emacs.desktop.lock +*.elc +auto-save-list +tramp +.\#* + +# Org-mode +.org-id-locations +*_archive + +# flymake-mode +*_flymake.* + +# eshell files +/eshell/history +/eshell/lastdir + +# elpa packages +/elpa/ + +# reftex files +*.rel + +# AUCTeX auto folder +/auto/ + +# cask packages +.cask/ +dist/ + +# Flycheck +flycheck_*.el + +# server auth directory +/server/ + +# projectiles files +.projectile + +# directory configuration +.dir-locals.el + +# network security +/network-security.data diff --git a/CIME/gitinterface.py b/CIME/gitinterface.py new file mode 100644 index 00000000000..cfb0c42fcb4 --- /dev/null +++ b/CIME/gitinterface.py @@ -0,0 +1,100 @@ +import sys +from CIME.utils import run_cmd_no_fail +from pathlib import Path + + +class GitInterface: + def __init__(self, repo_path, logger, branch=None): + logger.debug("Initialize GitInterface for {}".format(repo_path)) + if isinstance(repo_path, str): + self.repo_path = Path(repo_path).resolve() + elif isinstance(repo_path, Path): + self.repo_path = repo_path.resolve() + else: + raise TypeError("repo_path must be a str or Path object") + self.logger = logger + try: + import git + + self._use_module = True + try: + self.repo = git.Repo(str(self.repo_path)) # Initialize GitPython repo + except git.exc.InvalidGitRepositoryError: + self.git = git + self._init_git_repo(branch=branch) + msg = "Using GitPython interface to git" + except ImportError: + self._use_module = False + if not (self.repo_path / ".git").exists(): + self._init_git_repo(branch=branch) + msg = "Using shell interface to git" + self.logger.debug(msg) + + def _git_command(self, operation, *args): + self.logger.debug(operation) + if self._use_module and operation != "submodule": + try: + return getattr(self.repo.git, operation)(*args) + except Exception as e: + sys.exit(e) + else: + return ["git", "-C", str(self.repo_path), operation] + list(args) + + def _init_git_repo(self, branch=None): + if self._use_module: + self.repo = self.git.Repo.init(str(self.repo_path)) + if branch: + self.git_operation("checkout", "-b", branch) + else: + command = ["git", "-C", str(self.repo_path), "init"] + if branch: + command.extend(["-b", branch]) + run_cmd_no_fail(" ".join(command)) + + # pylint: disable=unused-argument + def git_operation(self, operation, *args, **kwargs): + command = self._git_command(operation, *args) + if isinstance(command, list): + try: + return run_cmd_no_fail(" ".join(command)) + except Exception as e: + sys.exit(e) + else: + return command + + def config_get_value(self, section, name): + if self._use_module: + config = self.repo.config_reader() + try: + val = config.get_value(section, name) + except: + val = None + return val + else: + cmd = ( + "git", + "-C", + str(self.repo_path), + "config", + "--get", + f"{section}.{name}", + ) + output = run_cmd_no_fail(cmd) + return output.strip() + + def config_set_value(self, section, name, value): + if self._use_module: + with self.repo.config_writer() as writer: + writer.set_value(section, name, value) + writer.release() # Ensure changes are saved + else: + cmd = ( + "git", + "-C", + str(self.repo_path), + "config", + f"{section}.{name}", + value, + ) + self.logger.debug(cmd) + run_cmd_no_fail(cmd) diff --git a/CIME/status.py b/CIME/status.py new file mode 100644 index 00000000000..23109b07645 --- /dev/null +++ b/CIME/status.py @@ -0,0 +1,132 @@ +# These routines were moved from utils.py to avoid circular dependancies +import time, os, sys, logging +from CIME.utils import Timeout, CASE_SUCCESS, CASE_FAILURE + +logger = logging.getLogger(__name__) + + +def append_status(msg, sfile, caseroot="."): + """ + Append msg to sfile in caseroot + """ + ctime = time.strftime("%Y-%m-%d %H:%M:%S: ") + + # Reduce empty lines in CaseStatus. It's a very concise file + # and does not need extra newlines for readability + line_ending = "\n" + with open(os.path.join(caseroot, sfile), "a") as fd: + fd.write(ctime + msg + line_ending) + fd.write(" ---------------------------------------------------" + line_ending) + + +def append_testlog(msg, caseroot="."): + """ + Add to TestStatus.log file + """ + append_status(msg, "TestStatus.log", caseroot) + + +def append_case_status(phase, status, msg=None, caseroot=".", gitinterface=None): + """ + Update CaseStatus file + """ + msg = msg if msg else "" + append_status( + "{} {} {}".format(phase, status, msg), + "CaseStatus", + caseroot, + ) + if gitinterface: + filelist = gitinterface.git_operation( + "ls-files", "--deleted", "--exclude-standard" + ).splitlines() + # First delete files that have been removed + for f in filelist: + logger.debug("removing file {}".format(f)) + gitinterface.git_operation("rm", f) + filelist = gitinterface.git_operation( + "ls-files", "--others", "--modified", "--exclude-standard" + ).splitlines() + # Files that should not be added should have been excluded by the .gitignore file + for f in filelist: + logger.debug("adding file {}".format(f)) + gitinterface.git_operation("add", f) + msg = msg if msg else " no message provided" + gitinterface.git_operation("commit", "-m", '"' + msg + '"') + remote = gitinterface.git_operation("remote") + if remote: + with Timeout(30): + gitinterface.git_operation("push", remote) + + +def run_and_log_case_status( + func, + phase, + caseroot=".", + custom_starting_msg_functor=None, + custom_success_msg_functor=None, + is_batch=False, + gitinterface=None, +): + starting_msg = None + + if custom_starting_msg_functor is not None: + starting_msg = custom_starting_msg_functor() + + # Delay appending "starting" on "case.subsmit" phase when batch system is + # present since we don't have the jobid yet + if phase != "case.submit" or not is_batch: + append_case_status( + phase, + "starting", + msg=starting_msg, + caseroot=caseroot, + gitinterface=gitinterface, + ) + rv = None + try: + rv = func() + except BaseException: + custom_success_msg = ( + custom_success_msg_functor(rv) + if custom_success_msg_functor and rv is not None + else None + ) + if phase == "case.submit" and is_batch: + append_case_status( + phase, + "starting", + msg=custom_success_msg, + caseroot=caseroot, + gitinterface=gitinterface, + ) + e = sys.exc_info()[1] + append_case_status( + phase, + CASE_FAILURE, + msg=("\n{}".format(e)), + caseroot=caseroot, + gitinterface=gitinterface, + ) + raise + else: + custom_success_msg = ( + custom_success_msg_functor(rv) if custom_success_msg_functor else None + ) + if phase == "case.submit" and is_batch: + append_case_status( + phase, + "starting", + msg=custom_success_msg, + caseroot=caseroot, + gitinterface=gitinterface, + ) + append_case_status( + phase, + CASE_SUCCESS, + msg=custom_success_msg, + caseroot=caseroot, + gitinterface=gitinterface, + ) + + return rv diff --git a/CIME/test_scheduler.py b/CIME/test_scheduler.py index b8b12ae08d2..e1dc99400f4 100644 --- a/CIME/test_scheduler.py +++ b/CIME/test_scheduler.py @@ -14,9 +14,8 @@ from CIME.XML.standard_module_setup import * from CIME.get_tests import get_recommended_test_time, get_build_groups, is_perf_test +from CIME.status import append_status, append_testlog from CIME.utils import ( - append_status, - append_testlog, TESTS_FAILED_ERR_CODE, parse_test_name, get_full_test_name, diff --git a/CIME/tests/test_unit_utils.py b/CIME/tests/test_unit_utils.py index 2994a87ffb1..8abde4d8fef 100755 --- a/CIME/tests/test_unit_utils.py +++ b/CIME/tests/test_unit_utils.py @@ -8,9 +8,9 @@ import unittest from unittest import mock +from CIME.status import run_and_log_case_status from CIME.utils import ( indent_string, - run_and_log_case_status, import_from_file, _line_defines_python_function, file_contains_python_function, diff --git a/CIME/utils.py b/CIME/utils.py index 864f0299b27..0973647823b 100644 --- a/CIME/utils.py +++ b/CIME/utils.py @@ -22,6 +22,8 @@ # Fix to pass user defined `srcroot` to `CIME.XML.generic_xml.GenericXML` # where it's used to resolve $SRCROOT in XML config files. GLOBAL = {} +CASE_SUCCESS = "success" +CASE_FAILURE = "error" def deprecate_action(message): @@ -1748,7 +1750,6 @@ def convert_to_type(value, type_str, vid=""): vid is only for generating better error messages. """ if value is not None: - if type_str == "char": pass @@ -1794,7 +1795,6 @@ def convert_to_unknown_type(value): Convert value to it's real type by probing conversions. """ if value is not None: - # Attempt to convert to logical if value.upper() in ["TRUE", "FALSE"]: return value.upper() == "TRUE" @@ -2056,39 +2056,6 @@ def format_time(time_format, input_format, input_time): return output_time -def append_status(msg, sfile, caseroot="."): - """ - Append msg to sfile in caseroot - """ - ctime = time.strftime("%Y-%m-%d %H:%M:%S: ") - - # Reduce empty lines in CaseStatus. It's a very concise file - # and does not need extra newlines for readability - line_ending = "\n" - - with open(os.path.join(caseroot, sfile), "a") as fd: - fd.write(ctime + msg + line_ending) - fd.write(" ---------------------------------------------------" + line_ending) - - -def append_testlog(msg, caseroot="."): - """ - Add to TestStatus.log file - """ - append_status(msg, "TestStatus.log", caseroot) - - -def append_case_status(phase, status, msg=None, caseroot="."): - """ - Update CaseStatus file - """ - append_status( - "{} {}{}".format(phase, status, " {}".format(msg if msg else "")), - "CaseStatus", - caseroot, - ) - - def does_file_have_string(filepath, text): """ Does the text string appear in the filepath file @@ -2457,60 +2424,6 @@ def verbatim_success_msg(return_val): return return_val -CASE_SUCCESS = "success" -CASE_FAILURE = "error" - - -def run_and_log_case_status( - func, - phase, - caseroot=".", - custom_starting_msg_functor=None, - custom_success_msg_functor=None, - is_batch=False, -): - starting_msg = None - - if custom_starting_msg_functor is not None: - starting_msg = custom_starting_msg_functor() - - # Delay appending "starting" on "case.subsmit" phase when batch system is - # present since we don't have the jobid yet - if phase != "case.submit" or not is_batch: - append_case_status(phase, "starting", msg=starting_msg, caseroot=caseroot) - rv = None - try: - rv = func() - except BaseException: - custom_success_msg = ( - custom_success_msg_functor(rv) - if custom_success_msg_functor and rv is not None - else None - ) - if phase == "case.submit" and is_batch: - append_case_status( - phase, "starting", msg=custom_success_msg, caseroot=caseroot - ) - e = sys.exc_info()[1] - append_case_status( - phase, CASE_FAILURE, msg=("\n{}".format(e)), caseroot=caseroot - ) - raise - else: - custom_success_msg = ( - custom_success_msg_functor(rv) if custom_success_msg_functor else None - ) - if phase == "case.submit" and is_batch: - append_case_status( - phase, "starting", msg=custom_success_msg, caseroot=caseroot - ) - append_case_status( - phase, CASE_SUCCESS, msg=custom_success_msg, caseroot=caseroot - ) - - return rv - - def _check_for_invalid_args(args): # Prevent circular import from CIME.config import Config diff --git a/docker/Dockerfile b/docker/Dockerfile index 05e5bbe5fb8..5a3cdb295c4 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -12,7 +12,6 @@ RUN mkdir -p \ wget -O /storage/inputdata/share/domains/domain.ocn.ne4np4_oQU240.160614.nc https://portal.nersc.gov/project/e3sm/inputdata/share/domains/domain.ocn.ne4np4_oQU240.160614.nc && \ wget -O /storage/inputdata/share/domains/domain.lnd.ne4np4_oQU240.160614.nc https://portal.nersc.gov/project/e3sm/inputdata/share/domains/domain.lnd.ne4np4_oQU240.160614.nc - COPY cime.yaml /cime.yaml RUN mamba env update -f /cime.yaml && \