From 1fa26aa59af2e51ec15e50760fd09e493a2444b2 Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Tue, 6 Aug 2024 11:08:31 -0600 Subject: [PATCH 01/16] works with local repo --- CIME/SystemTests/system_tests_common.py | 2 +- CIME/Tools/xmlchange | 4 +- CIME/build.py | 4 +- CIME/case/case.py | 13 ++-- CIME/case/case_cmpgen_namelists.py | 5 +- CIME/case/case_run.py | 7 +- CIME/case/case_setup.py | 14 +++- CIME/case/case_st_archive.py | 5 +- CIME/case/case_submit.py | 3 +- CIME/case/case_test.py | 3 +- CIME/gitinterface.py | 86 ++++++++++++++++++++++ CIME/non_py/gitignore.template | 77 ++++++++++++++++++++ CIME/status.py | 96 +++++++++++++++++++++++++ CIME/utils.py | 90 +---------------------- 14 files changed, 302 insertions(+), 107 deletions(-) create mode 100644 CIME/gitinterface.py create mode 100644 CIME/non_py/gitignore.template create mode 100644 CIME/status.py diff --git a/CIME/SystemTests/system_tests_common.py b/CIME/SystemTests/system_tests_common.py index 13057dd52e9..93ab4cf4c60 100644 --- a/CIME/SystemTests/system_tests_common.py +++ b/CIME/SystemTests/system_tests_common.py @@ -4,8 +4,8 @@ from CIME.XML.standard_module_setup import * from CIME.XML.env_run import EnvRun from CIME.XML.env_test import EnvTest +from CIME.status import append_testlog from CIME.utils import ( - append_testlog, get_model, safe_copy, get_timestamp, diff --git a/CIME/Tools/xmlchange b/CIME/Tools/xmlchange index 611e3116424..9e6927636d3 100755 --- a/CIME/Tools/xmlchange +++ b/CIME/Tools/xmlchange @@ -51,9 +51,9 @@ from standard_script_setup import * from CIME.utils import ( expect, convert_to_type, - append_case_status, get_batch_script_for_job, ) +from CIME.status import append_case_status from CIME.case import Case from CIME.locked_files import check_lockedfiles @@ -321,7 +321,7 @@ def xmlchange( for arg in sys.argv: argstr += "%s " % arg msg = " %s " % (argstr) - append_case_status("xmlchange", "success", msg=msg, caseroot=caseroot) + append_case_status("xmlchange", "success", msg=msg, caseroot=caseroot, gitinterface=case._gitinterface) def _main_func(description): diff --git a/CIME/build.py b/CIME/build.py index 2faa8b720b8..4440c9294ec 100644 --- a/CIME/build.py +++ b/CIME/build.py @@ -4,11 +4,11 @@ import glob, shutil, time, threading, subprocess from pathlib import Path from CIME.XML.standard_module_setup import * +from CIME.status import run_and_log_case_status from CIME.utils import ( get_model, analyze_build_log, stringify_bool, - run_and_log_case_status, get_timestamp, run_sub_or_cmd, run_cmd, @@ -1318,7 +1318,7 @@ def case_build( cb = cb + " (SHAREDLIB_BUILD)" if model_only == True: cb = cb + " (MODEL_BUILD)" - return run_and_log_case_status(functor, cb, caseroot=caseroot) + return run_and_log_case_status(functor, cb, caseroot=caseroot, gitinterface=case._gitinterface) ############################################################################### diff --git a/CIME/case/case.py b/CIME/case/case.py index 7aab433b9a1..c0c4f074cd8 100644 --- a/CIME/case/case.py +++ b/CIME/case/case.py @@ -13,10 +13,12 @@ # pylint: disable=import-error,redefined-builtin from CIME import utils from CIME.config import Config -from CIME.utils import expect, get_cime_root, append_status +from CIME.status import append_status +from CIME.utils import expect, get_cime_root from CIME.utils import convert_to_type, get_model, set_model from CIME.utils import get_project, get_charge_account, check_name from CIME.utils import get_current_commit, safe_copy, get_cime_default_driver +from CIME.gitinterface import GitInterface from CIME.locked_files import LOCKED_DIR, lock_file from CIME.XML.machines import Machines from CIME.XML.pes import Pes @@ -77,7 +79,7 @@ class Case(object): """ - from CIME.case.case_setup import case_setup + from CIME.case.case_setup import case_setup, _create_case_repo from CIME.case.case_clone import create_clone, _copy_user_modified_to_clone from CIME.case.case_test import case_test from CIME.case.case_submit import check_DA_settings, check_case, submit @@ -164,7 +166,7 @@ def __init__(self, case_root=None, read_only=True, record=False, non_local=False self._component_description = {} self._is_env_loaded = False self._loaded_envs = None - + self._gitinterface = None # these are user_mods as defined in the compset # Command Line user_mods are handled seperately @@ -199,9 +201,12 @@ def __init__(self, case_root=None, read_only=True, record=False, non_local=False mach == probed_machine, f"Current machine {probed_machine} does not match case machine {mach}.", ) + if os.path.exists(os.path.join(self.get_value("CASEROOT"),".git")): + self._gitinterface = GitInterface(self.get_value("CASEROOT"),logger) - self.initialize_derived_attributes() + self.initialize_derived_attributes() + def get_baseline_dir(self): baseline_root = self.get_value("BASELINE_ROOT") diff --git a/CIME/case/case_cmpgen_namelists.py b/CIME/case/case_cmpgen_namelists.py index c76ce826519..5407f1755e5 100644 --- a/CIME/case/case_cmpgen_namelists.py +++ b/CIME/case/case_cmpgen_namelists.py @@ -7,7 +7,8 @@ from CIME.compare_namelists import is_namelist_file, compare_namelist_files from CIME.simple_compare import compare_files, compare_runconfigfiles -from CIME.utils import append_status, safe_copy, SharedArea +from CIME.utils import safe_copy, SharedArea +from CIME.status import append_status from CIME.test_status import * import os, shutil, traceback, stat, glob @@ -186,7 +187,7 @@ def case_cmpgen_namelists( NAMELIST_PHASE, TEST_PASS_STATUS if success else TEST_FAIL_STATUS ) try: - append_status(output, logfile_name, caseroot=caseroot) + append_status(output, logfile_name, caseroot=caseroot, gitinterface=self._gitinterface) except IOError: pass diff --git a/CIME/case/case_run.py b/CIME/case/case_run.py index bc132a5aee7..8550d35fb25 100644 --- a/CIME/case/case_run.py +++ b/CIME/case/case_run.py @@ -3,9 +3,10 @@ '""" from CIME.XML.standard_module_setup import * from CIME.config import Config -from CIME.utils import gzip_existing_file, new_lid, run_and_log_case_status -from CIME.utils import run_sub_or_cmd, append_status, safe_copy, model_log, CIMEError +from CIME.utils import gzip_existing_file, new_lid +from CIME.utils import run_sub_or_cmd, safe_copy, model_log, CIMEError from CIME.utils import batch_jobid, is_comp_standalone +from CIME.status import append_status, run_and_log_case_status from CIME.get_timing import get_timing from CIME.locked_files import check_lockedfiles @@ -180,6 +181,7 @@ def _run_model_impl(case, lid, skip_pnl=False, da_cycle=0): custom_success_msg_functor=msg_func, caseroot=case.get_value("CASEROOT"), is_batch=is_batch, + gitinterface=case._gitinterface ) cmd_success = True except CIMEError: @@ -291,6 +293,7 @@ def _run_model(case, lid, skip_pnl=False, da_cycle=0): custom_success_msg_functor=msg_func, caseroot=case.get_value("CASEROOT"), is_batch=is_batch, + gitinterface=case._gitinterface, ) diff --git a/CIME/case/case_setup.py b/CIME/case/case_setup.py index dfbab723697..d7497f89b89 100644 --- a/CIME/case/case_setup.py +++ b/CIME/case/case_setup.py @@ -13,16 +13,17 @@ copy_depends_files, ) from CIME.utils import ( - run_and_log_case_status, get_batch_script_for_job, safe_copy, file_contains_python_function, import_from_file, copy_local_macros_to_dir, ) +from CIME.status import run_and_log_case_status, append_case_status from CIME.utils import batch_jobid from CIME.test_status import * from CIME.locked_files import unlock_file, lock_file, check_lockedfiles +from CIME.gitinterface import GitInterface import errno, shutil @@ -487,7 +488,7 @@ def case_setup(self, clean=False, test_mode=False, reset=False, keep=None): if is_batch: jobid = batch_jobid() msg_func = lambda *args: jobid if jobid is not None else "" - + if self.get_value("TEST") and not test_mode: test_name = casebaseid if casebaseid is not None else self.get_value("CASE") with TestStatus(test_dir=caseroot, test_name=test_name) as ts: @@ -517,3 +518,12 @@ def case_setup(self, clean=False, test_mode=False, reset=False, keep=None): caseroot=caseroot, is_batch=is_batch, ) + if not os.path.exists(os.path.join(caseroot, ".git")): + self._create_case_repo(caseroot) + +def _create_case_repo(self, caseroot): + self._gitinterface = GitInterface(caseroot,logger,branch=self.get_value("CASE")) + safe_copy(os.path.join(self.get_value("CIMEROOT"), "CIME","non_py","gitignore.template"), os.path.join(caseroot,".gitignore")) + # add all files in caseroot to local repository + self._gitinterface._git_command("add","*") + append_case_status("", "", "local git repository created", gitinterface=self._gitinterface) diff --git a/CIME/case/case_st_archive.py b/CIME/case/case_st_archive.py index d735d6c587a..b330db5d88e 100644 --- a/CIME/case/case_st_archive.py +++ b/CIME/case/case_st_archive.py @@ -8,13 +8,13 @@ from CIME.XML.standard_module_setup import * from CIME.utils import ( - run_and_log_case_status, ls_sorted_by_mtime, symlink_force, safe_copy, find_files, + batch_jobid, ) -from CIME.utils import batch_jobid +from CIME.status import run_and_log_case_status from CIME.date import get_file_date from CIME.XML.archive import Archive from CIME.XML.files import Files @@ -1050,6 +1050,7 @@ def case_st_archive( custom_success_msg_functor=msg_func, caseroot=caseroot, is_batch=is_batch, + gitinterface=self._gitinterface ) logger.info("st_archive completed") diff --git a/CIME/case/case_submit.py b/CIME/case/case_submit.py index 037a08f0615..22914fad588 100644 --- a/CIME/case/case_submit.py +++ b/CIME/case/case_submit.py @@ -8,7 +8,8 @@ """ import configparser from CIME.XML.standard_module_setup import * -from CIME.utils import expect, run_and_log_case_status, CIMEError, get_time_in_seconds +from CIME.utils import expect, CIMEError, get_time_in_seconds +from CIME.status import run_and_log_case_status from CIME.locked_files import ( unlock_file, lock_file, diff --git a/CIME/case/case_test.py b/CIME/case/case_test.py index 030c5f6618b..c607dd93786 100644 --- a/CIME/case/case_test.py +++ b/CIME/case/case_test.py @@ -4,8 +4,9 @@ """ from CIME.XML.standard_module_setup import * -from CIME.utils import expect, find_system_test, append_testlog, find_proc_id +from CIME.utils import expect, find_system_test, find_proc_id from CIME.SystemTests.system_tests_common import * +from CIME.status import append_testlog import sys, signal diff --git a/CIME/gitinterface.py b/CIME/gitinterface.py new file mode 100644 index 00000000000..455f6e6c68f --- /dev/null +++ b/CIME/gitinterface.py @@ -0,0 +1,86 @@ +import os +import sys +from . import utils +from pathlib import Path + +class GitInterface: + def __init__(self, repo_path, logger, branch=None): + logger.debug("Initialize GitInterface for {}".format(repo_path)) + if isinstance(repo_path, str): + self.repo_path = Path(repo_path).resolve() + elif isinstance(repo_path, Path): + self.repo_path = repo_path.resolve() + else: + raise TypeError("repo_path must be a str or Path object") + self.logger = logger + try: + import git + + self._use_module = True + try: + self.repo = git.Repo(str(self.repo_path)) # Initialize GitPython repo + except git.exc.InvalidGitRepositoryError: + self.git = git + self._init_git_repo(branch=branch) + msg = "Using GitPython interface to git" + except ImportError: + self._use_module = False + if not (self.repo_path / ".git").exists(): + self._init_git_repo(branch=branch) + msg = "Using shell interface to git" + self.logger.debug(msg) + + def _git_command(self, operation, *args): + self.logger.debug(operation) + if self._use_module and operation != "submodule": + try: + return getattr(self.repo.git, operation)(*args) + except Exception as e: + sys.exit(e) + else: + return ["git", "-C", str(self.repo_path), operation] + list(args) + + def _init_git_repo(self, branch=None): + if self._use_module: + self.repo = self.git.Repo.init(str(self.repo_path)) + if branch: + self.git_operation("checkout", "-b",branch) + else: + command = ("git", "-C", str(self.repo_path), "init") + if branch: + command.extend("-b", branch) + utils.execute_subprocess(command) + + # pylint: disable=unused-argument + def git_operation(self, operation, *args, **kwargs): + command = self._git_command(operation, *args) + if isinstance(command, list): + try: + return utils.execute_subprocess(command, output_to_caller=True) + except Exception as e: + sys.exit(e) + else: + return command + + def config_get_value(self, section, name): + if self._use_module: + config = self.repo.config_reader() + try: + val = config.get_value(section, name) + except: + val = None + return val + else: + cmd = ("git", "-C", str(self.repo_path), "config", "--get", f"{section}.{name}") + output = utils.execute_subprocess(cmd, output_to_caller=True) + return output.strip() + + def config_set_value(self, section, name, value): + if self._use_module: + with self.repo.config_writer() as writer: + writer.set_value(section, name, value) + writer.release() # Ensure changes are saved + else: + cmd = ("git", "-C", str(self.repo_path), "config", f"{section}.{name}", value) + self.logger.debug(cmd) + utils.execute_subprocess(cmd, output_to_caller=True) diff --git a/CIME/non_py/gitignore.template b/CIME/non_py/gitignore.template new file mode 100644 index 00000000000..02a3c097804 --- /dev/null +++ b/CIME/non_py/gitignore.template @@ -0,0 +1,77 @@ +# cime build and run directories +bld +run +# logger output files +*.log + +#python files +__pycache__/ +*.so + +# Swap +[._]*.s[a-v][a-z] +!*.svg # comment out if you don't need vector files +[._]*.sw[a-p] +[._]s[a-rt-v][a-z] +[._]ss[a-gi-z] +[._]sw[a-p] + +# Session +Session.vim +Sessionx.vim + +# Temporary +.netrwhist +*~ +# Auto-generated tag files +tags +# Persistent undo +[._]*.un~ +# -*- mode: gitignore; -*- +*~ +\#*\# +/.emacs.desktop +/.emacs.desktop.lock +*.elc +auto-save-list +tramp +.\#* + +# Org-mode +.org-id-locations +*_archive + +# flymake-mode +*_flymake.* + +# eshell files +/eshell/history +/eshell/lastdir + +# elpa packages +/elpa/ + +# reftex files +*.rel + +# AUCTeX auto folder +/auto/ + +# cask packages +.cask/ +dist/ + +# Flycheck +flycheck_*.el + +# server auth directory +/server/ + +# projectiles files +.projectile + +# directory configuration +.dir-locals.el + +# network security +/network-security.data diff --git a/CIME/status.py b/CIME/status.py new file mode 100644 index 00000000000..aa1832265b5 --- /dev/null +++ b/CIME/status.py @@ -0,0 +1,96 @@ +# These routines were moved from utils.py to avoid circular dependancies +import time, os, sys, logging +import CIME.utils + +logger = logging.getLogger(__name__) + +def append_status(msg, sfile, caseroot=".", gitinterface=None): + """ + Append msg to sfile in caseroot + """ + ctime = time.strftime("%Y-%m-%d %H:%M:%S: ") + + # Reduce empty lines in CaseStatus. It's a very concise file + # and does not need extra newlines for readability + line_ending = "\n" + + with open(os.path.join(caseroot, sfile), "a") as fd: + fd.write(ctime + msg + line_ending) + fd.write(" ---------------------------------------------------" + line_ending) + + +def append_testlog(msg, caseroot=".", gitinterface=None): + """ + Add to TestStatus.log file + """ + append_status(msg, "TestStatus.log", caseroot, gitinterface=gitinterface) + + +def append_case_status(phase, status, msg=None, caseroot=".", gitinterface=None): + """ + Update CaseStatus file + """ + msg = msg if msg else "no message provided" + append_status( + "{} {}{}".format(phase, status, msg), + "CaseStatus", + caseroot, + ) + if gitinterface: + filelist = gitinterface.git_operation("ls-files", "--others", "--modified", "--deleted", "--exclude-standard").splitlines() + for f in filelist: + logger.info("adding file {}".format(f)) + gitinterface.git_operation("add",f) + gitinterface.git_operation("commit","-m","\""+msg+"\"") + +def run_and_log_case_status( + func, + phase, + caseroot=".", + custom_starting_msg_functor=None, + custom_success_msg_functor=None, + is_batch=False, + gitinterface = None, +): + starting_msg = None + + if custom_starting_msg_functor is not None: + starting_msg = custom_starting_msg_functor() + + # Delay appending "starting" on "case.subsmit" phase when batch system is + # present since we don't have the jobid yet + if phase != "case.submit" or not is_batch: + append_case_status(phase, "starting", msg=starting_msg, caseroot=caseroot, gitinterface=gitinterface) + rv = None + try: + rv = func() + except BaseException: + custom_success_msg = ( + custom_success_msg_functor(rv) + if custom_success_msg_functor and rv is not None + else None + ) + if phase == "case.submit" and is_batch: + append_case_status( + phase, "starting", msg=custom_success_msg, caseroot=caseroot, gitinterface=gitinterface + ) + e = sys.exc_info()[1] + append_case_status( + phase, CIME.utils.CASE_FAILURE, msg=("\n{}".format(e)), caseroot=caseroot, gitinterface=gitinterface + ) + raise + else: + custom_success_msg = ( + custom_success_msg_functor(rv) if custom_success_msg_functor else None + ) + if phase == "case.submit" and is_batch: + append_case_status( + phase, "starting", msg=custom_success_msg, caseroot=caseroot, gitinterface=gitinterface + ) + append_case_status( + phase, CIME.utils.CASE_SUCCESS, msg=custom_success_msg, caseroot=caseroot, gitinterface=gitinterface + ) + + return rv + + diff --git a/CIME/utils.py b/CIME/utils.py index 864f0299b27..b2b82b74528 100644 --- a/CIME/utils.py +++ b/CIME/utils.py @@ -22,7 +22,8 @@ # Fix to pass user defined `srcroot` to `CIME.XML.generic_xml.GenericXML` # where it's used to resolve $SRCROOT in XML config files. GLOBAL = {} - +CASE_SUCCESS = "success" +CASE_FAILURE = "error" def deprecate_action(message): class ActionStoreDeprecated(Action): @@ -2056,39 +2057,6 @@ def format_time(time_format, input_format, input_time): return output_time -def append_status(msg, sfile, caseroot="."): - """ - Append msg to sfile in caseroot - """ - ctime = time.strftime("%Y-%m-%d %H:%M:%S: ") - - # Reduce empty lines in CaseStatus. It's a very concise file - # and does not need extra newlines for readability - line_ending = "\n" - - with open(os.path.join(caseroot, sfile), "a") as fd: - fd.write(ctime + msg + line_ending) - fd.write(" ---------------------------------------------------" + line_ending) - - -def append_testlog(msg, caseroot="."): - """ - Add to TestStatus.log file - """ - append_status(msg, "TestStatus.log", caseroot) - - -def append_case_status(phase, status, msg=None, caseroot="."): - """ - Update CaseStatus file - """ - append_status( - "{} {}{}".format(phase, status, " {}".format(msg if msg else "")), - "CaseStatus", - caseroot, - ) - - def does_file_have_string(filepath, text): """ Does the text string appear in the filepath file @@ -2457,60 +2425,6 @@ def verbatim_success_msg(return_val): return return_val -CASE_SUCCESS = "success" -CASE_FAILURE = "error" - - -def run_and_log_case_status( - func, - phase, - caseroot=".", - custom_starting_msg_functor=None, - custom_success_msg_functor=None, - is_batch=False, -): - starting_msg = None - - if custom_starting_msg_functor is not None: - starting_msg = custom_starting_msg_functor() - - # Delay appending "starting" on "case.subsmit" phase when batch system is - # present since we don't have the jobid yet - if phase != "case.submit" or not is_batch: - append_case_status(phase, "starting", msg=starting_msg, caseroot=caseroot) - rv = None - try: - rv = func() - except BaseException: - custom_success_msg = ( - custom_success_msg_functor(rv) - if custom_success_msg_functor and rv is not None - else None - ) - if phase == "case.submit" and is_batch: - append_case_status( - phase, "starting", msg=custom_success_msg, caseroot=caseroot - ) - e = sys.exc_info()[1] - append_case_status( - phase, CASE_FAILURE, msg=("\n{}".format(e)), caseroot=caseroot - ) - raise - else: - custom_success_msg = ( - custom_success_msg_functor(rv) if custom_success_msg_functor else None - ) - if phase == "case.submit" and is_batch: - append_case_status( - phase, "starting", msg=custom_success_msg, caseroot=caseroot - ) - append_case_status( - phase, CASE_SUCCESS, msg=custom_success_msg, caseroot=caseroot - ) - - return rv - - def _check_for_invalid_args(args): # Prevent circular import from CIME.config import Config From ab162a380c3a5184af9b22f1b9e78a72792db502 Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Tue, 6 Aug 2024 13:29:54 -0600 Subject: [PATCH 02/16] add remote repository support --- CIME/Tools/xmlchange | 30 +++++++++++++++++++++++++++++- CIME/XML/env_batch.py | 2 +- CIME/case/case_submit.py | 1 + CIME/status.py | 15 ++++++++++----- 4 files changed, 41 insertions(+), 7 deletions(-) diff --git a/CIME/Tools/xmlchange b/CIME/Tools/xmlchange index 9e6927636d3..24d50c8ddc8 100755 --- a/CIME/Tools/xmlchange +++ b/CIME/Tools/xmlchange @@ -52,6 +52,7 @@ from CIME.utils import ( expect, convert_to_type, get_batch_script_for_job, + Timeout, ) from CIME.status import append_case_status from CIME.case import Case @@ -241,7 +242,34 @@ def xmlchange_single_value( if xmlid == "JOB_QUEUE": case.set_value("USER_REQUESTED_QUEUE", xmlval, subgroup) - +# if CASE_GIT_REPOSITORY is updated set the remote to the new repo +# note that if origin exists and is different than CASE_GIT_REPOSITORY it +# will be deleted and replaced + if xmlid == "CASE_GIT_REPOSITORY": + if not case._gitinterface: + logger.warning("You must run case.setup before you can set CASE_GIT_REPOSITORY") + return + remote = case._gitinterface.git_operation("remote") + if remote: + if remote == xmlval: + return + else: + case._gitinterface.git_operation("remote","rm",remote) + branch = case._gitinterface.git_operation("branch").split()[1] + remote_branches = "" + with Timeout(30): + remote_branches = case._gitinterface.git_operation("ls-remote","--heads",xmlval) + chk = remote_branches.find("refs/heads/"+branch) + + if chk >= 0: + expect(False," Could not set {} to {}, remote branch {} already exists, Rename case and try again.".format(xmlid,xmlval,branch)) + with Timeout(30): + case._gitinterface.git_operation("remote", "add", "origin", xmlval) + case._gitinterface.git_operation("push","--set-upstream","origin", branch) + + + + def xmlchange( caseroot, listofsettings, diff --git a/CIME/XML/env_batch.py b/CIME/XML/env_batch.py index 2c705555fea..0fa71d7f8db 100644 --- a/CIME/XML/env_batch.py +++ b/CIME/XML/env_batch.py @@ -1385,7 +1385,7 @@ def make_all_batch_files(self, case): template = case.get_resolved_value( env_workflow.get_value("template", subgroup=job) ) - + print(f"template {template} job {job}") if os.path.isabs(template): input_batch_script = template else: diff --git a/CIME/case/case_submit.py b/CIME/case/case_submit.py index 22914fad588..7e0e09881e4 100644 --- a/CIME/case/case_submit.py +++ b/CIME/case/case_submit.py @@ -284,6 +284,7 @@ def submit( caseroot=caseroot, custom_success_msg_functor=lambda x: x.split(":")[-1], is_batch=is_batch, + gitinterface=self._gitinterface, ) except BaseException: # Want to catch KeyboardInterrupt too # If something failed in the batch system, make sure to mark diff --git a/CIME/status.py b/CIME/status.py index aa1832265b5..4b66300d9a5 100644 --- a/CIME/status.py +++ b/CIME/status.py @@ -1,6 +1,6 @@ # These routines were moved from utils.py to avoid circular dependancies import time, os, sys, logging -import CIME.utils +from CIME.utils import Timeout, CASE_SUCCESS, CASE_FAILURE logger = logging.getLogger(__name__) @@ -30,7 +30,7 @@ def append_case_status(phase, status, msg=None, caseroot=".", gitinterface=None) """ Update CaseStatus file """ - msg = msg if msg else "no message provided" + msg = msg if msg else "" append_status( "{} {}{}".format(phase, status, msg), "CaseStatus", @@ -38,10 +38,15 @@ def append_case_status(phase, status, msg=None, caseroot=".", gitinterface=None) ) if gitinterface: filelist = gitinterface.git_operation("ls-files", "--others", "--modified", "--deleted", "--exclude-standard").splitlines() + # Files that should not be added should have been excluded by the .gitignore file for f in filelist: - logger.info("adding file {}".format(f)) + logger.debug("adding file {}".format(f)) gitinterface.git_operation("add",f) gitinterface.git_operation("commit","-m","\""+msg+"\"") + remote = gitinterface.git_operation("remote") + if remote: + with Timeout(30): + gitinterface.git_operation("push",remote) def run_and_log_case_status( func, @@ -76,7 +81,7 @@ def run_and_log_case_status( ) e = sys.exc_info()[1] append_case_status( - phase, CIME.utils.CASE_FAILURE, msg=("\n{}".format(e)), caseroot=caseroot, gitinterface=gitinterface + phase, CASE_FAILURE, msg=("\n{}".format(e)), caseroot=caseroot, gitinterface=gitinterface ) raise else: @@ -88,7 +93,7 @@ def run_and_log_case_status( phase, "starting", msg=custom_success_msg, caseroot=caseroot, gitinterface=gitinterface ) append_case_status( - phase, CIME.utils.CASE_SUCCESS, msg=custom_success_msg, caseroot=caseroot, gitinterface=gitinterface + phase, CASE_SUCCESS, msg=custom_success_msg, caseroot=caseroot, gitinterface=gitinterface ) return rv From faadc4e01439f5a537d9e8a17104df0555218d34 Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Tue, 6 Aug 2024 13:34:49 -0600 Subject: [PATCH 03/16] cleanup whitespace --- CIME/Tools/xmlchange | 3 --- CIME/XML/env_batch.py | 1 - 2 files changed, 4 deletions(-) diff --git a/CIME/Tools/xmlchange b/CIME/Tools/xmlchange index 24d50c8ddc8..fa7c50e60fc 100755 --- a/CIME/Tools/xmlchange +++ b/CIME/Tools/xmlchange @@ -266,9 +266,6 @@ def xmlchange_single_value( with Timeout(30): case._gitinterface.git_operation("remote", "add", "origin", xmlval) case._gitinterface.git_operation("push","--set-upstream","origin", branch) - - - def xmlchange( caseroot, diff --git a/CIME/XML/env_batch.py b/CIME/XML/env_batch.py index 0fa71d7f8db..f158f29dd07 100644 --- a/CIME/XML/env_batch.py +++ b/CIME/XML/env_batch.py @@ -1385,7 +1385,6 @@ def make_all_batch_files(self, case): template = case.get_resolved_value( env_workflow.get_value("template", subgroup=job) ) - print(f"template {template} job {job}") if os.path.isabs(template): input_batch_script = template else: From cf698d6904b871f8d981bf786f43b73cbc8b4907 Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Wed, 7 Aug 2024 07:38:29 -0600 Subject: [PATCH 04/16] working on izumi --- CIME/build.py | 2 +- CIME/case/case_setup.py | 5 +++-- CIME/gitinterface.py | 14 +++++++------- CIME/status.py | 1 + 4 files changed, 12 insertions(+), 10 deletions(-) diff --git a/CIME/build.py b/CIME/build.py index 4440c9294ec..8a8b2e6ca81 100644 --- a/CIME/build.py +++ b/CIME/build.py @@ -1326,5 +1326,5 @@ def clean(case, cleanlist=None, clean_all=False, clean_depends=None): ############################################################################### functor = lambda: _clean_impl(case, cleanlist, clean_all, clean_depends) return run_and_log_case_status( - functor, "build.clean", caseroot=case.get_value("CASEROOT") + functor, "build.clean", caseroot=case.get_value("CASEROOT"), gitinterface=case._gitinterface ) diff --git a/CIME/case/case_setup.py b/CIME/case/case_setup.py index d7497f89b89..e4958296e3d 100644 --- a/CIME/case/case_setup.py +++ b/CIME/case/case_setup.py @@ -517,13 +517,14 @@ def case_setup(self, clean=False, test_mode=False, reset=False, keep=None): custom_success_msg_functor=msg_func, caseroot=caseroot, is_batch=is_batch, + ) - if not os.path.exists(os.path.join(caseroot, ".git")): self._create_case_repo(caseroot) def _create_case_repo(self, caseroot): self._gitinterface = GitInterface(caseroot,logger,branch=self.get_value("CASE")) - safe_copy(os.path.join(self.get_value("CIMEROOT"), "CIME","non_py","gitignore.template"), os.path.join(caseroot,".gitignore")) + if not os.path.exists(os.path.join(caseroot, ".gitignore")): + safe_copy(os.path.join(self.get_value("CIMEROOT"), "CIME","non_py","gitignore.template"), os.path.join(caseroot,".gitignore")) # add all files in caseroot to local repository self._gitinterface._git_command("add","*") append_case_status("", "", "local git repository created", gitinterface=self._gitinterface) diff --git a/CIME/gitinterface.py b/CIME/gitinterface.py index 455f6e6c68f..ec47c8b69b6 100644 --- a/CIME/gitinterface.py +++ b/CIME/gitinterface.py @@ -38,7 +38,7 @@ def _git_command(self, operation, *args): except Exception as e: sys.exit(e) else: - return ["git", "-C", str(self.repo_path), operation] + list(args) + return ["git","-C", str(self.repo_path), operation] + list(args) def _init_git_repo(self, branch=None): if self._use_module: @@ -46,17 +46,17 @@ def _init_git_repo(self, branch=None): if branch: self.git_operation("checkout", "-b",branch) else: - command = ("git", "-C", str(self.repo_path), "init") + command = ["git", "-C", str(self.repo_path), "init"] if branch: - command.extend("-b", branch) - utils.execute_subprocess(command) + command.extend(["-b", branch]) + utils.run_cmd_no_fail(" ".join(command)) # pylint: disable=unused-argument def git_operation(self, operation, *args, **kwargs): command = self._git_command(operation, *args) if isinstance(command, list): try: - return utils.execute_subprocess(command, output_to_caller=True) + return utils.run_cmd_no_fail(" ".join(command)) except Exception as e: sys.exit(e) else: @@ -72,7 +72,7 @@ def config_get_value(self, section, name): return val else: cmd = ("git", "-C", str(self.repo_path), "config", "--get", f"{section}.{name}") - output = utils.execute_subprocess(cmd, output_to_caller=True) + output = utils.run_cmd_no_fail(cmd) return output.strip() def config_set_value(self, section, name, value): @@ -83,4 +83,4 @@ def config_set_value(self, section, name, value): else: cmd = ("git", "-C", str(self.repo_path), "config", f"{section}.{name}", value) self.logger.debug(cmd) - utils.execute_subprocess(cmd, output_to_caller=True) + utils.run_cmd_no_fail(cmd) diff --git a/CIME/status.py b/CIME/status.py index 4b66300d9a5..29fa5165620 100644 --- a/CIME/status.py +++ b/CIME/status.py @@ -42,6 +42,7 @@ def append_case_status(phase, status, msg=None, caseroot=".", gitinterface=None) for f in filelist: logger.debug("adding file {}".format(f)) gitinterface.git_operation("add",f) + msg = msg if msg else " no message provided" gitinterface.git_operation("commit","-m","\""+msg+"\"") remote = gitinterface.git_operation("remote") if remote: From 8738ae87bc218d38ccf3a0c68f2cd491584e0cf5 Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Wed, 7 Aug 2024 09:00:58 -0500 Subject: [PATCH 05/16] port to tacc/vista --- CIME/XML/env_batch.py | 6 +++--- CIME/status.py | 7 ++++++- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/CIME/XML/env_batch.py b/CIME/XML/env_batch.py index f158f29dd07..03bbb2ca2f9 100644 --- a/CIME/XML/env_batch.py +++ b/CIME/XML/env_batch.py @@ -1080,19 +1080,19 @@ def _submit_single_job( submitargs, run_args, batchredirect, - get_batch_script_for_job(job), + os.path.join(self._caseroot,get_batch_script_for_job(job)), ) else: sequence = ( batchsubmit, submitargs, batchredirect, - get_batch_script_for_job(job), + os.path.join(self._caseroot,get_batch_script_for_job(job)), run_args, ) submitcmd = " ".join(s.strip() for s in sequence if s is not None) - if submitcmd.startswith("ssh"): + if submitcmd.startswith("ssh") and "$CASEROOT" in submitcmd: # add ` before cd $CASEROOT and at end of command submitcmd = submitcmd.replace("cd $CASEROOT", "'cd $CASEROOT") + "'" diff --git a/CIME/status.py b/CIME/status.py index 29fa5165620..010e5f6b60d 100644 --- a/CIME/status.py +++ b/CIME/status.py @@ -37,7 +37,12 @@ def append_case_status(phase, status, msg=None, caseroot=".", gitinterface=None) caseroot, ) if gitinterface: - filelist = gitinterface.git_operation("ls-files", "--others", "--modified", "--deleted", "--exclude-standard").splitlines() + filelist = gitinterface.git_operation("ls-files", "--deleted", "--exclude-standard").splitlines() + # First delete files that have been removed + for f in filelist: + logger.debug("removing file {}".format(f)) + gitinterface.git_operation("rm",f) + filelist = gitinterface.git_operation("ls-files", "--others", "--modified", "--exclude-standard").splitlines() # Files that should not be added should have been excluded by the .gitignore file for f in filelist: logger.debug("adding file {}".format(f)) From 12dcf56b28baaa9f47a2c5a5e186e202a5a96e6b Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Wed, 7 Aug 2024 08:26:32 -0600 Subject: [PATCH 06/16] update location of status files --- CIME/test_scheduler.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/CIME/test_scheduler.py b/CIME/test_scheduler.py index b8b12ae08d2..e1dc99400f4 100644 --- a/CIME/test_scheduler.py +++ b/CIME/test_scheduler.py @@ -14,9 +14,8 @@ from CIME.XML.standard_module_setup import * from CIME.get_tests import get_recommended_test_time, get_build_groups, is_perf_test +from CIME.status import append_status, append_testlog from CIME.utils import ( - append_status, - append_testlog, TESTS_FAILED_ERR_CODE, parse_test_name, get_full_test_name, From 1a8f2c05b2536654884ad0e4dfce869fb0709b8c Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Wed, 7 Aug 2024 09:04:42 -0600 Subject: [PATCH 07/16] black reformat --- CIME/Tools/xmlchange | 41 ++++++++++++++++++++++++++++------------- 1 file changed, 28 insertions(+), 13 deletions(-) diff --git a/CIME/Tools/xmlchange b/CIME/Tools/xmlchange index fa7c50e60fc..a5924389f10 100755 --- a/CIME/Tools/xmlchange +++ b/CIME/Tools/xmlchange @@ -63,6 +63,7 @@ import re # Set logger logger = logging.getLogger("xmlchange") + ############################################################################### def parse_command_line(args, description): ############################################################################### @@ -242,31 +243,41 @@ def xmlchange_single_value( if xmlid == "JOB_QUEUE": case.set_value("USER_REQUESTED_QUEUE", xmlval, subgroup) -# if CASE_GIT_REPOSITORY is updated set the remote to the new repo -# note that if origin exists and is different than CASE_GIT_REPOSITORY it -# will be deleted and replaced + # if CASE_GIT_REPOSITORY is updated set the remote to the new repo + # note that if origin exists and is different than CASE_GIT_REPOSITORY it + # will be deleted and replaced if xmlid == "CASE_GIT_REPOSITORY": if not case._gitinterface: - logger.warning("You must run case.setup before you can set CASE_GIT_REPOSITORY") + logger.warning( + "You must run case.setup before you can set CASE_GIT_REPOSITORY" + ) return remote = case._gitinterface.git_operation("remote") if remote: if remote == xmlval: return else: - case._gitinterface.git_operation("remote","rm",remote) + case._gitinterface.git_operation("remote", "rm", remote) branch = case._gitinterface.git_operation("branch").split()[1] remote_branches = "" with Timeout(30): - remote_branches = case._gitinterface.git_operation("ls-remote","--heads",xmlval) - chk = remote_branches.find("refs/heads/"+branch) + remote_branches = case._gitinterface.git_operation( + "ls-remote", "--heads", xmlval + ) + chk = remote_branches.find("refs/heads/" + branch) if chk >= 0: - expect(False," Could not set {} to {}, remote branch {} already exists, Rename case and try again.".format(xmlid,xmlval,branch)) + expect( + False, + " Could not set {} to {}, remote branch {} already exists, Rename case and try again.".format( + xmlid, xmlval, branch + ), + ) with Timeout(30): case._gitinterface.git_operation("remote", "add", "origin", xmlval) - case._gitinterface.git_operation("push","--set-upstream","origin", branch) - + case._gitinterface.git_operation("push", "--set-upstream", "origin", branch) + + def xmlchange( caseroot, listofsettings, @@ -280,7 +291,6 @@ def xmlchange( dryrun, non_local, ): - with Case(caseroot, read_only=False, record=True, non_local=non_local) as case: comp_classes = case.get_values("COMP_CLASSES") env_test = None @@ -300,7 +310,6 @@ def xmlchange( # Change values for setting in listofsettings: - pair = setting.split("=", 1) expect( len(pair) == 2, @@ -346,7 +355,13 @@ def xmlchange( for arg in sys.argv: argstr += "%s " % arg msg = " %s " % (argstr) - append_case_status("xmlchange", "success", msg=msg, caseroot=caseroot, gitinterface=case._gitinterface) + append_case_status( + "xmlchange", + "success", + msg=msg, + caseroot=caseroot, + gitinterface=case._gitinterface, + ) def _main_func(description): From 6752b161400be99307c792252a2f49941f763a05 Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Wed, 7 Aug 2024 09:16:17 -0600 Subject: [PATCH 08/16] black reformat --- CIME/XML/env_batch.py | 5 +- CIME/build.py | 9 +++- CIME/case/case.py | 12 ++--- CIME/case/case_cmpgen_namelists.py | 9 ++-- CIME/case/case_run.py | 3 +- CIME/case/case_setup.py | 19 +++++--- CIME/case/case_st_archive.py | 8 +-- CIME/gitinterface.py | 34 +++++++++---- CIME/status.py | 78 ++++++++++++++++++++---------- CIME/utils.py | 3 +- 10 files changed, 116 insertions(+), 64 deletions(-) diff --git a/CIME/XML/env_batch.py b/CIME/XML/env_batch.py index 03bbb2ca2f9..b444a29333a 100644 --- a/CIME/XML/env_batch.py +++ b/CIME/XML/env_batch.py @@ -919,7 +919,6 @@ def _submit_single_job( resubmit_immediate=False, workflow=True, ): - if not dry_run: logger.warning("Submit job {}".format(job)) batch_system = self.get_value("BATCH_SYSTEM", subgroup=None) @@ -1080,14 +1079,14 @@ def _submit_single_job( submitargs, run_args, batchredirect, - os.path.join(self._caseroot,get_batch_script_for_job(job)), + os.path.join(self._caseroot, get_batch_script_for_job(job)), ) else: sequence = ( batchsubmit, submitargs, batchredirect, - os.path.join(self._caseroot,get_batch_script_for_job(job)), + os.path.join(self._caseroot, get_batch_script_for_job(job)), run_args, ) diff --git a/CIME/build.py b/CIME/build.py index 8a8b2e6ca81..2d6c6414ffa 100644 --- a/CIME/build.py +++ b/CIME/build.py @@ -1318,7 +1318,9 @@ def case_build( cb = cb + " (SHAREDLIB_BUILD)" if model_only == True: cb = cb + " (MODEL_BUILD)" - return run_and_log_case_status(functor, cb, caseroot=caseroot, gitinterface=case._gitinterface) + return run_and_log_case_status( + functor, cb, caseroot=caseroot, gitinterface=case._gitinterface + ) ############################################################################### @@ -1326,5 +1328,8 @@ def clean(case, cleanlist=None, clean_all=False, clean_depends=None): ############################################################################### functor = lambda: _clean_impl(case, cleanlist, clean_all, clean_depends) return run_and_log_case_status( - functor, "build.clean", caseroot=case.get_value("CASEROOT"), gitinterface=case._gitinterface + functor, + "build.clean", + caseroot=case.get_value("CASEROOT"), + gitinterface=case._gitinterface, ) diff --git a/CIME/case/case.py b/CIME/case/case.py index c0c4f074cd8..61f48fcea53 100644 --- a/CIME/case/case.py +++ b/CIME/case/case.py @@ -100,7 +100,6 @@ class Case(object): ) def __init__(self, case_root=None, read_only=True, record=False, non_local=False): - if case_root is None: case_root = os.getcwd() expect( @@ -201,12 +200,13 @@ def __init__(self, case_root=None, read_only=True, record=False, non_local=False mach == probed_machine, f"Current machine {probed_machine} does not match case machine {mach}.", ) - if os.path.exists(os.path.join(self.get_value("CASEROOT"),".git")): - self._gitinterface = GitInterface(self.get_value("CASEROOT"),logger) - + if os.path.exists(os.path.join(self.get_value("CASEROOT"), ".git")): + self._gitinterface = GitInterface( + self.get_value("CASEROOT"), logger + ) self.initialize_derived_attributes() - + def get_baseline_dir(self): baseline_root = self.get_value("BASELINE_ROOT") @@ -687,7 +687,6 @@ def _set_compset(self, compset_name, files): # Loop through all of the files listed in COMPSETS_SPEC_FILE and find the file # that has a match for either the alias or the longname in that order for component in components: - # Determine the compsets file for this component compsets_filename = files.get_value( "COMPSETS_SPEC_FILE", {"component": component} @@ -1289,7 +1288,6 @@ def configure( gpu_type=None, gpu_offload=None, ): - expect( check_name(compset_name, additional_chars="."), "Invalid compset name {}".format(compset_name), diff --git a/CIME/case/case_cmpgen_namelists.py b/CIME/case/case_cmpgen_namelists.py index 5407f1755e5..10ded368147 100644 --- a/CIME/case/case_cmpgen_namelists.py +++ b/CIME/case/case_cmpgen_namelists.py @@ -12,7 +12,6 @@ from CIME.test_status import * import os, shutil, traceback, stat, glob -from distutils import dir_util logger = logging.getLogger(__name__) @@ -94,7 +93,7 @@ def _do_full_nl_gen_impl(case, test, generate_name, baseline_root=None): if os.path.isdir(baseline_casedocs): shutil.rmtree(baseline_casedocs) - dir_util.copy_tree(casedoc_dir, baseline_casedocs, preserve_mode=False) + shutil.copytree(casedoc_dir, baseline_casedocs) for item in glob.glob(os.path.join(test_dir, "user_nl*")): preexisting_baseline = os.path.join(baseline_dir, os.path.basename(item)) @@ -187,7 +186,11 @@ def case_cmpgen_namelists( NAMELIST_PHASE, TEST_PASS_STATUS if success else TEST_FAIL_STATUS ) try: - append_status(output, logfile_name, caseroot=caseroot, gitinterface=self._gitinterface) + append_status( + output, + logfile_name, + caseroot=caseroot, + ) except IOError: pass diff --git a/CIME/case/case_run.py b/CIME/case/case_run.py index 8550d35fb25..2c21bd4b076 100644 --- a/CIME/case/case_run.py +++ b/CIME/case/case_run.py @@ -16,6 +16,7 @@ logger = logging.getLogger(__name__) + ############################################################################### def _pre_run_check(case, lid, skip_pnl=False, da_cycle=0): ############################################################################### @@ -181,7 +182,7 @@ def _run_model_impl(case, lid, skip_pnl=False, da_cycle=0): custom_success_msg_functor=msg_func, caseroot=case.get_value("CASEROOT"), is_batch=is_batch, - gitinterface=case._gitinterface + gitinterface=case._gitinterface, ) cmd_success = True except CIMEError: diff --git a/CIME/case/case_setup.py b/CIME/case/case_setup.py index e4958296e3d..28cfbb5ab73 100644 --- a/CIME/case/case_setup.py +++ b/CIME/case/case_setup.py @@ -488,7 +488,7 @@ def case_setup(self, clean=False, test_mode=False, reset=False, keep=None): if is_batch: jobid = batch_jobid() msg_func = lambda *args: jobid if jobid is not None else "" - + if self.get_value("TEST") and not test_mode: test_name = casebaseid if casebaseid is not None else self.get_value("CASE") with TestStatus(test_dir=caseroot, test_name=test_name) as ts: @@ -517,14 +517,21 @@ def case_setup(self, clean=False, test_mode=False, reset=False, keep=None): custom_success_msg_functor=msg_func, caseroot=caseroot, is_batch=is_batch, - ) self._create_case_repo(caseroot) + def _create_case_repo(self, caseroot): - self._gitinterface = GitInterface(caseroot,logger,branch=self.get_value("CASE")) + self._gitinterface = GitInterface(caseroot, logger, branch=self.get_value("CASE")) if not os.path.exists(os.path.join(caseroot, ".gitignore")): - safe_copy(os.path.join(self.get_value("CIMEROOT"), "CIME","non_py","gitignore.template"), os.path.join(caseroot,".gitignore")) + safe_copy( + os.path.join( + self.get_value("CIMEROOT"), "CIME", "non_py", "gitignore.template" + ), + os.path.join(caseroot, ".gitignore"), + ) # add all files in caseroot to local repository - self._gitinterface._git_command("add","*") - append_case_status("", "", "local git repository created", gitinterface=self._gitinterface) + self._gitinterface._git_command("add", "*") + append_case_status( + "", "", "local git repository created", gitinterface=self._gitinterface + ) diff --git a/CIME/case/case_st_archive.py b/CIME/case/case_st_archive.py index b330db5d88e..1135741228b 100644 --- a/CIME/case/case_st_archive.py +++ b/CIME/case/case_st_archive.py @@ -22,6 +22,7 @@ logger = logging.getLogger(__name__) + ############################################################################### def _get_archive_fn_desc(archive_fn): ############################################################################### @@ -179,7 +180,6 @@ def _archive_rpointer_files( # Generate rpointer file(s) for interim restarts for the one datename and each # possible value of ninst_strings if save_interim_restart_files: - # parse env_archive.xml to determine the rpointer files # and contents for the given archive_entry tag rpointer_items = archive.get_rpointer_contents(archive_entry) @@ -445,7 +445,7 @@ def _archive_restarts_date( histfiles_savein_rundir_by_compname = {} - for (archive_entry, compname, compclass) in _get_component_archive_entries( + for archive_entry, compname, compclass in _get_component_archive_entries( components, archive ): if compclass: @@ -879,7 +879,7 @@ def _archive_process( # archive history files - for (_, compname, compclass) in _get_component_archive_entries(components, archive): + for _, compname, compclass in _get_component_archive_entries(components, archive): if compclass: logger.info( "Archiving history files for {} ({})".format(compname, compclass) @@ -1050,7 +1050,7 @@ def case_st_archive( custom_success_msg_functor=msg_func, caseroot=caseroot, is_batch=is_batch, - gitinterface=self._gitinterface + gitinterface=self._gitinterface, ) logger.info("st_archive completed") diff --git a/CIME/gitinterface.py b/CIME/gitinterface.py index ec47c8b69b6..cfb0c42fcb4 100644 --- a/CIME/gitinterface.py +++ b/CIME/gitinterface.py @@ -1,8 +1,8 @@ -import os import sys -from . import utils +from CIME.utils import run_cmd_no_fail from pathlib import Path + class GitInterface: def __init__(self, repo_path, logger, branch=None): logger.debug("Initialize GitInterface for {}".format(repo_path)) @@ -38,25 +38,25 @@ def _git_command(self, operation, *args): except Exception as e: sys.exit(e) else: - return ["git","-C", str(self.repo_path), operation] + list(args) + return ["git", "-C", str(self.repo_path), operation] + list(args) def _init_git_repo(self, branch=None): if self._use_module: self.repo = self.git.Repo.init(str(self.repo_path)) if branch: - self.git_operation("checkout", "-b",branch) + self.git_operation("checkout", "-b", branch) else: command = ["git", "-C", str(self.repo_path), "init"] if branch: command.extend(["-b", branch]) - utils.run_cmd_no_fail(" ".join(command)) + run_cmd_no_fail(" ".join(command)) # pylint: disable=unused-argument def git_operation(self, operation, *args, **kwargs): command = self._git_command(operation, *args) if isinstance(command, list): try: - return utils.run_cmd_no_fail(" ".join(command)) + return run_cmd_no_fail(" ".join(command)) except Exception as e: sys.exit(e) else: @@ -71,8 +71,15 @@ def config_get_value(self, section, name): val = None return val else: - cmd = ("git", "-C", str(self.repo_path), "config", "--get", f"{section}.{name}") - output = utils.run_cmd_no_fail(cmd) + cmd = ( + "git", + "-C", + str(self.repo_path), + "config", + "--get", + f"{section}.{name}", + ) + output = run_cmd_no_fail(cmd) return output.strip() def config_set_value(self, section, name, value): @@ -81,6 +88,13 @@ def config_set_value(self, section, name, value): writer.set_value(section, name, value) writer.release() # Ensure changes are saved else: - cmd = ("git", "-C", str(self.repo_path), "config", f"{section}.{name}", value) + cmd = ( + "git", + "-C", + str(self.repo_path), + "config", + f"{section}.{name}", + value, + ) self.logger.debug(cmd) - utils.run_cmd_no_fail(cmd) + run_cmd_no_fail(cmd) diff --git a/CIME/status.py b/CIME/status.py index 010e5f6b60d..3ce01a9cb3c 100644 --- a/CIME/status.py +++ b/CIME/status.py @@ -4,7 +4,8 @@ logger = logging.getLogger(__name__) -def append_status(msg, sfile, caseroot=".", gitinterface=None): + +def append_status(msg, sfile, caseroot="."): """ Append msg to sfile in caseroot """ @@ -13,17 +14,17 @@ def append_status(msg, sfile, caseroot=".", gitinterface=None): # Reduce empty lines in CaseStatus. It's a very concise file # and does not need extra newlines for readability line_ending = "\n" - + with open(os.path.join(caseroot, sfile), "a") as fd: fd.write(ctime + msg + line_ending) fd.write(" ---------------------------------------------------" + line_ending) - -def append_testlog(msg, caseroot=".", gitinterface=None): + +def append_testlog(msg, caseroot="."): """ Add to TestStatus.log file """ - append_status(msg, "TestStatus.log", caseroot, gitinterface=gitinterface) + append_status(msg, "TestStatus.log", caseroot) def append_case_status(phase, status, msg=None, caseroot=".", gitinterface=None): @@ -37,31 +38,36 @@ def append_case_status(phase, status, msg=None, caseroot=".", gitinterface=None) caseroot, ) if gitinterface: - filelist = gitinterface.git_operation("ls-files", "--deleted", "--exclude-standard").splitlines() + filelist = gitinterface.git_operation( + "ls-files", "--deleted", "--exclude-standard" + ).splitlines() # First delete files that have been removed for f in filelist: logger.debug("removing file {}".format(f)) - gitinterface.git_operation("rm",f) - filelist = gitinterface.git_operation("ls-files", "--others", "--modified", "--exclude-standard").splitlines() + gitinterface.git_operation("rm", f) + filelist = gitinterface.git_operation( + "ls-files", "--others", "--modified", "--exclude-standard" + ).splitlines() # Files that should not be added should have been excluded by the .gitignore file for f in filelist: logger.debug("adding file {}".format(f)) - gitinterface.git_operation("add",f) + gitinterface.git_operation("add", f) msg = msg if msg else " no message provided" - gitinterface.git_operation("commit","-m","\""+msg+"\"") + gitinterface.git_operation("commit", "-m", '"' + msg + '"') remote = gitinterface.git_operation("remote") if remote: with Timeout(30): - gitinterface.git_operation("push",remote) - + gitinterface.git_operation("push", remote) + + def run_and_log_case_status( - func, - phase, - caseroot=".", - custom_starting_msg_functor=None, - custom_success_msg_functor=None, - is_batch=False, - gitinterface = None, + func, + phase, + caseroot=".", + custom_starting_msg_functor=None, + custom_success_msg_functor=None, + is_batch=False, + gitinterface=None, ): starting_msg = None @@ -71,7 +77,13 @@ def run_and_log_case_status( # Delay appending "starting" on "case.subsmit" phase when batch system is # present since we don't have the jobid yet if phase != "case.submit" or not is_batch: - append_case_status(phase, "starting", msg=starting_msg, caseroot=caseroot, gitinterface=gitinterface) + append_case_status( + phase, + "starting", + msg=starting_msg, + caseroot=caseroot, + gitinterface=gitinterface, + ) rv = None try: rv = func() @@ -83,11 +95,19 @@ def run_and_log_case_status( ) if phase == "case.submit" and is_batch: append_case_status( - phase, "starting", msg=custom_success_msg, caseroot=caseroot, gitinterface=gitinterface + phase, + "starting", + msg=custom_success_msg, + caseroot=caseroot, + gitinterface=gitinterface, ) e = sys.exc_info()[1] append_case_status( - phase, CASE_FAILURE, msg=("\n{}".format(e)), caseroot=caseroot, gitinterface=gitinterface + phase, + CASE_FAILURE, + msg=("\n{}".format(e)), + caseroot=caseroot, + gitinterface=gitinterface, ) raise else: @@ -96,12 +116,18 @@ def run_and_log_case_status( ) if phase == "case.submit" and is_batch: append_case_status( - phase, "starting", msg=custom_success_msg, caseroot=caseroot, gitinterface=gitinterface + phase, + "starting", + msg=custom_success_msg, + caseroot=caseroot, + gitinterface=gitinterface, ) append_case_status( - phase, CASE_SUCCESS, msg=custom_success_msg, caseroot=caseroot, gitinterface=gitinterface + phase, + CASE_SUCCESS, + msg=custom_success_msg, + caseroot=caseroot, + gitinterface=gitinterface, ) return rv - - diff --git a/CIME/utils.py b/CIME/utils.py index b2b82b74528..0973647823b 100644 --- a/CIME/utils.py +++ b/CIME/utils.py @@ -25,6 +25,7 @@ CASE_SUCCESS = "success" CASE_FAILURE = "error" + def deprecate_action(message): class ActionStoreDeprecated(Action): def __call__(self, parser, namespace, values, option_string=None): @@ -1749,7 +1750,6 @@ def convert_to_type(value, type_str, vid=""): vid is only for generating better error messages. """ if value is not None: - if type_str == "char": pass @@ -1795,7 +1795,6 @@ def convert_to_unknown_type(value): Convert value to it's real type by probing conversions. """ if value is not None: - # Attempt to convert to logical if value.upper() in ["TRUE", "FALSE"]: return value.upper() == "TRUE" From f63b4233c232a0d5a69eb074814ecd97534de840 Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Wed, 7 Aug 2024 09:29:52 -0600 Subject: [PATCH 09/16] fix reference to append_status --- CIME/compare_test_results.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CIME/compare_test_results.py b/CIME/compare_test_results.py index d9220551986..a46dbe50fe7 100644 --- a/CIME/compare_test_results.py +++ b/CIME/compare_test_results.py @@ -1,5 +1,6 @@ import CIME.compare_namelists, CIME.simple_compare -from CIME.utils import append_status, EnvironmentContext, parse_test_name +from CIME.status import append_status +from CIME.utils import EnvironmentContext, parse_test_name from CIME.test_status import * from CIME.hist_utils import compare_baseline, get_ts_synopsis from CIME.case import Case From 25e8170c465686fb06fc05184d7e2ac720e2a20d Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Wed, 7 Aug 2024 09:40:49 -0600 Subject: [PATCH 10/16] another import change --- CIME/tests/test_unit_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CIME/tests/test_unit_utils.py b/CIME/tests/test_unit_utils.py index 2994a87ffb1..8abde4d8fef 100755 --- a/CIME/tests/test_unit_utils.py +++ b/CIME/tests/test_unit_utils.py @@ -8,9 +8,9 @@ import unittest from unittest import mock +from CIME.status import run_and_log_case_status from CIME.utils import ( indent_string, - run_and_log_case_status, import_from_file, _line_defines_python_function, file_contains_python_function, From 9de721b2449483760558cfb5dd49cf6b474457c4 Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Wed, 7 Aug 2024 09:55:36 -0600 Subject: [PATCH 11/16] another distutil change --- CIME/SystemTests/hommebaseclass.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/CIME/SystemTests/hommebaseclass.py b/CIME/SystemTests/hommebaseclass.py index bad27d4aa56..8ea4a03c6b9 100644 --- a/CIME/SystemTests/hommebaseclass.py +++ b/CIME/SystemTests/hommebaseclass.py @@ -4,11 +4,11 @@ from CIME.XML.standard_module_setup import * from CIME.SystemTests.system_tests_common import SystemTestsCommon from CIME.build import post_build -from CIME.utils import append_testlog, SharedArea +from CIME.status import append_testlog +from CIME.utils import SharedArea from CIME.test_status import * import shutil -from distutils import dir_util logger = logging.getLogger(__name__) @@ -97,10 +97,9 @@ def run_phase(self): shutil.rmtree(full_baseline_dir) with SharedArea(): - dir_util.copy_tree( + shutil.copytree( os.path.join(exeroot, "tests", "baseline"), full_baseline_dir, - preserve_mode=False, ) elif compare: From 772ae8e928b7dec8dbed48192ee8214a1f7d6874 Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Wed, 7 Aug 2024 10:28:09 -0600 Subject: [PATCH 12/16] more import changes --- CIME/SystemTests/funit.py | 3 ++- CIME/SystemTests/mvk.py | 8 ++++---- CIME/SystemTests/pgn.py | 10 +++++----- CIME/SystemTests/tsc.py | 8 ++++---- 4 files changed, 15 insertions(+), 14 deletions(-) diff --git a/CIME/SystemTests/funit.py b/CIME/SystemTests/funit.py index 1ebaf720604..a7c21a06944 100644 --- a/CIME/SystemTests/funit.py +++ b/CIME/SystemTests/funit.py @@ -5,7 +5,8 @@ from CIME.XML.standard_module_setup import * from CIME.SystemTests.system_tests_common import SystemTestsCommon from CIME.build import post_build -from CIME.utils import append_testlog, get_cime_root +from CIME.status import append_testlog +from CIME.utils import get_cime_root from CIME.test_status import * logger = logging.getLogger(__name__) diff --git a/CIME/SystemTests/mvk.py b/CIME/SystemTests/mvk.py index 2ab2f72cd33..854dce74472 100644 --- a/CIME/SystemTests/mvk.py +++ b/CIME/SystemTests/mvk.py @@ -10,10 +10,11 @@ import json import logging -from distutils import dir_util +from shutils import copytree import CIME.test_status import CIME.utils +from CIME.status import append_testlog from CIME.SystemTests.system_tests_common import SystemTestsCommon from CIME.case.case_setup import case_setup from CIME.XML.machines import Machines @@ -173,10 +174,9 @@ def _compare_baseline(self): urlroot = CIME.utils.get_urlroot(mach_obj) if htmlroot is not None: with CIME.utils.SharedArea(): - dir_util.copy_tree( + copytree( evv_out_dir, os.path.join(htmlroot, "evv", case_name), - preserve_mode=False, ) if urlroot is None: urlroot = "[{}_URL]".format(mach_name.capitalize()) @@ -203,4 +203,4 @@ def _compare_baseline(self): ) ) - CIME.utils.append_testlog(comments, self._orig_caseroot) + append_testlog(comments, self._orig_caseroot) diff --git a/CIME/SystemTests/pgn.py b/CIME/SystemTests/pgn.py index a8696c0afd2..83199ec674e 100644 --- a/CIME/SystemTests/pgn.py +++ b/CIME/SystemTests/pgn.py @@ -16,7 +16,7 @@ import logging from collections import OrderedDict -from distutils import dir_util +from shutils import copytree import pandas as pd import numpy as np @@ -24,6 +24,7 @@ import CIME.test_status import CIME.utils +from CIME.status import append_testlog from CIME.SystemTests.system_tests_common import SystemTestsCommon from CIME.case.case_setup import case_setup from CIME.XML.machines import Machines @@ -80,7 +81,7 @@ def build_phase(self, sharedlib_only=False, model_only=False): if not model_only: # Lay all of the components out concurrently logger.debug( - "PGN_INFO: Updating NINST for multi-instance in " "env_mach_pes.xml" + "PGN_INFO: Updating NINST for multi-instance in env_mach_pes.xml" ) for comp in ["ATM", "OCN", "WAV", "GLC", "ICE", "ROF", "LND"]: ntasks = self._case.get_value("NTASKS_{}".format(comp)) @@ -224,10 +225,9 @@ def _compare_baseline(self): urlroot = CIME.utils.get_urlroot(mach_obj) if htmlroot is not None: with CIME.utils.SharedArea(): - dir_util.copy_tree( + copytree( evv_out_dir, os.path.join(htmlroot, "evv", case_name), - preserve_mode=False, ) if urlroot is None: urlroot = "[{}_URL]".format(mach_name.capitalize()) @@ -253,7 +253,7 @@ def _compare_baseline(self): ) ) - CIME.utils.append_testlog(comments, self._orig_caseroot) + append_testlog(comments, self._orig_caseroot) def run_phase(self): logger.debug("PGN_INFO: RUN PHASE") diff --git a/CIME/SystemTests/tsc.py b/CIME/SystemTests/tsc.py index f50fd4c334b..be695b7f482 100644 --- a/CIME/SystemTests/tsc.py +++ b/CIME/SystemTests/tsc.py @@ -11,10 +11,11 @@ import json import logging -from distutils import dir_util +from shutil import copytree import CIME.test_status import CIME.utils +from CIME.status import append_testlog from CIME.SystemTests.system_tests_common import SystemTestsCommon from CIME.case.case_setup import case_setup from CIME.hist_utils import rename_all_hist_files @@ -213,10 +214,9 @@ def _compare_baseline(self): urlroot = CIME.utils.get_urlroot(mach_obj) if htmlroot is not None: with CIME.utils.SharedArea(): - dir_util.copy_tree( + copytree( evv_out_dir, os.path.join(htmlroot, "evv", case_name), - preserve_mode=False, ) if urlroot is None: urlroot = "[{}_URL]".format(mach_name.capitalize()) @@ -243,7 +243,7 @@ def _compare_baseline(self): ) ) - CIME.utils.append_testlog(comments, self._orig_caseroot) + append_testlog(comments, self._orig_caseroot) def _generate_baseline(self): super(TSC, self)._generate_baseline() From 3f0a0644146e93b2b3004607cbd9c88b7da59abf Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Wed, 7 Aug 2024 11:20:51 -0600 Subject: [PATCH 13/16] fix formatting to satisfy unit tests --- CIME/status.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/CIME/status.py b/CIME/status.py index 3ce01a9cb3c..23109b07645 100644 --- a/CIME/status.py +++ b/CIME/status.py @@ -14,7 +14,6 @@ def append_status(msg, sfile, caseroot="."): # Reduce empty lines in CaseStatus. It's a very concise file # and does not need extra newlines for readability line_ending = "\n" - with open(os.path.join(caseroot, sfile), "a") as fd: fd.write(ctime + msg + line_ending) fd.write(" ---------------------------------------------------" + line_ending) @@ -33,7 +32,7 @@ def append_case_status(phase, status, msg=None, caseroot=".", gitinterface=None) """ msg = msg if msg else "" append_status( - "{} {}{}".format(phase, status, msg), + "{} {} {}".format(phase, status, msg), "CaseStatus", caseroot, ) From 3d3814d16744ce50da26b5a244ebaa2f67e4c7e3 Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Wed, 7 Aug 2024 12:05:42 -0600 Subject: [PATCH 14/16] add minimum git version requirement --- CIME/case/case_setup.py | 44 +++++++++++++++++++++++++++++------------ 1 file changed, 31 insertions(+), 13 deletions(-) diff --git a/CIME/case/case_setup.py b/CIME/case/case_setup.py index 28cfbb5ab73..30562ba0e67 100644 --- a/CIME/case/case_setup.py +++ b/CIME/case/case_setup.py @@ -18,9 +18,10 @@ file_contains_python_function, import_from_file, copy_local_macros_to_dir, + batch_jobid, + run_cmd_no_fail, ) from CIME.status import run_and_log_case_status, append_case_status -from CIME.utils import batch_jobid from CIME.test_status import * from CIME.locked_files import unlock_file, lock_file, check_lockedfiles from CIME.gitinterface import GitInterface @@ -522,16 +523,33 @@ def case_setup(self, clean=False, test_mode=False, reset=False, keep=None): def _create_case_repo(self, caseroot): - self._gitinterface = GitInterface(caseroot, logger, branch=self.get_value("CASE")) - if not os.path.exists(os.path.join(caseroot, ".gitignore")): - safe_copy( - os.path.join( - self.get_value("CIMEROOT"), "CIME", "non_py", "gitignore.template" - ), - os.path.join(caseroot, ".gitignore"), + version = run_cmd_no_fail("git --version") + result = re.findall(r"([0-9]+)\.([0-9]+)\.?[0-9]*", version) + major = int(result[0][0]) + minor = int(result[0][1]) + + # gitinterface needs git version 2.28 or newer + if major > 2 or (major == 2 and minor >= 28): + self._gitinterface = GitInterface( + caseroot, logger, branch=self.get_value("CASE") + ) + if not os.path.exists(os.path.join(caseroot, ".gitignore")): + safe_copy( + os.path.join( + self.get_value("CIMEROOT"), "CIME", "non_py", "gitignore.template" + ), + os.path.join(caseroot, ".gitignore"), + ) + append_case_status( + "", "", "local git repository created", gitinterface=self._gitinterface + ) + # add all files in caseroot to local repository + self._gitinterface._git_command("add", "*") + else: + logger.warning("git interface requires git version 2.28 or newer") + + append_case_status( + "", + "", + f"local git version too old for cime git interface {major}.{minor}", ) - # add all files in caseroot to local repository - self._gitinterface._git_command("add", "*") - append_case_status( - "", "", "local git repository created", gitinterface=self._gitinterface - ) From 739b27e800b740328ccf20ce047663b2f40da171 Mon Sep 17 00:00:00 2001 From: Jason Boutte Date: Wed, 7 Aug 2024 11:48:56 -0700 Subject: [PATCH 15/16] Fixes minimum version of git --- docker/Dockerfile | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 7517c9fb6b0..c1224ae60db 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,5 +1,5 @@ -ARG MAMBAFORGE_VERSION=4.14.0-0 -FROM condaforge/mambaforge:${MAMBAFORGE_VERSION} AS base +ARG BASE_TAG=24.3.0-0 +FROM condaforge/miniforge3:${BASE_TAG} AS base SHELL ["/bin/bash", "-c"] @@ -14,6 +14,7 @@ RUN mkdir -p \ # Install common packages RUN mamba install --yes -c conda-forge \ + 'git>=2.28' \ cmake \ make \ wget \ @@ -99,7 +100,7 @@ COPY entrypoint.sh /entrypoint.sh ENTRYPOINT [ "/entrypoint.sh" ] -FROM base as slurm +FROM base AS slurm RUN apt-get update && \ DEBIAN_FRONTEND=noninteractive \ @@ -112,7 +113,7 @@ COPY slurm/slurm.conf /etc/slurm-llnl/ COPY slurm/config_batch.xml /root/.cime/ COPY slurm/entrypoint_batch.sh /entrypoint_batch.sh -FROM base as pbs +FROM base AS pbs RUN apt-get update && \ DEBIAN_FRONTEND=noninteractive \ From 32c65c6f9769ef0767b92d36210f848090757875 Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Thu, 8 Aug 2024 16:49:00 -0600 Subject: [PATCH 16/16] move template to templates directory --- CIME/case/case_setup.py | 6 +++++- CIME/{non_py => data/templates}/gitignore.template | 0 2 files changed, 5 insertions(+), 1 deletion(-) rename CIME/{non_py => data/templates}/gitignore.template (100%) diff --git a/CIME/case/case_setup.py b/CIME/case/case_setup.py index 30562ba0e67..d4b510f13f4 100644 --- a/CIME/case/case_setup.py +++ b/CIME/case/case_setup.py @@ -536,7 +536,11 @@ def _create_case_repo(self, caseroot): if not os.path.exists(os.path.join(caseroot, ".gitignore")): safe_copy( os.path.join( - self.get_value("CIMEROOT"), "CIME", "non_py", "gitignore.template" + self.get_value("CIMEROOT"), + "CIME", + "data", + "templates", + "gitignore.template", ), os.path.join(caseroot, ".gitignore"), ) diff --git a/CIME/non_py/gitignore.template b/CIME/data/templates/gitignore.template similarity index 100% rename from CIME/non_py/gitignore.template rename to CIME/data/templates/gitignore.template