From bf0e78ed63dd65d3d471f8fd74c02c8a2da909e3 Mon Sep 17 00:00:00 2001 From: David Huber Date: Thu, 21 Mar 2024 12:30:36 -0500 Subject: [PATCH 01/32] Added initial hsi and htar features. --- src/wxflow/archive_utils.py | 353 ++++++++++++++++++++++++++++++++++++ src/wxflow/file_utils.py | 206 ++++++++++++++++++++- src/wxflow/fsutils.py | 34 +++- 3 files changed, 590 insertions(+), 3 deletions(-) create mode 100644 src/wxflow/archive_utils.py diff --git a/src/wxflow/archive_utils.py b/src/wxflow/archive_utils.py new file mode 100644 index 0000000..a61382f --- /dev/null +++ b/src/wxflow/archive_utils.py @@ -0,0 +1,353 @@ +from logging import getLogger +from .executable import Executable, which + +from .fsutils import cp, mkdir, is_rstprod + +__all__ = ['hsi'] + +logger = getLogger(__name__.split('.')[-1]) + +class hsi: + """Class providing a set of functions to interact with the hsi utility. + + """ + + + def hsi(*args) -> None: + """Direct command builder function for hsi based on the input arguments. + + `args` should consist of a set of string arguments to send to hsi + For example, hsi.hsi("get","some_local_file : /some/hpss/file") will execute + hsi get some_local_file : /some/hpss/file + + """ + + cmd = which("hsi", required=True) + for arg in args: + cmd.add_default_arg(arg) + + cmd() + + + def get(source: str, target: str = "", hsi_flags: str = "") -> None: + """ Function to get a file from HPSS via hsi + + Parameters + ---------- + source : str + Full path location on HPSS of the file + + target : str + Location on the local machine to place the file. If not specified, + then the file will be placed in the current directory. + + hsi_flags : str + String of flags to send to hsi. See _parse_hsi_flags below for a + full list. + """ + args = () + + # Parse any hsi flags + if len(hsi_flags) > 0: + args = args + _parse_hsi_flags(hsi_flags) + + args = ("get",) + args = args + (target + " : " + source,) + hsi(*args) + + + def put(source: str, target: str, hsi_flags: str = "", listing_file : str = None) -> None: + """ Function to put a file onto HPSS via hsi + + Parameters + ---------- + source : str + Location on the local machine of the source file to send to HPSS. + + target : str + Full path of the target location of the file on HPSS. + + hsi_flags : str + String of flags to send to hsi. See _parse_hsi_flags below for a + full list. + """ + args = () + + # Parse any hsi flags + if len(hsi_flags) > 0: + args = args + _parse_hsi_flags(hsi_flags) + + args = args + ("put",) + args = args + (source + " : " + target,) + hsi(*args) + + + def chmod(mod: str, target: str, hsi_flags: str = "", flags: str = "") -> None: + """ Function to change the permissions of a file or directory on HPSS + + Parameters + ---------- + mod : str + Permissions to set for the file or directory, e.g. "640", "o+r", etc. + + target : str + Full path of the target location of the file on HPSS. + + hsi_flags : str + String of flags to send to hsi. See _parse_hsi_flags below for a + full list. + + flags : str + Flags to send to chmod. Valid flags are -R, -f, -c, and -v. See + "chmod --help" for more details. + """ + + args = () + + # Parse any hsi flags + if len(hsi_flags) > 0: + args = args + _parse_hsi_flags(hsi_flags) + + args = args + ("chmod",) + + if len(flags) > 0: + valid_flags = ["R", "f", "c", "v"] + args = args + _parse_flags(flags, valid_flags) + + args = args + (mod,) + args = args + (target,) + hsi(*args) + + + def chgrp(group_name: str, target: str, hsi_flags: str = "", flags: str = "") -> None: + """ Function to change the group of a file or directory on HPSS + + Parameters + ---------- + group_name : str + The group to which ownership of the file/directory is to be set. + + target : str + Full path of the target location of the file on HPSS. + + hsi_flags : str + String of flags to send to hsi. See _parse_hsi_flags below for a + full list. + + flags : str + Flags to send to chmod. Valid flags are -c, -v, -f, -h, and -R. See + "chgrp --help" for more details. + """ + + args = () + + # Parse any hsi flags + if len(hsi_flags) > 0: + args = args + _parse_hsi_flags(hsi_flags) + + args = args + ("chgrp",) + + if len(flags) > 0: + valid_flags = ["c", "v", "f", "h", "R"] + args = args + _parse_flags(flags, valid_flags) + + args = args + (group_name,) + args = args + (target,) + hsi(*args) + + + def rm(target: str, hsi_flags: str = "", flags: str = "") -> None: + """ Function to delete a file or directory on HPSS via hsi + + Parameters + ---------- + target : str + Full path of the target location of the file on HPSS. + + hsi_flags : str + String of flags to send to hsi. See _parse_hsi_flags below for a + full list. + + flags : str + Flags to send to chmod. Valid flags are -v, -f, and -r. See + "rm --help" for more details. + """ + + args = () + + # Parse any hsi flags + if len(hsi_flags) > 0: + args = args + _parse_hsi_flags(hsi_flags) + + args = args + ("rm",) + + if len(flags) > 0: + valid_flags = ['r', 'f', 'v'] + args = args + _parse_flags(flags, valid_flags) + + args = args + (target,) + hsi(*args) + + + def _parse_flags(flags: str, valid_flags: list, valid_flags_with_args: list = []) -> tuple: + """Expands the input flags into a tuple + + Inputs: + flags: str + String of flags to send to an hsi subcommand + valid_flags: list of strings + Set of valid flags to check against + valid_flags_with_args: list of strings + Set of valid flags that accept arguments + + Return: + Tuple of input flags + + Error: + Returns ValueError if an invalid flag is in flags + """ + + expanded_flags = () + flags_and_args = flags.split(" ") + n_args = len(flags_and_args) + i = 0 + while i < n_args: + flag = flags_and_args[i] + # Check that the flag is valid + if flag.startswith("-") + # E.g. "-p" or "-O output_file" + if len(flag) == 2: + flag = flag[1] + if flag in valid_flags_with_arg: + arg = flags_and_args[i+1] + i += 2 + expanded_flags = expanded_flags + (flag + arg,) + else: + expanded_flags = expanded_flags + (flag,) + # E.g. "-Ooutput_file" + else: + flag = flag[1] + arg = flag[2:] + i += 1 + expanded_flags = expanded_flags + (flag + arg,) + + else: + raise ValueError(f"One or more input flags '{flags}' is missing a '-', unable to parse") + + if flag not in valid_flags + valid_flags_with_arg: + raise ValueError(f"The input flags '{flags}' contains an invalid flag '{flag}'." + + return expanded_flags + + + def _parse_hsi_flags(flags: str) -> tuple: + """Expands the input flags into a tuple + + Inputs: + flags: str + String of flags to send to hsi. Valid flags (from hsi -?): +--------------------------------------------------------------------------- + hsi [-a acct_id|acct_name] [-A authmethod] [-c krb_cred_file] [-d level] [-e] [-G globus proxy path] + [-h host[/port] [-k keytabfile] [-l loginname] [-O listingFile] [-o] [-P] [-p port] [-s site] + [-q] [-v] [cmds] + Parameters: + -a acct_id | acct_name - specifies the HPSS account ID or account name to set + after login completes. This will be used for all new file creations + -A authmethod - authentication method. Case-insensitive legal method names are: + "combo","keytab","ident","gsi","local" + -c krb_cred_file - pathname to use for Kerberos credentials cache file + -d debug_level - debug message level (0-5) default is 0 + -e - command echo flag. Echos lines read from file(s) to the listable output file + -h host - specifies host name or IP address of the HPSS server, and optionally, + the port on which to connect + -k keytabfile - specifies pathname to DCE keytab file + -l loginname - specifies login name to use. + For kerberos, this is usually of the form "name@realm" + -O listingFile - specifies filename to contain all listable output, error messages,etc + This option is intended for use by programs that run hsi as a child process + and internally disables verbose (-v) mode, and sets quiet (-q) mode. (This can + be overridden by specifying the -v or -q parameters after the -O parameter) + -q - specifies "quiet" mode. Suppresses login message,file transfer progress messages, etc. + -s site - specifies the site name to connect to at startup. This name must match one of + the stanza names in either the global hsirc file, or in the user's private .hsirc file + -v - specifies "verbose" mode for listable output +------------------------------------------------------------------------------------- + + Return: + Tuple of input flags + + Error: + Returns ValueError if an invalid flag is in flags + """ + + valid_flags = ["e", "p", "q", "v"] + valid_flags_with_arg ["a", "A", "c", "d", "h", "k", "l", "O", "s"] + + return _parse_flags(flags, valid_flags, valid_flags_with_args) + +class htar: + """Class providing a set of functions to interact with the htar utility. + + """ + + def create(target: str, fileset: list, flags: str = None, ignore_missing: bool = False) -> None: + """ Build and execute the command + htar -cf + + Parameters: + ----------- + target : str + Path to the tarball to be created on HPSS + fileset: list + List of files to be added to + + Keyword Argument: + ----------------- + flags : str + Optional flags to add to the htar command + Each flag must be separated by a " " and start with a "-". + (-c and -f are assumed) + Invalid flags will raise a ValueError + + ignore_missing: bool + Flag specifying whether missing files are allowed. Warnings + will be printed instead. + + # Will raise a CommandNotFoundError if htar is not in $PATH + # Will raise a FileNotFoundError if an input file is not found + # Will raise a ValueError if an invalid flag is sent in + """ + cmd = which("htar", required=True) + + if flags is not None: + valid_flags = ['-v', '-h', '-q', '-V', '-v'] + ignore_flags = ['-c', '-f'] + for flag in flags.split(" ") + if flag in ignore_flags: + continue + if flag not in valid_flags: + raise ValueError(f"The input flags '{flags}' contains an invalid flag '{flag}'." + cmd.add_default_arg(flag) + + cmd.add_default_arg("-c") + cmd.add_default_arg("-f") + + cmd.add_default_arg(target) + + has_rstprod = False + + # Check files for existence and rstprod + for file in fileset + if not os.path.exists(file): + if ignore_missing: + print(f"WARNING input file '{file}' does not exist") + else: + raise FileNotFoundError(f"The input file '{file}' was not found") + + if not has_rstprod: + has_rstprod = is_rstprod(file) + + cmd.add_default_arg(file) + + if has_rstprod: + cmd() diff --git a/src/wxflow/file_utils.py b/src/wxflow/file_utils.py index a8220fc..34bb2e9 100644 --- a/src/wxflow/file_utils.py +++ b/src/wxflow/file_utils.py @@ -1,6 +1,6 @@ from logging import getLogger -from .fsutils import cp, mkdir +from .fsutils import cp, mkdir, hsi_put, hsi_get, hsi_chmod, hsi_chgrp __all__ = ['FileHandler'] @@ -63,6 +63,37 @@ def _copy_files(filelist): cp(src, dest) logger.info(f'Copied {src} to {dest}') + @staticmethod + def _write_tarball(filelist, tarball): + """Function to add all files specified in the list to the target tarball. + The target tarball can have extensions .tar, .tar.bz2, .tar.gz, or .tgz, + representing the desired compression. + + Parameters + ---------- + filelist : list + List of files + + tarball : str + Name of the tarball to create/append to + """ + + # Check for valid tar extension + valid_tar_extensions = [".tar", ".tar.gz", ".tgz", "tar.bz2"] + valid = False + + for ext in valid_tar_extensions: + if tarball.endswith(ext): + valid = True + + if not valid: + raise TypeError(f"Target tarball ({tarball}) has an invalid extension.") + + for file in filelist: + # Check for globs + write_tar(file, tarball) + logger.info(f'Wrote {file} to {tarball}') + @staticmethod def _make_dirs(dirlist): """Function to make all directories specified in the list @@ -75,3 +106,176 @@ def _make_dirs(dirlist): for dd in dirlist: mkdir(dd) logger.info(f'Created {dd}') + + +class ArchiveHandler: + """Class to interact with a target archive (e.g. .tar). Presently, only + archive creation is enabled. + + Parameters + ---------- + config : dict + A dictionary containing "protocol", "action", "target", and "fileset" + + NOTE + ---- + "action" can be presently only be "create" + "protocol" can be either "tar" or "htar" + "target" is the name of the archive to act on + "fileset" is a list of files to add to the archive + "verbose" is an optional boolean key for extra output + + Attributes + ---------- + config : dict + Dictionary of files to manipulate + """ + + def __init__(self, config): + + self.config = config + + def create(self): + """ + Method to create an archive based described in the configuration + + The input configuration should have the following keys + protocol: str + Name of the archiving program (currently tar or htar) + target: str + File name of the target archive + fileset: list + List of files to write to the archive + verbose: boolean (optional) + Whether to print verbosely while writing or not + """ + launcher = { + 'tar': self._create_tar, + 'htar': self._create_htar, + } + # Get configuration values + protocol = self.config['protocol'] + target = self.config['target'] + fileset = self.config['fileset'] + action = self.config['action'] + verbose = False + if "verbose" in self.config.keys(): + verbose = self.config['verbose'] + + launcher[protocol] (target, fileset, verbose) + + @staticmethod + def _create_tar(target, filelist, verbose=False): + """Function to add all files specified in the list to the target tarball. + The target tarball can have extensions .tar, .tar.bz2, .tar.gz, or .tgz, + representing the desired compression. This will overwrite an existing + tar file. + + Parameters + ---------- + target : str + Name of the tarball to create + + filelist : list + List of files + + verbose : boolean + Whether to print as files are written to the tarball. + """ + + # Check for valid tar extension + valid_tar_extensions = [".tar", ".tar.gz", ".tgz", "tar.bz2"] + valid = False + + for ext in valid_tar_extensions: + if target.endswith(ext): + valid = True + + if not valid: + raise TypeError(f"Target tarball {target} has an invalid extension.") + + # Create the parent directory if it does not yet exist + if not os.path.exists(os.path.dirname(archive_name)): + mkdir(os.path.dirname(archive_name)) + + has_rstprod = False + + with tarfile.open(target) as handle: + for file in fileset: + if not has_rstprod: + if(os.stat(file).st_gid == gid_rstprod): + has_rstprod = True + os.chown(target, -1, gid_rstprod) + logger.info(f'Changed group of {target} to rstprod.') + + tar.add(file) + write_tar(file, target) + + @staticmethod + def _create_htar(target, filelist, verbose=False): + """Function to add all files specified in the list to the target tarball + on HPSS. The target tarball may only have the extension .tar. This will + overwrite an existing tar file. + + Parameters + ---------- + target : str + Full path and filename of the tarball to create + + filelist : list + List of files + + verbose : boolean + Whether to print as files are written to the tarball. + """ + + # Check for valid tar extension + if target.endswith(ext): + valid = True + + if not target.endswith(".tar"): + raise TypeError(f"Target tarball {target} has an invalid extension.") + + # Get the htar command. + htar = which("htar", required=True) + + has_rstprod = False + + for file in fileset: + if not has_rstprod: + if(os.stat(file).st_gid == gid_rstprod): + has_rstprod = True + + # Build the htar command + if verbose: + htar.add_default_arg('-cvf') + else: + htar.add_default_arg('-cf') + + htar.add_default_arg(target) + htar.add_default_arg(" ".join(filelist)) + + # Attempt to run htar + try: + htar(output=str) + except: + if has_rstprod: + # Attempt to change the group of the target file + try: + hsi.chgrp("rstprod", target) + hsi.chmod("640", target) + except: + hsi.rm(target, flags="-f") + raise OSError(f"Failed to change the group for a restricted data file.\n" + f"Please verify that {target} was deleted!!") + + raise OSError(f"htar failed to archive {target}.") + + if has_rstprod: + try: + hsi.chgrp("rstprod", target) + hsi.chmod("640", target) + except: + hsi.rm(target, flags="-f") + raise OSError(f"Failed to change the group for a restricted data file.\n" + f"Please verify that {target} was deleted!!") diff --git a/src/wxflow/fsutils.py b/src/wxflow/fsutils.py index c266642..1653d5c 100644 --- a/src/wxflow/fsutils.py +++ b/src/wxflow/fsutils.py @@ -2,9 +2,11 @@ import errno import os import shutil +import pathlib +import grp +from .executable import Executable, which -__all__ = ['mkdir', 'mkdir_p', 'rmdir', 'chdir', 'rm_p', 'cp'] - +__all__ = ['mkdir', 'mkdir_p', 'rmdir', 'chdir', 'rm_p', 'cp', 'get_gid'] def mkdir_p(path): try: @@ -85,3 +87,31 @@ def cp(source: str, target: str) -> None: raise OSError(f"unable to copy {source} to {target}") except Exception as exc: raise Exception(exc) + + +# Group ID number for a given group name +def get_gid(group_name: str): + try: + group_id = grp.getgrnam(group_name) + except KeyError: + raise KeyError(f"{group_name} is not a valid group name.") + + return group_id + + +# Determine if a path (dir, file, link) belongs to the rstprod group +def is_rstprod(path: str) -> bool: + try: + rstprod_gid = get_gid("rstprod") + except KeyError: + # The rstprod group does not exist + return False + + if not os.path.exists(path): + print(f"WARNING '{path}' does not exist!") + return False + + if os.stat(path).st_gid == rstprod_gid: + return True + + return False From 17b2ff24e90f85fa58edf1b5dc116918f4d4ed0e Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Thu, 21 Mar 2024 19:43:48 +0000 Subject: [PATCH 02/32] Clean up hsi class. --- src/wxflow/__init__.py | 1 + src/wxflow/archive_utils.py | 139 ++++++++++++++++++------------------ src/wxflow/file_utils.py | 2 +- 3 files changed, 72 insertions(+), 70 deletions(-) diff --git a/src/wxflow/__init__.py b/src/wxflow/__init__.py index 1f9efa3..300b343 100644 --- a/src/wxflow/__init__.py +++ b/src/wxflow/__init__.py @@ -7,6 +7,7 @@ from .executable import CommandNotFoundError, Executable, which from .factory import Factory from .file_utils import FileHandler +from .archive_utils import Hsi, Htar from .fsutils import chdir, cp, mkdir, mkdir_p, rm_p, rmdir from .jinja import Jinja from .logger import Logger, logit diff --git a/src/wxflow/archive_utils.py b/src/wxflow/archive_utils.py index a61382f..76e8983 100644 --- a/src/wxflow/archive_utils.py +++ b/src/wxflow/archive_utils.py @@ -7,13 +7,12 @@ logger = getLogger(__name__.split('.')[-1]) -class hsi: +class Hsi: """Class providing a set of functions to interact with the hsi utility. """ - - def hsi(*args) -> None: + def hsi(self, *args) -> None: """Direct command builder function for hsi based on the input arguments. `args` should consist of a set of string arguments to send to hsi @@ -22,14 +21,15 @@ def hsi(*args) -> None: """ - cmd = which("hsi", required=True) + cmd = which("hsi", required = True) + for arg in args: cmd.add_default_arg(arg) cmd() - def get(source: str, target: str = "", hsi_flags: str = "") -> None: + def get(self, source: str, target: str = "", hsi_flags: str = "") -> None: """ Function to get a file from HPSS via hsi Parameters @@ -49,14 +49,17 @@ def get(source: str, target: str = "", hsi_flags: str = "") -> None: # Parse any hsi flags if len(hsi_flags) > 0: - args = args + _parse_hsi_flags(hsi_flags) + args = args + self._parse_hsi_flags(hsi_flags) args = ("get",) - args = args + (target + " : " + source,) - hsi(*args) + if len(target) == 0: + args = args + (source,) + else: + args = args + (target + " : " + source,) + self.hsi(*args) - def put(source: str, target: str, hsi_flags: str = "", listing_file : str = None) -> None: + def put(self, source: str, target: str, hsi_flags: str = "", listing_file : str = None) -> None: """ Function to put a file onto HPSS via hsi Parameters @@ -75,14 +78,14 @@ def put(source: str, target: str, hsi_flags: str = "", listing_file : str = None # Parse any hsi flags if len(hsi_flags) > 0: - args = args + _parse_hsi_flags(hsi_flags) + args = args + self._parse_hsi_flags(hsi_flags) args = args + ("put",) args = args + (source + " : " + target,) - hsi(*args) + self.hsi(*args) - def chmod(mod: str, target: str, hsi_flags: str = "", flags: str = "") -> None: + def chmod(self, mod: str, target: str, hsi_flags: str = "", flags: str = "") -> None: """ Function to change the permissions of a file or directory on HPSS Parameters @@ -98,28 +101,28 @@ def chmod(mod: str, target: str, hsi_flags: str = "", flags: str = "") -> None: full list. flags : str - Flags to send to chmod. Valid flags are -R, -f, -c, and -v. See - "chmod --help" for more details. + Flags to send to chmod. Valid flags are -d, -f, -h, -H, and -R. See + "hsi chmod -?" for more details. """ args = () # Parse any hsi flags if len(hsi_flags) > 0: - args = args + _parse_hsi_flags(hsi_flags) + args = args + self._parse_hsi_flags(hsi_flags) args = args + ("chmod",) if len(flags) > 0: - valid_flags = ["R", "f", "c", "v"] - args = args + _parse_flags(flags, valid_flags) + valid_flags = ["d", "h", "H", "R", "f"] + args = args + self._parse_flags(flags, valid_flags) args = args + (mod,) args = args + (target,) - hsi(*args) + self.hsi(*args) - def chgrp(group_name: str, target: str, hsi_flags: str = "", flags: str = "") -> None: + def chgrp(self, group_name: str, target: str, hsi_flags: str = "", flags: str = "") -> None: """ Function to change the group of a file or directory on HPSS Parameters @@ -135,7 +138,7 @@ def chgrp(group_name: str, target: str, hsi_flags: str = "", flags: str = "") -> full list. flags : str - Flags to send to chmod. Valid flags are -c, -v, -f, -h, and -R. See + Flags to send to chmod. Valid flags are -h, -L, -H, and -R. See "chgrp --help" for more details. """ @@ -143,20 +146,20 @@ def chgrp(group_name: str, target: str, hsi_flags: str = "", flags: str = "") -> # Parse any hsi flags if len(hsi_flags) > 0: - args = args + _parse_hsi_flags(hsi_flags) + args = args + self._parse_hsi_flags(hsi_flags) args = args + ("chgrp",) if len(flags) > 0: - valid_flags = ["c", "v", "f", "h", "R"] - args = args + _parse_flags(flags, valid_flags) + valid_flags = ["R", "h", "L", "H"] + args = args + self._parse_flags(flags, valid_flags) args = args + (group_name,) args = args + (target,) - hsi(*args) + self.hsi(*args) - def rm(target: str, hsi_flags: str = "", flags: str = "") -> None: + def rm(self, target: str, hsi_flags: str = "", flags: str = "") -> None: """ Function to delete a file or directory on HPSS via hsi Parameters @@ -169,27 +172,26 @@ def rm(target: str, hsi_flags: str = "", flags: str = "") -> None: full list. flags : str - Flags to send to chmod. Valid flags are -v, -f, and -r. See - "rm --help" for more details. + Flags to send to chmod. The only valid flag is -R (recursive). """ args = () # Parse any hsi flags if len(hsi_flags) > 0: - args = args + _parse_hsi_flags(hsi_flags) + args = args + self._parse_hsi_flags(hsi_flags) args = args + ("rm",) if len(flags) > 0: - valid_flags = ['r', 'f', 'v'] - args = args + _parse_flags(flags, valid_flags) + valid_flags = ['R'] + args = args + self._parse_flags(flags, valid_flags) args = args + (target,) - hsi(*args) + self.hsi(*args) - def _parse_flags(flags: str, valid_flags: list, valid_flags_with_args: list = []) -> tuple: + def _parse_flags(self, flags: str, valid_flags: list, valid_flags_with_arg: list = []) -> tuple: """Expands the input flags into a tuple Inputs: @@ -197,7 +199,7 @@ def _parse_flags(flags: str, valid_flags: list, valid_flags_with_args: list = [] String of flags to send to an hsi subcommand valid_flags: list of strings Set of valid flags to check against - valid_flags_with_args: list of strings + valid_flags_with_arg: list of strings Set of valid flags that accept arguments Return: @@ -214,63 +216,62 @@ def _parse_flags(flags: str, valid_flags: list, valid_flags_with_args: list = [] while i < n_args: flag = flags_and_args[i] # Check that the flag is valid - if flag.startswith("-") + if flag.startswith("-"): # E.g. "-p" or "-O output_file" if len(flag) == 2: flag = flag[1] if flag in valid_flags_with_arg: arg = flags_and_args[i+1] i += 2 - expanded_flags = expanded_flags + (flag + arg,) + expanded_flags = expanded_flags + ("-" + flag + arg,) else: - expanded_flags = expanded_flags + (flag,) + expanded_flags = expanded_flags + ("-" + flag,) + i += 1 # E.g. "-Ooutput_file" else: - flag = flag[1] + tmp_flag = flag[1] arg = flag[2:] + flag = tmp_flag i += 1 - expanded_flags = expanded_flags + (flag + arg,) + expanded_flags = expanded_flags + ("-" + flag + arg,) else: raise ValueError(f"One or more input flags '{flags}' is missing a '-', unable to parse") if flag not in valid_flags + valid_flags_with_arg: - raise ValueError(f"The input flags '{flags}' contains an invalid flag '{flag}'." + raise ValueError(f"The input flags '{flags}' contains an invalid flag '{flag}'.") return expanded_flags - def _parse_hsi_flags(flags: str) -> tuple: + def _parse_hsi_flags(self, flags: str) -> tuple: """Expands the input flags into a tuple Inputs: flags: str String of flags to send to hsi. Valid flags (from hsi -?): --------------------------------------------------------------------------- - hsi [-a acct_id|acct_name] [-A authmethod] [-c krb_cred_file] [-d level] [-e] [-G globus proxy path] - [-h host[/port] [-k keytabfile] [-l loginname] [-O listingFile] [-o] [-P] [-p port] [-s site] - [-q] [-v] [cmds] - Parameters: - -a acct_id | acct_name - specifies the HPSS account ID or account name to set - after login completes. This will be used for all new file creations - -A authmethod - authentication method. Case-insensitive legal method names are: - "combo","keytab","ident","gsi","local" - -c krb_cred_file - pathname to use for Kerberos credentials cache file - -d debug_level - debug message level (0-5) default is 0 - -e - command echo flag. Echos lines read from file(s) to the listable output file - -h host - specifies host name or IP address of the HPSS server, and optionally, - the port on which to connect - -k keytabfile - specifies pathname to DCE keytab file - -l loginname - specifies login name to use. - For kerberos, this is usually of the form "name@realm" - -O listingFile - specifies filename to contain all listable output, error messages,etc - This option is intended for use by programs that run hsi as a child process - and internally disables verbose (-v) mode, and sets quiet (-q) mode. (This can - be overridden by specifying the -v or -q parameters after the -O parameter) - -q - specifies "quiet" mode. Suppresses login message,file transfer progress messages, etc. - -s site - specifies the site name to connect to at startup. This name must match one of + Parameters: + -a acct_id | acct_name - specifies the HPSS account ID or account name to set + after login completes. This will be used for all new file creations + -A authmethod - authentication method. Case-insensitive legal method names are: + "combo","keytab","ident","gsi","local" + -c krb_cred_file - pathname to use for Kerberos credentials cache file + -d debug_level - debug message level (0-5) default is 0 + -e - command echo flag. Echos lines read from file(s) to the listable output file + -h host - specifies host name or IP address of the HPSS server, and optionally, + the port on which to connect + -k keytabfile - specifies pathname to DCE keytab file + -l loginname - specifies login name to use. + For kerberos, this is usually of the form "name@realm" + -O listingFile - specifies filename to contain all listable output, error messages,etc + This option is intended for use by programs that run hsi as a child process + and internally disables verbose (-v) mode, and sets quiet (-q) mode. (This can + be overridden by specifying the -v or -q parameters after the -O parameter) + -q - specifies "quiet" mode. Suppresses login message,file transfer progress messages, etc. + -s site - specifies the site name to connect to at startup. This name must match one of the stanza names in either the global hsirc file, or in the user's private .hsirc file - -v - specifies "verbose" mode for listable output + -v - specifies "verbose" mode for listable output ------------------------------------------------------------------------------------- Return: @@ -281,11 +282,11 @@ def _parse_hsi_flags(flags: str) -> tuple: """ valid_flags = ["e", "p", "q", "v"] - valid_flags_with_arg ["a", "A", "c", "d", "h", "k", "l", "O", "s"] + valid_flags_with_arg = ["a", "A", "c", "d", "h", "k", "l", "O", "s"] - return _parse_flags(flags, valid_flags, valid_flags_with_args) + return self._parse_flags(flags, valid_flags, valid_flags_with_arg) -class htar: +class Htar: """Class providing a set of functions to interact with the htar utility. """ @@ -322,11 +323,11 @@ def create(target: str, fileset: list, flags: str = None, ignore_missing: bool = if flags is not None: valid_flags = ['-v', '-h', '-q', '-V', '-v'] ignore_flags = ['-c', '-f'] - for flag in flags.split(" ") + for flag in flags.split(" "): if flag in ignore_flags: continue if flag not in valid_flags: - raise ValueError(f"The input flags '{flags}' contains an invalid flag '{flag}'." + raise ValueError(f"The input flags '{flags}' contains an invalid flag '{flag}'.") cmd.add_default_arg(flag) cmd.add_default_arg("-c") @@ -337,7 +338,7 @@ def create(target: str, fileset: list, flags: str = None, ignore_missing: bool = has_rstprod = False # Check files for existence and rstprod - for file in fileset + for file in fileset: if not os.path.exists(file): if ignore_missing: print(f"WARNING input file '{file}' does not exist") diff --git a/src/wxflow/file_utils.py b/src/wxflow/file_utils.py index 34bb2e9..267a057 100644 --- a/src/wxflow/file_utils.py +++ b/src/wxflow/file_utils.py @@ -1,6 +1,6 @@ from logging import getLogger -from .fsutils import cp, mkdir, hsi_put, hsi_get, hsi_chmod, hsi_chgrp +from .fsutils import cp, mkdir __all__ = ['FileHandler'] From 33098afdd54432bc6078b977130e5ec957752804 Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Thu, 21 Mar 2024 20:11:41 +0000 Subject: [PATCH 03/32] Add new hsi features. --- src/wxflow/__init__.py | 2 +- src/wxflow/archive_utils.py | 147 ++++++-------------------- src/wxflow/file_utils.py | 204 ------------------------------------ src/wxflow/fsutils.py | 1 - 4 files changed, 34 insertions(+), 320 deletions(-) diff --git a/src/wxflow/__init__.py b/src/wxflow/__init__.py index 300b343..72b5ef2 100644 --- a/src/wxflow/__init__.py +++ b/src/wxflow/__init__.py @@ -7,7 +7,7 @@ from .executable import CommandNotFoundError, Executable, which from .factory import Factory from .file_utils import FileHandler -from .archive_utils import Hsi, Htar +from .archive_utils import Hsi from .fsutils import chdir, cp, mkdir, mkdir_p, rm_p, rmdir from .jinja import Jinja from .logger import Logger, logit diff --git a/src/wxflow/archive_utils.py b/src/wxflow/archive_utils.py index 76e8983..ca2d2b6 100644 --- a/src/wxflow/archive_utils.py +++ b/src/wxflow/archive_utils.py @@ -3,7 +3,7 @@ from .fsutils import cp, mkdir, is_rstprod -__all__ = ['hsi'] +__all__ = ['Hsi'] logger = getLogger(__name__.split('.')[-1]) @@ -12,7 +12,8 @@ class Hsi: """ - def hsi(self, *args) -> None: + @staticmethod + def hsi(*args) -> None: """Direct command builder function for hsi based on the input arguments. `args` should consist of a set of string arguments to send to hsi @@ -29,7 +30,8 @@ def hsi(self, *args) -> None: cmd() - def get(self, source: str, target: str = "", hsi_flags: str = "") -> None: + @classmethod + def get(cls, source: str, target: str = "", hsi_flags: str = "") -> None: """ Function to get a file from HPSS via hsi Parameters @@ -49,17 +51,18 @@ def get(self, source: str, target: str = "", hsi_flags: str = "") -> None: # Parse any hsi flags if len(hsi_flags) > 0: - args = args + self._parse_hsi_flags(hsi_flags) + args = args + cls._parse_hsi_flags(hsi_flags) args = ("get",) if len(target) == 0: args = args + (source,) else: args = args + (target + " : " + source,) - self.hsi(*args) + cls.hsi(*args) - def put(self, source: str, target: str, hsi_flags: str = "", listing_file : str = None) -> None: + @classmethod + def put(cls, source: str, target: str, hsi_flags: str = "", listing_file : str = None) -> None: """ Function to put a file onto HPSS via hsi Parameters @@ -78,14 +81,15 @@ def put(self, source: str, target: str, hsi_flags: str = "", listing_file : str # Parse any hsi flags if len(hsi_flags) > 0: - args = args + self._parse_hsi_flags(hsi_flags) + args = args + cls._parse_hsi_flags(hsi_flags) args = args + ("put",) args = args + (source + " : " + target,) - self.hsi(*args) + cls.hsi(*args) - def chmod(self, mod: str, target: str, hsi_flags: str = "", flags: str = "") -> None: + @classmethod + def chmod(cls, mod: str, target: str, hsi_flags: str = "", flags: str = "") -> None: """ Function to change the permissions of a file or directory on HPSS Parameters @@ -109,20 +113,21 @@ def chmod(self, mod: str, target: str, hsi_flags: str = "", flags: str = "") -> # Parse any hsi flags if len(hsi_flags) > 0: - args = args + self._parse_hsi_flags(hsi_flags) + args = args + cls._parse_hsi_flags(hsi_flags) args = args + ("chmod",) if len(flags) > 0: valid_flags = ["d", "h", "H", "R", "f"] - args = args + self._parse_flags(flags, valid_flags) + args = args + cls._parse_flags(flags, valid_flags) args = args + (mod,) args = args + (target,) - self.hsi(*args) + cls.hsi(*args) - def chgrp(self, group_name: str, target: str, hsi_flags: str = "", flags: str = "") -> None: + @classmethod + def chgrp(cls, group_name: str, target: str, hsi_flags: str = "", flags: str = "") -> None: """ Function to change the group of a file or directory on HPSS Parameters @@ -146,20 +151,21 @@ def chgrp(self, group_name: str, target: str, hsi_flags: str = "", flags: str = # Parse any hsi flags if len(hsi_flags) > 0: - args = args + self._parse_hsi_flags(hsi_flags) + args = args + cls._parse_hsi_flags(hsi_flags) args = args + ("chgrp",) if len(flags) > 0: valid_flags = ["R", "h", "L", "H"] - args = args + self._parse_flags(flags, valid_flags) + args = args + cls._parse_flags(flags, valid_flags) args = args + (group_name,) args = args + (target,) - self.hsi(*args) + cls.hsi(*args) - def rm(self, target: str, hsi_flags: str = "", flags: str = "") -> None: + @classmethod + def rm(cls, target: str, hsi_flags: str = "", flags: str = "") -> None: """ Function to delete a file or directory on HPSS via hsi Parameters @@ -179,19 +185,20 @@ def rm(self, target: str, hsi_flags: str = "", flags: str = "") -> None: # Parse any hsi flags if len(hsi_flags) > 0: - args = args + self._parse_hsi_flags(hsi_flags) + args = args + cls._parse_hsi_flags(hsi_flags) args = args + ("rm",) if len(flags) > 0: valid_flags = ['R'] - args = args + self._parse_flags(flags, valid_flags) + args = args + cls._parse_flags(flags, valid_flags) args = args + (target,) - self.hsi(*args) + cls.hsi(*args) - def _parse_flags(self, flags: str, valid_flags: list, valid_flags_with_arg: list = []) -> tuple: + @staticmethod + def _parse_flags(flags: str, valid_flags: list, valid_flags_with_arg: list = []) -> tuple: """Expands the input flags into a tuple Inputs: @@ -244,35 +251,14 @@ def _parse_flags(self, flags: str, valid_flags: list, valid_flags_with_arg: list return expanded_flags - def _parse_hsi_flags(self, flags: str) -> tuple: + @classmethod + def _parse_hsi_flags(cls, flags: str) -> tuple: """Expands the input flags into a tuple Inputs: flags: str - String of flags to send to hsi. Valid flags (from hsi -?): ---------------------------------------------------------------------------- - Parameters: - -a acct_id | acct_name - specifies the HPSS account ID or account name to set - after login completes. This will be used for all new file creations - -A authmethod - authentication method. Case-insensitive legal method names are: - "combo","keytab","ident","gsi","local" - -c krb_cred_file - pathname to use for Kerberos credentials cache file - -d debug_level - debug message level (0-5) default is 0 - -e - command echo flag. Echos lines read from file(s) to the listable output file - -h host - specifies host name or IP address of the HPSS server, and optionally, - the port on which to connect - -k keytabfile - specifies pathname to DCE keytab file - -l loginname - specifies login name to use. - For kerberos, this is usually of the form "name@realm" - -O listingFile - specifies filename to contain all listable output, error messages,etc - This option is intended for use by programs that run hsi as a child process - and internally disables verbose (-v) mode, and sets quiet (-q) mode. (This can - be overridden by specifying the -v or -q parameters after the -O parameter) - -q - specifies "quiet" mode. Suppresses login message,file transfer progress messages, etc. - -s site - specifies the site name to connect to at startup. This name must match one of - the stanza names in either the global hsirc file, or in the user's private .hsirc file - -v - specifies "verbose" mode for listable output -------------------------------------------------------------------------------------- + String of flags to send to hsi. Valid flags are -a, -A, -c, + -d, -e, -h, -k, -l, -O, -q, -s, and -v. See hsi -q for details. Return: Tuple of input flags @@ -284,71 +270,4 @@ def _parse_hsi_flags(self, flags: str) -> tuple: valid_flags = ["e", "p", "q", "v"] valid_flags_with_arg = ["a", "A", "c", "d", "h", "k", "l", "O", "s"] - return self._parse_flags(flags, valid_flags, valid_flags_with_arg) - -class Htar: - """Class providing a set of functions to interact with the htar utility. - - """ - - def create(target: str, fileset: list, flags: str = None, ignore_missing: bool = False) -> None: - """ Build and execute the command - htar -cf - - Parameters: - ----------- - target : str - Path to the tarball to be created on HPSS - fileset: list - List of files to be added to - - Keyword Argument: - ----------------- - flags : str - Optional flags to add to the htar command - Each flag must be separated by a " " and start with a "-". - (-c and -f are assumed) - Invalid flags will raise a ValueError - - ignore_missing: bool - Flag specifying whether missing files are allowed. Warnings - will be printed instead. - - # Will raise a CommandNotFoundError if htar is not in $PATH - # Will raise a FileNotFoundError if an input file is not found - # Will raise a ValueError if an invalid flag is sent in - """ - cmd = which("htar", required=True) - - if flags is not None: - valid_flags = ['-v', '-h', '-q', '-V', '-v'] - ignore_flags = ['-c', '-f'] - for flag in flags.split(" "): - if flag in ignore_flags: - continue - if flag not in valid_flags: - raise ValueError(f"The input flags '{flags}' contains an invalid flag '{flag}'.") - cmd.add_default_arg(flag) - - cmd.add_default_arg("-c") - cmd.add_default_arg("-f") - - cmd.add_default_arg(target) - - has_rstprod = False - - # Check files for existence and rstprod - for file in fileset: - if not os.path.exists(file): - if ignore_missing: - print(f"WARNING input file '{file}' does not exist") - else: - raise FileNotFoundError(f"The input file '{file}' was not found") - - if not has_rstprod: - has_rstprod = is_rstprod(file) - - cmd.add_default_arg(file) - - if has_rstprod: - cmd() + return cls._parse_flags(flags, valid_flags, valid_flags_with_arg) diff --git a/src/wxflow/file_utils.py b/src/wxflow/file_utils.py index 267a057..a8220fc 100644 --- a/src/wxflow/file_utils.py +++ b/src/wxflow/file_utils.py @@ -63,37 +63,6 @@ def _copy_files(filelist): cp(src, dest) logger.info(f'Copied {src} to {dest}') - @staticmethod - def _write_tarball(filelist, tarball): - """Function to add all files specified in the list to the target tarball. - The target tarball can have extensions .tar, .tar.bz2, .tar.gz, or .tgz, - representing the desired compression. - - Parameters - ---------- - filelist : list - List of files - - tarball : str - Name of the tarball to create/append to - """ - - # Check for valid tar extension - valid_tar_extensions = [".tar", ".tar.gz", ".tgz", "tar.bz2"] - valid = False - - for ext in valid_tar_extensions: - if tarball.endswith(ext): - valid = True - - if not valid: - raise TypeError(f"Target tarball ({tarball}) has an invalid extension.") - - for file in filelist: - # Check for globs - write_tar(file, tarball) - logger.info(f'Wrote {file} to {tarball}') - @staticmethod def _make_dirs(dirlist): """Function to make all directories specified in the list @@ -106,176 +75,3 @@ def _make_dirs(dirlist): for dd in dirlist: mkdir(dd) logger.info(f'Created {dd}') - - -class ArchiveHandler: - """Class to interact with a target archive (e.g. .tar). Presently, only - archive creation is enabled. - - Parameters - ---------- - config : dict - A dictionary containing "protocol", "action", "target", and "fileset" - - NOTE - ---- - "action" can be presently only be "create" - "protocol" can be either "tar" or "htar" - "target" is the name of the archive to act on - "fileset" is a list of files to add to the archive - "verbose" is an optional boolean key for extra output - - Attributes - ---------- - config : dict - Dictionary of files to manipulate - """ - - def __init__(self, config): - - self.config = config - - def create(self): - """ - Method to create an archive based described in the configuration - - The input configuration should have the following keys - protocol: str - Name of the archiving program (currently tar or htar) - target: str - File name of the target archive - fileset: list - List of files to write to the archive - verbose: boolean (optional) - Whether to print verbosely while writing or not - """ - launcher = { - 'tar': self._create_tar, - 'htar': self._create_htar, - } - # Get configuration values - protocol = self.config['protocol'] - target = self.config['target'] - fileset = self.config['fileset'] - action = self.config['action'] - verbose = False - if "verbose" in self.config.keys(): - verbose = self.config['verbose'] - - launcher[protocol] (target, fileset, verbose) - - @staticmethod - def _create_tar(target, filelist, verbose=False): - """Function to add all files specified in the list to the target tarball. - The target tarball can have extensions .tar, .tar.bz2, .tar.gz, or .tgz, - representing the desired compression. This will overwrite an existing - tar file. - - Parameters - ---------- - target : str - Name of the tarball to create - - filelist : list - List of files - - verbose : boolean - Whether to print as files are written to the tarball. - """ - - # Check for valid tar extension - valid_tar_extensions = [".tar", ".tar.gz", ".tgz", "tar.bz2"] - valid = False - - for ext in valid_tar_extensions: - if target.endswith(ext): - valid = True - - if not valid: - raise TypeError(f"Target tarball {target} has an invalid extension.") - - # Create the parent directory if it does not yet exist - if not os.path.exists(os.path.dirname(archive_name)): - mkdir(os.path.dirname(archive_name)) - - has_rstprod = False - - with tarfile.open(target) as handle: - for file in fileset: - if not has_rstprod: - if(os.stat(file).st_gid == gid_rstprod): - has_rstprod = True - os.chown(target, -1, gid_rstprod) - logger.info(f'Changed group of {target} to rstprod.') - - tar.add(file) - write_tar(file, target) - - @staticmethod - def _create_htar(target, filelist, verbose=False): - """Function to add all files specified in the list to the target tarball - on HPSS. The target tarball may only have the extension .tar. This will - overwrite an existing tar file. - - Parameters - ---------- - target : str - Full path and filename of the tarball to create - - filelist : list - List of files - - verbose : boolean - Whether to print as files are written to the tarball. - """ - - # Check for valid tar extension - if target.endswith(ext): - valid = True - - if not target.endswith(".tar"): - raise TypeError(f"Target tarball {target} has an invalid extension.") - - # Get the htar command. - htar = which("htar", required=True) - - has_rstprod = False - - for file in fileset: - if not has_rstprod: - if(os.stat(file).st_gid == gid_rstprod): - has_rstprod = True - - # Build the htar command - if verbose: - htar.add_default_arg('-cvf') - else: - htar.add_default_arg('-cf') - - htar.add_default_arg(target) - htar.add_default_arg(" ".join(filelist)) - - # Attempt to run htar - try: - htar(output=str) - except: - if has_rstprod: - # Attempt to change the group of the target file - try: - hsi.chgrp("rstprod", target) - hsi.chmod("640", target) - except: - hsi.rm(target, flags="-f") - raise OSError(f"Failed to change the group for a restricted data file.\n" - f"Please verify that {target} was deleted!!") - - raise OSError(f"htar failed to archive {target}.") - - if has_rstprod: - try: - hsi.chgrp("rstprod", target) - hsi.chmod("640", target) - except: - hsi.rm(target, flags="-f") - raise OSError(f"Failed to change the group for a restricted data file.\n" - f"Please verify that {target} was deleted!!") diff --git a/src/wxflow/fsutils.py b/src/wxflow/fsutils.py index 1653d5c..b336d13 100644 --- a/src/wxflow/fsutils.py +++ b/src/wxflow/fsutils.py @@ -2,7 +2,6 @@ import errno import os import shutil -import pathlib import grp from .executable import Executable, which From ad45216fab0a993cc7f66d75a3d52771500b42a8 Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Fri, 22 Mar 2024 19:39:42 +0000 Subject: [PATCH 04/32] Add htar capabilities. --- src/wxflow/__init__.py | 5 +- src/wxflow/archive_utils.py | 273 ------------------------------------ src/wxflow/fsutils.py | 18 --- src/wxflow/hsi.py | 188 +++++++++++++++++++++++++ src/wxflow/htar.py | 146 +++++++++++++++++++ 5 files changed, 337 insertions(+), 293 deletions(-) delete mode 100644 src/wxflow/archive_utils.py create mode 100644 src/wxflow/hsi.py create mode 100644 src/wxflow/htar.py diff --git a/src/wxflow/__init__.py b/src/wxflow/__init__.py index 72b5ef2..1494ae6 100644 --- a/src/wxflow/__init__.py +++ b/src/wxflow/__init__.py @@ -7,8 +7,9 @@ from .executable import CommandNotFoundError, Executable, which from .factory import Factory from .file_utils import FileHandler -from .archive_utils import Hsi -from .fsutils import chdir, cp, mkdir, mkdir_p, rm_p, rmdir +from .hsi import Hsi +from .htar import Htar +from .fsutils import chdir, cp, mkdir, mkdir_p, rm_p, rmdir, get_gid from .jinja import Jinja from .logger import Logger, logit from .sqlitedb import SQLiteDB, SQLiteDBError diff --git a/src/wxflow/archive_utils.py b/src/wxflow/archive_utils.py deleted file mode 100644 index ca2d2b6..0000000 --- a/src/wxflow/archive_utils.py +++ /dev/null @@ -1,273 +0,0 @@ -from logging import getLogger -from .executable import Executable, which - -from .fsutils import cp, mkdir, is_rstprod - -__all__ = ['Hsi'] - -logger = getLogger(__name__.split('.')[-1]) - -class Hsi: - """Class providing a set of functions to interact with the hsi utility. - - """ - - @staticmethod - def hsi(*args) -> None: - """Direct command builder function for hsi based on the input arguments. - - `args` should consist of a set of string arguments to send to hsi - For example, hsi.hsi("get","some_local_file : /some/hpss/file") will execute - hsi get some_local_file : /some/hpss/file - - """ - - cmd = which("hsi", required = True) - - for arg in args: - cmd.add_default_arg(arg) - - cmd() - - - @classmethod - def get(cls, source: str, target: str = "", hsi_flags: str = "") -> None: - """ Function to get a file from HPSS via hsi - - Parameters - ---------- - source : str - Full path location on HPSS of the file - - target : str - Location on the local machine to place the file. If not specified, - then the file will be placed in the current directory. - - hsi_flags : str - String of flags to send to hsi. See _parse_hsi_flags below for a - full list. - """ - args = () - - # Parse any hsi flags - if len(hsi_flags) > 0: - args = args + cls._parse_hsi_flags(hsi_flags) - - args = ("get",) - if len(target) == 0: - args = args + (source,) - else: - args = args + (target + " : " + source,) - cls.hsi(*args) - - - @classmethod - def put(cls, source: str, target: str, hsi_flags: str = "", listing_file : str = None) -> None: - """ Function to put a file onto HPSS via hsi - - Parameters - ---------- - source : str - Location on the local machine of the source file to send to HPSS. - - target : str - Full path of the target location of the file on HPSS. - - hsi_flags : str - String of flags to send to hsi. See _parse_hsi_flags below for a - full list. - """ - args = () - - # Parse any hsi flags - if len(hsi_flags) > 0: - args = args + cls._parse_hsi_flags(hsi_flags) - - args = args + ("put",) - args = args + (source + " : " + target,) - cls.hsi(*args) - - - @classmethod - def chmod(cls, mod: str, target: str, hsi_flags: str = "", flags: str = "") -> None: - """ Function to change the permissions of a file or directory on HPSS - - Parameters - ---------- - mod : str - Permissions to set for the file or directory, e.g. "640", "o+r", etc. - - target : str - Full path of the target location of the file on HPSS. - - hsi_flags : str - String of flags to send to hsi. See _parse_hsi_flags below for a - full list. - - flags : str - Flags to send to chmod. Valid flags are -d, -f, -h, -H, and -R. See - "hsi chmod -?" for more details. - """ - - args = () - - # Parse any hsi flags - if len(hsi_flags) > 0: - args = args + cls._parse_hsi_flags(hsi_flags) - - args = args + ("chmod",) - - if len(flags) > 0: - valid_flags = ["d", "h", "H", "R", "f"] - args = args + cls._parse_flags(flags, valid_flags) - - args = args + (mod,) - args = args + (target,) - cls.hsi(*args) - - - @classmethod - def chgrp(cls, group_name: str, target: str, hsi_flags: str = "", flags: str = "") -> None: - """ Function to change the group of a file or directory on HPSS - - Parameters - ---------- - group_name : str - The group to which ownership of the file/directory is to be set. - - target : str - Full path of the target location of the file on HPSS. - - hsi_flags : str - String of flags to send to hsi. See _parse_hsi_flags below for a - full list. - - flags : str - Flags to send to chmod. Valid flags are -h, -L, -H, and -R. See - "chgrp --help" for more details. - """ - - args = () - - # Parse any hsi flags - if len(hsi_flags) > 0: - args = args + cls._parse_hsi_flags(hsi_flags) - - args = args + ("chgrp",) - - if len(flags) > 0: - valid_flags = ["R", "h", "L", "H"] - args = args + cls._parse_flags(flags, valid_flags) - - args = args + (group_name,) - args = args + (target,) - cls.hsi(*args) - - - @classmethod - def rm(cls, target: str, hsi_flags: str = "", flags: str = "") -> None: - """ Function to delete a file or directory on HPSS via hsi - - Parameters - ---------- - target : str - Full path of the target location of the file on HPSS. - - hsi_flags : str - String of flags to send to hsi. See _parse_hsi_flags below for a - full list. - - flags : str - Flags to send to chmod. The only valid flag is -R (recursive). - """ - - args = () - - # Parse any hsi flags - if len(hsi_flags) > 0: - args = args + cls._parse_hsi_flags(hsi_flags) - - args = args + ("rm",) - - if len(flags) > 0: - valid_flags = ['R'] - args = args + cls._parse_flags(flags, valid_flags) - - args = args + (target,) - cls.hsi(*args) - - - @staticmethod - def _parse_flags(flags: str, valid_flags: list, valid_flags_with_arg: list = []) -> tuple: - """Expands the input flags into a tuple - - Inputs: - flags: str - String of flags to send to an hsi subcommand - valid_flags: list of strings - Set of valid flags to check against - valid_flags_with_arg: list of strings - Set of valid flags that accept arguments - - Return: - Tuple of input flags - - Error: - Returns ValueError if an invalid flag is in flags - """ - - expanded_flags = () - flags_and_args = flags.split(" ") - n_args = len(flags_and_args) - i = 0 - while i < n_args: - flag = flags_and_args[i] - # Check that the flag is valid - if flag.startswith("-"): - # E.g. "-p" or "-O output_file" - if len(flag) == 2: - flag = flag[1] - if flag in valid_flags_with_arg: - arg = flags_and_args[i+1] - i += 2 - expanded_flags = expanded_flags + ("-" + flag + arg,) - else: - expanded_flags = expanded_flags + ("-" + flag,) - i += 1 - # E.g. "-Ooutput_file" - else: - tmp_flag = flag[1] - arg = flag[2:] - flag = tmp_flag - i += 1 - expanded_flags = expanded_flags + ("-" + flag + arg,) - - else: - raise ValueError(f"One or more input flags '{flags}' is missing a '-', unable to parse") - - if flag not in valid_flags + valid_flags_with_arg: - raise ValueError(f"The input flags '{flags}' contains an invalid flag '{flag}'.") - - return expanded_flags - - - @classmethod - def _parse_hsi_flags(cls, flags: str) -> tuple: - """Expands the input flags into a tuple - - Inputs: - flags: str - String of flags to send to hsi. Valid flags are -a, -A, -c, - -d, -e, -h, -k, -l, -O, -q, -s, and -v. See hsi -q for details. - - Return: - Tuple of input flags - - Error: - Returns ValueError if an invalid flag is in flags - """ - - valid_flags = ["e", "p", "q", "v"] - valid_flags_with_arg = ["a", "A", "c", "d", "h", "k", "l", "O", "s"] - - return cls._parse_flags(flags, valid_flags, valid_flags_with_arg) diff --git a/src/wxflow/fsutils.py b/src/wxflow/fsutils.py index b336d13..758bb28 100644 --- a/src/wxflow/fsutils.py +++ b/src/wxflow/fsutils.py @@ -96,21 +96,3 @@ def get_gid(group_name: str): raise KeyError(f"{group_name} is not a valid group name.") return group_id - - -# Determine if a path (dir, file, link) belongs to the rstprod group -def is_rstprod(path: str) -> bool: - try: - rstprod_gid = get_gid("rstprod") - except KeyError: - # The rstprod group does not exist - return False - - if not os.path.exists(path): - print(f"WARNING '{path}' does not exist!") - return False - - if os.stat(path).st_gid == rstprod_gid: - return True - - return False diff --git a/src/wxflow/hsi.py b/src/wxflow/hsi.py new file mode 100644 index 0000000..32c7993 --- /dev/null +++ b/src/wxflow/hsi.py @@ -0,0 +1,188 @@ +from logging import getLogger +from .executable import Executable, which + +__all__ = ['Hsi'] + +logger = getLogger(__name__.split('.')[-1]) + + +class Hsi: + """Class providing a set of functions to interact with the hsi utility. + + """ + + @staticmethod + def hsi(*args) -> None: + """Direct command builder function for hsi based on the input arguments. + + `args` should consist of a set of string arguments to send to hsi + For example, hsi.hsi("get","some_local_file : /some/hpss/file") will execute + hsi get some_local_file : /some/hpss/file + + """ + + cmd = which("hsi", required = True) + + for arg in args: + cmd.add_default_arg(arg) + + cmd() + + + @classmethod + def get(cls, source: str, target: str = "", hsi_flags: str = "") -> None: + """ Function to get a file from HPSS via hsi + + Parameters + ---------- + source : str + Full path location on HPSS of the file + + target : str + Location on the local machine to place the file. If not specified, + then the file will be placed in the current directory. + + hsi_flags : str + String of flags to send to hsi. + """ + args = () + + # Parse any hsi flags + if len(hsi_flags) > 0: + args += tuple(hsi_flags.split(" ")) + + args += ("get",) + if len(target) == 0: + args += (source,) + else: + args += (target + " : " + source,) + cls.hsi(*args) + + + @classmethod + def put(cls, source: str, target: str, hsi_flags: str = "", listing_file : str = None) -> None: + """ Function to put a file onto HPSS via hsi + + Parameters + ---------- + source : str + Location on the local machine of the source file to send to HPSS. + + target : str + Full path of the target location of the file on HPSS. + + hsi_flags : str + String of flags to send to hsi. + """ + args = () + + # Parse any hsi flags + if len(hsi_flags) > 0: + args += tuple(hsi_flags.split(" ")) + + args += ("put",) + args += (source + " : " + target,) + cls.hsi(*args) + + + @classmethod + def chmod(cls, mod: str, target: str, hsi_flags: str = "", flags: str = "") -> None: + """ Function to change the permissions of a file or directory on HPSS + + Parameters + ---------- + mod : str + Permissions to set for the file or directory, e.g. "640", "o+r", etc. + + target : str + Full path of the target location of the file on HPSS. + + hsi_flags : str + String of flags to send to hsi. + + flags : str + Flags to send to chmod. Valid flags are -d, -f, -h, -H, and -R. See + "hsi chmod -?" for more details. + """ + + args = () + + # Parse any hsi flags + if len(hsi_flags) > 0: + args += tuple(hsi_flags.split(" ")) + + args += ("chmod",) + + if len(flags) > 0: + args += tuple(flags.split(" ")) + + args += (mod,) + args += (target,) + cls.hsi(*args) + + + @classmethod + def chgrp(cls, group_name: str, target: str, hsi_flags: str = "", flags: str = "") -> None: + """ Function to change the group of a file or directory on HPSS + + Parameters + ---------- + group_name : str + The group to which ownership of the file/directory is to be set. + + target : str + Full path of the target location of the file on HPSS. + + hsi_flags : str + String of flags to send to hsi. + + flags : str + Flags to send to chmod. Valid flags are -h, -L, -H, and -R. See + "chgrp --help" for more details. + """ + + args = () + + # Parse any hsi flags + if len(hsi_flags) > 0: + args += tuple(hsi_flags.split(" ")) + + args += ("chgrp",) + + if len(flags) > 0: + args += tuple(flags.split(" ")) + + args += (group_name,) + args += (target,) + cls.hsi(*args) + + + @classmethod + def rm(cls, target: str, hsi_flags: str = "", flags: str = "") -> None: + """ Function to delete a file or directory on HPSS via hsi + + Parameters + ---------- + target : str + Full path of the target location of the file on HPSS. + + hsi_flags : str + String of flags to send to hsi. + + flags : str + Flags to send to chmod. The only valid flag is -R (recursive). + """ + + args = () + + # Parse any hsi flags + if len(hsi_flags) > 0: + args += tuple(hsi_flags.split(" ")) + + args += ("rm",) + + if len(flags) > 0: + args += tuple(flags.split(" ")) + + args += (target,) + cls.hsi(*args) diff --git a/src/wxflow/htar.py b/src/wxflow/htar.py new file mode 100644 index 0000000..722ee11 --- /dev/null +++ b/src/wxflow/htar.py @@ -0,0 +1,146 @@ +from logging import getLogger +from .executable import Executable, which + +__all__ = ['Htar'] + +logger = getLogger(__name__.split('.')[-1]) + + +class Htar: + """Class providing a set of functions to interact with the htar utility. + + """ + + @staticmethod + def htar(*args) -> None: + """Direct command builder function for htar based on the input arguments. + + `args` should consist of a set of string arguments to send to htar + For example, Htar.htar("-cvf","/path/to/hpss/archive.tar", "") will execute + htar -cvf /path/to/hpss/archive.tar + + """ + + cmd = which("htar", required = True) + + for arg in args: + cmd.add_default_arg(arg) + + cmd() + + + @classmethod + def create(cls, tarball: str, fileset: list, flags: str = "") -> None: + """ Function to write an archive to HPSS + + Parameters + ---------- + flags : str + String of flags to send to htar. + + tarball : str + Full path location on HPSS to create the archive. + + fileset : list + List containing filenames, patterns, or directories to archive + """ + args = ("-c",) + + # Parse any htar flags + if len(flags) > 0: + args += tuple(flags.split(" ")) + + args += ("-f", tarball,) + tuple(fileset) + + cls.htar(*args) + + + @classmethod + def cvf(cls, tarball: str, fileset: list) -> None: + """ Function to write an archive to HPSS verbosely (without flags). + + Parameters + ---------- + tarball : str + Full path location on HPSS to create the archive. + + fileset : list + List containing filenames, patterns, or directories to archive + """ + cls.create(tarball, fileset, flags = "-v") + + + @classmethod + def extract(cls, tarball: str, fileset: list = [], flags: str = "") -> None: + """ Function to extract an archive from HPSS via htar + + Parameters + ---------- + flags : str + String of flags to send to htar. + + tarball : str + Full path location of an archive on HPSS to extract from. + + fileset : list + List containing filenames, patterns, or directories to extract from + the archive. If empty, then all files will be extracted. + """ + args = ("-x",) + + # Parse any htar flags + if len(flags) > 0: + args += tuple(flags.split(" ")) + + args += ("-f", tarball,) + + if len(fileset) > 0: + args += tuple(fileset) + + cls.htar(*args) + + + @classmethod + def xvf(cls, tarball: str = "", fileset: list = []) -> None: + """ Function to extract an archive from HPSS verbosely (without flags). + + Parameters + ---------- + tarball : str + Full path location of an archive on HPSS to extract from. + + fileset : list + List containing filenames, patterns, or directories to extract from + the archive. If empty, then all files will be extracted. + """ + cls.extract(tarball, fileset, flags = "-v") + + + @classmethod + def tell(cls, tarball: str, flags: str = "", fileset: list = []) -> None: + """ Function to list the contents of an archive on HPSS + + Parameters + ---------- + flags : str + String of flags to send to htar. + + tarball : str + Full path location on HPSS to list the contents of. + + fileset : list + List containing filenames, patterns, or directories to list. + If empty, then all files will be listed. + """ + args = ("-t",) + + # Parse any htar flags + if len(flags) > 0: + args += tuple(flags.split(" ")) + + args += ("-f", tarball,) + + if len(fileset) > 0: + args += tuple(fileset) + + cls.htar(*args) From d54ea50c837749f22563d40d3e01771a1bbca10f Mon Sep 17 00:00:00 2001 From: David Huber Date: Mon, 1 Apr 2024 11:16:15 -0400 Subject: [PATCH 05/32] Refactor hsi and htar utils. (untested) --- src/wxflow/fsutils.py | 10 +- src/wxflow/hsi.py | 319 +++++++++++++++++++++++++----------------- src/wxflow/htar.py | 189 ++++++++++++------------- 3 files changed, 292 insertions(+), 226 deletions(-) diff --git a/src/wxflow/fsutils.py b/src/wxflow/fsutils.py index 758bb28..5cc57e1 100644 --- a/src/wxflow/fsutils.py +++ b/src/wxflow/fsutils.py @@ -5,7 +5,7 @@ import grp from .executable import Executable, which -__all__ = ['mkdir', 'mkdir_p', 'rmdir', 'chdir', 'rm_p', 'cp', 'get_gid'] +__all__ = ['mkdir', 'mkdir_p', 'rmdir', 'chdir', 'rm_p', 'cp', 'get_gid', 'chgrp'] def mkdir_p(path): try: @@ -96,3 +96,11 @@ def get_gid(group_name: str): raise KeyError(f"{group_name} is not a valid group name.") return group_id + + +# Change the group of a target file or directory +def f_chgrp(target, group_name, recursive=False): + #TODO add recursive option + gid = get_gid(group_name) + uid = os.stat(target).st_uid + os.chown(target, uid, gid) diff --git a/src/wxflow/hsi.py b/src/wxflow/hsi.py index 32c7993..dbdb1b6 100644 --- a/src/wxflow/hsi.py +++ b/src/wxflow/hsi.py @@ -1,188 +1,253 @@ from logging import getLogger from .executable import Executable, which -__all__ = ['Hsi'] +__all__ = ['hsi', 'get', 'put', 'ls', 'chmod', 'chgrp', 'rm', 'file_exists'] logger = getLogger(__name__.split('.')[-1]) -class Hsi: - """Class providing a set of functions to interact with the hsi utility. +def hsi(*args) -> str: + """Direct command builder function for hsi based on the input arguments. + `args` should consist of a set of string arguments to send to hsi + For example, hsi("get","some_local_file : /some/hpss/file") will execute + hsi get some_local_file : /some/hpss/file + + Return: str + Concatenated output and error of the hsi command. + """ + + cmd = which("hsi", required = True) + + for arg in args: + cmd.add_default_arg(arg) + + output = cmd(output = str.split, error = str.split) + + return output + + +def get(source: str, target: str = "", hsi_flags: str = "") -> str: + """ Function to get a file from HPSS via hsi + + Parameters + ---------- + source : str + Full path location on HPSS of the file + + target : str + Location on the local machine to place the file. If not specified, + then the file will be placed in the current directory. + + hsi_flags : str + String of flags to send to hsi. """ + args = [] + + # Parse any hsi flags + if len(hsi_flags) > 0: + args.extend(hsi_flags.split(" ")) - @staticmethod - def hsi(*args) -> None: - """Direct command builder function for hsi based on the input arguments. + args.append("get") + if len(target) == 0: + args.append(source) + else: + args.append(target + " : " + source) - `args` should consist of a set of string arguments to send to hsi - For example, hsi.hsi("get","some_local_file : /some/hpss/file") will execute - hsi get some_local_file : /some/hpss/file + output = hsi(*args) - """ + return output - cmd = which("hsi", required = True) - for arg in args: - cmd.add_default_arg(arg) +def put(source: str, target: str, hsi_flags: str = "", listing_file : str = None) -> str: + """ Function to put a file onto HPSS via hsi - cmd() + Parameters + ---------- + source : str + Location on the local machine of the source file to send to HPSS. + target : str + Full path of the target location of the file on HPSS. - @classmethod - def get(cls, source: str, target: str = "", hsi_flags: str = "") -> None: - """ Function to get a file from HPSS via hsi + hsi_flags : str + String of flags to send to hsi. + """ + args = [] - Parameters - ---------- - source : str - Full path location on HPSS of the file + # Parse any hsi flags + if len(hsi_flags) > 0: + args.extend(hsi_flags.split(" ")) - target : str - Location on the local machine to place the file. If not specified, - then the file will be placed in the current directory. + args.append("put") + args.append(source + " : " + target) + output = hsi(*args) - hsi_flags : str - String of flags to send to hsi. - """ - args = () + return output - # Parse any hsi flags - if len(hsi_flags) > 0: - args += tuple(hsi_flags.split(" ")) - args += ("get",) - if len(target) == 0: - args += (source,) - else: - args += (target + " : " + source,) - cls.hsi(*args) +def chmod(mod: str, target: str, hsi_flags: str = "", chmod_flags: str = "") -> str: + """ Function to change the permissions of a file or directory on HPSS + Parameters + ---------- + mod : str + Permissions to set for the file or directory, e.g. "640", "o+r", etc. - @classmethod - def put(cls, source: str, target: str, hsi_flags: str = "", listing_file : str = None) -> None: - """ Function to put a file onto HPSS via hsi + target : str + Full path of the target location of the file on HPSS. - Parameters - ---------- - source : str - Location on the local machine of the source file to send to HPSS. + hsi_flags : str + String of flags to send to hsi. - target : str - Full path of the target location of the file on HPSS. + flags : str + Flags to send to chmod. Valid flags are -d, -f, -h, -H, and -R. See + "hsi chmod -?" for more details. + """ - hsi_flags : str - String of flags to send to hsi. - """ - args = () + args = [] - # Parse any hsi flags - if len(hsi_flags) > 0: - args += tuple(hsi_flags.split(" ")) + # Parse any hsi flags + if len(hsi_flags) > 0: + args.extend(hsi_flags.split(" ")) - args += ("put",) - args += (source + " : " + target,) - cls.hsi(*args) + args.append("chmod") + if len(flags) > 0: + args.extend(chmod_flags.split(" ")) - @classmethod - def chmod(cls, mod: str, target: str, hsi_flags: str = "", flags: str = "") -> None: - """ Function to change the permissions of a file or directory on HPSS + args.append(mod) + args.append(target) + output = hsi(*args) - Parameters - ---------- - mod : str - Permissions to set for the file or directory, e.g. "640", "o+r", etc. + return output - target : str - Full path of the target location of the file on HPSS. - hsi_flags : str - String of flags to send to hsi. +def chgrp(group_name: str, target: str, hsi_flags: str = "", chgrp_flags: str = "") -> str: + """ Function to change the group of a file or directory on HPSS - flags : str - Flags to send to chmod. Valid flags are -d, -f, -h, -H, and -R. See - "hsi chmod -?" for more details. - """ + Parameters + ---------- + group_name : str + The group to which ownership of the file/directory is to be set. - args = () + target : str + Full path of the target location of the file on HPSS. - # Parse any hsi flags - if len(hsi_flags) > 0: - args += tuple(hsi_flags.split(" ")) + hsi_flags : str + String of flags to send to hsi. - args += ("chmod",) + flags : str + Flags to send to chmod. Valid flags are -h, -L, -H, and -R. See + "chgrp --help" for more details. + """ - if len(flags) > 0: - args += tuple(flags.split(" ")) + args = [] - args += (mod,) - args += (target,) - cls.hsi(*args) + # Parse any hsi flags + if len(hsi_flags) > 0: + args.extend(hsi_flags.split(" ")) + args.append("chgrp") - @classmethod - def chgrp(cls, group_name: str, target: str, hsi_flags: str = "", flags: str = "") -> None: - """ Function to change the group of a file or directory on HPSS + if len(flags) > 0: + args.extend(chgrp_flags.split(" ")) - Parameters - ---------- - group_name : str - The group to which ownership of the file/directory is to be set. + args.append(group_name) + args.append(target) + output = hsi(*args) - target : str - Full path of the target location of the file on HPSS. + return output - hsi_flags : str - String of flags to send to hsi. - flags : str - Flags to send to chmod. Valid flags are -h, -L, -H, and -R. See - "chgrp --help" for more details. - """ +def rm(target: str, hsi_flags: str = "", rm_flags: str = "") -> str: + """ Function to delete a file or directory on HPSS via hsi - args = () + Parameters + ---------- + target : str + Full path of the target location of the file on HPSS. - # Parse any hsi flags - if len(hsi_flags) > 0: - args += tuple(hsi_flags.split(" ")) + hsi_flags : str + String of flags to send to hsi. - args += ("chgrp",) + flags : str + Flags to send to chmod. The only valid flag is -R (recursive). + """ - if len(flags) > 0: - args += tuple(flags.split(" ")) + args = [] - args += (group_name,) - args += (target,) - cls.hsi(*args) + # Parse any hsi flags + if len(hsi_flags) > 0: + args.extend(hsi_flags.split(" ")) + args.append("rm") - @classmethod - def rm(cls, target: str, hsi_flags: str = "", flags: str = "") -> None: - """ Function to delete a file or directory on HPSS via hsi + if len(flags) > 0: + args.extend(rm_flags.split(" ")) - Parameters - ---------- - target : str - Full path of the target location of the file on HPSS. + args.append(target) + output = hsi(*args) - hsi_flags : str - String of flags to send to hsi. + return output - flags : str - Flags to send to chmod. The only valid flag is -R (recursive). - """ - args = () +def ls(target: str, hsi_flags: str = "", ls_flags: str = "") -> str: + """ Function to list files/directories on HPSS via hsi + + Parameters + ---------- + target : str + Full path of the target location on HPSS. + + hsi_flags : str + String of flags to send to hsi. + + flags : str + Flags to send to ls. + """ + + args = [] + + # Parse any hsi flags + if len(hsi_flags) > 0: + args.extend(hsi_flags.split(" ")) + + args.append("ls") + + if len(ls_flags) > 0: + args.extend(flags.split(" ")) + + args.append(target) + output = hsi(*args) + + return output + + +def file_exists(target: str) -> bool: + """ Function to list files/directories on HPSS via hsi + + Parameters + ---------- + target : str + Full path of the target location on HPSS. + + Return: bool + True if the file exists on HPSS. + """ - # Parse any hsi flags - if len(hsi_flags) > 0: - args += tuple(hsi_flags.split(" ")) + cmd = which("hsi", required = True) - args += ("rm",) + for arg in ["ls", "target"]: + cmd.add_default_arg(arg) - if len(flags) > 0: - args += tuple(flags.split(" ")) + # Do not exit if the file is not found; do not pipe output to stdout + output = cmd(output = str, error = str, ignore_errors=[64]) - args += (target,) - cls.hsi(*args) + if "HPSS_ENOENT" in output: + return False + # Catch wildcards + elif f"Warning: No matching names located for '{target}'" in target + return False + else: + return True diff --git a/src/wxflow/htar.py b/src/wxflow/htar.py index 722ee11..b24e8d5 100644 --- a/src/wxflow/htar.py +++ b/src/wxflow/htar.py @@ -6,141 +6,134 @@ logger = getLogger(__name__.split('.')[-1]) -class Htar: - """Class providing a set of functions to interact with the htar utility. +"""Class providing a set of functions to interact with the htar utility. - """ +""" + +def htar(*args) -> None: + """Direct command builder function for htar based on the input arguments. - @staticmethod - def htar(*args) -> None: - """Direct command builder function for htar based on the input arguments. + `args` should consist of a set of string arguments to send to htar + For example, Htar.htar("-cvf","/path/to/hpss/archive.tar", "") will execute + htar -cvf /path/to/hpss/archive.tar - `args` should consist of a set of string arguments to send to htar - For example, Htar.htar("-cvf","/path/to/hpss/archive.tar", "") will execute - htar -cvf /path/to/hpss/archive.tar + """ - """ + cmd = which("htar", required = True) - cmd = which("htar", required = True) + for arg in args: + cmd.add_default_arg(arg) - for arg in args: - cmd.add_default_arg(arg) + cmd() - cmd() +def create(tarball: str, fileset: list, flags: str = "") -> None: + """ Function to write an archive to HPSS - @classmethod - def create(cls, tarball: str, fileset: list, flags: str = "") -> None: - """ Function to write an archive to HPSS + Parameters + ---------- + flags : str + String of flags to send to htar. - Parameters - ---------- - flags : str - String of flags to send to htar. + tarball : str + Full path location on HPSS to create the archive. - tarball : str - Full path location on HPSS to create the archive. + fileset : list + List containing filenames, patterns, or directories to archive + """ + args = ("-c",) - fileset : list - List containing filenames, patterns, or directories to archive - """ - args = ("-c",) + # Parse any htar flags + if len(flags) > 0: + args += tuple(flags.split(" ")) - # Parse any htar flags - if len(flags) > 0: - args += tuple(flags.split(" ")) + args += ("-f", tarball,) + tuple(fileset) - args += ("-f", tarball,) + tuple(fileset) + htar(*args) - cls.htar(*args) +def cvf(tarball: str, fileset: list) -> None: + """ Function to write an archive to HPSS verbosely (without flags). - @classmethod - def cvf(cls, tarball: str, fileset: list) -> None: - """ Function to write an archive to HPSS verbosely (without flags). + Parameters + ---------- + tarball : str + Full path location on HPSS to create the archive. - Parameters - ---------- - tarball : str - Full path location on HPSS to create the archive. + fileset : list + List containing filenames, patterns, or directories to archive + """ + create(tarball, fileset, flags = "-v") - fileset : list - List containing filenames, patterns, or directories to archive - """ - cls.create(tarball, fileset, flags = "-v") +def extract(tarball: str, fileset: list = [], flags: str = "") -> None: + """ Function to extract an archive from HPSS via htar - @classmethod - def extract(cls, tarball: str, fileset: list = [], flags: str = "") -> None: - """ Function to extract an archive from HPSS via htar + Parameters + ---------- + flags : str + String of flags to send to htar. - Parameters - ---------- - flags : str - String of flags to send to htar. + tarball : str + Full path location of an archive on HPSS to extract from. - tarball : str - Full path location of an archive on HPSS to extract from. + fileset : list + List containing filenames, patterns, or directories to extract from + the archive. If empty, then all files will be extracted. + """ + args = ("-x",) - fileset : list - List containing filenames, patterns, or directories to extract from - the archive. If empty, then all files will be extracted. - """ - args = ("-x",) + # Parse any htar flags + if len(flags) > 0: + args += tuple(flags.split(" ")) - # Parse any htar flags - if len(flags) > 0: - args += tuple(flags.split(" ")) + args += ("-f", tarball,) - args += ("-f", tarball,) - - if len(fileset) > 0: - args += tuple(fileset) + if len(fileset) > 0: + args += tuple(fileset) - cls.htar(*args) + htar(*args) - @classmethod - def xvf(cls, tarball: str = "", fileset: list = []) -> None: - """ Function to extract an archive from HPSS verbosely (without flags). +def xvf(tarball: str = "", fileset: list = []) -> None: + """ Function to extract an archive from HPSS verbosely (without flags). - Parameters - ---------- - tarball : str - Full path location of an archive on HPSS to extract from. + Parameters + ---------- + tarball : str + Full path location of an archive on HPSS to extract from. - fileset : list - List containing filenames, patterns, or directories to extract from - the archive. If empty, then all files will be extracted. - """ - cls.extract(tarball, fileset, flags = "-v") + fileset : list + List containing filenames, patterns, or directories to extract from + the archive. If empty, then all files will be extracted. + """ + extract(tarball, fileset, flags = "-v") - @classmethod - def tell(cls, tarball: str, flags: str = "", fileset: list = []) -> None: - """ Function to list the contents of an archive on HPSS +def tell(tarball: str, flags: str = "", fileset: list = []) -> None: + """ Function to list the contents of an archive on HPSS - Parameters - ---------- - flags : str - String of flags to send to htar. + Parameters + ---------- + flags : str + String of flags to send to htar. - tarball : str - Full path location on HPSS to list the contents of. + tarball : str + Full path location on HPSS to list the contents of. - fileset : list - List containing filenames, patterns, or directories to list. - If empty, then all files will be listed. - """ - args = ("-t",) + fileset : list + List containing filenames, patterns, or directories to list. + If empty, then all files will be listed. + """ + args = ("-t",) - # Parse any htar flags - if len(flags) > 0: - args += tuple(flags.split(" ")) + # Parse any htar flags + if len(flags) > 0: + args += tuple(flags.split(" ")) - args += ("-f", tarball,) + args += ("-f", tarball,) - if len(fileset) > 0: - args += tuple(fileset) + if len(fileset) > 0: + args += tuple(fileset) - cls.htar(*args) + htar(*args) From d43b9119b3e7789977fc897ab0b7942cfc812a3c Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Mon, 1 Apr 2024 18:41:47 +0000 Subject: [PATCH 06/32] Fix some hsi bugs. --- src/wxflow/__init__.py | 5 ++--- src/wxflow/hsi.py | 30 ++++++++++++++---------------- 2 files changed, 16 insertions(+), 19 deletions(-) diff --git a/src/wxflow/__init__.py b/src/wxflow/__init__.py index c1f6f7a..afc2222 100644 --- a/src/wxflow/__init__.py +++ b/src/wxflow/__init__.py @@ -7,10 +7,9 @@ from .executable import CommandNotFoundError, Executable, ProcessError, which from .factory import Factory from .file_utils import FileHandler -from .hsi import Hsi -from .htar import Htar from .fsutils import chdir, cp, mkdir, mkdir_p, rm_p, rmdir, get_gid -from .jinja import Jinja +from . import hsi, htar +#from .jinja import Jinja from .logger import Logger, logit from .sqlitedb import SQLiteDB, SQLiteDBError from .task import Task diff --git a/src/wxflow/hsi.py b/src/wxflow/hsi.py index dbdb1b6..2235c5e 100644 --- a/src/wxflow/hsi.py +++ b/src/wxflow/hsi.py @@ -100,9 +100,8 @@ def chmod(mod: str, target: str, hsi_flags: str = "", chmod_flags: str = "") -> hsi_flags : str String of flags to send to hsi. - flags : str - Flags to send to chmod. Valid flags are -d, -f, -h, -H, and -R. See - "hsi chmod -?" for more details. + chmod_flags : str + Flags to send to chmod. See "hsi chmod -?" for more details. """ args = [] @@ -113,7 +112,7 @@ def chmod(mod: str, target: str, hsi_flags: str = "", chmod_flags: str = "") -> args.append("chmod") - if len(flags) > 0: + if len(chmod_flags) > 0: args.extend(chmod_flags.split(" ")) args.append(mod) @@ -137,9 +136,8 @@ def chgrp(group_name: str, target: str, hsi_flags: str = "", chgrp_flags: str = hsi_flags : str String of flags to send to hsi. - flags : str - Flags to send to chmod. Valid flags are -h, -L, -H, and -R. See - "chgrp --help" for more details. + chgrp_flags : str + Flags to send to chgrp. See "hsi chgrp -?" for more details. """ args = [] @@ -150,7 +148,7 @@ def chgrp(group_name: str, target: str, hsi_flags: str = "", chgrp_flags: str = args.append("chgrp") - if len(flags) > 0: + if len(chgrp_flags) > 0: args.extend(chgrp_flags.split(" ")) args.append(group_name) @@ -171,8 +169,8 @@ def rm(target: str, hsi_flags: str = "", rm_flags: str = "") -> str: hsi_flags : str String of flags to send to hsi. - flags : str - Flags to send to chmod. The only valid flag is -R (recursive). + rm_flags : str + Flags to send to rm. See "hsi rm -?" for more details. """ args = [] @@ -183,7 +181,7 @@ def rm(target: str, hsi_flags: str = "", rm_flags: str = "") -> str: args.append("rm") - if len(flags) > 0: + if len(rm_flags) > 0: args.extend(rm_flags.split(" ")) args.append(target) @@ -203,8 +201,8 @@ def ls(target: str, hsi_flags: str = "", ls_flags: str = "") -> str: hsi_flags : str String of flags to send to hsi. - flags : str - Flags to send to ls. + ls_flags : str + Flags to send to ls. See "hsi ls -?" for more details. """ args = [] @@ -216,7 +214,7 @@ def ls(target: str, hsi_flags: str = "", ls_flags: str = "") -> str: args.append("ls") if len(ls_flags) > 0: - args.extend(flags.split(" ")) + args.extend(ls_flags.split(" ")) args.append(target) output = hsi(*args) @@ -238,7 +236,7 @@ def file_exists(target: str) -> bool: cmd = which("hsi", required = True) - for arg in ["ls", "target"]: + for arg in ["ls", target]: cmd.add_default_arg(arg) # Do not exit if the file is not found; do not pipe output to stdout @@ -247,7 +245,7 @@ def file_exists(target: str) -> bool: if "HPSS_ENOENT" in output: return False # Catch wildcards - elif f"Warning: No matching names located for '{target}'" in target + elif f"Warning: No matching names located for '{target}'" in output: return False else: return True From 453f6ac421d6d8731724dec6e93ce4673b0c2b30 Mon Sep 17 00:00:00 2001 From: David Huber Date: Tue, 2 Apr 2024 17:51:25 +0000 Subject: [PATCH 07/32] Add mkdir function to hsi. --- src/wxflow/hsi.py | 40 ++++++++++++++++++++++++++++++++++++---- 1 file changed, 36 insertions(+), 4 deletions(-) diff --git a/src/wxflow/hsi.py b/src/wxflow/hsi.py index 2235c5e..59519a6 100644 --- a/src/wxflow/hsi.py +++ b/src/wxflow/hsi.py @@ -1,7 +1,7 @@ from logging import getLogger from .executable import Executable, which -__all__ = ['hsi', 'get', 'put', 'ls', 'chmod', 'chgrp', 'rm', 'file_exists'] +__all__ = ['hsi', 'get', 'put', 'ls', 'chmod', 'chgrp', 'rm', 'mkdir', 'file_exists'] logger = getLogger(__name__.split('.')[-1]) @@ -190,6 +190,38 @@ def rm(target: str, hsi_flags: str = "", rm_flags: str = "") -> str: return output +def mkdir(target: str, hsi_flags: str = "", mkdir_flags: str = "") -> str: + """ Function to delete a file or directory on HPSS via hsi + + Parameters + ---------- + target : str + Full path of the target location of the file on HPSS. + + hsi_flags : str + String of flags to send to hsi. + + mkdir_flags : str + Flags to send to mkdir (-p is assumed). See "hsi mkdir -?" for more details. + """ + + args = [] + + # Parse any hsi flags + if len(hsi_flags) > 0: + args.extend(hsi_flags.split(" ")) + + args.extend(["mkdir", "-p"]) + + if len(mkdir_flags) > 0: + args.extend(mkdir_flags.split(" ")) + + args.append(target) + output = hsi(*args) + + return output + + def ls(target: str, hsi_flags: str = "", ls_flags: str = "") -> str: """ Function to list files/directories on HPSS via hsi @@ -222,8 +254,8 @@ def ls(target: str, hsi_flags: str = "", ls_flags: str = "") -> str: return output -def file_exists(target: str) -> bool: - """ Function to list files/directories on HPSS via hsi +def exists(target: str) -> bool: + """ Function to test the existence of a file/directory/glob on HPSS Parameters ---------- @@ -231,7 +263,7 @@ def file_exists(target: str) -> bool: Full path of the target location on HPSS. Return: bool - True if the file exists on HPSS. + True if the target exists on HPSS. """ cmd = which("hsi", required = True) From 3f9a529982ecd4598cab9002dbe12863a4d1948f Mon Sep 17 00:00:00 2001 From: David Huber Date: Tue, 2 Apr 2024 18:04:10 +0000 Subject: [PATCH 08/32] Rename chgrp function --- src/wxflow/__init__.py | 4 ++-- src/wxflow/fsutils.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/wxflow/__init__.py b/src/wxflow/__init__.py index afc2222..136f95d 100644 --- a/src/wxflow/__init__.py +++ b/src/wxflow/__init__.py @@ -7,9 +7,9 @@ from .executable import CommandNotFoundError, Executable, ProcessError, which from .factory import Factory from .file_utils import FileHandler -from .fsutils import chdir, cp, mkdir, mkdir_p, rm_p, rmdir, get_gid +from .fsutils import chdir, cp, mkdir, mkdir_p, rm_p, rmdir, get_gid, chgrp from . import hsi, htar -#from .jinja import Jinja +from .jinja import Jinja from .logger import Logger, logit from .sqlitedb import SQLiteDB, SQLiteDBError from .task import Task diff --git a/src/wxflow/fsutils.py b/src/wxflow/fsutils.py index 5cc57e1..d4654c1 100644 --- a/src/wxflow/fsutils.py +++ b/src/wxflow/fsutils.py @@ -99,7 +99,7 @@ def get_gid(group_name: str): # Change the group of a target file or directory -def f_chgrp(target, group_name, recursive=False): +def chgrp(target, group_name, recursive=False): #TODO add recursive option gid = get_gid(group_name) uid = os.stat(target).st_uid From 9997b85be9f0acefb12bed4bb81d2e06a8352631 Mon Sep 17 00:00:00 2001 From: David Huber Date: Tue, 2 Apr 2024 18:05:54 +0000 Subject: [PATCH 09/32] Remove unused executable module from fsutils --- src/wxflow/fsutils.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/wxflow/fsutils.py b/src/wxflow/fsutils.py index d4654c1..fd0466a 100644 --- a/src/wxflow/fsutils.py +++ b/src/wxflow/fsutils.py @@ -3,7 +3,6 @@ import os import shutil import grp -from .executable import Executable, which __all__ = ['mkdir', 'mkdir_p', 'rmdir', 'chdir', 'rm_p', 'cp', 'get_gid', 'chgrp'] From ac337ac3adbfcfe876bfdb33e0204d1e854867dc Mon Sep 17 00:00:00 2001 From: David Huber Date: Tue, 2 Apr 2024 18:40:03 +0000 Subject: [PATCH 10/32] Fix pynorms issues. --- src/wxflow/__init__.py | 2 +- src/wxflow/fsutils.py | 8 +++++--- src/wxflow/hsi.py | 29 ++++++++++++++++------------- src/wxflow/htar.py | 22 +++++++++------------- 4 files changed, 31 insertions(+), 30 deletions(-) diff --git a/src/wxflow/__init__.py b/src/wxflow/__init__.py index 136f95d..31e36c5 100644 --- a/src/wxflow/__init__.py +++ b/src/wxflow/__init__.py @@ -1,5 +1,6 @@ import os +from . import hsi, htar from .attrdict import AttrDict from .configuration import (Configuration, cast_as_dtype, cast_strdict_as_dtypedict) @@ -8,7 +9,6 @@ from .factory import Factory from .file_utils import FileHandler from .fsutils import chdir, cp, mkdir, mkdir_p, rm_p, rmdir, get_gid, chgrp -from . import hsi, htar from .jinja import Jinja from .logger import Logger, logit from .sqlitedb import SQLiteDB, SQLiteDBError diff --git a/src/wxflow/fsutils.py b/src/wxflow/fsutils.py index fd0466a..9dee60d 100644 --- a/src/wxflow/fsutils.py +++ b/src/wxflow/fsutils.py @@ -1,10 +1,12 @@ import contextlib import errno +import grp import os import shutil -import grp -__all__ = ['mkdir', 'mkdir_p', 'rmdir', 'chdir', 'rm_p', 'cp', 'get_gid', 'chgrp'] +__all__ = ['mkdir', 'mkdir_p', 'rmdir', 'chdir', 'rm_p', 'cp', + 'get_gid', 'chgrp'] + def mkdir_p(path): try: @@ -99,7 +101,7 @@ def get_gid(group_name: str): # Change the group of a target file or directory def chgrp(target, group_name, recursive=False): - #TODO add recursive option + # TODO add recursive option gid = get_gid(group_name) uid = os.stat(target).st_uid os.chown(target, uid, gid) diff --git a/src/wxflow/hsi.py b/src/wxflow/hsi.py index 59519a6..e37cfc2 100644 --- a/src/wxflow/hsi.py +++ b/src/wxflow/hsi.py @@ -1,9 +1,7 @@ -from logging import getLogger from .executable import Executable, which -__all__ = ['hsi', 'get', 'put', 'ls', 'chmod', 'chgrp', 'rm', 'mkdir', 'file_exists'] - -logger = getLogger(__name__.split('.')[-1]) +__all__ = ['hsi', 'get', 'put', 'ls', 'chmod', 'chgrp', 'rm', + 'mkdir', 'file_exists'] def hsi(*args) -> str: @@ -17,12 +15,12 @@ def hsi(*args) -> str: Concatenated output and error of the hsi command. """ - cmd = which("hsi", required = True) + cmd = which("hsi", required=True) for arg in args: cmd.add_default_arg(arg) - output = cmd(output = str.split, error = str.split) + output = cmd(output=str.split, error=str.split) return output @@ -59,7 +57,8 @@ def get(source: str, target: str = "", hsi_flags: str = "") -> str: return output -def put(source: str, target: str, hsi_flags: str = "", listing_file : str = None) -> str: +def put(source: str, target: str, hsi_flags: str = "", + listing_file: str = None) -> str: """ Function to put a file onto HPSS via hsi Parameters @@ -86,13 +85,15 @@ def put(source: str, target: str, hsi_flags: str = "", listing_file : str = None return output -def chmod(mod: str, target: str, hsi_flags: str = "", chmod_flags: str = "") -> str: +def chmod(mod: str, target: str, hsi_flags: str = "", + chmod_flags: str = "") -> str: """ Function to change the permissions of a file or directory on HPSS Parameters ---------- mod : str - Permissions to set for the file or directory, e.g. "640", "o+r", etc. + Permissions to set for the file or directory, + e.g. "640", "o+r", etc. target : str Full path of the target location of the file on HPSS. @@ -122,7 +123,8 @@ def chmod(mod: str, target: str, hsi_flags: str = "", chmod_flags: str = "") -> return output -def chgrp(group_name: str, target: str, hsi_flags: str = "", chgrp_flags: str = "") -> str: +def chgrp(group_name: str, target: str, hsi_flags: str = "", + chgrp_flags: str = "") -> str: """ Function to change the group of a file or directory on HPSS Parameters @@ -202,7 +204,8 @@ def mkdir(target: str, hsi_flags: str = "", mkdir_flags: str = "") -> str: String of flags to send to hsi. mkdir_flags : str - Flags to send to mkdir (-p is assumed). See "hsi mkdir -?" for more details. + Flags to send to mkdir (-p is assumed). + See "hsi mkdir -?" for more details. """ args = [] @@ -266,13 +269,13 @@ def exists(target: str) -> bool: True if the target exists on HPSS. """ - cmd = which("hsi", required = True) + cmd = which("hsi", required=True) for arg in ["ls", target]: cmd.add_default_arg(arg) # Do not exit if the file is not found; do not pipe output to stdout - output = cmd(output = str, error = str, ignore_errors=[64]) + output = cmd(output=str, error=str, ignore_errors=[64]) if "HPSS_ENOENT" in output: return False diff --git a/src/wxflow/htar.py b/src/wxflow/htar.py index b24e8d5..8e7f721 100644 --- a/src/wxflow/htar.py +++ b/src/wxflow/htar.py @@ -1,25 +1,21 @@ -from logging import getLogger from .executable import Executable, which -__all__ = ['Htar'] +__all__ = ['htar', 'create', 'cvf', 'extract', 'xvf', 'tell'] -logger = getLogger(__name__.split('.')[-1]) - - -"""Class providing a set of functions to interact with the htar utility. - -""" def htar(*args) -> None: - """Direct command builder function for htar based on the input arguments. + """ + Direct command builder function for htar based on the input arguments. `args` should consist of a set of string arguments to send to htar - For example, Htar.htar("-cvf","/path/to/hpss/archive.tar", "") will execute + For example, + htar.htar("-cvf","/path/to/hpss/archive.tar", "") + will execute htar -cvf /path/to/hpss/archive.tar """ - cmd = which("htar", required = True) + cmd = which("htar", required=True) for arg in args: cmd.add_default_arg(arg) @@ -63,7 +59,7 @@ def cvf(tarball: str, fileset: list) -> None: fileset : list List containing filenames, patterns, or directories to archive """ - create(tarball, fileset, flags = "-v") + create(tarball, fileset, flags="-v") def extract(tarball: str, fileset: list = [], flags: str = "") -> None: @@ -107,7 +103,7 @@ def xvf(tarball: str = "", fileset: list = []) -> None: List containing filenames, patterns, or directories to extract from the archive. If empty, then all files will be extracted. """ - extract(tarball, fileset, flags = "-v") + extract(tarball, fileset, flags="-v") def tell(tarball: str, flags: str = "", fileset: list = []) -> None: From af941c8c564f174c4d032575dff4b0345fc06b53 Mon Sep 17 00:00:00 2001 From: David Huber Date: Tue, 2 Apr 2024 18:42:18 +0000 Subject: [PATCH 11/32] Sort fsutils functions. --- src/wxflow/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/wxflow/__init__.py b/src/wxflow/__init__.py index 31e36c5..a9a5f58 100644 --- a/src/wxflow/__init__.py +++ b/src/wxflow/__init__.py @@ -8,7 +8,7 @@ from .executable import CommandNotFoundError, Executable, ProcessError, which from .factory import Factory from .file_utils import FileHandler -from .fsutils import chdir, cp, mkdir, mkdir_p, rm_p, rmdir, get_gid, chgrp +from .fsutils import chdir, chgrp, cp, get_gid, mkdir, mkdir_p, rm_p, rmdir from .jinja import Jinja from .logger import Logger, logit from .sqlitedb import SQLiteDB, SQLiteDBError From c47a0039751a25701ced8d3822501a82bf635a33 Mon Sep 17 00:00:00 2001 From: David Huber Date: Tue, 2 Apr 2024 19:15:37 +0000 Subject: [PATCH 12/32] add documentation for HPSS functions --- docs/api.rst | 2 ++ docs/hsi.rst | 11 +++++++++++ docs/htar.rst | 8 ++++++++ 3 files changed, 21 insertions(+) create mode 100644 docs/hsi.rst create mode 100644 docs/htar.rst diff --git a/docs/api.rst b/docs/api.rst index 476d94d..e5f3c4a 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -8,5 +8,7 @@ API timetools jinja logger + hsi + htar diff --git a/docs/hsi.rst b/docs/hsi.rst new file mode 100644 index 0000000..71a4f8f --- /dev/null +++ b/docs/hsi.rst @@ -0,0 +1,11 @@ +.. currentmodule:: wxflow.hsi + +.. autofunction:: hsi +.. autofunction:: get +.. autofunction:: put +.. autofunction:: chmod +.. autofunction:: chgrp +.. autofunction:: rm +.. autofunction:: mkdir +.. autofunction:: ls +.. autofunction:: exists diff --git a/docs/htar.rst b/docs/htar.rst new file mode 100644 index 0000000..03bfc3d --- /dev/null +++ b/docs/htar.rst @@ -0,0 +1,8 @@ +.. currentmodule:: wxflow.htar + +.. autofunction:: htar +.. autofunction:: create +.. autofunction:: extract +.. autofunction:: xvf +.. autofunction:: cvf +.. autofunction:: tell From 6609f7c2f8e5fed323575f4c44fc2e3c92db0f9d Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Thu, 4 Apr 2024 12:19:35 +0000 Subject: [PATCH 13/32] Convert hsi, htar functions to class methods. --- src/wxflow/__init__.py | 3 +- src/wxflow/hsi.py | 461 ++++++++++++++++++++++++----------------- src/wxflow/htar.py | 215 +++++++++++-------- 3 files changed, 395 insertions(+), 284 deletions(-) diff --git a/src/wxflow/__init__.py b/src/wxflow/__init__.py index a9a5f58..ab9b8e3 100644 --- a/src/wxflow/__init__.py +++ b/src/wxflow/__init__.py @@ -1,6 +1,5 @@ import os -from . import hsi, htar from .attrdict import AttrDict from .configuration import (Configuration, cast_as_dtype, cast_strdict_as_dtypedict) @@ -9,6 +8,8 @@ from .factory import Factory from .file_utils import FileHandler from .fsutils import chdir, chgrp, cp, get_gid, mkdir, mkdir_p, rm_p, rmdir +from .hsi import Hsi +from .htar import Htar from .jinja import Jinja from .logger import Logger, logit from .sqlitedb import SQLiteDB, SQLiteDBError diff --git a/src/wxflow/hsi.py b/src/wxflow/hsi.py index e37cfc2..f3eba8c 100644 --- a/src/wxflow/hsi.py +++ b/src/wxflow/hsi.py @@ -1,286 +1,363 @@ -from .executable import Executable, which +from .executable import which -__all__ = ['hsi', 'get', 'put', 'ls', 'chmod', 'chgrp', 'rm', - 'mkdir', 'file_exists'] +__all__ = ['Hsi'] -def hsi(*args) -> str: - """Direct command builder function for hsi based on the input arguments. +class Hsi: + """ + Class offering an interface to HPSS via the hsi utility. - `args` should consist of a set of string arguments to send to hsi - For example, hsi("get","some_local_file : /some/hpss/file") will execute - hsi get some_local_file : /some/hpss/file + Examples: + -------- - Return: str - Concatenated output and error of the hsi command. + >>> from wxflow import Hsi + >>> hsi = Hsi() # Generates an Executable object of "hsi" + >>> output = hsi.put("some_local_file", "/HPSS/path/to/some_file") # Put a file onto HPSS + >>> output = hsi.ls("/HPSS/path/to/some_file") # List the file + >>> output = hsi.chgrp("rstprod", "/HPSS/pth/to/some_file") # Change the group to rstprod """ - cmd = which("hsi", required=True) + def __init__(self): + """Instantiate the hsi command + """ - for arg in args: - cmd.add_default_arg(arg) + self.exe = which("hsi", required=True) - output = cmd(output=str.split, error=str.split) + def hsi(self, args: list, silent: bool = False, ignore_errors: list = []) -> str: + """Direct command builder function for hsi based on the input arguments. - return output + args: list + A list of arguments to sent to hsi + silent: bool + Whether the output of the hsi command should be written to stdout -def get(source: str, target: str = "", hsi_flags: str = "") -> str: - """ Function to get a file from HPSS via hsi + ignore_errors: list + List of error numbers to ignore. For example, hsi returns error + number 64 if a target file does not exist on HPSS. - Parameters - ---------- - source : str - Full path location on HPSS of the file + Return: str + Concatenated output and error of the hsi command. - target : str - Location on the local machine to place the file. If not specified, - then the file will be placed in the current directory. + Example: + -------- + >>> hsi = Hsi() + >>> # Execute `hsi get some_local_file : /some/hpss/file` + >>> hsi.hsi(["get","some_local_file : /some/hpss/file"]) + """ - hsi_flags : str - String of flags to send to hsi. - """ - args = [] + if silent: + output = self.exe(*args, output=str, error=str, + ignore_errors=ignore_errors) + else: + output = self.exe(*args, output=str.split, error=str.split, + ignore_errors=ignore_errors) - # Parse any hsi flags - if len(hsi_flags) > 0: - args.extend(hsi_flags.split(" ")) + return output - args.append("get") - if len(target) == 0: - args.append(source) - else: - args.append(target + " : " + source) + def get(self, source: str, target: str = "", hsi_flags: str = "-q -e") -> str: + """ Function to get a file from HPSS via hsi - output = hsi(*args) + Parameters + ---------- + source : str + Full path location on HPSS of the file - return output + target : str + Location on the local machine to place the file. If not specified, + then the file will be placed in the current directory. + hsi_flags : str + String of flags to send to hsi. By default, suppress login info and + echo the get command. + """ + args = [] -def put(source: str, target: str, hsi_flags: str = "", - listing_file: str = None) -> str: - """ Function to put a file onto HPSS via hsi + # Convert to str to handle Path objects + target = str(target) + source = str(source) - Parameters - ---------- - source : str - Location on the local machine of the source file to send to HPSS. + # Parse any hsi flags + if len(hsi_flags) > 0: + args.extend(hsi_flags.split(" ")) - target : str - Full path of the target location of the file on HPSS. + args.append("get") + if len(target) == 0: + args.append(source) + else: + args.append(target + " : " + source) - hsi_flags : str - String of flags to send to hsi. - """ - args = [] + output = self.hsi(args) - # Parse any hsi flags - if len(hsi_flags) > 0: - args.extend(hsi_flags.split(" ")) + return output - args.append("put") - args.append(source + " : " + target) - output = hsi(*args) + def put(self, source: str, target: str, hsi_flags: str = "-q -e", + listing_file: str = None) -> str: + """ Function to put a file onto HPSS via hsi - return output + Parameters + ---------- + source : str + Location on the local machine of the source file to send to HPSS. + target : str + Full path of the target location of the file on HPSS. -def chmod(mod: str, target: str, hsi_flags: str = "", - chmod_flags: str = "") -> str: - """ Function to change the permissions of a file or directory on HPSS + hsi_flags : str + String of flags to send to hsi. By default, suppress login info + and echo the put command. + """ + args = [] - Parameters - ---------- - mod : str - Permissions to set for the file or directory, - e.g. "640", "o+r", etc. + # Convert to str to handle Path objects + target = str(target) + source = str(source) - target : str - Full path of the target location of the file on HPSS. + # Parse any hsi flags + if len(hsi_flags) > 0: + args.extend(hsi_flags.split(" ")) - hsi_flags : str - String of flags to send to hsi. + args.append("put") + args.append(source + " : " + target) + output = self.hsi(args) - chmod_flags : str - Flags to send to chmod. See "hsi chmod -?" for more details. - """ + return output - args = [] + def chmod(self, mod: str, target: str, hsi_flags: str = "-q -e", + chmod_flags: str = "") -> str: + """ Function to change the permissions of a file or directory on HPSS - # Parse any hsi flags - if len(hsi_flags) > 0: - args.extend(hsi_flags.split(" ")) + Parameters + ---------- + mod : str + Permissions to set for the file or directory, + e.g. "640", "o+r", etc. - args.append("chmod") + target : str + Full path of the target location of the file on HPSS. - if len(chmod_flags) > 0: - args.extend(chmod_flags.split(" ")) + hsi_flags : str + String of flags to send to hsi. By default, suppress login info. - args.append(mod) - args.append(target) - output = hsi(*args) + chmod_flags : str + Flags to send to chmod. See "hsi chmod -?" for more details. + """ - return output + args = [] + # Parse any hsi flags + if len(hsi_flags) > 0: + args.extend(hsi_flags.split(" ")) -def chgrp(group_name: str, target: str, hsi_flags: str = "", - chgrp_flags: str = "") -> str: - """ Function to change the group of a file or directory on HPSS + args.append("chmod") - Parameters - ---------- - group_name : str - The group to which ownership of the file/directory is to be set. + if len(chmod_flags) > 0: + args.extend(chmod_flags.split(" ")) - target : str - Full path of the target location of the file on HPSS. + args.append(mod) + args.append(target) + output = self.hsi(args) - hsi_flags : str - String of flags to send to hsi. + return output - chgrp_flags : str - Flags to send to chgrp. See "hsi chgrp -?" for more details. - """ + def chgrp(self, group_name: str, target: str, hsi_flags: str = "-q", + chgrp_flags: str = "") -> str: + """ Function to change the group of a file or directory on HPSS - args = [] + Parameters + ---------- + group_name : str + The group to which ownership of the file/directory is to be set. - # Parse any hsi flags - if len(hsi_flags) > 0: - args.extend(hsi_flags.split(" ")) + target : str + Full path of the target location of the file on HPSS. - args.append("chgrp") + hsi_flags : str + String of flags to send to hsi. By default, suppress login info. - if len(chgrp_flags) > 0: - args.extend(chgrp_flags.split(" ")) + chgrp_flags : str + Flags to send to chgrp. See "hsi chgrp -?" for more details. + """ - args.append(group_name) - args.append(target) - output = hsi(*args) + args = [] - return output + # Parse any hsi flags + if len(hsi_flags) > 0: + args.extend(hsi_flags.split(" ")) + args.append("chgrp") -def rm(target: str, hsi_flags: str = "", rm_flags: str = "") -> str: - """ Function to delete a file or directory on HPSS via hsi + if len(chgrp_flags) > 0: + args.extend(chgrp_flags.split(" ")) - Parameters - ---------- - target : str - Full path of the target location of the file on HPSS. + args.append(group_name) + args.append(target) + output = self.hsi(args) - hsi_flags : str - String of flags to send to hsi. + return output - rm_flags : str - Flags to send to rm. See "hsi rm -?" for more details. - """ + def rm(self, target: str, hsi_flags: str = "-q -e", rm_flags: str = "") -> str: + """ Function to delete a file or directory on HPSS via hsi - args = [] + Parameters + ---------- + target : str + Full path of the target location of the file on HPSS. - # Parse any hsi flags - if len(hsi_flags) > 0: - args.extend(hsi_flags.split(" ")) + hsi_flags : str + String of flags to send to hsi. By default, suppress login info + and echo the rm command. - args.append("rm") + rm_flags : str + Flags to send to rm. See "hsi rm -?" for more details. + """ - if len(rm_flags) > 0: - args.extend(rm_flags.split(" ")) + # Call rmdir if recursive (-r) flag present + if "-r" in rm_flags: + rmdir_flags = rm_flags.replace("-r", "") + output = self.rmdir(target, hsi_flags, rmdir_flags) + return output - args.append(target) - output = hsi(*args) + args = [] - return output + # Parse any hsi flags + if len(hsi_flags) > 0: + args.extend(hsi_flags.split(" ")) + args.append("rm") -def mkdir(target: str, hsi_flags: str = "", mkdir_flags: str = "") -> str: - """ Function to delete a file or directory on HPSS via hsi + if len(rm_flags) > 0: + args.extend(rm_flags.split(" ")) - Parameters - ---------- - target : str - Full path of the target location of the file on HPSS. + args.append(target) + output = self.hsi(args) - hsi_flags : str - String of flags to send to hsi. + return output - mkdir_flags : str - Flags to send to mkdir (-p is assumed). - See "hsi mkdir -?" for more details. - """ + def rmdir(self, target: str, hsi_flags: str = "-q -e", rmdir_flags: str = "") -> str: + """ Function to delete a file or directory on HPSS via hsi + + Parameters + ---------- + target : str + Full path of the target location of the file on HPSS. + + hsi_flags : str + String of flags to send to hsi. By default, suppress login info + and echo the rmdir command. + + rmdir_flags : str + Flags to send to rmdir. See "hsi rmdir -?" for more details. + """ + + args = [] - args = [] + # Parse any hsi flags + if len(hsi_flags) > 0: + args.extend(hsi_flags.split(" ")) - # Parse any hsi flags - if len(hsi_flags) > 0: - args.extend(hsi_flags.split(" ")) + args.append("rmdir") - args.extend(["mkdir", "-p"]) + if len(rmdir_flags) > 0: + args.extend(rmdir_flags.split(" ")) - if len(mkdir_flags) > 0: - args.extend(mkdir_flags.split(" ")) + args.append(target) + output = self.hsi(args) - args.append(target) - output = hsi(*args) + return output - return output + def mkdir(self, target: str, hsi_flags: str = "-q -e", mkdir_flags: str = "") -> str: + """ Function to delete a file or directory on HPSS via hsi + Parameters + ---------- + target : str + Full path of the target location of the file on HPSS. -def ls(target: str, hsi_flags: str = "", ls_flags: str = "") -> str: - """ Function to list files/directories on HPSS via hsi + hsi_flags : str + String of flags to send to hsi. By default, suppress login info + and echo the mkdir command. + """ - Parameters - ---------- - target : str + args = [] + + # Parse any hsi flags + if len(hsi_flags) > 0: + args.extend(hsi_flags.split(" ")) + + # The only flag available for mkdir is -p, which we will use. + args.extend(["mkdir", "-p"]) + + if len(mkdir_flags) > 0: + args.extend(mkdir_flags.split(" ")) + + args.append(target) + output = self.hsi(args) + + return output + + def ls(self, target: str, hsi_flags: str = "-q", ls_flags: str = "", + ignore_missing: bool = False) -> str: + """ Function to list files/directories on HPSS via hsi + + Parameters + ---------- + target : str Full path of the target location on HPSS. - hsi_flags : str - String of flags to send to hsi. + hsi_flags : str + String of flags to send to hsi. By default, suppress login info. - ls_flags : str + ls_flags : str Flags to send to ls. See "hsi ls -?" for more details. - """ - args = [] + ignore_missing: bool + Flag to ignore missing files + """ - # Parse any hsi flags - if len(hsi_flags) > 0: - args.extend(hsi_flags.split(" ")) + args = [] - args.append("ls") + if ignore_missing: + ignore_errors = [64] + else: + ignore_errors = [] - if len(ls_flags) > 0: - args.extend(ls_flags.split(" ")) + # Parse any hsi flags + if len(hsi_flags) > 0: + args.extend(hsi_flags.split(" ")) - args.append(target) - output = hsi(*args) + args.append("ls") - return output + # Parse any ls flags + if len(ls_flags) > 0: + args.extend(ls_flags.split(" ")) + args.append(target) + output = self.hsi(args, ignore_errors=ignore_errors) -def exists(target: str) -> bool: - """ Function to test the existence of a file/directory/glob on HPSS + return output - Parameters - ---------- - target : str - Full path of the target location on HPSS. + def exists(self, target: str) -> bool: + """ Function to test the existence of a file/directory/glob on HPSS - Return: bool - True if the target exists on HPSS. - """ + Parameters + ---------- + target : str + Full path of the target location on HPSS. - cmd = which("hsi", required=True) + Return: bool + True if the target exists on HPSS. + """ - for arg in ["ls", target]: - cmd.add_default_arg(arg) + args = ["-q", "ls", target] - # Do not exit if the file is not found; do not pipe output to stdout - output = cmd(output=str, error=str, ignore_errors=[64]) + # Do not exit if the file is not found; do not pipe output to stdout + output = self.hsi(args, silent=True, ignore_errors=[64]) - if "HPSS_ENOENT" in output: - return False - # Catch wildcards - elif f"Warning: No matching names located for '{target}'" in output: - return False - else: - return True + if "HPSS_ENOENT" in output: + return False + # Catch wildcards + elif f"Warning: No matching names located for '{target}'" in output: + return False + else: + return True diff --git a/src/wxflow/htar.py b/src/wxflow/htar.py index 8e7f721..190fb5d 100644 --- a/src/wxflow/htar.py +++ b/src/wxflow/htar.py @@ -1,135 +1,168 @@ -from .executable import Executable, which +from .executable import which -__all__ = ['htar', 'create', 'cvf', 'extract', 'xvf', 'tell'] +__all__ = ['Htar'] -def htar(*args) -> None: +class Htar: """ - Direct command builder function for htar based on the input arguments. + Class offering an interface to HPSS via the htar utility. - `args` should consist of a set of string arguments to send to htar - For example, - htar.htar("-cvf","/path/to/hpss/archive.tar", "") - will execute - htar -cvf /path/to/hpss/archive.tar + Examples: + -------- + >>> from wxflow import Htar + >>> htar = Htar() # Generates an Executable object of "htar" + >>> output = htar.cvf("/HPSS/path/to/archive.tar", "file1 file2") # Create an HPSS archive from two local files + >>> output = htar.tell("/HPSS/path/to/archive.tar") # List the contents of an archive """ + def __init__(self) -> None: + self.exe = which("htar") - cmd = which("htar", required=True) + def htar(self, args, silent: bool = False) -> str: + """ + Direct command builder function for htar based on the input arguments. - for arg in args: - cmd.add_default_arg(arg) + Parameters: + ----------- + args: list + List of string arguments to send to htar - cmd() + silent: bool + Flag to suppress output to stdout + Return: str + Output from the htar command -def create(tarball: str, fileset: list, flags: str = "") -> None: - """ Function to write an archive to HPSS + Examples: + --------- + >>> htar = Htar() + >>> # Run `htar -cvf /path/to/hpss/archive.tar file1 file2 file-* + >>> htar.htar("-cvf", "/path/to/hpss/archive.tar", "file1 file2 file-*") + """ - Parameters - ---------- - flags : str - String of flags to send to htar. + if silent: + output = self.exe(*args, output=str, error=str) + else: + output = self.exe(*args, output=str.split, error=str.split) - tarball : str - Full path location on HPSS to create the archive. + return output - fileset : list - List containing filenames, patterns, or directories to archive - """ - args = ("-c",) + def create(self, tarball: str, fileset: list, flags: str = "") -> str: + """ Function to write an archive to HPSS - # Parse any htar flags - if len(flags) > 0: - args += tuple(flags.split(" ")) + Parameters + ---------- + flags : str + String of flags to send to htar. - args += ("-f", tarball,) + tuple(fileset) + tarball : str + Full path location on HPSS to create the archive. - htar(*args) + fileset : list + List containing filenames, patterns, or directories to archive + """ + args = ["-c"] + # Parse any htar flags + if len(flags) > 0: + args += flags.split(" ") -def cvf(tarball: str, fileset: list) -> None: - """ Function to write an archive to HPSS verbosely (without flags). + if len(fileset) == 0: + raise ValueError("Input fileset is empty, cannot create archive") - Parameters - ---------- - tarball : str - Full path location on HPSS to create the archive. + args += ["-f", tarball, ' '.join(fileset)] - fileset : list - List containing filenames, patterns, or directories to archive - """ - create(tarball, fileset, flags="-v") + output = self.htar(args) + return output -def extract(tarball: str, fileset: list = [], flags: str = "") -> None: - """ Function to extract an archive from HPSS via htar + def cvf(self, tarball: str, fileset: list) -> str: + """ Function to write an archive to HPSS verbosely (without flags). - Parameters - ---------- - flags : str - String of flags to send to htar. + Parameters + ---------- + tarball : str + Full path location on HPSS to create the archive. - tarball : str - Full path location of an archive on HPSS to extract from. + fileset : list + List containing filenames, patterns, or directories to archive + """ + output = self.create(tarball, fileset, flags="-v -P") - fileset : list - List containing filenames, patterns, or directories to extract from - the archive. If empty, then all files will be extracted. - """ - args = ("-x",) + return output - # Parse any htar flags - if len(flags) > 0: - args += tuple(flags.split(" ")) + def extract(self, tarball: str, fileset: list = [], flags: str = "") -> str: + """ Function to extract an archive from HPSS via htar - args += ("-f", tarball,) + Parameters + ---------- + flags : str + String of flags to send to htar. - if len(fileset) > 0: - args += tuple(fileset) + tarball : str + Full path location of an archive on HPSS to extract from. - htar(*args) + fileset : list + List containing filenames, patterns, or directories to extract from + the archive. If empty, then all files will be extracted. + """ + args = ["-x"] + # Parse any htar flags + if len(flags) > 0: + args += flags.split(" ") -def xvf(tarball: str = "", fileset: list = []) -> None: - """ Function to extract an archive from HPSS verbosely (without flags). + args += ["-f", tarball] - Parameters - ---------- - tarball : str - Full path location of an archive on HPSS to extract from. + if len(fileset) > 0: + args.append(' '.join(fileset)) - fileset : list - List containing filenames, patterns, or directories to extract from - the archive. If empty, then all files will be extracted. - """ - extract(tarball, fileset, flags="-v") + output = self.htar(args) + return output -def tell(tarball: str, flags: str = "", fileset: list = []) -> None: - """ Function to list the contents of an archive on HPSS + def xvf(self, tarball: str = "", fileset: list = []) -> str: + """ Function to extract an archive from HPSS verbosely (without flags). - Parameters - ---------- - flags : str - String of flags to send to htar. + Parameters + ---------- + tarball : str + Full path location of an archive on HPSS to extract from. - tarball : str - Full path location on HPSS to list the contents of. + fileset : list + List containing filenames, patterns, or directories to extract from + the archive. If empty, then all files will be extracted. + """ + output = self.extract(tarball, fileset, flags="-v") - fileset : list - List containing filenames, patterns, or directories to list. - If empty, then all files will be listed. - """ - args = ("-t",) + return output + + def tell(self, tarball: str, flags: str = "", fileset: list = []) -> str: + """ Function to list the contents of an archive on HPSS + + Parameters + ---------- + flags : str + String of flags to send to htar. + + tarball : str + Full path location on HPSS to list the contents of. + + fileset : list + List containing filenames, patterns, or directories to list. + If empty, then all files will be listed. + """ + args = ["-t"] + + # Parse any htar flags + if len(flags) > 0: + args += [flags.split(" ")] - # Parse any htar flags - if len(flags) > 0: - args += tuple(flags.split(" ")) + args += ["-f", tarball] - args += ("-f", tarball,) + if len(fileset) > 0: + args += " ".join(fileset) - if len(fileset) > 0: - args += tuple(fileset) + output = self.htar(args) - htar(*args) + return output From c0021f70a3906bbecbf9b812ba74a62daef0506c Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Thu, 4 Apr 2024 12:29:32 +0000 Subject: [PATCH 14/32] Created hsi and htar tests. --- tests/test_hsi.py | 160 +++++++++++++++++++++++++++++++++++++++++++++ tests/test_htar.py | 79 ++++++++++++++++++++++ 2 files changed, 239 insertions(+) create mode 100644 tests/test_hsi.py create mode 100644 tests/test_htar.py diff --git a/tests/test_hsi.py b/tests/test_hsi.py new file mode 100644 index 0000000..752e5c4 --- /dev/null +++ b/tests/test_hsi.py @@ -0,0 +1,160 @@ +import os + +from pathlib import Path + +import pytest + +import random + +import string + +from wxflow import Hsi, CommandNotFoundError + +# These tests do not run on the GH runner as they it is not connected to HPSS. +# It is intended that these tests should only be run on Hera or WCOSS2. +# This test also assumes that you are a member of rstprod on HPSS. If not, +# then the test_chgrp test will fail. + +try: + hsi = Hsi() +except CommandNotFoundError: + hsi = None + +test_hash = ''.join(random.choices(string.ascii_lowercase + string.digits, k=10)) +user = os.environ['USER'] +test_path = f'/NCEPDEV/emc-global/1year/{user}/hsi_test/test-{test_hash}' + + +@pytest.mark.skipif(not hsi, reason="Only runs on Hera/WCOSS2 with hpss module loaded.") +def test_hsi(): + """ + Test for the hsi command builder: + """ + output = hsi.hsi(["ls", "/NCEPDEV/"]) + + assert type(output) is str + assert "emc-global" in output + + +@pytest.mark.skipif(not hsi, reason="Only runs on Hera/WCOSS2 with hpss module loaded.") +def test_exists(): + """ + Test for checking if a target exists on HPSS + """ + assert hsi.exists("/NCEPDEV") + assert not hsi.exists("/not_a_file") + + +@pytest.mark.skipif(not hsi, reason="Only runs on Hera/WCOSS2 with hpss module loaded.") +def test_ls(): + """ + Test HPSS listing + """ + output = hsi.ls("/NCEPDEV/") + assert "emc-global" in output + + output = hsi.ls("/NCEPDEV/", ls_flags="-l") + assert "drwxr-xr-x" in output + + +@pytest.mark.skipif(not hsi, reason="Only runs on Hera/WCOSS2 with hpss module loaded.") +def test_mkdir_rmdir(): + """ + Test for creating a directory: + """ + + # Create the test_path + output = hsi.mkdir(test_path) + + # Check that the test path was created + assert hsi.exists(test_path) + + # Remove the test_path + output = hsi.rmdir(test_path) + + # Check that the test path was removed + assert not hsi.exists(test_path) + + +@pytest.mark.skipif(not hsi, reason="Only runs on Hera/WCOSS2 with hpss module loaded.") +def test_chmod(): + """ + Test for chmod: + """ + + # Create the test_path + output = hsi.mkdir(test_path) + + # Change the mode of the test path + output = hsi.chmod("750", test_path, chmod_flags="-R") + + # Check that the mode was changed + output = hsi.ls(test_path, ls_flags="-d -l") + + assert "drwxr-x---" in output + + # Remove the test_path + output = hsi.rmdir(test_path) + + +@pytest.mark.skipif(not hsi, reason="Only runs on Hera/WCOSS2 with hpss module loaded.") +def test_chgrp(): + """ + Test for chgrp: + """ + + # Create the test_path + output = hsi.mkdir(test_path) + + # Change the group of the test path + output = hsi.chgrp("rstprod", test_path, chgrp_flags="-R") + + # Check that the group was changed + output = hsi.ls(test_path, ls_flags="-d -l") + + assert "rstprod" in output + + # Remove the test_path + output = hsi.rmdir(test_path) + + +@pytest.mark.skipif(not hsi, reason="Only runs on Hera/WCOSS2 with hpss module loaded.") +def test_put_get(tmp_path): + """ + Test for sending/getting a file to/from HPSS: + Parameters + ---------- + tmp_path - pytest fixture + """ + + # Create temporary directories + input_dir_path = tmp_path / 'my_input_dir' + input_dir_path.mkdir() + output_dir_path = tmp_path / 'my_output_dir' + output_dir_path.mkdir() + # Create an empty file to send + in_tmp_file = input_dir_path / 'a.txt' + in_tmp_file.touch() + + # Name the output file + out_tmp_file = output_dir_path / 'a.txt' + + # Create test_path if it doesn't exist + if not hsi.exists(test_path): + hsi.mkdir(test_path) + + # Send the temporary file + output = hsi.put(in_tmp_file, test_path + "/a.txt") + + assert "a.txt" in output + assert hsi.exists(test_path + "/a.txt") + + # Get the temporary file + output = hsi.get(test_path + "/a.txt", out_tmp_file) + + assert "a.txt" in output + assert out_tmp_file.exists() + + # Remove the test directory + output = hsi.rm(test_path + "/a.txt") + output = hsi.rmdir(test_path) diff --git a/tests/test_htar.py b/tests/test_htar.py new file mode 100644 index 0000000..d3b8f03 --- /dev/null +++ b/tests/test_htar.py @@ -0,0 +1,79 @@ +import os + +from pathlib import Path + +import pytest + +import random + +import string + +from wxflow import Htar, Hsi, CommandNotFoundError + +# These tests do not run on the GH runner as they it is not connected to HPSS. +# It is intended that these tests should only be run on Hera or WCOSS2. + +try: + htar = Htar() + hsi = Hsi() +except CommandNotFoundError: + hsi = None + htar = None + +test_hash = ''.join(random.choices(string.ascii_lowercase + string.digits, k=10)) +user = os.environ['USER'] +test_path = f'/NCEPDEV/emc-global/1year/{user}/hsi_test/test-{test_hash}' + + +@pytest.mark.skipif(not htar, reason="Only runs on Hera/WCOSS2 with hpss module loaded.") +def test_htar(): + """ + Test for the htar command builder: + """ + output = htar.htar(["-?"]) + + assert type(output) is str + assert "Usage" in output + + +@pytest.mark.skipif(not htar, reason="Only runs on Hera/WCOSS2 with hpss module loaded.") +def test_cvf_xvf_tell(tmp_path): + """ + Test creating, extracting, and listing a tarball on HPSS: + Parameters + ---------- + tmp_path - pytest fixture + """ + + # Create temporary directories + input_dir_path = tmp_path / 'my_input_dir' + input_dir_path.mkdir() + # Create an empty file to send + in_tmp_file = input_dir_path / 'a.txt' + in_tmp_file.touch() + in_tmp_file.write_text("Contents of a.txt") + + test_tarball = test_path + "/test.tar" + + # Create the archive file + output = htar.cvf(test_tarball, [str(in_tmp_file)]) + print("output::") + print(output) + + assert "a.txt" in output + assert hsi.exists(test_tarball) + + # Extract the test archive + output = htar.xvf(test_tarball) + + assert "a.txt" in output + + # List the contents of the test archive + output = htar.tell(test_tarball) + + assert "a.txt" in output + + # Remove the test directory + output = hsi.rm(test_tarball) + output = hsi.rm(test_tarball + ".idx") + output = hsi.rmdir(test_path) From 958bd67ec53f3675aeed57c2e5e40de6126c9503 Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Thu, 4 Apr 2024 12:33:05 +0000 Subject: [PATCH 15/32] Sorted hsi and htar tests. --- tests/test_hsi.py | 9 +++------ tests/test_htar.py | 9 +++------ 2 files changed, 6 insertions(+), 12 deletions(-) diff --git a/tests/test_hsi.py b/tests/test_hsi.py index 752e5c4..74edff0 100644 --- a/tests/test_hsi.py +++ b/tests/test_hsi.py @@ -1,14 +1,11 @@ import os - +import random +import string from pathlib import Path import pytest -import random - -import string - -from wxflow import Hsi, CommandNotFoundError +from wxflow import CommandNotFoundError, Hsi # These tests do not run on the GH runner as they it is not connected to HPSS. # It is intended that these tests should only be run on Hera or WCOSS2. diff --git a/tests/test_htar.py b/tests/test_htar.py index d3b8f03..9e256b1 100644 --- a/tests/test_htar.py +++ b/tests/test_htar.py @@ -1,14 +1,11 @@ import os - +import random +import string from pathlib import Path import pytest -import random - -import string - -from wxflow import Htar, Hsi, CommandNotFoundError +from wxflow import CommandNotFoundError, Hsi, Htar # These tests do not run on the GH runner as they it is not connected to HPSS. # It is intended that these tests should only be run on Hera or WCOSS2. From 838d156033285fb4d5d985e16e6426db78375d92 Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Thu, 4 Apr 2024 12:40:38 +0000 Subject: [PATCH 16/32] Update documentation. --- docs/hsi.rst | 13 +++---------- docs/htar.rst | 10 +++------- src/wxflow/hsi.py | 18 +++++++++--------- src/wxflow/htar.py | 10 +++++----- 4 files changed, 20 insertions(+), 31 deletions(-) diff --git a/docs/hsi.rst b/docs/hsi.rst index 71a4f8f..8ebfe10 100644 --- a/docs/hsi.rst +++ b/docs/hsi.rst @@ -1,11 +1,4 @@ -.. currentmodule:: wxflow.hsi +.. currentmodule:: wxflow -.. autofunction:: hsi -.. autofunction:: get -.. autofunction:: put -.. autofunction:: chmod -.. autofunction:: chgrp -.. autofunction:: rm -.. autofunction:: mkdir -.. autofunction:: ls -.. autofunction:: exists +.. autoclass:: Hsi + :members: diff --git a/docs/htar.rst b/docs/htar.rst index 03bfc3d..4061bd1 100644 --- a/docs/htar.rst +++ b/docs/htar.rst @@ -1,8 +1,4 @@ -.. currentmodule:: wxflow.htar +.. currentmodule:: wxflow -.. autofunction:: htar -.. autofunction:: create -.. autofunction:: extract -.. autofunction:: xvf -.. autofunction:: cvf -.. autofunction:: tell +.. autoclass:: Htar + :members: diff --git a/src/wxflow/hsi.py b/src/wxflow/hsi.py index f3eba8c..731b876 100644 --- a/src/wxflow/hsi.py +++ b/src/wxflow/hsi.py @@ -56,7 +56,7 @@ def hsi(self, args: list, silent: bool = False, ignore_errors: list = []) -> str return output def get(self, source: str, target: str = "", hsi_flags: str = "-q -e") -> str: - """ Function to get a file from HPSS via hsi + """ Method to get a file from HPSS via hsi Parameters ---------- @@ -93,7 +93,7 @@ def get(self, source: str, target: str = "", hsi_flags: str = "-q -e") -> str: def put(self, source: str, target: str, hsi_flags: str = "-q -e", listing_file: str = None) -> str: - """ Function to put a file onto HPSS via hsi + """ Method to put a file onto HPSS via hsi Parameters ---------- @@ -125,7 +125,7 @@ def put(self, source: str, target: str, hsi_flags: str = "-q -e", def chmod(self, mod: str, target: str, hsi_flags: str = "-q -e", chmod_flags: str = "") -> str: - """ Function to change the permissions of a file or directory on HPSS + """ Method to change the permissions of a file or directory on HPSS Parameters ---------- @@ -162,7 +162,7 @@ def chmod(self, mod: str, target: str, hsi_flags: str = "-q -e", def chgrp(self, group_name: str, target: str, hsi_flags: str = "-q", chgrp_flags: str = "") -> str: - """ Function to change the group of a file or directory on HPSS + """ Method to change the group of a file or directory on HPSS Parameters ---------- @@ -197,7 +197,7 @@ def chgrp(self, group_name: str, target: str, hsi_flags: str = "-q", return output def rm(self, target: str, hsi_flags: str = "-q -e", rm_flags: str = "") -> str: - """ Function to delete a file or directory on HPSS via hsi + """ Method to delete a file or directory on HPSS via hsi Parameters ---------- @@ -235,7 +235,7 @@ def rm(self, target: str, hsi_flags: str = "-q -e", rm_flags: str = "") -> str: return output def rmdir(self, target: str, hsi_flags: str = "-q -e", rmdir_flags: str = "") -> str: - """ Function to delete a file or directory on HPSS via hsi + """ Method to delete a file or directory on HPSS via hsi Parameters ---------- @@ -267,7 +267,7 @@ def rmdir(self, target: str, hsi_flags: str = "-q -e", rmdir_flags: str = "") -> return output def mkdir(self, target: str, hsi_flags: str = "-q -e", mkdir_flags: str = "") -> str: - """ Function to delete a file or directory on HPSS via hsi + """ Method to delete a file or directory on HPSS via hsi Parameters ---------- @@ -298,7 +298,7 @@ def mkdir(self, target: str, hsi_flags: str = "-q -e", mkdir_flags: str = "") -> def ls(self, target: str, hsi_flags: str = "-q", ls_flags: str = "", ignore_missing: bool = False) -> str: - """ Function to list files/directories on HPSS via hsi + """ Method to list files/directories on HPSS via hsi Parameters ---------- @@ -338,7 +338,7 @@ def ls(self, target: str, hsi_flags: str = "-q", ls_flags: str = "", return output def exists(self, target: str) -> bool: - """ Function to test the existence of a file/directory/glob on HPSS + """ Method to test the existence of a file/directory/glob on HPSS Parameters ---------- diff --git a/src/wxflow/htar.py b/src/wxflow/htar.py index 190fb5d..48ba118 100644 --- a/src/wxflow/htar.py +++ b/src/wxflow/htar.py @@ -48,7 +48,7 @@ def htar(self, args, silent: bool = False) -> str: return output def create(self, tarball: str, fileset: list, flags: str = "") -> str: - """ Function to write an archive to HPSS + """ Method to write an archive to HPSS Parameters ---------- @@ -77,7 +77,7 @@ def create(self, tarball: str, fileset: list, flags: str = "") -> str: return output def cvf(self, tarball: str, fileset: list) -> str: - """ Function to write an archive to HPSS verbosely (without flags). + """ Method to write an archive to HPSS verbosely (without flags). Parameters ---------- @@ -92,7 +92,7 @@ def cvf(self, tarball: str, fileset: list) -> str: return output def extract(self, tarball: str, fileset: list = [], flags: str = "") -> str: - """ Function to extract an archive from HPSS via htar + """ Method to extract an archive from HPSS via htar Parameters ---------- @@ -122,7 +122,7 @@ def extract(self, tarball: str, fileset: list = [], flags: str = "") -> str: return output def xvf(self, tarball: str = "", fileset: list = []) -> str: - """ Function to extract an archive from HPSS verbosely (without flags). + """ Method to extract an archive from HPSS verbosely (without flags). Parameters ---------- @@ -138,7 +138,7 @@ def xvf(self, tarball: str = "", fileset: list = []) -> str: return output def tell(self, tarball: str, flags: str = "", fileset: list = []) -> str: - """ Function to list the contents of an archive on HPSS + """ Method to list the contents of an archive on HPSS Parameters ---------- From faa4c0285375c7d1f97aa96f2235dedb1c6d2691 Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Thu, 4 Apr 2024 15:00:46 +0000 Subject: [PATCH 17/32] Ignore missing files when removing them. --- src/wxflow/hsi.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/wxflow/hsi.py b/src/wxflow/hsi.py index 731b876..fa17735 100644 --- a/src/wxflow/hsi.py +++ b/src/wxflow/hsi.py @@ -230,12 +230,14 @@ def rm(self, target: str, hsi_flags: str = "-q -e", rm_flags: str = "") -> str: args.extend(rm_flags.split(" ")) args.append(target) - output = self.hsi(args) + + # Ignore missing files + output = self.hsi(args, ignore_errors=[72]) return output def rmdir(self, target: str, hsi_flags: str = "-q -e", rmdir_flags: str = "") -> str: - """ Method to delete a file or directory on HPSS via hsi + """ Method to delete a directory on HPSS via hsi Parameters ---------- From d5f2cb626ee8c7c95d271b095a4499f6ac102d57 Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Thu, 4 Apr 2024 15:01:19 +0000 Subject: [PATCH 18/32] Create parent directories by default when creating hpss tarballs. --- src/wxflow/htar.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/wxflow/htar.py b/src/wxflow/htar.py index 48ba118..fdfa0de 100644 --- a/src/wxflow/htar.py +++ b/src/wxflow/htar.py @@ -47,7 +47,7 @@ def htar(self, args, silent: bool = False) -> str: return output - def create(self, tarball: str, fileset: list, flags: str = "") -> str: + def create(self, tarball: str, fileset: list, flags: str = "-P") -> str: """ Method to write an archive to HPSS Parameters From 34043c23e9301c6d346dc4a47573b18548ece646 Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Thu, 4 Apr 2024 16:53:17 +0000 Subject: [PATCH 19/32] Hide the command builder, provide default args to __init__. --- src/wxflow/hsi.py | 46 +++++++++++++++++++++++++++------------------- 1 file changed, 27 insertions(+), 19 deletions(-) diff --git a/src/wxflow/hsi.py b/src/wxflow/hsi.py index fa17735..4c08cd6 100644 --- a/src/wxflow/hsi.py +++ b/src/wxflow/hsi.py @@ -17,13 +17,21 @@ class Hsi: >>> output = hsi.chgrp("rstprod", "/HPSS/pth/to/some_file") # Change the group to rstprod """ - def __init__(self): + def __init__(self, def_hsi_args = "-q -e"): """Instantiate the hsi command + + def_hsi_args: str + List of default arguments to send to hsi. The defaults are + -q: run in quiet mode (do not print login information) + -e: echo each command """ self.exe = which("hsi", required=True) - def hsi(self, args: list, silent: bool = False, ignore_errors: list = []) -> str: + for arg in def_hsi_args.split(" "): + self.exe.add_default_arg(args) + + def _hsi(self, args: list, silent: bool = False, ignore_errors: list = []) -> str: """Direct command builder function for hsi based on the input arguments. args: list @@ -55,7 +63,7 @@ def hsi(self, args: list, silent: bool = False, ignore_errors: list = []) -> str return output - def get(self, source: str, target: str = "", hsi_flags: str = "-q -e") -> str: + def get(self, source: str, target: str = "", hsi_flags: str = "") -> str: """ Method to get a file from HPSS via hsi Parameters @@ -87,11 +95,11 @@ def get(self, source: str, target: str = "", hsi_flags: str = "-q -e") -> str: else: args.append(target + " : " + source) - output = self.hsi(args) + output = self._hsi(args) return output - def put(self, source: str, target: str, hsi_flags: str = "-q -e", + def put(self, source: str, target: str, hsi_flags: str = "", listing_file: str = None) -> str: """ Method to put a file onto HPSS via hsi @@ -119,11 +127,11 @@ def put(self, source: str, target: str, hsi_flags: str = "-q -e", args.append("put") args.append(source + " : " + target) - output = self.hsi(args) + output = self._hsi(args) return output - def chmod(self, mod: str, target: str, hsi_flags: str = "-q -e", + def chmod(self, mod: str, target: str, hsi_flags: str = "", chmod_flags: str = "") -> str: """ Method to change the permissions of a file or directory on HPSS @@ -156,11 +164,11 @@ def chmod(self, mod: str, target: str, hsi_flags: str = "-q -e", args.append(mod) args.append(target) - output = self.hsi(args) + output = self._hsi(args) return output - def chgrp(self, group_name: str, target: str, hsi_flags: str = "-q", + def chgrp(self, group_name: str, target: str, hsi_flags: str = "", chgrp_flags: str = "") -> str: """ Method to change the group of a file or directory on HPSS @@ -192,11 +200,11 @@ def chgrp(self, group_name: str, target: str, hsi_flags: str = "-q", args.append(group_name) args.append(target) - output = self.hsi(args) + output = self._hsi(args) return output - def rm(self, target: str, hsi_flags: str = "-q -e", rm_flags: str = "") -> str: + def rm(self, target: str, hsi_flags: str = "", rm_flags: str = "") -> str: """ Method to delete a file or directory on HPSS via hsi Parameters @@ -232,11 +240,11 @@ def rm(self, target: str, hsi_flags: str = "-q -e", rm_flags: str = "") -> str: args.append(target) # Ignore missing files - output = self.hsi(args, ignore_errors=[72]) + output = self._hsi(args, ignore_errors=[72]) return output - def rmdir(self, target: str, hsi_flags: str = "-q -e", rmdir_flags: str = "") -> str: + def rmdir(self, target: str, hsi_flags: str = "", rmdir_flags: str = "") -> str: """ Method to delete a directory on HPSS via hsi Parameters @@ -264,11 +272,11 @@ def rmdir(self, target: str, hsi_flags: str = "-q -e", rmdir_flags: str = "") -> args.extend(rmdir_flags.split(" ")) args.append(target) - output = self.hsi(args) + output = self._hsi(args) return output - def mkdir(self, target: str, hsi_flags: str = "-q -e", mkdir_flags: str = "") -> str: + def mkdir(self, target: str, hsi_flags: str = "", mkdir_flags: str = "") -> str: """ Method to delete a file or directory on HPSS via hsi Parameters @@ -294,11 +302,11 @@ def mkdir(self, target: str, hsi_flags: str = "-q -e", mkdir_flags: str = "") -> args.extend(mkdir_flags.split(" ")) args.append(target) - output = self.hsi(args) + output = self._hsi(args) return output - def ls(self, target: str, hsi_flags: str = "-q", ls_flags: str = "", + def ls(self, target: str, hsi_flags: str = "", ls_flags: str = "", ignore_missing: bool = False) -> str: """ Method to list files/directories on HPSS via hsi @@ -335,7 +343,7 @@ def ls(self, target: str, hsi_flags: str = "-q", ls_flags: str = "", args.extend(ls_flags.split(" ")) args.append(target) - output = self.hsi(args, ignore_errors=ignore_errors) + output = self._hsi(args, ignore_errors=ignore_errors) return output @@ -354,7 +362,7 @@ def exists(self, target: str) -> bool: args = ["-q", "ls", target] # Do not exit if the file is not found; do not pipe output to stdout - output = self.hsi(args, silent=True, ignore_errors=[64]) + output = self._hsi(args, silent=True, ignore_errors=[64]) if "HPSS_ENOENT" in output: return False From 759e2b3eda8bd73641aa033af81409086c5d246a Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Thu, 4 Apr 2024 16:54:55 +0000 Subject: [PATCH 20/32] Hide the htar command builder. --- src/wxflow/htar.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/wxflow/htar.py b/src/wxflow/htar.py index fdfa0de..49206d7 100644 --- a/src/wxflow/htar.py +++ b/src/wxflow/htar.py @@ -18,7 +18,7 @@ class Htar: def __init__(self) -> None: self.exe = which("htar") - def htar(self, args, silent: bool = False) -> str: + def _htar(self, args, silent: bool = False) -> str: """ Direct command builder function for htar based on the input arguments. @@ -72,7 +72,7 @@ def create(self, tarball: str, fileset: list, flags: str = "-P") -> str: args += ["-f", tarball, ' '.join(fileset)] - output = self.htar(args) + output = self._htar(args) return output @@ -117,7 +117,7 @@ def extract(self, tarball: str, fileset: list = [], flags: str = "") -> str: if len(fileset) > 0: args.append(' '.join(fileset)) - output = self.htar(args) + output = self._htar(args) return output @@ -163,6 +163,6 @@ def tell(self, tarball: str, flags: str = "", fileset: list = []) -> str: if len(fileset) > 0: args += " ".join(fileset) - output = self.htar(args) + output = self._htar(args) return output From ac8110400fe6bead953e8b166ab5cd5263551a03 Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Thu, 4 Apr 2024 17:05:44 +0000 Subject: [PATCH 21/32] Rename args to arg. --- src/wxflow/hsi.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/wxflow/hsi.py b/src/wxflow/hsi.py index 4c08cd6..2b5d6f0 100644 --- a/src/wxflow/hsi.py +++ b/src/wxflow/hsi.py @@ -29,7 +29,7 @@ def __init__(self, def_hsi_args = "-q -e"): self.exe = which("hsi", required=True) for arg in def_hsi_args.split(" "): - self.exe.add_default_arg(args) + self.exe.add_default_arg(arg) def _hsi(self, args: list, silent: bool = False, ignore_errors: list = []) -> str: """Direct command builder function for hsi based on the input arguments. From 3045a8e990eb49317f42320721d93cd63da0954e Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Thu, 4 Apr 2024 17:06:13 +0000 Subject: [PATCH 22/32] Cleanup hsi and htar tests. --- tests/test_hsi.py | 23 ++++++----------------- tests/test_htar.py | 15 ++------------- 2 files changed, 8 insertions(+), 30 deletions(-) diff --git a/tests/test_hsi.py b/tests/test_hsi.py index 74edff0..d54e00a 100644 --- a/tests/test_hsi.py +++ b/tests/test_hsi.py @@ -22,18 +22,7 @@ test_path = f'/NCEPDEV/emc-global/1year/{user}/hsi_test/test-{test_hash}' -@pytest.mark.skipif(not hsi, reason="Only runs on Hera/WCOSS2 with hpss module loaded.") -def test_hsi(): - """ - Test for the hsi command builder: - """ - output = hsi.hsi(["ls", "/NCEPDEV/"]) - - assert type(output) is str - assert "emc-global" in output - - -@pytest.mark.skipif(not hsi, reason="Only runs on Hera/WCOSS2 with hpss module loaded.") +@pytest.mark.skipif(not hsi, reason="Did not find the hsi command") def test_exists(): """ Test for checking if a target exists on HPSS @@ -42,7 +31,7 @@ def test_exists(): assert not hsi.exists("/not_a_file") -@pytest.mark.skipif(not hsi, reason="Only runs on Hera/WCOSS2 with hpss module loaded.") +@pytest.mark.skipif(not hsi, reason="Did not find the hsi command") def test_ls(): """ Test HPSS listing @@ -54,7 +43,7 @@ def test_ls(): assert "drwxr-xr-x" in output -@pytest.mark.skipif(not hsi, reason="Only runs on Hera/WCOSS2 with hpss module loaded.") +@pytest.mark.skipif(not hsi, reason="Did not find the hsi command") def test_mkdir_rmdir(): """ Test for creating a directory: @@ -73,7 +62,7 @@ def test_mkdir_rmdir(): assert not hsi.exists(test_path) -@pytest.mark.skipif(not hsi, reason="Only runs on Hera/WCOSS2 with hpss module loaded.") +@pytest.mark.skipif(not hsi, reason="Did not find the hsi command") def test_chmod(): """ Test for chmod: @@ -94,7 +83,7 @@ def test_chmod(): output = hsi.rmdir(test_path) -@pytest.mark.skipif(not hsi, reason="Only runs on Hera/WCOSS2 with hpss module loaded.") +@pytest.mark.skipif(not hsi, reason="Did not find the hsi command") def test_chgrp(): """ Test for chgrp: @@ -115,7 +104,7 @@ def test_chgrp(): output = hsi.rmdir(test_path) -@pytest.mark.skipif(not hsi, reason="Only runs on Hera/WCOSS2 with hpss module loaded.") +@pytest.mark.skipif(not hsi, reason="Did not find the hsi command") def test_put_get(tmp_path): """ Test for sending/getting a file to/from HPSS: diff --git a/tests/test_htar.py b/tests/test_htar.py index 9e256b1..ebfc34e 100644 --- a/tests/test_htar.py +++ b/tests/test_htar.py @@ -19,21 +19,10 @@ test_hash = ''.join(random.choices(string.ascii_lowercase + string.digits, k=10)) user = os.environ['USER'] -test_path = f'/NCEPDEV/emc-global/1year/{user}/hsi_test/test-{test_hash}' +test_path = f'/NCEPDEV/emc-global/1year/{user}/htar_test/test-{test_hash}' -@pytest.mark.skipif(not htar, reason="Only runs on Hera/WCOSS2 with hpss module loaded.") -def test_htar(): - """ - Test for the htar command builder: - """ - output = htar.htar(["-?"]) - - assert type(output) is str - assert "Usage" in output - - -@pytest.mark.skipif(not htar, reason="Only runs on Hera/WCOSS2 with hpss module loaded.") +@pytest.mark.skipif(not htar, reason="Did not find the htar command") def test_cvf_xvf_tell(tmp_path): """ Test creating, extracting, and listing a tarball on HPSS: From 470eb4af967af1d8fbd4fdc1ce1e9aa52b1704df Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Thu, 4 Apr 2024 17:08:28 +0000 Subject: [PATCH 23/32] Fix spacing for pynorms --- src/wxflow/hsi.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/wxflow/hsi.py b/src/wxflow/hsi.py index 2b5d6f0..64590d3 100644 --- a/src/wxflow/hsi.py +++ b/src/wxflow/hsi.py @@ -17,7 +17,7 @@ class Hsi: >>> output = hsi.chgrp("rstprod", "/HPSS/pth/to/some_file") # Change the group to rstprod """ - def __init__(self, def_hsi_args = "-q -e"): + def __init__(self, def_hsi_args="-q -e"): """Instantiate the hsi command def_hsi_args: str From cc92cd167419627454f4239b9ddddefe5dddb67d Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Thu, 4 Apr 2024 19:25:59 +0000 Subject: [PATCH 24/32] Rename args to arg_list. --- src/wxflow/hsi.py | 114 ++++++++++++++++++++++----------------------- src/wxflow/htar.py | 36 +++++++------- 2 files changed, 75 insertions(+), 75 deletions(-) diff --git a/src/wxflow/hsi.py b/src/wxflow/hsi.py index 64590d3..0599ef1 100644 --- a/src/wxflow/hsi.py +++ b/src/wxflow/hsi.py @@ -17,7 +17,7 @@ class Hsi: >>> output = hsi.chgrp("rstprod", "/HPSS/pth/to/some_file") # Change the group to rstprod """ - def __init__(self, def_hsi_args="-q -e"): + def __init__(self, def_hsi_args: list = ["-q", "-e"]): """Instantiate the hsi command def_hsi_args: str @@ -28,13 +28,13 @@ def __init__(self, def_hsi_args="-q -e"): self.exe = which("hsi", required=True) - for arg in def_hsi_args.split(" "): + for arg in def_hsi_args: self.exe.add_default_arg(arg) - def _hsi(self, args: list, silent: bool = False, ignore_errors: list = []) -> str: + def _hsi(self, arg_list: list, silent: bool = False, ignore_errors: list = []) -> str: """Direct command builder function for hsi based on the input arguments. - args: list + arg_list: list A list of arguments to sent to hsi silent: bool @@ -55,10 +55,10 @@ def _hsi(self, args: list, silent: bool = False, ignore_errors: list = []) -> st """ if silent: - output = self.exe(*args, output=str, error=str, + output = self.exe(*arg_list, output=str, error=str, ignore_errors=ignore_errors) else: - output = self.exe(*args, output=str.split, error=str.split, + output = self.exe(*arg_list, output=str.split, error=str.split, ignore_errors=ignore_errors) return output @@ -79,7 +79,7 @@ def get(self, source: str, target: str = "", hsi_flags: str = "") -> str: String of flags to send to hsi. By default, suppress login info and echo the get command. """ - args = [] + arg_list = [] # Convert to str to handle Path objects target = str(target) @@ -87,15 +87,15 @@ def get(self, source: str, target: str = "", hsi_flags: str = "") -> str: # Parse any hsi flags if len(hsi_flags) > 0: - args.extend(hsi_flags.split(" ")) + arg_list.extend(hsi_flags.split(" ")) - args.append("get") + arg_list.append("get") if len(target) == 0: - args.append(source) + arg_list.append(source) else: - args.append(target + " : " + source) + arg_list.append(target + " : " + source) - output = self._hsi(args) + output = self._hsi(arg_list) return output @@ -115,7 +115,7 @@ def put(self, source: str, target: str, hsi_flags: str = "", String of flags to send to hsi. By default, suppress login info and echo the put command. """ - args = [] + arg_list = [] # Convert to str to handle Path objects target = str(target) @@ -123,11 +123,11 @@ def put(self, source: str, target: str, hsi_flags: str = "", # Parse any hsi flags if len(hsi_flags) > 0: - args.extend(hsi_flags.split(" ")) + arg_list.extend(hsi_flags.split(" ")) - args.append("put") - args.append(source + " : " + target) - output = self._hsi(args) + arg_list.append("put") + arg_list.append(source + " : " + target) + output = self._hsi(arg_list) return output @@ -151,20 +151,20 @@ def chmod(self, mod: str, target: str, hsi_flags: str = "", Flags to send to chmod. See "hsi chmod -?" for more details. """ - args = [] + arg_list = [] # Parse any hsi flags if len(hsi_flags) > 0: - args.extend(hsi_flags.split(" ")) + arg_list.extend(hsi_flags.split(" ")) - args.append("chmod") + arg_list.append("chmod") if len(chmod_flags) > 0: - args.extend(chmod_flags.split(" ")) + arg_list.extend(chmod_flags.split(" ")) - args.append(mod) - args.append(target) - output = self._hsi(args) + arg_list.append(mod) + arg_list.append(target) + output = self._hsi(arg_list) return output @@ -187,20 +187,20 @@ def chgrp(self, group_name: str, target: str, hsi_flags: str = "", Flags to send to chgrp. See "hsi chgrp -?" for more details. """ - args = [] + arg_list = [] # Parse any hsi flags if len(hsi_flags) > 0: - args.extend(hsi_flags.split(" ")) + arg_list.extend(hsi_flags.split(" ")) - args.append("chgrp") + arg_list.append("chgrp") if len(chgrp_flags) > 0: - args.extend(chgrp_flags.split(" ")) + arg_list.extend(chgrp_flags.split(" ")) - args.append(group_name) - args.append(target) - output = self._hsi(args) + arg_list.append(group_name) + arg_list.append(target) + output = self._hsi(arg_list) return output @@ -226,21 +226,21 @@ def rm(self, target: str, hsi_flags: str = "", rm_flags: str = "") -> str: output = self.rmdir(target, hsi_flags, rmdir_flags) return output - args = [] + arg_list = [] # Parse any hsi flags if len(hsi_flags) > 0: - args.extend(hsi_flags.split(" ")) + arg_list.extend(hsi_flags.split(" ")) - args.append("rm") + arg_list.append("rm") if len(rm_flags) > 0: - args.extend(rm_flags.split(" ")) + arg_list.extend(rm_flags.split(" ")) - args.append(target) + arg_list.append(target) # Ignore missing files - output = self._hsi(args, ignore_errors=[72]) + output = self._hsi(arg_list, ignore_errors=[72]) return output @@ -260,19 +260,19 @@ def rmdir(self, target: str, hsi_flags: str = "", rmdir_flags: str = "") -> str: Flags to send to rmdir. See "hsi rmdir -?" for more details. """ - args = [] + arg_list = [] # Parse any hsi flags if len(hsi_flags) > 0: - args.extend(hsi_flags.split(" ")) + arg_list.extend(hsi_flags.split(" ")) - args.append("rmdir") + arg_list.append("rmdir") if len(rmdir_flags) > 0: - args.extend(rmdir_flags.split(" ")) + arg_list.extend(rmdir_flags.split(" ")) - args.append(target) - output = self._hsi(args) + arg_list.append(target) + output = self._hsi(arg_list) return output @@ -289,20 +289,20 @@ def mkdir(self, target: str, hsi_flags: str = "", mkdir_flags: str = "") -> str: and echo the mkdir command. """ - args = [] + arg_list = [] # Parse any hsi flags if len(hsi_flags) > 0: - args.extend(hsi_flags.split(" ")) + arg_list.extend(hsi_flags.split(" ")) # The only flag available for mkdir is -p, which we will use. - args.extend(["mkdir", "-p"]) + arg_list.extend(["mkdir", "-p"]) if len(mkdir_flags) > 0: - args.extend(mkdir_flags.split(" ")) + arg_list.extend(mkdir_flags.split(" ")) - args.append(target) - output = self._hsi(args) + arg_list.append(target) + output = self._hsi(arg_list) return output @@ -325,7 +325,7 @@ def ls(self, target: str, hsi_flags: str = "", ls_flags: str = "", Flag to ignore missing files """ - args = [] + arg_list = [] if ignore_missing: ignore_errors = [64] @@ -334,16 +334,16 @@ def ls(self, target: str, hsi_flags: str = "", ls_flags: str = "", # Parse any hsi flags if len(hsi_flags) > 0: - args.extend(hsi_flags.split(" ")) + arg_list.extend(hsi_flags.split(" ")) - args.append("ls") + arg_list.append("ls") # Parse any ls flags if len(ls_flags) > 0: - args.extend(ls_flags.split(" ")) + arg_list.extend(ls_flags.split(" ")) - args.append(target) - output = self._hsi(args, ignore_errors=ignore_errors) + arg_list.append(target) + output = self._hsi(arg_list, ignore_errors=ignore_errors) return output @@ -359,10 +359,10 @@ def exists(self, target: str) -> bool: True if the target exists on HPSS. """ - args = ["-q", "ls", target] + arg_list = ["-q", "ls", target] # Do not exit if the file is not found; do not pipe output to stdout - output = self._hsi(args, silent=True, ignore_errors=[64]) + output = self._hsi(arg_list, silent=True, ignore_errors=[64]) if "HPSS_ENOENT" in output: return False diff --git a/src/wxflow/htar.py b/src/wxflow/htar.py index 49206d7..c8f3ddd 100644 --- a/src/wxflow/htar.py +++ b/src/wxflow/htar.py @@ -18,13 +18,13 @@ class Htar: def __init__(self) -> None: self.exe = which("htar") - def _htar(self, args, silent: bool = False) -> str: + def _htar(self, arg_list: list, silent: bool = False) -> str: """ Direct command builder function for htar based on the input arguments. Parameters: ----------- - args: list + arg_list: list List of string arguments to send to htar silent: bool @@ -41,9 +41,9 @@ def _htar(self, args, silent: bool = False) -> str: """ if silent: - output = self.exe(*args, output=str, error=str) + output = self.exe(*arg_list, output=str, error=str) else: - output = self.exe(*args, output=str.split, error=str.split) + output = self.exe(*arg_list, output=str.split, error=str.split) return output @@ -61,18 +61,18 @@ def create(self, tarball: str, fileset: list, flags: str = "-P") -> str: fileset : list List containing filenames, patterns, or directories to archive """ - args = ["-c"] + arg_list = ["-c"] # Parse any htar flags if len(flags) > 0: - args += flags.split(" ") + arg_list += flags.split(" ") if len(fileset) == 0: raise ValueError("Input fileset is empty, cannot create archive") - args += ["-f", tarball, ' '.join(fileset)] + arg_list += ["-f", tarball, ' '.join(fileset)] - output = self._htar(args) + output = self._htar(arg_list) return output @@ -106,18 +106,18 @@ def extract(self, tarball: str, fileset: list = [], flags: str = "") -> str: List containing filenames, patterns, or directories to extract from the archive. If empty, then all files will be extracted. """ - args = ["-x"] + arg_list = ["-x"] # Parse any htar flags if len(flags) > 0: - args += flags.split(" ") + arg_list += flags.split(" ") - args += ["-f", tarball] + arg_list += ["-f", tarball] if len(fileset) > 0: - args.append(' '.join(fileset)) + arg_list.append(' '.join(fileset)) - output = self._htar(args) + output = self._htar(arg_list) return output @@ -152,17 +152,17 @@ def tell(self, tarball: str, flags: str = "", fileset: list = []) -> str: List containing filenames, patterns, or directories to list. If empty, then all files will be listed. """ - args = ["-t"] + arg_list = ["-t"] # Parse any htar flags if len(flags) > 0: - args += [flags.split(" ")] + arg_list += [flags.split(" ")] - args += ["-f", tarball] + arg_list += ["-f", tarball] if len(fileset) > 0: - args += " ".join(fileset) + arg_list += " ".join(fileset) - output = self._htar(args) + output = self._htar(arg_list) return output From 0798c10432988c2a52cf09bf3de1184a9201a533 Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Fri, 5 Apr 2024 18:05:06 +0000 Subject: [PATCH 25/32] Parse file lists to htar as individual arguments. --- src/wxflow/htar.py | 7 +++++-- tests/test_htar.py | 15 +++++++-------- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/src/wxflow/htar.py b/src/wxflow/htar.py index c8f3ddd..39a6d41 100644 --- a/src/wxflow/htar.py +++ b/src/wxflow/htar.py @@ -70,7 +70,10 @@ def create(self, tarball: str, fileset: list, flags: str = "-P") -> str: if len(fileset) == 0: raise ValueError("Input fileset is empty, cannot create archive") - arg_list += ["-f", tarball, ' '.join(fileset)] + arg_list += ["-f", tarball] + + # Convert filenames in fileset to strings to handle Path objects + arg_list.extend([str(filename) for filename in fileset]) output = self._htar(arg_list) @@ -115,7 +118,7 @@ def extract(self, tarball: str, fileset: list = [], flags: str = "") -> str: arg_list += ["-f", tarball] if len(fileset) > 0: - arg_list.append(' '.join(fileset)) + arg_list.extend([str(filename) for filename in fileset]) output = self._htar(arg_list) diff --git a/tests/test_htar.py b/tests/test_htar.py index ebfc34e..387dd57 100644 --- a/tests/test_htar.py +++ b/tests/test_htar.py @@ -34,23 +34,22 @@ def test_cvf_xvf_tell(tmp_path): # Create temporary directories input_dir_path = tmp_path / 'my_input_dir' input_dir_path.mkdir() - # Create an empty file to send - in_tmp_file = input_dir_path / 'a.txt' - in_tmp_file.touch() - in_tmp_file.write_text("Contents of a.txt") + # Create files to send + in_tmp_files = [input_dir_path / 'a.txt', input_dir_path / 'b.txt'] + for f in in_tmp_files: + f.touch() + f.write_text("Some contents") test_tarball = test_path + "/test.tar" # Create the archive file - output = htar.cvf(test_tarball, [str(in_tmp_file)]) - print("output::") - print(output) + output = htar.cvf(test_tarball, in_tmp_files) assert "a.txt" in output assert hsi.exists(test_tarball) # Extract the test archive - output = htar.xvf(test_tarball) + output = htar.xvf(test_tarball, in_tmp_files) assert "a.txt" in output From 1652b54080c8471ce3cbf5456d39fabf41e2d5f8 Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Mon, 8 Apr 2024 14:35:50 +0000 Subject: [PATCH 26/32] hsi: better documentation; add helper function; catch empty args. --- src/wxflow/hsi.py | 289 ++++++++++++++++++++++++++++------------------ 1 file changed, 179 insertions(+), 110 deletions(-) diff --git a/src/wxflow/hsi.py b/src/wxflow/hsi.py index 0599ef1..97e0805 100644 --- a/src/wxflow/hsi.py +++ b/src/wxflow/hsi.py @@ -1,5 +1,7 @@ from .executable import which +from typing import Union, List + __all__ = ['Hsi'] @@ -17,43 +19,63 @@ class Hsi: >>> output = hsi.chgrp("rstprod", "/HPSS/pth/to/some_file") # Change the group to rstprod """ - def __init__(self, def_hsi_args: list = ["-q", "-e"]): + def __init__(self, quiet: bool = True, echo_commands:bool = True, opts: Union[str, List] = []): """Instantiate the hsi command - def_hsi_args: str - List of default arguments to send to hsi. The defaults are - -q: run in quiet mode (do not print login information) - -e: echo each command + Parameters: + ----------- + quiet : bool + Run hsi in quiet mode (suppress login information, transfer info, etc) + echo_commands : bool + Echo each command. Some commands will still not echo (e.g. chmod). + opts : str | list + Additional arguments to send to each hsi command. """ self.exe = which("hsi", required=True) - for arg in def_hsi_args: + hsi_args = [] + + if quiet: + hsi_args.append("-q") + + if echo_commands: + hsi_args.append("-e") + + hsi_args.extend(Hsi._split_opts(hsi_args)) + + for arg in hsi_args: self.exe.add_default_arg(arg) def _hsi(self, arg_list: list, silent: bool = False, ignore_errors: list = []) -> str: """Direct command builder function for hsi based on the input arguments. - arg_list: list - A list of arguments to sent to hsi + Parameters: + arg_list : list + A list of arguments to sent to hsi - silent: bool - Whether the output of the hsi command should be written to stdout + silent : bool + Whether the output of the hsi command should be written to stdout - ignore_errors: list - List of error numbers to ignore. For example, hsi returns error - number 64 if a target file does not exist on HPSS. + ignore_errors : list + List of error numbers to ignore. For example, hsi returns error + number 64 if a target file does not exist on HPSS. - Return: str - Concatenated output and error of the hsi command. + Returns + ------- + output : str + Concatenated output and error of the hsi command. Example: -------- >>> hsi = Hsi() >>> # Execute `hsi get some_local_file : /some/hpss/file` - >>> hsi.hsi(["get","some_local_file : /some/hpss/file"]) + >>> hsi._hsi(["get","some_local_file : /some/hpss/file"]) """ + # Remove any empty arguments which can cause issues for hsi + arg_list = [arg for arg in arg_list if arg != ""] + if silent: output = self.exe(*arg_list, output=str, error=str, ignore_errors=ignore_errors) @@ -63,7 +85,7 @@ def _hsi(self, arg_list: list, silent: bool = False, ignore_errors: list = []) - return output - def get(self, source: str, target: str = "", hsi_flags: str = "") -> str: + def get(self, source: str, target = None, opts: Union[List, str] = []) -> str: """ Method to get a file from HPSS via hsi Parameters @@ -75,31 +97,31 @@ def get(self, source: str, target: str = "", hsi_flags: str = "") -> str: Location on the local machine to place the file. If not specified, then the file will be placed in the current directory. - hsi_flags : str - String of flags to send to hsi. By default, suppress login info and - echo the get command. + opts : str | list + List or string of additional options to send to hsi command. + + Returns + ------- + output : str + Concatenated output and error of the hsi get command. """ arg_list = [] # Convert to str to handle Path objects - target = str(target) source = str(source) + target = str(target) if target is not None else None - # Parse any hsi flags - if len(hsi_flags) > 0: - arg_list.extend(hsi_flags.split(" ")) + # Parse any hsi options + arg_list.extend(Hsi._split_opts(opts)) arg_list.append("get") - if len(target) == 0: - arg_list.append(source) - else: - arg_list.append(target + " : " + source) + args_list.append(source) if target is None else arg_list.append(f"{target} : {source}") output = self._hsi(arg_list) return output - def put(self, source: str, target: str, hsi_flags: str = "", + def put(self, source: str, target: str, opts: Union[List, str] = [], listing_file: str = None) -> str: """ Method to put a file onto HPSS via hsi @@ -111,9 +133,13 @@ def put(self, source: str, target: str, hsi_flags: str = "", target : str Full path of the target location of the file on HPSS. - hsi_flags : str - String of flags to send to hsi. By default, suppress login info - and echo the put command. + opts : str | List + List or string of additional options to send to hsi. + + Returns + ------- + output : str + Concatenated output and error of the hsi put command. """ arg_list = [] @@ -121,9 +147,8 @@ def put(self, source: str, target: str, hsi_flags: str = "", target = str(target) source = str(source) - # Parse any hsi flags - if len(hsi_flags) > 0: - arg_list.extend(hsi_flags.split(" ")) + # Parse any hsi options + arg_list.extend(Hsi._split_opts(opts)) arg_list.append("put") arg_list.append(source + " : " + target) @@ -131,8 +156,8 @@ def put(self, source: str, target: str, hsi_flags: str = "", return output - def chmod(self, mod: str, target: str, hsi_flags: str = "", - chmod_flags: str = "") -> str: + def chmod(self, mod: str, target: str, hsi_opts: Union[List, str] = "", + chmod_opts: Union[List, str] = "") -> str: """ Method to change the permissions of a file or directory on HPSS Parameters @@ -144,23 +169,27 @@ def chmod(self, mod: str, target: str, hsi_flags: str = "", target : str Full path of the target location of the file on HPSS. - hsi_flags : str - String of flags to send to hsi. By default, suppress login info. + hsi_opts : list | str + Options to send to hsi. - chmod_flags : str - Flags to send to chmod. See "hsi chmod -?" for more details. + chmod_opts : list | str + Options to send to chmod. See "hsi chmod -?" for more details. + + Return + ------ + output : str + Concatenated output and error of the hsi chmod command. """ arg_list = [] - # Parse any hsi flags - if len(hsi_flags) > 0: - arg_list.extend(hsi_flags.split(" ")) + # Parse any hsi options + arg_list.extend(Hsi._split_opts(hsi_opts)) arg_list.append("chmod") - if len(chmod_flags) > 0: - arg_list.extend(chmod_flags.split(" ")) + # Parse any chmod options + arg_list.extend(Hsi._split_opts(chmod_opts)) arg_list.append(mod) arg_list.append(target) @@ -168,8 +197,8 @@ def chmod(self, mod: str, target: str, hsi_flags: str = "", return output - def chgrp(self, group_name: str, target: str, hsi_flags: str = "", - chgrp_flags: str = "") -> str: + def chgrp(self, group_name: str, target: str, hsi_opts: str = "", + chgrp_opts: str = "") -> str: """ Method to change the group of a file or directory on HPSS Parameters @@ -180,23 +209,27 @@ def chgrp(self, group_name: str, target: str, hsi_flags: str = "", target : str Full path of the target location of the file on HPSS. - hsi_flags : str - String of flags to send to hsi. By default, suppress login info. + hsi_opts : str + String of options to send to hsi. + + chgrp_opts : str + Options to send to chgrp. See "hsi chgrp -?" for more details. - chgrp_flags : str - Flags to send to chgrp. See "hsi chgrp -?" for more details. + Returns + ------- + output : str + Concatenated output and error of the hsi chgrp command. """ arg_list = [] - # Parse any hsi flags - if len(hsi_flags) > 0: - arg_list.extend(hsi_flags.split(" ")) + # Parse any hsi options + arg_list.extend(Hsi._split_opts(hsi_opts)) arg_list.append("chgrp") - if len(chgrp_flags) > 0: - arg_list.extend(chgrp_flags.split(" ")) + # Parse any chgrp options + arg_list.extend(Hsi._split_opts(chgrp_opts)) arg_list.append(group_name) arg_list.append(target) @@ -204,7 +237,7 @@ def chgrp(self, group_name: str, target: str, hsi_flags: str = "", return output - def rm(self, target: str, hsi_flags: str = "", rm_flags: str = "") -> str: + def rm(self, target: str, hsi_opts: str = "", rm_opts: str = "") -> str: """ Method to delete a file or directory on HPSS via hsi Parameters @@ -212,30 +245,34 @@ def rm(self, target: str, hsi_flags: str = "", rm_flags: str = "") -> str: target : str Full path of the target location of the file on HPSS. - hsi_flags : str - String of flags to send to hsi. By default, suppress login info - and echo the rm command. + hsi_opts : str + String of options to send to hsi. + + rm_opts : str + Options to send to rm. See "hsi rm -?" for more details. - rm_flags : str - Flags to send to rm. See "hsi rm -?" for more details. + Returns + ------- + output : str + Concatenated output and error of the hsi rm command. """ # Call rmdir if recursive (-r) flag present - if "-r" in rm_flags: - rmdir_flags = rm_flags.replace("-r", "") - output = self.rmdir(target, hsi_flags, rmdir_flags) + # NOTE this will ONLY remove empty directories + if "-r" in rm_opts: + rmdir_opts = [ opt for opts in Hsi._split_opts(rm_opts) if opt != "-r" ] + output = self.rmdir(target, hsi_opts, rmdir_opts) return output arg_list = [] - # Parse any hsi flags - if len(hsi_flags) > 0: - arg_list.extend(hsi_flags.split(" ")) + # Parse any hsi options + arg_list.extend(Hsi._split_opts(hsi_opts)) arg_list.append("rm") - if len(rm_flags) > 0: - arg_list.extend(rm_flags.split(" ")) + # Parse any rm options + arg_list.extend(Hsi._split_opts(rm_opts)) arg_list.append(target) @@ -244,39 +281,42 @@ def rm(self, target: str, hsi_flags: str = "", rm_flags: str = "") -> str: return output - def rmdir(self, target: str, hsi_flags: str = "", rmdir_flags: str = "") -> str: - """ Method to delete a directory on HPSS via hsi + def rmdir(self, target: str, hsi_opts: str = "", rmdir_opts: str = "") -> str: + """ Method to delete an empty directory on HPSS via hsi Parameters ---------- target : str Full path of the target location of the file on HPSS. - hsi_flags : str - String of flags to send to hsi. By default, suppress login info - and echo the rmdir command. + hsi_opts : str + String of options to send to hsi. - rmdir_flags : str - Flags to send to rmdir. See "hsi rmdir -?" for more details. + rmdir_opts : str + Options to send to rmdir. See "hsi rmdir -?" for more details. + + Returns + ------- + output : str + Concatenated output and error of the hsi rmdir command. """ arg_list = [] - # Parse any hsi flags - if len(hsi_flags) > 0: - arg_list.extend(hsi_flags.split(" ")) + # Parse any hsi options + arg_list.extend(Hsi._split_opts(hsi_opts)) arg_list.append("rmdir") - if len(rmdir_flags) > 0: - arg_list.extend(rmdir_flags.split(" ")) + # Parse any rmdir options + arg_list.extend(Hsi._split_opts(rmdir_opts)) arg_list.append(target) output = self._hsi(arg_list) return output - def mkdir(self, target: str, hsi_flags: str = "", mkdir_flags: str = "") -> str: + def mkdir(self, target: str, hsi_opts: str = "", mkdir_opts: str = "") -> str: """ Method to delete a file or directory on HPSS via hsi Parameters @@ -284,45 +324,53 @@ def mkdir(self, target: str, hsi_flags: str = "", mkdir_flags: str = "") -> str: target : str Full path of the target location of the file on HPSS. - hsi_flags : str - String of flags to send to hsi. By default, suppress login info - and echo the mkdir command. + hsi_opts : str + String of options to send to hsi. + + Returns + ------- + output : str + Concatenated output and error of the hsi mkdir command. """ arg_list = [] - # Parse any hsi flags - if len(hsi_flags) > 0: - arg_list.extend(hsi_flags.split(" ")) + # Parse any hsi options + arg_list.extend(Hsi._split_opts(hsi_opts)) # The only flag available for mkdir is -p, which we will use. arg_list.extend(["mkdir", "-p"]) - if len(mkdir_flags) > 0: - arg_list.extend(mkdir_flags.split(" ")) + # Parse any mkdir options + arg_list.extend(Hsi._split_opts(mkdir_opts)) arg_list.append(target) output = self._hsi(arg_list) return output - def ls(self, target: str, hsi_flags: str = "", ls_flags: str = "", + def ls(self, target: str, hsi_opts: str = "", ls_opts: str = "", ignore_missing: bool = False) -> str: """ Method to list files/directories on HPSS via hsi Parameters ---------- target : str - Full path of the target location on HPSS. + Full path of the target location on HPSS. + + hsi_opts : str + String of options to send to hsi. - hsi_flags : str - String of flags to send to hsi. By default, suppress login info. + ls_opts : str + Options to send to ls. See "hsi ls -?" for more details. - ls_flags : str - Flags to send to ls. See "hsi ls -?" for more details. + ignore_missing : bool + Flag to ignore missing files - ignore_missing: bool - Flag to ignore missing files + Returns + ------- + output : str + Concatenated output and error of the hsi ls command. """ arg_list = [] @@ -332,15 +380,13 @@ def ls(self, target: str, hsi_flags: str = "", ls_flags: str = "", else: ignore_errors = [] - # Parse any hsi flags - if len(hsi_flags) > 0: - arg_list.extend(hsi_flags.split(" ")) + # Parse any hsi options + arg_list.extend(Hsi._split_opts(hsi_opts)) arg_list.append("ls") - # Parse any ls flags - if len(ls_flags) > 0: - arg_list.extend(ls_flags.split(" ")) + # Parse any ls options + arg_list.extend(Hsi._split_opts(ls_opts)) arg_list.append(target) output = self._hsi(arg_list, ignore_errors=ignore_errors) @@ -355,7 +401,9 @@ def exists(self, target: str) -> bool: target : str Full path of the target location on HPSS. - Return: bool + Returns + ------- + pattern_exists : bool True if the target exists on HPSS. """ @@ -365,9 +413,30 @@ def exists(self, target: str) -> bool: output = self._hsi(arg_list, silent=True, ignore_errors=[64]) if "HPSS_ENOENT" in output: - return False + pattern_exists = False # Catch wildcards elif f"Warning: No matching names located for '{target}'" in output: - return False + pattern_exists = False else: - return True + pattern_exists = True + + return pattern_exists + + @staticmethod + def _split_opts(opts: Union[List, str] = "") -> list: + """ Method to split input list or string of hsi options + + Parameters + ---------- + opts : list | str + Input list or string of options to send to hsi or subcommand + + Returns + ------- + split_opts : list + List of options to send to hsi or the hsi subcommand + """ + + split_opts = opts.split(" ") if isinstance(opts, str) else opts + + return split_opts From ef745acf910960e8b7fbab7cd742cc268f4adc5c Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Mon, 8 Apr 2024 14:37:06 +0000 Subject: [PATCH 27/32] htar: better documentation; add helper function; catch empty args --- src/wxflow/htar.py | 139 ++++++++++++++++++++++++++++++--------------- 1 file changed, 94 insertions(+), 45 deletions(-) diff --git a/src/wxflow/htar.py b/src/wxflow/htar.py index 39a6d41..c87b13f 100644 --- a/src/wxflow/htar.py +++ b/src/wxflow/htar.py @@ -1,5 +1,7 @@ from .executable import which +from typing import Union, List + __all__ = ['Htar'] @@ -24,22 +26,27 @@ def _htar(self, arg_list: list, silent: bool = False) -> str: Parameters: ----------- - arg_list: list - List of string arguments to send to htar + arg_list : list + List of string arguments to send to htar - silent: bool - Flag to suppress output to stdout + silent : bool + Flag to suppress output to stdout - Return: str - Output from the htar command + Returns + ------- + output : str + Concatenated output and error from the htar command Examples: --------- >>> htar = Htar() - >>> # Run `htar -cvf /path/to/hpss/archive.tar file1 file2 file-* - >>> htar.htar("-cvf", "/path/to/hpss/archive.tar", "file1 file2 file-*") + >>> # Run `htar -cvPf /path/to/hpss/archive.tar file1 file2 file-* + >>> htar._htar("-cvPf", "/path/to/hpss/archive.tar", "file1 file2 file-*") """ + # Remove any empty arguments which can cause issues for htar + arg_list = [arg for arg in arg_list if arg != ""] + if silent: output = self.exe(*arg_list, output=str, error=str) else: @@ -47,85 +54,98 @@ def _htar(self, arg_list: list, silent: bool = False) -> str: return output - def create(self, tarball: str, fileset: list, flags: str = "-P") -> str: + def create(self, tarball: str, fileset: Union[List, str], opts: Union[List, str] = "-P") -> str: """ Method to write an archive to HPSS Parameters ---------- - flags : str - String of flags to send to htar. + opts : str | list + Options to send to htar. By default, "-P" (create parent directories). tarball : str Full path location on HPSS to create the archive. - fileset : list + fileset : list | str List containing filenames, patterns, or directories to archive + + Returns + ------- + output : str + Concatenated output and error of the htar command. """ arg_list = ["-c"] - # Parse any htar flags - if len(flags) > 0: - arg_list += flags.split(" ") + # Parse any htar options + arg_list.extend(Htar._split_opts(opts)) if len(fileset) == 0: raise ValueError("Input fileset is empty, cannot create archive") - arg_list += ["-f", tarball] + arg_list.extend(["-f", tarball]) # Convert filenames in fileset to strings to handle Path objects - arg_list.extend([str(filename) for filename in fileset]) + arg_list.extend([str(filename) for filename in Htar._split_opts(fileset)]) output = self._htar(arg_list) return output - def cvf(self, tarball: str, fileset: list) -> str: - """ Method to write an archive to HPSS verbosely (without flags). + def cvf(self, tarball: str, fileset: Union[List, str]) -> str: + """ Method to write an archive to HPSS verbosely (without options). Parameters ---------- tarball : str Full path location on HPSS to create the archive. - fileset : list + fileset : list | str List containing filenames, patterns, or directories to archive + + Returns + ------- + output : str + Concatenated output and error from the htar command """ - output = self.create(tarball, fileset, flags="-v -P") + output = self.create(tarball, fileset, opts="-v -P") return output - def extract(self, tarball: str, fileset: list = [], flags: str = "") -> str: + def extract(self, tarball: str, fileset: Union[List, str] = [], opts: Union[List, str] = "") -> str: """ Method to extract an archive from HPSS via htar Parameters ---------- - flags : str - String of flags to send to htar. + opts : str + String of options to send to htar. tarball : str Full path location of an archive on HPSS to extract from. - fileset : list - List containing filenames, patterns, or directories to extract from + fileset : list | str + Filenames, patterns, or directories to extract from the archive. If empty, then all files will be extracted. + + Returns + ------- + output : str + Concatenated output and error from the htar command """ arg_list = ["-x"] - # Parse any htar flags - if len(flags) > 0: - arg_list += flags.split(" ") + # Parse any htar options + arg_list.extend(Htar._split_opts(opts)) arg_list += ["-f", tarball] - if len(fileset) > 0: - arg_list.extend([str(filename) for filename in fileset]) + # Convert filename(s) to str to handle Path objects + arg_list.extend([str(filename) for filename in Htar._split_opts(fileset)]) output = self._htar(arg_list) return output def xvf(self, tarball: str = "", fileset: list = []) -> str: - """ Method to extract an archive from HPSS verbosely (without flags). + """ Method to extract an archive from HPSS verbosely (without options). Parameters ---------- @@ -135,37 +155,66 @@ def xvf(self, tarball: str = "", fileset: list = []) -> str: fileset : list List containing filenames, patterns, or directories to extract from the archive. If empty, then all files will be extracted. + + Returns + ------- + output : str + Concatenated output and error from the htar command """ - output = self.extract(tarball, fileset, flags="-v") + output = self.extract(tarball, fileset, opts="-v") return output - def tell(self, tarball: str, flags: str = "", fileset: list = []) -> str: + def tell(self, tarball: str, opts: Union[List, str] = "", fileset: Union[List, str] = []) -> str: """ Method to list the contents of an archive on HPSS Parameters ---------- - flags : str - String of flags to send to htar. + opts : str + String of options to send to htar. tarball : str Full path location on HPSS to list the contents of. - fileset : list - List containing filenames, patterns, or directories to list. - If empty, then all files will be listed. + fileset : list | str + Filenames, patterns, or directories to list from + the archive. If empty, then all files will be listed. + + Returns + ------- + output : str + Concatenated output and error from the htar command """ + print("enter") arg_list = ["-t"] - # Parse any htar flags - if len(flags) > 0: - arg_list += [flags.split(" ")] + # Parse any htar options + arg_list.extend(Htar._split_opts(opts)) - arg_list += ["-f", tarball] + arg_list.extend(["-f", tarball]) - if len(fileset) > 0: - arg_list += " ".join(fileset) + # Convert filename(s) to str to handle Path objects + arg_list.extend([str(filename) for filename in Htar._split_opts(fileset)]) output = self._htar(arg_list) return output + + @staticmethod + def _split_opts(opts: Union[List, str] = "") -> list: + """ Method to split input list or string of htar options + + Parameters + ---------- + opts : list | str + Input list or string of options to send to htar + + Returns + ------- + split_opts : list + List of options to send to htar + """ + + split_opts = opts.split(" ") if isinstance(opts, str) else opts + + return split_opts From 0c9049fc726a229bee67b5a0532c55ca9de1e8fe Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Mon, 8 Apr 2024 14:39:24 +0000 Subject: [PATCH 28/32] Actually return group ID. --- src/wxflow/fsutils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/wxflow/fsutils.py b/src/wxflow/fsutils.py index 9dee60d..af9e5b8 100644 --- a/src/wxflow/fsutils.py +++ b/src/wxflow/fsutils.py @@ -92,7 +92,7 @@ def cp(source: str, target: str) -> None: # Group ID number for a given group name def get_gid(group_name: str): try: - group_id = grp.getgrnam(group_name) + group_id = grp.getgrnam(group_name).gr_gid except KeyError: raise KeyError(f"{group_name} is not a valid group name.") @@ -100,7 +100,7 @@ def get_gid(group_name: str): # Change the group of a target file or directory -def chgrp(target, group_name, recursive=False): +def chgrp(group_name, target, recursive=False): # TODO add recursive option gid = get_gid(group_name) uid = os.stat(target).st_uid From 1a661b1ea393287d00c2b43379e90f7b83292021 Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Mon, 8 Apr 2024 14:41:07 +0000 Subject: [PATCH 29/32] Update hsi tests for renamed keywords. --- tests/test_hsi.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/test_hsi.py b/tests/test_hsi.py index d54e00a..8d2efec 100644 --- a/tests/test_hsi.py +++ b/tests/test_hsi.py @@ -39,7 +39,7 @@ def test_ls(): output = hsi.ls("/NCEPDEV/") assert "emc-global" in output - output = hsi.ls("/NCEPDEV/", ls_flags="-l") + output = hsi.ls("/NCEPDEV/", ls_opts="-l") assert "drwxr-xr-x" in output @@ -72,10 +72,10 @@ def test_chmod(): output = hsi.mkdir(test_path) # Change the mode of the test path - output = hsi.chmod("750", test_path, chmod_flags="-R") + output = hsi.chmod("750", test_path, chmod_opts="-R") # Check that the mode was changed - output = hsi.ls(test_path, ls_flags="-d -l") + output = hsi.ls(test_path, ls_opts="-d -l") assert "drwxr-x---" in output @@ -93,10 +93,10 @@ def test_chgrp(): output = hsi.mkdir(test_path) # Change the group of the test path - output = hsi.chgrp("rstprod", test_path, chgrp_flags="-R") + output = hsi.chgrp("rstprod", test_path, chgrp_opts="-R") # Check that the group was changed - output = hsi.ls(test_path, ls_flags="-d -l") + output = hsi.ls(test_path, ls_opts="-d -l") assert "rstprod" in output From ccfcf92761b1b84130f08b46103870065a89bc23 Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Mon, 8 Apr 2024 14:42:19 +0000 Subject: [PATCH 30/32] Sort hsi and htar imports. --- src/wxflow/hsi.py | 4 ++-- src/wxflow/htar.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/wxflow/hsi.py b/src/wxflow/hsi.py index 97e0805..72ad107 100644 --- a/src/wxflow/hsi.py +++ b/src/wxflow/hsi.py @@ -1,6 +1,6 @@ -from .executable import which +from typing import List, Union -from typing import Union, List +from .executable import which __all__ = ['Hsi'] diff --git a/src/wxflow/htar.py b/src/wxflow/htar.py index c87b13f..03fdbe8 100644 --- a/src/wxflow/htar.py +++ b/src/wxflow/htar.py @@ -1,6 +1,6 @@ -from .executable import which +from typing import List, Union -from typing import Union, List +from .executable import which __all__ = ['Htar'] From 2961846ccb7f4e277e9f26bce0d6e98029e6985c Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Mon, 8 Apr 2024 14:43:52 +0000 Subject: [PATCH 31/32] Fix pycodestyle in hsi.py. --- src/wxflow/hsi.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/wxflow/hsi.py b/src/wxflow/hsi.py index 72ad107..d1da7f3 100644 --- a/src/wxflow/hsi.py +++ b/src/wxflow/hsi.py @@ -19,7 +19,7 @@ class Hsi: >>> output = hsi.chgrp("rstprod", "/HPSS/pth/to/some_file") # Change the group to rstprod """ - def __init__(self, quiet: bool = True, echo_commands:bool = True, opts: Union[str, List] = []): + def __init__(self, quiet: bool = True, echo_commands: bool = True, opts: Union[str, List] = []): """Instantiate the hsi command Parameters: @@ -85,7 +85,7 @@ def _hsi(self, arg_list: list, silent: bool = False, ignore_errors: list = []) - return output - def get(self, source: str, target = None, opts: Union[List, str] = []) -> str: + def get(self, source: str, target=None, opts: Union[List, str] = []) -> str: """ Method to get a file from HPSS via hsi Parameters @@ -260,7 +260,7 @@ def rm(self, target: str, hsi_opts: str = "", rm_opts: str = "") -> str: # Call rmdir if recursive (-r) flag present # NOTE this will ONLY remove empty directories if "-r" in rm_opts: - rmdir_opts = [ opt for opts in Hsi._split_opts(rm_opts) if opt != "-r" ] + rmdir_opts = [opt for opts in Hsi._split_opts(rm_opts) if opt != "-r"] output = self.rmdir(target, hsi_opts, rmdir_opts) return output From 7d7c83dc95515b9997d96d233849ae2876915a25 Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Mon, 8 Apr 2024 20:34:00 +0000 Subject: [PATCH 32/32] Add recursive flag for rm. --- src/wxflow/hsi.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/wxflow/hsi.py b/src/wxflow/hsi.py index d1da7f3..25333d5 100644 --- a/src/wxflow/hsi.py +++ b/src/wxflow/hsi.py @@ -237,7 +237,7 @@ def chgrp(self, group_name: str, target: str, hsi_opts: str = "", return output - def rm(self, target: str, hsi_opts: str = "", rm_opts: str = "") -> str: + def rm(self, target: str, recursive: bool = False, hsi_opts: str = "", rm_opts: str = "") -> str: """ Method to delete a file or directory on HPSS via hsi Parameters @@ -251,16 +251,18 @@ def rm(self, target: str, hsi_opts: str = "", rm_opts: str = "") -> str: rm_opts : str Options to send to rm. See "hsi rm -?" for more details. + recursive : bool + Flag to indicate a call to rmdir. + Returns ------- output : str Concatenated output and error of the hsi rm command. """ - # Call rmdir if recursive (-r) flag present + # Call rmdir if recursive is set # NOTE this will ONLY remove empty directories - if "-r" in rm_opts: - rmdir_opts = [opt for opts in Hsi._split_opts(rm_opts) if opt != "-r"] + if recursive: output = self.rmdir(target, hsi_opts, rmdir_opts) return output