Add tests for configuration.py (#1192)

Add tests to configuration.py
Update README.md to illustrate developers how to run tests locally and manually
Add .gitignore to pygw

.github/workflows/pytests.yaml

@@ -15,12 +15,12 @@ jobs:
         with:
           python-version: ${{ matrix.python }}
 
-      - name: Install (upgrade) dependencies
+      - name: Install (upgrade) python dependencies
         run: |
           pip install --upgrade pip
 
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
         with:
           path: global-workflow
 

ush/python/pygw/.gitignore

@@ -0,0 +1,139 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+pip-wheel-metadata/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+.python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# Sphinx documentation
+docs/_build/
+
+# Editor backup files (Emacs, vim)
+*~
+*.sw[a-p]
+
+# Pycharm IDE files
+.idea/

ush/python/pygw/README.md

@@ -6,16 +6,31 @@ Python tools specifically for global applications
 Simple installation instructions
 ```sh
 $> git clone https://github.com/noaa-emc/global-workflow
-$> cd global-workflow/ush/python
+$> cd global-workflow/ush/python/pygw
 $> pip install .
 ```
 
 It is not required to install this package.  Instead, 
 ```sh
-$> cd global-workflow/ush/python
+$> cd global-workflow/ush/python/pygw
 $> export PYTHONPATH=$PWD/src/pygw
 ```
 would put this package in the `PYTHONPATH`
 
 ### Note:
 These instructions will be updated and the tools are under development.
+
+### Running python tests:
+Simple instructions to enable executing pytests manually
+```sh
+# Create a python virtual environment and step into it
+$> cd global-workflow/ush/python/pygw
+$> python3 -m venv venv
+$> source venv/bin/activate
+
+# Install pygw with the developer requirements
+(venv) $> pip install .[dev]
+
+# Run pytests
+(venv) $> pytest -v
+```

ush/python/pygw/setup.cfg

@@ -52,7 +52,7 @@ where=src
 * = *.txt, *.md
 
 [options.extras_require]
-dev = pytest-cov>=3
+dev = pytest>=7; pytest-cov>=3
 
 [green]
 file-pattern = test_*.py

ush/python/pygw/src/pygw/configuration.py

@@ -2,14 +2,14 @@
 import os
 import random
 import subprocess
-from datetime import datetime
 from pathlib import Path
 from pprint import pprint
 from typing import Union, List, Dict, Any
 
 from pygw.attrdict import AttrDict
+from pygw.timetools import to_datetime
 
-__all__ = ['Configuration']
+__all__ = ['Configuration', 'cast_as_dtype', 'cast_strdict_as_dtypedict']
 
 
 class ShellScriptException(Exception):
@@ -32,11 +32,6 @@ class Configuration:
     (or generally for sourcing a shell script into a python dictionary)
     """
 
-    DATE_ENV_VARS = ['CDATE', 'SDATE', 'EDATE']
-    TRUTHS = ['y', 'yes', 't', 'true', '.t.', '.true.']
-    BOOLS = ['n', 'no', 'f', 'false', '.f.', '.false.'] + TRUTHS
-    BOOLS = [x.upper() for x in BOOLS] + BOOLS
-
     def __init__(self, config_dir: Union[str, Path]):
         """
         Given a directory containing config files (config.XYZ),
@@ -84,18 +79,7 @@ def parse_config(self, files: Union[str, bytes, list]) -> Dict[str, Any]:
         if isinstance(files, (str, bytes)):
             files = [files]
         files = [self.find_config(file) for file in files]
-        varbles = AttrDict()
-        for key, value in self._get_script_env(files).items():
-            if key in self.DATE_ENV_VARS:  # likely a date, convert to datetime
-                varbles[key] = datetime.strptime(value, '%Y%m%d%H')
-            elif value in self.BOOLS:  # Likely a boolean, convert to True/False
-                varbles[key] = self._true_or_not(value)
-            elif '.' in value:  # Likely a number and that too a float
-                varbles[key] = self._cast_or_not(float, value)
-            else:  # Still could be a number, may be an integer
-                varbles[key] = self._cast_or_not(int, value)
-
-        return varbles
+        return cast_strdict_as_dtypedict(self._get_script_env(files))
 
     def print_config(self, files: Union[str, bytes, list]) -> None:
         """
@@ -137,16 +121,59 @@ def _get_shell_env(scripts: List) -> Dict[str, Any]:
             varbls[entry[0:iequal]] = entry[iequal + 1:]
         return varbls
 
-    @staticmethod
-    def _cast_or_not(type, value):
+
+def cast_strdict_as_dtypedict(ctx: Dict[str, str]) -> Dict[str, Any]:
+    """
+    Environment variables are typically stored as str
+    This method attempts to translate those into datatypes
+    Parameters
+    ----------
+    ctx : dict
+          dictionary with values as str
+    Returns
+    -------
+    varbles : dict
+              dictionary with values as datatypes
+    """
+    varbles = AttrDict()
+    for key, value in ctx.items():
+        varbles[key] = cast_as_dtype(value)
+    return varbles
+
+
+def cast_as_dtype(string: str) -> Union[str, int, float, bool, Any]:
+    """
+    Cast a value into known datatype
+    Parameters
+    ----------
+    string: str
+    Returns
+    -------
+    value : str or int or float or datetime
+            default: str
+    """
+    TRUTHS = ['y', 'yes', 't', 'true', '.t.', '.true.']
+    BOOLS = ['n', 'no', 'f', 'false', '.f.', '.false.'] + TRUTHS
+    BOOLS = [x.upper() for x in BOOLS] + BOOLS + ['Yes', 'No', 'True', 'False']
+
+    def _cast_or_not(type: Any, string: str):
         try:
-            return type(value)
+            return type(string)
         except ValueError:
-            return value
+            return string
 
-    @staticmethod
-    def _true_or_not(value):
+    def _true_or_not(string: str):
         try:
-            return value.lower() in Configuration.TRUTHS
+            return string.lower() in TRUTHS
         except AttributeError:
-            return value
+            return string
+
+    try:
+        return to_datetime(string)  # Try as a datetime
+    except Exception as exc:
+        if string in BOOLS:  # Likely a boolean, convert to True/False
+            return _true_or_not(string)
+        elif '.' in string:  # Likely a number and that too a float
+            return _cast_or_not(float, string)
+        else:  # Still could be a number, may be an integer
+            return _cast_or_not(int, string)

ush/python/pygw/src/pygw/timetools.py

@@ -9,12 +9,15 @@
 
 
 _DATETIME_RE = re.compile(
-    r"(?P<year>\d{4})(-)?(?P<month>\d{2})(-)?(?P<day>\d{2})(T)?(?P<hour>\d{2})?(:)?(?P<minute>\d{2})?(:)?(?P<second>\d{2})?(Z)?")
+    r"(?P<year>\d{4})(-)?(?P<month>\d{2})(-)?(?P<day>\d{2})"
+    r"(T)?(?P<hour>\d{2})?(:)?(?P<minute>\d{2})?(:)?(?P<second>\d{2})?(Z)?")
 
 _TIMEDELTA_HOURS_RE = re.compile(
-    r"(?P<sign>[+-])?((?P<days>\d+)[d])?(T)?((?P<hours>\d+)[H])?((?P<minutes>\d+)[M])?((?P<seconds>\d+)[S])?(Z)?")
+    r"(?P<sign>[+-])?"
+    r"((?P<days>\d+)[d])?(T)?((?P<hours>\d+)[H])?((?P<minutes>\d+)[M])?((?P<seconds>\d+)[S])?(Z)?")
 _TIMEDELTA_TIME_RE = re.compile(
-    r"(?P<sign>[+-])?((?P<days>\d+)\s+day(s)?,\s)?(T)?(?P<hours>\d{1,2})?(:(?P<minutes>\d{1,2}))?(:(?P<seconds>\d{1,2}))?")
+    r"(?P<sign>[+-])?"
+    r"((?P<days>\d+)\s+day(s)?,\s)?(T)?(?P<hours>\d{1,2})?(:(?P<minutes>\d{1,2}))?(:(?P<seconds>\d{1,2}))?")
 
 
 def to_datetime(dtstr):