Merge branch 'develop' into feature/gefs_v13_822_add_atm_fcst

.github/workflows/pytests.yaml

@@ -15,12 +15,12 @@ jobs:
         with:
           python-version: ${{ matrix.python }}
 
-      - name: Install (upgrade) dependencies
+      - name: Install (upgrade) python dependencies
         run: |
           pip install --upgrade pip
 
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
         with:
           path: global-workflow
 

jobs/JGDAS_GLOBAL_OCEAN_ANALYSIS_POST

@@ -5,7 +5,7 @@ source "${HOMEgfs}/ush/preamble.sh"
 ##############################################
 # make temp directory
 ##############################################
-export DATA=${DATA:-${DATAROOT}/ocnanal_${CDATE}} # TODO (G): Switch to {cyc} when the downstream code is ready
+export DATA=${DATA:-${DATAROOT}/${RUN}ocnanal_${cyc}} 
 mkdir -p "${DATA}"
 cd "${DATA}" || (echo "${DATA} does not exist. ABORT!"; exit 1)
 

jobs/JGDAS_GLOBAL_OCEAN_ANALYSIS_PREP

@@ -5,7 +5,7 @@ source "${HOMEgfs}/ush/preamble.sh"
 ##############################################
 # make temp directory
 ##############################################
-export DATA=${DATA:-${DATAROOT}/ocnanal_${CDATE}} # TODO (G): Switch to {cyc} when the downstream code is ready
+export DATA=${DATA:-${DATAROOT}/${RUN}ocnanal_${cyc}} 
 rm -rf "${DATA}"  # Ensure starting with a clean DATA
 mkdir -p "${DATA}"
 cd "${DATA}" || (echo "${DATA} does not exist. ABORT!"; exit 1)
@@ -50,7 +50,6 @@ status=$?
 ##############################################
 # Set variables used in the script
 ##############################################
-export CDATE=${CDATE:-${PDY}${cyc}}
 export CDUMP=${CDUMP:-${RUN:-"gfs"}}
 export COMPONENT="ocean"
 

jobs/JGDAS_GLOBAL_OCEAN_ANALYSIS_RUN

@@ -6,7 +6,7 @@ source "${HOMEgfs}/ush/preamble.sh"
 ##############################################
 # make temp directory
 ##############################################
-export DATA=${DATA:-${DATAROOT}/ocnanal_${CDATE}} # TODO (G): Switch to {cyc} when the downstream code is ready
+export DATA=${DATA:-${DATAROOT}/${RUN}ocnanal_${cyc}}
 mkdir -p "${DATA}"
 cd "${DATA}" || (echo "${DATA} does not exist. ABORT!"; exit 1)
 

parm/config/config.base.emc.dyn

@@ -117,8 +117,11 @@ export ATARDIR="@ATARDIR@"
 
 # Commonly defined parameters in JJOBS
 export envir=${envir:-"prod"}
-export NET="gfs"
-export RUN=${RUN:-${CDUMP:-"gfs"}}
+export NET="gfs"  # NET is defined in the job-card (ecf)
+export RUN=${RUN:-${CDUMP:-"gfs"}}  # RUN is defined in the job-card (ecf); CDUMP is used at EMC as a RUN proxy
+# TODO: determine where is RUN actually used in the workflow other than here
+# TODO: is it possible to replace all instances of ${CDUMP} to ${RUN} to be
+#       consistent w/ EE2?
 export COMIN_OBS=${DMPDIR}/${CDUMP}.${PDY}/$cyc/atmos
 export COMIN_GES_OBS=${DMPDIR}/${CDUMP}.${PDY}/$cyc/atmos
 export COMINatmos=${ROTDIR}/${CDUMP}.${PDY}/${cyc}/atmos
@@ -345,7 +348,7 @@ if [ $DOHYBVAR = "YES" ]; then
     fi
 fi
 
-# if 3DVAR and IAU 
+# if 3DVAR and IAU
 if [[ ${DOHYBVAR} == "NO" && ${DOIAU} == "YES"  ]]; then
     export IAUFHRS="6"
     export IAU_FHROT="3"

sorc/build_ufs.sh

@@ -13,7 +13,7 @@ source "${cwd}/ufs_model.fd/tests/module-setup.sh"
 
 while getopts ":da:v" option; do
   case "${option}" in
-    d) BUILD_TYPE="Debug";;
+    d) BUILD_TYPE="DEBUG";;
     a) APP="${OPTARG}" ;;
     v) export BUILD_VERBOSE="YES";;
     :)

ush/forecast_det.sh

@@ -40,7 +40,7 @@ FV3_GFS_det(){
   #-------------------------------------------------------
   # determine if restart IC exists to continue from a previous forecast
   RERUN="NO"
-  [[ ${CDUMP} = "gfs" ]] && filecount=$(find "${RSTDIR_ATM}" -type f | wc -l)
+  filecount=$(find "${RSTDIR_ATM:-/dev/null}" -type f | wc -l)
   if [ ${CDUMP} = "gfs" -a ${rst_invt1} -gt 0 -a ${FHMAX} -gt ${rst_invt1} -a ${filecount} -gt 10 ]; then
     reverse=$(echo "${restart_interval[@]} " | tac -s ' ')
     for xfh in ${reverse} ; do

ush/python/pygw/.gitignore

@@ -0,0 +1,139 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+pip-wheel-metadata/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+.python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# Sphinx documentation
+docs/_build/
+
+# Editor backup files (Emacs, vim)
+*~
+*.sw[a-p]
+
+# Pycharm IDE files
+.idea/

ush/python/pygw/README.md

@@ -6,16 +6,31 @@ Python tools specifically for global applications
 Simple installation instructions
 ```sh
 $> git clone https://github.com/noaa-emc/global-workflow
-$> cd global-workflow/ush/python
+$> cd global-workflow/ush/python/pygw
 $> pip install .
 ```
 
 It is not required to install this package.  Instead, 
 ```sh
-$> cd global-workflow/ush/python
+$> cd global-workflow/ush/python/pygw
 $> export PYTHONPATH=$PWD/src/pygw
 ```
 would put this package in the `PYTHONPATH`
 
 ### Note:
 These instructions will be updated and the tools are under development.
+
+### Running python tests:
+Simple instructions to enable executing pytests manually
+```sh
+# Create a python virtual environment and step into it
+$> cd global-workflow/ush/python/pygw
+$> python3 -m venv venv
+$> source venv/bin/activate
+
+# Install pygw with the developer requirements
+(venv) $> pip install .[dev]
+
+# Run pytests
+(venv) $> pytest -v
+```

ush/python/pygw/setup.cfg

@@ -52,7 +52,7 @@ where=src
 * = *.txt, *.md
 
 [options.extras_require]
-dev = pytest-cov>=3
+dev = pytest>=7; pytest-cov>=3
 
 [green]
 file-pattern = test_*.py

ush/python/pygw/src/pygw/configuration.py

@@ -2,14 +2,14 @@
 import os
 import random
 import subprocess
-from datetime import datetime
 from pathlib import Path
 from pprint import pprint
 from typing import Union, List, Dict, Any
 
 from pygw.attrdict import AttrDict
+from pygw.timetools import to_datetime
 
-__all__ = ['Configuration']
+__all__ = ['Configuration', 'cast_as_dtype', 'cast_strdict_as_dtypedict']
 
 
 class ShellScriptException(Exception):
@@ -32,11 +32,6 @@ class Configuration:
     (or generally for sourcing a shell script into a python dictionary)
     """
 
-    DATE_ENV_VARS = ['CDATE', 'SDATE', 'EDATE']
-    TRUTHS = ['y', 'yes', 't', 'true', '.t.', '.true.']
-    BOOLS = ['n', 'no', 'f', 'false', '.f.', '.false.'] + TRUTHS
-    BOOLS = [x.upper() for x in BOOLS] + BOOLS
-
     def __init__(self, config_dir: Union[str, Path]):
         """
         Given a directory containing config files (config.XYZ),
@@ -84,18 +79,7 @@ def parse_config(self, files: Union[str, bytes, list]) -> Dict[str, Any]:
         if isinstance(files, (str, bytes)):
             files = [files]
         files = [self.find_config(file) for file in files]
-        varbles = AttrDict()
-        for key, value in self._get_script_env(files).items():
-            if key in self.DATE_ENV_VARS:  # likely a date, convert to datetime
-                varbles[key] = datetime.strptime(value, '%Y%m%d%H')
-            elif value in self.BOOLS:  # Likely a boolean, convert to True/False
-                varbles[key] = self._true_or_not(value)
-            elif '.' in value:  # Likely a number and that too a float
-                varbles[key] = self._cast_or_not(float, value)
-            else:  # Still could be a number, may be an integer
-                varbles[key] = self._cast_or_not(int, value)
-
-        return varbles
+        return cast_strdict_as_dtypedict(self._get_script_env(files))
 
     def print_config(self, files: Union[str, bytes, list]) -> None:
         """
@@ -137,16 +121,59 @@ def _get_shell_env(scripts: List) -> Dict[str, Any]:
             varbls[entry[0:iequal]] = entry[iequal + 1:]
         return varbls
 
-    @staticmethod
-    def _cast_or_not(type, value):
+
+def cast_strdict_as_dtypedict(ctx: Dict[str, str]) -> Dict[str, Any]:
+    """
+    Environment variables are typically stored as str
+    This method attempts to translate those into datatypes
+    Parameters
+    ----------
+    ctx : dict
+          dictionary with values as str
+    Returns
+    -------
+    varbles : dict
+              dictionary with values as datatypes
+    """
+    varbles = AttrDict()
+    for key, value in ctx.items():
+        varbles[key] = cast_as_dtype(value)
+    return varbles
+
+
+def cast_as_dtype(string: str) -> Union[str, int, float, bool, Any]:
+    """
+    Cast a value into known datatype
+    Parameters
+    ----------
+    string: str
+    Returns
+    -------
+    value : str or int or float or datetime
+            default: str
+    """
+    TRUTHS = ['y', 'yes', 't', 'true', '.t.', '.true.']
+    BOOLS = ['n', 'no', 'f', 'false', '.f.', '.false.'] + TRUTHS
+    BOOLS = [x.upper() for x in BOOLS] + BOOLS + ['Yes', 'No', 'True', 'False']
+
+    def _cast_or_not(type: Any, string: str):
         try:
-            return type(value)
+            return type(string)
         except ValueError:
-            return value
+            return string
 
-    @staticmethod
-    def _true_or_not(value):
+    def _true_or_not(string: str):
         try:
-            return value.lower() in Configuration.TRUTHS
+            return string.lower() in TRUTHS
         except AttributeError:
-            return value
+            return string
+
+    try:
+        return to_datetime(string)  # Try as a datetime
+    except Exception as exc:
+        if string in BOOLS:  # Likely a boolean, convert to True/False
+            return _true_or_not(string)
+        elif '.' in string:  # Likely a number and that too a float
+            return _cast_or_not(float, string)
+        else:  # Still could be a number, may be an integer
+            return _cast_or_not(int, string)