Update workflow python scripts (#760)

* update python scripts
ufs-community · May 9, 2022 · 89a0c7f · 89a0c7f
1 parent 00e2155
commit 89a0c7f
Show file tree

Hide file tree

Showing 2 changed files with 40 additions and 23 deletions.
diff --git a/ush/config_defaults.yaml b/ush/config_defaults.yaml
@@ -238,19 +238,6 @@ EXEC_SUBDIR: "bin"
 # need in order to create initial and boundary condition files for a given 
 # cycle on the native FV3-LAM grid.
 #
-# FIXLAM_NCO_BASEDIR:
-# The base directory containing pregenerated grid, orography, and surface 
-# climatology files.  For the pregenerated grid specified by PREDEF_GRID_NAME, 
-# these "fixed" files are located in:
-#
-#   ${FIXLAM_NCO_BASEDIR}/${PREDEF_GRID_NAME}
-#
-# The workflow scripts will create a symlink in the experiment directory
-# that will point to a subdirectory (having the name of the grid being
-# used) under this directory.  This variable should be set to a null 
-# string in this file, but it can be specified in the user-specified 
-# workflow configuration file (EXPT_CONFIG_FN).
-#
 # envir, NET, model_ver, RUN:
 # Standard environment variables defined in the NCEP Central Operations WCOSS
 # Implementation Standards document as follows:
@@ -292,7 +279,6 @@ EXEC_SUBDIR: "bin"
 #-----------------------------------------------------------------------
 #
 COMIN: "/path/of/directory/containing/data/files/for/IC/LBCS"
-FIXLAM_NCO_BASEDIR: ""
 envir: "para"
 NET: "rrfs"
 model_ver: "v1.0.0"
@@ -711,13 +697,21 @@ EXTRN_MDL_SYSBASEDIR_LBCS: ''
 # EXTRN_MDL_FILES_LBCS:
 # Analogous to EXTRN_MDL_FILES_ICS but for LBCs instead of ICs.
 #
+# EXTRN_MDL_DATA_STORES:
+# A list of data stores where the scripts should look for external model
+# data. The list is in priority order. If disk information is provided
+# via USE_USER_STAGED_EXTRN_FILES or a known location on the platform,
+# the disk location will be highest priority. Options are disk, hpss,
+# aws, and nomads.
+#
 #-----------------------------------------------------------------------
 #
 USE_USER_STAGED_EXTRN_FILES: "FALSE"
 EXTRN_MDL_SOURCE_BASEDIR_ICS: ""
 EXTRN_MDL_FILES_ICS: ""
 EXTRN_MDL_SOURCE_BASEDIR_LBCS: ""
 EXTRN_MDL_FILES_LBCS: ""
+EXTRN_MDL_DATA_STORES: ""
 #
 #-----------------------------------------------------------------------
 #
@@ -1305,6 +1299,22 @@ VX_ENSPOINT_PROB_TN: "run_enspointvx_prob"
 # SFC_CLIMO_DIR:
 # Same as GRID_DIR but for the MAKE_SFC_CLIMO_TN task.
 #
+# DOMAIN_PREGEN_BASEDIR:
+# The base directory containing pregenerated grid, orography, and surface 
+# climatology files. This is an alternative for setting GRID_DIR,
+# OROG_DIR, and SFC_CLIMO_DIR individually
+# 
+# For the pregenerated grid specified by PREDEF_GRID_NAME, 
+# these "fixed" files are located in:
+#
+#   ${DOMAIN_PREGEN_BASEDIR}/${PREDEF_GRID_NAME}
+#
+# The workflow scripts will create a symlink in the experiment directory
+# that will point to a subdirectory (having the name of the grid being
+# used) under this directory.  This variable should be set to a null 
+# string in this file, but it can be specified in the user-specified 
+# workflow configuration file (EXPT_CONFIG_FN).
+#
 # RUN_TASK_GET_EXTRN_ICS:
 # Flag that determines whether the GET_EXTRN_ICS_TN task is to be run.
 #
@@ -1351,6 +1361,8 @@ OROG_DIR: "/path/to/pregenerated/orog/files"
 RUN_TASK_MAKE_SFC_CLIMO: "TRUE"
 SFC_CLIMO_DIR: "/path/to/pregenerated/surface/climo/files"
 
+DOMAIN_PREGEN_BASEDIR: ""
+
 RUN_TASK_GET_EXTRN_ICS: "TRUE"
 RUN_TASK_GET_EXTRN_LBCS: "TRUE"
 RUN_TASK_MAKE_ICS: "TRUE"

diff --git a/ush/setup.py b/ush/setup.py
@@ -337,7 +337,7 @@ def setup():
     #
     global MACHINE
     global MACHINE_FILE
-    global FIXgsm, FIXaer, FIXlut, TOPO_DIR, SFC_CLIMO_INPUT_DIR, FIXLAM_NCO_BASEDIR, \
+    global FIXgsm, FIXaer, FIXlut, TOPO_DIR, SFC_CLIMO_INPUT_DIR, DOMAIN_PREGEN_BASEDIR, \
            RELATIVE_LINK_FLAG, WORKFLOW_MANAGER, NCORES_PER_NODE, SCHED, \
            QUEUE_DEFAULT, QUEUE_HPSS, QUEUE_FCST, \
            PARTITION_DEFAULT, PARTITION_HPSS, PARTITION_FCST
@@ -362,7 +362,7 @@ def setup():
           FIXlut = \"{FIXlut or ""}
           TOPO_DIR = \"{TOPO_DIR or ""}
           SFC_CLIMO_INPUT_DIR = \"{SFC_CLIMO_INPUT_DIR or ""}
-          FIXLAM_NCO_BASEDIR = \"{FIXLAM_NCO_BASEDIR or ""}
+          DOMAIN_PREGEN_BASEDIR = \"{DOMAIN_PREGEN_BASEDIR or ""}
         You can specify the missing location(s) in config.sh''')
 
     #
@@ -1042,12 +1042,17 @@ def setup():
     if USE_USER_STAGED_EXTRN_FILES:
 
       if not os.path.exists(EXTRN_MDL_SOURCE_BASEDIR_ICS):
+      # Check for the base directory up to the first templated field.
+      idx = EXTRN_MDL_SOURCE_BASEDIR_ICS.find("$")
+      if not os.path.exists(EXTRN_MDL_SOURCE_BASEDIR_ICS[:idx]):
         print_err_msg_exit(f'''
             The directory (EXTRN_MDL_SOURCE_BASEDIR_ICS) in which the user-staged 
             external model files for generating ICs should be located does not exist:
               EXTRN_MDL_SOURCE_BASEDIR_ICS = \"{EXTRN_MDL_SOURCE_BASEDIR_ICS}\"''')
 
       if not os.path.exists(EXTRN_MDL_SOURCE_BASEDIR_LBCS):
+      idx = EXTRN_MDL_SOURCE_BASEDIR_LBCS.find("$")
+      if not os.path.exists(EXTRN_MDL_SOURCE_BASEDIR_LBCS[:idx]): 
         print_err_msg_exit(f'''
             The directory (EXTRN_MDL_SOURCE_BASEDIR_LBCS) in which the user-staged 
             external model files for generating LBCs should be located does not exist:
@@ -1153,7 +1158,7 @@ def setup():
 
     if RUN_ENVIR == "nco":
 
-      nco_fix_dir = os.path.join(FIXLAM_NCO_BASEDIR, PREDEF_GRID_NAME)
+      nco_fix_dir = os.path.join(DOMAIN_PREGEN_BASEDIR, PREDEF_GRID_NAME)
       if not os.path.exists(nco_fix_dir):
         print_err_msg_exit(f'''
             The directory (nco_fix_dir) that should contain the pregenerated grid,
@@ -1168,11 +1173,11 @@ def setup():
             When RUN_ENVIR is set to \"nco\", the workflow assumes that pregenerated
             grid files already exist in the directory 
             
-              {FIXLAM_NCO_BASEDIR}/{PREDEF_GRID_NAME}
+              {DOMAIN_PREGEN_BASEDIR}/{PREDEF_GRID_NAME}
             
             where
             
-              FIXLAM_NCO_BASEDIR = \"{FIXLAM_NCO_BASEDIR}\"
+              DOMAIN_PREGEN_BASEDIR = \"{DOMAIN_PREGEN_BASEDIR}\"
               PREDEF_GRID_NAME = \"{PREDEF_GRID_NAME}\"
             
             Thus, the MAKE_GRID_TN task must not be run (i.e. RUN_TASK_MAKE_GRID must 
@@ -1205,11 +1210,11 @@ def setup():
         msg=f'''
             When RUN_ENVIR is set to \"nco\", the workflow assumes that pregenerated
             orography files already exist in the directory 
-              {FIXLAM_NCO_BASEDIR}/{PREDEF_GRID_NAME}
+              {DOMAIN_PREGEN_BASEDIR}/{PREDEF_GRID_NAME}
             
             where
             
-              FIXLAM_NCO_BASEDIR = \"{FIXLAM_NCO_BASEDIR}\"
+              DOMAIN_PREGEN_BASEDIR = \"{DOMAIN_PREGEN_BASEDIR}\"
               PREDEF_GRID_NAME = \"{PREDEF_GRID_NAME}\"
             
             Thus, the MAKE_OROG_TN task must not be run (i.e. RUN_TASK_MAKE_OROG must 
@@ -1243,11 +1248,11 @@ def setup():
             When RUN_ENVIR is set to \"nco\", the workflow assumes that pregenerated
             surface climatology files already exist in the directory 
             
-              {FIXLAM_NCO_BASEDIR}/{PREDEF_GRID_NAME}
+              {DOMAIN_PREGEN_BASEDIR}/{PREDEF_GRID_NAME}
             
             where
             
-              FIXLAM_NCO_BASEDIR = \"{FIXLAM_NCO_BASEDIR}\"
+              DOMAIN_PREGEN_BASEDIR = \"{DOMAIN_PREGEN_BASEDIR}\"
               PREDEF_GRID_NAME = \"{PREDEF_GRID_NAME}\"
             
             Thus, the MAKE_SFC_CLIMO_TN task must not be run (i.e. RUN_TASK_MAKE_SFC_CLIMO