Merge pull request #2823 from jedwards4b/multi_inst_check_input

fix inputdata issue for multi_instance cases

For multi instance cases append the input_data_list from each instance so that all required files will be downloaded. This is a backward compatible change but requires a change in each external component to only delete the input_data_list file prior to the first instance and append with each instance.

Test suite: scripts_regression_tests.py
Test baseline:
Test namelist changes:
Test status: bit for bit

Fixes #2822

User interface changes?:

Update gh-pages html (Y/N)?:

Code review:

scripts/Tools/archive_metadata

@@ -79,6 +79,7 @@ class PasswordPromptAction(argparse.Action):
 # -------------------------------------------------------------------------------
     """ SVN developer's password class handler
     """
+    # pylint: disable=redefined-builtin
     def __init__(self,
                  option_strings=None,
                  dest=None,
@@ -521,7 +522,7 @@ def get_run_last_date(casename, run_path):
     cpl_files = sorted(glob.glob(os.path.join(run_path, pattern)))
 
     if cpl_files:
-        head, cpl_file = os.path.split(cpl_files[-1])
+        _, cpl_file = os.path.split(cpl_files[-1])
         fparts = cpl_file.split('.')
         return fparts[-2]
 
@@ -541,7 +542,7 @@ def get_sta_last_date(sta_path):
     rest_dirs = sorted(glob.glob(os.path.join(sta_path, 'rest/*')))
 
     if rest_dirs:
-        head, rest_dir = os.path.split(rest_dirs[-1])
+        _, rest_dir = os.path.split(rest_dirs[-1])
         return rest_dir
 
     return '0000-00-00'
@@ -591,7 +592,7 @@ def get_case_status(case_dict):
                                'lnd/hist', 'logs', 'ocn/hist', 'rest', 'rof/hist',
                                'wav/hist']:
                     path = os.path.join(case_dict['sta_path'], subdir)
-                    (raw_disk, toss) = get_disk_usage(path)
+                    (raw_disk, _) = get_disk_usage(path)
                     sta_size += raw_disk
                 case_dict['sta_size'] = bytes_2_human(sta_size)
 
@@ -1454,7 +1455,7 @@ def main_func(options):
                 sys.exit(0)
             else:
                 logger.info('ERROR archive_metadata failed to find "%s" '\
-                            'in experiments database at "%s".', 
+                            'in experiments database at "%s".',
                             case_dict['q_casename'], case_dict['query_expdb_url'])
                 sys.exit(1)
 
@@ -1518,4 +1519,4 @@ if __name__ == "__main__":
         sys.exit(__status__)
     except Exception as error:
         print str(error)
-        sys.exit(1)
+        sys.exit(1)

scripts/lib/CIME/nmlgen.py

@@ -8,6 +8,7 @@
 
 import datetime
 import re
+import hashlib
 
 from CIME.XML.standard_module_setup import *
 from CIME.namelist import Namelist, parse, \
@@ -467,17 +468,25 @@ def create_stream_file_and_update_shr_strdata_nml(self, config, #pylint:disable=
 
         with open(stream_path, 'w') as stream_file:
             stream_file.write(stream_file_text)
+
+        lines_hash = self._get_input_file_hash(data_list_path)
         with open(data_list_path, 'a') as input_data_list:
             for i, filename in enumerate(domain_filenames.split("\n")):
                 if filename.strip() == '':
                     continue
                 filepath = os.path.join(domain_filepath, filename.strip())
-                input_data_list.write("domain{:d} = {}\n".format(i+1, filepath))
+                string = "domain{:d} = {}\n".format(i+1, filepath)
+                hashValue = hashlib.md5(string.rstrip().encode('utf-8')).hexdigest()
+                if hashValue not in lines_hash:
+                    input_data_list.write(string)
             for i, filename in enumerate(data_filenames.split("\n")):
                 if filename.strip() == '':
                     continue
                 filepath = os.path.join(data_filepath, filename.strip())
-                input_data_list.write("file{:d} = {}\n".format(i+1, filepath))
+                string = "file{:d} = {}\n".format(i+1, filepath)
+                hashValue = hashlib.md5(string.rstrip().encode('utf-8')).hexdigest()
+                if hashValue not in lines_hash:
+                    input_data_list.write(string)
         self.update_shr_strdata_nml(config, stream, stream_path)
 
     def update_shr_strdata_nml(self, config, stream, stream_path):
@@ -589,39 +598,57 @@ def create_shr_strdata_nml(self):
     def get_group_variables(self, group_name):
         return self._namelist.get_group_variables(group_name)
 
-
-    def _write_input_files(self, input_data_list):
+    def _get_input_file_hash(self, data_list_path):
+        lines_hash = set()
+        if os.path.isfile(data_list_path):
+            with open(data_list_path, "r") as input_data_list:
+                for line in input_data_list:
+                    hashValue = hashlib.md5(line.rstrip().encode('utf-8')).hexdigest()
+                    logger.debug( "Found line {} with hash {}".format(line,hashValue))
+                    lines_hash.add(hashValue)
+        return lines_hash
+
+    def _write_input_files(self, data_list_path):
         """Write input data files to list."""
-        for group_name in self._namelist.get_group_names():
-            for variable_name in self._namelist.get_variable_names(group_name):
-                input_pathname = self._definition.get_node_element_info(variable_name, "input_pathname")
-                if input_pathname is not None:
-                    # This is where we end up for all variables that are paths
-                    # to input data files.
-                    literals = self._namelist.get_variable_value(group_name, variable_name)
-                    for literal in literals:
-                        file_path = character_literal_to_string(literal)
-                        # NOTE - these are hard-coded here and a better way is to make these extensible
-                        if file_path == 'UNSET' or file_path == 'idmap':
-                            continue
-                        if input_pathname == 'abs':
-                            # No further mangling needed for absolute paths.
-                            # At this point, there are overwrites that should be ignored
-                            if not os.path.isabs(file_path):
+        # append to input_data_list file
+        lines_hash = self._get_input_file_hash(data_list_path)
+        with open(data_list_path, "a") as input_data_list:
+            for group_name in self._namelist.get_group_names():
+                for variable_name in self._namelist.get_variable_names(group_name):
+                    input_pathname = self._definition.get_node_element_info(variable_name, "input_pathname")
+                    if input_pathname is not None:
+                        # This is where we end up for all variables that are paths
+                        # to input data files.
+                        literals = self._namelist.get_variable_value(group_name, variable_name)
+                        for literal in literals:
+                            file_path = character_literal_to_string(literal)
+                            # NOTE - these are hard-coded here and a better way is to make these extensible
+                            if file_path == 'UNSET' or file_path == 'idmap':
                                 continue
+                            if input_pathname == 'abs':
+                                # No further mangling needed for absolute paths.
+                                # At this point, there are overwrites that should be ignored
+                                if not os.path.isabs(file_path):
+                                    continue
+                                else:
+                                    pass
+                            elif input_pathname.startswith('rel:'):
+                                # The part past "rel" is the name of a variable that
+                                # this variable specifies its path relative to.
+                                root_var = input_pathname[4:]
+                                root_dir = self.get_value(root_var)
+                                file_path = os.path.join(root_dir, file_path)
+                            else:
+                                expect(False,
+                                       "Bad input_pathname value: {}.".format(input_pathname))
+                            # Write to the input data list.
+                            string = "{} = {}".format(variable_name, file_path)
+                            hashValue = hashlib.md5(string.rstrip().encode('utf-8')).hexdigest()
+                            if hashValue not in lines_hash:
+                                logger.debug("Adding line {} with hash {}".format(string,hashValue))
+                                input_data_list.write(string+"\n")
                             else:
-                                pass
-                        elif input_pathname.startswith('rel:'):
-                            # The part past "rel" is the name of a variable that
-                            # this variable specifies its path relative to.
-                            root_var = input_pathname[4:]
-                            root_dir = self.get_value(root_var)
-                            file_path = os.path.join(root_dir, file_path)
-                        else:
-                            expect(False,
-                                   "Bad input_pathname value: {}.".format(input_pathname))
-                        # Write to the input data list.
-                        input_data_list.write("{} = {}\n".format(variable_name, file_path))
+                                logger.debug("Line already in file {}".format(string))
 
     def write_output_file(self, namelist_file, data_list_path=None, groups=None, sorted_groups=True):
         """Write out the namelists and input data files.
@@ -645,9 +672,7 @@ def write_output_file(self, namelist_file, data_list_path=None, groups=None, sor
         self._namelist.write(namelist_file, groups=groups, sorted_groups=sorted_groups)
 
         if data_list_path is not None:
-            # append to input_data_list file
-            with open(data_list_path, "a") as input_data_list:
-                self._write_input_files(input_data_list)
+            self._write_input_files(data_list_path)
 
     def add_nmlcontents(self, filename, group, append=True, format_="nmlcontents", sorted_groups=True):
         """ Write only contents of nml group """

src/components/data_comps/datm/cime_config/buildnml

@@ -25,7 +25,7 @@ logger = logging.getLogger(__name__)
 
 # pylint: disable=too-many-arguments,too-many-locals,too-many-branches,too-many-statements
 ####################################################################################
-def _create_namelists(case, confdir, inst_string, infile, nmlgen):
+def _create_namelists(case, confdir, inst_string, infile, nmlgen, data_list_path):
 ####################################################################################
     """Write out the namelist for this component.
 
@@ -65,14 +65,6 @@ def _create_namelists(case, confdir, inst_string, infile, nmlgen):
     logger.debug("DATM presaero mode is {}".format(datm_presaero))
     logger.debug("DATM topo mode is {}".format(datm_topo))
 
-    #----------------------------------------------------
-    # Clear out old data.
-    #----------------------------------------------------
-    data_list_path = os.path.join(case.get_case_root(), "Buildconf",
-                                  "datm.input_data_list")
-    if os.path.exists(data_list_path):
-        os.remove(data_list_path)
-
     #----------------------------------------------------
     # Create configuration information.
     #----------------------------------------------------
@@ -204,6 +196,14 @@ def buildnml(case, caseroot, compname):
     # Create the namelist generator object - independent of instance
     nmlgen = NamelistGenerator(case, definition_file, files=files)
 
+    #----------------------------------------------------
+    # Clear out old data.
+    #----------------------------------------------------
+    data_list_path = os.path.join(case.get_case_root(), "Buildconf",
+                                  "datm.input_data_list")
+    if os.path.exists(data_list_path):
+        os.remove(data_list_path)
+
     #----------------------------------------------------
     # Loop over instances
     #----------------------------------------------------
@@ -234,7 +234,7 @@ def buildnml(case, caseroot, compname):
         namelist_infile = [infile]
 
         # create namelist and stream file(s) data component
-        _create_namelists(case, confdir, inst_string, namelist_infile, nmlgen)
+        _create_namelists(case, confdir, inst_string, namelist_infile, nmlgen, data_list_path)
 
         # copy namelist files and stream text files, to rundir
         if os.path.isdir(rundir):

src/components/data_comps/desp/cime_config/buildnml

@@ -24,7 +24,7 @@ logger = logging.getLogger(__name__)
 
 # pylint: disable=too-many-arguments,too-many-locals,too-many-branches,too-many-statements
 ####################################################################################
-def _create_namelists(case, confdir, infile, nmlgen):
+def _create_namelists(case, confdir, infile, nmlgen, data_list_path):
 ####################################################################################
     """Write out the namelist for this component.
 
@@ -48,14 +48,6 @@ def _create_namelists(case, confdir, infile, nmlgen):
     #----------------------------------------------------
     logger.debug("DESP mode is %s", desp_mode)
 
-    #----------------------------------------------------
-    # Clear out old data.
-    #----------------------------------------------------
-    data_list_path = os.path.join(case.get_case_root(), "Buildconf",
-                                  "desp.input_data_list")
-    if os.path.exists(data_list_path):
-        os.remove(data_list_path)
-
     #----------------------------------------------------
     # Create configuration information.
     #----------------------------------------------------
@@ -116,6 +108,13 @@ def buildnml(case, caseroot, compname):
     # Create the namelist generator object - independent of instance
     nmlgen = NamelistGenerator(case, definition_file)
 
+    #----------------------------------------------------
+    # Clear out old data.
+    #----------------------------------------------------
+    data_list_path = os.path.join(case.get_case_root(), "Buildconf",
+                                  "desp.input_data_list")
+    if os.path.exists(data_list_path):
+        os.remove(data_list_path)
     #----------------------------------------------------
     # Loop over instances
     #----------------------------------------------------
@@ -146,7 +145,7 @@ def buildnml(case, caseroot, compname):
         namelist_infile = [infile]
 
         # create namelist and stream file(s) data component
-        _create_namelists(case, confdir, namelist_infile, nmlgen)
+        _create_namelists(case, confdir, namelist_infile, nmlgen, data_list_path)
 
         # copy namelist files and stream text files, to rundir
         if os.path.isdir(rundir):

src/components/data_comps/dice/cime_config/buildnml

@@ -25,7 +25,7 @@ logger = logging.getLogger(__name__)
 
 # pylint: disable=too-many-arguments,too-many-locals,too-many-branches,too-many-statements
 ####################################################################################
-def _create_namelists(case, confdir, inst_string, infile, nmlgen):
+def _create_namelists(case, confdir, inst_string, infile, nmlgen, data_list_path):
 ####################################################################################
     """Write out the namelist for this component.
 
@@ -58,14 +58,6 @@ def _create_namelists(case, confdir, inst_string, infile, nmlgen):
     logger.debug("DICE mode is {}".format(dice_mode))
     logger.debug("DICE grid is {}".format(ice_grid))
 
-    #----------------------------------------------------
-    # Clear out old data.
-    #----------------------------------------------------
-    data_list_path = os.path.join(case.get_case_root(), "Buildconf",
-                                  "dice.input_data_list")
-    if os.path.exists(data_list_path):
-        os.remove(data_list_path)
-
     #----------------------------------------------------
     # Create configuration information.
     #----------------------------------------------------
@@ -162,6 +154,14 @@ def buildnml(case, caseroot, compname):
     # Create the namelist generator object - independent of instance
     nmlgen = NamelistGenerator(case, definition_file, files=files)
 
+    #----------------------------------------------------
+    # Clear out old data.
+    #----------------------------------------------------
+    data_list_path = os.path.join(case.get_case_root(), "Buildconf",
+                                  "dice.input_data_list")
+    if os.path.exists(data_list_path):
+        os.remove(data_list_path)
+
     #----------------------------------------------------
     # Loop over instances
     #----------------------------------------------------
@@ -192,7 +192,7 @@ def buildnml(case, caseroot, compname):
         namelist_infile = [infile]
 
         # create namelist and stream file(s) data component
-        _create_namelists(case, confdir, inst_string, namelist_infile, nmlgen)
+        _create_namelists(case, confdir, inst_string, namelist_infile, nmlgen, data_list_path)
 
         # copy namelist files and stream text files, to rundir
         if os.path.isdir(rundir):

src/components/data_comps/dlnd/cime_config/buildnml

@@ -25,7 +25,7 @@ logger = logging.getLogger(__name__)
 
 # pylint: disable=too-many-arguments,too-many-locals,too-many-branches,too-many-statements
 ####################################################################################
-def _create_namelists(case, confdir, inst_string, infile, nmlgen):
+def _create_namelists(case, confdir, inst_string, infile, nmlgen, data_list_path):
 ####################################################################################
     """Write out the namelist for this component.
 
@@ -57,14 +57,6 @@ def _create_namelists(case, confdir, inst_string, infile, nmlgen):
     logger.debug("DLND grid is {}".format(lnd_grid))
     logger.debug("DLND glc_nec is {}".format(glc_nec))
 
-    #----------------------------------------------------
-    # Clear out old data.
-    #----------------------------------------------------
-    data_list_path = os.path.join(case.get_case_root(), "Buildconf",
-                                  "dlnd.input_data_list")
-    if os.path.exists(data_list_path):
-        os.remove(data_list_path)
-
     #----------------------------------------------------
     # Create configuration information.
     #----------------------------------------------------
@@ -170,6 +162,14 @@ def buildnml(case, caseroot, compname):
 
     # Create the namelist generator object - independent of instance
     nmlgen = NamelistGenerator(case, definition_file, files=files)
+    #----------------------------------------------------
+    # Clear out old data.
+    #----------------------------------------------------
+    data_list_path = os.path.join(case.get_case_root(), "Buildconf",
+                                  "dlnd.input_data_list")
+    if os.path.exists(data_list_path):
+        os.remove(data_list_path)
+
     #----------------------------------------------------
     # Loop over instances
     #----------------------------------------------------
@@ -200,7 +200,7 @@ def buildnml(case, caseroot, compname):
         namelist_infile = [infile]
 
         # create namelist and stream file(s) data component
-        _create_namelists(case, confdir, inst_string, namelist_infile, nmlgen)
+        _create_namelists(case, confdir, inst_string, namelist_infile, nmlgen, data_list_path)
 
         # copy namelist files and stream text files, to rundir
         if os.path.isdir(rundir):