correct generate_ctest and run_ctest

core/ctest_const.py

@@ -105,7 +105,7 @@
         os.path.join(CTEST_ALLUXIO_DIR, "core/alluxio-ctest.properties")
     ],
     HUDI: [
-        os.path.join(CTEST_HUDI_DIR, "hudi-common/core-ctest.yaml")
+        os.path.join(CTEST_HUDI_DIR, "hudi-common/src/main/resources/hudi-ctest.conf")
     ],
 }
 

core/generate_ctest/inject.py

@@ -9,6 +9,23 @@
 from program_input import p_input
 
 project = p_input["project"]
+hudi_inject_comment = """#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+"""
 
 def inject_config(param_value_pairs):
     for p, v in param_value_pairs.items():
@@ -25,8 +42,9 @@ def inject_config(param_value_pairs):
          for inject_path in INJECTION_PATH[project]:
              print(">>>>[ctest_core] injecting into file: {}".format(inject_path))
              file = open(inject_path, "w")
+             file.write(hudi_inject_comment)
              for p, v in param_value_pairs.items():
-                 file.write(p + ": " + v + "\n")
+                 file.write(p + " " + v)
              file.close()
     elif project in [HCOMMON, HDFS, HBASE]:
         conf = ET.Element("configuration")
@@ -48,11 +66,16 @@ def inject_config(param_value_pairs):
 
 def clean_conf_file(project):
     print(">>>> cleaning injected configuration from file")
-    if project in [ZOOKEEPER, ALLUXIO, HUDI]:
+    if project in [ZOOKEEPER, ALLUXIO]:
         for inject_path in INJECTION_PATH[project]:
             file = open(inject_path, "w")
             file.write("\n")
             file.close()
+    elif project in [HUDI]:
+        for inject_path in INJECTION_PATH[project]:
+            file = open(inject_path, "w")
+            file.write(hudi_inject_comment)
+            file.close()
     elif project in [HCOMMON, HDFS, HBASE]:
         conf = ET.Element("configuration")
         for inject_path in INJECTION_PATH[project]:

core/generate_ctest/program_input.py

@@ -8,7 +8,7 @@
     # path to param -> tests json mapping
     "mapping_path": "../../data/ctest_mapping/opensource-hudi-common.json", # string
     # good values of params tests will be run against
-    "param_value_tsv": "hudi-common-generated-values.tsv", # string
+    "param_value_tsv": "../generate_value/hudi-common-generated-values.tsv", # string
     # display the terminal output live, without saving any results
     "display_mode": False, # bool
     # whether to use mvn test or mvn surefire:test

core/run_ctest/inject.py

@@ -9,6 +9,23 @@
 from program_input import p_input
 
 project = p_input["project"]
+hudi_inject_comment = """#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+"""
 
 def inject_config(param_value_pairs):
     for p, v in param_value_pairs.items():
@@ -25,8 +42,9 @@ def inject_config(param_value_pairs):
          for inject_path in INJECTION_PATH[project]:
              print(">>>>[ctest_core] injecting into file: {}".format(inject_path))
              file = open(inject_path, "w")
+             file.write(hudi_inject_comment)
              for p, v in param_value_pairs.items():
-                 file.write(p + ": " + v + "\n")
+                 file.write(p + " " + v)
              file.close()
     elif project in [HCOMMON, HDFS, HBASE]:
         conf = ET.Element("configuration")
@@ -48,11 +66,16 @@ def inject_config(param_value_pairs):
 
 def clean_conf_file(project):
     print(">>>> cleaning injected configuration from file")
-    if project in [ZOOKEEPER, ALLUXIO, HUDI]:
+    if project in [ZOOKEEPER, ALLUXIO]:
         for inject_path in INJECTION_PATH[project]:
             file = open(inject_path, "w")
             file.write("\n")
             file.close()
+    elif project in [HUDI]:
+        for inject_path in INJECTION_PATH[project]:
+            file = open(inject_path, "w")
+            file.write(hudi_inject_comment)
+            file.close()
     elif project in [HCOMMON, HDFS, HBASE]:
         conf = ET.Element("configuration")
         for inject_path in INJECTION_PATH[project]:

core/run_ctest/parse_input.py

@@ -46,7 +46,7 @@ def parse_conf_file(path):
         if "no default configuration file" in path:
             return {}
         if project in [HUDI]:
-             return parse_conf_file_yaml(path)
+            return parse_conf_file_conf
         return parse_conf_file_properties(path)
 
 
@@ -91,22 +91,22 @@ def parse_conf_file_properties(path):
                 conf_map[cur_key] = cur_value
     return conf_map
 
-def parse_conf_file_yaml(path):
-     deprecate_conf = load_deprecate_config_map()
-     conf_map = {}
-     for line in open(path):
-         if line.startswith("#"):
-             continue
-         seg = line.strip("\n").split(": ")
-         if len(seg) == 2:
-             cur_key, cur_value = [x.strip() for x in seg]
-             if cur_key not in conf_map:
-                 if cur_key in deprecate_conf:
-                     print(">>>>[ctest_core] {} in your input conf file is deprecated in the project,".format(cur_key)
-                      + " replaced with {}".format(deprecate_conf[cur_key]))
-                     cur_key = deprecate_conf[cur_key]
-                 conf_map[cur_key] = cur_value
-     return conf_map
+def parse_conf_file_conf(path):
+    deprecate_conf = load_deprecate_config_map()
+    conf_map = {}
+    for line in open(path):
+        if line.startswith("#"):
+            continue
+        seg = line.strip("\n").split(" ")
+        if len(seg) == 2:
+            cur_key, cur_value = [x.strip() for x in seg]
+            if cur_key not in conf_map:
+                if cur_key in deprecate_conf:
+                    print(">>>>[ctest_core] {} in your input conf file is deprecated in the project,".format(cur_key)
+                     + " replaced with {}".format(deprecate_conf[cur_key]))
+                    cur_key = deprecate_conf[cur_key]
+                conf_map[cur_key] = cur_value
+    return conf_map
 
 def extract_conf_diff(path):
     """get the config diff"""