diff --git a/core/patch/hudi/interception.patch b/core/patch/hudi/interception.patch new file mode 100644 index 00000000..5dcbde71 --- /dev/null +++ b/core/patch/hudi/interception.patch @@ -0,0 +1,127 @@ +From 031e2381080ee63343abf17f0f36ac22adb2fd5f Mon Sep 17 00:00:00 2001 +From: yifanj4 +Date: Sun, 20 Nov 2022 16:57:11 -0600 +Subject: [PATCH] ctest-injection + +--- + .../config/DFSPropertiesConfiguration.java | 31 +++++++++++++++++++ + .../hudi/common/config/HoodieConfig.java | 12 +++++++ + .../src/main/resources/hudi-ctest.conf | 19 ++++++++++++ + 3 files changed, 62 insertions(+) + create mode 100644 hudi-common/src/main/resources/hudi-ctest.conf + +diff --git a/hudi-common/src/main/java/org/apache/hudi/common/config/DFSPropertiesConfiguration.java b/hudi-common/src/main/java/org/apache/hudi/common/config/DFSPropertiesConfiguration.java +index 08cbd568df..32271cc76f 100644 +--- a/hudi-common/src/main/java/org/apache/hudi/common/config/DFSPropertiesConfiguration.java ++++ b/hudi-common/src/main/java/org/apache/hudi/common/config/DFSPropertiesConfiguration.java +@@ -90,6 +90,13 @@ public class DFSPropertiesConfiguration { + this.visitedFilePaths = new HashSet<>(); + } + ++ public DFSPropertiesConfiguration(boolean isCtest) { ++ this.hadoopConfig = null; ++ this.currentFilePath = null; ++ this.hoodieConfig = new HoodieConfig(isCtest); ++ this.visitedFilePaths = new HashSet<>(); ++ } ++ + /** + * Load global props from hudi-defaults.conf which is under class loader or CONF_FILE_DIR_ENV_NAME. + * @return Typed Properties +@@ -122,6 +129,22 @@ public class DFSPropertiesConfiguration { + return conf.getProps(); + } + ++ //Ctest ++ public static TypedProperties loadCtestProps() { ++ DFSPropertiesConfiguration conf = new DFSPropertiesConfiguration(true); ++ URL configFile = Thread.currentThread().getContextClassLoader().getResource("hudi-ctest.conf"); ++ if (configFile != null) { ++ try (BufferedReader br = new BufferedReader(new InputStreamReader(configFile.openStream()))) { ++ conf.addPropsFromStream(br); ++ return conf.getProps(); ++ } catch (IOException ioe) { ++ throw new HoodieIOException( ++ String.format("Failed to read %s from class loader", "hudi-ctest.conf"), ioe); ++ } ++ } ++ return conf.getProps(); ++ } ++ + public static void refreshGlobalProps() { + GLOBAL_PROPS = loadGlobalProps(); + } +@@ -196,6 +219,14 @@ public class DFSPropertiesConfiguration { + return globalProps; + } + ++ // Ctest ++ public static TypedProperties getCtestPropsFF() { ++ final TypedProperties globalProps = new TypedProperties(); ++ GLOBAL_PROPS = loadCtestProps(); ++ globalProps.putAll(GLOBAL_PROPS); ++ return globalProps; ++ } ++ + // test only + public static TypedProperties addToGlobalProps(String key, String value) { + GLOBAL_PROPS.put(key, value); +diff --git a/hudi-common/src/main/java/org/apache/hudi/common/config/HoodieConfig.java b/hudi-common/src/main/java/org/apache/hudi/common/config/HoodieConfig.java +index 366d19fe6e..63dd4f71b0 100644 +--- a/hudi-common/src/main/java/org/apache/hudi/common/config/HoodieConfig.java ++++ b/hudi-common/src/main/java/org/apache/hudi/common/config/HoodieConfig.java +@@ -52,6 +52,11 @@ public class HoodieConfig implements Serializable { + + public HoodieConfig() { + this.props = new TypedProperties(); ++ this.props = getCtestProps(); // Ctest ++ } ++ ++ public HoodieConfig(boolean isCtest) { ++ this.props = new TypedProperties(); + } + + public HoodieConfig(Properties props) { +@@ -219,6 +224,13 @@ public class HoodieConfig implements Serializable { + } + } + ++ // Ctest ++ public TypedProperties getCtestProps() { ++ TypedProperties mergedProps = DFSPropertiesConfiguration.getCtestPropsFF(); ++ mergedProps.putAll(props); ++ return mergedProps; ++ } ++ + public void setDefaultOnCondition(boolean condition, HoodieConfig config) { + if (condition) { + setDefault(config); +diff --git a/hudi-common/src/main/resources/hudi-ctest.conf b/hudi-common/src/main/resources/hudi-ctest.conf +new file mode 100644 +index 0000000000..dd87c567d9 +--- /dev/null ++++ b/hudi-common/src/main/resources/hudi-ctest.conf +@@ -0,0 +1,19 @@ ++# ++# Licensed to the Apache Software Foundation (ASF) under one or more ++# contributor license agreements. See the NOTICE file distributed with ++# this work for additional information regarding copyright ownership. ++# The ASF licenses this file to You under the Apache License, Version 2.0 ++# (the "License"); you may not use this file except in compliance with ++# the License. You may obtain a copy of the License at ++# ++# http://www.apache.org/licenses/LICENSE-2.0 ++# ++# Unless required by applicable law or agreed to in writing, software ++# distributed under the License is distributed on an "AS IS" BASIS, ++# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++# See the License for the specific language governing permissions and ++# limitations under the License. ++# ++ ++hoodie.index.type BLOOM ++hoodie.metadata.enable true +\ No newline at end of file +-- +2.25.1 + diff --git a/core/patch/hudi/logging.patch b/core/patch/hudi/logging.patch new file mode 100644 index 00000000..f3d7b797 --- /dev/null +++ b/core/patch/hudi/logging.patch @@ -0,0 +1,119 @@ +From 582e8253e2e594b9844d12f45ef8926637a3d617 Mon Sep 17 00:00:00 2001 +From: yifanj4 +Date: Wed, 2 Nov 2022 18:56:14 -0500 +Subject: [PATCH] first attempt to ctest-set-get + +--- + hudi-common/pom.xml | 9 ++++++++ + .../hudi/common/config/HoodieConfig.java | 21 +++++++++++++++++++ + 2 files changed, 30 insertions(+) + +diff --git a/hudi-common/pom.xml b/hudi-common/pom.xml +index 9f149f464f..6d1ebe6905 100644 +--- a/hudi-common/pom.xml ++++ b/hudi-common/pom.xml +@@ -97,6 +97,15 @@ + + + ++ ++ org.apache.maven.plugins ++ maven-surefire-plugin ++ 3.0.0-M4 ++ ++ plain ++ true ++ ++ + + + +diff --git a/hudi-common/src/main/java/org/apache/hudi/common/config/HoodieConfig.java b/hudi-common/src/main/java/org/apache/hudi/common/config/HoodieConfig.java +index 366d19fe6e..9362d79a24 100644 +--- a/hudi-common/src/main/java/org/apache/hudi/common/config/HoodieConfig.java ++++ b/hudi-common/src/main/java/org/apache/hudi/common/config/HoodieConfig.java +@@ -32,6 +32,7 @@ import java.lang.reflect.Modifier; + import java.util.Arrays; + import java.util.List; + import java.util.Properties; ++import java.util.Map; + + /** + * This class deals with {@link ConfigProperty} and provides get/set functionalities. +@@ -60,14 +61,21 @@ public class HoodieConfig implements Serializable { + + public void setValue(ConfigProperty cfg, String val) { + cfg.checkValues(val); ++ LOG.warn("[CTEST][SET-PARAM] " + cfg.key() + getStackTrace()); // CTEST + props.setProperty(cfg.key(), val); + } + + public void setValue(String key, String val) { ++ LOG.warn("[CTEST][SET-PARAM] " + key + getStackTrace()); // CTEST + props.setProperty(key, val); + } + + public void setAll(Properties properties) { ++ for (Map.Entry e : properties.entrySet()) { ++ if (!props.containsKey(String.valueOf(e.getKey()))) { ++ LOG.warn("[CTEST][SET-PARAM] " + e.getKey() + getStackTrace()); // CTEST ++ } ++ } + props.putAll(properties); + } + +@@ -77,12 +85,14 @@ public class HoodieConfig implements Serializable { + if (configProperty.getInferFunc().isPresent()) { + inferValue = configProperty.getInferFunc().get().apply(this); + } ++ LOG.warn("[CTEST][SET-PARAM] " + configProperty.key() + getStackTrace()); // CTEST + props.setProperty(configProperty.key(), inferValue.isPresent() ? inferValue.get().toString() : configProperty.defaultValue().toString()); + } + } + + public void setDefaultValue(ConfigProperty configProperty, T defaultVal) { + if (!contains(configProperty)) { ++ LOG.warn("[CTEST][SET-PARAM] " + configProperty.key() + getStackTrace()); // CTEST + props.setProperty(configProperty.key(), defaultVal.toString()); + } + } +@@ -100,6 +110,7 @@ public class HoodieConfig implements Serializable { + + private Option getRawValue(ConfigProperty configProperty) { + if (props.containsKey(configProperty.key())) { ++ LOG.warn("[CTEST][GET-PARAM] " + configProperty.key()); // CTEST + return Option.ofNullable(props.get(configProperty.key())); + } + for (String alternative : configProperty.getAlternatives()) { +@@ -107,6 +118,7 @@ public class HoodieConfig implements Serializable { + LOG.warn(String.format("The configuration key '%s' has been deprecated " + + "and may be removed in the future. Please use the new key '%s' instead.", + alternative, configProperty.key())); ++ LOG.warn("[CTEST][GET-PARAM] " + configProperty.key()); // CTEST + return Option.ofNullable(props.get(alternative)); + } + } +@@ -144,6 +156,7 @@ public class HoodieConfig implements Serializable { + } + + public String getString(String key) { ++ LOG.warn("[CTEST][GET-PARAM] " + key); // CTEST + return props.getProperty(key); + } + +@@ -237,4 +250,12 @@ public class HoodieConfig implements Serializable { + throw new HoodieException(errorMessage); + } + } ++ ++ private String getStackTrace() { // ctest ++ String stacktrace = " "; ++ for (StackTraceElement element : Thread.currentThread().getStackTrace()) { ++ stacktrace = stacktrace.concat(element.getClassName() + "\t"); ++ } ++ return stacktrace; ++ } + } +-- +2.25.1 +