Port Hive, Yarn, Flink, Kylin, Camel, Nifi to framework

core/Dockerfile

@@ -7,7 +7,7 @@ RUN \
   apt-get install -y git && \
 # Install python
   apt-get update && \
-  apt-get install -y python python-dev python-pip python-virtualenv && \
+  apt-get install -y python python-dev python-pip python-virtualenv python3-pip python3-virtualenv && \
   rm -rf /var/lib/apt/lists/* && \
 # Install misc
   apt-get update && \

core/README.md

@@ -59,7 +59,7 @@ To generate ctests or run ctest, you need to first clone the target project.
 1. In `openctest/core`, run `./add_project.sh <main project>` to clone the project, switch to and build the branch `ctest-injection`. This branch will be later used by `generate_ctest` and `run_ctest`.
 2. In `openctest/core/identify_param`, run `./add_project.sh <main project>` to clone the project, switch to and build the branch `ctest-logging`. This branch will be later used by `identify_param`.
 
-`<main project>` can be `hadoop`, `hbase`, `zookeeper` or `alluxio`.
+`<main project>` can be `hadoop`, `hbase`, `zookeeper`, `hive`, `alluxio`, `yarn`, `flink`, `kylin`, `camel` or `nifi`.
 
 ## Usage
 

core/add_project.sh

@@ -11,6 +11,8 @@ function setup_hadoop() {
     mvn clean install -DskipTests
     cd $home_dir/hadoop-hdfs-project/hadoop-hdfs
     mvn package -DskipTests
+    cd $home_dir/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common
+    mvn package -DskipTests
 }
 
 function setup_hbase() {
@@ -47,6 +49,43 @@ function setup_alluxio() {
     cd core
     mvn clean install -DskipTests -Dcheckstyle.skip -Dlicense.skip -Dfindbugs.skip -Dmaven.javadoc.skip=true
 }
+function setup_nifi(){
+    [ ! -d "app/ctest-nifi" ] && git clone https://github.com/lilacyl/nifi.git app/ctest-nifi
+    cd app/ctest-nifi
+    git fetch && git checkout ctest-injection 
+    mvn clean install -DskipTest
+}
+
+function setup_hive(){
+    [ ! -d "app/ctest-hive" ] && git clone https://github.com/lilacyl/hive.git app/ctest-hive
+    cd app/ctest-hive
+    git fetch && git checkout ctest-injection
+    cd common
+    mvn clean install -DskipTests
+}
+
+function setup_flink() {
+    [ ! -d "app/ctest-flink" ] && git clone https://github.com/jessicahuang523/flink app/ctest-flink
+    cd app/ctest-flink
+    git fetch && git checkout ctest-injection
+    cd flink-core
+    mvn clean install -DskipTests
+}
+
+function setup_camel() {
+    [ ! -d "app/ctest-camel" ] && git clone https://github.com/wenhsinghuang/camel.git app/ctest-camel
+    cd app/ctest-camel
+    git fetch && git checkout ctest-logging
+    mvn clean install -DskipTests
+}
+
+
+function setup_kylin(){
+  [ ! -d "app/ctest-kylin" ] && git clone https://github.com/rtao6/kylin.git app/ctest-kylin
+  cd app/ctest-kylin
+  git fetch && git checkout ctest-injection
+  mvn clean install -DskipTests -Dcheckstyle.skip -Dlicense.skip -Dfindbugs.skip -Dmaven.javadoc.skip=true
+}
 
 function usage() {
     echo "Usage: add_project.sh <main project>"
@@ -64,7 +103,12 @@ function main() {
             hbase) setup_hbase ;;
             zookeeper) setup_zookeeper ;;
             alluxio) setup_alluxio ;;
-            *) echo "Unexpected project: $project - only support hadoop, hbase, zookeeper and alluxio." ;;
+            hive) setup_hive ;;
+            nifi) setup_nifi ;;
+            flink) setup_flink ;;
+            camel) setup_camel ;;
+            kylin) setup_kylin ;;
+            *) echo "Unexpected project: $project - only support hadoop, hbase, zookeeper, alluxio, hive, nifi, flink, kylin and camel." ;;
         esac
     fi
 }

core/ctest_const.py

@@ -12,18 +12,36 @@
 HBASE = "hbase-server"
 ZOOKEEPER = "zookeeper-server"
 ALLUXIO = "alluxio-core"
+HIVE = "hive-common"
+NIFI = "nifi-common"
+FLINK = "flink-core"
+CAMEL = "camel-core"
+HYARNCOMMON = "hadoop-yarn-common"
+KCOMMON = "kylin-common"
+
 
 CTEST_HADOOP_DIR = os.path.join(APP_DIR, "ctest-hadoop")
 CTEST_HBASE_DIR = os.path.join(APP_DIR, "ctest-hbase")
 CTEST_ZK_DIR = os.path.join(APP_DIR, "ctest-zookeeper")
 CTEST_ALLUXIO_DIR = os.path.join(APP_DIR, "ctest-alluxio")
+CTEST_HIVE_DIR = os.path.join(APP_DIR, "ctest-hive")
+CTEST_NIFI_DIR = os.path.join(APP_DIR, "ctest-nifi")
+CTEST_FLINK_DIR = os.path.join(APP_DIR, "ctest-flink")
+CTEST_CAMEL_DIR = os.path.join(APP_DIR, "ctest-camel")
+CTEST_KYLIN_DIR = os.path.join(APP_DIR, "ctest-kylin")
 
 PROJECT_DIR = {
     HCOMMON: CTEST_HADOOP_DIR,
     HDFS: CTEST_HADOOP_DIR,
     HBASE: CTEST_HBASE_DIR,
     ZOOKEEPER: CTEST_ZK_DIR,
     ALLUXIO: CTEST_ALLUXIO_DIR,
+    HIVE: CTEST_HIVE_DIR,
+    NIFI: CTEST_NIFI_DIR,
+    FLINK: CTEST_FLINK_DIR,
+    CAMEL: CTEST_CAMEL_DIR,
+    HYARNCOMMON: CTEST_HADOOP_DIR,
+    KCOMMON: CTEST_KYLIN_DIR,
 }
 
 
@@ -34,12 +52,20 @@
     HBASE: "hbase-server",
     ZOOKEEPER: "zookeeper-server",
     ALLUXIO: "core",
+    HIVE: "common",
+    NIFI: "nifi-common",
+    FLINK: "flink-core",
+    CAMEL: "core/camel-core",
+    HYARNCOMMON: "hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common",
+    KCOMMON: "core-common",
 }
 
 
 # surefire report
-SUREFIRE_SUBDIR = "target/surefire-reports/"
+SUREFIRE_SUBDIR = "../target/surefire-reports/"
 SUREFIRE_XML = "TEST-{}.xml" # slot is the classname
+HIVE_SUREFIRE_XML =  "TEST-org.apache.hadoop.hive.conf.{}.xml" # slot is the classname
+SUREFIRE_XML_NIFI = "TEST-org.apache.nifi.util.{}.xml" # slot is the classname
 SUREFIRE_TXT = "{}.txt" # testclass
 SUREFIRE_OUTTXT = "{}-output.txt" #testclass 
 
@@ -58,6 +84,12 @@
         os.path.join(CTEST_ALLUXIO_DIR, MODULE_SUBDIR[ALLUXIO], "server/worker", SUREFIRE_SUBDIR),
         os.path.join(CTEST_ALLUXIO_DIR, MODULE_SUBDIR[ALLUXIO], "server/master", SUREFIRE_SUBDIR),
     ],
+    HIVE: [os.path.join(CTEST_HIVE_DIR, MODULE_SUBDIR[HIVE], SUREFIRE_SUBDIR)],
+    NIFI: [os.path.join(CTEST_NIFI_DIR, MODULE_SUBDIR[NIFI],"nifi-properties", SUREFIRE_SUBDIR)],
+    FLINK: [os.path.join(CTEST_FLINK_DIR, MODULE_SUBDIR[FLINK], SUREFIRE_SUBDIR)],
+    CAMEL: [os.path.join(CTEST_HADOOP_DIR, MODULE_SUBDIR[CAMEL], SUREFIRE_SUBDIR)],
+    HYARNCOMMON: [os.path.join(CTEST_HADOOP_DIR, MODULE_SUBDIR[HYARNCOMMON], SUREFIRE_SUBDIR)],
+    KCOMMON: [os.path.join(CTEST_KYLIN_DIR, MODULE_SUBDIR[KCOMMON], SUREFIRE_SUBDIR)],
 }
 
 # default or deprecate conf path
@@ -74,7 +106,13 @@
     HDFS: os.path.join(DEFAULT_CONF_DIR, HDFS + "-default.tsv"),
     HBASE: os.path.join(DEFAULT_CONF_DIR, HBASE + "-default.tsv"),
     ALLUXIO: os.path.join(DEFAULT_CONF_DIR, ALLUXIO + "-default.tsv"),
-    ZOOKEEPER: os.path.join(DEFAULT_CONF_DIR, ZOOKEEPER + "-default.tsv")
+    ZOOKEEPER: os.path.join(DEFAULT_CONF_DIR, ZOOKEEPER + "-default.tsv"),
+    HIVE: os.path.join(DEFAULT_CONF_DIR, HIVE + "-default.tsv"),
+    NIFI: os.path.join(DEFAULT_CONF_DIR, NIFI + "-default.tsv"),
+    FLINK: os.path.join(DEFAULT_CONF_DIR, FLINK + "-default.tsv"),
+    CAMEL: os.path.join(DEFAULT_CONF_DIR, CAMEL + "-default.tsv"),
+    HYARNCOMMON: os.path.join(DEFAULT_CONF_DIR, HYARNCOMMON + "-default.tsv"),
+    KCOMMON: os.path.join(DEFAULT_CONF_DIR, KCOMMON + "-default.tsv"),
 }
 
 
@@ -96,7 +134,26 @@
     ],
     ALLUXIO: [
         os.path.join(CTEST_ALLUXIO_DIR, "core/alluxio-ctest.properties")
-    ]
+    ],
+    HIVE: [
+        os.path.join(CTEST_HIVE_DIR, "conf/hive-ctest.xml")
+    ],
+    NIFI: [
+        os.path.join(CTEST_NIFI_DIR, "nifi-commons/nifi-properties/src/test/resources/NiFiProperties/conf/ctest.properties")
+    ],
+    FLINK: [
+        os.path.join(CTEST_FLINK_DIR, "flink-core/core-ctest.yaml")
+    ],
+    CAMEL: [
+        os.path.join(CTEST_CAMEL_DIR, "core/camel-core/camel-ctest.properties")
+    ],
+    HYARNCOMMON: [
+        os.path.join(CTEST_HADOOP_DIR, "hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/target/classes/yarn-common-ctest.xml")
+    ],
+    KCOMMON: [
+        os.path.join(CTEST_KYLIN_DIR, "core-common/src/main/resources/ctest.properties")
+        # os.path.join(CTEST_KYLIN_DIR, "core-common/target/ctest.properties")
+    ],
 }
 
 

core/default_configs/camel-core-default.tsv

@@ -0,0 +1,60 @@
+cool.leading	" Leading space"	Default String
+cool.trailing	"Trailing space"	Default String
+cool.both	" Both leading and trailing space"	Default String
+space.leading	"   \r\n"	Default String
+space.trailing	"\t"	Default String
+space.both	"  \r   \t  \n"	Default String
+mixed.leading	"   Leading space\r\n"	Default String
+mixed.trailing 	"Trailing space\t"  	Default String
+mixed.both  	"  Both leading and trailing space\r   \t  \n"  	Default String
+empty.line  	""  	Default String
+cool.end	mock:result	Default endpoint
+cool.result	result	Default result
+cool.result.xx	result	parameterized propertySuffix
+cool.end.xx	mock:result	parameterized propertySuffix
+cool.concat	mock:{{cool.result}}	parameterized propertySuffix
+cool.concat.escaped	mock:\\{{cool.result\\}}{"query":{"match_all":{}\\}}	parameterized propertySuffix
+cool.start	direct:cool	parameterized propertySuffix
+cool.showid	true	parameterized propertySuffix
+cool.name	Camel	parameterized propertySuffix
+cool.other.name	Cheese	parameterized propertySuffix
+cool.a	{{cool.b}}	circular reference test
+cool.b	{{cool.c}}	circular reference test
+cool.c	{{cool.a}}	circular reference test
+cool.mock	mock	circular reference test
+myCoolCharset	iso-8859-1	circular reference test
+slipDelimiter	##	circular reference test
+myQueueSize	10	circular reference test
+myDelayPattern	3:10;5:30;10:50;20:100	circular reference test
+stop	true	circular reference test
+onlytwo	2	circular reference test
+integration.ftpEnabled	true	circular reference test
+cheese.end	mock:cheese	No description
+cheese.quote	Camel rocks	No description
+cheese.type	Gouda	No description
+cheese.server	http://mycoolserver	No description
+bean.foo	foo	No description
+elephant	Hello Thai Elephant จ	No description
+sslContextParameters.protocol	TLS	No description
+keyManagersParameters.algorithm	SunX509	No description
+trustManagersParameters.algorithm	PKIX	No description
+sslContextParameters.provider	SunJSSE	No description
+sslContextParameters.sessionTimeout	2	No description
+keyStoreParameters.provider	SUN	No description
+trustManagersParameters.provider	SunJSSE	No description
+keyStoreParamerers.password	changeit	No description
+keyManagersParameters.keyPassword	changeit	No description
+keyStoreParameters.resource	org/apache/camel/support/jsse/localhost.p12	No description
+cipherSuite.0	TLS_AES_256_GCM_SHA384	No description
+filterParameters.exclude	exclude	No description
+secureSocketProtocol.0	TLSv1.3	No description
+secureRandomParameters.algorithm	SHA1PRNG	No description
+sslContextServerParameters.clientAuthentication	REQUIRE	No description
+keyStoreParameters.type	pkcs12	No description
+filterParameters.include	include	No description
+keyManagersParameters.provider	SunJSSE	No description
+secureRandomParameters.provider	SUN	No description
+autoStartupProp	true	No description
+noAutoStartupProp	false	No description
+maxKeep	1	Hardcoded In OptionalPropertyPlaceholderTest.java
+queue	foo	No description

core/default_configs/flink-core-default.tsv

@@ -0,0 +1,32 @@
+fs.default-scheme	(none)	The default filesystem scheme, used for paths that do not declare a scheme explicitly. May contain an authority, e.g. host:port in case of an HDFS NameNode.
+fs.allowed-fallback-filesystems	(none)	A (semicolon-separated) list of file schemes, for which Hadoop can be used instead of an appropriate Flink plugin. (example: s3;wasb)
+pipeline.auto-type-registration	true	Controls whether Flink is automatically registering all types in the user programs with Kryo.
+pipeline.auto-generate-uids	true	When auto-generated UIDs are disabled, users are forced to manually specify UIDs on DataStream applications.It is highly recommended that users specify UIDs before deploying to production since they are used to match state in savepoints to operators in a job. Because auto-generated ID's are likely to change when modifying a job, specifying custom IDs allow an application to evolve over time without discarding state.
+pipeline.auto-watermark-interval	0 ms	The interval of the automatic watermark emission. Watermarks are used throughout the streaming system to keep track of the progress of time. They are used, for example, for time based windowing.
+pipeline.closure-cleaner-level	RECURSIVE	Configures the mode in which the closure cleaner works.Possible values:"NONE": Disables the closure cleaner completely."TOP_LEVEL": Cleans only the top-level class without recursing into fields."RECURSIVE": Cleans all fields recursively.
+pipeline.force-avro	false	Forces Flink to use the Apache Avro serializer for POJOs.Important: Make sure to include the flink-avro module.
+pipeline.generic-types	true	If the use of generic types is disabled, Flink will throw an UnsupportedOperationException whenever it encounters a data type that would go through Kryo for serialization.Disabling generic types can be helpful to eagerly find and eliminate the use of types that would go through Kryo serialization during runtime. Rather than checking types individually, using this option will throw exceptions eagerly in the places where generic types are used.We recommend to use this option only during development and pre-production phases, not during actual production use. The application program and/or the input data may be such that new, previously unseen, types occur at some point. In that case, setting this option would cause the program to fail.
+pipeline.force-kryo	false	If enabled, forces TypeExtractor to use Kryo serializer for POJOS even though we could analyze as POJO. In some cases this might be preferable. For example, when using interfaces with subclasses that cannot be analyzed as POJO.
+pipeline.global-job-parameters	(none)	Register a custom, serializable user configuration object. The configuration can be  accessed in operators
+metrics.latency.interval	0	Defines the interval at which latency tracking marks are emitted from the sources. Disables latency tracking if set to 0 or a negative value. Enabling this feature can significantly impact the performance of the cluster.
+state.backend.changelog.periodic-materialize.interval	10 min	Defines the interval in milliseconds to perform periodic materialization for state backend. The periodic materialization will be disabled when the value is negative
+state.backend.changelog.max-failures-allowed	3	Max number of consecutive materialization failures allowed.
+pipeline.max-parallelism	-1	The program-wide maximum parallelism used for operators which haven't specified a maximum parallelism. The maximum parallelism specifies the upper limit for dynamic scaling and the number of key groups used for partitioned state.
+parallelism.default	1	Default parallelism for jobs.
+pipeline.object-reuse	false	When enabled objects that Flink internally uses for deserialization and passing data to user-code functions will be reused. Keep in mind that this can lead to bugs when the user-code function of an operation is not aware of this behaviour.
+task.cancellation.interval	30000	Time interval between two successive task cancellation attempts in milliseconds.
+task.cancellation.timeout	180000	Timeout in milliseconds after which a task cancellation times out and leads to a fatal TaskManager error. A value of 0 deactivates the watch dog. Notice that a task cancellation is different from both a task failure and a clean shutdown.  Task cancellation timeout only applies to task cancellation and does not apply to task closing/clean-up caused by a task failure or a clean shutdown.
+execution.checkpointing.snapshot-compression	false	Tells if we should use compression for the state snapshot data or not
+restart-strategy	(none)	Defines the restart strategy to use in case of job failures.Accepted values are:none, off, disable: No restart strategy.fixeddelay, fixed-delay: Fixed delay restart strategy. More details can be found here.failurerate, failure-rate: Failure rate restart strategy. More details can be found here.exponentialdelay, exponential-delay: Exponential delay restart strategy. More details can be found here.If checkpointing is disabled, the default value is none. If checkpointing is enabled, the default value is fixed-delay with Integer.MAX_VALUE restart attempts and '1 s' delay.
+pipeline.default-kryo-serializers	(none)	Semicolon separated list of pairs of class names and Kryo serializers class names to be used as Kryo default serializersExample:class:org.example.ExampleClass,serializer:org.example.ExampleSerializer1; class:org.example.ExampleClass2,serializer:org.example.ExampleSerializer2
+pipeline.registered-pojo-types	(none)	Semicolon separated list of types to be registered with the serialization stack. If the type is eventually serialized as a POJO, then the type is registered with the POJO serializer. If the type ends up being serialized with Kryo, then it will be registered at Kryo to make sure that only tags are written.
+pipeline.registered-kryo-types	(none)	Semicolon separated list of types to be registered with the serialization stack. If the type is eventually serialized as a POJO, then the type is registered with the POJO serializer. If the type ends up being serialized with Kryo, then it will be registered at Kryo to make sure that only tags are written.
+jobmanager.scheduler	Default	Determines which scheduler implementation is used to schedule tasks. Accepted values are:'Default': Default scheduler'Adaptive': Adaptive scheduler. More details can be found here.'AdaptiveBatch': Adaptive batch scheduler. More details can be found here.Possible values:"Default""Adaptive""AdaptiveBatch"
+compiler.delimited-informat.max-line-samples	10	The maximum number of line samples taken by the compiler for delimited inputs. The samples are used to estimate the number of records. This value can be overridden for a specific input with the input format’s parameters.
+compiler.delimited-informat.min-line-samples	2	The minimum number of line samples taken by the compiler for delimited inputs. The samples are used to estimate the number of records. This value can be overridden for a specific input with the input format’s parameters
+compiler.delimited-informat.max-sample-len	2097152	The maximal length of a line sample that the compiler takes for delimited inputs. If the length of a single sample exceeds this value (possible because of misconfiguration of the parser), the sampling aborts. This value can be overridden for a specific input with the input format’s parameters.
+cluster.intercept-user-system-exit	DISABLED	Flag to check user code exiting system by terminating JVM (e.g., System.exit()). Note that this configuration option can interfere with cluster.processes.halt-on-fatal-error: In intercepted user-code, a call to System.exit() will not cause the JVM to halt, when THROW is configured.Possible values:"DISABLED": Flink is not monitoring or intercepting calls to System.exit()"LOG": Log exit attempt with stack trace but still allowing exit to be performed"THROW": Throw exception when exit is attempted disallowing JVM termination
+cluster.processes.halt-on-fatal-error	false	Whether processes should halt on fatal errors instead of performing a graceful shutdown. In some environments (e.g. Java 8 with the G1 garbage collector), a regular graceful shutdown can lead to a JVM deadlock. See FLINK-16510 for details.
+fs.overwrite-files	false	Specifies whether file output writers should overwrite existing files by default. Set to "true" to overwrite by default,"false" otherwise.
+fs.output.always-create-directory	false	File writers running with a parallelism larger than one create a directory for the output file path and put the different result files (one per parallel writer task) into that directory. If this option is set to "true", writers with a parallelism of 1 will also create a directory and place a single result file into it. If the option is set to "false", the writer will directly create the file directly at the output path, without creating a containing directory.
+rest.bind-address	(none)	The address that the server binds itself.