Adding support for Apache Rocketmq

core/add_project.sh

@@ -48,6 +48,14 @@ function setup_alluxio() {
     mvn clean install -DskipTests -Dcheckstyle.skip -Dlicense.skip -Dfindbugs.skip -Dmaven.javadoc.skip=true
 }
 
+function setup_rocketmq() {
+    [ ! -d "app/ctest-rocketmq" ] && git clone https://github.com/Stranger3333/rocketmq.git app/ctest-rocketmq
+    cd app/ctest-rocketmq
+    git fetch && git checkout ctest-intercept
+    cd acl
+    mvn clean install -DskipTests -Dcheckstyle.skip -Dlicense.skip -Dfindbugs.skip -Dmaven.javadoc.skip=true
+}
+
 function usage() {
     echo "Usage: add_project.sh <main project>"
     exit 1
@@ -64,7 +72,8 @@ function main() {
             hbase) setup_hbase ;;
             zookeeper) setup_zookeeper ;;
             alluxio) setup_alluxio ;;
-            *) echo "Unexpected project: $project - only support hadoop, hbase, zookeeper and alluxio." ;;
+            rocketmq) setup_rocketmq ;;
+            *) echo "Unexpected project: $project - only support hadoop, hbase, zookeeper alluxio, and rocketmq." ;;
         esac
     fi
 }

core/ctest_const.py

@@ -12,18 +12,20 @@
 HBASE = "hbase-server"
 ZOOKEEPER = "zookeeper-server"
 ALLUXIO = "alluxio-core"
+ROCKETMQ = "rocketmq-acl"
 
 CTEST_HADOOP_DIR = os.path.join(APP_DIR, "ctest-hadoop")
 CTEST_HBASE_DIR = os.path.join(APP_DIR, "ctest-hbase")
 CTEST_ZK_DIR = os.path.join(APP_DIR, "ctest-zookeeper")
 CTEST_ALLUXIO_DIR = os.path.join(APP_DIR, "ctest-alluxio")
-
+CTEST_ROCKETMQ_DIR = os.path.join(APP_DIR, "ctest-rocketmq")
 PROJECT_DIR = {
     HCOMMON: CTEST_HADOOP_DIR,
     HDFS: CTEST_HADOOP_DIR,
     HBASE: CTEST_HBASE_DIR,
     ZOOKEEPER: CTEST_ZK_DIR,
     ALLUXIO: CTEST_ALLUXIO_DIR,
+    ROCKETMQ: CTEST_ROCKETMQ_DIR,
 }
 
 
@@ -34,6 +36,7 @@
     HBASE: "hbase-server",
     ZOOKEEPER: "zookeeper-server",
     ALLUXIO: "core",
+    ROCKETMQ: "acl",
 }
 
 
@@ -58,6 +61,7 @@
         os.path.join(CTEST_ALLUXIO_DIR, MODULE_SUBDIR[ALLUXIO], "server/worker", SUREFIRE_SUBDIR),
         os.path.join(CTEST_ALLUXIO_DIR, MODULE_SUBDIR[ALLUXIO], "server/master", SUREFIRE_SUBDIR),
     ],
+    ROCKETMQ: [os.path.join(CTEST_ROCKETMQ_DIR, MODULE_SUBDIR[ROCKETMQ], SUREFIRE_SUBDIR)],
 }
 
 # default or deprecate conf path
@@ -74,7 +78,8 @@
     HDFS: os.path.join(DEFAULT_CONF_DIR, HDFS + "-default.tsv"),
     HBASE: os.path.join(DEFAULT_CONF_DIR, HBASE + "-default.tsv"),
     ALLUXIO: os.path.join(DEFAULT_CONF_DIR, ALLUXIO + "-default.tsv"),
-    ZOOKEEPER: os.path.join(DEFAULT_CONF_DIR, ZOOKEEPER + "-default.tsv")
+    ZOOKEEPER: os.path.join(DEFAULT_CONF_DIR, ZOOKEEPER + "-default.tsv"),
+    ROCKETMQ: os.path.join(DEFAULT_CONF_DIR, ROCKETMQ + "-default.tsv")
 }
 
 
@@ -96,6 +101,9 @@
     ],
     ALLUXIO: [
         os.path.join(CTEST_ALLUXIO_DIR, "core/alluxio-ctest.properties")
+    ],
+    ROCKETMQ: [
+        os.path.join(CTEST_ROCKETMQ_DIR, "acl/src/test/resources/conf/ctest.yml")
     ]
 }
 

core/default_configs/rocketmq-acl-default.tsv

@@ -0,0 +1,9 @@
+admin	false	true
+accessKey	RocketMQ	RocketMQ5
+secretKey	6172839	24691356
+whiteRemoteAddress	192.168.1.*	192.168.0.*
+defaultTopicPerm	DENY	SUB
+defaultGroupPerm	DENY	SUB
+topicPerms	topicA=DENY	topicC=SUB
+groupPerms	groupA=DENY	groupB=SUB
+globalWhiteAddrs	10.10.103.*	192.168.0.*

core/generate_ctest/identify.py

@@ -13,6 +13,7 @@
 def identify_ctest(project):
     test_result_dir = os.path.join(GENCTEST_TR_DIR, project)
     ctest_file = os.path.join(CTESTS_DIR, CTESTS_FILE.format(project=project))
+    print("ctest file: " + ctest_file)
     try:
         ctests = json.load(open(ctest_file, "r"))
         ctest_file.close()

core/generate_ctest/inject.py

@@ -7,7 +7,8 @@
 from ctest_const import *
 
 from program_input import p_input
-
+# FOR ROCKETMQ
+import yaml
 project = p_input["project"]
 
 def inject_config(param_value_pairs):
@@ -35,6 +36,38 @@ def inject_config(param_value_pairs):
             file.write(str.encode("<?xml version=\"1.0\"?>\n<?xml-stylesheet type=\"text/xsl\" href=\"configuration.xsl\"?>\n"))
             file.write(ET.tostring(conf))
             file.close()
+    elif project in [ROCKETMQ]:
+        for inject_path in INJECTION_PATH[project]:
+            print(">>>>[ctest_core] injecting into file: {}".format(inject_path))
+            file = open(inject_path, "w")
+            # '10.10.104.*','192.168.0.*'
+            dict_global_addr = {'globalWhiteRemoteAddresses':[]}
+            # {"accessKey":"rocketmq2","secretKey":12345678,"whiteRemoteAddress":"192.168.1.*","admin":True}
+            dict_accounts = {'accounts':[{"accessKey":"rocketmq","secretKey":1234567,"whiteRemoteAddress":"192.168.0.*","admin":False}]}
+
+            for p, v in param_value_pairs.items():
+                if dict_accounts['accounts'][0].__contains__(p):
+                    if v == 'true':
+                        v = True
+                    elif v == 'false':
+                        v = False 
+                    dict_accounts['accounts'][0][p] = v
+                else:
+                    if p in ['defaultTopicPerm', 'defaultGroupPerm']:
+                        dict_accounts['accounts'][0][p] = v
+                    elif p in ['topicPerms', 'groupPerms']:
+                        dict_accounts['accounts'][0][p] = []
+                        dict_accounts['accounts'][0][p].append(v)
+                    elif p == 'globalWhiteAddrs':
+                        dict_global_addr['globalWhiteRemoteAddresses'].append(v)
+
+
+            # if not dict_global_addr['globalWhiteRemoteAddresses']:
+            #     dict_global_addr['globalWhiteRemoteAddresses'] = ['10.10.103.*','192.168.0.*']
+            if dict_global_addr['globalWhiteRemoteAddresses']:
+                yaml.dump(dict_global_addr, file)
+            yaml.dump(dict_accounts, file)
+            file.close()
     else:
         sys.exit(">>>>[ctest_core] value injection for {} is not supported yet".format(project))
 
@@ -53,5 +86,10 @@ def clean_conf_file(project):
             file.write(str.encode("<?xml version=\"1.0\"?>\n<?xml-stylesheet type=\"text/xsl\" href=\"configuration.xsl\"?>\n"))
             file.write(ET.tostring(conf))
             file.close()
+    elif project in [ROCKETMQ]:
+        for inject_path in INJECTION_PATH[project]:
+            file = open(inject_path, "w")
+            file.write("\n")
+            file.close()
     else:
         sys.exit(">>>>[ctest_core] value injection for {} is not supported yet".format(project))

core/generate_ctest/parse_output.py

@@ -14,15 +14,22 @@
 def parse_surefire(clsname, expected_methods):
     """method expected to show up in surefire"""
     expected_methods = set(expected_methods)
+    # print(expected_methods)
     times = {}
     errors = {}
+    # print("abfcc")
     try:
         fpath = None
+        # print("surefire dir list: ")
         for surefire_path in SUREFIRE_DIR[project]:
+
+            # print("surefire path: " + surefire_path)
             xml_path = os.path.join(surefire_path, SUREFIRE_XML.format(clsname))
+            # print(xml_path)
             if os.path.exists(xml_path):
                 print(">>>>[ctest_core] surefire report path: " + xml_path)
                 fpath = open(xml_path)
+        # print("---debug")
         tree = ET.parse(fpath)
         root = tree.getroot()
         tsinfo = root.attrib
@@ -62,6 +69,7 @@ def parse_surefire(clsname, expected_methods):
         if set(times.keys()) != expected_methods:
             print(">>>>[ctest_core] [strange] tests run not the same as expected tests")
     except Exception as e:
+        print(clsname)
         print(">>>>[ctest_core] failed to parse surefire file: {}".format(e))
 
     # pretty printing

core/generate_ctest/program_input.py

@@ -4,11 +4,11 @@
     # run mode
     "run_mode": "generate_ctest", # string
     # name of the project, i.e. hadoop-common, hadoop-hdfs, see constant.py
-    "project": "hadoop-common", # string
+    "project": "rocketmq-acl", # string
     # path to param -> tests json mapping
-    "mapping_path": "../../data/ctest_mapping/opensource-hadoop-common.json", # string
+    "mapping_path": "../../data/ctest_mapping/opensource-rocketmq-acl.json", # string
     # good values of params tests will be run against
-    "param_value_tsv": "sample-hadoop-common.tsv", # string
+    "param_value_tsv": "../default_configs/rocketmq-acl-default.tsv", # string
     # display the terminal output live, without saving any results
     "display_mode": False, # bool
     # whether to use mvn test or mvn surefire:test

core/generate_ctest/rocketmq-acl-generated-values.tsv

@@ -0,0 +1,9 @@
+admin	false	true
+accessKey	RocketMQ	RocketMQ5
+secretKey	6172839	24691356
+whiteRemoteAddress	192.168.1.*	192.168.0.*
+defaultTopicPerm	DENY	SUB
+defaultGroupPerm	DENY	SUB
+topicPerms	topicA=DENY	topicC=SUB
+groupPerms	groupA=DENY	groupB=SUB
+globalWhiteAddrs	10.10.103.*	192.168.0.*

core/generate_ctest/test_result/rocketmq-acl/missing_test_accessKey.list

@@ -0,0 +1,2 @@
+>>>>[ctest_core] missing test for rocketmq-acl	accessKey	RocketMQ
+>>>>[ctest_core] missing test for rocketmq-acl	accessKey	RocketMQ5

core/generate_ctest/test_result/rocketmq-acl/missing_test_admin.list

@@ -0,0 +1,2 @@
+>>>>[ctest_core] missing test for rocketmq-acl	admin	false
+>>>>[ctest_core] missing test for rocketmq-acl	admin	true

core/generate_ctest/test_result/rocketmq-acl/missing_test_defaultGroupPerm.list

@@ -0,0 +1,2 @@
+>>>>[ctest_core] missing test for rocketmq-acl	defaultGroupPerm	DENY
+>>>>[ctest_core] missing test for rocketmq-acl	defaultGroupPerm	SUB

core/generate_ctest/test_result/rocketmq-acl/missing_test_defaultTopicPerm.list

@@ -0,0 +1,2 @@
+>>>>[ctest_core] missing test for rocketmq-acl	defaultTopicPerm	DENY
+>>>>[ctest_core] missing test for rocketmq-acl	defaultTopicPerm	SUB

core/generate_ctest/test_result/rocketmq-acl/missing_test_globalWhiteAddrs.list

@@ -0,0 +1,2 @@
+>>>>[ctest_core] missing test for rocketmq-acl	globalWhiteAddrs	10.10.103.*
+>>>>[ctest_core] missing test for rocketmq-acl	globalWhiteAddrs	192.168.0.*

core/generate_ctest/test_result/rocketmq-acl/missing_test_groupPerms.list

@@ -0,0 +1,2 @@
+>>>>[ctest_core] missing test for rocketmq-acl	groupPerms	groupA=DENY
+>>>>[ctest_core] missing test for rocketmq-acl	groupPerms	groupB=SUB

core/generate_ctest/test_result/rocketmq-acl/missing_test_secretKey.list

@@ -0,0 +1,2 @@
+>>>>[ctest_core] missing test for rocketmq-acl	secretKey	6172839
+>>>>[ctest_core] missing test for rocketmq-acl	secretKey	24691356

core/generate_ctest/test_result/rocketmq-acl/missing_test_topicPerms.list

@@ -0,0 +1,2 @@
+>>>>[ctest_core] missing test for rocketmq-acl	topicPerms	topicA=DENY
+>>>>[ctest_core] missing test for rocketmq-acl	topicPerms	topicC=SUB

core/generate_ctest/test_result/rocketmq-acl/missing_test_whiteRemoteAddress.list

@@ -0,0 +1,2 @@
+>>>>[ctest_core] missing test for rocketmq-acl	whiteRemoteAddress	192.168.1.*
+>>>>[ctest_core] missing test for rocketmq-acl	whiteRemoteAddress	192.168.0.*