feat(regression): Allow different build options

Fixes #402.

Allow the regression test to cover different build options including
`--proxy_passthrough`, `--define mqtt=enable`, `--define db=enable`,
`--define build_type=dbg`, `--define build_type=profile`.

tests/regression/common.py

@@ -5,8 +5,8 @@
 import requests
 import statistics
 import subprocess
+import argparse
 
-DEBUG_FLAG = False
 TIMES_TOTAL = 100
 TIMEOUT = 100  # [sec]
 STATUS_CODE_500 = "500"
@@ -24,19 +24,23 @@
 
 def parse_cli_arg():
     global URL
-    if len(sys.argv) == 2:
-        raw_url = sys.argv[1]
-    elif len(sys.argv) == 4:
-        raw_url = sys.argv[1]
-        if sys.argv[2] == 'Y':
-            DEBUG_FLAG = True
-
-        # the 3rd arg is the option which determine if use the debugging mode of statistical tests
-        if sys.argv[3] == 'Y':
-            TIMES_TOTAL = 2
+    parser = argparse.ArgumentParser('Regression test runner program')
+    parser.add_argument('-u',
+                        '--url',
+                        dest='raw_url',
+                        default="localhost:8000")
+    parser.add_argument('-d', '--debug', dest="debug", action="store_true")
+    parser.add_argument('--nostat', dest="no_stat", action="store_true")
+    args = parser.parse_args()
+
+    if args.no_stat:
+        global TIMES_TOTAL
+        TIMES_TOTAL = 2
+    if args.debug:
+        logging.basicConfig(level=logging.DEBUG)
     else:
-        raw_url = "localhost:8000"
-    URL = "http://" + raw_url
+        logging.basicConfig(level=logging.INFO)
+    URL = "http://" + args.raw_url
 
 
 def eval_stat(time_cost, func_name):
@@ -61,6 +65,17 @@ def gen_rand_trytes(tryte_len):
     return trytes
 
 
+def test_logger(f):
+    logger = logging.getLogger(f.__module__)
+    name = f.__name__
+
+    def decorate(instance):
+        logger.debug(f"Testing case = {name}")
+        return instance
+
+    return decorate(f)
+
+
 def valid_trytes(trytes, trytes_len):
     if len(trytes) != trytes_len:
         return False
@@ -81,18 +96,16 @@ def map_field(key, value):
 
 def API(get_query, get_data=None, post_data=None):
     global URL
+    command = "curl {} -X POST -H 'Content-Type: application/json' -w \", %{{http_code}}\" -d '{}'"
     try:
         response = {}
         if get_data is not None:
-            r = requests.get(str(URL + get_query + get_data), timeout=TIMEOUT)
+            command = str(URL + get_query + get_data)
+            r = requests.get(command, timeout=TIMEOUT)
             response = {"content": r.text, "status_code": str(r.status_code)}
 
         elif post_data is not None:
-            command = "curl " + str(
-                URL + get_query
-            ) + " -X POST -H 'Content-Type: application/json' -w \", %{http_code}\" -d '" + str(
-                post_data) + "'"
-            logging.debug("curl command = " + command)
+            command = command.format(URL + get_query, post_data)
             p = subprocess.Popen(command,
                                  shell=True,
                                  stdout=subprocess.PIPE,
@@ -114,4 +127,6 @@ def API(get_query, get_data=None, post_data=None):
     if not response:
         response = None
 
+    logging.debug(f"Command = {command}, response = {response}")
+
     return response

tests/regression/common.sh

@@ -0,0 +1,54 @@
+# Build options
+setup_build_opts() {
+    # The options are separated by '|', the format is as follows
+    # <bazel build args> | <binary cli arguments> 
+    OPTIONS=(
+        "|"
+        "|--iri_host ${IRI_HOST}"
+        "|--iri_port ${IRI_PORT}"
+        "|--ta_host ${TA_HOST}"
+        "|--db_host ${DB_HOST}"
+        "|--verbose"
+        "|--proxy_passthrough"
+        "--define db=enable|"
+        "--define build_type=debug|"
+        "--define build_type=profile|"
+    )
+    success=()
+    fail=()
+}
+
+# Check environment variables
+check_env() {
+    ENV_NAME=(
+        "IRI_HOST" 
+        "IRI_PORT" 
+        "TA_HOST" 
+        "TA_PORT" 
+        "DB_HOST" 
+    )
+
+    echo "Checking environment variables"
+    echo "=============================="
+
+    for (( i = 0; i < ${#ENV_NAME[@]}; i++ )); do
+        name=${ENV_NAME[${i}]}
+        if [[ -z ${!name} ]]; then
+            echo "${name} not set"
+            fail=1
+        else
+            echo "${name} is set to ${!name}"
+        fi
+    done
+
+    echo "=============================="
+
+    [ -z ${fail} ] || exit 1
+}
+
+# Parse command line arguments
+get_cli_args () {
+    sleep_time=$1
+    shift
+    remaining_args=$@ # Get the remaining arguments
+}

tests/regression/requirements.txt

@@ -1 +1,5 @@
+certifi==2019.11.28
+chardet==3.0.4
+idna==2.7
 requests==2.20.0
+urllib3==1.24.3

tests/regression/run-api-with-mqtt.sh

@@ -0,0 +1,46 @@
+#!/bin/bash
+
+source tests/regression/common.sh
+
+check_env
+setup_build_opts
+
+# Get command line arguments
+# Current arguments parsed are <sleep_time> <remaining_args>
+get_cli_args $@
+
+# Install prerequisites
+make MQTT
+pip install --user -r tests/regression/requirements.txt
+redis-server &
+
+# Iterate over all available build options
+for (( i = 0; i < ${#OPTIONS[@]}; i++ )); do
+    option=${OPTIONS[${i}]}
+    cli_arg=${option} | cut -d '|' -f 1
+    build_arg=${option} | cut -d '|' -f 2
+
+    bazel run accelerator ${build_arg} -- --ta_port=${TA_PORT} ${cli_arg} &
+    TA=$!
+    sleep ${sleep_time} # TA takes time to be built
+    trap "kill -9 ${TA};" INT # Trap SIGINT from Ctrl-C to stop TA
+
+    python3 tests/regression/runner.py ${remaining_args} --url localhost:${TA_PORT}
+    rc=$?
+
+    if [ $rc -ne 0 ]
+    then
+        echo "Build option '${option}' failed"
+        fail+=("${option}")
+    else
+        success+=("${option}")
+    fi
+
+    bazel clean
+    wait $(kill -9 ${TA})
+done
+
+echo "--------- Successful build options ---------"
+for (( i = 0; i < ${#success[@]}; i++ )); do echo ${success[${i}]}; done
+echo "----------- Failed build options -----------"
+for (( i = 0; i < ${#fail[@]}; i++ )); do echo ${fail[${i}]}; done

tests/regression/run-api.sh

@@ -0,0 +1,46 @@
+#!/bin/bash
+
+source tests/regression/common.sh
+
+check_env
+setup_build_opts
+
+# Get command line arguments
+# Current arguments parsed are <sleep_time> <remaining_args>
+get_cli_args $@
+
+# Install prerequisites
+make
+pip install --user -r tests/regression/requirements.txt
+redis-server &
+
+# Iterate over all available build options
+for (( i = 0; i < ${#OPTIONS[@]}; i++ )); do
+    option=${OPTIONS[${i}]}
+    cli_arg=${option} | cut -d '|' -f 1
+    build_arg=${option} | cut -d '|' -f 2
+
+    bazel run accelerator ${build_arg} -- --ta_port=${TA_PORT} ${cli_arg} &
+    TA=$!
+    sleep ${sleep_time} # TA takes time to be built
+    trap "kill -9 ${TA};" INT # Trap SIGINT from Ctrl-C to stop TA
+
+    python3 tests/regression/runner.py ${remaining_args} --url localhost:${TA_PORT}
+    rc=$?
+
+    if [ $rc -ne 0 ]
+    then
+        echo "Build option '${option}' failed"
+        fail+=("${option}")
+    else
+        success+=("${option}")
+    fi
+
+    bazel clean
+    wait $(kill -9 ${TA})
+done
+
+echo "--------- Successful build options ---------"
+for (( i = 0; i < ${#success[@]}; i++ )); do echo ${success[${i}]}; done
+echo "----------- Failed build options -----------"
+for (( i = 0; i < ${#fail[@]}; i++ )); do echo ${fail[${i}]}; done

tests/regression/runner.py

@@ -15,10 +15,10 @@
 
 # Run all the API Test here
 if __name__ == '__main__':
-    if DEBUG_FLAG == True:
-        logging.basicConfig(level=logging.DEBUG)
-    else:
-        logging.basicConfig(level=logging.INFO)
+    ver = sys.version_info
+    if ver.major < 3 or (ver.major == 3 and ver.minor < 6):
+        raise Exception("Must be using Python 3.6 or greater")
+
     parse_cli_arg()
 
     suite_path = os.path.join(os.path.dirname(__file__), "test_suite")
@@ -27,4 +27,6 @@
         if module[-3:] == ".py":
             mod = __import__(module[:-3], locals(), globals())
             suite = unittest.TestLoader().loadTestsFromModule(mod)
-            unittest.TextTestRunner().run(suite)
+            result = unittest.TextTestRunner().run(suite)
+            if not result.wasSuccessful():
+                exit(1)

tests/regression/test_suite/generate_address.py

@@ -8,6 +8,7 @@
 class GenerateAddress(unittest.TestCase):
 
     # Without additional GET parameter (pass)
+    @test_logger
     def test_normal(self):
         res = API("/address/", get_data=self.query_string[0])
         res_json = json.loads(res["content"])
@@ -16,16 +17,19 @@ def test_normal(self):
         self.assertTrue(valid_trytes(res_json[0], LEN_ADDR))
 
     # Ascii string (fail)
+    @test_logger
     def test_ascii_string(self):
         res = API("/address/", get_data=self.query_string[1])
         self.assertEqual(STATUS_CODE_400, res["status_code"])
 
     # Unicode string (fail)
+    @test_logger
     def test_unicode_string(self):
         res = API("/address/", get_data=self.query_string[2])
         self.assertEqual(STATUS_CODE_400, res["status_code"])
 
     # Time statistics
+    @test_logger
     def test_time_statistics(self):
         time_cost = []
         for i in range(TIMES_TOTAL):

tests/regression/test_suite/get_tips.py

@@ -8,6 +8,7 @@
 class GetTips(unittest.TestCase):
 
     # Without additional GET parameter (pass)
+    @test_logger
     def test_normal(self):
         res = API("/tips/", get_data=self.query_string[0])
         self.assertEqual(STATUS_CODE_200, res["status_code"])
@@ -17,16 +18,19 @@ def test_normal(self):
             self.assertTrue(valid_trytes(tx_hash, LEN_ADDR))
 
     # Ascii string (fail)
+    @test_logger
     def test_ascii_string(self):
         res = API("/tips/", get_data=self.query_string[1])
         self.assertEqual(STATUS_CODE_400, res["status_code"])
 
     # Unicode string (fail)
+    @test_logger
     def test_unicode_string(self):
         res = API("/tips/", get_data=self.query_string[2])
         self.assertEqual(STATUS_CODE_400, res["status_code"])
 
     # Time statistics
+    @test_logger
     def test_time_statistics(self):
         time_cost = []
         for i in range(TIMES_TOTAL):

tests/regression/test_suite/get_tips_pair.py

@@ -8,6 +8,7 @@
 class GetTipsPair(unittest.TestCase):
 
     # Without additional GET parameter (pass)
+    @test_logger
     def test_normal(self):
         res = API("/tips/pair/", get_data=self.query_string[0])
         self.assertEqual(STATUS_CODE_200, res["status_code"])
@@ -17,16 +18,19 @@ def test_normal(self):
         self.assertTrue(valid_trytes(tips_hash["branchTransaction"], LEN_ADDR))
 
     # Ascii string (fail)
+    @test_logger
     def test_ascii_string(self):
         res = API("/tips/pair/", get_data=self.query_string[1])
         self.assertEqual(STATUS_CODE_400, res["status_code"])
 
     # Unicode string (fail)
+    @test_logger
     def test_unicode_string(self):
         res = API("/tips/pair/", get_data=self.query_string[2])
         self.assertEqual(STATUS_CODE_400, res["status_code"])
 
     # Time statistics
+    @test_logger
     def test_time_statistics(self):
         time_cost = []
         for i in range(TIMES_TOTAL):