diff --git a/.gitignore b/.gitignore index 2792325..f38b6ab 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ .DS_Store __pycache__ /ext/* +/var/uls_install_id diff --git a/Dockerfile b/Dockerfile index 2dd6467..bd4b77f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -11,7 +11,7 @@ ARG ULS_DIR="$HOMEDIR/uls" ARG EXT_DIR="$ULS_DIR/ext" ARG ETP_CLI_VERSION="0.3.9" -ARG EAA_CLI_VERSION="0.5.0.2" +ARG EAA_CLI_VERSION="0.5.1" ARG MFA_CLI_VERSION="0.0.9" ARG GC_CLI_VERSION="dev" ARG LINODE_CLI_VERSION="dev" @@ -23,14 +23,15 @@ ENV HOMEDIR=$HOMEDIR # PREPARE ENVIRONMENT -# ENV PREP RUN apt-get update && \ apt-get --no-install-recommends -y install \ ca-certificates \ git \ curl \ telnet \ - gcc libssl-dev libffi-dev && \ + gcc \ + libssl-dev \ + libffi-dev && \ rm -rf /var/lib/apt/lists/ # USER & GROUP @@ -39,12 +40,12 @@ RUN groupadd akamai && \ USER akamai WORKDIR ${HOMEDIR} -RUN mkdir -p ${ULS_DIR} +RUN mkdir -p ${ULS_DIR} && \ + mkdir -p ${ULS_DIR}/var # Install ULS COPY bin/ ${ULS_DIR}/bin -COPY var/ ${ULS_DIR}/var WORKDIR ${ULS_DIR} RUN pip3 install -r ${ULS_DIR}/bin/requirements.txt @@ -65,7 +66,7 @@ ENV MFA-CLI_VERSION=$MFA_CLI_VERSION RUN git clone --depth 1 -b "${MFA_CLI_VERSION}" --single-branch https://github.com/akamai/cli-mfa.git ${EXT_DIR}/cli-mfa && \ pip3 install -r ${EXT_DIR}/cli-mfa/requirements.txt -## GC CLI +## GuardiCore CLI ENV GC_CLI_VERSION=$GC_CLI_VERSION RUN git clone --depth 1 -b "${GC_CLI_VERSION}" --single-branch https://github.com/MikeSchiessl/gc-logs.git ${EXT_DIR}/cli-gc && \ pip3 install -r ${EXT_DIR}/cli-gc/bin/requirements.txt @@ -78,5 +79,4 @@ RUN git clone --depth 1 -b "${LINODE_CLI_VERSION}" --single-branch h # ENTRYPOINTS / CMD VOLUME ["${ULS_DIR}/var"] ENTRYPOINT ["/usr/local/bin/python3","-u","bin/uls.py"] -#CMD ["--help"] # EOF diff --git a/bin/config/global_config.py b/bin/config/global_config.py index 9d00285..dbeaaec 100644 --- a/bin/config/global_config.py +++ b/bin/config/global_config.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # Common global variables / constants -__version__ = "1.6.1" +__version__ = "1.6.2" __tool_name_long__ = "Akamai Unified Log Streamer" __tool_name_short__ = "ULS" diff --git a/bin/modules/UlsInputCli.py b/bin/modules/UlsInputCli.py index 9d33662..ae79819 100644 --- a/bin/modules/UlsInputCli.py +++ b/bin/modules/UlsInputCli.py @@ -132,10 +132,13 @@ def _prep_start_endtime(self, cli_param, time): def _uls_useragent(self, product, feed): + install_id = UlsTools.get_install_id()['install_id'] + if install_id: + header_install_id = f"-{install_id}" if UlsTools.check_docker(): - my_useragent = f'ULS/{uls_config.__version__}_{product}-{feed}-docker' + my_useragent = f'ULS/{uls_config.__version__}_{product}-{feed}{header_install_id}-DKR' else: - my_useragent = f'ULS/{uls_config.__version__}_{product}-{feed}' + my_useragent = f'ULS/{uls_config.__version__}_{product}-{feed}{header_install_id}' return ["--user-agent-prefix", my_useragent] def proc_create(self): diff --git a/bin/modules/UlsTools.py b/bin/modules/UlsTools.py index f2c847f..c4bc5f4 100644 --- a/bin/modules/UlsTools.py +++ b/bin/modules/UlsTools.py @@ -107,6 +107,7 @@ def _get_cli_version(cli_bin, edgerc_mock_file): f"Docker Status\t\t{check_docker()}\n" f"RootPath \t\t{root_path}\n" f"TimeZone (UTC OFST) \t{check_timezone()} ({-time.timezone / 3600})\n" + f"Installation ID \t{get_install_id()['install_id']}" ) # Delete the mocked edgerc file @@ -181,7 +182,7 @@ def root_path(): def check_autoresume(input, feed, checkpoint_dir=uls_config.autoresume_checkpoint_path): # Check if we're in a supported stream / feed - if input not in uls_config.autoresume_supported_inputs or feed == "CONHEALTH": + if input not in uls_config.autoresume_supported_inputs or feed == "CONHEALTH" or feed == "DEVINV" : aka_log.log.critical(f"Input {input} or feed {feed} currently not supported by AUTORESUME - Exiting.") sys.exit(1) @@ -264,3 +265,38 @@ def write_autoresume_ckpt(input, feed, autoresume_file, logline): except Exception as write_error: aka_log.log.critical(f"AUTORESUME - Failure writing data to {autoresume_file} - Data: {autoresume_data} - error: {write_error} - Exiting") sys.exit(1) + + +def create_install_id(install_id_file=str(root_path()) + "/var/uls_install_id"): + if os.path.isfile(install_id_file): + aka_log.log.info(f"Install ID file - found") + install_id = get_install_id(install_id_file)['install_id'] + else: + aka_log.log.info(f"No install ID file found - creating ID + File") + import base64 + import random + import string + my_time = int(time.strftime("%Y%m%d")) + token = ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(6)) + raw_id = f"{token}-{my_time}-{uls_config.__version__}" + install_id = base64.b64encode(raw_id.encode()) + install_id_json = {'install_date': str(my_time), 'install_id': str(install_id.decode()), 'install_version': str(uls_config.__version__)} + try: + with open(install_id_file, "w") as inst_f: + json.dump(install_id_json, inst_f) + except Exception as error: + aka_log.log.warning(f"Not able to write install id file - not saving file !! Error: {error}") + aka_log.log.info(f"Created & saved a new installation id: {install_id}") + return install_id + +def get_install_id(install_id_file=str(root_path()) + "/var/uls_install_id"): + try: + with open(install_id_file, "r") as inst_f: + data = json.load(inst_f) + #print(data) + install_id = data['install_id'] + except Exception as error: + aka_log.log.debug(f"Not able to read install file - returning mocked data. Error: {error}") + data = {'install_id': "ERROR-GETTING-INSTALLATION-ID"} + #return install_id + return data \ No newline at end of file diff --git a/bin/uls.py b/bin/uls.py index ceaf456..63fd814 100755 --- a/bin/uls.py +++ b/bin/uls.py @@ -66,6 +66,9 @@ def main(): # Determine root directory root_path = str(UlsTools.root_path()) + # Check / Create install id + UlsTools.create_install_id() + # OUTPUT Version Information if uls_args.version: UlsTools.uls_version(root_path=root_path) diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index cf4d636..2af5e1b 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -1,10 +1,26 @@ # Version History +## v1.6.2 + +||| +|---|---| +|Date|2022-10 +|Kind| Minor release +|Author|mschiess@akamai.com + +- **Minor improvements** + - Bumped EAA CLI to version 0.5.1 (additional SIEM fields - EAA release 2022.02) + - Amended FAQ to [handle self-signed certificates alongside Guardicore](./FAQ.md#uls-throws-tls-an-error-when-connecting-towards-guardicore-api---input-gc) + - Added installation ID ("random string" + "current date YMD" + "first installed version") to support debugging process + - fixed a bug in the Dockerfile that left uls/var unusable +- **Housekeeping** + - fixed some bugs in testing (false negative) & speeded up testing process + ## v1.6.1 ||| |---|---| -|Date|2022-09 +|Date|2022-10 |Kind| BUGFIX release |Author|mschiess@akamai.com diff --git a/docs/FAQ.md b/docs/FAQ.md index 882a2f0..de31a67 100644 --- a/docs/FAQ.md +++ b/docs/FAQ.md @@ -10,7 +10,7 @@ - [Logs are not showing up in my SIEM](#logs-are-not-showing-up-in-siem) - [ULS on Windows error: "[WinError 2] The system cannot find the file specified"](#uls-on-windows-error-winerror-2-the-system-cannot-find-the-file-specified) - [ULS does not start due to missing field in config](#uls-does-not-start-due-to-missing-field-in-config) - +- [ULS throws TLS an error when connecting towards Guardicore API (--input GC)](#uls-throws-tls-an-error-when-connecting-towards-guardicore-api---input-gc) ---- ## FAQ @@ -109,3 +109,28 @@ There seems to be an issue within the module that actually parses the config (co Please watch out to specify the section exactly the same way (case sensitivity) as you have specified it in your .edgerc file. We will follow up on this topic within an [GitHub issue](https://github.com/akamai/uls/issues/20) +--- +### ULS throws TLS an error when connecting towards Guardicore API (--input GC) +When using an internal Guardicore installation that has no valid TLS certificate, ULS might throw the following error: +```bash +self._sslobj.do_handshake() +[SSL: CERTIFICATE_VERIFY_FAILED] +``` + +In order to work with self-signed certificates, you have 2 options: +- Recommended: + You provide the root CA of your self-signed certifcate to the python process + ```bash + export REQUESTS_CA_BUNDLE=/path/to/your/certificate.pem + ``` + + +- Insecure (not recommended): + You skip the TLS certificate (this is very insecure) + Set the following ENV variable on your system + ```bash + export GC_SKIP_TLS_VALIDATION=True + ``` +Both options also work for docker / kubernetes installations + +--- diff --git a/docs/examples/kubernetes/helm/akamai-uls/Chart.yaml b/docs/examples/kubernetes/helm/akamai-uls/Chart.yaml index 899a938..6eb05a4 100644 --- a/docs/examples/kubernetes/helm/akamai-uls/Chart.yaml +++ b/docs/examples/kubernetes/helm/akamai-uls/Chart.yaml @@ -3,5 +3,5 @@ name: akamai-uls description: Akamai Universal Log Streamer Helm installation type: application -version: 1.6.1 -appVersion: "1.6.1" +version: 1.6.2 +appVersion: "1.6.2" diff --git a/test/positive_test.bats b/test/positive_test.bats index 90e328f..751bafc 100644 --- a/test/positive_test.bats +++ b/test/positive_test.bats @@ -9,16 +9,22 @@ uls_bin=bin/uls.py # Should we using a mocked edgerc (TRUE/FALSE) ? mocked_edgerc=FALSE - # TIMEOUT -uls_test_timeout=90 -uls_kill_timeout=120 - +# TIMEOUT + # How much time is timeout alklowed to run + uls_test_timeout=20 + # Send a kill signal after + uls_kill_timeout=30 + # Used for regular timeout + uls_timeout_signal="TERM" + uls_timeout_params=" --preserve-status --kill-after $uls_kill_timeout --signal ${uls_timeout_signal} ${uls_test_timeout} " ### Switch between mocked and real edgerc if [ "$mocked_edgerc"=="FALSE" ] ; then # REAL EDGERC FILE uls_edgerc=~/.edgerc uls_section=akamaidemo + + # Variables eaa_access_assert="username" eaa_devinv_assert="client_version" @@ -46,140 +52,174 @@ load 'bats/bats-assert/load.bash' # POSITIVE tests ## EAA +#@test "EAA - ACCESS" { +# run timeout --kill-after=$uls_kill_timeout --signal=2 --preserve-status $uls_test_timeout $uls_bin --input eaa --feed access --output raw --edgerc $uls_edgerc --section $uls_section +# assert_output --partial $eaa_access_assert +# #assert_output --partial "The specified directory tmp does not exist or privileges are missing - exiting" +# #[ "$status" -eq 124 ] #return value from timeout without --preserve status +# [ "$status" -eq 100 ] || [ "$status" -eq 130 ] || [ "$status" -eq 2 ] #return value from uls when interrupted --> with --preserve status on timeout +#} + @test "EAA - ACCESS" { - run timeout --kill-after=$uls_kill_timeout --signal=2 --preserve-status $uls_test_timeout $uls_bin --input eaa --feed access --output raw --edgerc $uls_edgerc --section $uls_section - assert_output --partial $eaa_access_assert + run timeout ${uls_timeout_params} ${uls_bin} --input eaa --feed access --output raw --edgerc $uls_edgerc --section $uls_section --loglevel info + #assert_output --partial $eaa_access_assert + assert_line --partial "UlsInputCli - started PID" + refute_line --partial "was found stale -" #assert_output --partial "The specified directory tmp does not exist or privileges are missing - exiting" #[ "$status" -eq 124 ] #return value from timeout without --preserve status - [ "$status" -eq 100 ] || [ "$status" -eq 130 ] || [ "$status" -eq 2 ] #return value from uls when interrupted --> with --preserve status on timeout + [ "$status" -eq 100 ] || [ "$status" -eq 130 ] || [ "$status" -eq 137 ] #return value from uls when interrupted --> with --preserve status on timeout } + @test "EAA - ADMIN" { - run timeout --kill-after=$uls_kill_timeout --signal=2 --preserve-status $uls_test_timeout $uls_bin --input eaa --feed admin --output raw --edgerc $uls_edgerc --section $uls_section - assert_output --partial "" + run timeout ${uls_timeout_params} ${uls_bin} --input eaa --feed admin --output raw --edgerc $uls_edgerc --section $uls_section --loglevel info + #assert_output --partial "" #assert_output --partial "The specified directory tmp does not exist or privileges are missing - exiting" + assert_line --partial "UlsInputCli - started PID" + refute_line --partial "was found stale -" #[ "$status" -eq 124 ] #return value from timeout without --preserve status - [ "$status" -eq 100 ] || [ "$status" -eq 130 ] || [ "$status" -eq 2 ] #return value from uls when interrupted --> with --preserve status on timeout + [ "$status" -eq 100 ] || [ "$status" -eq 130 ] || [ "$status" -eq 137 ] #return value from uls when interrupted --> with --preserve status on timeout } @test "EAA - CONHEALTH" { - run timeout --kill-after=$uls_kill_timeout --signal=2 --preserve-status $uls_test_timeout $uls_bin --input eaa --feed admin --output raw --edgerc $uls_edgerc --section $uls_section - assert_output "" + run timeout ${uls_timeout_params} ${uls_bin} --input eaa --feed admin --output raw --edgerc $uls_edgerc --section $uls_section --loglevel info + #assert_output "" + assert_line --partial "UlsInputCli - started PID" + refute_line --partial "was found stale -" #assert_output --partial "The specified directory tmp does not exist or privileges are missing - exiting" #[ "$status" -eq 124 ] #return value from timeout without --preserve status - [ "$status" -eq 100 ] || [ "$status" -eq 130 ] || [ "$status" -eq 2 ] #return value from uls when interrupted --> with --preserve status on timeout + [ "$status" -eq 100 ] || [ "$status" -eq 130 ] || [ "$status" -eq 137 ] #return value from uls when interrupted --> with --preserve status on timeout } @test "EAA - DEVINV" { - run timeout --kill-after=$uls_kill_timeout --signal=2 --preserve-status $uls_test_timeout $uls_bin --input eaa --feed devinv --output raw --edgerc $uls_edgerc --section $uls_section - assert_output --partial $eaa_devinv_assert + run timeout ${uls_timeout_params} ${uls_bin} --input eaa --feed devinv --output raw --edgerc $uls_edgerc --section $uls_section --loglevel info + #assert_output --partial $eaa_devinv_assert + assert_line --partial "UlsInputCli - started PID" + refute_line --partial "was found stale -" #assert_output --partial "The specified directory tmp does not exist or privileges are missing - exiting" #[ "$status" -eq 124 ] #return value from timeout without --preserve status - [ "$status" -eq 100 ] || [ "$status" -eq 130 ] || [ "$status" -eq 2 ] #return value from uls when interrupted --> with --preserve status on timeout + [ "$status" -eq 100 ] || [ "$status" -eq 130 ] || [ "$status" -eq 137 ] #return value from uls when interrupted --> with --preserve status on timeout } ## ETP @test "ETP - THREAT" { - run timeout --kill-after=$uls_kill_timeout --signal=2 --preserve-status $uls_test_timeout $uls_bin --input etp --feed threat --output raw --edgerc $uls_edgerc --section $uls_section - assert_output --partial $etp_assert + run timeout ${uls_timeout_params} ${uls_bin} --input etp --feed threat --output raw --edgerc $uls_edgerc --section $uls_section --loglevel info + #assert_output --partial $etp_assert + assert_line --partial "UlsInputCli - started PID" + refute_line --partial "was found stale -" #assert_output --partial "The specified directory tmp does not exist or privileges are missing - exiting" #[ "$status" -eq 124 ] #return value from timeout without --preserve status - [ "$status" -eq 100 ] || [ "$status" -eq 130 ] || [ "$status" -eq 2 ] #return value from uls when interrupted --> with --preserve status on timeout + [ "$status" -eq 100 ] || [ "$status" -eq 130 ] || [ "$status" -eq 137 ] #return value from uls when interrupted --> with --preserve status on timeout } @test "ETP - AUP" { - run timeout --kill-after=$uls_kill_timeout --signal=2 --preserve-status $uls_test_timeout $uls_bin --input etp --feed aup --output raw --edgerc $uls_edgerc --section $uls_section - assert_output --partial $etp_assert + run timeout ${uls_timeout_params} ${uls_bin} --input etp --feed aup --output raw --edgerc $uls_edgerc --section $uls_section --loglevel info + #assert_output --partial $etp_assert + assert_line --partial "UlsInputCli - started PID" + refute_line --partial "was found stale -" #assert_output --partial "The specified directory tmp does not exist or privileges are missing - exiting" #[ "$status" -eq 124 ] #return value from timeout without --preserve status - [ "$status" -eq 100 ] || [ "$status" -eq 130 ] || [ "$status" -eq 2 ] #return value from uls when interrupted --> with --preserve status on timeout + [ "$status" -eq 100 ] || [ "$status" -eq 130 ] || [ "$status" -eq 137 ] #return value from uls when interrupted --> with --preserve status on timeout } @test "ETP - DNS" { - run timeout --kill-after=$uls_kill_timeout --signal=2 --preserve-status $uls_test_timeout $uls_bin --input etp --feed dns --output raw --edgerc $uls_edgerc --section $uls_section - assert_output --partial $etp_assert + run timeout ${uls_timeout_params} ${uls_bin} --input etp --feed dns --output raw --edgerc $uls_edgerc --section $uls_section --loglevel info + #assert_output --partial $etp_assert + assert_line --partial "UlsInputCli - started PID" + refute_line --partial "was found stale -" #assert_output --partial "The specified directory tmp does not exist or privileges are missing - exiting" #[ "$status" -eq 124 ] #return value from timeout without --preserve status - [ "$status" -eq 100 ] || [ "$status" -eq 130 ] || [ "$status" -eq 2 ] #return value from uls when interrupted --> with --preserve status on timeout + [ "$status" -eq 100 ] || [ "$status" -eq 130 ] || [ "$status" -eq 137 ] #return value from uls when interrupted --> with --preserve status on timeout } @test "ETP - PROXY" { - run timeout --kill-after=$uls_kill_timeout --signal=2 --preserve-status $uls_test_timeout $uls_bin --input etp --feed proxy --output raw --edgerc $uls_edgerc --section $uls_section - assert_output --partial $etp_assert + run timeout ${uls_timeout_params} ${uls_bin} --input etp --feed proxy --output raw --edgerc $uls_edgerc --section $uls_section --loglevel info + #assert_output --partial $etp_assert + assert_line --partial "UlsInputCli - started PID" + refute_line --partial "was found stale -" #assert_output --partial "The specified directory tmp does not exist or privileges are missing - exiting" #[ "$status" -eq 124 ] #return value from timeout without --preserve status - [ "$status" -eq 100 ] || [ "$status" -eq 130 ] || [ "$status" -eq 2 ] #return value from uls when interrupted --> with --preserve status on timeout + [ "$status" -eq 100 ] || [ "$status" -eq 130 ] || [ "$status" -eq 137 ] #return value from uls when interrupted --> with --preserve status on timeout } ## MFA @test "MFA - EVENT" { - run timeout --kill-after=$uls_kill_timeout --signal=2 --preserve-status $uls_test_timeout $uls_bin --input mfa --feed event --output raw --edgerc $uls_edgerc --section $uls_section - assert_output "" + run timeout ${uls_timeout_params} ${uls_bin} --input mfa --feed event --output raw --edgerc $uls_edgerc --section $uls_section --loglevel info + #assert_output "" + assert_line --partial "UlsInputCli - started PID" + refute_line --partial "was found stale -" #assert_output --partial "The specified directory tmp does not exist or privileges are missing - exiting" #[ "$status" -eq 124 ] #return value from timeout without --preserve status - [ "$status" -eq 100 ] || [ "$status" -eq 130 ] || [ "$status" -eq 2 ] #return value from uls when interrupted --> with --preserve status on timeout + [ "$status" -eq 100 ] || [ "$status" -eq 130 ] || [ "$status" -eq 137 ] #return value from uls when interrupted --> with --preserve status on timeout } ## GUARDICORE @test "GC - NETLOG" { - run timeout --kill-after=$uls_kill_timeout --signal=2 --preserve-status $uls_test_timeout $uls_bin --input gc --feed netlog --output raw --edgerc $uls_edgerc --section $uls_section - assert_output --partial $gc_assert + run timeout ${uls_timeout_params} ${uls_bin} --input gc --feed netlog --output raw --edgerc $uls_edgerc --section $uls_section --loglevel info + #assert_output --partial $gc_assert + assert_line --partial "UlsInputCli - started PID" + refute_line --partial "was found stale -" #assert_output --partial "The specified directory tmp does not exist or privileges are missing - exiting" #[ "$status" -eq 124 ] #return value from timeout without --preserve status - [ "$status" -eq 100 ] || [ "$status" -eq 130 ] || [ "$status" -eq 2 ] #return value from uls when interrupted --> with --preserve status on timeout + [ "$status" -eq 100 ] || [ "$status" -eq 130 ] || [ "$status" -eq 137 ] #return value from uls when interrupted --> with --preserve status on timeout } ## LINODE @test "LINODE - AUDIT" { - run timeout --kill-after=$uls_kill_timeout --signal=2 --preserve-status $uls_test_timeout $uls_bin --input linode --feed audit --output raw --edgerc $uls_edgerc --section $uls_section - assert_output --partial "" + run timeout ${uls_timeout_params} ${uls_bin} --input linode --feed audit --output raw --edgerc $uls_edgerc --section $uls_section --loglevel info + #assert_output --partial "" + assert_line --partial "UlsInputCli - started PID" + refute_line --partial "was found stale -" #assert_output --partial "The specified directory tmp does not exist or privileges are missing - exiting" #[ "$status" -eq 124 ] #return value from timeout without --preserve status - [ "$status" -eq 100 ] || [ "$status" -eq 130 ] || [ "$status" -eq 2 ] #return value from uls when interrupted --> with --preserve status on timeout + [ "$status" -eq 100 ] || [ "$status" -eq 130 ] || [ "$status" -eq 137 ] #return value from uls when interrupted --> with --preserve status on timeout } ## FILE OUTPUT @test "FILE: ETP - THREAT" { - run timeout --kill-after=$uls_kill_timeout --signal=2 --preserve-status $uls_test_timeout $uls_bin --input etp --feed threat --output file --filename "/tmp/uls_tmplogfile1.log" --edgerc $uls_edgerc --section $uls_section - assert_output --partial "" + run timeout ${uls_timeout_params} ${uls_bin} --input etp --feed threat --output file --filename "/tmp/uls_tmplogfile1.log" --edgerc $uls_edgerc --section $uls_section --loglevel info + assert_line --partial "UlsInputCli - started PID" + refute_line --partial "was found stale -" #assert_output --partial "The specified directory tmp does not exist or privileges are missing - exiting" #[ "$status" -eq 124 ] #return value from timeout without --preserve status - [ "$status" -eq 100 ] || [ "$status" -eq 130 ] || [ "$status" -eq 2 ] #return value from uls when interrupted --> with --preserve status on timeout + [ "$status" -eq 100 ] || [ "$status" -eq 130 ] || [ "$status" -eq 137 ] #return value from uls when interrupted --> with --preserve status on timeout rm -fr /tmp/uls_tmplogfile.log } @test "FILEACTION: ETP - THREAT" { - run timeout --kill-after=$uls_kill_timeout --signal=2 --preserve-status $uls_test_timeout $uls_bin --input etp --feed threat --output file --filename "/tmp/uls_tmplogfile2.log" --filebackup 1 --fileaction "/bin/zip '%s'" --edgerc $uls_edgerc --section $uls_section - assert_output --partial "" + run timeout ${uls_timeout_params} ${uls_bin} --input etp --feed threat --output file --filename "/tmp/uls_tmplogfile2.log" --filebackup 1 --fileaction "/bin/zip '%s'" --edgerc $uls_edgerc --section $uls_section --loglevel info + #assert_output --partial "" + assert_line --partial "UlsInputCli - started PID" + refute_line --partial "was found stale -" #assert_output --partial "The specified directory tmp does not exist or privileges are missing - exiting" #[ "$status" -eq 124 ] #return value from timeout without --preserve status - [ "$status" -eq 100 ] || [ "$status" -eq 130 ] || [ "$status" -eq 2 ] #return value from uls when interrupted --> with --preserve status on timeout + [ "$status" -eq 100 ] || [ "$status" -eq 130 ] || [ "$status" -eq 137 ] #return value from uls when interrupted --> with --preserve status on timeout rm -fr /tmp/uls_tmplogfile.log } ## Transformation @test "TRANSFORM - MCAS" { - run timeout --kill-after=$uls_kill_timeout --signal=2 --preserve-status $uls_test_timeout $uls_bin --input etp --feed dns --output raw --transformation mcas --edgerc $uls_edgerc --section $uls_section + run timeout ${uls_timeout_params} ${uls_bin} --input etp --feed dns --output raw --transformation mcas --edgerc $uls_edgerc --section $uls_section assert_output --partial "detection_time" #assert_output --partial "The specified directory tmp does not exist or privileges are missing - exiting" #[ "$status" -eq 124 ] #return value from timeout without --preserve status - [ "$status" -eq 100 ] || [ "$status" -eq 130 ] || [ "$status" -eq 2 ] #return value from uls when interrupted --> with --preserve status on timeout + [ "$status" -eq 100 ] || [ "$status" -eq 130 ] || [ "$status" -eq 137 ] #return value from uls when interrupted --> with --preserve status on timeout } @test "TRANSFORM - JMESPATH" { - run timeout --kill-after=$uls_kill_timeout --signal=2 --preserve-status $uls_test_timeout $uls_bin --input eaa --feed access --output raw --transformation jmespath --transformationpattern '[geo_country, geo_state]' --edgerc $uls_edgerc --section $uls_section + run timeout ${uls_timeout_params} ${uls_bin} --input eaa --feed access --output raw --transformation jmespath --transformationpattern '[geo_country, geo_state]' --edgerc $uls_edgerc --section $uls_section assert_output --partial $jmespath_assert #assert_output --partial "The specified directory tmp does not exist or privileges are missing - exiting" #[ "$status" -eq 124 ] #return value from timeout without --preserve status - [ "$status" -eq 100 ] || [ "$status" -eq 130 ] || [ "$status" -eq 2 ] #return value from uls when interrupted --> with --preserve status on timeout + [ "$status" -eq 100 ] || [ "$status" -eq 130 ] || [ "$status" -eq 137 ] #return value from uls when interrupted --> with --preserve status on timeout } ## AUTORESUME @test "AUTORESUME - Create File" { rm -f /tmp/uls_eaa_access.ckpt - run timeout --kill-after=$uls_kill_timeout --signal=2 --preserve-status $uls_test_timeout $uls_bin --input eaa --feed access --output raw --edgerc $uls_edgerc --section $uls_section --autoresume --autoresumepath /tmp/ + run timeout ${uls_timeout_params} ${uls_bin} --input eaa --feed access --output raw --edgerc $uls_edgerc --section $uls_section --autoresume --autoresumepath /tmp/ assert_output --partial $eaa_access_assert #assert_output --partial " seems to be empty" #assert_output --partial "The specified directory tmp does not exist or privileges are missing - exiting" #[ "$status" -eq 124 ] #return value from timeout without --preserve status - [ "$status" -eq 100 ] || [ "$status" -eq 130 ] || [ "$status" -eq 2 ] #return value from uls when interrupted --> with --preserve status on timeout + [ "$status" -eq 100 ] || [ "$status" -eq 130 ] || [ "$status" -eq 137 ] #return value from uls when interrupted --> with --preserve status on timeout rm -f /tmp/uls_eaa_access.ckpt } diff --git a/test/test.sh b/test/test.sh index 05934d3..7505c35 100644 --- a/test/test.sh +++ b/test/test.sh @@ -1,7 +1,9 @@ #!/bin/bash # test.sh [file - $tests_available] [regex merge] tests_available="basic positive negative " -parallel_tests=10 +parallel_tests=20 + + function do_test() { if [ -f "test/$1_test.bats" ] ; then @@ -26,6 +28,14 @@ function do_tag() { git tag "BATS-TEST-SUCCESSFUL__$my_date" } +function pre_cleanup() { + # We saw some issues with stale (unkilled) processes + killall timeout -9 +} + + +# The code + if [ "$1" == "all" ] || [ "$1" == "" ] ; then select="all" elif [[ $tests_available =~ "$1 " ]] ; then @@ -43,7 +53,7 @@ else fi - +pre_cleanup if [ "$select" == "all" ] ; then for i in $tests_available ; do echo "> $i"