v.1.3.3 (#22)

* preparing v.1.2.0 * bugfix for version_display_bug * merged automated testing framework * updated docs to represenet no-default-filepath * Added configureable stderr logging, fixed a bug in stderr output handling * amended changelog * fixed pip -> pip3 in CLI docs * re-enable STDERR disabling for stability * fixed a windows specifc bug * removed .bat file windows fix * bumped etp-cli 0.3.6 * amended changelog for etp 3.6 * merged pre v1.3.0 * fixed submodules issue * v1.3.0 finalized * modified bug report template * bugfix for ETP Threat checkpoint not working * fixed addiontial features doc * amended changelog * v1.3.2a * Dockerfille fix * v1.3.2 * v1.3.3
akamai · Feb 28, 2022 · 9f26f79 · 9f26f79
1 parent f5408db
commit 9f26f79
Show file tree

Hide file tree

Showing 22 changed files with 52 additions and 46 deletions.
diff --git a/Dockerfile b/Dockerfile
@@ -12,7 +12,7 @@ ARG             EXT_DIR="$ULS_DIR/ext"
 
 ARG             ETP_CLI_VERSION="0.3.7"
 ARG             EAA_CLI_VERSION="0.4.6"
-ARG             MFA_CLI_VERSION="0.0.6"
+ARG             MFA_CLI_VERSION="0.0.7"
 
 # ENV VARS
 ENV             ULS_DIR=$ULS_DIR
@@ -64,6 +64,7 @@ RUN             git clone --depth 1 -b "${MFA_CLI_VERSION}" --single-branch http
                 pip3 install -r ${EXT_DIR}/cli-mfa/requirements.txt
 
 # ENTRYPOINTS / CMD
+VOLUME          ["${ULS_DIR}/var"]
 ENTRYPOINT      ["/usr/local/bin/python3","-u","bin/uls.py"]
 #CMD             ["--help"]
 # EOF
diff --git a/README.md b/README.md
@@ -43,8 +43,7 @@ It can be run directly as Python code, as a provided Docker container or through
       - [DNS](docs/LOG_OVERVIEW.md#dns)
       - [PROXY](docs/LOG_OVERVIEW.md#proxy)
     - [Akamai Phish-proof Multi Factor Authenticator (AKAMAI-MFA)](https://www.akamai.com/us/en/products/security/akamai-mfa.jsp)
-      - [AUTH](docs/LOG_OVERVIEW.md#authentication-logs-auth)
-      - [POLICY](docs/LOG_OVERVIEW.md#policy-logs-policy)
+      - [EVENT](docs/LOG_OVERVIEW.md#authentication-logs-auth)
 
 
 - Supported data outputs

diff --git a/bin/config/global_config.py b/bin/config/global_config.py
@@ -1,7 +1,7 @@
 #!/usr/bin/env python3
 
 # Common global variables / constants
-__version__ = "1.3.2"
+__version__ = "1.3.3"
 __tool_name_long__ = "Akamai Unified Log Streamer"
 __tool_name_short__ = "ULS"
 
@@ -29,7 +29,7 @@
 
     # MFA
 bin_mfa_cli = "ext/cli-mfa/bin/akamai-mfa"      # Path to the MFA CLI Executable
-mfa_cli_feeds = ['POLICY', 'AUTH']              # Available MFA CLI feeds
+mfa_cli_feeds = ['EVENT']              # Available MFA CLI feeds
 
     # INPUT Choices
 input_choices = ['EAA', 'ETP', 'MFA']           # Available input types

diff --git a/bin/modules/UlsArgsParser.py b/bin/modules/UlsArgsParser.py
@@ -1,4 +1,4 @@
-# Copyright 2021 Akamai Technologies, Inc. All Rights Reserved
+# Copyright 2022 Akamai Technologies, Inc. All Rights Reserved
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

diff --git a/bin/modules/UlsInputCli.py b/bin/modules/UlsInputCli.py
@@ -1,4 +1,4 @@
-# Copyright 2021 Akamai Technologies, Inc. All Rights Reserved
+# Copyright 2022 Akamai Technologies, Inc. All Rights Reserved
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -226,7 +226,7 @@ def proc_create(self):
                                               self.credentials_file_section,
                                               uls_config.edgerc_mfa)
                     my_feed = self._feed_selector(self.feed, product_feeds)
-                    cli_command = [self.bin_python, '-u', product_path, 'event', my_feed.lower(), '-f']
+                    cli_command = [self.bin_python, '-u', product_path, 'event', '-f']
                     cli_command[3:3] = self._uls_useragent(self.product, my_feed)
                     cli_command[3:3] = edgegrid_auth
                     cli_command[3:3] = self._prep_proxy(self.inproxy)

diff --git a/bin/modules/UlsMonitoring.py b/bin/modules/UlsMonitoring.py
@@ -1,4 +1,4 @@
-# Copyright 2021 Akamai Technologies, Inc. All Rights Reserved
+# Copyright 2022 Akamai Technologies, Inc. All Rights Reserved
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

diff --git a/bin/modules/UlsOutput.py b/bin/modules/UlsOutput.py
@@ -1,4 +1,4 @@
-# Copyright 2021 Akamai Technologies, Inc. All Rights Reserved
+# Copyright 2022 Akamai Technologies, Inc. All Rights Reserved
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

diff --git a/bin/modules/UlsTools.py b/bin/modules/UlsTools.py
@@ -1,4 +1,4 @@
-# Copyright 2021 Akamai Technologies, Inc. All Rights Reserved
+# Copyright 2022 Akamai Technologies, Inc. All Rights Reserved
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

diff --git a/bin/modules/UlsTransformation.py b/bin/modules/UlsTransformation.py
@@ -1,4 +1,4 @@
-# Copyright 2021 Akamai Technologies, Inc. All Rights Reserved
+# Copyright 2022 Akamai Technologies, Inc. All Rights Reserved
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

diff --git a/bin/modules/aka_log.py b/bin/modules/aka_log.py
@@ -1,4 +1,4 @@
-# Copyright 2021 Akamai Technologies, Inc. All Rights Reserved
+# Copyright 2022 Akamai Technologies, Inc. All Rights Reserved
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -27,3 +27,4 @@ def init(loglevel='WARNING', loggername=None):
     return log
 
 # EOF
+
diff --git a/docs/ADDITIONAL_FEATURES.md b/docs/ADDITIONAL_FEATURES.md
@@ -55,7 +55,7 @@ Please see the dedicated "[Transformations docs](TRANSFORMATIONS.md)" available.
 
 # AUTORESUME / RESUME
 This feature was introduced in ULS 1.3.0.  
-Different circumstances (network isse, server maintainence, ...) could lead to an interruption of the log stream or ULS itself.
+Different circumstances (network issues, server maintainence, ...) could lead to an interruption of the log stream or ULS itself.
 As this could cause a gap in the continuous log delivery, ULS now offers the option to enable automated resume upon the last recorded checkpoint.
 
 AUTORESUME will create a checkpoint every 1000 lines of log (configureable) to prevent too many FS operations.

diff --git a/docs/ARGUMENTS_ENV_VARS.md b/docs/ARGUMENTS_ENV_VARS.md
@@ -12,17 +12,17 @@ The following tables list all available command line parameters and their corres
 
 
 ## INPUT
-| Parameter                 | Env - Var       | Options                                                                                               | Default       | Description                                                                                                                                                                                 |
-|---------------------------|-----------------|-------------------------------------------------------------------------------------------------------|---------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
-| -i <br> --input           | ULS_INPUT       | 'EAA', 'ETP', 'MFA'                                                                                   | None          | Specify the desired INPUT source                                                                                                                                                            |
-| --feed                    | ULS_FEED        | EAA: 'ACCESS', 'ADMIN', 'CONHEALTH'<br> ETP: 'THREAT', 'AUP', 'DNS', 'PROXY'<br> MFA: 'AUTH','POLICY' | None          | Specify the desired INPUT feed                                                                                                                                                              |
-| --format                  | ULS_FORMAT      | 'JSON', 'TEXT'                                                                                        | JSON          | Specify the desired INPUT (=OUTPUT) format                                                                                                                                                  |
-| --inproxy<br>--inputproxy | ULS_INPUT_PROXY | HOST:PORT                                                                                             | None          | Adjust proxy usage for INPUT data collection (cli) <br>If this parameter does not work as expected, [please read more about it here](./FAQ.md#--inputproxy-proxy-does-not-work-as-expected) |
-| --rawcmd                  | ULS_RAWCMD      | \<cli command\>                                                                                       | None          | USE with caution /!\ <br> This is meant only to be used when told by AKAMAI [Click here for more information](ADDITIONAL_FEATURES.md#rawcmd---rawcmd-feature)                               |
-| --edgerc                  | ULS_EDGERC      | /path/to/your/.edgerc                                                                                 | '~/.edgerc'   | Specify the location of the .edgerc EDGE GRID AUTH file                                                                                                                                     |
-| --section                 | ULS_SECTION     | edgerc_config_section                                                                                 | 'default'     | Specify the desired section within the .edgerc file                                                                                                                                         |
-| --starttime               | ULS_STARTTIME   | EPOCH timestamp                                                                                       | `cli_default` | Specify an EPOCH timestamp from where to start the log collection.                                                                                                                          |
-| --endtime                 | ULS_ENDTIME     | EPOCH timestamp                                                                                       | None          | Specify an EPOCH timestamp up until where to fetch logs. ULS will exit after reaching this point.<br>ULS will not continue reading logs on CLI errors !!!                                   |
+| Parameter                 | Env - Var       | Options                                                                                        | Default       | Description                                                                                                                                                                                 |
+|---------------------------|-----------------|------------------------------------------------------------------------------------------------|---------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| -i <br> --input           | ULS_INPUT       | 'EAA', 'ETP', 'MFA'                                                                            | None          | Specify the desired INPUT source                                                                                                                                                            |
+| --feed                    | ULS_FEED        | EAA: 'ACCESS', 'ADMIN', 'CONHEALTH'<br> ETP: 'THREAT', 'AUP', 'DNS', 'PROXY'<br> MFA: 'EVENT'  | None          | Specify the desired INPUT feed                                                                                                                                                              |
+| --format                  | ULS_FORMAT      | 'JSON', 'TEXT'                                                                                 | JSON          | Specify the desired INPUT (=OUTPUT) format                                                                                                                                                  |
+| --inproxy<br>--inputproxy | ULS_INPUT_PROXY | HOST:PORT                                                                                      | None          | Adjust proxy usage for INPUT data collection (cli) <br>If this parameter does not work as expected, [please read more about it here](./FAQ.md#--inputproxy-proxy-does-not-work-as-expected) |
+| --rawcmd                  | ULS_RAWCMD      | \<cli command\>                                                                                | None          | USE with caution /!\ <br> This is meant only to be used when told by AKAMAI [Click here for more information](ADDITIONAL_FEATURES.md#rawcmd---rawcmd-feature)                               |
+| --edgerc                  | ULS_EDGERC      | /path/to/your/.edgerc                                                                          | '~/.edgerc'   | Specify the location of the .edgerc EDGE GRID AUTH file                                                                                                                                     |
+| --section                 | ULS_SECTION     | edgerc_config_section                                                                          | 'default'     | Specify the desired section within the .edgerc file                                                                                                                                         |
+| --starttime               | ULS_STARTTIME   | EPOCH timestamp                                                                                | `cli_default` | Specify an EPOCH timestamp from where to start the log collection.                                                                                                                          |
+| --endtime                 | ULS_ENDTIME     | EPOCH timestamp                                                                                | None          | Specify an EPOCH timestamp up until where to fetch logs. ULS will exit after reaching this point.<br>ULS will not continue reading logs on CLI errors !!!                                   |
 
 
 ## OUTPUT

diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
@@ -1,4 +1,14 @@
 # Version History
+## v1.3.3
+|||
+|---|---|
+|Date|2022-02-28
+|Kind| Bugfix release
+|Author|mschiess@akamai.com
+- **Bugfix**
+  - Adopted to new MFA CLI Version (only single feed "EVENT" available anymore)
+  - Amended new dates to the file headers
+  - Added volume to dockerfile as data storage for "autoresume"
 
 ## v1.3.2
 |||

diff --git a/docs/COMMAND_LINE_USAGE.md b/docs/COMMAND_LINE_USAGE.md
@@ -91,7 +91,7 @@ All log output will be directed to STDOUT by default.
 - MFA AUTH LOG ==> HTTP LISTENER (SPLUNK) 
   disabled TLS verification
   ```bash
-   python3 bin/uls.py --input=MFA --feed auth --output HTTP --httpformat '{"event": %s}' --httpauthheader '{"Authorization": "Splunk xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx"}' --httpurl "https://127.0.0.1:9091/services/collector/event" --httpinsecure
+   python3 bin/uls.py --input=MFA --feed event --output HTTP --httpformat '{"event": %s}' --httpauthheader '{"Authorization": "Splunk xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx"}' --httpurl "https://127.0.0.1:9091/services/collector/event" --httpinsecure
   ```
 
 - Logging to a file and sending process to the background

diff --git a/docs/DOCKER-COMPOSE_USAGE.md b/docs/DOCKER-COMPOSE_USAGE.md
@@ -67,4 +67,4 @@ docker compose up -d
     - [docker-compose.yml](examples/docker-compose/complex/docker-compose.yml)
     - [etp-threat.env](examples/docker-compose/complex/etp-threat.env)
     - [eaa-admin.env](examples/docker-compose/complex/eaa-access.env)
-    - [eaa-access.env](examples/docker-compose/complex/eaa-access.env)
+    - [mfa.env](examples/docker-compose/complex/mfa.env)
diff --git a/docs/TRANSFORMATIONS.md b/docs/TRANSFORMATIONS.md
@@ -49,4 +49,9 @@ bin/uls.py --input eaa --feed access --section akamaidemo --output raw --transfo
 
 # EAA ACCESSS logs transformed to output only country, state and city in LIST format (sent to RAW output)
 bin/uls.py --input eaa --feed access --section akamaidemo --output raw --transformation jmespath --transformationpattern '[geo_country, geo_state, geo_city]'
+
+# EAA ACCESS logs transformed to a "NON-JSON" syslog'ish format, joining a static text with dynamic json values (dropping the keys)
+pattern='join(``, [`"<14>uls-hostname {"`, query.time, query.domain, `"}"`])'
+bin/uls.py --section akamaidemo --input etp --feed dns --output raw  --transformation jmespath --transformationpattern "$pattern"
 ```
+
diff --git a/docs/examples/docker-compose/complex/README.md b/docs/examples/docker-compose/complex/README.md
@@ -17,7 +17,7 @@ All docker related controls can be found in [docker-compose.yml](docker-compose.
 ## MFA-AUTH
 |IN|OUT|FILE|
 |---|---|---|
-|MFA-AUTH|UDP|[mfa-auth.env](mfa-auth.env)|
+|MFA-AUTH|UDP|[mfa-auth.env](mfa.env)|
 
 
 

diff --git a/docs/examples/docker-compose/complex/docker-compose.yml b/docs/examples/docker-compose/complex/docker-compose.yml
@@ -18,10 +18,10 @@ services:
         source: /path/to/your/.edgerc
         target: /opt/akamai-uls/.edgerc
         read_only: true
-  eaa-admin:
+  mfa:
     image: akamai/uls:latest
     restart: always
-    env_file: mfa-auth.env
+    env_file: mfa.env
     volumes:
       - type: bind
         source: /path/to/your/.edgerc

diff --git a/...mples/docker-compose/complex/mfa-auth.env → docs/examples/docker-compose/complex/mfa.env b/...mples/docker-compose/complex/mfa-auth.env → docs/examples/docker-compose/complex/mfa.env
@@ -5,7 +5,7 @@ ULS_LOGLEVEL=INFO
 
 # INPUT CONFIGURATION
     ULS_INPUT=MFA
-    ULS_FEED=AUTH
+    ULS_FEED=EVENT
     ULS_FORMAT=JSON
     ULS_EDGERC=/opt/akamai-uls/.edgerc
 

diff --git a/docs/examples/kubernetes/helm/akamai-uls/Chart.yaml b/docs/examples/kubernetes/helm/akamai-uls/Chart.yaml
@@ -3,5 +3,5 @@ name: akamai-uls
 description: Akamai Universal Log Streamer Helm installation
 
 type: application
-version: 0.1.0
-appVersion: "1.3.2"
+version: 0.1.1
+appVersion: "1.3.3"
diff --git a/test/positive_test.bats b/test/positive_test.bats
@@ -81,18 +81,8 @@ load 'test/bats/bats-assert/load.bash'
 }
 
 ## MFA
-@test "MFA - AUTH" {
-    skip "MFA API currently broken"
-    run timeout --preserve-status $uls_test_timeout $uls_bin --input mfa --feed auth --output raw --edgerc $uls_edgerc --section $uls_section
-    assert_output ""
-    #assert_output --partial "The specified directory tmp does not exist or privileges are missing - exiting"
-    #[ "$status" -eq 124 ]      #return value from timeout without --preserve status
-    [ "$status" -eq 100 ]       #return value from uls when interrupted --> with --preserve status on timeout
-}
-
-@test "MFA - POLICY" {
-    skip "MFA API currently broken"
-    run timeout --preserve-status $uls_test_timeout $uls_bin --input mfa --feed policy --output raw --edgerc $uls_edgerc --section $uls_section
+@test "MFA - EVENT" {
+    run timeout --preserve-status $uls_test_timeout $uls_bin --input mfa --feed event --output raw --edgerc $uls_edgerc --section $uls_section
     assert_output ""
     #assert_output --partial "The specified directory tmp does not exist or privileges are missing - exiting"
     #[ "$status" -eq 124 ]      #return value from timeout without --preserve status

diff --git a/test/test.sh b/test/test.sh
@@ -17,7 +17,7 @@ function do_test() {
 }
 
 function do_tag() {
-  my_date=$(date +%Y%M%d-%H%M%S-%Z)
+  my_date=$(date +%Y%m%d-%H%M%S-%Z)
   git tag "BATS-TEST-SUCCESSFUL__$my_date"
 }