v1.6.6 (#44)

||| |---|---| |Date|2023-08-23 |Kind| Minor release |Author|mschiess@akamai.com, androcho@akamai.com - **Features** - Added '--httpliveness' to disable HTTP(S) OPTIONS request for liveness checking - Added new feed for ETP: Network traffic connections details (netcon) [Requires CLI-ETP >= 0.4.2] - **Minor improvements** - DOC Fix for manual CLI installation - [docker] bumped CLI-EAA version to "0.5.9" - [docker] bumped CLI-ETP version to "0.4.2" - **Housekeeping** - Updated the ETP Links from developer.akamai.com to techdocs.akamai.com - Added "docker file liniting" into test scripts
akamai · Aug 23, 2023 · 758786f · 758786f
1 parent c406560
commit 758786f
Show file tree

Hide file tree

Showing 13 changed files with 93 additions and 40 deletions.
diff --git a/Dockerfile b/Dockerfile
@@ -1,5 +1,5 @@
 FROM            python:3.11.4-slim-bookworm
-LABEL 	        MAINTAINER="Mike Schiessl - mike.schiessl@akamai.com"
+LABEL           MAINTAINER="Mike Schiessl - mike.schiessl@akamai.com"
 LABEL	        APP_LONG="Akamai Universal Log Streamer"
 LABEL           APP_SHORT="ULS"
 LABEL           VENDOR="Akamai Technologies"
@@ -10,8 +10,8 @@ ARG             HOMEDIR="/opt/akamai-uls"
 ARG             ULS_DIR="$HOMEDIR/uls"
 ARG             EXT_DIR="$ULS_DIR/ext"
 
-ARG             ETP_CLI_VERSION="0.4.0"
-ARG             EAA_CLI_VERSION="0.5.7"
+ARG             ETP_CLI_VERSION="0.4.2"
+ARG             EAA_CLI_VERSION="0.5.9"
 ARG             MFA_CLI_VERSION="0.1.1"
 ARG             GC_CLI_VERSION="v0.0.1(beta)"
 ARG             LINODE_CLI_VERSION="dev"
@@ -23,6 +23,7 @@ ENV             HOMEDIR=$HOMEDIR
 ENV             CONTAINERIZED=TRUE
 
 # PREPARE ENVIRONMENT
+    # hadolint ignore=DL3008
 RUN	            apt-get update && \
 	            apt-get --no-install-recommends -y install \
 		        ca-certificates \
@@ -47,34 +48,34 @@ RUN             mkdir -p ${ULS_DIR} && \
 # Install ULS
 COPY            bin/ ${ULS_DIR}/bin
 WORKDIR         ${ULS_DIR}
-RUN             pip3 install -r ${ULS_DIR}/bin/requirements.txt
+RUN             pip3 install --no-cache-dir -r ${ULS_DIR}/bin/requirements.txt
 
 # Install external CLI'S
 ## ETP CLI
 ENV             ETP_CLI_VERSION=$ETP_CLI_VERSION
 RUN             git clone --depth 1 -b "${ETP_CLI_VERSION}" --single-branch https://github.com/akamai/cli-etp.git ${EXT_DIR}/cli-etp && \
-                pip3 install -r ${EXT_DIR}/cli-etp/requirements.txt
+                pip3 install --no-cache-dir -r ${EXT_DIR}/cli-etp/requirements.txt
 
 ## EAA CLI
 ENV             EAA-CLI_VERSION=$EAA_CLI_VERSION
 RUN             git clone --depth 1 -b "${EAA_CLI_VERSION}" --single-branch https://github.com/akamai/cli-eaa.git ${EXT_DIR}/cli-eaa && \
-                pip3 install -r ${EXT_DIR}/cli-eaa/requirements.txt
+                pip3 install --no-cache-dir -r ${EXT_DIR}/cli-eaa/requirements.txt
 
 
 ## MFA CLI
 ENV             MFA-CLI_VERSION=$MFA_CLI_VERSION
 RUN             git clone --depth 1 -b "${MFA_CLI_VERSION}" --single-branch https://github.com/akamai/cli-mfa.git ${EXT_DIR}/cli-mfa && \
-                pip3 install -r ${EXT_DIR}/cli-mfa/requirements.txt
+                pip3 install --no-cache-dir -r ${EXT_DIR}/cli-mfa/requirements.txt
 
 ## GuardiCore CLI
 ENV             GC_CLI_VERSION=$GC_CLI_VERSION
 RUN             git clone --depth 1 -b "${GC_CLI_VERSION}" --single-branch https://github.com/MikeSchiessl/gc-logs.git ${EXT_DIR}/cli-gc && \
-                pip3 install -r ${EXT_DIR}/cli-gc/bin/requirements.txt
+                pip3 install --no-cache-dir -r ${EXT_DIR}/cli-gc/bin/requirements.txt
 
 ## LINODE CLI
 ENV             LINODE_CLI_VERSION=$LINODE_CLI_VERSION
 RUN             git clone --depth 1 -b "${LINODE_CLI_VERSION}" --single-branch https://github.com/MikeSchiessl/ln-logs.git ${EXT_DIR}/cli-linode && \
-                pip3 install -r ${EXT_DIR}/cli-linode/bin/requirements.txt
+                pip3 install --no-cache-dir -r ${EXT_DIR}/cli-linode/bin/requirements.txt
 
 # ENTRYPOINTS / CMD
 VOLUME          ["${ULS_DIR}/var"]

diff --git a/README.md b/README.md
@@ -43,6 +43,7 @@ It can be run directly as Python code, as a provided Docker container,  through
       - [AUP](docs/LOG_OVERVIEW.md#accceptable-use-policy-logs-aup)
       - [DNS](docs/LOG_OVERVIEW.md#dns)
       - [PROXY](docs/LOG_OVERVIEW.md#proxy)
+      - [NETCON](docs/LOG_OVERVIEW.md#netcon)
     - [Akamai Phish-proof Multi Factor Authenticator (AKAMAI-MFA)](https://www.akamai.com/us/en/products/security/akamai-mfa.jsp)
       - [EVENT](docs/LOG_OVERVIEW.md#authentication-logs-auth)
     - [Akamai Guardicore Segmentation](https://www.akamai.com/lp/guardicore) (beta)
@@ -90,7 +91,7 @@ Anyway, details for some specific SIEM solutions can be found in [this directory
 - Python 3.9+ OR docker / docker-compose OR kubernetes / helm
 - AKAMAI .edgerc file ([see further documentation here](docs/AKAMAI_API_CREDENTIALS.md))
 - Outbound Internet access (Akamai API, Dockerhub OR Python repositories)
-
+- Compute resource footprint is minimal: 1 vCPU, 1 GB RAM, 500 MB free disk. See also [High availability](docs/HIGH_AVAILABILITY.md) doc.
 
 ### Command Line Usage
 ![ULS command line usage](docs/images/uls_cli_help_example.png)  

diff --git a/bin/config/global_config.py b/bin/config/global_config.py
@@ -1,7 +1,7 @@
 #!/usr/bin/env python3
 
 # Common global variables / constants
-__version__ = "1.6.5"
+__version__ = "1.6.6"
 __tool_name_long__ = "Akamai Unified Log Streamer"
 __tool_name_short__ = "ULS"
 
@@ -25,7 +25,7 @@
         # Path to the ETP CLI Executable
 bin_etp_cli = "ext/cli-etp/bin/akamai-etp"
         # Available ETP CLI feeds
-etp_cli_feeds = ['THREAT', 'AUP', 'DNS', 'PROXY']
+etp_cli_feeds = ['THREAT', 'AUP', 'DNS', 'PROXY', 'NETCON']
 
     # MFA
 bin_mfa_cli = "ext/cli-mfa/bin/akamai-mfa"      # Path to the MFA CLI Executable
@@ -76,6 +76,7 @@
 output_http_aggregate_count = 500               # Number of events to aggregate in POST request to HTTP Collector. 1 mean no aggregation
 output_http_aggregate_idle = 5                  # Aggregate will send the data regardless of the count if the previous event was x secs ago
 output_http_expected_status_code = 200          # Return Code for successful delivery
+output_http_liveness_check = True               # Send an OPTIONS request to probe the HTTP Server is live
     ## FILE
 output_file_encoding = "utf-8"                  # FILE Encoding setting
 output_file_handler_choices = ['SIZE', 'TIME']  # Available Choices for the file handler

diff --git a/bin/modules/UlsArgsParser.py b/bin/modules/UlsArgsParser.py
@@ -14,6 +14,7 @@
 
 import argparse
 import os
+from distutils.util import strtobool
 
 import config.global_config as uls_config
 
@@ -39,7 +40,6 @@ def init():
                         const=True,
                         help=f'Display {uls_config.__tool_name_short__} version and operational information')
 
-
     # ----------------------
     # Input GROUP
     input_group = parser.add_argument_group(title="Input",
@@ -190,6 +190,17 @@ def init():
                                    f"Example: %%s = [{{'event1': 'data1'}},{{'event2': 'data2'}},...] - "
                                    f"Default: {uls_config.output_http_aggregate_count}")
 
+    ## HTTP LIVENESS CHECK
+    output_group.add_argument('--httpliveness',
+                              action='store',
+                              type=lambda x: bool(strtobool(x)),
+                              default=(os.environ.get('ULS_HTTP_LIVENESS_CHECK') or uls_config.output_http_liveness_check),
+                              help=f"ULS to send a OPTIONS request to the HTTP Server "
+                                   f"to ensure its liveness. ULS will fail if server is not "
+                                   f"responding with HTTP/200 or HTTP/204. Set to False to "
+                                   f"disable. Default: {uls_config.output_http_liveness_check}"
+    )
+
     # FILE STUFF
     ## File Handler
     output_group.add_argument('--filehandler',

diff --git a/bin/modules/UlsOutput.py b/bin/modules/UlsOutput.py
@@ -46,6 +46,7 @@ def __init__(self, output_type: str,
                  http_out_auth_header=None,
                  http_url=None,
                  http_insecure=False,
+                 http_liveness=True,
                  filehandler=None,
                  filename=None,
                  filebackupcount=None,
@@ -74,6 +75,7 @@ def __init__(self, output_type: str,
         self.http_out_format = None
         self.http_out_aggregate_count = None
         self.http_url = None
+        self.http_liveness = None
         self.httpSession = None
         self.port = None
         self.host = None
@@ -144,6 +146,7 @@ def __init__(self, output_type: str,
                 self.http_out_auth_header = http_out_auth_header
 
             self.http_insecure = http_insecure
+            self.http_liveness = http_liveness
             self.http_timeout = uls_config.output_http_timeout
 
         elif self.output_type in ['HTTP'] and not http_url:
@@ -261,22 +264,26 @@ def connect(self):
                         aka_log.log.info(f"{self.name} attempting to connect via "
                                          f"HTTP(S) to {self.http_url} ")
 
-                    # Let'S do an options request
-                    resp = self.httpSession.options(url=self.http_url,
-                                                    data='{"event":"connection test"}',
-                                                    verify=self.http_verify_tls, timeout=self.http_timeout)
-
-                    if resp.status_code == 200:
-                        reconnect_counter = 1
-                        self.connected = True
-                        aka_log.log.info(f"{self.name} successful connected to {self.http_url} ")
+                    # Probe the HTTP Server for liveness issuing OPTIONS request
+                    if self.http_liveness:
+                        resp = self.httpSession.options(url=self.http_url,
+                                                        data='{"event":"connection test"}',
+                                                        verify=self.http_verify_tls, timeout=self.http_timeout)
+
+                        if resp.status_code in (200, 204):
+                            reconnect_counter = 1
+                            self.connected = True
+                            aka_log.log.info(f"{self.name} Liveness check, connected to {self.http_url} ")
+                        else:
+                            aka_log.log.error(f"{self.name} Liveness check, error connecting to {self.http_url}. "
+                                              f"StatusCode: {resp.status_code} Reason: "
+                                              f"{resp.text} [{reconnect_counter}]")
+                            time.sleep(uls_config.output_reconnect_delay)
+                            self.connected = False
+                            reconnect_counter = reconnect_counter + 1
                     else:
-                        aka_log.log.error(f"{self.name} error connecting to {self.http_url}. "
-                                          f"StatusCode: {resp.status_code} Reason: "
-                                          f"{resp.text} [{reconnect_counter}]")
-                        time.sleep(uls_config.output_reconnect_delay)
-                        self.connected = False
-                        reconnect_counter = reconnect_counter + 1
+                        aka_log.log.info("Bypassing HTTP liveness check, reason: manually disabled.")
+                        self.connected = True
 
                 # RAW OUTPUT
                 elif self.output_type == "RAW":

diff --git a/bin/uls.py b/bin/uls.py
@@ -120,6 +120,7 @@ def main():
                                     http_out_aggregate_count=uls_args.httpaggregate,
                                     http_url=uls_args.httpurl,
                                     http_insecure=uls_args.httpinsecure,
+                                    http_liveness=uls_args.httpliveness,
                                     filehandler=uls_args.filehandler,
                                     filename=uls_args.filename,
                                     filebackupcount=uls_args.filebackupcount,

diff --git a/docs/ARGUMENTS_ENV_VARS.md b/docs/ARGUMENTS_ENV_VARS.md
@@ -38,6 +38,7 @@ The following tables list all available command line parameters and their corres
 | --httpformat     | HTTP(S)     | ULS_HTTP_FORMAT      | '<http_output_format>'                 | '{"event": %s}'          | Specify the expected output format (e.g. json) where %s will be replaced with the event data. /!\ %s can only be used once                                                                                                                                                                       |
 | --httpauthheader | HTTP(S)     | ULS_HTTP_AUTH_HEADER | '{"Authorization": "VALUE"}'           | None                     | Specify an Auhtorization header to auth against the HTTP Server (HTTP only) <br>Example:<br>'{"Authorization": "Splunk xxxx-xxxx-xxxx-xxxx-xxxxxxxxxxx"}'                                                                                                                                        |
 | --httpinsecure   | HTTP(S)     | ULS_HTTP_INSECURE    | True                                   | False                    | Disable TLS CA certificate verification                                                                                                                                                                                                                                                          |
+| --httpliveness | HTTP(S) | ULS_HTTP_LIVENESS_CHECK | True, False | True | Perform liveness check with OPTIONS request that must return 200 or 204 if enabled|
 | --httpaggregate  | HTTP(S)     | ULS_HTTP_AGGREGATE   | xxxx                                   | 500                      | Number of events to aggregate for one output request the %s in the httpformat will be replaced by a LIST of events.<br> A value of 1 means no aggregation.<br>Example: %s = [{'event1': 'data1'},{'event2': 'data2'},...]                                                |
 |                  |             |                      |                                        |                          |                                                                                                                                                                                                                                                                                                  | 
 | --filehandler    | FILE        | ULS_FILE_HANDLER     | 'SIZE','TIME'                          | SIZE                     | Select the handler which decides how the files are rotated if either specific SIZE or TIME has been reached                                                                                                                                                                                      |

diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
@@ -1,4 +1,22 @@
 # Version History
+
+## v1.6.6
+|||
+|---|---|
+|Date|2023-08-23
+|Kind| Minor release
+|Author|mschiess@akamai.com, androcho@akamai.com
+- **Features**
+  - Added '--httpliveness' to disable HTTP(S) OPTIONS request for liveness checking
+  - Added new feed for ETP: Network traffic connections details (netcon) [Requires CLI-ETP >= 0.4.2]
+- **Minor improvements**
+  - DOC Fix for manual CLI installation
+  - [docker] bumped CLI-EAA version to "0.5.9"
+  - [docker] bumped CLI-ETP version to "0.4.2"
+- **Housekeeping**
+  - Updated the ETP Links from developer.akamai.com to techdocs.akamai.com
+  - Added "docker file liniting" into test scripts
+
 ## v1.6.5
 |||
 |---|---|

diff --git a/docs/COMMAND_LINE_USAGE.md b/docs/COMMAND_LINE_USAGE.md
@@ -226,7 +226,7 @@ git -C ext/cli-eaa pull -q
 pip3 install -q -r ext/cli-eaa/requirements.txt
 
 # ETP CLI (only if installed)
-git -C ext/cli-etppull -q
+git -C ext/cli-etp pull -q
 pip3 install -q -r ext/cli-etp/requirements.txt
 
 # MFA CLI (only if installed)

diff --git a/docs/LOG_OVERVIEW.md b/docs/LOG_OVERVIEW.md
@@ -18,6 +18,7 @@ Here are some examples (per product) and links to additional information.
     - [Accceptable Use Policy Logs (AUP)](#accceptable-use-policy-logs-aup)
     - [DNS](#dns)
     - [PROXY](#proxy)
+    - [NETCON](#netcon)
   - [Akamai MFA (MFA)](#akamai-mfa-mfa)
     - [Authentication Logs (AUTH)](#authentication-logs-auth)
   - [Guardicore](#guardicore)
@@ -253,7 +254,7 @@ Each event will be one device as a JSON document, example provided with the cli-
 ## Enterprise Threat Protector (ETP)
 
 ### Threat Log (THREAT)
-Additional information regarding the log fields can be found on [here](https://developer.akamai.com/api/enterprise_security/enterprise_threat_protector_reporting/v3.html#threatevent)
+Additional information regarding the log fields can be found [here](https://techdocs.akamai.com/etp-reporting/reference/post-threat-event-details)
 ```json
 {
     "pageInfo": {
@@ -644,7 +645,7 @@ Additional information regarding the log fields can be found on [here](https://d
 ```
 
 ### Accceptable Use Policy Logs (AUP)
-Additional information regarding the log fields can be found on [here](https://developer.akamai.com/api/enterprise_security/enterprise_threat_protector_reporting/v3.html#event)
+Additional information regarding the log fields can be found [here](https://techdocs.akamai.com/etp-reporting/reference/get-events-details)
 ```json
 {
     "pageInfo": {
@@ -1035,7 +1036,7 @@ Additional information regarding the log fields can be found on [here](https://d
 ```
 
 ### DNS
-Additional information regarding the log fields can be found on [here](https://developer.akamai.com/api/enterprise_security/enterprise_threat_protector_reporting/v3.html#dnsactivityevent)
+Additional information regarding the log fields can be found [here](https://techdocs.akamai.com/etp-reporting/reference/post-dns-activities-details)
 ```json
 {
     "pageInfo": {
@@ -1262,7 +1263,7 @@ Additional information regarding the log fields can be found on [here](https://d
 ```
 
 ### PROXY
-Additional information regarding the log fields can be found on [here](https://developer.akamai.com/api/enterprise_security/enterprise_threat_protector_reporting/v3.html#proxytraffictransaction)
+Additional information regarding the log fields can be found [here](https://techdocs.akamai.com/etp-reporting/reference/post-traffic-transaction-details)
 ```json
 {
     "pageInfo": {
@@ -2086,6 +2087,11 @@ Additional information regarding the log fields can be found on [here](https://d
 }
 ```
 
+### NETCON
+Additional information regarding the log fields can be found [here](https://techdocs.akamai.com/etp-reporting/reference/post-network-traffic-connections-details)
+```json
+{"id": "123", "connectionId": "0xABCDEF1234567890", "domain": "123.123.123.123", "connStartTime": "2023-08-23T07:59:11Z", "connEndTime": "2023-08-23T07:59:11Z", "clientIP": "222.111.222.111", "clientPort": 35593, "destinationIP": "111.222.111.222", "destinationPort": 80, "siteId": 1234536, "siteName": "ETP DEMO", "policyAction": "onramp", "onrampType": "explicit_proxy_tls", "internalClientIP": "", "httpVersion": "N/A", "httpUserAgent": "", "machineId": "", "machineName": "", "clientRequestId": "", "ovfActionId": -1, "ovfActionName": "N/A", "stats": {"httpRequestCount": 1, "inBytes": 0, "outBytes": 0}, "dropInfo": {"wasDropped": true, "droppedReason": "Destination Filter - Internal Host IP"}, "encryptedInternalClientIP": "123123123123123123/ABCDEF", "decryptedInternalClientIP": "192.168.11.168", "sublocationId": "-1", "sublocationName": "N/A", "deviceOwnerId": "", "encryptedInternalClientName": ""}
+```
 
 ## Akamai MFA (MFA)
 Additional information regarding the MFA log fields can be found on [here](https://techdocs.akamai.com/mfa/docs/splunk-app).

diff --git a/docs/examples/kubernetes/helm/akamai-uls/Chart.yaml b/docs/examples/kubernetes/helm/akamai-uls/Chart.yaml
@@ -3,5 +3,5 @@ name: akamai-uls
 description: Akamai Universal Log Streamer Helm installation
 
 type: application
-version: 1.6.5
-appVersion: "1.6.5"
+version: 2.0.0
+appVersion: "1.6.6"
diff --git a/test/README.md b/test/README.md
@@ -36,6 +36,7 @@ For better testing stability, we packed the required dependencies into the test
 ```
 brew install bats-core
 brew install helm
+brew install hadolint
 
 git clone https://github.com/ztombol/bats-assert.git test/bats/bats-assert
 git clone https://github.com/ztombol/bats-support.git test/bats/bats-support