v1.7.0 (#48)

preparing v1.7.0 release
akamai · Oct 10, 2023 · 9896496 · 9896496
1 parent 758786f
commit 9896496
Show file tree

Hide file tree

Showing 16 changed files with 375 additions and 72 deletions.
diff --git a/Dockerfile b/Dockerfile
@@ -1,4 +1,4 @@
-FROM            python:3.11.4-slim-bookworm
+FROM            python:3.12.0-slim-bookworm
 LABEL           MAINTAINER="Mike Schiessl - mike.schiessl@akamai.com"
 LABEL	        APP_LONG="Akamai Universal Log Streamer"
 LABEL           APP_SHORT="ULS"
@@ -10,10 +10,10 @@ ARG             HOMEDIR="/opt/akamai-uls"
 ARG             ULS_DIR="$HOMEDIR/uls"
 ARG             EXT_DIR="$ULS_DIR/ext"
 
-ARG             ETP_CLI_VERSION="0.4.2"
-ARG             EAA_CLI_VERSION="0.5.9"
+ARG             ETP_CLI_VERSION="0.4.4"
+ARG             EAA_CLI_VERSION="0.6.2"
 ARG             MFA_CLI_VERSION="0.1.1"
-ARG             GC_CLI_VERSION="v0.0.1(beta)"
+ARG             GC_CLI_VERSION="v0.0.2(beta)"
 ARG             LINODE_CLI_VERSION="dev"
 
 # ENV VARS

diff --git a/README.md b/README.md
@@ -38,6 +38,7 @@ It can be run directly as Python code, as a provided Docker container,  through
       - [ADMIN](docs/LOG_OVERVIEW.md#admin-logs-admin)
       - [CONHEALTH](docs/LOG_OVERVIEW.md#connector-health-conhealth)
       - [DEVINV](docs/LOG_OVERVIEW.md#device-inventory-devinv)
+      - [DIRHEALTH](docs/LOG_OVERVIEW.md#directory-health-dirhealth)
     - [Enterprise Threat Protectors (ETP)](https://www.akamai.com/us/en/products/security/enterprise-threat-protector.jsp)
       - [THREAT](docs/LOG_OVERVIEW.md#threat-log-threat)
       - [AUP](docs/LOG_OVERVIEW.md#accceptable-use-policy-logs-aup)
@@ -101,7 +102,7 @@ Example commands:
 python3.9 bin/uls.py --input etp --feed threat --output raw
 
 # EAA - ACCESS to TCP
-python3.9  bin/uls.py --input eaa --feed access -output tcp --host 10.99.10.99 --port 8081
+python3.9  bin/uls.py --input eaa --feed access --output tcp --host 10.99.10.99 --port 8081
 ```
 For more information, please visit [this document](./docs/COMMAND_LINE_USAGE.md)
 

diff --git a/bin/config/global_config.py b/bin/config/global_config.py
@@ -1,7 +1,7 @@
 #!/usr/bin/env python3
 
 # Common global variables / constants
-__version__ = "1.6.6"
+__version__ = "1.7.0"
 __tool_name_long__ = "Akamai Unified Log Streamer"
 __tool_name_short__ = "ULS"
 
@@ -19,7 +19,7 @@
         # Path to the EAA CLI Executable
 bin_eaa_cli = "ext/cli-eaa/bin/akamai-eaa"
         # Available EAA CLI feeds
-eaa_cli_feeds = ['ACCESS', 'ADMIN', 'CONHEALTH', 'DEVINV']
+eaa_cli_feeds = ['ACCESS', 'ADMIN', 'CONHEALTH', 'DEVINV', 'DIRHEALTH']
 
     # ETP
         # Path to the ETP CLI Executable
@@ -60,7 +60,7 @@
 input_rerun_delay = 1                           # Time in seconds between rerun attempts
 input_disable_stderr = True                     # Enable STDERR output disabling (see value below to specify when this should happen)
 input_disable_stderr_after = 25                 # Disable stderr output after x input_cli cycles --> to prevent buffer overflow
-input_queue_size = 10000                        # Maximum number of events we want to store in-memory, default is 10000
+input_queue_size = 15000                        # Maximum number of events we want to store in-memory, default is 10000
 
 # OUTPUT Configuration
 output_reconnect_retries = 10                   # Number of reconnect attempts before giving up
@@ -77,6 +77,10 @@
 output_http_aggregate_idle = 5                  # Aggregate will send the data regardless of the count if the previous event was x secs ago
 output_http_expected_status_code = 200          # Return Code for successful delivery
 output_http_liveness_check = True               # Send an OPTIONS request to probe the HTTP Server is live
+output_http_default_formattype = 'json-list'   # The default "formattype" being used in standard operation
+output_http_formattypes = ['json-list', 'single-event']     # List of choices (valid formattypes)
+
+
     ## FILE
 output_file_encoding = "utf-8"                  # FILE Encoding setting
 output_file_handler_choices = ['SIZE', 'TIME']  # Available Choices for the file handler

diff --git a/bin/modules/UlsArgsParser.py b/bin/modules/UlsArgsParser.py
@@ -110,6 +110,14 @@ def init():
                              default=(os.environ.get('ULS_ENDTIME') or None),
                              help="End time (EPOCH SECONDS) until when to stop getting logs ('default': cli_default (never), example: '1631556101')")
 
+    # INPUT QUEUE SIZE
+    input_group.add_argument('--inputqueuesize',
+                             action='store',
+                             type=int,
+                             dest="input_queue_size",
+                             default=(os.environ.get('ULS_INPUT_QUEUESIZE') or uls_config.input_queue_size ),
+                             help=f"Maximum threshold of the input queue. (Default: {uls_config.input_queue_size})")
+
     # ----------------------
     # Output GROUP
     output_group = parser.add_argument_group(title="Output",
@@ -201,6 +209,16 @@ def init():
                                    f"disable. Default: {uls_config.output_http_liveness_check}"
     )
 
+    ## HTTP FORMATTYPE
+    output_group.add_argument('--httpformattype',
+                              action='store',
+                              type=str.lower,
+                              default=(os.environ.get('ULS_HTTP_FORMAT_TYPE') or
+                                       uls_config.output_http_default_formattype),
+                              choices=uls_config.output_http_formattypes,
+                              help=f"Specifies the type how the given http format is being wrapped (controls, how the httpformat is being rendered in http output) "
+                                   f" Default: {uls_config.output_http_default_formattype}, Valid Choices: {uls_config.output_http_formattypes}")
+
     # FILE STUFF
     ## File Handler
     output_group.add_argument('--filehandler',

diff --git a/bin/modules/UlsInputCli.py b/bin/modules/UlsInputCli.py
@@ -175,6 +175,16 @@ def proc_create(self):
                                        'dp',
                                        'inventory',
                                        '--tail']
+                    elif my_feed == "DIRHEALTH":
+                        UlsTools.uls_check_edgerc(self.credentials_file,
+                                                  self.credentials_file_section,
+                                                  uls_config.edgerc_openapi)
+                        cli_command = [self.bin_python,
+                                       '-u',
+                                       product_path,
+                                       'dir',
+                                       'list',
+                                       '--tail']
                     else:
                         UlsTools.uls_check_edgerc(self.credentials_file,
                                                   self.credentials_file_section,

diff --git a/bin/modules/UlsOutput.py b/bin/modules/UlsOutput.py
@@ -47,6 +47,7 @@ def __init__(self, output_type: str,
                  http_url=None,
                  http_insecure=False,
                  http_liveness=True,
+                 http_formattype=None,
                  filehandler=None,
                  filename=None,
                  filebackupcount=None,
@@ -116,6 +117,9 @@ def __init__(self, output_type: str,
             self.http_out_aggregate_count = http_out_aggregate_count        # Added for easier CLI configuration
             self.aggregateListTick = None # Last time we added items in the list
             # ---- End change for EME-588 ----
+            self.http_formattype = http_formattype
+
+
 
             self.http_url = http_url
             # apply other variables if SET
@@ -406,7 +410,6 @@ def send_data(self, data):
         """
         try:
             aka_log.log.debug(f"{self.name} Trying to send data via {self.output_type}")
-
             if self.output_type == "TCP":
                 send_data = bytes(self.tcpudp_out_format, 'utf-8') % data
                 out_data = send_data + uls_config.output_line_breaker.encode()
@@ -429,7 +432,23 @@ def send_data(self, data):
                     self.aggregateListTick is not None and
                     self.aggregateListTick < time.time() - uls_config.output_http_aggregate_idle
                 ):
-                    request = requests.Request('POST', url=self.http_url, data=(self.http_out_format % json.dumps(self.aggregateList)))
+
+
+                    # JSON-LIST EVENT FORMAT: '{"event": [{logline1},{logline2},{logline3},{….},{logline500}]}'
+                    # See https://github.com/akamai/uls/issues/45
+                    if self.http_formattype.lower() == "json-list":
+                        request = requests.Request('POST', url=self.http_url, data=(self.http_out_format % json.dumps(self.aggregateList)))
+
+                    # Single EVENT FORMAT: '{"event": {logline1}}{"event": {logline2}}{"event": {….}}{"event": {logline500}}'
+                    # See https://github.com/akamai/uls/issues/45
+                    elif self.http_formattype.lower() == "single-event":
+                        #[print(fruit + " juice") for fruit in fruits]
+                        single_event_data = ""
+                        for logline in self.aggregateList:
+                            #print(f"logline: {self.http_out_format % logline}")
+                            single_event_data = f"{single_event_data}{self.http_out_format % json.dumps(logline)}"
+                        request = requests.Request('POST', url=self.http_url, data=(single_event_data))
+
                     prepped = self.httpSession.prepare_request(request)
                     payload_length = prepped.headers["Content-Length"]
 

diff --git a/bin/uls.py b/bin/uls.py
@@ -121,6 +121,7 @@ def main():
                                     http_url=uls_args.httpurl,
                                     http_insecure=uls_args.httpinsecure,
                                     http_liveness=uls_args.httpliveness,
+                                    http_formattype=uls_args.httpformattype,
                                     filehandler=uls_args.filehandler,
                                     filename=uls_args.filename,
                                     filebackupcount=uls_args.filebackupcount,
@@ -162,7 +163,7 @@ def main():
     # New ULS/1.5: the input module is ingesting messages
     # into a thread safe queue. The function call will immediately
     # return
-    event_q = queue.Queue(uls_config.input_queue_size)
+    event_q = queue.Queue(uls_args.input_queue_size)
     my_input.ingest(stopEvent, event_q, my_monitor)
 
     # Now we are back to the main thread to process the message

diff --git a/docs/AKAMAI_API_CREDENTIALS.md b/docs/AKAMAI_API_CREDENTIALS.md
@@ -16,19 +16,19 @@ This document describes how to create Akamai API credentials and configure them
   - [Guardicore](#guardicore)
     - [Guardicore API Integration](#guardicore-api-integration)
   - [Linode](#linode)
-    - [Linode API Token](#linode-api-credentials)
+    - [Linode API Credentials](#linode-api-credentials)
 - [Advanced .edgerc usage](#advanced-edgerc-usage)
   - [Multiple customer contracts](#multiple-customer-contracts)
-  - [Partner & employee enhancement](#partner--employee-enhancement)
+  - [Partner \& employee enhancement](#partner--employee-enhancement)
   - [ETP API EVENT Filters](#etp-api-event-filters)
 
 ## Feeds / API overview
 
 |Product long name|Acronym| Feed(s)                         | API                                                                                   |
 |---|---|---------------------------------|---------------------------------------------------------------------------------------|
 |Enterprise Application Access|EAA| ACCESS, ADMIN                   | [EAA Legacy API](#eaa-legacy-api-for-access-and-admin-audit-feeds)                    |
-|Enterprise Application Access|EAA| HEALTH                          | [{OPEN} API / Enterprise Application Access](#eaa-open-api-for-connector-health-feed) |
-|Enterprise Threat Protector|ETP| THREAT, AUP, DNS, PROXY         | [{OPEN} API / ETP Report](#etp-open-api-reporting)                                    |
+|Enterprise Application Access|EAA| CONHEALTH, DEVINV, DIRHEALTH                          | [{OPEN} API / Enterprise Application Access](#eaa-open-api-for-connector-health-feed) |
+|Secure Internet Access Enterprise|ETP| THREAT, AUP, DNS, PROXY, NETCON         | [{OPEN} API / ETP Report](#etp-open-api-reporting)                                    |
 |Akamai MFA|MFA| EVENTS                          | [MFA Integration](#mfa-integration-for-logging)                                       |
 |Guardicore|GC| NETLOG, INCIDENT, AGENT, SYSTEM | [Guardicore API Integration](#guardicore-api-integration)                             |
 |Linode|LN| AUDIT                           | [Linode API Credentials](#linode-api-credentials) |