From 919a05ee3a26f6aa330785c406394972fa25256e Mon Sep 17 00:00:00 2001
From: Mike Schiessl <77062930+MikeSchiessl@users.noreply.github.com>
Date: Tue, 2 Apr 2024 11:10:41 +0200
Subject: [PATCH] Development (#61)
v1.7.3 preparation
---
Dockerfile | 4 +-
README.md | 1 +
bin/modules/UlsInputCli.py | 24 ++++---
bin/modules/UlsOutput.py | 39 ++++++++++-
bin/modules/UlsTools.py | 29 +++++++-
bin/uls.py | 11 +++
bin/uls_config/global_config.py | 6 +-
docs/ARGUMENTS_ENV_VARS.md | 67 +++++++++++++++----
docs/CHANGELOG.md | 56 +++++++++++++---
docs/COMMAND_LINE_USAGE.md | 11 +--
.../kubernetes/helm/akamai-uls/Chart.yaml | 2 +-
scripts/get-uls.sh | 47 +++++++++++--
test/positive_test.bats | 8 +++
13 files changed, 254 insertions(+), 51 deletions(-)
diff --git a/Dockerfile b/Dockerfile
index 77e2295..f37924a 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -10,10 +10,10 @@ ARG HOMEDIR="/opt/akamai-uls"
ARG ULS_DIR="$HOMEDIR/uls"
ARG EXT_DIR="$ULS_DIR/ext"
-ARG ETP_CLI_VERSION="0.4.5"
+ARG ETP_CLI_VERSION="0.4.7"
ARG EAA_CLI_VERSION="0.6.3"
ARG MFA_CLI_VERSION="0.1.1"
-ARG GC_CLI_VERSION="v0.0.3(beta)"
+ARG GC_CLI_VERSION="v0.0.4(beta)"
ARG LINODE_CLI_VERSION="dev"
# ENV VARS
diff --git a/README.md b/README.md
index 4db740a..4254867 100644
--- a/README.md
+++ b/README.md
@@ -52,6 +52,7 @@ It can be run directly as Python code, as a provided Docker container, through
- [INCIDENT](docs/LOG_OVERVIEW.md#incident)
- AGENT
- SYSTEM
+ - AUDIT
- [Linode](https://www.linode.com/) (experimental)
- [AUDIT](docs/LOG_OVERVIEW.md#audit-logs)
diff --git a/bin/modules/UlsInputCli.py b/bin/modules/UlsInputCli.py
index a633584..5faf067 100644
--- a/bin/modules/UlsInputCli.py
+++ b/bin/modules/UlsInputCli.py
@@ -86,6 +86,9 @@ def __init__(self,
self.disable_stderr = uls_config.input_disable_stderr #Specify if STDERR should be disabled at all after $disable_stderr_after cycles
self.disable_stderr_after = uls_config.input_disable_stderr_after # Disable StdErr Output after # cycles
+ # Generic vars
+ self.edgerc_hostname = None
+
def _feed_selector(self, feed, product_feeds):
if feed in product_feeds:
# feed matches the given list
@@ -155,7 +158,7 @@ def proc_create(self):
if not self.rawcmd:
my_feed = self._feed_selector(self.feed, product_feeds)
if my_feed == "CONHEALTH":
- UlsTools.uls_check_edgerc(self.credentials_file,
+ self.edgerc_hostname = UlsTools.uls_check_edgerc(self.credentials_file,
self.credentials_file_section,
uls_config.edgerc_openapi)
cli_command = [self.bin_python,
@@ -166,7 +169,7 @@ def proc_create(self):
'--perf',
'--tail']
elif my_feed == "DEVINV":
- UlsTools.uls_check_edgerc(self.credentials_file,
+ self.edgerc_hostname = UlsTools.uls_check_edgerc(self.credentials_file,
self.credentials_file_section,
uls_config.edgerc_openapi)
cli_command = [self.bin_python,
@@ -176,7 +179,7 @@ def proc_create(self):
'inventory',
'--tail']
elif my_feed == "DIRHEALTH":
- UlsTools.uls_check_edgerc(self.credentials_file,
+ self.edgerc_hostname = UlsTools.uls_check_edgerc(self.credentials_file,
self.credentials_file_section,
uls_config.edgerc_openapi)
cli_command = [self.bin_python,
@@ -186,7 +189,7 @@ def proc_create(self):
'list',
'--tail']
else:
- UlsTools.uls_check_edgerc(self.credentials_file,
+ self.edgerc_hostname = UlsTools.uls_check_edgerc(self.credentials_file,
self.credentials_file_section,
uls_config.edgerc_eaa_legacy)
cli_command = [self.bin_python, '-u', product_path, 'log', my_feed.lower(), '-f']
@@ -218,7 +221,7 @@ def proc_create(self):
aka_log.log.warning(f"{self.name} - Selected LOG Format ({self.cliformat}) "
f"not available for {product_path}, continuing with JSON.")
if not self.rawcmd:
- UlsTools.uls_check_edgerc(self.credentials_file, self.credentials_file_section,
+ self.edgerc_hostname = UlsTools.uls_check_edgerc(self.credentials_file, self.credentials_file_section,
uls_config.edgerc_openapi + ["etp_config_id"])
my_feed = self._feed_selector(self.feed, product_feeds)
cli_command = [self.bin_python, '-u', product_path, 'event', my_feed.lower(), '-f']
@@ -249,7 +252,7 @@ def proc_create(self):
aka_log.log.warning(f"{self.name} - Selected LOG Format ({self.cliformat}) "
f"not available for {product_path}, continuing with JSON.")
if not self.rawcmd:
- UlsTools.uls_check_edgerc(self.credentials_file,
+ self.edgerc_hostname = UlsTools.uls_check_edgerc(self.credentials_file,
self.credentials_file_section,
uls_config.edgerc_mfa)
my_feed = self._feed_selector(self.feed, product_feeds)
@@ -279,7 +282,7 @@ def proc_create(self):
aka_log.log.warning(f"{self.name} - Selected LOG Format ({self.cliformat}) "
f"not available for {product_path}, continuing with JSON.")
if not self.rawcmd:
- UlsTools.uls_check_edgerc(self.credentials_file,
+ self.edgerc_hostname = UlsTools.uls_check_edgerc(self.credentials_file,
self.credentials_file_section,
uls_config.edgerc_gc)
my_feed = self._feed_selector(self.feed, product_feeds)
@@ -309,7 +312,7 @@ def proc_create(self):
aka_log.log.warning(f"{self.name} - Selected LOG Format ({self.cliformat}) "
f"not available for {product_path}, continuing with JSON.")
if not self.rawcmd:
- UlsTools.uls_check_edgerc(self.credentials_file,
+ self.edgerc_hostname = UlsTools.uls_check_edgerc(self.credentials_file,
self.credentials_file_section,
uls_config.edgerc_linode)
my_feed = self._feed_selector(self.feed, product_feeds)
@@ -476,4 +479,7 @@ def ingest_loop(self):
except Exception:
aka_log.log.exception("Error in ingest_loop")
-# EOF
+ def get_edgerc_hostname(self):
+ return self.edgerc_hostname
+
+ # EOF
diff --git a/bin/modules/UlsOutput.py b/bin/modules/UlsOutput.py
index 10dde6f..2e52e54 100644
--- a/bin/modules/UlsOutput.py
+++ b/bin/modules/UlsOutput.py
@@ -97,7 +97,7 @@ def __init__(self, output_type: str,
if self.output_type in ['TCP', 'UDP'] and host and port:
self.host = host
self.port = port
- if "%s" in tcpudp_out_format:
+ if "%s" in str(tcpudp_out_format):
self.tcpudp_out_format = tcpudp_out_format
else:
aka_log.log.critical(
@@ -513,4 +513,41 @@ def tear_down(self):
self.httpSession.close()
self.connected = False
+### FEATURE REQ 20240318 - https://github.com/akamai/uls/issues/57
+
+ def ingest_vars_into_output_format(self, placeholder: str = None, replacement: str = None):
+ if not placeholder or not replacement:
+ aka_log.log.debug(f"{self.name} Variable substition triggered but no value given (inline code issue)")
+ #print(f"PLACEHOLDER: {placeholder}")
+ #print(f"REPLACEMENT: {replacement}")
+ #return True
+ sys.exit(1)
+
+ if placeholder in str(self.tcpudp_out_format) and self.tcpudp_out_format:
+ self.tcpudp_out_format = str(self.tcpudp_out_format).replace(placeholder, replacement)
+ aka_log.log.debug(f"{self.name} Replacing {placeholder} in TCPUDP string with: {replacement} ")
+ aka_log.log.debug(f"{self.name} SUBSTITUTION new TCPUDP output string: {self.tcpudp_out_format} ")
+
+ if placeholder in str(self.http_out_format) and self.http_out_format:
+ self.http_out_format = str(self.http_out_format).replace(placeholder, replacement)
+ aka_log.log.debug(f"{self.name} Replacing {placeholder} in HTTP string with: {replacement} ")
+ aka_log.log.debug(f"{self.name} SUBSTITUTION new HTTP output string: {self.http_out_format} ")
+
+
+ return True
+
+ def ingest_os_vars_into_output_format(self):
+ aka_log.log.debug(f"{self.name} Replacing ENV VARS in output FORMAT")
+ if self.tcpudp_out_format:
+ self.tcpudp_out_format = os.path.expandvars(self.tcpudp_out_format)
+ aka_log.log.debug(f"{self.name} OS_ENV_VARS new TCPUDP output string: {self.tcpudp_out_format} ")
+ if self.http_out_format:
+ self.http_out_format = os.path.expandvars(self.http_out_format)
+ aka_log.log.debug(f"{self.name} OS_ENV_VARS new HTTP output string: {self.http_out_format} ")
+
+
+ return True
+
+ ### / FEATURE REQ 20240318 - https://github.com/akamai/uls/issues/57
+
# EOF
diff --git a/bin/modules/UlsTools.py b/bin/modules/UlsTools.py
index 2579b40..92d190d 100644
--- a/bin/modules/UlsTools.py
+++ b/bin/modules/UlsTools.py
@@ -121,9 +121,11 @@ def uls_check_edgerc(configfile, configsection, configvalues):
Verify the given "edgerc" file to contain all required variables (for the desired stream) within the given section
see https://github.com/akamai/uls/blob/main/docs/AKAMAI_API_CREDENTIALS.md for more information
:param configfile: The path to the config file
+
:param configsection: The section within the config file [default]
:param configvalues: A list of desiresd config values ["val1", "val2", ...]
:return:
+ api_hostname
"""
config = configparser.ConfigParser()
# Load config file
@@ -141,13 +143,23 @@ def uls_check_edgerc(configfile, configsection, configvalues):
aka_log.log.debug(f"Section '{configsection}' found.")
# check for specified values
+ my_hostname = None
for configvalue in configvalues:
if not configvalue in config[configsection]:
aka_log.log.critical(f"Required configuration value '{configvalue}' not found in section / file. Please see: {uls_config.edgerc_documentation_url} - Exiting")
sys.exit(1)
else:
+
+ ### FEATURE REQ 20240318
+ if 'host' in configvalue:
+ my_hostname = config[configsection][configvalue]
+ #print(f"HOSTNAME{my_hostname}")
+ ### FEATURE REQ 20240318
+
aka_log.log.debug(f"Required configuration value '{configvalue}' found.")
- return 0
+ if not my_hostname:
+ my_hostname = "no_hostname_available"
+ return my_hostname
def uls_check_args(input, output):
@@ -190,6 +202,7 @@ def check_autoresume(input, feed, checkpoint_dir=uls_config.autoresume_checkpoin
if (input not in uls_config.autoresume_supported_inputs or
feed == "CONHEALTH" or
feed == "DEVINV" or
+ feed == "AGENT" or
feed == "DIRHEALTH"):
aka_log.log.critical(f"Input {input} or feed {feed} currently not supported by AUTORESUME - Exiting.")
sys.exit(1)
@@ -205,7 +218,7 @@ def check_autoresume(input, feed, checkpoint_dir=uls_config.autoresume_checkpoin
checkpoint = None
else:
try:
- with open (checkpoint_full, "r") as ckpt_f:
+ with open(checkpoint_full, "r") as ckpt_f:
data = json.load(ckpt_f)
if data['creation_time'] and data['checkpoint']:
aka_log.log.debug(f"Autoresume Checkpoint successfully loaded. Checkpoint Time: {data['checkpoint']}, Creation_time: {data['creation_time']}")
@@ -215,6 +228,8 @@ def check_autoresume(input, feed, checkpoint_dir=uls_config.autoresume_checkpoin
mytime = data['checkpoint'].split("Z")[0]
elif data['input'] == "EAA":
mytime = data['checkpoint'].split("+")[0]
+ elif data['input'] == "GC":
+ mytime = data['checkpoint'].split(".")[0]
else:
aka_log.log.critical(
f"Unhandeled input data in checkpointfile \'{checkpoint_full}\' --> {input} / {feed} - Exiting.")
@@ -263,6 +278,12 @@ def write_autoresume_ckpt(input, feed, autoresume_file, logline, current_count):
checkpoint_timestamp = json.loads(checkpoint_line)['datetime']
elif input == "ETP" and feed == "NETCON":
checkpoint_timestamp = json.loads(checkpoint_line)['connStartTime']
+ elif input == "GC" and feed == "AUDIT":
+ checkpoint_timestamp = json.loads(checkpoint_line)['time']
+ elif input == "GC" and feed == "INCIDENT":
+ checkpoint_timestamp = json.loads(checkpoint_line)['closed_time']
+ elif input == "GC" and feed == "NETLOG":
+ checkpoint_timestamp = json.loads(checkpoint_line)['db_insert_time']
else:
aka_log.log.critical(
f"AUTORESUME - Unhandled Input / Feed detected: '{input} / {feed}' (this should never happen !!)- Exiting")
@@ -311,4 +332,6 @@ def get_install_id(install_id_file=str(root_path()) + "/var/uls_install_id"):
aka_log.log.debug(f"Not able to read install file - returning mocked data. Error: {error}")
data = {'install_id': "ERROR-GETTING-INSTALLATION-ID"}
#return install_id
- return data
\ No newline at end of file
+ return data
+
+
diff --git a/bin/uls.py b/bin/uls.py
index c597f81..d21dc6e 100755
--- a/bin/uls.py
+++ b/bin/uls.py
@@ -89,6 +89,7 @@ def main():
# Check CLI Environment
UlsTools.uls_check_sys(root_path=root_path, uls_input=uls_args.input)
+
# Create & Start monitoring Instance
my_monitor = UlsMonitoring.UlsMonitoring(stopEvent=stopEvent,
product=uls_args.input,
@@ -109,6 +110,7 @@ def main():
endtime=uls_args.endtime,
root_path=root_path)
+
# Connect to the selected input UlsOutput
my_output = UlsOutput.UlsOutput(output_type=uls_args.output,
host=uls_args.host,
@@ -151,11 +153,20 @@ def main():
else:
filter_pattern = None
+
# Now let's handle the data and send input to output
# Initiate the Input handler
my_input.proc_create()
+ # Append extra vars to the output
+ #my_output.ingest_vars_into_output_format(api_hostname=my_input.get_edgerc_hostname())
+ my_output.ingest_vars_into_output_format(placeholder='{api_hostname}', replacement=my_input.get_edgerc_hostname())
+ my_output.ingest_vars_into_output_format(placeholder='{uls_input}', replacement=uls_args.input)
+ my_output.ingest_vars_into_output_format(placeholder='{uls_feed}', replacement=uls_args.feed)
+ my_output.ingest_os_vars_into_output_format()
+
+
# Connect the output handler
my_output.connect()
diff --git a/bin/uls_config/global_config.py b/bin/uls_config/global_config.py
index 979ee62..e1f0360 100644
--- a/bin/uls_config/global_config.py
+++ b/bin/uls_config/global_config.py
@@ -1,7 +1,7 @@
#!/usr/bin/env python3
# Common global variables / constants
-__version__ = "1.7.2"
+__version__ = "1.7.3"
__tool_name_long__ = "Akamai Unified Log Streamer"
__tool_name_short__ = "ULS"
@@ -33,7 +33,7 @@
# Guardicore
bin_gc_cli = "ext/cli-gc/bin/akamai-gc" # Path to the GC CLI Executable
-gc_cli_feeds = ['NETLOG', 'INCIDENT', 'AGENT', 'SYSTEM'] # Available GC CLI feeds
+gc_cli_feeds = ['NETLOG', 'INCIDENT', 'AGENT', 'SYSTEM', 'AUDIT'] # Available GC CLI feeds
# LINODE
bin_linode_cli = "ext/cli-linode/bin/akamai-linode" # Path to the LINODE CLI Executable
@@ -107,6 +107,6 @@
# Autoresume Configuration
autoresume_checkpoint_path = "var/" # (Default) Path, where the checkpointfiles should be stored to
-autoresume_supported_inputs = ['ETP', 'EAA'] # Internal Var only, to adjust supported inputs
+autoresume_supported_inputs = ['ETP', 'EAA', 'GC'] # Internal Var only, to adjust supported inputs
autoresume_write_after = 1000 # Write checkpoint only every ${autoresume_write_every} loglines
diff --git a/docs/ARGUMENTS_ENV_VARS.md b/docs/ARGUMENTS_ENV_VARS.md
index fab9783..c55b5ce 100644
--- a/docs/ARGUMENTS_ENV_VARS.md
+++ b/docs/ARGUMENTS_ENV_VARS.md
@@ -21,18 +21,18 @@ The following tables list all available command line parameters and their corres
## INPUT
-| Parameter | Env - Var | Options | Default | Description |
-|---------------------------|-----------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
-| -i
--input | ULS_INPUT | 'EAA', 'ETP', 'MFA', 'GC', 'LINODE' | None | Specify the desired INPUT source |
-| --feed | ULS_FEED | EAA: 'ACCESS', 'ADMIN', 'CONHEALTH', 'DEVINV'
ETP: 'THREAT', 'AUP', 'DNS', 'PROXY'
MFA: 'EVENT'
GC: 'NETLOG', 'INCIDENT', 'AGENT', 'SYSTEM'
LINODE: 'AUDIT' | None | Specify the desired INPUT feed |
-| --format | ULS_FORMAT | 'JSON', 'TEXT' | JSON | Specify the desired INPUT (=OUTPUT) format |
-| --inproxy
--inputproxy | ULS_INPUT_PROXY | HOST:PORT | None | Adjust proxy usage for INPUT data collection (cli)
If this parameter does not work as expected, [please read more about it here](./FAQ.md#--inputproxy-proxy-does-not-work-as-expected) |
-| --rawcmd | ULS_RAWCMD | \ | None | USE with caution /!\
This is meant only to be used when told by AKAMAI [Click here for more information](ADDITIONAL_FEATURES.md#rawcmd---rawcmd-feature) |
-| --edgerc | ULS_EDGERC | /path/to/your/.edgerc | '~/.edgerc' | Specify the location of the .edgerc EDGE GRID AUTH file |
-| --section | ULS_SECTION | edgerc_config_section | 'default' | Specify the desired section within the .edgerc file |
-| --starttime | ULS_STARTTIME | EPOCH timestamp (in seconds) | `cli_default` | Specify an EPOCH timestamp from where to start the log collection. |
-| --endtime | ULS_ENDTIME | EPOCH timestamp (in seconds) | None | Specify an EPOCH timestamp up until where to fetch logs. ULS will exit after reaching this point.
ULS will not continue reading logs on CLI errors !!! |
-| --inputqueuesize | ULS_INPUT_QUEUESIZE | INPUT_QUEUE_SIZE(int) | 15000 | Maximum threshold of the input queue. When threshold is reached, ULS will stop operations and exit "Capacity exceeded, too many incoming data vs. slow output" |
+| Parameter | Env - Var | Options | Default | Description |
+|---------------------------|-----------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| -i
--input | ULS_INPUT | 'EAA', 'ETP', 'MFA', 'GC', 'LINODE' | None | Specify the desired INPUT source |
+| --feed | ULS_FEED | EAA: 'ACCESS', 'ADMIN', 'CONHEALTH', 'DEVINV'
ETP: 'THREAT', 'AUP', 'DNS', 'PROXY'
MFA: 'EVENT'
GC: 'NETLOG', 'INCIDENT', 'AGENT', 'SYSTEM', 'AUDIT'
LINODE: 'AUDIT' | None | Specify the desired INPUT feed |
+| --format | ULS_FORMAT | 'JSON', 'TEXT' | JSON | Specify the desired INPUT (=OUTPUT) format |
+| --inproxy
--inputproxy | ULS_INPUT_PROXY | HOST:PORT | None | Adjust proxy usage for INPUT data collection (cli)
If this parameter does not work as expected, [please read more about it here](./FAQ.md#--inputproxy-proxy-does-not-work-as-expected) |
+| --rawcmd | ULS_RAWCMD | \ | None | USE with caution /!\
This is meant only to be used when told by AKAMAI [Click here for more information](ADDITIONAL_FEATURES.md#rawcmd---rawcmd-feature) |
+| --edgerc | ULS_EDGERC | /path/to/your/.edgerc | '~/.edgerc' | Specify the location of the .edgerc EDGE GRID AUTH file |
+| --section | ULS_SECTION | edgerc_config_section | 'default' | Specify the desired section within the .edgerc file |
+| --starttime | ULS_STARTTIME | EPOCH timestamp (in seconds) | `cli_default` | Specify an EPOCH timestamp from where to start the log collection. |
+| --endtime | ULS_ENDTIME | EPOCH timestamp (in seconds) | None | Specify an EPOCH timestamp up until where to fetch logs. ULS will exit after reaching this point.
ULS will not continue reading logs on CLI errors !!! |
+| --inputqueuesize | ULS_INPUT_QUEUESIZE | INPUT_QUEUE_SIZE(int) | 15000 | Maximum threshold of the input queue. When threshold is reached, ULS will stop operations and exit "Capacity exceeded, too many incoming data vs. slow output" |
## OUTPUT
@@ -78,3 +78,46 @@ The following tables list all available command line parameters and their corres
| --autoresume | ULS_AUTORESUME | [True, False] | False | Enable automated resume on based on a checkpoint upon api failure or crash (do not use alongside --starttime) |
| --autoresumepath | ULS_AUTORESUME_PATH | '/path/to/store/checkpoints/' | var/ | Specify the path where checkpoint files should be written to. (Trailing /) |
| --autoresumewriteafter | ULS_AUTORESUME_WRITEAFTER | | 1000 | Specify after how many loglines a checkpoint should be written. |
+
+
+## Customizing HTTP & TCPUDP Formatting
+
+Applicable to argument `--tcpudpformat` or environment variable `ULS_TCPUDP_FORMAT`.
+
+By default ULS will write the exact payload received from the API to
+the configured ULS output. Thus, `%s` is the default value.
+
+### Payload decoration (Variable Substition)
+
+If you need to surround the payload with extra information (i.e. metadata for your SIEM),
+you can surround the `%s`. In the example below, didn't surround the `%s` by double-quote
+since we want the output to remain a valid JSON:
+
+```json
+{"event": %s}
+```
+
+### Output Variables
+
+While customizing you might want to use dynamic variables. Substitution happens when ULS
+software is starting, if you change your configuration file API hostname, you'll need
+to restart ULS so it can take effect.
+Besides the below replacements, ULS will also interprete OS (or DOCKER) ENV Vars.
+
+
+| Variable | Description |
+|----------------|--------------------------------------------------------------|
+| {api_hostname} | This variable will be replaced with the Akamai API Hostname |
+| {uls_input} | This variable will be replaced with the ULS Input |
+| {uls_feed} | This variable will be replaced with the ULS Feed |
+
+
+### Example
+```bash
+# Substitution example
+'{"api_host": "{api_hostname}", "input_feed": "{uls_input}-{uls_feed}", "event": %s}'
+
+# OS ENV Var example
+export MY_ENV_VAR="THIS IS MY ENV VAR"
+'{"api_host": "{api_hostname}", "input_feed": "{uls_input}-{uls_feed}", "additional_env": "$MY_ENV_VAR", "event": %s}'
+```
\ No newline at end of file
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 65f6110..9489c04 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -1,4 +1,25 @@
# Version History
+## v1.7.3
+|||
+|---|---|
+|Date|2024-04-02
+|Kind| MINOR release
+|Author|mschiess@akamai.com, androcho@akamai.com
+
+- **Features**
+ - introduced "audit logs" for Guardicore
+ - introduced "AUTORESUME" functionality for GC: NETLOG, INCIDENTS and AUDIT
+ - TCPUDP/HTTP format string now support [varialbe substitution](ARGUMENTS_ENV_VARS.md#payload-decoration-variable-substition):
+ - Substitution: {api_hostname}, {uls_input}, {uls_feed}
+ - OS ENV VARS: $VAR_NAME
+
+- **Minor improvements**
+ - [docker] bumped CLI-ETP to "0.4.7" - thx to @Antoine for a couple of bugfixes
+ - [docker] bumped CLI-EAA to "0.6.3"
+ - [docker] bumped CLI-GC to "v0.0.4(beta)"
+ - [CLI] Fixed an auto installer [Issue #58](https://github.com/akamai/uls/issues/58) - thx @Antoine
+---
+
## v1.7.2
|||
|---|---|
@@ -20,6 +41,7 @@
- Fixed a bug in the autoresume function that created a problem with timezones in certain circumstances
- **Housekeeping**
- improved local container testing
+---
## v1.7.1
|||
@@ -33,6 +55,7 @@
- [docker] bumped CLI-ETP version to "0.4.5"
- **Housekeeping**
- Added additional automated testing to the docker release process
+---
## v1.7.0
|||
@@ -54,7 +77,7 @@
- **Housekeeping**
- DocFix Readme.md (thx [@ihommani](https://github.com/akamai/uls/pull/47))
- Increased default input_queue_size from 10000 to 15000 to avoid race conditions when an API is answering very fast
-
+---
## v1.6.6
|||
@@ -72,7 +95,8 @@
- **Housekeeping**
- Updated the ETP Links from developer.akamai.com to techdocs.akamai.com
- Added "docker file liniting" into test scripts
-
+---
+
## v1.6.5
|||
|---|---|
@@ -82,6 +106,7 @@
- **Minor improvements**
- Allow manipulation of the [TCP & UDP output format](ARGUMENTS_ENV_VARS.md#list-of-parameters--environmental-variables) (--tcpudpformat / ULS_TCPUDP_FORMAT).
- [docker] bumped source image to 3.11.4-slim-bookworm (new debian release)
+---
## v1.6.4
|||
@@ -98,6 +123,7 @@
- **Bugfix**
- `--endtime ` didn't cause ULS to eventually stop ops. This is now fixed.
- improved container detection (only cosmetic improvement)
+---
## v1.6.3
@@ -116,6 +142,7 @@
- fixed a bug in the "get_uls.sh" script which stated the wrong error message when pip was not found
- minor fix to properly detect "podman" as docker alternative
- Amendend installation steps for Guardicore and Linode log-fetcher(s)
+---
## v1.6.2
@@ -132,6 +159,7 @@
- fixed a bug in the Dockerfile that left uls/var unusable
- **Housekeeping**
- fixed some bugs in testing (false negative) & speeded up testing process
+---
## v1.6.1
@@ -146,7 +174,7 @@
- **Housekeeping**
- Added parallel testing processes to speed up testing (see [Testing Readme](../test/README.md))
- added randomization tokens for "mocked" edgerc file (to avoid race condition in prallel testing)
-
+---
## v1.6.0
@@ -170,6 +198,7 @@
- bumped python container (docker) to version 3.10.7
- bumped ETP-CLI version to 0.3.9 which should massively reduce the fetch lag
- Added [documentation](./HIGH_AVAILABILITY.md) to explain high availability options for ULS
+---
## v1.5.1
@@ -182,6 +211,7 @@
- **Bugfix**
- Use cli-eaa version 0.5.0.2 fixing a timezone issue on the EAA ADMIN feed
+---
## v1.5.0
@@ -207,7 +237,7 @@
- `event_bytes_interval` # total size in bytes processed
- **Housekeeping**
- Documented missing dependencies in [test/README.md](../test/README.md)
-
+---
## v1.4.0
|||
@@ -226,6 +256,7 @@
- **Bugfix**
- Fixed a bug in the test scripts to support real `.edgerc`
+---
## v1.3.5
|||
@@ -236,6 +267,7 @@
- **Minor improvements**
- More QRADAR log source type definitions (thx to bitonio)
- Added docker-compose ETP - Tenant example
+---
## v1.3.4
|||
@@ -248,6 +280,7 @@
- Added SUMO Logic (thx to huskar20 for the contribution)
- bumped CLI-MFA to v0.0.9
- added resources, nodeSelector, tolerations and affinity to the helm values.yaml / template
+---
## v1.3.3
@@ -260,7 +293,7 @@
- Adopted to new MFA CLI Version (only single feed "EVENT" available anymore)
- Amended new dates to the file headers
- Added volume to dockerfile as data storage for "autoresume"
-
+---
## v1.3.2
|||
@@ -284,6 +317,7 @@
- added a sanity check for "HTTP_OUT_FORMAT" to avoid issues with the ´%s´ seclector
- removed forced http authentication token "--httpauthheader" (allow None)
- discovered a bug in configparser -> [see FAQ entry](FAQ.md#uls-does-not-start-due-to-missing-field-in-config)
+---
## v1.3.1
@@ -295,6 +329,7 @@
- **Bugfixes**
- fixed a checkpoint issue when using ETP / THREAD
- some doc fixes
+---
## v1.3.0
|||
@@ -320,7 +355,7 @@
- Fixed an output issue on "CLI failure", added configureable output handling to the config
- replaced pip with pip3 in CLI usage docs
- Fixed a windows bug (bypass blocking on windows) + added a [FAQ entry on how fix a installation specific bug](FAQ.md#uls-on-windows-error-winerror-2-the-system-cannot-find-the-file-specified)
-
+---
## v1.2.0
|||
@@ -349,6 +384,7 @@
- bumped EAA CLI Version to 0.4.4 (docker only)
- Introduced dedicated ["OUTPUT" documentation](OUTPUTS.md)
- introduced uls own requirements.txt in the [bin directory](../bin/requirements.txt) - still trying to keep req's as low as possible.
+---
## v1.1.0
|||
@@ -366,6 +402,7 @@
- added better error guidance when basic stuff is unset (input / output)
- moved docker-compose from root dir to /docs
- added `read_only: true` to the docker-compose.yml files (security enhancement)
+---
## v1.0.0
|||
@@ -380,7 +417,7 @@
- simplified cli - command re-usage (visual parsing of subprocess cmd)
- cleaned up the Dockerfile
- added [Log_Overview](LOG_OVERVIEW.md) page to extend background on logged data
-
+---
## v0.9.0
|||
@@ -399,7 +436,7 @@
- added FAQ documents
- Feature:
- [FILTER (--filter) feature](ADDITIONAL_FEATURES.md#filter---filter-feature) introduced to reduce number of sent log lines towards SIEM
-
+---
## v0.0.4
|||
@@ -416,6 +453,7 @@
- Feature:
- EAA CONNECTOR HEALTH (CONHEALTH) now available
- Preflight (forced) check for available cli's
+---
## v0.0.3
|||
@@ -428,6 +466,7 @@
- fixed a bug that caused Popen PIPE to hang in certain circumstances
- bumped Dockerfile to newer CLI versions
- [introduced RAW output](ADDITIONAL_FEATURES.md#rawcmd---rawcmd-feature) (send data to stdout)
+---
## v0.0.2
|||
@@ -438,6 +477,7 @@
- fixed monitoring output bug in docker-compose
- fixed bug in Dockerfile that prevented development builds
- fixed a bug in EAA CLI handler
+---
## v0.0.1 (Initial Commit)
|version|v0.0.1|
diff --git a/docs/COMMAND_LINE_USAGE.md b/docs/COMMAND_LINE_USAGE.md
index 72ea121..3919e9c 100644
--- a/docs/COMMAND_LINE_USAGE.md
+++ b/docs/COMMAND_LINE_USAGE.md
@@ -6,7 +6,7 @@ All commands referenced in this document are run from the repositories root leve
## Table of contents
- [ULS Command Line Usage](#uls-command-line-usage)
- - [Requirements](#requirements)
+ - [Pre-requisites](#pre-requisites)
- [Installation](#installation)
- [Automated Installation](#automated-installation)
- [Manual Installation](#manual-installation)
@@ -20,18 +20,19 @@ All commands referenced in this document are run from the repositories root leve
- [Automated Update](#automated-update)
- [Manual Update](#manual-update)
-## Requirements
+## Pre-requisites
+
+To run the operations in this documentation, make sure you have the following tools installed:
-To run the operations within the following documentation, you need to have the following tools installed:
- git
-- python >= 3.9 (including pip)
+- python >= `3.9` (including pip >= `22.2`)
- Akamai API credentials file - `.edgerc` (see [API Credentials](AKAMAI_API_CREDENTIALS.md) for creation instructions)
- Understanding of available [ULS CLI PARAMETERS](ARGUMENTS_ENV_VARS.md)
- Access to `github.com`, `pypi.org`, `pythonhosted.org` and `pypi.python.org` within your firewall
## Installation
-To install ULS, you can choose 2 different ways: automated or manual
+To install ULS, you can choose 2 different ways: automated or manual.
### Automated Installation
The automated installation actually does everything, the described below in the manual installation but saves you from the copying the blocks.
diff --git a/docs/examples/kubernetes/helm/akamai-uls/Chart.yaml b/docs/examples/kubernetes/helm/akamai-uls/Chart.yaml
index 3a3f671..577d52e 100644
--- a/docs/examples/kubernetes/helm/akamai-uls/Chart.yaml
+++ b/docs/examples/kubernetes/helm/akamai-uls/Chart.yaml
@@ -4,4 +4,4 @@ description: Akamai Universal Log Streamer Helm installation
type: application
version: 2.0.0
-appVersion: "1.7.2"
+appVersion: "1.7.3"
diff --git a/scripts/get-uls.sh b/scripts/get-uls.sh
index 2b8ed58..93da53c 100755
--- a/scripts/get-uls.sh
+++ b/scripts/get-uls.sh
@@ -5,6 +5,21 @@
default_modules="eaa,etp,mfa,gc,ln"
default_install_dir="$(pwd)/uls"
+function min_version() {
+ # Test if a given version string matches the requirements
+ # Usage:
+ # min_version required_min_version version
+ # Function returns:
+ # 0 if the version is equal or greater than the requirement
+ # 1 if the version is lower
+ local required_ver="$1"
+ local current_ver="$2"
+ if [ "$(printf '%s\n' "$required_ver" "$current_ver" | sort -V | head -n1)" = "$required_ver" ]; then
+ return 0
+ else
+ return 1
+ fi
+}
echo -e "
AKAMAI
@@ -14,7 +29,17 @@ echo -e "
_/ _/ _/ _/
_/_/ _/_/_/_/ _/_/_/
-Universal Log Stream - Installer"
+Unified Log Streamer - Installer
+
+Available ULS modules:
+ eaa: Enterprise Application Access
+ etp: Secure Internet Access Enterprise
+ mfa: Akamai phishproof MFA
+ gc: Akamai Guardicore Segmentation
+ ln: Linode Audit log
+
+More about supported feed:
+https://github.com/akamai/uls/blob/main/docs/LOG_OVERVIEW.md"
echo -e "\n\n\n"
# Preflight checks
@@ -32,12 +57,12 @@ if [[ -z $(which python3) ]] ; then
fi
### check python version is correct for ULS
+py_min_version="3.9"
py_version=$(python3 --version | cut -d " " -f 2)
-if [[ $(echo ${py_version} | cut -d "." -f 1 ) -ne 3 ]] || [[ $(echo ${py_version} | cut -d "." -f 2 ) -lt 9 ]]; then
- echo "Wrong Python version - exiting"
+min_version "${py_min_version}" "${py_version}" || {
+ echo "Python version must be >= ${py_min_version}, found ${py_version} - exiting"
exit 1
-fi
-
+}
## pip3
if [[ -z $(which pip3) ]] ; then
@@ -45,6 +70,16 @@ if [[ -z $(which pip3) ]] ; then
exit 1
fi
+### Check PIP version
+pip3_min_version="22.2"
+pip3_version="$(pip3 --version|cut -d' ' -f2)"
+min_version "${pip3_min_version}" "${pip3_version}" || {
+ echo "pip3 version must be >= ${pip3_min_version}, version detected: ${pip3_version}"
+ echo "Consider upgrading your PIP version with command:"
+ echo " $(which pip3) install --upgrade pip"
+ exit 1
+}
+
### Show versions to verify the correct binaries for python and pip are being used
echo "We will use the following python binaries to install ULS:"
echo -ne "python3: \t $(ls $(which python3))\n"
@@ -131,8 +166,6 @@ function py_reqs() {
;;
esac
fi
-
-
}
## Grab EAA-CLI
diff --git a/test/positive_test.bats b/test/positive_test.bats
index a9eecfc..7e0b45c 100644
--- a/test/positive_test.bats
+++ b/test/positive_test.bats
@@ -256,6 +256,14 @@ load 'bats/bats-assert/load.bash'
rm -f /tmp/uls_eaa_access.ckpt
}
+## OUTPUTFORMAT SUBSITUTION
+@test "OUTPUTFORMAT SUBSITUTION" {
+ export BATS_VAR="BATSTEST123"
+ run timeout ${uls_timeout_params} ${uls_bin} --input eaa --feed access --output tcp --host 127.0.0.1 --port 1234 --edgerc $uls_edgerc --section $uls_section --loglevel debug --tcpudpformat '$BATS_VAR - {api_hostname} %s'
+ assert_output --partial "OS_ENV_VARS new TCPUDP output string: BATSTEST123 - manage.akamai-access.com %s"
+ unset BATS_VAR
+}
+
## HELM LINT
@test "LINT the HELM CHART" {
run helm lint docs/examples/kubernetes/helm/akamai-uls --strict