Development (#61)

v1.7.3 preparation
akamai · Apr 2, 2024 · 919a05e · 919a05e
1 parent 52174bd
commit 919a05e
Show file tree

Hide file tree

Showing 13 changed files with 254 additions and 51 deletions.
diff --git a/Dockerfile b/Dockerfile
@@ -10,10 +10,10 @@ ARG             HOMEDIR="/opt/akamai-uls"
 ARG             ULS_DIR="$HOMEDIR/uls"
 ARG             EXT_DIR="$ULS_DIR/ext"
 
-ARG             ETP_CLI_VERSION="0.4.5"
+ARG             ETP_CLI_VERSION="0.4.7"
 ARG             EAA_CLI_VERSION="0.6.3"
 ARG             MFA_CLI_VERSION="0.1.1"
-ARG             GC_CLI_VERSION="v0.0.3(beta)"
+ARG             GC_CLI_VERSION="v0.0.4(beta)"
 ARG             LINODE_CLI_VERSION="dev"
 
 # ENV VARS

diff --git a/README.md b/README.md
@@ -52,6 +52,7 @@ It can be run directly as Python code, as a provided Docker container,  through
       - [INCIDENT](docs/LOG_OVERVIEW.md#incident)
       - AGENT
       - SYSTEM
+      - AUDIT
     - [Linode](https://www.linode.com/) (experimental)
       - [AUDIT](docs/LOG_OVERVIEW.md#audit-logs)
 

diff --git a/bin/modules/UlsInputCli.py b/bin/modules/UlsInputCli.py
@@ -86,6 +86,9 @@ def __init__(self,
         self.disable_stderr = uls_config.input_disable_stderr    #Specify if STDERR should be disabled at all after $disable_stderr_after cycles
         self.disable_stderr_after = uls_config.input_disable_stderr_after   # Disable StdErr Output after # cycles
 
+        # Generic vars
+        self.edgerc_hostname = None
+
     def _feed_selector(self, feed, product_feeds):
         if feed in product_feeds:
             # feed matches the given list
@@ -155,7 +158,7 @@ def proc_create(self):
                 if not self.rawcmd:
                     my_feed = self._feed_selector(self.feed, product_feeds)
                     if my_feed == "CONHEALTH":
-                        UlsTools.uls_check_edgerc(self.credentials_file,
+                        self.edgerc_hostname = UlsTools.uls_check_edgerc(self.credentials_file,
                                                   self.credentials_file_section,
                                                   uls_config.edgerc_openapi)
                         cli_command = [self.bin_python,
@@ -166,7 +169,7 @@ def proc_create(self):
                                        '--perf',
                                        '--tail']
                     elif my_feed == "DEVINV":
-                        UlsTools.uls_check_edgerc(self.credentials_file,
+                        self.edgerc_hostname = UlsTools.uls_check_edgerc(self.credentials_file,
                                                   self.credentials_file_section,
                                                   uls_config.edgerc_openapi)
                         cli_command = [self.bin_python,
@@ -176,7 +179,7 @@ def proc_create(self):
                                        'inventory',
                                        '--tail']
                     elif my_feed == "DIRHEALTH":
-                        UlsTools.uls_check_edgerc(self.credentials_file,
+                        self.edgerc_hostname = UlsTools.uls_check_edgerc(self.credentials_file,
                                                   self.credentials_file_section,
                                                   uls_config.edgerc_openapi)
                         cli_command = [self.bin_python,
@@ -186,7 +189,7 @@ def proc_create(self):
                                        'list',
                                        '--tail']
                     else:
-                        UlsTools.uls_check_edgerc(self.credentials_file,
+                        self.edgerc_hostname = UlsTools.uls_check_edgerc(self.credentials_file,
                                                   self.credentials_file_section,
                                                   uls_config.edgerc_eaa_legacy)
                         cli_command = [self.bin_python, '-u', product_path, 'log', my_feed.lower(), '-f']
@@ -218,7 +221,7 @@ def proc_create(self):
                     aka_log.log.warning(f"{self.name} - Selected LOG Format ({self.cliformat}) "
                                         f"not available for {product_path}, continuing with JSON.")
                 if not self.rawcmd:
-                    UlsTools.uls_check_edgerc(self.credentials_file, self.credentials_file_section,
+                    self.edgerc_hostname = UlsTools.uls_check_edgerc(self.credentials_file, self.credentials_file_section,
                                               uls_config.edgerc_openapi + ["etp_config_id"])
                     my_feed = self._feed_selector(self.feed, product_feeds)
                     cli_command = [self.bin_python, '-u', product_path, 'event', my_feed.lower(), '-f']
@@ -249,7 +252,7 @@ def proc_create(self):
                     aka_log.log.warning(f"{self.name} - Selected LOG Format ({self.cliformat}) "
                                         f"not available for {product_path}, continuing with JSON.")
                 if not self.rawcmd:
-                    UlsTools.uls_check_edgerc(self.credentials_file,
+                    self.edgerc_hostname = UlsTools.uls_check_edgerc(self.credentials_file,
                                               self.credentials_file_section,
                                               uls_config.edgerc_mfa)
                     my_feed = self._feed_selector(self.feed, product_feeds)
@@ -279,7 +282,7 @@ def proc_create(self):
                     aka_log.log.warning(f"{self.name} - Selected LOG Format ({self.cliformat}) "
                                         f"not available for {product_path}, continuing with JSON.")
                 if not self.rawcmd:
-                    UlsTools.uls_check_edgerc(self.credentials_file,
+                    self.edgerc_hostname = UlsTools.uls_check_edgerc(self.credentials_file,
                                               self.credentials_file_section,
                                               uls_config.edgerc_gc)
                     my_feed = self._feed_selector(self.feed, product_feeds)
@@ -309,7 +312,7 @@ def proc_create(self):
                     aka_log.log.warning(f"{self.name} - Selected LOG Format ({self.cliformat}) "
                                         f"not available for {product_path}, continuing with JSON.")
                 if not self.rawcmd:
-                    UlsTools.uls_check_edgerc(self.credentials_file,
+                    self.edgerc_hostname = UlsTools.uls_check_edgerc(self.credentials_file,
                                               self.credentials_file_section,
                                               uls_config.edgerc_linode)
                     my_feed = self._feed_selector(self.feed, product_feeds)
@@ -476,4 +479,7 @@ def ingest_loop(self):
             except Exception:
                 aka_log.log.exception("Error in ingest_loop")
 
-# EOF
+    def get_edgerc_hostname(self):
+        return self.edgerc_hostname
+
+    # EOF
diff --git a/bin/modules/UlsOutput.py b/bin/modules/UlsOutput.py
@@ -97,7 +97,7 @@ def __init__(self, output_type: str,
         if self.output_type in ['TCP', 'UDP'] and host and port:
             self.host = host
             self.port = port
-            if "%s" in tcpudp_out_format:
+            if "%s" in str(tcpudp_out_format):
                 self.tcpudp_out_format = tcpudp_out_format
             else:
                 aka_log.log.critical(
@@ -513,4 +513,41 @@ def tear_down(self):
                 self.httpSession.close()
         self.connected = False
 
+### FEATURE REQ 20240318 - https://github.com/akamai/uls/issues/57
+
+    def ingest_vars_into_output_format(self, placeholder: str = None, replacement: str = None):
+        if not placeholder or not replacement:
+            aka_log.log.debug(f"{self.name} Variable substition triggered but no value given (inline code issue)")
+            #print(f"PLACEHOLDER: {placeholder}")
+            #print(f"REPLACEMENT: {replacement}")
+            #return True
+            sys.exit(1)
+
+        if placeholder in str(self.tcpudp_out_format) and self.tcpudp_out_format:
+            self.tcpudp_out_format = str(self.tcpudp_out_format).replace(placeholder, replacement)
+            aka_log.log.debug(f"{self.name} Replacing {placeholder} in TCPUDP string with: {replacement} ")
+            aka_log.log.debug(f"{self.name} SUBSTITUTION new TCPUDP output string: {self.tcpudp_out_format} ")
+
+        if placeholder in str(self.http_out_format) and self.http_out_format:
+            self.http_out_format = str(self.http_out_format).replace(placeholder, replacement)
+            aka_log.log.debug(f"{self.name} Replacing {placeholder} in HTTP string with: {replacement} ")
+            aka_log.log.debug(f"{self.name} SUBSTITUTION new HTTP output string: {self.http_out_format} ")
+
+
+        return True
+
+    def ingest_os_vars_into_output_format(self):
+        aka_log.log.debug(f"{self.name} Replacing ENV VARS in output FORMAT")
+        if self.tcpudp_out_format:
+            self.tcpudp_out_format = os.path.expandvars(self.tcpudp_out_format)
+            aka_log.log.debug(f"{self.name} OS_ENV_VARS new TCPUDP output string: {self.tcpudp_out_format} ")
+        if self.http_out_format:
+            self.http_out_format = os.path.expandvars(self.http_out_format)
+            aka_log.log.debug(f"{self.name} OS_ENV_VARS new HTTP output string: {self.http_out_format} ")
+
+
+        return True
+
+    ### / FEATURE REQ 20240318 - https://github.com/akamai/uls/issues/57
+
 # EOF
diff --git a/bin/modules/UlsTools.py b/bin/modules/UlsTools.py
@@ -121,9 +121,11 @@ def uls_check_edgerc(configfile, configsection, configvalues):
     Verify the given "edgerc" file to contain all required variables (for the desired stream) within the given section
     see https://github.com/akamai/uls/blob/main/docs/AKAMAI_API_CREDENTIALS.md for more information
     :param configfile: The path to the config file
+
     :param configsection: The section within the config file [default]
     :param configvalues: A list of desiresd config values ["val1", "val2", ...]
     :return:
+    api_hostname
     """
     config = configparser.ConfigParser()
     # Load config file
@@ -141,13 +143,23 @@ def uls_check_edgerc(configfile, configsection, configvalues):
         aka_log.log.debug(f"Section '{configsection}' found.")
 
     # check for specified values
+    my_hostname = None
     for configvalue in configvalues:
         if not configvalue in config[configsection]:
             aka_log.log.critical(f"Required configuration value '{configvalue}' not found in section / file. Please see: {uls_config.edgerc_documentation_url} - Exiting")
             sys.exit(1)
         else:
+
+            ### FEATURE REQ 20240318
+            if 'host' in configvalue:
+                my_hostname = config[configsection][configvalue]
+                #print(f"HOSTNAME{my_hostname}")
+            ### FEATURE REQ 20240318
+
             aka_log.log.debug(f"Required configuration value '{configvalue}' found.")
-    return 0
+        if not my_hostname:
+            my_hostname = "no_hostname_available"
+    return my_hostname
 
 
 def uls_check_args(input, output):
@@ -190,6 +202,7 @@ def check_autoresume(input, feed, checkpoint_dir=uls_config.autoresume_checkpoin
     if (input not in uls_config.autoresume_supported_inputs or
             feed == "CONHEALTH" or
             feed == "DEVINV" or
+            feed == "AGENT" or
             feed == "DIRHEALTH"):
         aka_log.log.critical(f"Input {input} or feed {feed} currently not supported by AUTORESUME - Exiting.")
         sys.exit(1)
@@ -205,7 +218,7 @@ def check_autoresume(input, feed, checkpoint_dir=uls_config.autoresume_checkpoin
             checkpoint = None
         else:
             try:
-                with open (checkpoint_full, "r") as ckpt_f:
+                with open(checkpoint_full, "r") as ckpt_f:
                     data = json.load(ckpt_f)
                     if data['creation_time'] and data['checkpoint']:
                         aka_log.log.debug(f"Autoresume Checkpoint successfully loaded. Checkpoint Time: {data['checkpoint']}, Creation_time: {data['creation_time']}")
@@ -215,6 +228,8 @@ def check_autoresume(input, feed, checkpoint_dir=uls_config.autoresume_checkpoin
                             mytime = data['checkpoint'].split("Z")[0]
                         elif data['input'] == "EAA":
                             mytime = data['checkpoint'].split("+")[0]
+                        elif data['input'] == "GC":
+                            mytime = data['checkpoint'].split(".")[0]
                         else:
                             aka_log.log.critical(
                                 f"Unhandeled input data in checkpointfile  \'{checkpoint_full}\' --> {input} / {feed} - Exiting.")
@@ -263,6 +278,12 @@ def write_autoresume_ckpt(input, feed, autoresume_file, logline, current_count):
         checkpoint_timestamp = json.loads(checkpoint_line)['datetime']
     elif input == "ETP" and feed == "NETCON":
         checkpoint_timestamp = json.loads(checkpoint_line)['connStartTime']
+    elif input == "GC" and feed == "AUDIT":
+        checkpoint_timestamp = json.loads(checkpoint_line)['time']
+    elif input == "GC" and feed == "INCIDENT":
+        checkpoint_timestamp = json.loads(checkpoint_line)['closed_time']
+    elif input == "GC" and feed == "NETLOG":
+        checkpoint_timestamp = json.loads(checkpoint_line)['db_insert_time']
     else:
         aka_log.log.critical(
             f"AUTORESUME - Unhandled Input / Feed detected:  '{input} / {feed}' (this should never happen !!)- Exiting")
@@ -311,4 +332,6 @@ def get_install_id(install_id_file=str(root_path()) + "/var/uls_install_id"):
         aka_log.log.debug(f"Not able to read install file - returning mocked data. Error: {error}")
         data = {'install_id': "ERROR-GETTING-INSTALLATION-ID"}
     #return install_id
-    return data
+    return data
+
+
diff --git a/bin/uls.py b/bin/uls.py
@@ -89,6 +89,7 @@ def main():
     # Check CLI Environment
     UlsTools.uls_check_sys(root_path=root_path, uls_input=uls_args.input)
 
+
     # Create & Start monitoring Instance
     my_monitor = UlsMonitoring.UlsMonitoring(stopEvent=stopEvent,
                                              product=uls_args.input,
@@ -109,6 +110,7 @@ def main():
                                        endtime=uls_args.endtime,
                                        root_path=root_path)
 
+
     # Connect to the selected input UlsOutput
     my_output = UlsOutput.UlsOutput(output_type=uls_args.output,
                                     host=uls_args.host,
@@ -151,11 +153,20 @@ def main():
     else:
         filter_pattern = None
 
+
     # Now let's handle the data and send input to output
 
     # Initiate the Input handler
     my_input.proc_create()
 
+    # Append extra vars to the output
+    #my_output.ingest_vars_into_output_format(api_hostname=my_input.get_edgerc_hostname())
+    my_output.ingest_vars_into_output_format(placeholder='{api_hostname}', replacement=my_input.get_edgerc_hostname())
+    my_output.ingest_vars_into_output_format(placeholder='{uls_input}', replacement=uls_args.input)
+    my_output.ingest_vars_into_output_format(placeholder='{uls_feed}', replacement=uls_args.feed)
+    my_output.ingest_os_vars_into_output_format()
+
+
     # Connect the output handler
     my_output.connect()
 

diff --git a/bin/uls_config/global_config.py b/bin/uls_config/global_config.py
@@ -1,7 +1,7 @@
 #!/usr/bin/env python3
 
 # Common global variables / constants
-__version__ = "1.7.2"
+__version__ = "1.7.3"
 __tool_name_long__ = "Akamai Unified Log Streamer"
 __tool_name_short__ = "ULS"
 
@@ -33,7 +33,7 @@
 
     # Guardicore
 bin_gc_cli = "ext/cli-gc/bin/akamai-gc"                                         # Path to the GC CLI Executable
-gc_cli_feeds = ['NETLOG', 'INCIDENT', 'AGENT', 'SYSTEM']                       # Available GC CLI feeds
+gc_cli_feeds = ['NETLOG', 'INCIDENT', 'AGENT', 'SYSTEM', 'AUDIT']                       # Available GC CLI feeds
 
     # LINODE
 bin_linode_cli = "ext/cli-linode/bin/akamai-linode"         # Path to the LINODE CLI Executable
@@ -107,6 +107,6 @@
 
 # Autoresume Configuration
 autoresume_checkpoint_path = "var/"              # (Default) Path, where the checkpointfiles should be stored to
-autoresume_supported_inputs = ['ETP', 'EAA']     # Internal Var only, to adjust supported inputs
+autoresume_supported_inputs = ['ETP', 'EAA', 'GC']     # Internal Var only, to adjust supported inputs
 autoresume_write_after = 1000                    # Write checkpoint only every ${autoresume_write_every} loglines