v1.6.2 (#34)

approved

.gitignore

@@ -1,3 +1,4 @@
 .DS_Store
 __pycache__
 /ext/*
+/var/uls_install_id

Dockerfile

@@ -11,7 +11,7 @@ ARG             ULS_DIR="$HOMEDIR/uls"
 ARG             EXT_DIR="$ULS_DIR/ext"
 
 ARG             ETP_CLI_VERSION="0.3.9"
-ARG             EAA_CLI_VERSION="0.5.0.2"
+ARG             EAA_CLI_VERSION="0.5.1"
 ARG             MFA_CLI_VERSION="0.0.9"
 ARG             GC_CLI_VERSION="dev"
 ARG             LINODE_CLI_VERSION="dev"
@@ -23,14 +23,15 @@ ENV             HOMEDIR=$HOMEDIR
 
 
 # PREPARE ENVIRONMENT
-# ENV PREP
 RUN	            apt-get update && \
 	            apt-get --no-install-recommends -y install \
 		        ca-certificates \
 		        git \
 		        curl \
                 telnet \
-                gcc libssl-dev libffi-dev  && \
+                gcc \
+                libssl-dev \
+                libffi-dev  && \
 		        rm -rf /var/lib/apt/lists/
 
 # USER & GROUP
@@ -39,12 +40,12 @@ RUN 	        groupadd akamai && \
 
 USER            akamai
 WORKDIR         ${HOMEDIR}
-RUN             mkdir -p ${ULS_DIR}
+RUN             mkdir -p ${ULS_DIR} && \
+                mkdir -p ${ULS_DIR}/var
 
 
 # Install ULS
 COPY            bin/ ${ULS_DIR}/bin
-COPY            var/ ${ULS_DIR}/var
 WORKDIR         ${ULS_DIR}
 RUN             pip3 install -r ${ULS_DIR}/bin/requirements.txt
 
@@ -65,7 +66,7 @@ ENV             MFA-CLI_VERSION=$MFA_CLI_VERSION
 RUN             git clone --depth 1 -b "${MFA_CLI_VERSION}" --single-branch https://github.com/akamai/cli-mfa.git ${EXT_DIR}/cli-mfa && \
                 pip3 install -r ${EXT_DIR}/cli-mfa/requirements.txt
 
-## GC CLI
+## GuardiCore CLI
 ENV             GC_CLI_VERSION=$GC_CLI_VERSION
 RUN             git clone --depth 1 -b "${GC_CLI_VERSION}" --single-branch https://github.com/MikeSchiessl/gc-logs.git ${EXT_DIR}/cli-gc && \
                 pip3 install -r ${EXT_DIR}/cli-gc/bin/requirements.txt
@@ -78,5 +79,4 @@ RUN             git clone --depth 1 -b "${LINODE_CLI_VERSION}" --single-branch h
 # ENTRYPOINTS / CMD
 VOLUME          ["${ULS_DIR}/var"]
 ENTRYPOINT      ["/usr/local/bin/python3","-u","bin/uls.py"]
-#CMD             ["--help"]
 # EOF

bin/config/global_config.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python3
 
 # Common global variables / constants
-__version__ = "1.6.1"
+__version__ = "1.6.2"
 __tool_name_long__ = "Akamai Unified Log Streamer"
 __tool_name_short__ = "ULS"
 

bin/modules/UlsInputCli.py

@@ -132,10 +132,13 @@ def _prep_start_endtime(self, cli_param, time):
 
 
     def _uls_useragent(self, product, feed):
+        install_id = UlsTools.get_install_id()['install_id']
+        if install_id:
+            header_install_id = f"-{install_id}"
         if UlsTools.check_docker():
-            my_useragent = f'ULS/{uls_config.__version__}_{product}-{feed}-docker'
+            my_useragent = f'ULS/{uls_config.__version__}_{product}-{feed}{header_install_id}-DKR'
         else:
-            my_useragent = f'ULS/{uls_config.__version__}_{product}-{feed}'
+            my_useragent = f'ULS/{uls_config.__version__}_{product}-{feed}{header_install_id}'
         return ["--user-agent-prefix", my_useragent]
 
     def proc_create(self):

bin/modules/UlsTools.py

@@ -107,6 +107,7 @@ def _get_cli_version(cli_bin, edgerc_mock_file):
           f"Docker Status\t\t{check_docker()}\n"
           f"RootPath \t\t{root_path}\n"
           f"TimeZone (UTC OFST) \t{check_timezone()} ({-time.timezone / 3600})\n"
+          f"Installation ID \t{get_install_id()['install_id']}"
           )
 
     # Delete the mocked edgerc file
@@ -181,7 +182,7 @@ def root_path():
 
 def check_autoresume(input, feed, checkpoint_dir=uls_config.autoresume_checkpoint_path):
     # Check if we're in a supported stream / feed
-    if input not in uls_config.autoresume_supported_inputs or feed == "CONHEALTH":
+    if input not in uls_config.autoresume_supported_inputs or feed == "CONHEALTH" or feed == "DEVINV" :
         aka_log.log.critical(f"Input {input} or feed {feed} currently not supported by AUTORESUME - Exiting.")
         sys.exit(1)
 
@@ -264,3 +265,38 @@ def write_autoresume_ckpt(input, feed, autoresume_file, logline):
     except Exception as write_error:
         aka_log.log.critical(f"AUTORESUME - Failure writing data to {autoresume_file} - Data: {autoresume_data} - error: {write_error} - Exiting")
         sys.exit(1)
+
+
+def create_install_id(install_id_file=str(root_path()) + "/var/uls_install_id"):
+    if os.path.isfile(install_id_file):
+        aka_log.log.info(f"Install ID file - found")
+        install_id = get_install_id(install_id_file)['install_id']
+    else:
+        aka_log.log.info(f"No install ID file found - creating ID + File")
+        import base64
+        import random
+        import string
+        my_time = int(time.strftime("%Y%m%d"))
+        token = ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(6))
+        raw_id = f"{token}-{my_time}-{uls_config.__version__}"
+        install_id = base64.b64encode(raw_id.encode())
+        install_id_json = {'install_date': str(my_time), 'install_id': str(install_id.decode()), 'install_version': str(uls_config.__version__)}
+        try:
+            with open(install_id_file, "w") as inst_f:
+                json.dump(install_id_json, inst_f)
+        except Exception as error:
+            aka_log.log.warning(f"Not able to write install id file - not saving file !! Error: {error}")
+        aka_log.log.info(f"Created & saved a new installation id: {install_id}")
+    return install_id
+
+def get_install_id(install_id_file=str(root_path()) + "/var/uls_install_id"):
+    try:
+        with open(install_id_file, "r") as inst_f:
+            data = json.load(inst_f)
+            #print(data)
+        install_id = data['install_id']
+    except Exception as error:
+        aka_log.log.debug(f"Not able to read install file - returning mocked data. Error: {error}")
+        data = {'install_id': "ERROR-GETTING-INSTALLATION-ID"}
+    #return install_id
+    return data

bin/uls.py

@@ -66,6 +66,9 @@ def main():
     # Determine root directory
     root_path = str(UlsTools.root_path())
 
+    # Check / Create install id
+    UlsTools.create_install_id()
+
     # OUTPUT Version Information
     if uls_args.version:
         UlsTools.uls_version(root_path=root_path)

docs/CHANGELOG.md

@@ -1,10 +1,26 @@
 # Version History
 
+## v1.6.2
+
+|||
+|---|---|
+|Date|2022-10
+|Kind| Minor release
+|Author|mschiess@akamai.com
+
+- **Minor improvements**
+  - Bumped EAA CLI to version 0.5.1 (additional SIEM fields - EAA release 2022.02)
+  - Amended FAQ to [handle self-signed certificates alongside Guardicore](./FAQ.md#uls-throws-tls-an-error-when-connecting-towards-guardicore-api---input-gc)
+  - Added installation ID ("random string" + "current date YMD" + "first installed version") to support debugging process
+  - fixed a bug in the Dockerfile that left uls/var unusable
+- **Housekeeping**
+  - fixed some bugs in testing (false negative) & speeded up testing process
+
 ## v1.6.1
 
 |||
 |---|---|
-|Date|2022-09
+|Date|2022-10
 |Kind| BUGFIX release
 |Author|mschiess@akamai.com
 

docs/FAQ.md

@@ -10,7 +10,7 @@
 - [Logs are not showing up in my SIEM](#logs-are-not-showing-up-in-siem)
 - [ULS on Windows error: "[WinError 2] The system cannot find the file specified"](#uls-on-windows-error-winerror-2-the-system-cannot-find-the-file-specified)
 - [ULS does not start due to missing field in config](#uls-does-not-start-due-to-missing-field-in-config)
-
+- [ULS throws TLS an error when connecting towards Guardicore API (--input GC)](#uls-throws-tls-an-error-when-connecting-towards-guardicore-api---input-gc)
 
 ----
 ## FAQ
@@ -109,3 +109,28 @@ There seems to be an issue within the module that actually parses the config (co
 Please watch out to specify the section exactly the same way (case sensitivity) as you have specified it in your .edgerc file.  
 We will follow up on this topic within an [GitHub issue](https://github.com/akamai/uls/issues/20)
 
+---
+### ULS throws TLS an error when connecting towards Guardicore API (--input GC)
+When using an internal Guardicore installation that has no valid TLS certificate, ULS might throw the following error:
+```bash
+self._sslobj.do_handshake()
+[SSL: CERTIFICATE_VERIFY_FAILED]
+```
+
+In order to work with self-signed certificates, you have 2 options:
+- Recommended:  
+  You provide the root CA of your self-signed certifcate to the python process 
+  ```bash
+  export REQUESTS_CA_BUNDLE=/path/to/your/certificate.pem
+  ```
+
+
+- Insecure (not recommended):  
+  You skip the TLS certificate (this is very insecure)
+  Set the following ENV variable on your system
+  ```bash
+  export GC_SKIP_TLS_VALIDATION=True
+  ```
+Both options also work for docker / kubernetes installations 
+
+---

docs/examples/kubernetes/helm/akamai-uls/Chart.yaml

@@ -3,5 +3,5 @@ name: akamai-uls
 description: Akamai Universal Log Streamer Helm installation
 
 type: application
-version: 1.6.1
-appVersion: "1.6.1"
+version: 1.6.2
+appVersion: "1.6.2"