Rebase to fix merge conflicts in #21

.pre-commit-config.yaml

@@ -1,6 +1,6 @@
 repos:
 -   repo: https://github.com/phantomcyber/dev-cicd-tools
-    rev: v1.16
+    rev: v1.17
     hooks:
     -   id: org-hook
     -   id: package-app-dependencies

LICENSE

@@ -186,7 +186,7 @@
       same "printed page" as the copyright notice for easier
       identification within third-party archives.
 
-   Copyright (c) 2017-2023 Splunk Inc.
+   Copyright (c) 2017-2024 Splunk Inc.
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.

NOTICE

@@ -1,5 +1,5 @@
 Splunk SOAR G Suite for GMail
-Copyright (c) 2017-2023 Splunk Inc.
+Copyright (c) 2017-2024 Splunk Inc.
 
 Third-party Software Attributions:
 
@@ -32,11 +32,6 @@ Version: 0.0.3
 License: Apache 2.0
 Unknown copyright
 
-Library: httplib2
-Version: 0.19.1
-License: MIT
-Copyright 2006 by Joe Gregorio
-
 Library: oauth2client
 Version: 4.1.2
 License: Apache 2.0
@@ -52,16 +47,6 @@ Version: 0.4.18
 License: MIT
 Copyright 2001-2014 Adam Hupp
 
-Library: requests
-Version: 2.25.0
-License: Apache 2.0
-Kenneth Reitz
-
-Library: rsa
-Version: 4.7.2
-License: Apache 2.0
-Copyright 2011 Sybren A
-
 Library: uritemplate
 Version: 3.0.1
 License: Apache 2.0

README.md

@@ -2,11 +2,11 @@
 # G Suite for GMail
 
 Publisher: Splunk  
-Connector Version: 2.5.0  
+Connector Version: 2.5.2  
 Product Vendor: Google  
 Product Name: GMail  
 Product Version Supported (regex): ".\*"  
-Minimum Product Version: 6.0.0  
+Minimum Product Version: 6.1.1  
 
 Integrates with G Suite for various investigative and containment actions
 

__init__.py

@@ -1,6 +1,6 @@
 # File: __init__.py
 #
-# Copyright (c) 2017-2023 Splunk Inc.
+# Copyright (c) 2017-2024 Splunk Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

gsgmail.json

@@ -10,104 +10,16 @@
     "package_name": "phantom_gsgmail",
     "product_vendor": "Google",
     "product_name": "GMail",
-    "min_phantom_version": "6.0.0",
+    "min_phantom_version": "6.1.1",
     "fips_compliant": true,
     "python_version": "3",
     "latest_tested_versions": [
         "Cloud, May 26, 2023"
     ],
-    "app_version": "2.5.0",
+    "app_version": "2.5.2",
     "product_version_regex": ".*",
-    "license": "Copyright (c) 2017-2023 Splunk Inc.",
-    "utctime_updated": "2022-01-25T00:07:36.000000Z",
-    "pip_dependencies": {
-        "wheel": [
-            {
-                "module": "beautifulsoup4",
-                "input_file": "wheels/py3/beautifulsoup4-4.9.1-py3-none-any.whl"
-            },
-            {
-                "module": "cachetools",
-                "input_file": "wheels/py3/cachetools-4.0.0-py3-none-any.whl"
-            },
-            {
-                "module": "certifi",
-                "input_file": "wheels/py3/certifi-2022.12.7-py3-none-any.whl"
-            },
-            {
-                "module": "chardet",
-                "input_file": "wheels/shared/chardet-3.0.4-py2.py3-none-any.whl"
-            },
-            {
-                "module": "google_api_python_client",
-                "input_file": "wheels/py3/google_api_python_client-1.7.11-py3-none-any.whl"
-            },
-            {
-                "module": "google_auth",
-                "input_file": "wheels/shared/google_auth-1.30.2-py2.py3-none-any.whl"
-            },
-            {
-                "module": "google_auth_httplib2",
-                "input_file": "wheels/shared/google_auth_httplib2-0.0.3-py2.py3-none-any.whl"
-            },
-            {
-                "module": "httplib2",
-                "input_file": "wheels/py3/httplib2-0.19.1-py3-none-any.whl"
-            },
-            {
-                "module": "idna",
-                "input_file": "wheels/shared/idna-2.10-py2.py3-none-any.whl"
-            },
-            {
-                "module": "oauth2client",
-                "input_file": "wheels/shared/oauth2client-4.1.2-py2.py3-none-any.whl"
-            },
-            {
-                "module": "pyasn1",
-                "input_file": "wheels/shared/pyasn1-0.4.8-py2.py3-none-any.whl"
-            },
-            {
-                "module": "pyasn1_modules",
-                "input_file": "wheels/shared/pyasn1_modules-0.2.8-py2.py3-none-any.whl"
-            },
-            {
-                "module": "pyparsing",
-                "input_file": "wheels/shared/pyparsing-2.4.7-py2.py3-none-any.whl"
-            },
-            {
-                "module": "python_magic",
-                "input_file": "wheels/shared/python_magic-0.4.18-py2.py3-none-any.whl"
-            },
-            {
-                "module": "requests",
-                "input_file": "wheels/shared/requests-2.25.0-py2.py3-none-any.whl"
-            },
-            {
-                "module": "rsa",
-                "input_file": "wheels/py3/rsa-4.7.2-py3-none-any.whl"
-            },
-            {
-                "module": "setuptools",
-                "input_file": "wheels/py3/setuptools-59.6.0-py3-none-any.whl"
-            },
-            {
-                "module": "six",
-                "input_file": "wheels/shared/six-1.16.0-py2.py3-none-any.whl"
-            },
-            {
-                "module": "soupsieve",
-                "input_file": "wheels/py3/soupsieve-2.3.2.post1-py3-none-any.whl"
-            },
-            {
-                "module": "uritemplate",
-                "input_file": "wheels/shared/uritemplate-3.0.1-py2.py3-none-any.whl"
-            },
-            {
-                "module": "urllib3",
-                "input_file": "wheels/shared/urllib3-1.26.13-py2.py3-none-any.whl"
-            }
-        ]
-    },
+    "license": "Copyright (c) 2017-2024 Splunk Inc.",
+    "utctime_updated": "2024-03-18T08:57:36.000000Z",
     "configuration": {
         "login_email": {
             "required": true,
@@ -1349,7 +1261,7 @@
             },
             {
                 "module": "pyparsing",
-                "input_file": "wheels/py3/pyparsing-3.0.9-py3-none-any.whl"
+                "input_file": "wheels/py3/pyparsing-3.1.2-py3-none-any.whl"
             },
             {
                 "module": "python_magic",

gsgmail_connector.py

@@ -1,6 +1,6 @@
 # File: gsgmail_connector.py
 #
-# Copyright (c) 2017-2023 Splunk Inc.
+# Copyright (c) 2017-2024 Splunk Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -259,7 +259,7 @@ def _parse_multipart_msg(self, action_result, msg, email_details, extract_attach
 
         email_details['email_headers'] = []
         for part in msg.walk():
-            type = part.get_content_type()
+            part_type = part.get_content_type()
             headers = self._get_email_headers_from_part(part)
             # split out important headers (for output table rendering)
             if headers.get('to'):
@@ -274,13 +274,13 @@ def _parse_multipart_msg(self, action_result, msg, email_details, extract_attach
             disp = str(part.get('Content-Disposition'))
             file_name = part.get_filename()
             # look for plain text parts, but skip attachments
-            if type == 'text/plain' and 'attachment' not in disp:
+            if part_type == 'text/plain' and 'attachment' not in disp:
                 charset = part.get_content_charset() or 'utf8'
                 # decode the base64 unicode bytestring into plain text
                 plain_body = part.get_payload(decode=True).decode(encoding=charset, errors="ignore")
                 # Add to list of plan text bodies
                 plain_bodies.append(plain_body)
-            if type == 'text/html' and 'attachment' not in disp:
+            if part_type == 'text/html' and 'attachment' not in disp:
                 charset = part.get_content_charset() or 'utf8'
                 # decode the base64 unicode bytestring into plain text
                 html_body = part.get_payload(decode=True).decode(encoding=charset, errors="ignore")
@@ -289,11 +289,15 @@ def _parse_multipart_msg(self, action_result, msg, email_details, extract_attach
             elif file_name and extract_attachments:
                 attach_resp = None
                 try:
+                    if part_type.startswith("message/"):
+                        content = part.get_payload(0).as_string()
+                    else:
+                        content = part.get_payload(decode=True)
                     # Create vault item with attachment payload
-                    attach_resp = Vault.create_attachment(part.get_payload(decode=True), container_id=container_id, file_name=file_name)
+                    attach_resp = Vault.create_attachment(content, container_id=container_id, file_name=file_name)
                 except Exception as e:
                     message = self._get_error_message_from_exception(e)
-                    self.error_print('Unable to add attachment: {} Error: {}').format(str(file_name), message)
+                    return action_result.set_status(phantom.APP_ERROR, f"Unable to add attachment: {file_name} Error: {message}")
                 if attach_resp.get('succeeded'):
                     # Create vault artifact
                     artifact = {
@@ -460,7 +464,7 @@ def _handle_get_email(self, param):
                     email_details_resp['parsed_plain_body'] = msg.get_payload(decode=True).decode(encoding=charset, errors="ignore")
                 except Exception as e:
                     message = self._get_error_message_from_exception(e)
-                    self.error_print("Unable to add email body: {}").format(message)
+                    self.error_print(f"Unable to add email body: {message}")
 
             action_result.add_data(email_details_resp)
 

gsgmail_consts.py

@@ -1,6 +1,6 @@
 # File: gsgmail_consts.py
 #
-# Copyright (c) 2017-2023 Splunk Inc.
+# Copyright (c) 2017-2024 Splunk Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

gsgmail_get_email.html

@@ -10,7 +10,7 @@
 {% block widget_content %} <!-- Main Start Block -->
 
 <!-- File: gsgmail_get_email.html
-    Copyright (c) 2017-2023 Splunk Inc.
+    Copyright (c) 2017-2024 Splunk Inc.
     Licensed under the Apache License, Version 2.0 (the "License");
     you may not use this file except in compliance with the License.
     You may obtain a copy of the License at

gsgmail_list_users.html

@@ -10,7 +10,7 @@
 {% block widget_content %} <!-- Main Start Block -->
 
 <!-- File: gsgmail_list_users.html
-    Copyright (c) 2017-2023 Splunk Inc.
+    Copyright (c) 2017-2024 Splunk Inc.
     Licensed under the Apache License, Version 2.0 (the "License");
     you may not use this file except in compliance with the License.
     You may obtain a copy of the License at

gsgmail_process_email.py

@@ -1,6 +1,6 @@
 # File: gsgmail_process_email.py
 #
-# Copyright (c) 2017-2023 Splunk Inc.
+# Copyright (c) 2017-2024 Splunk Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

gsgmail_view.py

@@ -1,6 +1,6 @@
 # File: gsgmail_view.py
 #
-# Copyright (c) 2017-2023 Splunk Inc.
+# Copyright (c) 2017-2024 Splunk Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

manual_readme_content.md

@@ -0,0 +1,73 @@
+[comment]: # " File: README.md"
+[comment]: # "  Copyright (c) 2017-2023 Splunk Inc."
+[comment]: # ""
+[comment]: # "  Licensed under Apache 2.0 (https://www.apache.org/licenses/LICENSE-2.0.txt)"
+[comment]: # ""
+### Service Account
+
+This app requires a pre-configured service account to operate. Please follow the procedure outlined
+at [this link](https://support.google.com/a/answer/7378726?hl=en) to create a service account.  
+The following APIs will need to be enabled:
+
+-   AdminSDK
+-   GMail API
+
+At the end of the creation process, the admin console should ask you to save the config as a JSON
+file. Copy the contents of the JSON file in the clipboard and paste it as the value of the
+**key_json** asset configuration parameter.
+
+### Scopes
+
+Once the service account has been created and APIs enabled, the next step is to configure scopes on
+these APIs to allow the App to access them. Every action requires different scopes to operate, these
+are listed in the action documentation.  
+To enable scopes please complete the following steps:
+
+-   Go to your G Suite domain's [Admin console.](http://admin.google.com/)
+-   Select **Security** from the list of controls. If you don't see **Security** listed, select
+    **Show More** , then select **Security** from the list of controls. If you can't see the
+    controls, make sure you're signed in as an administrator for the domain.
+-   Select **API controls** in the **Access and data control** section.
+-   Select **MANAGE DOMAIN WIDE DELEGATIONS** in the **Domain wide delegation** section.
+-   Select **Add new** in the API clients section
+-   In the **Client ID** field enter the service account's **Client ID** . You can find your service
+    account's client ID in the [Service accounts credentials
+    page](https://console.developers.google.com/apis/credentials) or the service account JSON file
+    (key named **client_id** ).
+-   In the **One or More API Scopes** field enter the list of scopes that you wish to grant access
+    to the App. For example, to enable all the scopes required by this app enter:
+    https://mail.google.com/, https://www.googleapis.com/auth/admin.directory.user.readonly,
+    https://www.googleapis.com/auth/gmail.readonly
+-   Click **Authorize** .
+
+### On-Poll
+
+-   API provides created time of the email and gmail searches based on the received time of the
+    email.
+
+-   Use the large container numbers in asset to avoid any kind of data loss for emails which
+    received at the same time.
+
+
+
+
+    **Configuration:**  
+
+<!-- -->
+
+-   label - To fetch the emails from the given folder name (default - all folders).  
+    **Note:-** Reply email in the email thread would not be ingested if you provide a specific label
+    in the configuration (eg. Inbox). It will ingest the reply email only if you leave the label
+    configuration parameter empty.  
+-   ingest_manner - To select the oldest first or newest first preference for ingestion (default -
+    oldest first).
+-   first_run_max_emails - Maximum containers to poll for the first scheduled polling (default -
+    1000).
+-   max_containers - Maximum containers to poll after the first scheduled poll completes (default -
+    100).
+-   extract_attachments - Extract all the attachments included in emails.
+-   download_eml_attachments - Downloads the EML file attached with the mail.
+-   extract_urls - Extracts the URLs present in the emails.
+-   extract_ips - Extracts the IP addresses present in the emails.
+-   extract_domains - Extract the domain names present in the emails.
+-   extract_hashes - Extract the hashes present in the emails (MD5).