From 2372f93021f565e9f1215ef160b8121c41235de7 Mon Sep 17 00:00:00 2001 From: Tapish Jain Date: Fri, 26 Jul 2024 17:08:46 -0700 Subject: [PATCH 01/16] PAPP-34389: All features done except templates --- gsgmail.json | 246 +++++++++++++++++++++++++++++++++++++++++++ gsgmail_connector.py | 173 ++++++++++++++++++++++++++++++ gsgmail_consts.py | 4 + 3 files changed, 423 insertions(+) diff --git a/gsgmail.json b/gsgmail.json index b088cf6..eb4bcfd 100644 --- a/gsgmail.json +++ b/gsgmail.json @@ -1229,6 +1229,252 @@ "title": "Get Email", "view": "gsgmail_view.get_email" } + }, + { + "action": "send email", + "description": "Send emails", + "type": "contain", + "verbose": "Action uses the GMail API. Requires authorization with the following scope: https://www.googleapis.com and https://www.googleapis.com/auth/gmail.settings.sharing.", + "read_only": false, + "identifier": "send_email", + "parameters": { + "from": { + "description": "From field", + "data_type": "string", + "order": 0, + "primary": true, + "contains": [ + "email" + ] + }, + "to": { + "description": "List of recipients email addresses", + "data_type": "string", + "order": 1, + "required": true, + "primary": true, + "contains": [ + "email" + ], + "allow_list": true + }, + "subject": { + "description": "Message Subject", + "data_type": "string", + "order": 2, + "required": true + }, + "cc": { + "description": "List of recipients email addresses to include on cc line", + "data_type": "string", + "order": 3, + "primary": true, + "contains": [ + "email" + ], + "allow_list": true + }, + "bcc": { + "description": "List of recipients email addresses to include on bcc line", + "data_type": "string", + "order": 4, + "contains": [ + "email" + ], + "allow_list": true, + "primary": true + }, + "reply_to": { + "description": "Address that should recieve replies to the sent email", + "data_type": "string", + "order": 5, + "contains": [ + "email" + ], + "allow_list": true, + "primary": true + }, + "headers": { + "description": "Serialized json dictionary. Additional email headers to be added to the message", + "data_type": "string", + "order": 6 + }, + "body": { + "description": "Html rendering of message", + "data_type": "string", + "order": 7, + "required": true + }, + "attachments": { + "description": "List of vault ids of files to attach to the email. Vault id is used as content id", + "data_type": "string", + "order": 8, + "allow_list": true, + "contains": [ + "sha1", + "vault id" + ], + "primary": true + }, + "alias_email": { + "description": "Custom from send-as alias email", + "data_type": "string", + "order": 9, + "contains": [ + "email" + ] + }, + "alias_name": { + "description": "Custom from send-as alias name", + "data_type": "string", + "order": 10 + } + }, + "output": [ + { + "data_path": "action_result.status", + "data_type": "string", + "example_values": [ + "success", + "failed" + ] + }, + { + "data_path": "action_result.parameter.alias_email", + "data_type": "string", + "contains": [ + "email" + ], + "example_values": [ + "test@testdomain.abc.com" + ] + }, + { + "data_path": "action_result.parameter.alias_name", + "data_type": "string" + }, + { + "data_path": "action_result.parameter.attachments", + "data_type": "string", + "contains": [ + "sha1", + "vault id" + ], + "example_values": [ + "da39a3ee5e6b4b0d3255bfef95601890afd80709" + ] + }, + { + "data_path": "action_result.parameter.bcc", + "data_type": "string", + "contains": [ + "email" + ], + "example_values": [ + "test@testdomain.abc.com" + ] + }, + { + "data_path": "action_result.parameter.body", + "data_type": "string", + "example_values": [ + "

Have a good time with these.

" + ] + }, + { + "data_path": "action_result.parameter.cc", + "data_type": "string", + "contains": [ + "email" + ], + "example_values": [ + "test@testdomain.abc.com" + ] + }, + { + "data_path": "action_result.parameter.from", + "data_type": "string", + "contains": [ + "email" + ], + "example_values": [ + "test@testdomain.abc.com" + ] + }, + { + "data_path": "action_result.parameter.headers", + "data_type": "string", + "example_values": [ + "{\"x-custom-header\":\"Custom value\"}" + ] + }, + { + "data_path": "action_result.parameter.subject", + "data_type": "string", + "example_values": [ + "Example subject" + ] + }, + { + "data_path": "action_result.parameter.to", + "data_type": "string", + "contains": [ + "email" + ], + "example_values": [ + "test@testdomain.abc.com" + ] + }, + { + "data_path": "action_result.data.*.draft_id", + "data_type": "string", + "example_values": [ + "rfc822t1500000000t3a1d2e0fghijklm" + ] + }, + { + "data_path": "action_result.data.*.threadId", + "data_type": "string", + "example_values": [ + "16d1234567890abcdef" + ] + }, + { + "data_path": "action_result.data.*.labelIds", + "data_type": "string", + "example_values": [ + "INBOX" + ] + }, + { + "data_path": "action_result.message", + "data_type": "string", + "example_values": [ + "All the provided emails were already deleted" + ] + }, + { + "data_path": "summary.total_objects", + "data_type": "numeric", + "example_values": [ + 1 + ] + }, + { + "data_path": "summary.total_objects_successful", + "data_type": "numeric", + "example_values": [ + 1 + ] + } + ], + "render": { + "width": 12, + "title": "Delete Email", + "type": "table", + "height": 5 + }, + "versions": "EQ(*)" } ], "pip39_dependencies": { diff --git a/gsgmail_connector.py b/gsgmail_connector.py index 7d91490..39292bf 100644 --- a/gsgmail_connector.py +++ b/gsgmail_connector.py @@ -24,8 +24,15 @@ from copy import deepcopy from datetime import datetime +from email.mime.text import MIMEText +from email.mime import application, multipart, text, base, image, audio +from email import encoders +from googleapiclient.http import MediaIoBaseUpload +from io import BytesIO + import phantom.app as phantom import phantom.utils as ph_utils +import phantom.vault as phantom_vault import requests from google.oauth2 import service_account from googleapiclient import errors @@ -854,6 +861,170 @@ def _handle_on_poll(self, param): return phantom.APP_SUCCESS + def _create_message(self, sender, to, cc, bcc, subject, message_text, reply_to=None, additional_headers={}, vault_ids=[]): + message = multipart.MIMEMultipart('alternative') + message['to'] = to + message['from'] = sender + message['subject'] = subject + if cc: + message['cc'] = cc + if bcc: + message['bcc'] = bcc + if reply_to: + message['Reply-To'] = reply_to + + for key, value in additional_headers.items(): + message[key] = value + + part1 = MIMEText(message_text, 'plain') + message.attach(part1) + + # Attach HTML part with plain text content + part2 = MIMEText(message_text, 'html') + message.attach(part2) + + current_size = 0 + mime_consumer = {'text': text.MIMEText, + 'image': image.MIMEImage, + 'audio': audio.MIMEAudio } + + for vault_id in vault_ids: + vault_info = self._get_vault_info(vault_id) + if not vault_info: + self.debug_print("Failed to find vault entry {}".format(vault_id)) + continue + + current_size += vault_info["size"] + if current_size > GSGMAIL_ATTACHMENTS_CUTOFF_SIZE: + self.debug_print("Total attachment size reached max capacity. No longer adding attachments after vault id {0}".format(vault_id)) + break + + content_type = vault_info['mime_type'] + main_type, sub_type = content_type.split('/', 1) + + consumer = None + if main_type in mime_consumer: + consumer = main_type[mime_consumer] + elif main_type == "application" and sub_type == "pdf": + consumer = application.MIMEApplication + + self.debug_print("Content type is {0}".format(content_type)) + attachment_part = None + if not consumer: + attachment_part = base.MIMEBase(main_type, sub_type) + with open(vault_info['path'], mode='rb') as file: + file_content = file.read() + attachment_part.set_payload(file_content) + else: + with open(vault_info['path'], mode='rb') as file: + attachment_part = consumer(file.read(), _subtype=sub_type) + + encoders.encode_base64(attachment_part) + + attachment_part.add_header( + 'Content-Disposition', + 'attachment; filename={0}'.format(vault_info['name']) + ) + attachment_part.add_header( + 'Content-Length', + str(vault_info['size']) # File size in bytes + ) + attachment_part.add_header( + 'Content-ID', + vault_info['vault_id'] + ) + message.attach(attachment_part) + + return message + + def _send_email(self, service, user_id, media, action_result): + try: + sent_message = service.users().messages().send(userId=user_id, body={}, media_body=media).execute() + return phantom.APP_SUCCESS, sent_message + except Exception as error: + self.debug_print("Error occured when sending draft: {0}".format(error)) + return action_result.set_status(phantom.APP_ERROR, "Message not sent because of {0}".format(error)), None + + def _get_vault_info(self, vault_id): + _, _, vault_infos = phantom_vault.vault_info(container_id=self.get_container_id(), vault_id=vault_id) + if not vault_infos: + _, _, vault_infos = phantom_vault.vault_info(vault_id=vault_id) + return vault_infos[0] if vault_infos else None + + def _create_send_as_alias(self, service, user_id, alias_email, display_name=None): + send_as = { + "sendAsEmail": alias_email, + "treatAsAlias": True, + "isPrimary": False + } + + if display_name: + send_as["displayName"] = display_name + + try: + result = service.users().settings().sendAs().create(userId=user_id, body=send_as).execute() + return phantom.APP_SUCCESS, result + except errors.HttpError as error: + return phantom.APP_ERROR, error + + def _handle_send_email(self, param): + self.save_progress("In action handler for: {0}".format(self.get_action_identifier())) + + # Add an action result object to self (BaseConnector) to represent the action for this param + action_result = self.add_action_result(ActionResult(dict(param))) + + # Create the credentials with the required scope + scopes = [GSGMAIL_DELETE_EMAIL] + + # Create a service here + self.save_progress("Creating GMail service object") + + ret_val, service = self._create_service(action_result, scopes, "gmail", "v1", self._login_email) + + if phantom.is_fail(ret_val): + return action_result.get_status() + + from_email = param["from"] if param.get("from", "") else self._login_email + + try: + headers = json.loads(param.get("headers", "{}")) + except json.JSONDecodeError as e: + return action_result.set_status(phantom.APP_ERROR, e), None + + vault_ids = [vault_id for x in param.get('attachments', '').split(',') if (vault_id := x.strip())] + + if param.get("alias_email"): + alias_email = param.get("alias_email") + alias_name = param.get("alias_name", "") + SETTINGS_SCOPE = [GSMAIL_SETTINGS_CHANGE] + ret_val, settings_service = self._create_service(action_result, SETTINGS_SCOPE, "gmail", "v1", self._login_email) + ret_val, res = self._create_send_as_alias(settings_service, "me", alias_email, alias_name) + if ret_val == phantom.APP_SUCCESS: + self.debug_print("Successfully created alias {0}".format(alias_email)) + from_email = alias_email + elif res.resp.status == 409 and 'alreadyExists' in res._get_reason(): + self.debug_print("Alias {0} already exists. Using to send emails".format(alias_email)) + from_email = alias_email + else: + self.debug_print("Could not create alias {0} because of {1}".format(alias_email, res)) + + message = self._create_message( + from_email, param["to"], + param.get("cc", ""), + param.get("bcc", ""), + param.get("subject", ""), + param["body"], + param.get("reply_to"), + headers, + vault_ids + ) + + media = MediaIoBaseUpload(BytesIO(message.as_bytes()), mimetype='message/rfc822', resumable=True) + ret_val, sent_message = self._send_email(service, "me", media, action_result) + if phantom.is_fail(ret_val): + return action_result.get_status() + return action_result.set_status(phantom.APP_SUCCESS, "Email sent with id {0}".format(sent_message["id"])) + def _process_email_ids(self, action_result, config, service, email_ids): for i, emid in enumerate(email_ids): self.send_progress("Parsing email id: {0}".format(emid)) @@ -920,6 +1091,8 @@ def handle_action(self, param): ret_val = self._handle_on_poll(param) elif action_id == 'test_connectivity': ret_val = self._handle_test_connectivity(param) + elif action_id == 'send_email': + ret_val = self._handle_send_email(param) return ret_val diff --git a/gsgmail_consts.py b/gsgmail_consts.py index 6b316db..fb6cc8a 100644 --- a/gsgmail_consts.py +++ b/gsgmail_consts.py @@ -27,6 +27,10 @@ GSGMAIL_AUTH_GMAIL_READ = 'https://www.googleapis.com/auth/gmail.readonly' GSGMAIL_AUTH_GMAIL_ADMIN_DIR = 'https://www.googleapis.com/auth/admin.directory.user.readonly' GSGMAIL_DELETE_EMAIL = 'https://mail.google.com/' +GSGMAIL_SEND_EMAIL = 'https://www.googleapis.com/auth/gmail.send' +GSGMAIL_CREATE_DRAFT = 'https://www.googleapis.com/auth/gmail.compose' +GSMAIL_SETTINGS_CHANGE = 'https://www.googleapis.com/auth/gmail.settings.sharing' +GSGMAIL_ATTACHMENTS_CUTOFF_SIZE = 26214400 # 25mb GSMAIL_DEFAULT_FIRST_RUN_MAX_EMAIL = 1000 GSMAIL_DEFAULT_MAX_CONTAINER = 100 From 4babce164e1bd8945508b37465bcc776f835b6a8 Mon Sep 17 00:00:00 2001 From: splunk-soar-connectors-admin Date: Sat, 27 Jul 2024 00:09:29 +0000 Subject: [PATCH 02/16] Update README.md --- README.md | 45 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/README.md b/README.md index c06114d..e645222 100644 --- a/README.md +++ b/README.md @@ -110,6 +110,7 @@ VARIABLE | REQUIRED | TYPE | DESCRIPTION [delete email](#action-delete-email) - Delete emails [on poll](#action-on-poll) - Callback action for the on-poll ingest functionality [get email](#action-get-email) - Retrieve email details via internet message id +[send email](#action-send-email) - Send emails ## action: 'test connectivity' Validate the asset configuration for connectivity @@ -346,4 +347,48 @@ action_result.data.\*.to | string | `email` | admin@testcorp.biz action_result.summary.total_messages_returned | numeric | | 1 action_result.message | string | | Total messages returned: 1 summary.total_objects | numeric | | 1 +summary.total_objects_successful | numeric | | 1 + +## action: 'send email' +Send emails + +Type: **contain** +Read only: **False** + +Action uses the GMail API. Requires authorization with the following scope: https://www.googleapis.com and https://www.googleapis.com/auth/gmail.settings.sharing. + +#### Action Parameters +PARAMETER | REQUIRED | DESCRIPTION | TYPE | CONTAINS +--------- | -------- | ----------- | ---- | -------- +**from** | optional | From field | string | `email` +**to** | required | List of recipients email addresses | string | `email` +**subject** | required | Message Subject | string | +**cc** | optional | List of recipients email addresses to include on cc line | string | `email` +**bcc** | optional | List of recipients email addresses to include on bcc line | string | `email` +**reply_to** | optional | Address that should recieve replies to the sent email | string | `email` +**headers** | optional | Serialized json dictionary. Additional email headers to be added to the message | string | +**body** | required | Html rendering of message | string | +**attachments** | optional | List of vault ids of files to attach to the email. Vault id is used as content id | string | `sha1` `vault id` +**alias_email** | optional | Custom from send-as alias email | string | `email` +**alias_name** | optional | Custom from send-as alias name | string | + +#### Action Output +DATA PATH | TYPE | CONTAINS | EXAMPLE VALUES +--------- | ---- | -------- | -------------- +action_result.status | string | | success failed +action_result.parameter.alias_email | string | `email` | test@testdomain.abc.com +action_result.parameter.alias_name | string | | +action_result.parameter.attachments | string | `sha1` `vault id` | da39a3ee5e6b4b0d3255bfef95601890afd80709 +action_result.parameter.bcc | string | `email` | test@testdomain.abc.com +action_result.parameter.body | string | |

Have a good time with these.

+action_result.parameter.cc | string | `email` | test@testdomain.abc.com +action_result.parameter.from | string | `email` | test@testdomain.abc.com +action_result.parameter.headers | string | | {"x-custom-header":"Custom value"} +action_result.parameter.subject | string | | Example subject +action_result.parameter.to | string | `email` | test@testdomain.abc.com +action_result.data.\*.draft_id | string | | rfc822t1500000000t3a1d2e0fghijklm +action_result.data.\*.threadId | string | | 16d1234567890abcdef +action_result.data.\*.labelIds | string | | INBOX +action_result.message | string | | All the provided emails were already deleted +summary.total_objects | numeric | | 1 summary.total_objects_successful | numeric | | 1 \ No newline at end of file From 17aabcac9017cbcc1ce5447b7ef49378d3cf9a54 Mon Sep 17 00:00:00 2001 From: Tapish Jain Date: Mon, 29 Jul 2024 12:01:38 -0700 Subject: [PATCH 03/16] PAPP-34389: Adding reply to output parameter to json --- gsgmail.json | 7 +++++++ gsgmail_connector.py | 11 ++++++----- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/gsgmail.json b/gsgmail.json index eb4bcfd..15fcef0 100644 --- a/gsgmail.json +++ b/gsgmail.json @@ -1374,6 +1374,13 @@ "test@testdomain.abc.com" ] }, + { + "data_path": "action_result.parameter.reply_to", + "data_type": "string", + "contains": [ + "email" + ] + }, { "data_path": "action_result.parameter.body", "data_type": "string", diff --git a/gsgmail_connector.py b/gsgmail_connector.py index 39292bf..045d1a3 100644 --- a/gsgmail_connector.py +++ b/gsgmail_connector.py @@ -24,10 +24,9 @@ from copy import deepcopy from datetime import datetime -from email.mime.text import MIMEText -from email.mime import application, multipart, text, base, image, audio from email import encoders -from googleapiclient.http import MediaIoBaseUpload +from email.mime import application, multipart, text, base, image, audio +from email.mime.text import MIMEText from io import BytesIO import phantom.app as phantom @@ -36,6 +35,7 @@ import requests from google.oauth2 import service_account from googleapiclient import errors +from googleapiclient.http import MediaIoBaseUpload from phantom.action_result import ActionResult from phantom.base_connector import BaseConnector from phantom.vault import Vault @@ -1009,11 +1009,12 @@ def _handle_send_email(self, param): self.debug_print("Could not create alias {0} because of {1}".format(alias_email, res)) message = self._create_message( - from_email, param["to"], + from_email, + param.get("to", ""), param.get("cc", ""), param.get("bcc", ""), param.get("subject", ""), - param["body"], + param.get("body", ""), param.get("reply_to"), headers, vault_ids From 97d391ba209f5f23ffcd51323a99752cbb6fb105 Mon Sep 17 00:00:00 2001 From: splunk-soar-connectors-admin Date: Mon, 29 Jul 2024 19:02:41 +0000 Subject: [PATCH 04/16] Update README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index e645222..a21702b 100644 --- a/README.md +++ b/README.md @@ -380,6 +380,7 @@ action_result.parameter.alias_email | string | `email` | test@testdomain.abc action_result.parameter.alias_name | string | | action_result.parameter.attachments | string | `sha1` `vault id` | da39a3ee5e6b4b0d3255bfef95601890afd80709 action_result.parameter.bcc | string | `email` | test@testdomain.abc.com +action_result.parameter.reply_to | string | `email` | action_result.parameter.body | string | |

Have a good time with these.

action_result.parameter.cc | string | `email` | test@testdomain.abc.com action_result.parameter.from | string | `email` | test@testdomain.abc.com From 25718d3ed916c0388596943eb3486074d53105a2 Mon Sep 17 00:00:00 2001 From: Tapish Jain Date: Thu, 1 Aug 2024 12:13:20 -0700 Subject: [PATCH 05/16] PAPP-34389: Getting rid of unused constants --- gsgmail_consts.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/gsgmail_consts.py b/gsgmail_consts.py index fb6cc8a..841a7f0 100644 --- a/gsgmail_consts.py +++ b/gsgmail_consts.py @@ -27,8 +27,6 @@ GSGMAIL_AUTH_GMAIL_READ = 'https://www.googleapis.com/auth/gmail.readonly' GSGMAIL_AUTH_GMAIL_ADMIN_DIR = 'https://www.googleapis.com/auth/admin.directory.user.readonly' GSGMAIL_DELETE_EMAIL = 'https://mail.google.com/' -GSGMAIL_SEND_EMAIL = 'https://www.googleapis.com/auth/gmail.send' -GSGMAIL_CREATE_DRAFT = 'https://www.googleapis.com/auth/gmail.compose' GSMAIL_SETTINGS_CHANGE = 'https://www.googleapis.com/auth/gmail.settings.sharing' GSGMAIL_ATTACHMENTS_CUTOFF_SIZE = 26214400 # 25mb From f81807141cc49765ca458e80f283b50fdb8dab17 Mon Sep 17 00:00:00 2001 From: Tapish Jain Date: Thu, 1 Aug 2024 16:15:59 -0700 Subject: [PATCH 06/16] PAPP-34389: fixing small bug with from email setting --- gsgmail_connector.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/gsgmail_connector.py b/gsgmail_connector.py index 045d1a3..6d3c18c 100644 --- a/gsgmail_connector.py +++ b/gsgmail_connector.py @@ -23,9 +23,8 @@ import sys from copy import deepcopy from datetime import datetime - from email import encoders -from email.mime import application, multipart, text, base, image, audio +from email.mime import application, audio, base, image, multipart, text from email.mime.text import MIMEText from io import BytesIO @@ -979,13 +978,13 @@ def _handle_send_email(self, param): # Create a service here self.save_progress("Creating GMail service object") - ret_val, service = self._create_service(action_result, scopes, "gmail", "v1", self._login_email) + from_email = param.get("from") if param.get("from", "") else self._login_email + + ret_val, service = self._create_service(action_result, scopes, "gmail", "v1", from_email) if phantom.is_fail(ret_val): return action_result.get_status() - from_email = param["from"] if param.get("from", "") else self._login_email - try: headers = json.loads(param.get("headers", "{}")) except json.JSONDecodeError as e: From dbfd4442e3b02c64907e4b3fdd9e3e9b0ee57329 Mon Sep 17 00:00:00 2001 From: Tapish Jain Date: Thu, 25 Jul 2024 10:43:02 -0700 Subject: [PATCH 07/16] PAPP-34391: ASCII for polling --- gsgmail.json | 40 ++++++++++++++++++---- gsgmail_connector.py | 16 +++++---- gsgmail_process_email.py | 72 +++++++++++++++++++++++++++++----------- 3 files changed, 97 insertions(+), 31 deletions(-) diff --git a/gsgmail.json b/gsgmail.json index 15fcef0..e5aa5e2 100644 --- a/gsgmail.json +++ b/gsgmail.json @@ -61,41 +61,59 @@ "default": 100, "order": 5 }, + "data_type": { + "description": "Ingestion data type when polling", + "data_type": "string", + "value_list": [ + "utf-8", + "ascii" + ], + "default": "utc-8", + "order": 6 + }, + "forwarding_address": { + "description": "Address to forward all emails to", + "data_type": "string", + "contains": [ + "email" + ], + "order": 7 + }, "extract_attachments": { "description": "Extract Attachments", "data_type": "boolean", "default": false, - "order": 6 + "order": 8 }, "extract_urls": { "description": "Extract URLs", "data_type": "boolean", "default": false, - "order": 7 + "order": 9 }, "extract_ips": { "description": "Extract IPs", "data_type": "boolean", "default": false, - "order": 8 + "order": 10 }, "extract_domains": { "description": "Extract Domain Names", "data_type": "boolean", "default": false, - "order": 9 + "order": 11 }, "extract_hashes": { "description": "Extract Hashes", "data_type": "boolean", "default": false, - "order": 10 + "order": 12 }, "download_eml_attachments": { "description": "Download EML attachments", "data_type": "boolean", "default": false, - "order": 11 + "order": 13 } }, "actions": [ @@ -828,6 +846,16 @@ "required": true, "value": 1000, "order": 4 + }, + "data_type": { + "description": "Encode ingested emails as ASCII or UTF-8", + "data_type": "string", + "value_list": [ + "utf-8", + "ascii" + ], + "default": "utf-8", + "order": 5 } }, "output": [], diff --git a/gsgmail_connector.py b/gsgmail_connector.py index 6d3c18c..63dad9d 100644 --- a/gsgmail_connector.py +++ b/gsgmail_connector.py @@ -130,9 +130,7 @@ def initialize(self): _, _, self._domain = self._login_email.partition('@') except Exception: return self.set_status(phantom.APP_ERROR, "Unable to extract domain from login_email") - - return phantom.APP_SUCCESS - + def _validate_integer(self, action_result, parameter, key, allow_zero=False): """ Validate an integer. @@ -833,6 +831,8 @@ def _handle_on_poll(self, param): else: max_emails = max_containers + ingestion_data_type = config.get("data_type", "utf-8") + run_limit = deepcopy(max_emails) action_result = self.add_action_result(ActionResult(dict(param))) email_id = param.get(phantom.APP_JSON_CONTAINER_ID, False) @@ -850,7 +850,10 @@ def _handle_on_poll(self, param): if not self.is_poll_now(): self._update_state() - self._process_email_ids(action_result, config, service, email_ids) + self.debug_print("On poll data type {0}".format(ingestion_data_type)) + if not ingestion_data_type: + ingestion_data_type = "utf-8" + self._process_email_ids(action_result, config, service, email_ids, ingestion_data_type) total_ingested += max_emails - self._dup_emails if ingest_manner == GSMAIL_LATEST_INGEST_MANNER or total_ingested >= run_limit or self.is_poll_now(): @@ -1025,7 +1028,8 @@ def _handle_send_email(self, param): return action_result.get_status() return action_result.set_status(phantom.APP_SUCCESS, "Email sent with id {0}".format(sent_message["id"])) - def _process_email_ids(self, action_result, config, service, email_ids): + def _process_email_ids(self, action_result, config, service, email_ids, data_type="utf-8"): + self.debug_print("The format type is {0}".format(data_type)) for i, emid in enumerate(email_ids): self.send_progress("Parsing email id: {0}".format(emid)) try: @@ -1040,7 +1044,7 @@ def _process_email_ids(self, action_result, config, service, email_ids): # but base64 can be represented in ascii with no possible issues raw_decode = base64.urlsafe_b64decode(message['raw'].encode("utf-8")).decode("utf-8") process_email = ProcessMail(self, config) - process_email.process_email(raw_decode, emid, timestamp) + process_email.process_email(raw_decode, emid, timestamp, data_type) def _update_state(self): utc_now = datetime.utcnow() diff --git a/gsgmail_process_email.py b/gsgmail_process_email.py index 02db856..8c240e1 100644 --- a/gsgmail_process_email.py +++ b/gsgmail_process_email.py @@ -201,7 +201,7 @@ def _get_ips(self, file_data, ips): if ips_in_mail: ips |= set(ips_in_mail) - def _handle_body(self, body, parsed_mail, email_id): + def _handle_body(self, body, parsed_mail, email_id, data_type="utf-8"): local_file_path = body['file_path'] ips = parsed_mail[PROC_EMAIL_JSON_IPS] @@ -353,6 +353,7 @@ def _decode_uni_string(self, input_str, def_name): value = decoded_string.get('value') encoding = decoded_string.get('encoding') + self._base_connector.debug_string("Decode uni string pair, value: {0}".format(value)) if not encoding or not value: # nothing to replace with @@ -398,7 +399,7 @@ def _get_container_name(self, parsed_mail, email_id): except Exception: return self._decode_uni_string(subject, def_cont_name) - def _handle_if_body(self, content_disp, content_type, part, bodies, file_path, parsed_mail): + def _handle_if_body(self, content_disp, content_type, part, bodies, file_path, parsed_mail, data_type="utf-8"): process_as_body = False @@ -425,10 +426,10 @@ def _handle_if_body(self, content_disp, content_type, part, bodies, file_path, p bodies.append({'file_path': file_path, 'charset': part.get_content_charset()}) - self._add_body_in_email_headers(parsed_mail, file_path, charset, content_type) + self._add_body_in_email_headers(parsed_mail, file_path, charset, content_type, data_type) return phantom.APP_SUCCESS, False - def _handle_part(self, part, part_index, tmp_dir, extract_attach, parsed_mail): + def _handle_part(self, part, part_index, tmp_dir, extract_attach, parsed_mail, data_type="utf-8"): bodies = parsed_mail[PROC_EMAIL_JSON_BODIES] files = parsed_mail[PROC_EMAIL_JSON_FILES] @@ -465,7 +466,7 @@ def _handle_part(self, part, part_index, tmp_dir, extract_attach, parsed_mail): self._base_connector.debug_print("file_path: {0}".format(file_path)) # is the part representing the body of the email - status, process_further = self._handle_if_body(content_disp, content_type, part, bodies, file_path, parsed_mail) + status, process_further = self._handle_if_body(content_disp, content_type, part, bodies, file_path, parsed_mail, data_type) if not process_further: return phantom.APP_SUCCESS @@ -499,7 +500,7 @@ def _get_file_name(self, input_str): except Exception: return self._decode_uni_string(input_str, input_str) - def _parse_email_headers(self, parsed_mail, part, charset=None, add_email_id=None): + def _parse_email_headers(self, parsed_mail, part, charset=None, add_email_id=None, data_type="utf-8"): email_header_artifacts = parsed_mail[PROC_EMAIL_JSON_EMAIL_HEADERS] email_headers = part.items() @@ -508,7 +509,8 @@ def _parse_email_headers(self, parsed_mail, part, charset=None, add_email_id=Non # Parse email keys first - headers = self._get_email_headers_from_part(part, charset) + headers = self._get_email_headers_from_part(part, charset, data_type) + self._base_connector.debug_print("parse emails headers {0}", headers) cef_artifact = {} cef_types = {} @@ -551,7 +553,7 @@ def _parse_email_headers(self, parsed_mail, part, charset=None, add_email_id=Non return len(email_header_artifacts) - def _get_email_headers_from_part(self, part, charset=None): + def _get_email_headers_from_part(self, part, charset=None, data_type="utf-8"): email_headers = list(part.items()) @@ -592,6 +594,9 @@ def _get_email_headers_from_part(self, part, charset=None): headers['decodedSubject'] = str(make_header(decode_header(subject))) except Exception: headers['decodedSubject'] = self._decode_uni_string(subject, subject) + if data_type == "ascii": + headers['decodedSubject'] = self._remove_non_ascii_characters(headers['decodedSubject']) + self._base_connector.debug_print("part headers {0}".format(headers)) return dict(headers) def _get_error_message_from_exception(self, e): @@ -617,7 +622,7 @@ def _get_error_message_from_exception(self, e): return error_code, error_msg - def _handle_mail_object(self, mail, email_id, rfc822_email, tmp_dir, start_time_epoch): + def _handle_mail_object(self, mail, email_id, rfc822_email, tmp_dir, start_time_epoch, data_type="utf-8"): parsed_mail = OrderedDict() @@ -635,6 +640,10 @@ def _handle_mail_object(self, mail, email_id, rfc822_email, tmp_dir, start_time_ # Extract fields and place it in a dictionary parsed_mail[PROC_EMAIL_JSON_SUBJECT] = mail.get('Subject', '') + self._base_connector.debug_print("handle mail object subject {0}".format(parsed_mail[PROC_EMAIL_JSON_SUBJECT])) + if data_type == "ascii": + parsed_mail[PROC_EMAIL_JSON_SUBJECT] = self._remove_non_ascii_characters(parsed_mail[PROC_EMAIL_JSON_SUBJECT]) + self._base_connector.debug_print("handle mail object subject {0}".format(parsed_mail[PROC_EMAIL_JSON_SUBJECT])) parsed_mail[PROC_EMAIL_JSON_FROM] = mail.get('From', '') parsed_mail[PROC_EMAIL_JSON_TO] = mail.get('To', '') parsed_mail[PROC_EMAIL_JSON_DATE] = mail.get('Date', '') @@ -651,14 +660,14 @@ def _handle_mail_object(self, mail, email_id, rfc822_email, tmp_dir, start_time_ if i == 0: add_email_id = email_id - self._parse_email_headers(parsed_mail, part, add_email_id=add_email_id) + self._parse_email_headers(parsed_mail, part, add_email_id=add_email_id, data_type=data_type) self._base_connector.debug_print("part type", type(part)) if part.is_multipart(): self.check_and_update_eml(part) continue try: - ret_val = self._handle_part(part, i, tmp_dir, extract_attach, parsed_mail) + ret_val = self._handle_part(part, i, tmp_dir, extract_attach, parsed_mail, data_type) except Exception as e: self._base_connector.debug_print("ErrorExp in _handle_part # {0}".format(i), e) continue @@ -667,13 +676,13 @@ def _handle_mail_object(self, mail, email_id, rfc822_email, tmp_dir, start_time_ continue else: - self._parse_email_headers(parsed_mail, mail, add_email_id=email_id) + self._parse_email_headers(parsed_mail, mail, add_email_id=email_id, data_type=data_type) # parsed_mail[PROC_EMAIL_JSON_EMAIL_HEADERS].append(mail.items()) file_path = "{0}/part_1.text".format(tmp_dir) with open(file_path, 'wb') as f: # noqa f.write(mail.get_payload(decode=True)) bodies.append({'file_path': file_path, 'charset': charset}) - self._add_body_in_email_headers(parsed_mail, file_path, mail.get_content_charset(), 'text/plain') + self._add_body_in_email_headers(parsed_mail, file_path, mail.get_content_charset(), 'text/plain', data_type) # get the container name container_name = self._get_container_name(parsed_mail, email_id) @@ -705,8 +714,10 @@ def _handle_mail_object(self, mail, email_id, rfc822_email, tmp_dir, start_time_ if not body: continue + self._base_connector.debug_print("in handle mail object body is {0}".format(body)) + self._base_connector.debug_print("in handle mail object parsed mail is {0}".format(parsed_mail)) try: - self._handle_body(body, parsed_mail, email_id) + self._handle_body(body, parsed_mail, email_id, data_type) except Exception as e: self._base_connector.debug_print_debug_print("ErrorExp in _handle_body # {0}: {1}".format(i, str(e))) continue @@ -718,7 +729,22 @@ def _handle_mail_object(self, mail, email_id, rfc822_email, tmp_dir, start_time_ return phantom.APP_SUCCESS - def _add_body_in_email_headers(self, parsed_mail, file_path, charset, content_type): + def _remove_non_ascii_characters(self, text): + return re.sub(r'[^\x00-\x7F]', '', text) + + def _remove_non_ascii_from_html(self, html_content): + def clean_ascii(self, node): + if node.string: + node.string.replace_with(self._remove_non_ascii_characters(node.string)) + else: + for child in node.contents: + clean_ascii(self, child) + + soup = BeautifulSoup(html_content, 'html.parser') + clean_ascii(self, soup) + return str(soup) + + def _add_body_in_email_headers(self, parsed_mail, file_path, charset, content_type, data_type="utf-8"): # Add email_bodies to email_headers email_headers = parsed_mail[PROC_EMAIL_JSON_EMAIL_HEADERS] @@ -732,6 +758,8 @@ def _add_body_in_email_headers(self, parsed_mail, file_path, charset, content_ty self._base_connector.debug_print("Reading file data using binary mode") # Add body to the last added Email artifact body_content = UnicodeDammit(body_content).unicode_markup.encode('utf-8').decode('utf-8') + self._base_connector.debug_print("add body in email headers body content {0}".format(body_content)) + if 'text/plain' in content_type: try: email_headers[-1]['cef']['bodyText'] = self._get_string( @@ -745,6 +773,9 @@ def _add_body_in_email_headers(self, parsed_mail, file_path, charset, content_ty err = "Error occurred while parsing text/plain body content for creating artifacts" self._base_connector.debug_print("{}. {}. {}".format(err, error_code, error_msg)) + if data_type == "ascii": + email_headers[-1]['cef']['bodyText'] = self._remove_non_ascii_characters(email_headers[-1]['cef']['bodyText']) + elif 'text/html' in content_type: try: email_headers[-1]['cef']['bodyHtml'] = self._get_string( @@ -758,6 +789,9 @@ def _add_body_in_email_headers(self, parsed_mail, file_path, charset, content_ty err = "Error occurred while parsing text/html body content for creating artifacts" self._base_connector.debug_print("{}. {}. {}".format(err, error_code, error_msg)) + if data_type == "ascii": + email_headers[-1]['cef']['bodyHtml'] = self._remove_non_ascii_from_html(email_headers[-1]['cef']['bodyHtml']) + else: if not email_headers[-1]['cef'].get('bodyOther'): email_headers[-1]['cef']['bodyOther'] = {} @@ -809,12 +843,12 @@ def _set_email_id_contains(self, email_id): return - def _int_process_email(self, rfc822_email, email_id, start_time_epoch): + def _int_process_email(self, rfc822_email, email_id, start_time_epoch, data_type="utf-8"): mail = email.message_from_string(rfc822_email) tmp_dir = tempfile.mkdtemp(prefix='ph_email') self._tmp_dirs.append(tmp_dir) try: - ret_val = self._handle_mail_object(mail, email_id, rfc822_email, tmp_dir, start_time_epoch) + ret_val = self._handle_mail_object(mail, email_id, rfc822_email, tmp_dir, start_time_epoch, data_type) except Exception as e: message = "ErrorExp in _handle_mail_object: {0}".format(e) self._base_connector.debug_print(message) @@ -863,13 +897,13 @@ def write_with_new_filename(self, tmp_dir, data, file_extension, dict_to_fill, f except Exception as e: self._base_connector.debug_print('Exception while writing file: {}'.format(e)) - def process_email(self, rfc822_email, email_id, epoch): + def process_email(self, rfc822_email, email_id, epoch, data_type="utf-8"): try: self._set_email_id_contains(email_id) except Exception: pass - ret_val, message, results = self._int_process_email(rfc822_email, email_id, epoch) + ret_val, message, results = self._int_process_email(rfc822_email, email_id, epoch, data_type) if not ret_val: self._del_tmp_dirs() From 70cfa91a4df95c23b6fabe0d370608353ac11d90 Mon Sep 17 00:00:00 2001 From: splunk-soar-connectors-admin Date: Thu, 25 Jul 2024 17:43:46 +0000 Subject: [PATCH 08/16] Update README.md --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index a21702b..43bdf4a 100644 --- a/README.md +++ b/README.md @@ -96,6 +96,8 @@ VARIABLE | REQUIRED | TYPE | DESCRIPTION **ingest_manner** | optional | string | How to ingest **first_run_max_emails** | optional | numeric | Maximum Containers for scheduled polling first time **max_containers** | optional | numeric | Maximum Containers for scheduled polling +**data_type** | optional | string | Ingestion data type when polling +**forwarding_address** | optional | string | Address to forward all emails to **extract_attachments** | optional | boolean | Extract Attachments **extract_urls** | optional | boolean | Extract URLs **extract_ips** | optional | boolean | Extract IPs @@ -278,6 +280,7 @@ PARAMETER | REQUIRED | DESCRIPTION | TYPE | CONTAINS **container_id** | optional | Parameter Ignored in this app | string | **container_count** | required | Maximum number of emails to ingest | numeric | **artifact_count** | required | Maximum number of artifact to ingest | numeric | +**data_type** | optional | Encode ingested emails as ASCII or UTF-8 | string | #### Action Output No Output From e1553d920e98482af72e6656a67ba5ce7e9a5195 Mon Sep 17 00:00:00 2001 From: Tapish Jain Date: Wed, 31 Jul 2024 15:39:18 -0700 Subject: [PATCH 09/16] PAPP-34391: forwarding and auto reply features --- gsgmail.json | 19 +++++++---- gsgmail_connector.py | 74 ++++++++++++++++++++++++++++++++++++++-- gsgmail_process_email.py | 8 +++-- 3 files changed, 89 insertions(+), 12 deletions(-) diff --git a/gsgmail.json b/gsgmail.json index e5aa5e2..c77ddb8 100644 --- a/gsgmail.json +++ b/gsgmail.json @@ -72,48 +72,53 @@ "order": 6 }, "forwarding_address": { - "description": "Address to forward all emails to", + "description": "Address to forward polled emails to", "data_type": "string", "contains": [ "email" ], "order": 7 }, + "auto_reply": { + "description": "Auto reply to emails with a set body", + "data_type": "string", + "order": 8 + }, "extract_attachments": { "description": "Extract Attachments", "data_type": "boolean", "default": false, - "order": 8 + "order": 9 }, "extract_urls": { "description": "Extract URLs", "data_type": "boolean", "default": false, - "order": 9 + "order": 10 }, "extract_ips": { "description": "Extract IPs", "data_type": "boolean", "default": false, - "order": 10 + "order": 11 }, "extract_domains": { "description": "Extract Domain Names", "data_type": "boolean", "default": false, - "order": 11 + "order": 12 }, "extract_hashes": { "description": "Extract Hashes", "data_type": "boolean", "default": false, - "order": 12 + "order": 13 }, "download_eml_attachments": { "description": "Download EML attachments", "data_type": "boolean", "default": false, - "order": 13 + "order": 14 } }, "actions": [ diff --git a/gsgmail_connector.py b/gsgmail_connector.py index 63dad9d..b520dd9 100644 --- a/gsgmail_connector.py +++ b/gsgmail_connector.py @@ -130,7 +130,9 @@ def initialize(self): _, _, self._domain = self._login_email.partition('@') except Exception: return self.set_status(phantom.APP_ERROR, "Unable to extract domain from login_email") - + + return phantom.APP_SUCCESS + def _validate_integer(self, action_result, parameter, key, allow_zero=False): """ Validate an integer. @@ -482,6 +484,7 @@ def _parse_multipart_message( msg, email_details, action_result, extract_attachments, extract_nested ) self._join_email_bodies(email_details) + self.debug_print("parse multirpart {0}".format(email_details)) return ret_val def _handle_get_email(self, param): @@ -1029,7 +1032,6 @@ def _handle_send_email(self, param): return action_result.set_status(phantom.APP_SUCCESS, "Email sent with id {0}".format(sent_message["id"])) def _process_email_ids(self, action_result, config, service, email_ids, data_type="utf-8"): - self.debug_print("The format type is {0}".format(data_type)) for i, emid in enumerate(email_ids): self.send_progress("Parsing email id: {0}".format(emid)) try: @@ -1043,8 +1045,74 @@ def _process_email_ids(self, action_result, config, service, email_ids, data_typ # the api libraries return the base64 encoded message as a unicode string, # but base64 can be represented in ascii with no possible issues raw_decode = base64.urlsafe_b64decode(message['raw'].encode("utf-8")).decode("utf-8") + + if config.get("auto_reply"): + ret_val, sent_message = self._auto_reply(message, config, action_result) + if phantom.is_fail(ret_val): + self.send_progress("Auto reply to email with id {0} failed".format(emid)) + else: + self.send_progress("Auto reply to email with id {0} succeeded. Id of reply: {1}".format(emid, sent_message["id"])) process_email = ProcessMail(self, config) - process_email.process_email(raw_decode, emid, timestamp, data_type) + ret_val, msg, vault_ids = process_email.process_email(raw_decode, emid, timestamp, data_type) + + if config.get("forwarding_address"): + ret_val, sent_message = self._forward_email(message, vault_ids, config, action_result) + if phantom.is_fail(ret_val): + self.send_progress("Forwarded email with id {0} to {1} failed".format(emid, config["forwarding_address"])) + else: + self.send_progress("Forwarded email with id {0} to {1}. Id of forwarded message: {2}".format(emid, config["forwarding_address"], sent_message["id"])) + + def _auto_reply(self, message, config, action_result): + raw_encoded = base64.urlsafe_b64decode(message["raw"].encode('UTF8')) + msg = email.message_from_bytes(raw_encoded) + headers = self._get_email_headers_from_part(msg) + to_address = headers.get("from", "") + subject = headers.get("subject", "") + reply_message = self._create_message(config["login_email"], to_address, None, None, 'Re: ' + subject, config["auto_reply"]) + media = MediaIoBaseUpload(BytesIO(reply_message.as_bytes()), mimetype='message/rfc822', resumable=True) + + scopes = [GSGMAIL_DELETE_EMAIL] + ret_val, service = self._create_service(action_result, scopes, "gmail", "v1", self._login_email) + + ret_val, sent_message = self._send_email(service, "me", media, action_result) + if phantom.is_fail(ret_val): + return action_result.get_status(), sent_message + return phantom.APP_SUCCESS, sent_message + + def _forward_email(self, email_details, attachment_vault_ids, config, action_result): + raw_encoded = base64.urlsafe_b64decode(email_details.pop('raw').encode('UTF8')) + msg = email.message_from_bytes(raw_encoded) + + if msg.is_multipart(): + ret_val = self._parse_multipart_message(action_result, msg, email_details, False, False) + + if phantom.is_fail(ret_val): + return action_result.get_status() + else: + # not multipart + email_details['email_headers'] = [] + charset = msg.get_content_charset() + headers = self._get_email_headers_from_part(msg) + email_details['email_headers'].append(headers) + try: + email_details['parsed_plain_body'] = msg.get_payload(decode=True).decode(encoding=charset, errors="ignore") + except Exception as e: + message = self._get_error_message_from_exception(e) + self.error_print(f"Unable to add email body: {message}") + + subject = "Fwd: " + email_details["email_headers"][0]["subject"] + body = email_details['parsed_plain_body'] or email_details.get("parsed_html_body", None) + forwrded_message = self._create_message(config["login_email"], config["forwarding_address"], None, None, subject, body, vault_ids=attachment_vault_ids) + + media = MediaIoBaseUpload(BytesIO(forwrded_message.as_bytes()), mimetype='message/rfc822', resumable=True) + + scopes = [GSGMAIL_DELETE_EMAIL] + ret_val, service = self._create_service(action_result, scopes, "gmail", "v1", self._login_email) + + ret_val, sent_message = self._send_email(service, "me", media, action_result) + if phantom.is_fail(ret_val): + return action_result.get_status(), sent_message + return phantom.APP_SUCCESS, sent_message def _update_state(self): utc_now = datetime.utcnow() diff --git a/gsgmail_process_email.py b/gsgmail_process_email.py index 8c240e1..088f64c 100644 --- a/gsgmail_process_email.py +++ b/gsgmail_process_email.py @@ -87,6 +87,7 @@ def __init__(self, base_connector, config): self._artifacts = list() self._attachments = list() self._tmp_dirs = list() + self._vault_ids = list() def _get_file_contains(self, file_path): @@ -909,6 +910,8 @@ def process_email(self, rfc822_email, email_id, epoch, data_type="utf-8"): self._del_tmp_dirs() return phantom.APP_ERROR, message + self._base_connector.debug_print("results files after processing {0}".format(results[0].get("files"))) + self._base_connector.debug_print("results artifacts after processing {0}".format(results[0].get("artifacts"))) try: self._parse_results(results) self._del_tmp_dirs() @@ -917,9 +920,9 @@ def process_email(self, rfc822_email, email_id, epoch, data_type="utf-8"): error_message = self._base_connector._get_error_message_from_exception(e) message = "Parsing results failed. {0}".format(error_message) self._debug_print(message) - return phantom.APP_ERROR, message + return phantom.APP_ERROR, message, self._vault_ids - return phantom.APP_SUCCESS, PROC_EMAIL_PROCESSED + return phantom.APP_SUCCESS, PROC_EMAIL_PROCESSED, self._vault_ids def _parse_results(self, results): @@ -1062,6 +1065,7 @@ def _handle_file(self, curr_file, container_id): self._base_connector.debug_print(PROC_EMAIL_FAILED_VAULT_ADD_FILE.format(message)) return phantom.APP_ERROR, phantom.APP_ERROR + self._vault_ids.append(vault_id) # add the vault id artifact to the container cef_artifact = {} if file_name: From 5ea3afffffdf2eb540a6a27c4cc422f6514967b9 Mon Sep 17 00:00:00 2001 From: Tapish Jain Date: Thu, 1 Aug 2024 11:50:28 -0700 Subject: [PATCH 10/16] PAPP-34391: ascii working as expected --- gsgmail.json | 2 +- gsgmail_connector.py | 1 - gsgmail_process_email.py | 40 ++++++++++++++++++++-------------------- 3 files changed, 21 insertions(+), 22 deletions(-) diff --git a/gsgmail.json b/gsgmail.json index c77ddb8..c73e23c 100644 --- a/gsgmail.json +++ b/gsgmail.json @@ -68,7 +68,7 @@ "utf-8", "ascii" ], - "default": "utc-8", + "default": "utf-8", "order": 6 }, "forwarding_address": { diff --git a/gsgmail_connector.py b/gsgmail_connector.py index b520dd9..4cf02b6 100644 --- a/gsgmail_connector.py +++ b/gsgmail_connector.py @@ -853,7 +853,6 @@ def _handle_on_poll(self, param): if not self.is_poll_now(): self._update_state() - self.debug_print("On poll data type {0}".format(ingestion_data_type)) if not ingestion_data_type: ingestion_data_type = "utf-8" self._process_email_ids(action_result, config, service, email_ids, ingestion_data_type) diff --git a/gsgmail_process_email.py b/gsgmail_process_email.py index 088f64c..7acfdd0 100644 --- a/gsgmail_process_email.py +++ b/gsgmail_process_email.py @@ -24,7 +24,7 @@ import string import tempfile from collections import OrderedDict -from email.header import decode_header, make_header +from email.header import decode_header, make_header, Header import magic import phantom.app as phantom @@ -202,7 +202,7 @@ def _get_ips(self, file_data, ips): if ips_in_mail: ips |= set(ips_in_mail) - def _handle_body(self, body, parsed_mail, email_id, data_type="utf-8"): + def _handle_body(self, body, parsed_mail, email_id): local_file_path = body['file_path'] ips = parsed_mail[PROC_EMAIL_JSON_IPS] @@ -354,7 +354,6 @@ def _decode_uni_string(self, input_str, def_name): value = decoded_string.get('value') encoding = decoded_string.get('encoding') - self._base_connector.debug_string("Decode uni string pair, value: {0}".format(value)) if not encoding or not value: # nothing to replace with @@ -509,9 +508,7 @@ def _parse_email_headers(self, parsed_mail, part, charset=None, add_email_id=Non return 0 # Parse email keys first - headers = self._get_email_headers_from_part(part, charset, data_type) - self._base_connector.debug_print("parse emails headers {0}", headers) cef_artifact = {} cef_types = {} @@ -595,9 +592,13 @@ def _get_email_headers_from_part(self, part, charset=None, data_type="utf-8"): headers['decodedSubject'] = str(make_header(decode_header(subject))) except Exception: headers['decodedSubject'] = self._decode_uni_string(subject, subject) + if data_type == "ascii": - headers['decodedSubject'] = self._remove_non_ascii_characters(headers['decodedSubject']) - self._base_connector.debug_print("part headers {0}".format(headers)) + ascii_subject = self._remove_non_ascii_characters(headers['decodedSubject']) + headers['decodedSubject'] = ascii_subject + mime_format_header = Header(ascii_subject, "utf-8").encode() + headers[subject] = mime_format_header + return dict(headers) def _get_error_message_from_exception(self, e): @@ -641,10 +642,15 @@ def _handle_mail_object(self, mail, email_id, rfc822_email, tmp_dir, start_time_ # Extract fields and place it in a dictionary parsed_mail[PROC_EMAIL_JSON_SUBJECT] = mail.get('Subject', '') - self._base_connector.debug_print("handle mail object subject {0}".format(parsed_mail[PROC_EMAIL_JSON_SUBJECT])) if data_type == "ascii": - parsed_mail[PROC_EMAIL_JSON_SUBJECT] = self._remove_non_ascii_characters(parsed_mail[PROC_EMAIL_JSON_SUBJECT]) - self._base_connector.debug_print("handle mail object subject {0}".format(parsed_mail[PROC_EMAIL_JSON_SUBJECT])) + decoded_subject = None + try: + decoded_subject = str(make_header(decode_header(parsed_mail[PROC_EMAIL_JSON_SUBJECT]))) + except Exception: + decoded_subject = self._decode_uni_string(parsed_mail[PROC_EMAIL_JSON_SUBJECT], None) + ascii_subject = self._remove_non_ascii_characters(decoded_subject) + mime_format_header = Header(ascii_subject, "utf-8").encode() + parsed_mail[PROC_EMAIL_JSON_SUBJECT] = mime_format_header parsed_mail[PROC_EMAIL_JSON_FROM] = mail.get('From', '') parsed_mail[PROC_EMAIL_JSON_TO] = mail.get('To', '') parsed_mail[PROC_EMAIL_JSON_DATE] = mail.get('Date', '') @@ -677,14 +683,13 @@ def _handle_mail_object(self, mail, email_id, rfc822_email, tmp_dir, start_time_ continue else: - self._parse_email_headers(parsed_mail, mail, add_email_id=email_id, data_type=data_type) + self._parse_email_headers(parsed_mail, mail, add_email_id=email_id) # parsed_mail[PROC_EMAIL_JSON_EMAIL_HEADERS].append(mail.items()) file_path = "{0}/part_1.text".format(tmp_dir) with open(file_path, 'wb') as f: # noqa f.write(mail.get_payload(decode=True)) bodies.append({'file_path': file_path, 'charset': charset}) self._add_body_in_email_headers(parsed_mail, file_path, mail.get_content_charset(), 'text/plain', data_type) - # get the container name container_name = self._get_container_name(parsed_mail, email_id) @@ -715,12 +720,10 @@ def _handle_mail_object(self, mail, email_id, rfc822_email, tmp_dir, start_time_ if not body: continue - self._base_connector.debug_print("in handle mail object body is {0}".format(body)) - self._base_connector.debug_print("in handle mail object parsed mail is {0}".format(parsed_mail)) try: - self._handle_body(body, parsed_mail, email_id, data_type) + self._handle_body(body, parsed_mail, email_id) except Exception as e: - self._base_connector.debug_print_debug_print("ErrorExp in _handle_body # {0}: {1}".format(i, str(e))) + self._base_connector.debug_print("ErrorExp in _handle_body # {0}: {1}".format(i, str(e))) continue # Files @@ -759,7 +762,6 @@ def _add_body_in_email_headers(self, parsed_mail, file_path, charset, content_ty self._base_connector.debug_print("Reading file data using binary mode") # Add body to the last added Email artifact body_content = UnicodeDammit(body_content).unicode_markup.encode('utf-8').decode('utf-8') - self._base_connector.debug_print("add body in email headers body content {0}".format(body_content)) if 'text/plain' in content_type: try: @@ -908,10 +910,8 @@ def process_email(self, rfc822_email, email_id, epoch, data_type="utf-8"): if not ret_val: self._del_tmp_dirs() - return phantom.APP_ERROR, message + return phantom.APP_ERROR, message, self._vault_ids - self._base_connector.debug_print("results files after processing {0}".format(results[0].get("files"))) - self._base_connector.debug_print("results artifacts after processing {0}".format(results[0].get("artifacts"))) try: self._parse_results(results) self._del_tmp_dirs() From 06084fff336d756b4275e710af2bc23a977aac49 Mon Sep 17 00:00:00 2001 From: splunk-soar-connectors-admin Date: Wed, 31 Jul 2024 22:40:15 +0000 Subject: [PATCH 11/16] Update README.md --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 43bdf4a..c682e44 100644 --- a/README.md +++ b/README.md @@ -97,7 +97,8 @@ VARIABLE | REQUIRED | TYPE | DESCRIPTION **first_run_max_emails** | optional | numeric | Maximum Containers for scheduled polling first time **max_containers** | optional | numeric | Maximum Containers for scheduled polling **data_type** | optional | string | Ingestion data type when polling -**forwarding_address** | optional | string | Address to forward all emails to +**forwarding_address** | optional | string | Address to forward polled emails to +**auto_reply** | optional | string | Auto reply to emails with a set body **extract_attachments** | optional | boolean | Extract Attachments **extract_urls** | optional | boolean | Extract URLs **extract_ips** | optional | boolean | Extract IPs From 5b4c74ddff9198ea69440e7490dcd29f5ef23583 Mon Sep 17 00:00:00 2001 From: Tapish Jain Date: Thu, 1 Aug 2024 12:09:57 -0700 Subject: [PATCH 12/16] PAPP-34391: small styling changes --- gsgmail_connector.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/gsgmail_connector.py b/gsgmail_connector.py index 4cf02b6..642dcae 100644 --- a/gsgmail_connector.py +++ b/gsgmail_connector.py @@ -1056,10 +1056,11 @@ def _process_email_ids(self, action_result, config, service, email_ids, data_typ if config.get("forwarding_address"): ret_val, sent_message = self._forward_email(message, vault_ids, config, action_result) + to = config["forwarding_address"] if phantom.is_fail(ret_val): - self.send_progress("Forwarded email with id {0} to {1} failed".format(emid, config["forwarding_address"])) + self.send_progress("Forwarded email with id {0} to {1} failed".format(emid, to)) else: - self.send_progress("Forwarded email with id {0} to {1}. Id of forwarded message: {2}".format(emid, config["forwarding_address"], sent_message["id"])) + self.send_progress("Forwarded email with id {0} to {1}. Forwarded message id: {2}".format(emid, to, sent_message["id"])) def _auto_reply(self, message, config, action_result): raw_encoded = base64.urlsafe_b64decode(message["raw"].encode('UTF8')) @@ -1101,7 +1102,15 @@ def _forward_email(self, email_details, attachment_vault_ids, config, action_res subject = "Fwd: " + email_details["email_headers"][0]["subject"] body = email_details['parsed_plain_body'] or email_details.get("parsed_html_body", None) - forwrded_message = self._create_message(config["login_email"], config["forwarding_address"], None, None, subject, body, vault_ids=attachment_vault_ids) + forwrded_message = self._create_message( + config["login_email"], + config["forwarding_address"], + None, + None, + subject, + body, + vault_ids=attachment_vault_ids + ) media = MediaIoBaseUpload(BytesIO(forwrded_message.as_bytes()), mimetype='message/rfc822', resumable=True) From 4087fe16a56b472ac75b1bf493e1feda36ad8348 Mon Sep 17 00:00:00 2001 From: Tapish Jain Date: Thu, 1 Aug 2024 16:23:23 -0700 Subject: [PATCH 13/16] PAPP-34391: getting rid of debug statement --- gsgmail_connector.py | 1 - 1 file changed, 1 deletion(-) diff --git a/gsgmail_connector.py b/gsgmail_connector.py index 642dcae..54cb217 100644 --- a/gsgmail_connector.py +++ b/gsgmail_connector.py @@ -484,7 +484,6 @@ def _parse_multipart_message( msg, email_details, action_result, extract_attachments, extract_nested ) self._join_email_bodies(email_details) - self.debug_print("parse multirpart {0}".format(email_details)) return ret_val def _handle_get_email(self, param): From 2cbded1c527cc2f912f8d84cbff6620092d588cb Mon Sep 17 00:00:00 2001 From: Tapish Jain Date: Thu, 8 Aug 2024 10:00:54 -0700 Subject: [PATCH 14/16] PAPP-34391: linting change --- gsgmail_process_email.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gsgmail_process_email.py b/gsgmail_process_email.py index 7acfdd0..0215d1a 100644 --- a/gsgmail_process_email.py +++ b/gsgmail_process_email.py @@ -24,7 +24,7 @@ import string import tempfile from collections import OrderedDict -from email.header import decode_header, make_header, Header +from email.header import Header, decode_header, make_header import magic import phantom.app as phantom From 610919a0824010cec902eb289e54a14c412012d2 Mon Sep 17 00:00:00 2001 From: Tapish Jain Date: Thu, 8 Aug 2024 14:27:23 -0700 Subject: [PATCH 15/16] PAPP-34389: small changes to action result data --- gsgmail.json | 4 ++-- gsgmail_connector.py | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/gsgmail.json b/gsgmail.json index 15fcef0..b91de4d 100644 --- a/gsgmail.json +++ b/gsgmail.json @@ -1433,7 +1433,7 @@ ] }, { - "data_path": "action_result.data.*.draft_id", + "data_path": "action_result.data.*.id", "data_type": "string", "example_values": [ "rfc822t1500000000t3a1d2e0fghijklm" @@ -1477,7 +1477,7 @@ ], "render": { "width": 12, - "title": "Delete Email", + "title": "Send Email", "type": "table", "height": 5 }, diff --git a/gsgmail_connector.py b/gsgmail_connector.py index 6d3c18c..f5e8acb 100644 --- a/gsgmail_connector.py +++ b/gsgmail_connector.py @@ -1023,6 +1023,7 @@ def _handle_send_email(self, param): ret_val, sent_message = self._send_email(service, "me", media, action_result) if phantom.is_fail(ret_val): return action_result.get_status() + action_result.add_data(sent_message) return action_result.set_status(phantom.APP_SUCCESS, "Email sent with id {0}".format(sent_message["id"])) def _process_email_ids(self, action_result, config, service, email_ids): From 910dcee929be1c025e40304564c0142f671133a4 Mon Sep 17 00:00:00 2001 From: splunk-soar-connectors-admin Date: Thu, 8 Aug 2024 21:28:03 +0000 Subject: [PATCH 16/16] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index a21702b..a26951c 100644 --- a/README.md +++ b/README.md @@ -387,7 +387,7 @@ action_result.parameter.from | string | `email` | test@testdomain.abc.com action_result.parameter.headers | string | | {"x-custom-header":"Custom value"} action_result.parameter.subject | string | | Example subject action_result.parameter.to | string | `email` | test@testdomain.abc.com -action_result.data.\*.draft_id | string | | rfc822t1500000000t3a1d2e0fghijklm +action_result.data.\*.id | string | | rfc822t1500000000t3a1d2e0fghijklm action_result.data.\*.threadId | string | | 16d1234567890abcdef action_result.data.\*.labelIds | string | | INBOX action_result.message | string | | All the provided emails were already deleted