From c3b502c6e4637ebcde94b5ab2f19ff8a0ea95f10 Mon Sep 17 00:00:00 2001 From: grokas Date: Mon, 28 Oct 2024 17:49:45 -0400 Subject: [PATCH 1/9] PAPP-34983 new get mailbox messages action --- office365.json | 260 ++++++++++++++++++++++++++++++++++++++++- office365_connector.py | 43 +++++++ office365_consts.py | 7 ++ 3 files changed, 309 insertions(+), 1 deletion(-) diff --git a/office365.json b/office365.json index 78ea226..d6c8b60 100644 --- a/office365.json +++ b/office365.json @@ -33,7 +33,7 @@ } ], "license": "Copyright (c) 2017-2024 Splunk Inc.", - "app_version": "3.0.1", + "app_version": "3.1.1", "utctime_updated": "2024-09-11T08:34:15.000000Z", "package_name": "phantom_msgraphoffice365", "main_module": "office365_connector.py", @@ -7917,6 +7917,264 @@ "view": "office365_view.display_view" }, "versions": "EQ(*)" + }, + { + "action": "get mailbox messages", + "identifier": "get_mailbox_messages", + "description": "Retrieves messages from a specified mailbox folder", + "type": "investigate", + "read_only": true, + "parameters": { + "email_address": { + "description": "Email address of the mailbox", + "data_type": "string", + "required": true, + "primary": true, + "order": 0 + }, + "folder": { + "description": "Folder to retrieve messages", + "data_type": "string", + "default": "inbox", + "order": 1 + }, + "limit": { + "description": "Maximum number of messages to retrieve", + "data_type": "numeric", + "default": 100, + "order": 2 + }, + "start_date": { + "description": "Start date for filtering messages (format: YYYY-MM-DD)", + "data_type": "string", + "order": 3 + }, + "end_date": { + "description": "End date for filtering messages (format: YYYY-MM-DD)", + "data_type": "string", + "order": 4 + } + }, + "output": [ + { + "data_path": "action_result.status", + "data_type": "string", + "example_values": ["success", "failed"] + }, + { + "data_path": "action_result.parameter.email_address", + "data_type": "string" + }, + { + "data_path": "action_result.parameter.folder", + "data_type": "string" + }, + { + "data_path": "action_result.parameter.limit", + "data_type": "numeric" + }, + { + "data_path": "action_result.parameter.start_date", + "data_type": "string" + }, + { + "data_path": "action_result.parameter.end_date", + "data_type": "string" + }, + { + "data_path": "action_result.data.*.id", + "data_type": "string", + "column_name": "ID", + "column_order": 0 + }, + { + "data_path": "action_result.data.*.body.content", + "data_type": "string" + }, + { + "data_path": "action_result.data.*.body.contentType", + "data_type": "string" + }, + { + "data_path": "action_result.data.*.flag.flagStatus", + "data_type": "string" + }, + { + "data_path": "action_result.data.*.from.emailAddress.name", + "data_type": "string" + }, + { + "data_path": "action_result.data.*.from.emailAddress.address", + "data_type": "string", + "column_name": "From", + "column_order": 1 + }, + { + "data_path": "action_result.data.*.isRead", + "data_type": "boolean", + "column_name": "Is Read", + "column_order": 2 + }, + { + "data_path": "action_result.data.*.sender.emailAddress.name", + "data_type": "string" + }, + { + "data_path": "action_result.data.*.sender.emailAddress.address", + "data_type": "string" + }, + { + "data_path": "action_result.data.*.isDraft", + "data_type": "boolean" + }, + { + "data_path": "action_result.data.*.replyTo.*.emailAddress.address", + "data_type": "string" + }, + { + "data_path": "action_result.data.*.replyTo.*.emailAddress.name", + "data_type": "string" + }, + { + "data_path": "action_result.data.*.subject", + "data_type": "string", + "column_name": "Subject", + "column_order": 3 + }, + { + "data_path": "action_result.data.*.webLink", + "data_type": "string", + "contains": ["url"] + }, + { + "data_path": "action_result.data.*.changeKey", + "data_type": "string" + }, + { + "data_path": "action_result.data.*.categories.*.name", + "data_type": "string" + }, + { + "data_path": "action_result.data.*.importance", + "data_type": "string" + }, + { + "data_path": "action_result.data.*.uniqueBody.content", + "data_type": "string" + }, + { + "data_path": "action_result.data.*.uniqueBody.contentType", + "data_type": "string" + }, + { + "data_path": "action_result.data.*.bodyPreview", + "data_type": "string" + }, + { + "data_path": "action_result.data.*.ccRecipients.*.emailAddress.address", + "data_type": "string" + }, + { + "data_path": "action_result.data.*.ccRecipients.*.emailAddress.name", + "data_type": "string" + }, + { + "data_path": "action_result.data.*.sentDateTime", + "data_type": "string" + }, + { + "data_path": "action_result.data.*.toRecipients.*.emailAddress.name", + "data_type": "string" + }, + { + "data_path": "action_result.data.*.toRecipients.*.emailAddress.address", + "data_type": "string" + }, + { + "data_path": "action_result.data.*.bccRecipients.*.emailAddress.address", + "data_type": "string" + }, + { + "data_path": "action_result.data.*.bccRecipients.*.emailAddress.name", + "data_type": "string" + }, + { + "data_path": "action_result.data.*.conversationId", + "data_type": "string" + }, + { + "data_path": "action_result.data.*.hasAttachments", + "data_type": "boolean", + "column_name": "Has Attachments", + "column_order": 4 + }, + { + "data_path": "action_result.data.*.parentFolderId", + "data_type": "string" + }, + { + "data_path": "action_result.data.*.createdDateTime", + "data_type": "string" + }, + { + "data_path": "action_result.data.*.receivedDateTime", + "data_type": "string", + "column_name": "Received Time", + "column_order": 5 + }, + { + "data_path": "action_result.data.*.conversationIndex", + "data_type": "string" + }, + { + "data_path": "action_result.data.*.internetMessageId", + "data_type": "string" + }, + { + "data_path": "action_result.data.*.lastModifiedDateTime", + "data_type": "string" + }, + { + "data_path": "action_result.data.*.internetMessageHeaders.*.name", + "data_type": "string" + }, + { + "data_path": "action_result.data.*.internetMessageHeaders.*.value", + "data_type": "string" + }, + { + "data_path": "action_result.data.*.isReadReceiptRequested", + "data_type": "boolean" + }, + { + "data_path": "action_result.data.*.inferenceClassification", + "data_type": "string" + }, + { + "data_path": "action_result.data.*.isDeliveryReceiptRequested", + "data_type": "boolean" + }, + { + "data_path": "action_result.summary.total_messages", + "data_type": "numeric" + }, + { + "data_path": "action_result.message", + "data_type": "string" + }, + { + "data_path": "summary.total_objects", + "data_type": "numeric" + }, + { + "data_path": "summary.total_objects_successful", + "data_type": "numeric" + } + ], + "render": { + "type": "table" + }, + "versions": "EQ(*)" } ], "pip_dependencies": { diff --git a/office365_connector.py b/office365_connector.py index a15200e..8480709 100644 --- a/office365_connector.py +++ b/office365_connector.py @@ -3055,6 +3055,46 @@ def _handle_resolve_name(self, param): return action_result.set_status(phantom.APP_SUCCESS) + def _handle_get_mailbox_messages(self, param): + self.save_progress("In action handler for: {0}".format(self.get_action_identifier())) + action_result = self.add_action_result(ActionResult(dict(param))) + + email_address = param["email_address"] + folder = param.get("folder", MSGOFFICE365_DEFAULT_FOLDER) + limit = param.get("limit", MSGOFFICE365_DEFAULT_LIMIT) + + ret_val, limit = _validate_integer(action_result, limit, "'limit' action") + if phantom.is_fail(ret_val): + return action_result.get_status() + + endpoint = MSGOFFICE365_MAILBOX_MESSAGES_ENDPOINT.format(email_address, folder) + params = {"$top": limit, "$orderby": MSGOFFICE365_ORDERBY_RECEIVED_DESC, "$select": ",".join(MSGOFFICE365_SELECT_PARAMETER_LIST)} + + # Add optional filters + date_filters = [] + if param.get("start_date"): + date_filters.append(MSGOFFICE365_RECEIVED_DATE_FILTER.format(operator="ge", date=param.get("start_date"))) + if param.get("end_date"): + date_filters.append(MSGOFFICE365_RECEIVED_DATE_FILTER.format(operator="le", date=param.get("end_date"))) + + if date_filters: + params["$filter"] = MSGOFFICE365_DATE_FILTER_AND.join(date_filters) + + ret_val, messages = self._paginator(action_result, endpoint, limit=limit, params=params) + + if phantom.is_fail(ret_val): + return action_result.get_status() + + for message in messages: + action_result.add_data(message) + + summary = action_result.update_summary({}) + summary["total_messages"] = len(messages) + + return action_result.set_status( + phantom.APP_SUCCESS, f"Successfully retrieved {len(messages)} messages from {email_address}'s {folder} folder" + ) + def handle_action(self, param): ret_val = phantom.APP_SUCCESS @@ -3139,6 +3179,9 @@ def handle_action(self, param): elif action_id == "update_email": ret_val = self._handle_update_email(param) + elif action_id == "get_mailbox_messages": + ret_val = self._handle_get_mailbox_messages(param) + return ret_val def _get_token(self, action_result): diff --git a/office365_consts.py b/office365_consts.py index 8a699bf..941d2f1 100644 --- a/office365_consts.py +++ b/office365_consts.py @@ -123,3 +123,10 @@ "webLink", "internetMessageId", ] + +MSGOFFICE365_MAILBOX_MESSAGES_ENDPOINT = "/users/{}/mailFolders/{}/messages" +MSGOFFICE365_DEFAULT_FOLDER = "Inbox" +MSGOFFICE365_DEFAULT_LIMIT = 100 +MSGOFFICE365_ORDERBY_RECEIVED_DESC = "receivedDateTime desc" +MSGOFFICE365_RECEIVED_DATE_FILTER = "receivedDateTime {operator} {date}" +MSGOFFICE365_DATE_FILTER_AND = " and " From aa7299753eb1b310d84f481a73a39a9bc796f675 Mon Sep 17 00:00:00 2001 From: grokas Date: Mon, 4 Nov 2024 09:45:24 -0800 Subject: [PATCH 2/9] PAPP-34983 updates for get mailbox messages action --- office365.json | 33 +++++- office365_connector.py | 236 +++++++++++++++++++++++++---------------- 2 files changed, 178 insertions(+), 91 deletions(-) diff --git a/office365.json b/office365.json index d6c8b60..ad86ef4 100644 --- a/office365.json +++ b/office365.json @@ -7944,15 +7944,44 @@ "default": 100, "order": 2 }, + "offset": { + "description": "Number of messages to skip before retrieving results", + "data_type": "numeric", + "default": 0, + "order": 3 + }, "start_date": { "description": "Start date for filtering messages (format: YYYY-MM-DD)", "data_type": "string", - "order": 3 + "order": 4 }, "end_date": { "description": "End date for filtering messages (format: YYYY-MM-DD)", "data_type": "string", - "order": 4 + "order": 5 + }, + "download_attachments": { + "description": "Download email attachments to vault", + "data_type": "boolean", + "default": false, + "order": 6 + }, + "download_email": { + "description": "Download email as EML file to vault", + "data_type": "boolean", + "default": false, + "order": 7 + }, + "extract_headers": { + "description": "Include email headers in results", + "data_type": "boolean", + "default": false, + "order": 8 + }, + "plus_ingest": { + "description": "If enabled, messages will be also ingested like on_poll", + "data_type": "boolean", + "order": 9 } }, "output": [ diff --git a/office365_connector.py b/office365_connector.py index 8480709..52ef8ad 100644 --- a/office365_connector.py +++ b/office365_connector.py @@ -1294,6 +1294,70 @@ def _process_email_data(self, config, action_result, endpoint, email): return phantom.APP_SUCCESS + def _process_email_details(self, action_result, email, email_address, extract_headers=False, download_attachments=False, download_email=False): + """ + Helper function to process email details including headers, attachments and email downloads + + Args: + action_result: The action result object + email: The email object to process + email_address: The email address of the mailbox + extract_headers: Whether to extract email headers + download_attachments: Whether to download attachments + download_email: Whether to download the email as EML + + Returns: + dict: Updated email object with additional details + """ + try: + # Get headers if requested + if extract_headers: + header_endpoint = f"/users/{email_address}/messages/{email['id']}?$select=internetMessageHeaders" + ret_val, header_response = self._make_rest_call_helper(action_result, header_endpoint) + if phantom.is_fail(ret_val): + self.debug_print(f"Failed to get headers for email {email['id']}") + else: + email["internetMessageHeaders"] = self._flatten_headers(header_response.get("internetMessageHeaders")) + + # Handle attachments if requested + if download_attachments and email.get("hasAttachments"): + attachment_endpoint = f"/users/{email_address}/messages/{email['id']}/attachments" + attach_endpoint = f"{attachment_endpoint}?$expand=microsoft.graph.itemattachment/item" + ret_val, attach_resp = self._make_rest_call_helper(action_result, attach_endpoint) + + if phantom.is_success(ret_val): + for attachment in attach_resp.get("value", []): + if attachment.get("@odata.type") == "#microsoft.graph.fileAttachment": + if not self._handle_attachment(attachment, self.get_container_id()): + self.debug_print(f"Could not process attachment for email {email['id']}") + elif attachment.get("@odata.type") == "#microsoft.graph.itemAttachment": + if not self._handle_item_attachment(attachment, self.get_container_id(), attachment_endpoint, action_result): + self.debug_print(f"Could not process item attachment for email {email['id']}") + email["attachments"] = attach_resp["value"] + + # Download email as EML if requested + if download_email: + subject = email.get("subject") + email_message = { + "id": email["id"], + "name": subject if subject else f"email_message_{email['id']}" + } + if self._handle_item_attachment( + email_message, + self.get_container_id(), + f"/users/{email_address}/messages", + action_result + ): + email["vaultId"] = email_message["vaultId"] + else: + self.debug_print(f"Could not download email {email['id']}") + + return email + + except Exception as e: + self.debug_print(f"Error processing email details: {str(e)}") + return email + def _remove_tokens(self, action_result): # checks whether the message includes any of the known error codes @@ -1994,88 +2058,16 @@ def _handle_get_email(self, param): if phantom.is_fail(ret_val): return action_result.get_status() - if param.get("extract_headers"): - header_endpoint = endpoint + "?$select=internetMessageHeaders" - ret_val, header_response = self._make_rest_call_helper(action_result, header_endpoint) - - if phantom.is_fail(ret_val): - return action_result.get_status() - # For Drafts there might not be any internetMessageHeaders, - # so we have to use get() fetching instead of directly fetching from dictionary - response["internetMessageHeaders"] = header_response.get("internetMessageHeaders") - - if param.get("download_attachments", False) and response.get("hasAttachments"): - endpoint += "/attachments" - attachment_endpoint = "{}?$expand=microsoft.graph.itemattachment/item".format(endpoint) - ret_val, attach_resp = self._make_rest_call_helper(action_result, attachment_endpoint) - - if phantom.is_fail(ret_val): - return action_result.get_status() - - for attachment in attach_resp.get("value", []): - # If it is fileAttachment, then we have to ingest it - if attachment.get("@odata.type") == "#microsoft.graph.fileAttachment": - if not self._handle_attachment(attachment, self.get_container_id()): - return action_result.set_status( - phantom.APP_ERROR, - "Could not process attachment. See logs for details", - ) - elif attachment.get("@odata.type") == "#microsoft.graph.itemAttachment": - if not self._handle_item_attachment(attachment, self.get_container_id(), endpoint, action_result): - return action_result.set_status( - phantom.APP_ERROR, - "Could not process item attachment. See logs for details", - ) - - response["attachments"] = attach_resp["value"] - - if response.get("@odata.type") in [ - "#microsoft.graph.eventMessage", - "#microsoft.graph.eventMessageRequest", - "#microsoft.graph.eventMessageResponse", - ]: - - event_endpoint = "{}/?$expand=Microsoft.Graph.EventMessage/Event".format(endpoint) - ret_val, event_resp = self._make_rest_call_helper(action_result, event_endpoint) - if phantom.is_fail(ret_val): - return action_result.get_status() - - response["event"] = event_resp["event"] - - if "internetMessageHeaders" in response: - response["internetMessageHeaders"] = self._flatten_headers(response["internetMessageHeaders"]) - - # If the response has attachments, update every attachment data with its type - # 'attachmentType' key - indicates type of attachment - # and if an email has any itemAttachment, then also add itemType in the response - # 'itemType' key - indicates type of itemAttachment - if response.get("attachments", []): - for attachment in response["attachments"]: - attachment_type = attachment.get("@odata.type", "") - attachment["attachmentType"] = attachment_type - if attachment_type == "#microsoft.graph.itemAttachment": - attachment["itemType"] = attachment.get("item", {}).get("@odata.type", "") - - if param.get("download_email"): - subject = response.get("subject") - email_message = { - "id": message_id, - "name": subject if subject else "email_message_{}".format(message_id), - } - if not self._handle_item_attachment( - email_message, - self.get_container_id(), - "/users/{0}/messages".format(email_addr), - action_result, - ): - return action_result.set_status( - phantom.APP_ERROR, - "Could not download the email. See logs for details", - ) - response["vaultId"] = email_message["vaultId"] + response = self._process_email_details( + action_result, + response, + email_addr, + extract_headers=param.get("extract_headers"), + download_attachments=param.get("download_attachments", False), + download_email=param.get("download_email", False) + ) action_result.add_data(response) - return action_result.set_status(phantom.APP_SUCCESS, "Successfully fetched email") def _handle_get_email_properties(self, param): @@ -3062,15 +3054,28 @@ def _handle_get_mailbox_messages(self, param): email_address = param["email_address"] folder = param.get("folder", MSGOFFICE365_DEFAULT_FOLDER) limit = param.get("limit", MSGOFFICE365_DEFAULT_LIMIT) + offset = param.get("offset", 0) + ingest = param.get("plus_ingest", False) + download_attachments = param.get("download_attachments", False) + download_email = param.get("download_email", False) + extract_headers = param.get("extract_headers", False) ret_val, limit = _validate_integer(action_result, limit, "'limit' action") if phantom.is_fail(ret_val): return action_result.get_status() + ret_val, offset = _validate_integer(action_result, offset, "'offset' action", allow_zero=True) + if phantom.is_fail(ret_val): + return action_result.get_status() + endpoint = MSGOFFICE365_MAILBOX_MESSAGES_ENDPOINT.format(email_address, folder) - params = {"$top": limit, "$orderby": MSGOFFICE365_ORDERBY_RECEIVED_DESC, "$select": ",".join(MSGOFFICE365_SELECT_PARAMETER_LIST)} + params = { + "$top": limit, + "$orderby": MSGOFFICE365_ORDERBY_RECEIVED_DESC, + "$select": ",".join(MSGOFFICE365_SELECT_PARAMETER_LIST), + "$skip": offset + } - # Add optional filters date_filters = [] if param.get("start_date"): date_filters.append(MSGOFFICE365_RECEIVED_DATE_FILTER.format(operator="ge", date=param.get("start_date"))) @@ -3085,15 +3090,68 @@ def _handle_get_mailbox_messages(self, param): if phantom.is_fail(ret_val): return action_result.get_status() - for message in messages: - action_result.add_data(message) + failed_email_ids = 0 + duplicate_count = 0 + total_emails = len(messages) - summary = action_result.update_summary({}) - summary["total_messages"] = len(messages) + for index, email in enumerate(messages): + try: + # Perform additional processing of attachments/data for email + email = self._process_email_details( + action_result, + email, + email_address, + extract_headers=extract_headers, + download_attachments=download_attachments, + download_email=download_email + ) - return action_result.set_status( - phantom.APP_SUCCESS, f"Successfully retrieved {len(messages)} messages from {email_address}'s {folder} folder" - ) + action_result.add_data(email) + + if ingest: + try: + # Ingest email data + ret_val = self._process_email_data(self.get_config(), action_result, endpoint, email) + if phantom.is_fail(ret_val): + failed_email_ids += 1 + continue + + # Check return message for duplication + if action_result.get_message() == "Duplicate container found": + duplicate_count += 1 + self.debug_print(f"Duplicate container found for email ID: {email.get('id')}") + continue + + except Exception as e: + failed_email_ids += 1 + self.debug_print(f"Exception occurred while processing email ID: {email.get('id')}. Error: {str(e)}") + + except Exception as e: + failed_email_ids += 1 + self.debug_print(f"Exception occurred while processing email ID: {email.get('id')}. Error: {str(e)}") + + if failed_email_ids == total_emails and total_emails > 0: + return action_result.set_status( + phantom.APP_ERROR, + f"Error occurred while processing all {total_emails} email IDs." + ) + + summary = action_result.update_summary({}) + summary["total_messages"] = total_emails + if ingest: + summary["new_emails_ingested"] = total_emails - failed_email_ids - duplicate_count + summary["duplicate_emails"] = duplicate_count + summary["failed_emails"] = failed_email_ids + + status_msg = f"Successfully retrieved {total_emails} messages from {email_address}'s {folder} folder (offset: {offset})" + if ingest: + status_msg += f" and ingested {total_emails - failed_email_ids - duplicate_count} new messages" + if duplicate_count: + status_msg += f" ({duplicate_count} duplicates skipped)" + if failed_email_ids: + status_msg += f" ({failed_email_ids} failed)" + + return action_result.set_status(phantom.APP_SUCCESS, status_msg) def handle_action(self, param): From 2d3583d4d8fd7efe0367225a3d8cab63c4c3ae38 Mon Sep 17 00:00:00 2001 From: grokas Date: Mon, 4 Nov 2024 11:29:20 -0800 Subject: [PATCH 3/9] PAPP-34983 get mailbox messages json and action updated --- office365.json | 434 ++++++++++++++++++++++++++++++++++++++++- office365_connector.py | 13 +- 2 files changed, 439 insertions(+), 8 deletions(-) diff --git a/office365.json b/office365.json index ad86ef4..e2150b5 100644 --- a/office365.json +++ b/office365.json @@ -7939,7 +7939,7 @@ "order": 1 }, "limit": { - "description": "Maximum number of messages to retrieve", + "description": "Maximum number of messages to retrieve (should not exceed 100 per request)", "data_type": "numeric", "default": 100, "order": 2 @@ -8002,6 +8002,10 @@ "data_path": "action_result.parameter.limit", "data_type": "numeric" }, + { + "data_path": "action_result.parameter.offset", + "data_type": "numeric" + }, { "data_path": "action_result.parameter.start_date", "data_type": "string" @@ -8171,6 +8175,422 @@ "data_path": "action_result.data.*.internetMessageHeaders.*.value", "data_type": "string" }, + { + "data_path": "action_result.data.*.internetMessageHeaders.Accept-Language", + "data_type": "string", + "example_values": [ + "en-US" + ] + }, + { + "data_path": "action_result.data.*.internetMessageHeaders.Authentication-Results", + "data_type": "string", + "example_values": [ + "spf=pass (sender IP is 209.85.210.171) smtp.mailfrom=testdomain.com; .abc.com; dkim=pass (signature was verified) header.d=testdomain.com.20150623.gappssmtp.com;.abc.com; dmarc=pass action=none header.from=testdomain.com;compauth=pass reason=100" + ] + }, + { + "data_path": "action_result.data.*.internetMessageHeaders.Content-Language", + "data_type": "string", + "example_values": [ + "en-US" + ] + }, + { + "data_path": "action_result.data.*.internetMessageHeaders.Content-Transfer-Encoding", + "data_type": "string", + "example_values": [ + "binary" + ] + }, + { + "data_path": "action_result.data.*.internetMessageHeaders.Content-Type", + "data_type": "string", + "example_values": [ + "multipart/related" + ] + }, + { + "data_path": "action_result.data.*.internetMessageHeaders.DKIM-Signature", + "data_type": "string", + "example_values": [ + "v=1; a=rsa-sha256; c=relaxed/relaxed; d=testdomain.com.20150623.gappssmtp.com; s=20150623; h=message-id:date:mime-version:from:to:subject; bh=tlTaRbacq4aWozhUPvcWg8i8flbpYQGZNs27nncn83I=; b=avAAeJ8jF08K4oIBhxTirRmyB+SXHwdU0zdxv7eqs/zWaWWcgmT0007KP560TTgo5u oD4nb6TvKxpRyWW4QwmkbuMIwHsMvehd2l1gispV3AawyGJjpmN7ErVYfLtIkz2Tap3V YxmluV+SqeyyxTU8pFAEZ7+2C2lOb1DO5TC7xCMv+dyzevSscJdbeN0dFkG+C93zCqkg w2fxubx2HDD7b/U6m2wXllYhH608wKJ/qYzyvQyqxYqNiQOtPRg2gw4sZ2UgN3+UQyVq 8ubO39ZuqakJpzEzYMw10d6E7SQhvHDJH7mFwhBlzhvOpb2gLJDN8n8dJaZo05BozQqq MsvA==" + ] + }, + { + "data_path": "action_result.data.*.internetMessageHeaders.Date", + "data_type": "string", + "example_values": [ + "Thu, 18 Jun 2020 02:11:26 -0700" + ] + }, + { + "data_path": "action_result.data.*.internetMessageHeaders.From", + "data_type": "string", + "example_values": [ + "\"Test\" " + ] + }, + { + "data_path": "action_result.data.*.internetMessageHeaders.In-Reply-To", + "data_type": "string", + "example_values": [ + "" + ] + }, + { + "data_path": "action_result.data.*.internetMessageHeaders.MIME-Version", + "data_type": "string", + "example_values": [ + "1.0" + ] + }, + { + "data_path": "action_result.data.*.internetMessageHeaders.Message-ID", + "data_type": "string", + "example_values": [ + "<5eeb2fbe.1c69fb81.22b4b.676a@mx.test.com>" + ] + }, + { + "data_path": "action_result.data.*.internetMessageHeaders.Received", + "data_type": "string", + "example_values": [ + "from localhost.localdomain (host-240.test.com. [204.107.141.240]) by tset.abc.com with UTF8SMTPSA id ng12sm1923252pjb.15.2020.06.18.02.11.26 for (version=TLS1_2 cipher=ECDHE-ECDSA-AES128-GCM-SHA256 bits=128/128); Thu, 18 Jun 2020 02:11:26 -0700 (PDT)" + ] + }, + { + "data_path": "action_result.data.*.internetMessageHeaders.Received-SPF", + "data_type": "string", + "example_values": [ + "Pass (protection.test.com: domain of testdomain.com designates 209.85.210.171 as permitted sender) receiver=protection.test.com; client-ip=209.85.210.171; helo=mail-pf1-f171.test.com;" + ] + }, + { + "data_path": "action_result.data.*.internetMessageHeaders.References", + "data_type": "string", + "example_values": [ + "" + ] + }, + { + "data_path": "action_result.data.*.internetMessageHeaders.Return-Path", + "data_type": "string", + "example_values": [ + "notifications@testdomain.com" + ], + "contains": [ + "email" + ] + }, + { + "data_path": "action_result.data.*.internetMessageHeaders.Subject", + "data_type": "string", + "example_values": [ + "Fw: Email having different attachments" + ] + }, + { + "data_path": "action_result.data.*.internetMessageHeaders.Thread-Index", + "data_type": "string", + "example_values": [ + "AQHWZLqyXR4k4Sc6skyFCMPITcMsbKpGS7Bm" + ] + }, + { + "data_path": "action_result.data.*.internetMessageHeaders.Thread-Topic", + "data_type": "string", + "example_values": [ + "Email having different attachments" + ] + }, + { + "data_path": "action_result.data.*.internetMessageHeaders.To", + "data_type": "string", + "example_values": [ + "\"Test\" " + ] + }, + { + "data_path": "action_result.data.*.internetMessageHeaders.X-EOPAttributedMessage", + "data_type": "string", + "example_values": [ + "0" + ] + }, + { + "data_path": "action_result.data.*.internetMessageHeaders.X-EOPTenantAttributedMessage", + "data_type": "string", + "example_values": [ + "a417c578-c7ee-480d-a225-d48057e74df5:0" + ] + }, + { + "data_path": "action_result.data.*.internetMessageHeaders.X-Forefront-Antispam-Report", + "data_type": "string", + "example_values": [ + "CIP:209.85.210.171;CTRY:US;LANG:en;SCL:-1;SRV:;IPV:NLI;SFV:SFE;H:mail-pf1-f171.test.com;PTR:mail-pf1-f171.test.com;CAT:NONE;SFTY:;SFS:;DIR:INB;SFP:;" + ] + }, + { + "data_path": "action_result.data.*.internetMessageHeaders.X-Gm-Message-State", + "data_type": "string", + "example_values": [ + "AOAM533ynFERIhSIewEEkj4b8B1rPNOEeie1IxBdrd55treEMtBa1jkL\tcO5ee4Ff6p0FYedfFtVtHKiCglGTpFTOSw==" + ] + }, + { + "data_path": "action_result.data.*.internetMessageHeaders.X-Google-DKIM-Signature", + "data_type": "string", + "example_values": [ + "v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20161025; h=x-gm-message-state:message-id:date:mime-version:from:to:subject; bh=tlTaRbacq4aWozhUPvcWg8i8flbpYQGZNs27nncn83I=; b=fPT47NIiheeY6GM0bxUOlsmnOgN4WuiOlalFvZqrAiFiOoYk6zrznvgIcAtiHZ4nxE naQAa+mZs5svqRjib3YI52OvR5U8MitIYaa0Rt3LyYSUO1s3iKTUs4nHyRnqPt1skNl7 2OUwsZPXo3ShJDw/uxZRu/cuN1iIfeuE02PrbR04p4D8+1XRslqt/Xqm/bOWKUauqZWe dH1E7meFY01hXxODreO4nWHIhsZgr49TpP/OqRyFcyKHHFFg2sPGXz+QNah6jP4YQUYd Tty2wzOX3nc/YS7TkVo3ORmbzh9o+UZaqH8wHbQlyTdklYxoMPvJwZTo72rTxZeqiJ9E J7PQ==" + ] + }, + { + "data_path": "action_result.data.*.internetMessageHeaders.X-Google-Smtp-Source", + "data_type": "string", + "example_values": [ + "ABdhPJxrYC7raBubCCIOmauxmxryzS9KsihTN6XCRgaNp2rDrG71TVxryzYCtelFOZ2Xj1LzcYIiMA==" + ] + }, + { + "data_path": "action_result.data.*.internetMessageHeaders.X-MS-Exchange-AntiSpam-MessageData", + "data_type": "string", + "example_values": [ + "VSM9HTzub/OH3NCwKXEQqkkzjnhdw5kXsgd9WM0SRgZ0qRdPg5D9/o3LA7lf8ziXc5k0mm9M5mHvFoYePXNXs/MGhGdBGxa/qUQ+FVHA2mDgfPkamJCEZxz//OX/uruTDo+zF4p9D1dQJpnIpx1M75OhuvrHX/BxWWzyAh78DXfF214YHdyFBCYepwl56CS7+fSGQL/r3p+OvWIBnIkISC+HJljSro2k47pPPAkspMhoUkb+zklyENFjez+JcEHYlih2FiNeUO8kb9b7qvlm3zPK98HLspzDh4BojpQ6Ff330iy7nfIK726tCMByxjOdnEQSB9Ua2sbE5gxSeeWL8MB5DHcQSSsXg+sR8w4gXrXLO3meE0lNQKRoAv2b1U0Q+yM0QBqeQWlymZG21bKeuH4gtAFQvfXNjoCtIbBQK1n7ZnL7fI21FJZRcMcKEneus6gLYUqD4PdLEq9FEGbfgiLmVYeUAL2A0Q/gectvL1OVudtHVR5gFMJKt65F1OtS04CPulfLLFSl1F4AzpjjtBSyQcK9R7bOsjoHxQXPMd9fMCzMSIq5f551pO0klKqWY7l11Un2Noj6CA7EtXiD1bTv8JmYQEKR+0HTZagNd+79GeTvKjxTvt9MkyO8k3aqWyNqT331ITnVICtksN1TVMCp8GVeDudNMr2PLSW0alOduR5unuEgTWrqHoaTGOovQx0PVjudNlpZ80ANK9hqaC/ZhLLOtNpJ3fZnjs06PzrPLGhE/IeccY1n8sYDvGm1QA9TN6JaaGPl1Pj6ecy16k0XuF/PKGHTL0M4LCpxSS6T87oFFH1zHkKtmbJp3aAI4bt3ihbQmwFb29JyMgL7ZOy+zrIwXGILh1KQGWQQv1uXXnAuqQy29HeFXs6D2hDHxHlBk5ZQ+vgRtsvRvGnq58vJ3CapjntfL3pOINUj1avLyAZxjasBWMTwaZs9JQ4ZIMekzkIk05lh9XfDSeULk2yKaH8YSCC6ENUHxSWa6pPHJfOdp9kXwOtlp09/VTTAikKy862k9ybN4bRWZB45B9Pv5scna8IX3rthIXUih8c=" + ] + }, + { + "data_path": "action_result.data.*.internetMessageHeaders.X-MS-Exchange-CrossTenant-AuthAs", + "data_type": "string", + "example_values": [ + "Internal" + ] + }, + { + "data_path": "action_result.data.*.internetMessageHeaders.X-MS-Exchange-CrossTenant-AuthSource", + "data_type": "string", + "example_values": [ + "SJ0QA11MB4941.namprd11.prod.test.com" + ] + }, + { + "data_path": "action_result.data.*.internetMessageHeaders.X-MS-Exchange-CrossTenant-FromEntityHeader", + "data_type": "string", + "example_values": [ + "Internet" + ] + }, + { + "data_path": "action_result.data.*.internetMessageHeaders.X-MS-Exchange-CrossTenant-Id", + "data_type": "string", + "example_values": [ + "a417c578-c7ee-480d-a225-d48057e74df5" + ] + }, + { + "data_path": "action_result.data.*.internetMessageHeaders.X-MS-Exchange-CrossTenant-MailboxType", + "data_type": "string", + "example_values": [ + "HOSTED" + ] + }, + { + "data_path": "action_result.data.*.internetMessageHeaders.X-MS-Exchange-CrossTenant-Network-Message-Id", + "data_type": "string", + "example_values": [ + "4b1ef179-4fe7-4248-7ec0-08d81367956e" + ] + }, + { + "data_path": "action_result.data.*.internetMessageHeaders.X-MS-Exchange-CrossTenant-OriginalArrivalTime", + "data_type": "string", + "example_values": [ + "18 Jun 2020 09:11:28.2511 (UTC)" + ] + }, + { + "data_path": "action_result.data.*.internetMessageHeaders.X-MS-Exchange-CrossTenant-UserPrincipalName", + "data_type": "string", + "example_values": [ + "bs91VnpEPjrqCnvlIeymwO6ye4Q8rggHggVNUPUbV/tC9uuFPVFOYg7e/Cd0MeGmSqT4AlLW0Nn4ZeEqNieSf/D1gp5iLz/YkwjXhYUSJnLRb/csQN4sRMMZsX3LUkKkwVpifaeJzoukLu8qSWn7og==" + ] + }, + { + "data_path": "action_result.data.*.internetMessageHeaders.X-MS-Exchange-Organization-AuthAs", + "data_type": "string", + "example_values": [ + "Anonymous" + ] + }, + { + "data_path": "action_result.data.*.internetMessageHeaders.X-MS-Exchange-Organization-AuthMechanism", + "data_type": "string", + "example_values": [ + "04" + ] + }, + { + "data_path": "action_result.data.*.internetMessageHeaders.X-MS-Exchange-Organization-AuthSource", + "data_type": "string", + "example_values": [ + "DM6NAM11FT055.eop-nam11.prod.protection.test.com" + ] + }, + { + "data_path": "action_result.data.*.internetMessageHeaders.X-MS-Exchange-Organization-ExpirationInterval", + "data_type": "string", + "example_values": [ + "1:00:00:00.0000000" + ] + }, + { + "data_path": "action_result.data.*.internetMessageHeaders.X-MS-Exchange-Organization-ExpirationIntervalReason", + "data_type": "string", + "example_values": [ + "OriginalSubmit" + ] + }, + { + "data_path": "action_result.data.*.internetMessageHeaders.X-MS-Exchange-Organization-ExpirationStartTime", + "data_type": "string", + "example_values": [ + "18 Jun 2020 09:11:28.2531 (UTC)" + ] + }, + { + "data_path": "action_result.data.*.internetMessageHeaders.X-MS-Exchange-Organization-ExpirationStartTimeReason", + "data_type": "string", + "example_values": [ + "OriginalSubmit" + ] + }, + { + "data_path": "action_result.data.*.internetMessageHeaders.X-MS-Exchange-Organization-MessageDirectionality", + "data_type": "string", + "example_values": [ + "Incoming" + ] + }, + { + "data_path": "action_result.data.*.internetMessageHeaders.X-MS-Exchange-Organization-Network-Message-Id", + "data_type": "string", + "example_values": [ + "4b1ef179-4fe7-4248-7ec0-08d81367956e" + ] + }, + { + "data_path": "action_result.data.*.internetMessageHeaders.X-MS-Exchange-Organization-SCL", + "data_type": "string", + "example_values": [ + "-1" + ] + }, + { + "data_path": "action_result.data.*.internetMessageHeaders.X-MS-Exchange-Processed-By-BccFoldering", + "data_type": "string", + "example_values": [ + "15.20.3109.017" + ] + }, + { + "data_path": "action_result.data.*.internetMessageHeaders.X-MS-Exchange-Transport-CrossTenantHeadersStamped", + "data_type": "string", + "example_values": [ + "BN6PR18MB1492" + ] + }, + { + "data_path": "action_result.data.*.internetMessageHeaders.X-MS-Exchange-Transport-EndToEndLatency", + "data_type": "string", + "example_values": [ + "00:00:02.7417647" + ] + }, + { + "data_path": "action_result.data.*.internetMessageHeaders.X-MS-Has-Attach", + "data_type": "string", + "example_values": [ + "yes" + ] + }, + { + "data_path": "action_result.data.*.internetMessageHeaders.X-MS-Office365-Filtering-Correlation-Id", + "data_type": "string", + "example_values": [ + "4b1ef179-4fe7-4248-7ec0-08d81367956e" + ] + }, + { + "data_path": "action_result.data.*.internetMessageHeaders.X-MS-Oob-TLC-OOBClassifiers", + "data_type": "string", + "example_values": [ + "OLM:1728;" + ] + }, + { + "data_path": "action_result.data.*.internetMessageHeaders.X-MS-PublicTrafficType", + "data_type": "string", + "example_values": [ + "Email" + ] + }, + { + "data_path": "action_result.data.*.internetMessageHeaders.X-MS-TNEF-Correlator", + "data_type": "string", + "example_values": [ + "" + ] + }, + { + "data_path": "action_result.data.*.internetMessageHeaders.X-MS-TrafficTypeDiagnostic", + "data_type": "string", + "example_values": [ + "BN6PR18MB1492:" + ] + }, + { + "data_path": "action_result.data.*.internetMessageHeaders.X-Microsoft-Antispam", + "data_type": "string", + "example_values": [ + "BCL:0;" + ] + }, + { + "data_path": "action_result.data.*.internetMessageHeaders.X-Microsoft-Antispam-Mailbox-Delivery", + "data_type": "string", + "example_values": [ + "wl:1;pcwl:1;ucf:0;jmr:0;auth:0;dest:I;ENG:(750128)(520011016)(520004050)(702028)(944506458)(944626604);" + ] + }, + { + "data_path": "action_result.data.*.internetMessageHeaders.X-Microsoft-Antispam-Message-Info", + "data_type": "string", + "example_values": [ + "La+CSxAnpzVJOXq7njrFPhIbsh0khleSwldy+W8NYDRsoyyPruPIiId4Avama7JyfzrxoExzhLk5pDn2lGPAJIpdcguiDSsDQg5T+iBCJgFeaEJXjhstECMi842/JGawB9WsiGw9Q/PpvjO5H/2fNLlQZVZW3AAQVZSsX3az4iOsv1Ggj4aYZRMKHmPAtniWOEtQD7zAEWC0jIZf613lWy3vxHfb/3+pV9X8zqPqazbyGy5Q14PICSNkKnvIw8rmeqJV8eSHhvR51Lchib6OIN4xOpLWxkSkBTt5B95RUPnpgPvgp2yLo0Q+EYRIabLDQ0kMsv+24+RnFmr9vo2gRNuFusw8iEPsVEQyhfgIWtBtsBpyvyykxcfa6lIdzQhixZH3Tlkdh1kb15wFS3Ooz3CjaWbY8jcUot5l1p08Ypsj6r7CpIo3xE6jE0x/EeUkDK3Fu/Ol0pOsJ1N5W4iJLdjqSQM3l/t9QWlcPhD8s6D7D7JM5OUHCeFEPr7sSL+P/5zTgBaeUvwtZrlQSH2GHc+5gPW8rkwlwJLJftVEid0gO2PUOrzItzME5PXYAcdx++sF3XC1YMPLet/jMpX8T7/z7+hxFxNyifgmGJ+DkNOec7yGkkcLBz6iCaHx7OrRGwDHIcdAtV85wCk3NEDDiKyHivQpwp/gY55W+wkLe7aqSHmFzm1rUSslx+DWz8w2EgSjJxOmf0JkoNKbTFl3FObkocR0lUUQUnETuoAXUqvpWGD5B69W9XXUM8c43ozz2oBZseheSAtkLil3tMIr/CMCMILPX/LdoErNtkmiFXCPqaLFSSeyO61oCMl6Ezndtwp22nwMPUg5ofG0kdqFuTW122umhy9C6h5BcREaLhWclSyqDoZPB9RvkRlI2kTRwuwbuFW3iOMzmVwxLIQH9K5JkxdMvC3hvNpjVgz7Q2ZnEF3xSNqeoWVQvkaIe8rQLUc8s+HMRUmSERGdfSuQJAx47g8PDs9s3rS/ThUSzIaljJPbUgXEnFg/G6h3I/yXLj2Nj2OG50snoI5jJmE4+69YmNwasdDZuYpnuQeFgu11HtsLniDthJdjEJyYC1utZNt9hgA+6JlLnm7Dxb43cSIiW8ev+3X+b2kREj2k/m8fSz7YgtoCB8AkuiVXRaH3EUiq8XCExbbWeynKRgwCZ6bzvfSiT3+cg+QQKPHFc/cgot56ta6X80tjhFodpTQNTE6V6C9QFHJ3JCVhsSzVifJAc8crI5hAcPbKFEIjinENcfpF/8reo2Yr1xFElhoX" + ] + }, + { + "data_path": "action_result.data.*.internetMessageHeaders.X-Originating-IP", + "data_type": "string", + "example_values": [ + "[2.39.180.162]" + ] + }, + { + "data_path": "action_result.data.*.internetMessageHeaders.X-Received", + "data_type": "string", + "example_values": [ + "by 2002:aa7:84d9:: with SMTP id x25mr2807688pfn.300.1592471487394; Thu, 18 Jun 2020 02:11:27 -0700 (PDT)" + ] + }, + { + "data_path": "action_result.data.*.internetMessageHeaders.subject", + "data_type": "string", + "example_values": [ + "test html" + ] + }, { "data_path": "action_result.data.*.isReadReceiptRequested", "data_type": "boolean" @@ -8187,6 +8607,18 @@ "data_path": "action_result.summary.total_messages", "data_type": "numeric" }, + { + "data_path": "action_result.summary.duplicate_emails", + "data_type": "numeric" + }, + { + "data_path": "action_result.summary.failed_emails", + "data_type": "numeric" + }, + { + "data_path": "action_result.summary.new_emails_ingested", + "data_type": "numeric" + }, { "data_path": "action_result.message", "data_type": "string" diff --git a/office365_connector.py b/office365_connector.py index 52ef8ad..2c720f2 100644 --- a/office365_connector.py +++ b/office365_connector.py @@ -3064,6 +3064,10 @@ def _handle_get_mailbox_messages(self, param): if phantom.is_fail(ret_val): return action_result.get_status() + # Limit should not exceed 100 per request for timeout reasons + if limit > 100: + return action_result.set_status(phantom.APP_ERROR, "Limit should not exceed 100 messages per request") + ret_val, offset = _validate_integer(action_result, offset, "'offset' action", allow_zero=True) if phantom.is_fail(ret_val): return action_result.get_status() @@ -3091,7 +3095,7 @@ def _handle_get_mailbox_messages(self, param): return action_result.get_status() failed_email_ids = 0 - duplicate_count = 0 + duplicate_count = self._duplicate_count total_emails = len(messages) for index, email in enumerate(messages): @@ -3116,12 +3120,6 @@ def _handle_get_mailbox_messages(self, param): failed_email_ids += 1 continue - # Check return message for duplication - if action_result.get_message() == "Duplicate container found": - duplicate_count += 1 - self.debug_print(f"Duplicate container found for email ID: {email.get('id')}") - continue - except Exception as e: failed_email_ids += 1 self.debug_print(f"Exception occurred while processing email ID: {email.get('id')}. Error: {str(e)}") @@ -3139,6 +3137,7 @@ def _handle_get_mailbox_messages(self, param): summary = action_result.update_summary({}) summary["total_messages"] = total_emails if ingest: + duplicate_count = self._duplicate_count - duplicate_count summary["new_emails_ingested"] = total_emails - failed_email_ids - duplicate_count summary["duplicate_emails"] = duplicate_count summary["failed_emails"] = failed_email_ids From ee34b0e272d7038f6425d36dca88ee63658ff528 Mon Sep 17 00:00:00 2001 From: grokas Date: Mon, 4 Nov 2024 12:26:57 -0800 Subject: [PATCH 4/9] PAPP-34983 formatting --- office365.json | 2 +- office365_connector.py | 27 +++++++++------------------ 2 files changed, 10 insertions(+), 19 deletions(-) diff --git a/office365.json b/office365.json index e2150b5..d93c94e 100644 --- a/office365.json +++ b/office365.json @@ -8017,7 +8017,7 @@ { "data_path": "action_result.data.*.id", "data_type": "string", - "column_name": "ID", + "column_name": "MESSAGE ID", "column_order": 0 }, { diff --git a/office365_connector.py b/office365_connector.py index 2c720f2..2b10ae0 100644 --- a/office365_connector.py +++ b/office365_connector.py @@ -1294,7 +1294,9 @@ def _process_email_data(self, config, action_result, endpoint, email): return phantom.APP_SUCCESS - def _process_email_details(self, action_result, email, email_address, extract_headers=False, download_attachments=False, download_email=False): + def _process_email_details( + self, action_result, email, email_address, extract_headers=False, download_attachments=False, download_email=False + ): """ Helper function to process email details including headers, attachments and email downloads @@ -1338,16 +1340,8 @@ def _process_email_details(self, action_result, email, email_address, extract_he # Download email as EML if requested if download_email: subject = email.get("subject") - email_message = { - "id": email["id"], - "name": subject if subject else f"email_message_{email['id']}" - } - if self._handle_item_attachment( - email_message, - self.get_container_id(), - f"/users/{email_address}/messages", - action_result - ): + email_message = {"id": email["id"], "name": subject if subject else f"email_message_{email['id']}"} + if self._handle_item_attachment(email_message, self.get_container_id(), f"/users/{email_address}/messages", action_result): email["vaultId"] = email_message["vaultId"] else: self.debug_print(f"Could not download email {email['id']}") @@ -2064,7 +2058,7 @@ def _handle_get_email(self, param): email_addr, extract_headers=param.get("extract_headers"), download_attachments=param.get("download_attachments", False), - download_email=param.get("download_email", False) + download_email=param.get("download_email", False), ) action_result.add_data(response) @@ -3077,7 +3071,7 @@ def _handle_get_mailbox_messages(self, param): "$top": limit, "$orderby": MSGOFFICE365_ORDERBY_RECEIVED_DESC, "$select": ",".join(MSGOFFICE365_SELECT_PARAMETER_LIST), - "$skip": offset + "$skip": offset, } date_filters = [] @@ -3107,7 +3101,7 @@ def _handle_get_mailbox_messages(self, param): email_address, extract_headers=extract_headers, download_attachments=download_attachments, - download_email=download_email + download_email=download_email, ) action_result.add_data(email) @@ -3129,10 +3123,7 @@ def _handle_get_mailbox_messages(self, param): self.debug_print(f"Exception occurred while processing email ID: {email.get('id')}. Error: {str(e)}") if failed_email_ids == total_emails and total_emails > 0: - return action_result.set_status( - phantom.APP_ERROR, - f"Error occurred while processing all {total_emails} email IDs." - ) + return action_result.set_status(phantom.APP_ERROR, f"Error occurred while processing all {total_emails} email IDs.") summary = action_result.update_summary({}) summary["total_messages"] = total_emails From 8a6135409eb543da5e520c9bac14a49846dd948d Mon Sep 17 00:00:00 2001 From: splunk-soar-connectors-admin Date: Mon, 4 Nov 2024 20:27:39 +0000 Subject: [PATCH 5/9] Update README.md --- README.md | 140 +++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 139 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 5121f71..6e7700d 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ # MS Graph for Office 365 Publisher: Splunk -Connector Version: 3.0.1 +Connector Version: 3.1.1 Product Vendor: Microsoft Product Name: Office 365 (MS Graph) Product Version Supported (regex): ".\*" @@ -354,6 +354,7 @@ VARIABLE | REQUIRED | TYPE | DESCRIPTION [block sender](#action-block-sender) - Add the sender email into the block list [unblock sender](#action-unblock-sender) - Remove the sender email from the block list [resolve name](#action-resolve-name) - Verify aliases and resolve display names to the appropriate user +[get mailbox messages](#action-get-mailbox-messages) - Retrieves messages from a specified mailbox folder ## action: 'test connectivity' Use supplied credentials to generate a token with MS Graph @@ -1685,4 +1686,141 @@ action_result.summary | string | | action_result.status | string | | success failed action_result.message | string | | summary.total_objects | numeric | | +summary.total_objects_successful | numeric | | + +## action: 'get mailbox messages' +Retrieves messages from a specified mailbox folder + +Type: **investigate** +Read only: **True** + +#### Action Parameters +PARAMETER | REQUIRED | DESCRIPTION | TYPE | CONTAINS +--------- | -------- | ----------- | ---- | -------- +**email_address** | required | Email address of the mailbox | string | +**folder** | optional | Folder to retrieve messages | string | +**limit** | optional | Maximum number of messages to retrieve (should not exceed 100 per request) | numeric | +**offset** | optional | Number of messages to skip before retrieving results | numeric | +**start_date** | optional | Start date for filtering messages (format: YYYY-MM-DD) | string | +**end_date** | optional | End date for filtering messages (format: YYYY-MM-DD) | string | +**download_attachments** | optional | Download email attachments to vault | boolean | +**download_email** | optional | Download email as EML file to vault | boolean | +**extract_headers** | optional | Include email headers in results | boolean | +**plus_ingest** | optional | If enabled, messages will be also ingested like on_poll | boolean | + +#### Action Output +DATA PATH | TYPE | CONTAINS | EXAMPLE VALUES +--------- | ---- | -------- | -------------- +action_result.status | string | | success failed +action_result.parameter.email_address | string | | +action_result.parameter.folder | string | | +action_result.parameter.limit | numeric | | +action_result.parameter.offset | numeric | | +action_result.parameter.start_date | string | | +action_result.parameter.end_date | string | | +action_result.data.\*.id | string | | +action_result.data.\*.body.content | string | | +action_result.data.\*.body.contentType | string | | +action_result.data.\*.flag.flagStatus | string | | +action_result.data.\*.from.emailAddress.name | string | | +action_result.data.\*.from.emailAddress.address | string | | +action_result.data.\*.isRead | boolean | | +action_result.data.\*.sender.emailAddress.name | string | | +action_result.data.\*.sender.emailAddress.address | string | | +action_result.data.\*.isDraft | boolean | | +action_result.data.\*.replyTo.\*.emailAddress.address | string | | +action_result.data.\*.replyTo.\*.emailAddress.name | string | | +action_result.data.\*.subject | string | | +action_result.data.\*.webLink | string | `url` | +action_result.data.\*.changeKey | string | | +action_result.data.\*.categories.\*.name | string | | +action_result.data.\*.importance | string | | +action_result.data.\*.uniqueBody.content | string | | +action_result.data.\*.uniqueBody.contentType | string | | +action_result.data.\*.bodyPreview | string | | +action_result.data.\*.ccRecipients.\*.emailAddress.address | string | | +action_result.data.\*.ccRecipients.\*.emailAddress.name | string | | +action_result.data.\*.sentDateTime | string | | +action_result.data.\*.toRecipients.\*.emailAddress.name | string | | +action_result.data.\*.toRecipients.\*.emailAddress.address | string | | +action_result.data.\*.bccRecipients.\*.emailAddress.address | string | | +action_result.data.\*.bccRecipients.\*.emailAddress.name | string | | +action_result.data.\*.conversationId | string | | +action_result.data.\*.hasAttachments | boolean | | +action_result.data.\*.parentFolderId | string | | +action_result.data.\*.createdDateTime | string | | +action_result.data.\*.receivedDateTime | string | | +action_result.data.\*.conversationIndex | string | | +action_result.data.\*.internetMessageId | string | | +action_result.data.\*.lastModifiedDateTime | string | | +action_result.data.\*.internetMessageHeaders.\*.name | string | | +action_result.data.\*.internetMessageHeaders.\*.value | string | | +action_result.data.\*.internetMessageHeaders.Accept-Language | string | | en-US +action_result.data.\*.internetMessageHeaders.Authentication-Results | string | | spf=pass (sender IP is 209.85.210.171) smtp.mailfrom=testdomain.com; .abc.com; dkim=pass (signature was verified) header.d=testdomain.com.20150623.gappssmtp.com;.abc.com; dmarc=pass action=none header.from=testdomain.com;compauth=pass reason=100 +action_result.data.\*.internetMessageHeaders.Content-Language | string | | en-US +action_result.data.\*.internetMessageHeaders.Content-Transfer-Encoding | string | | binary +action_result.data.\*.internetMessageHeaders.Content-Type | string | | multipart/related +action_result.data.\*.internetMessageHeaders.DKIM-Signature | string | | v=1; a=rsa-sha256; c=relaxed/relaxed; d=testdomain.com.20150623.gappssmtp.com; s=20150623; h=message-id:date:mime-version:from:to:subject; bh=tlTaRbacq4aWozhUPvcWg8i8flbpYQGZNs27nncn83I=; b=avAAeJ8jF08K4oIBhxTirRmyB+SXHwdU0zdxv7eqs/zWaWWcgmT0007KP560TTgo5u oD4nb6TvKxpRyWW4QwmkbuMIwHsMvehd2l1gispV3AawyGJjpmN7ErVYfLtIkz2Tap3V YxmluV+SqeyyxTU8pFAEZ7+2C2lOb1DO5TC7xCMv+dyzevSscJdbeN0dFkG+C93zCqkg w2fxubx2HDD7b/U6m2wXllYhH608wKJ/qYzyvQyqxYqNiQOtPRg2gw4sZ2UgN3+UQyVq 8ubO39ZuqakJpzEzYMw10d6E7SQhvHDJH7mFwhBlzhvOpb2gLJDN8n8dJaZo05BozQqq MsvA== +action_result.data.\*.internetMessageHeaders.Date | string | | Thu, 18 Jun 2020 02:11:26 -0700 +action_result.data.\*.internetMessageHeaders.From | string | | "Test" +action_result.data.\*.internetMessageHeaders.In-Reply-To | string | | +action_result.data.\*.internetMessageHeaders.MIME-Version | string | | 1.0 +action_result.data.\*.internetMessageHeaders.Message-ID | string | | <5eeb2fbe.1c69fb81.22b4b.676a@mx.test.com> +action_result.data.\*.internetMessageHeaders.Received | string | | from localhost.localdomain (host-240.test.com. [204.107.141.240]) by tset.abc.com with UTF8SMTPSA id ng12sm1923252pjb.15.2020.06.18.02.11.26 for (version=TLS1_2 cipher=ECDHE-ECDSA-AES128-GCM-SHA256 bits=128/128); Thu, 18 Jun 2020 02:11:26 -0700 (PDT) +action_result.data.\*.internetMessageHeaders.Received-SPF | string | | Pass (protection.test.com: domain of testdomain.com designates 209.85.210.171 as permitted sender) receiver=protection.test.com; client-ip=209.85.210.171; helo=mail-pf1-f171.test.com; +action_result.data.\*.internetMessageHeaders.References | string | | +action_result.data.\*.internetMessageHeaders.Return-Path | string | `email` | notifications@testdomain.com +action_result.data.\*.internetMessageHeaders.Subject | string | | Fw: Email having different attachments +action_result.data.\*.internetMessageHeaders.Thread-Index | string | | AQHWZLqyXR4k4Sc6skyFCMPITcMsbKpGS7Bm +action_result.data.\*.internetMessageHeaders.Thread-Topic | string | | Email having different attachments +action_result.data.\*.internetMessageHeaders.To | string | | "Test" +action_result.data.\*.internetMessageHeaders.X-EOPAttributedMessage | string | | 0 +action_result.data.\*.internetMessageHeaders.X-EOPTenantAttributedMessage | string | | a417c578-c7ee-480d-a225-d48057e74df5:0 +action_result.data.\*.internetMessageHeaders.X-Forefront-Antispam-Report | string | | CIP:209.85.210.171;CTRY:US;LANG:en;SCL:-1;SRV:;IPV:NLI;SFV:SFE;H:mail-pf1-f171.test.com;PTR:mail-pf1-f171.test.com;CAT:NONE;SFTY:;SFS:;DIR:INB;SFP:; +action_result.data.\*.internetMessageHeaders.X-Gm-Message-State | string | | AOAM533ynFERIhSIewEEkj4b8B1rPNOEeie1IxBdrd55treEMtBa1jkL cO5ee4Ff6p0FYedfFtVtHKiCglGTpFTOSw== +action_result.data.\*.internetMessageHeaders.X-Google-DKIM-Signature | string | | v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20161025; h=x-gm-message-state:message-id:date:mime-version:from:to:subject; bh=tlTaRbacq4aWozhUPvcWg8i8flbpYQGZNs27nncn83I=; b=fPT47NIiheeY6GM0bxUOlsmnOgN4WuiOlalFvZqrAiFiOoYk6zrznvgIcAtiHZ4nxE naQAa+mZs5svqRjib3YI52OvR5U8MitIYaa0Rt3LyYSUO1s3iKTUs4nHyRnqPt1skNl7 2OUwsZPXo3ShJDw/uxZRu/cuN1iIfeuE02PrbR04p4D8+1XRslqt/Xqm/bOWKUauqZWe dH1E7meFY01hXxODreO4nWHIhsZgr49TpP/OqRyFcyKHHFFg2sPGXz+QNah6jP4YQUYd Tty2wzOX3nc/YS7TkVo3ORmbzh9o+UZaqH8wHbQlyTdklYxoMPvJwZTo72rTxZeqiJ9E J7PQ== +action_result.data.\*.internetMessageHeaders.X-Google-Smtp-Source | string | | ABdhPJxrYC7raBubCCIOmauxmxryzS9KsihTN6XCRgaNp2rDrG71TVxryzYCtelFOZ2Xj1LzcYIiMA== +action_result.data.\*.internetMessageHeaders.X-MS-Exchange-AntiSpam-MessageData | string | | VSM9HTzub/OH3NCwKXEQqkkzjnhdw5kXsgd9WM0SRgZ0qRdPg5D9/o3LA7lf8ziXc5k0mm9M5mHvFoYePXNXs/MGhGdBGxa/qUQ+FVHA2mDgfPkamJCEZxz//OX/uruTDo+zF4p9D1dQJpnIpx1M75OhuvrHX/BxWWzyAh78DXfF214YHdyFBCYepwl56CS7+fSGQL/r3p+OvWIBnIkISC+HJljSro2k47pPPAkspMhoUkb+zklyENFjez+JcEHYlih2FiNeUO8kb9b7qvlm3zPK98HLspzDh4BojpQ6Ff330iy7nfIK726tCMByxjOdnEQSB9Ua2sbE5gxSeeWL8MB5DHcQSSsXg+sR8w4gXrXLO3meE0lNQKRoAv2b1U0Q+yM0QBqeQWlymZG21bKeuH4gtAFQvfXNjoCtIbBQK1n7ZnL7fI21FJZRcMcKEneus6gLYUqD4PdLEq9FEGbfgiLmVYeUAL2A0Q/gectvL1OVudtHVR5gFMJKt65F1OtS04CPulfLLFSl1F4AzpjjtBSyQcK9R7bOsjoHxQXPMd9fMCzMSIq5f551pO0klKqWY7l11Un2Noj6CA7EtXiD1bTv8JmYQEKR+0HTZagNd+79GeTvKjxTvt9MkyO8k3aqWyNqT331ITnVICtksN1TVMCp8GVeDudNMr2PLSW0alOduR5unuEgTWrqHoaTGOovQx0PVjudNlpZ80ANK9hqaC/ZhLLOtNpJ3fZnjs06PzrPLGhE/IeccY1n8sYDvGm1QA9TN6JaaGPl1Pj6ecy16k0XuF/PKGHTL0M4LCpxSS6T87oFFH1zHkKtmbJp3aAI4bt3ihbQmwFb29JyMgL7ZOy+zrIwXGILh1KQGWQQv1uXXnAuqQy29HeFXs6D2hDHxHlBk5ZQ+vgRtsvRvGnq58vJ3CapjntfL3pOINUj1avLyAZxjasBWMTwaZs9JQ4ZIMekzkIk05lh9XfDSeULk2yKaH8YSCC6ENUHxSWa6pPHJfOdp9kXwOtlp09/VTTAikKy862k9ybN4bRWZB45B9Pv5scna8IX3rthIXUih8c= +action_result.data.\*.internetMessageHeaders.X-MS-Exchange-CrossTenant-AuthAs | string | | Internal +action_result.data.\*.internetMessageHeaders.X-MS-Exchange-CrossTenant-AuthSource | string | | SJ0QA11MB4941.namprd11.prod.test.com +action_result.data.\*.internetMessageHeaders.X-MS-Exchange-CrossTenant-FromEntityHeader | string | | Internet +action_result.data.\*.internetMessageHeaders.X-MS-Exchange-CrossTenant-Id | string | | a417c578-c7ee-480d-a225-d48057e74df5 +action_result.data.\*.internetMessageHeaders.X-MS-Exchange-CrossTenant-MailboxType | string | | HOSTED +action_result.data.\*.internetMessageHeaders.X-MS-Exchange-CrossTenant-Network-Message-Id | string | | 4b1ef179-4fe7-4248-7ec0-08d81367956e +action_result.data.\*.internetMessageHeaders.X-MS-Exchange-CrossTenant-OriginalArrivalTime | string | | 18 Jun 2020 09:11:28.2511 (UTC) +action_result.data.\*.internetMessageHeaders.X-MS-Exchange-CrossTenant-UserPrincipalName | string | | bs91VnpEPjrqCnvlIeymwO6ye4Q8rggHggVNUPUbV/tC9uuFPVFOYg7e/Cd0MeGmSqT4AlLW0Nn4ZeEqNieSf/D1gp5iLz/YkwjXhYUSJnLRb/csQN4sRMMZsX3LUkKkwVpifaeJzoukLu8qSWn7og== +action_result.data.\*.internetMessageHeaders.X-MS-Exchange-Organization-AuthAs | string | | Anonymous +action_result.data.\*.internetMessageHeaders.X-MS-Exchange-Organization-AuthMechanism | string | | 04 +action_result.data.\*.internetMessageHeaders.X-MS-Exchange-Organization-AuthSource | string | | DM6NAM11FT055.eop-nam11.prod.protection.test.com +action_result.data.\*.internetMessageHeaders.X-MS-Exchange-Organization-ExpirationInterval | string | | 1:00:00:00.0000000 +action_result.data.\*.internetMessageHeaders.X-MS-Exchange-Organization-ExpirationIntervalReason | string | | OriginalSubmit +action_result.data.\*.internetMessageHeaders.X-MS-Exchange-Organization-ExpirationStartTime | string | | 18 Jun 2020 09:11:28.2531 (UTC) +action_result.data.\*.internetMessageHeaders.X-MS-Exchange-Organization-ExpirationStartTimeReason | string | | OriginalSubmit +action_result.data.\*.internetMessageHeaders.X-MS-Exchange-Organization-MessageDirectionality | string | | Incoming +action_result.data.\*.internetMessageHeaders.X-MS-Exchange-Organization-Network-Message-Id | string | | 4b1ef179-4fe7-4248-7ec0-08d81367956e +action_result.data.\*.internetMessageHeaders.X-MS-Exchange-Organization-SCL | string | | -1 +action_result.data.\*.internetMessageHeaders.X-MS-Exchange-Processed-By-BccFoldering | string | | 15.20.3109.017 +action_result.data.\*.internetMessageHeaders.X-MS-Exchange-Transport-CrossTenantHeadersStamped | string | | BN6PR18MB1492 +action_result.data.\*.internetMessageHeaders.X-MS-Exchange-Transport-EndToEndLatency | string | | 00:00:02.7417647 +action_result.data.\*.internetMessageHeaders.X-MS-Has-Attach | string | | yes +action_result.data.\*.internetMessageHeaders.X-MS-Office365-Filtering-Correlation-Id | string | | 4b1ef179-4fe7-4248-7ec0-08d81367956e +action_result.data.\*.internetMessageHeaders.X-MS-Oob-TLC-OOBClassifiers | string | | OLM:1728; +action_result.data.\*.internetMessageHeaders.X-MS-PublicTrafficType | string | | Email +action_result.data.\*.internetMessageHeaders.X-MS-TNEF-Correlator | string | | +action_result.data.\*.internetMessageHeaders.X-MS-TrafficTypeDiagnostic | string | | BN6PR18MB1492: +action_result.data.\*.internetMessageHeaders.X-Microsoft-Antispam | string | | BCL:0; +action_result.data.\*.internetMessageHeaders.X-Microsoft-Antispam-Mailbox-Delivery | string | | wl:1;pcwl:1;ucf:0;jmr:0;auth:0;dest:I;ENG:(750128)(520011016)(520004050)(702028)(944506458)(944626604); +action_result.data.\*.internetMessageHeaders.X-Microsoft-Antispam-Message-Info | string | | La+CSxAnpzVJOXq7njrFPhIbsh0khleSwldy+W8NYDRsoyyPruPIiId4Avama7JyfzrxoExzhLk5pDn2lGPAJIpdcguiDSsDQg5T+iBCJgFeaEJXjhstECMi842/JGawB9WsiGw9Q/PpvjO5H/2fNLlQZVZW3AAQVZSsX3az4iOsv1Ggj4aYZRMKHmPAtniWOEtQD7zAEWC0jIZf613lWy3vxHfb/3+pV9X8zqPqazbyGy5Q14PICSNkKnvIw8rmeqJV8eSHhvR51Lchib6OIN4xOpLWxkSkBTt5B95RUPnpgPvgp2yLo0Q+EYRIabLDQ0kMsv+24+RnFmr9vo2gRNuFusw8iEPsVEQyhfgIWtBtsBpyvyykxcfa6lIdzQhixZH3Tlkdh1kb15wFS3Ooz3CjaWbY8jcUot5l1p08Ypsj6r7CpIo3xE6jE0x/EeUkDK3Fu/Ol0pOsJ1N5W4iJLdjqSQM3l/t9QWlcPhD8s6D7D7JM5OUHCeFEPr7sSL+P/5zTgBaeUvwtZrlQSH2GHc+5gPW8rkwlwJLJftVEid0gO2PUOrzItzME5PXYAcdx++sF3XC1YMPLet/jMpX8T7/z7+hxFxNyifgmGJ+DkNOec7yGkkcLBz6iCaHx7OrRGwDHIcdAtV85wCk3NEDDiKyHivQpwp/gY55W+wkLe7aqSHmFzm1rUSslx+DWz8w2EgSjJxOmf0JkoNKbTFl3FObkocR0lUUQUnETuoAXUqvpWGD5B69W9XXUM8c43ozz2oBZseheSAtkLil3tMIr/CMCMILPX/LdoErNtkmiFXCPqaLFSSeyO61oCMl6Ezndtwp22nwMPUg5ofG0kdqFuTW122umhy9C6h5BcREaLhWclSyqDoZPB9RvkRlI2kTRwuwbuFW3iOMzmVwxLIQH9K5JkxdMvC3hvNpjVgz7Q2ZnEF3xSNqeoWVQvkaIe8rQLUc8s+HMRUmSERGdfSuQJAx47g8PDs9s3rS/ThUSzIaljJPbUgXEnFg/G6h3I/yXLj2Nj2OG50snoI5jJmE4+69YmNwasdDZuYpnuQeFgu11HtsLniDthJdjEJyYC1utZNt9hgA+6JlLnm7Dxb43cSIiW8ev+3X+b2kREj2k/m8fSz7YgtoCB8AkuiVXRaH3EUiq8XCExbbWeynKRgwCZ6bzvfSiT3+cg+QQKPHFc/cgot56ta6X80tjhFodpTQNTE6V6C9QFHJ3JCVhsSzVifJAc8crI5hAcPbKFEIjinENcfpF/8reo2Yr1xFElhoX +action_result.data.\*.internetMessageHeaders.X-Originating-IP | string | | [2.39.180.162] +action_result.data.\*.internetMessageHeaders.X-Received | string | | by 2002:aa7:84d9:: with SMTP id x25mr2807688pfn.300.1592471487394; Thu, 18 Jun 2020 02:11:27 -0700 (PDT) +action_result.data.\*.internetMessageHeaders.subject | string | | test html +action_result.data.\*.isReadReceiptRequested | boolean | | +action_result.data.\*.inferenceClassification | string | | +action_result.data.\*.isDeliveryReceiptRequested | boolean | | +action_result.summary.total_messages | numeric | | +action_result.summary.duplicate_emails | numeric | | +action_result.summary.failed_emails | numeric | | +action_result.summary.new_emails_ingested | numeric | | +action_result.message | string | | +summary.total_objects | numeric | | summary.total_objects_successful | numeric | | \ No newline at end of file From a1e313ea66f1af7af0010216686d17fa45c590a2 Mon Sep 17 00:00:00 2001 From: grokas Date: Mon, 4 Nov 2024 17:10:17 -0800 Subject: [PATCH 6/9] PAPP-34983 release notes added --- release_notes/unreleased.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/release_notes/unreleased.md b/release_notes/unreleased.md index fbcb2fd..01d1fb3 100644 --- a/release_notes/unreleased.md +++ b/release_notes/unreleased.md @@ -1 +1,3 @@ **Unreleased** + +* Added 'get mailbox messages' action to retrieve emails from a user's mailbox in an advanced manner. Supporting bulk messages date filtering, pagination, attachment downloads, EML file extraction, and ingestion. From a627cbfd0556dd088abae721e61c555f0eab3227 Mon Sep 17 00:00:00 2001 From: grokas Date: Mon, 4 Nov 2024 17:25:53 -0800 Subject: [PATCH 7/9] PAPP-34983 static test var name fix --- office365_connector.py | 6 +++--- office365_consts.py | 5 ++--- process_email.py | 4 ++-- 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/office365_connector.py b/office365_connector.py index 2b10ae0..eba433c 100644 --- a/office365_connector.py +++ b/office365_connector.py @@ -158,7 +158,7 @@ def _get_error_msg_from_exception(e, app_connector=None): :return: error message """ error_code = None - error_msg = ERROR_MSG_UNAVAILABLE + error_msg = MSGOFFICE365_ERROR_MSG_UNAVAILABLE if app_connector: app_connector.error_print("Error occurred.", dump_object=e) @@ -777,7 +777,7 @@ def _make_rest_call_helper( # If token is expired, generate a new token msg = action_result.get_message() - if msg and (("token" in msg and "expired" in msg) or any(failure_msg in msg for failure_msg in AUTH_FAILURE_MSG)): + if msg and (("token" in msg and "expired" in msg) or any(failure_msg in msg for failure_msg in MSGOFFICE365_AUTH_FAILURE_MSG)): self.debug_print("MSGRAPH", f"Error '{msg}' found in API response. Requesting new access token using refresh token") ret_val = self._get_token(action_result) if phantom.is_fail(ret_val): @@ -3066,7 +3066,7 @@ def _handle_get_mailbox_messages(self, param): if phantom.is_fail(ret_val): return action_result.get_status() - endpoint = MSGOFFICE365_MAILBOX_MESSAGES_ENDPOINT.format(email_address, folder) + endpoint = f"/users/{email_address}/mailFolders/{folder}/messages" params = { "$top": limit, "$orderby": MSGOFFICE365_ORDERBY_RECEIVED_DESC, diff --git a/office365_consts.py b/office365_consts.py index 941d2f1..cfb9131 100644 --- a/office365_consts.py +++ b/office365_consts.py @@ -69,12 +69,12 @@ # Constants relating to '_get_error_message_from_exception' -ERROR_MSG_UNAVAILABLE = "Error msg unavailable. Please check the asset configuration and|or action parameters" +MSGOFFICE365_ERROR_MSG_UNAVAILABLE = "Error msg unavailable. Please check the asset configuration and|or action parameters" # Constants relating to 'validate_integer' MSGOFFICE365_VALID_INT_MSG = "Please provide a valid integer value in the {param} parameter" MSGOFFICE365_NON_NEG_NON_ZERO_INT_MSG = "Please provide a valid non-zero positive integer value in the {param} parameter" -AUTH_FAILURE_MSG = [ +MSGOFFICE365_AUTH_FAILURE_MSG = [ "token is invalid", "Access token has expired", "ExpiredAuthenticationToken", @@ -124,7 +124,6 @@ "internetMessageId", ] -MSGOFFICE365_MAILBOX_MESSAGES_ENDPOINT = "/users/{}/mailFolders/{}/messages" MSGOFFICE365_DEFAULT_FOLDER = "Inbox" MSGOFFICE365_DEFAULT_LIMIT = 100 MSGOFFICE365_ORDERBY_RECEIVED_DESC = "receivedDateTime desc" diff --git a/process_email.py b/process_email.py index 616eefe..e449098 100644 --- a/process_email.py +++ b/process_email.py @@ -36,7 +36,7 @@ from phantom.vault import Vault from requests.structures import CaseInsensitiveDict -from office365_consts import ERROR_MSG_UNAVAILABLE +from office365_consts import MSGOFFICE365_ERROR_MSG_UNAVAILABLE _container_common = {"run_automation": False} # Don't run any playbooks, when this artifact is added @@ -136,7 +136,7 @@ def _get_error_msg_from_exception(e): """ error_code = None - error_msg = ERROR_MSG_UNAVAILABLE + error_msg = MSGOFFICE365_ERROR_MSG_UNAVAILABLE try: if hasattr(e, "args"): From cd7514278e3bf1ec1e55454d28cdac4757a8a56e Mon Sep 17 00:00:00 2001 From: grokas Date: Wed, 6 Nov 2024 00:00:35 -0800 Subject: [PATCH 8/9] PAPP-34983 function cleanup --- office365.json | 2 +- office365_connector.py | 146 +++++++++++++++++++++++++++-------------- 2 files changed, 96 insertions(+), 52 deletions(-) diff --git a/office365.json b/office365.json index d93c94e..65e462f 100644 --- a/office365.json +++ b/office365.json @@ -7921,7 +7921,7 @@ { "action": "get mailbox messages", "identifier": "get_mailbox_messages", - "description": "Retrieves messages from a specified mailbox folder", + "description": "Retrieves messages from a specified mailbox folder with advanced functionality", "type": "investigate", "read_only": true, "parameters": { diff --git a/office365_connector.py b/office365_connector.py index eba433c..4172025 100644 --- a/office365_connector.py +++ b/office365_connector.py @@ -1295,62 +1295,103 @@ def _process_email_data(self, config, action_result, endpoint, email): return phantom.APP_SUCCESS def _process_email_details( - self, action_result, email, email_address, extract_headers=False, download_attachments=False, download_email=False + self, action_result, email, email_address, endpoint, extract_headers=False, download_attachments=False, download_email=False ): """ - Helper function to process email details including headers, attachments and email downloads - - Args: - action_result: The action result object - email: The email object to process - email_address: The email address of the mailbox - extract_headers: Whether to extract email headers - download_attachments: Whether to download attachments - download_email: Whether to download the email as EML - - Returns: - dict: Updated email object with additional details + Process email details including headers, attachments and email downloads. + + :param action_result: Action result object for tracking status + :param email: Email object to process + :param email_address: Email address of the mailbox + :param endpoint: Base endpoint for API calls + :param extract_headers: Whether to extract email headers (default: False) + :param download_attachments: Whether to download attachments (default: False) + :param download_email: Whether to download the email as EML (default: False) + :return: Updated email object with additional details including: + - internetMessageHeaders: Flattened email headers if extract_headers=True + - attachments: List of processed attachments if download_attachments=True + - vaultId: Vault ID of downloaded EML file if download_email=True """ - try: - # Get headers if requested - if extract_headers: - header_endpoint = f"/users/{email_address}/messages/{email['id']}?$select=internetMessageHeaders" - ret_val, header_response = self._make_rest_call_helper(action_result, header_endpoint) - if phantom.is_fail(ret_val): - self.debug_print(f"Failed to get headers for email {email['id']}") - else: - email["internetMessageHeaders"] = self._flatten_headers(header_response.get("internetMessageHeaders")) - - # Handle attachments if requested - if download_attachments and email.get("hasAttachments"): - attachment_endpoint = f"/users/{email_address}/messages/{email['id']}/attachments" - attach_endpoint = f"{attachment_endpoint}?$expand=microsoft.graph.itemattachment/item" - ret_val, attach_resp = self._make_rest_call_helper(action_result, attach_endpoint) - - if phantom.is_success(ret_val): - for attachment in attach_resp.get("value", []): - if attachment.get("@odata.type") == "#microsoft.graph.fileAttachment": - if not self._handle_attachment(attachment, self.get_container_id()): - self.debug_print(f"Could not process attachment for email {email['id']}") - elif attachment.get("@odata.type") == "#microsoft.graph.itemAttachment": - if not self._handle_item_attachment(attachment, self.get_container_id(), attachment_endpoint, action_result): - self.debug_print(f"Could not process item attachment for email {email['id']}") - email["attachments"] = attach_resp["value"] - - # Download email as EML if requested - if download_email: - subject = email.get("subject") - email_message = {"id": email["id"], "name": subject if subject else f"email_message_{email['id']}"} - if self._handle_item_attachment(email_message, self.get_container_id(), f"/users/{email_address}/messages", action_result): - email["vaultId"] = email_message["vaultId"] - else: - self.debug_print(f"Could not download email {email['id']}") + if extract_headers: + header_endpoint = endpoint + "?$select=internetMessageHeaders" + ret_val, header_response = self._make_rest_call_helper(action_result, header_endpoint) - return email + if phantom.is_fail(ret_val): + return action_result.get_status() + # For Drafts there might not be any internetMessageHeaders, + # so we have to use get() fetching instead of directly fetching from dictionary + email["internetMessageHeaders"] = header_response.get("internetMessageHeaders") - except Exception as e: - self.debug_print(f"Error processing email details: {str(e)}") - return email + if download_attachments and email.get("hasAttachments"): + endpoint += "/attachments" + attachment_endpoint = "{}?$expand=microsoft.graph.itemattachment/item".format(endpoint) + ret_val, attach_resp = self._make_rest_call_helper(action_result, attachment_endpoint) + + if phantom.is_fail(ret_val): + return action_result.get_status() + + for attachment in attach_resp.get("value", []): + # If it is fileAttachment, then we have to ingest it + if attachment.get("@odata.type") == "#microsoft.graph.fileAttachment": + if not self._handle_attachment(attachment, self.get_container_id()): + return action_result.set_status( + phantom.APP_ERROR, + "Could not process attachment. See logs for details", + ) + elif attachment.get("@odata.type") == "#microsoft.graph.itemAttachment": + if not self._handle_item_attachment(attachment, self.get_container_id(), endpoint, action_result): + return action_result.set_status( + phantom.APP_ERROR, + "Could not process item attachment. See logs for details", + ) + + email["attachments"] = attach_resp["value"] + + if email.get("@odata.type") in [ + "#microsoft.graph.eventMessage", + "#microsoft.graph.eventMessageRequest", + "#microsoft.graph.eventMessageResponse", + ]: + event_endpoint = "{}/?$expand=Microsoft.Graph.EventMessage/Event".format(endpoint) + ret_val, event_resp = self._make_rest_call_helper(action_result, event_endpoint) + if phantom.is_fail(ret_val): + return action_result.get_status() + + email["event"] = event_resp["event"] + + if "internetMessageHeaders" in email: + email["internetMessageHeaders"] = self._flatten_headers(email["internetMessageHeaders"]) + + # If the response has attachments, update every attachment data with its type + # 'attachmentType' key - indicates type of attachment + # and if an email has any itemAttachment, then also add itemType in the response + # 'itemType' key - indicates type of itemAttachment + if email.get("attachments", []): + for attachment in email["attachments"]: + attachment_type = attachment.get("@odata.type", "") + attachment["attachmentType"] = attachment_type + if attachment_type == "#microsoft.graph.itemAttachment": + attachment["itemType"] = attachment.get("item", {}).get("@odata.type", "") + + if download_email: + subject = email.get("subject") + email_message = { + "id": email["id"], + "name": subject if subject else "email_message_{}".format(email["id"]), + } + if not self._handle_item_attachment( + email_message, + self.get_container_id(), + "/users/{0}/messages".format(email_address), + action_result, + ): + return action_result.set_status( + phantom.APP_ERROR, + "Could not download the email. See logs for details", + ) + email["vaultId"] = email_message["vaultId"] + + return email def _remove_tokens(self, action_result): # checks whether the message includes any of the known error codes @@ -2056,6 +2097,7 @@ def _handle_get_email(self, param): action_result, response, email_addr, + endpoint, extract_headers=param.get("extract_headers"), download_attachments=param.get("download_attachments", False), download_email=param.get("download_email", False), @@ -3095,10 +3137,12 @@ def _handle_get_mailbox_messages(self, param): for index, email in enumerate(messages): try: # Perform additional processing of attachments/data for email + message_endpoint = f"/users/{email_address}/messages/{email['id']}" email = self._process_email_details( action_result, email, email_address, + message_endpoint, extract_headers=extract_headers, download_attachments=download_attachments, download_email=download_email, From 1306679f77528ea9c7aa6b029d80df5e8b584e7b Mon Sep 17 00:00:00 2001 From: splunk-soar-connectors-admin Date: Wed, 6 Nov 2024 23:41:10 +0000 Subject: [PATCH 9/9] Update README.md --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 6e7700d..c0bd84c 100644 --- a/README.md +++ b/README.md @@ -354,7 +354,7 @@ VARIABLE | REQUIRED | TYPE | DESCRIPTION [block sender](#action-block-sender) - Add the sender email into the block list [unblock sender](#action-unblock-sender) - Remove the sender email from the block list [resolve name](#action-resolve-name) - Verify aliases and resolve display names to the appropriate user -[get mailbox messages](#action-get-mailbox-messages) - Retrieves messages from a specified mailbox folder +[get mailbox messages](#action-get-mailbox-messages) - Retrieves messages from a specified mailbox folder with advanced functionality ## action: 'test connectivity' Use supplied credentials to generate a token with MS Graph @@ -1689,7 +1689,7 @@ summary.total_objects | numeric | | summary.total_objects_successful | numeric | | ## action: 'get mailbox messages' -Retrieves messages from a specified mailbox folder +Retrieves messages from a specified mailbox folder with advanced functionality Type: **investigate** Read only: **True**