From 5e7b38a3cb3e70cc0828a2cc904e853096bdbdb1 Mon Sep 17 00:00:00 2001 From: Aditi Bhoge Date: Wed, 21 Oct 2026 23:33:47 -0700 Subject: [PATCH 01/10] Add Proofpoint POD MailLog ASIM EmailEvent parsers Add MailLog source and filtering parsers, unifier registrations, generated ARM, fixtures and mapping documentation. Live validation is deferred; repository KQL checks need the protected custom-table catalog updated. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 321517f1-a1c5-4242-be85-2d144a9cc295 --- .../ARM/ASimEmailEvent/ASimEmailEvent.json | 2 +- .../ASimEmailEventProofpointPODMailLog.json | 36 +++++ .../README.md | 23 +++ .../ARM/FullDeploymentEmailEvent.json | 40 +++++ .../ARM/imEmailEvent/imEmailEvent.json | 2 +- .../README.md | 24 +++ .../vimEmailEventProofpointPODMailLog.json | 36 +++++ .../CHANGELOG/ASimEmailEvent.md | 5 + .../ASimEmailEventProofpointPODMailLog.md | 7 + .../ASimEmailEvent/CHANGELOG/imEmailEvent.md | 5 + .../vimEmailEventProofpointPODMailLog.md | 7 + .../Parsers/ASimEmailEvent.yaml | 8 +- .../ASimEmailEventProofpointPODMailLog.yaml | 96 +++++++++++ .../ASimEmailEvent/Parsers/imEmailEvent.yaml | 13 +- .../vimEmailEventProofpointPODMailLog.yaml | 149 ++++++++++++++++++ Parsers/ASimEmailEvent/README.md | 89 ++++++++++- .../ASIM/ProofpointPODMailLog_CL_Schema.csv | 9 ++ ...nt_POD MailLog_EmailEvent_IngestedLogs.csv | 21 +++ 18 files changed, 563 insertions(+), 9 deletions(-) create mode 100644 Parsers/ASimEmailEvent/ARM/ASimEmailEventProofpointPODMailLog/ASimEmailEventProofpointPODMailLog.json create mode 100644 Parsers/ASimEmailEvent/ARM/ASimEmailEventProofpointPODMailLog/README.md create mode 100644 Parsers/ASimEmailEvent/ARM/vimEmailEventProofpointPODMailLog/README.md create mode 100644 Parsers/ASimEmailEvent/ARM/vimEmailEventProofpointPODMailLog/vimEmailEventProofpointPODMailLog.json create mode 100644 Parsers/ASimEmailEvent/CHANGELOG/ASimEmailEventProofpointPODMailLog.md create mode 100644 Parsers/ASimEmailEvent/CHANGELOG/vimEmailEventProofpointPODMailLog.md create mode 100644 Parsers/ASimEmailEvent/Parsers/ASimEmailEventProofpointPODMailLog.yaml create mode 100644 Parsers/ASimEmailEvent/Parsers/vimEmailEventProofpointPODMailLog.yaml create mode 100644 Sample Data/ASIM/ProofpointPODMailLog_CL_Schema.csv create mode 100644 Sample Data/ASIM/Proofpoint_POD MailLog_EmailEvent_IngestedLogs.csv diff --git a/Parsers/ASimEmailEvent/ARM/ASimEmailEvent/ASimEmailEvent.json b/Parsers/ASimEmailEvent/ARM/ASimEmailEvent/ASimEmailEvent.json index b1cd990b7ce..8468dbff75b 100644 --- a/Parsers/ASimEmailEvent/ARM/ASimEmailEvent/ASimEmailEvent.json +++ b/Parsers/ASimEmailEvent/ARM/ASimEmailEvent/ASimEmailEvent.json @@ -27,7 +27,7 @@ "displayName": "Email Event ASIM parser", "category": "ASIM", "FunctionAlias": "ASimEmailEvent", - "query": "let DisabledParsers=materialize(_GetWatchlist('ASimDisabledParsers') | where SearchKey in ('Any', 'ExcludeASimEmailEvent') | extend SourceSpecificParser=column_ifexists('SourceSpecificParser','') | distinct SourceSpecificParser| where isnotempty(SourceSpecificParser));\nlet ASimBuiltInDisabled=toscalar('ExcludeASimEmailEvent' in (DisabledParsers) or 'Any' in (DisabledParsers)); \nlet parser=(pack:bool=false){\nunion isfuzzy=true\n vimEmailEventEmpty\n}; \nparser (pack=pack)", + "query": "let DisabledParsers=materialize(_GetWatchlist('ASimDisabledParsers') | where SearchKey in ('Any', 'ExcludeASimEmailEvent') | extend SourceSpecificParser=column_ifexists('SourceSpecificParser','') | distinct SourceSpecificParser| where isnotempty(SourceSpecificParser));\nlet ASimBuiltInDisabled=toscalar('ExcludeASimEmailEvent' in (DisabledParsers) or 'Any' in (DisabledParsers)); \nlet parser=(pack:bool=false){\nunion isfuzzy=true\n vimEmailEventEmpty,\n ASimEmailEventProofpointPODMailLog(pack=pack, disabled=(ASimBuiltInDisabled or ('ExcludeASimEmailEventProofpointPODMailLog' in (DisabledParsers))))\n}; \nparser (pack=pack)", "version": 1, "functionParameters": "pack:bool=False" } diff --git a/Parsers/ASimEmailEvent/ARM/ASimEmailEventProofpointPODMailLog/ASimEmailEventProofpointPODMailLog.json b/Parsers/ASimEmailEvent/ARM/ASimEmailEventProofpointPODMailLog/ASimEmailEventProofpointPODMailLog.json new file mode 100644 index 00000000000..36932c82b74 --- /dev/null +++ b/Parsers/ASimEmailEvent/ARM/ASimEmailEventProofpointPODMailLog/ASimEmailEventProofpointPODMailLog.json @@ -0,0 +1,36 @@ +{ + "$schema": "https://schema.management.azure.com/schemas/2019-08-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "Workspace": { + "type": "string", + "metadata": { + "description": "The Microsoft Sentinel workspace into which the function will be deployed. Has to be in the selected Resource Group." + } + }, + "WorkspaceRegion": { + "type": "string", + "defaultValue": "[resourceGroup().location]", + "metadata": { + "description": "The region of the selected workspace. The default value will use the Region selection above." + } + } + }, + "resources": [ + { + "type": "Microsoft.OperationalInsights/workspaces/savedSearches", + "apiVersion": "2020-08-01", + "name": "[concat(parameters('Workspace'), '/ASimEmailEventProofpointPODMailLog')]", + "location": "[parameters('WorkspaceRegion')]", + "properties": { + "etag": "*", + "displayName": "Email Event ASIM parser for Proofpoint POD MailLog", + "category": "ASIM", + "FunctionAlias": "ASimEmailEventProofpointPODMailLog", + "query": "let parser = (disabled:bool=false, pack:bool=false) {\n ProofpointPODMailLog_CL\n | where not(disabled)\n | where isnotempty(tostring(sm.qid)) and (bag_has_key(sm, \"from\") or array_length(sm[\"to\"]) > 0)\n | extend\n IsDelivery = coalesce(array_length(sm[\"to\"]), 0) > 0,\n Relay = tostring(sm.relay),\n Dvc = coalesce(tostring(pps.agent), tostring(metadata.origin.data.agent)),\n EventStartTime = coalesce(eventTime, TimeGenerated),\n EmailMailFromAddress = trim(@\"[<>\\s]\", tostring(sm[\"from\"])),\n EmailSizeBytes = tolong(sm.sizeBytes),\n Dsn = tostring(sm.dsn),\n DeliveryStatus = tostring(sm.stat)\n | parse Relay with RelayHost \" [\" RelayIp \"]\" *\n | extend\n EmailMailFromAddress = iff(EmailMailFromAddress contains \"@\", EmailMailFromAddress, \"\"),\n RelayHost = trim_end(@\"\\.\", coalesce(RelayHost, Relay)),\n RelayIp = iff(isnotnull(parse_ipv4(RelayIp)) or isnotempty(parse_ipv6(RelayIp)), RelayIp, \"\"),\n EventResult = case(Dsn startswith \"2.\", \"Success\", Dsn startswith \"4.\" or Dsn startswith \"5.\", \"Failure\", \"NA\"),\n EmailProtocolName = case(tostring(sm.proto) =~ \"SMTP\" or tostring(sm.mailer) in~ (\"smtp\", \"esmtp\"), \"SMTP\", \"\"),\n EventSubType = iff(IsDelivery, \"EmailSent\", \"EmailReceived\")\n | extend\n SrcIpAddr = iff(IsDelivery, \"\", RelayIp),\n DstIpAddr = iff(IsDelivery, RelayIp, \"\"),\n Src = iff(IsDelivery, Dvc, coalesce(RelayIp, RelayHost)),\n Dst = iff(IsDelivery, coalesce(RelayIp, RelayHost), Dvc),\n SrcFQDN = iff(not(IsDelivery) and RelayHost contains \".\", RelayHost, \"\"),\n SrcHostname = iff(IsDelivery, tostring(split(Dvc, \".\")[0]), tostring(split(RelayHost, \".\")[0])),\n SrcDomain = iff(IsDelivery, substring(Dvc, indexof(Dvc, \".\") + 1), substring(RelayHost, indexof(RelayHost, \".\") + 1)),\n DvcHostname = tostring(split(Dvc, \".\")[0]),\n DvcDomain = iff(Dvc contains \".\", substring(Dvc, indexof(Dvc, \".\") + 1), \"\"),\n DvcFQDN = iff(Dvc contains \".\", Dvc, \"\"),\n EmailMailFromDomain = iff(EmailMailFromAddress contains \"@\", tostring(split(EmailMailFromAddress, \"@\")[1]), \"\"),\n DvcAction = iff(IsDelivery and EventResult == \"Success\" and DeliveryStatus startswith \"Sent\", \"Delivered\", \"NA\")\n | extend\n SrcFQDN = iff(IsDelivery, DvcFQDN, SrcFQDN),\n SrcDomain = iff(iff(IsDelivery, Dvc, RelayHost) contains \".\", SrcDomain, \"\"),\n DvcDomainType = iff(isnotempty(DvcDomain), \"FQDN\", \"\"),\n EventEndTime = EventStartTime,\n EventCount = int(1),\n EventVendor = \"Proofpoint\",\n EventProduct = \"POD MailLog\",\n EventSchema = \"EmailEvent\",\n EventSchemaVersion = \"1.0.0\",\n EventType = \"EmailDelivery\",\n EventOriginalType = \"maillog\",\n EmailDirection = \"Unknown\",\n EventOriginalResultDetails = strcat(Dsn, iff(isnotempty(Dsn) and isnotempty(DeliveryStatus), \"; \", \"\"), DeliveryStatus),\n EventUid = iff(isnotempty(id), strcat(\"ProofpointPODMailLog_CL:\", id), \"\"),\n DvcEntityKey = \"\",\n AdditionalEntities = dynamic([]),\n AdditionalFields = iff(pack, bag_pack(\"sm\", sm, \"tls\", tls, \"pps\", pps, \"metadata\", metadata), dynamic({})),\n Type = \"ProofpointPODMailLog_CL\"\n | extend SrcDomainType = iff(isnotempty(SrcDomain), \"FQDN\", \"\"), IpAddr = SrcIpAddr\n | project-rename EventOriginalUid = id, EventMessage = data\n | project\n TimeGenerated, Type, EventStartTime, EventEndTime, EventCount,\n EventVendor, EventProduct, EventSchema, EventSchemaVersion,\n EventType, EventSubType, EventOriginalType, EventResult,\n EventOriginalResultDetails, EventOriginalUid, EventUid, EventMessage,\n Dvc, DvcHostname, DvcDomain, DvcDomainType, DvcFQDN, DvcAction, DvcEntityKey,\n Src, SrcIpAddr, SrcHostname, SrcDomain, SrcDomainType, SrcFQDN, Dst, DstIpAddr, IpAddr,\n EmailMailFromAddress, EmailMailFromDomain, EmailSizeBytes, EmailProtocolName, EmailDirection,\n AdditionalFields, AdditionalEntities\n};\nparser(disabled=disabled, pack=pack)\n", + "version": 1, + "functionParameters": "disabled:bool=False,pack:bool=False" + } + } + ] +} \ No newline at end of file diff --git a/Parsers/ASimEmailEvent/ARM/ASimEmailEventProofpointPODMailLog/README.md b/Parsers/ASimEmailEvent/ARM/ASimEmailEventProofpointPODMailLog/README.md new file mode 100644 index 00000000000..742834533b9 --- /dev/null +++ b/Parsers/ASimEmailEvent/ARM/ASimEmailEventProofpointPODMailLog/README.md @@ -0,0 +1,23 @@ +# Proofpoint POD MailLog ASIM EmailEvent Normalization Parser + +ARM template for ASIM EmailEvent schema parser for Proofpoint POD MailLog. + +Normalizes ProofpointPODMailLog_CL SMTP receipt and delivery records to EmailEvent. +Uses the CCP table schema and repository samples; live validation is deferred. +Envelope recipients are retained in AdditionalFields, not mapped to header recipients. + + +The Advanced Security Information Model (ASIM) enables you to use and create source-agnostic content, simplifying your analysis of the data in your Microsoft Sentinel workspace. + +For more information, see: + +- [Normalization and the Advanced Security Information Model (ASIM)](https://aka.ms/AboutASIM) +- [Deploy all of ASIM](https://aka.ms/DeployASIM) +- [ASIM EmailEvent normalization schema reference](https://aka.ms/ASimEmailEventDoc) + +For the changelog, see: +- [CHANGELOG](https://github.com/Azure/Azure-Sentinel/blob/master/Parsers/ASimEmailEvent/CHANGELOG/ASimEmailEventProofpointPODMailLog.md) + +
+ +[![Deploy to Azure](https://aka.ms/deploytoazurebutton)](https://portal.azure.com/#create/Microsoft.Template/uri/https%3A%2F%2Fraw.githubusercontent.com%2FAzure%2FAzure-Sentinel%2Fmaster%2FParsers%2FASimEmailEvent%2FARM%2FASimEmailEventProofpointPODMailLog%2FASimEmailEventProofpointPODMailLog.json) [![Deploy to Azure Gov](https://aka.ms/deploytoazuregovernbutton)](https://portal.azure.us/#create/Microsoft.Template/uri/https%3A%2F%2Fraw.githubusercontent.com%2FAzure%2FAzure-Sentinel%2Fmaster%2FParsers%2FASimEmailEvent%2FARM%2FASimEmailEventProofpointPODMailLog%2FASimEmailEventProofpointPODMailLog.json) diff --git a/Parsers/ASimEmailEvent/ARM/FullDeploymentEmailEvent.json b/Parsers/ASimEmailEvent/ARM/FullDeploymentEmailEvent.json index 0cec41ce310..b5e89e872ec 100644 --- a/Parsers/ASimEmailEvent/ARM/FullDeploymentEmailEvent.json +++ b/Parsers/ASimEmailEvent/ARM/FullDeploymentEmailEvent.json @@ -38,6 +38,26 @@ } } }, + { + "type": "Microsoft.Resources/deployments", + "apiVersion": "2020-10-01", + "name": "linkedASimEmailEventProofpointPODMailLog", + "properties": { + "mode": "Incremental", + "templateLink": { + "uri": "https://raw.githubusercontent.com/Azure/Azure-Sentinel/master/Parsers/ASimEmailEvent/ARM/ASimEmailEventProofpointPODMailLog/ASimEmailEventProofpointPODMailLog.json", + "contentVersion": "1.0.0.0" + }, + "parameters": { + "Workspace": { + "value": "[parameters('Workspace')]" + }, + "WorkspaceRegion": { + "value": "[parameters('WorkspaceRegion')]" + } + } + } + }, { "type": "Microsoft.Resources/deployments", "apiVersion": "2020-10-01", @@ -77,6 +97,26 @@ } } } + }, + { + "type": "Microsoft.Resources/deployments", + "apiVersion": "2020-10-01", + "name": "linkedvimEmailEventProofpointPODMailLog", + "properties": { + "mode": "Incremental", + "templateLink": { + "uri": "https://raw.githubusercontent.com/Azure/Azure-Sentinel/master/Parsers/ASimEmailEvent/ARM/vimEmailEventProofpointPODMailLog/vimEmailEventProofpointPODMailLog.json", + "contentVersion": "1.0.0.0" + }, + "parameters": { + "Workspace": { + "value": "[parameters('Workspace')]" + }, + "WorkspaceRegion": { + "value": "[parameters('WorkspaceRegion')]" + } + } + } } ], "outputs": {} diff --git a/Parsers/ASimEmailEvent/ARM/imEmailEvent/imEmailEvent.json b/Parsers/ASimEmailEvent/ARM/imEmailEvent/imEmailEvent.json index f598cd155ef..59e3e47e991 100644 --- a/Parsers/ASimEmailEvent/ARM/imEmailEvent/imEmailEvent.json +++ b/Parsers/ASimEmailEvent/ARM/imEmailEvent/imEmailEvent.json @@ -27,7 +27,7 @@ "displayName": "Email Event ASIM filtering parser", "category": "ASIM", "FunctionAlias": "imEmailEvent", - "query": "let DisabledParsers=materialize(_GetWatchlist('ASimDisabledParsers') | where SearchKey in ('Any', 'ExcludeimEmailEvent') | extend SourceSpecificParser=column_ifexists('SourceSpecificParser','') | distinct SourceSpecificParser | where isnotempty(SourceSpecificParser));\nlet imBuiltInDisabled=toscalar('ExcludeimEmailEvent' in (DisabledParsers) or 'Any' in (DisabledParsers)); \nlet parser=(\n starttime: datetime=datetime(null), \n endtime: datetime=datetime(null), \n ipaddr_has_any_prefix: dynamic=dynamic([]),\n emailfromaddress_has_any: dynamic=dynamic([]),\n emailrecipient_has_any: dynamic=dynamic([]),\n emailsubject_has_any: dynamic=dynamic([]),\n emaildirection_in: dynamic=dynamic([]),\n emaildeliverylocation_in: dynamic=dynamic([]),\n eventresult: string='*',\n pack:bool=false)\n{\nunion isfuzzy=true\n vimEmailEventEmpty\n};\nparser (starttime=starttime, endtime=endtime, ipaddr_has_any_prefix=ipaddr_has_any_prefix, emailfromaddress_has_any=emailfromaddress_has_any, emailrecipient_has_any=emailrecipient_has_any, emailsubject_has_any=emailsubject_has_any, emaildirection_in=emaildirection_in, emaildeliverylocation_in=emaildeliverylocation_in, eventresult=eventresult, pack=pack)", + "query": "let DisabledParsers=materialize(_GetWatchlist('ASimDisabledParsers') | where SearchKey in ('Any', 'ExcludeimEmailEvent') | extend SourceSpecificParser=column_ifexists('SourceSpecificParser','') | distinct SourceSpecificParser | where isnotempty(SourceSpecificParser));\nlet imBuiltInDisabled=toscalar('ExcludeimEmailEvent' in (DisabledParsers) or 'Any' in (DisabledParsers)); \nlet parser=(\n starttime: datetime=datetime(null), \n endtime: datetime=datetime(null), \n ipaddr_has_any_prefix: dynamic=dynamic([]),\n emailfromaddress_has_any: dynamic=dynamic([]),\n emailrecipient_has_any: dynamic=dynamic([]),\n emailsubject_has_any: dynamic=dynamic([]),\n emaildirection_in: dynamic=dynamic([]),\n emaildeliverylocation_in: dynamic=dynamic([]),\n eventresult: string='*',\n pack:bool=false)\n{\nunion isfuzzy=true\n vimEmailEventEmpty,\n vimEmailEventProofpointPODMailLog(\n starttime=starttime, endtime=endtime, ipaddr_has_any_prefix=ipaddr_has_any_prefix,\n emailfromaddress_has_any=emailfromaddress_has_any, emailrecipient_has_any=emailrecipient_has_any,\n emailsubject_has_any=emailsubject_has_any, emaildirection_in=emaildirection_in,\n emaildeliverylocation_in=emaildeliverylocation_in, eventresult=eventresult, pack=pack,\n disabled=(imBuiltInDisabled or ('ExcludevimEmailEventProofpointPODMailLog' in (DisabledParsers))))\n};\nparser (starttime=starttime, endtime=endtime, ipaddr_has_any_prefix=ipaddr_has_any_prefix, emailfromaddress_has_any=emailfromaddress_has_any, emailrecipient_has_any=emailrecipient_has_any, emailsubject_has_any=emailsubject_has_any, emaildirection_in=emaildirection_in, emaildeliverylocation_in=emaildeliverylocation_in, eventresult=eventresult, pack=pack)", "version": 1, "functionParameters": "starttime:datetime=datetime(null),endtime:datetime=datetime(null),ipaddr_has_any_prefix:dynamic=dynamic([]),emailfromaddress_has_any:dynamic=dynamic([]),emailrecipient_has_any:dynamic=dynamic([]),emailsubject_has_any:dynamic=dynamic([]),emaildirection_in:dynamic=dynamic([]),emaildeliverylocation_in:dynamic=dynamic([]),eventresult:string='*',pack:bool=False" } diff --git a/Parsers/ASimEmailEvent/ARM/vimEmailEventProofpointPODMailLog/README.md b/Parsers/ASimEmailEvent/ARM/vimEmailEventProofpointPODMailLog/README.md new file mode 100644 index 00000000000..0433444dd2c --- /dev/null +++ b/Parsers/ASimEmailEvent/ARM/vimEmailEventProofpointPODMailLog/README.md @@ -0,0 +1,24 @@ +# Proofpoint POD MailLog ASIM EmailEvent Normalization Parser + +ARM template for ASIM EmailEvent schema parser for Proofpoint POD MailLog. + +Filters and normalizes ProofpointPODMailLog_CL SMTP receipt and delivery records to EmailEvent. +Uses the CCP table schema and repository samples; live validation is deferred. +Header sender, header recipients, subject and delivery location are unavailable; +nonempty filters for these fields return no records. + + +The Advanced Security Information Model (ASIM) enables you to use and create source-agnostic content, simplifying your analysis of the data in your Microsoft Sentinel workspace. + +For more information, see: + +- [Normalization and the Advanced Security Information Model (ASIM)](https://aka.ms/AboutASIM) +- [Deploy all of ASIM](https://aka.ms/DeployASIM) +- [ASIM EmailEvent normalization schema reference](https://aka.ms/ASimEmailEventDoc) + +For the changelog, see: +- [CHANGELOG](https://github.com/Azure/Azure-Sentinel/blob/master/Parsers/ASimEmailEvent/CHANGELOG/vimEmailEventProofpointPODMailLog.md) + +
+ +[![Deploy to Azure](https://aka.ms/deploytoazurebutton)](https://portal.azure.com/#create/Microsoft.Template/uri/https%3A%2F%2Fraw.githubusercontent.com%2FAzure%2FAzure-Sentinel%2Fmaster%2FParsers%2FASimEmailEvent%2FARM%2FvimEmailEventProofpointPODMailLog%2FvimEmailEventProofpointPODMailLog.json) [![Deploy to Azure Gov](https://aka.ms/deploytoazuregovernbutton)](https://portal.azure.us/#create/Microsoft.Template/uri/https%3A%2F%2Fraw.githubusercontent.com%2FAzure%2FAzure-Sentinel%2Fmaster%2FParsers%2FASimEmailEvent%2FARM%2FvimEmailEventProofpointPODMailLog%2FvimEmailEventProofpointPODMailLog.json) diff --git a/Parsers/ASimEmailEvent/ARM/vimEmailEventProofpointPODMailLog/vimEmailEventProofpointPODMailLog.json b/Parsers/ASimEmailEvent/ARM/vimEmailEventProofpointPODMailLog/vimEmailEventProofpointPODMailLog.json new file mode 100644 index 00000000000..1427541ca7e --- /dev/null +++ b/Parsers/ASimEmailEvent/ARM/vimEmailEventProofpointPODMailLog/vimEmailEventProofpointPODMailLog.json @@ -0,0 +1,36 @@ +{ + "$schema": "https://schema.management.azure.com/schemas/2019-08-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "Workspace": { + "type": "string", + "metadata": { + "description": "The Microsoft Sentinel workspace into which the function will be deployed. Has to be in the selected Resource Group." + } + }, + "WorkspaceRegion": { + "type": "string", + "defaultValue": "[resourceGroup().location]", + "metadata": { + "description": "The region of the selected workspace. The default value will use the Region selection above." + } + } + }, + "resources": [ + { + "type": "Microsoft.OperationalInsights/workspaces/savedSearches", + "apiVersion": "2020-08-01", + "name": "[concat(parameters('Workspace'), '/vimEmailEventProofpointPODMailLog')]", + "location": "[parameters('WorkspaceRegion')]", + "properties": { + "etag": "*", + "displayName": "Email Event ASIM filtering parser for Proofpoint POD MailLog", + "category": "ASIM", + "FunctionAlias": "vimEmailEventProofpointPODMailLog", + "query": "let parser = (\n starttime:datetime=datetime(null),\n endtime:datetime=datetime(null),\n ipaddr_has_any_prefix:dynamic=dynamic([]),\n emailfromaddress_has_any:dynamic=dynamic([]),\n emailrecipient_has_any:dynamic=dynamic([]),\n emailsubject_has_any:dynamic=dynamic([]),\n emaildirection_in:dynamic=dynamic([]),\n emaildeliverylocation_in:dynamic=dynamic([]),\n eventresult:string='*',\n disabled:bool=false,\n pack:bool=false)\n{\n ProofpointPODMailLog_CL\n | where not(disabled)\n | where (isnull(starttime) or eventTime >= starttime or (isnull(eventTime) and TimeGenerated >= starttime))\n and (isnull(endtime) or eventTime <= endtime or (isnull(eventTime) and TimeGenerated <= endtime))\n | where array_length(emailfromaddress_has_any) == 0 and array_length(emailrecipient_has_any) == 0\n and array_length(emailsubject_has_any) == 0 and array_length(emaildeliverylocation_in) == 0\n | where array_length(emaildirection_in) == 0 or \"Unknown\" in (emaildirection_in)\n | where isnotempty(tostring(sm.qid)) and (bag_has_key(sm, \"from\") or array_length(sm[\"to\"]) > 0)\n | extend\n IsDelivery = coalesce(array_length(sm[\"to\"]), 0) > 0,\n Dsn = tostring(sm.dsn)\n | extend EventResult = case(Dsn startswith \"2.\", \"Success\", Dsn startswith \"4.\" or Dsn startswith \"5.\", \"Failure\", \"NA\")\n | where eventresult == \"*\" or EventResult == eventresult\n | extend\n Relay = tostring(sm.relay),\n Dvc = coalesce(tostring(pps.agent), tostring(metadata.origin.data.agent)),\n EventStartTime = coalesce(eventTime, TimeGenerated),\n EmailMailFromAddress = trim(@\"[<>\\s]\", tostring(sm[\"from\"])),\n EmailSizeBytes = tolong(sm.sizeBytes),\n DeliveryStatus = tostring(sm.stat)\n | parse Relay with RelayHost \" [\" RelayIp \"]\" *\n | extend\n EmailMailFromAddress = iff(EmailMailFromAddress contains \"@\", EmailMailFromAddress, \"\"),\n RelayHost = trim_end(@\"\\.\", coalesce(RelayHost, Relay)),\n RelayIp = iff(isnotnull(parse_ipv4(RelayIp)) or isnotempty(parse_ipv6(RelayIp)), RelayIp, \"\"),\n EmailProtocolName = case(tostring(sm.proto) =~ \"SMTP\" or tostring(sm.mailer) in~ (\"smtp\", \"esmtp\"), \"SMTP\", \"\"),\n EventSubType = iff(IsDelivery, \"EmailSent\", \"EmailReceived\")\n | extend\n SrcIpAddr = iff(IsDelivery, \"\", RelayIp),\n DstIpAddr = iff(IsDelivery, RelayIp, \"\"),\n Src = iff(IsDelivery, Dvc, coalesce(RelayIp, RelayHost)),\n Dst = iff(IsDelivery, coalesce(RelayIp, RelayHost), Dvc),\n SrcFQDN = iff(not(IsDelivery) and RelayHost contains \".\", RelayHost, \"\"),\n SrcHostname = iff(IsDelivery, tostring(split(Dvc, \".\")[0]), tostring(split(RelayHost, \".\")[0])),\n SrcDomain = iff(IsDelivery, substring(Dvc, indexof(Dvc, \".\") + 1), substring(RelayHost, indexof(RelayHost, \".\") + 1)),\n DvcHostname = tostring(split(Dvc, \".\")[0]),\n DvcDomain = iff(Dvc contains \".\", substring(Dvc, indexof(Dvc, \".\") + 1), \"\"),\n DvcFQDN = iff(Dvc contains \".\", Dvc, \"\"),\n EmailMailFromDomain = iff(EmailMailFromAddress contains \"@\", tostring(split(EmailMailFromAddress, \"@\")[1]), \"\"),\n DvcAction = iff(IsDelivery and EventResult == \"Success\" and DeliveryStatus startswith \"Sent\", \"Delivered\", \"NA\")\n | where array_length(ipaddr_has_any_prefix) == 0\n or has_any_ipv4_prefix(SrcIpAddr, ipaddr_has_any_prefix)\n or has_any_ipv4_prefix(DstIpAddr, ipaddr_has_any_prefix)\n | extend\n SrcFQDN = iff(IsDelivery, DvcFQDN, SrcFQDN),\n SrcDomain = iff(iff(IsDelivery, Dvc, RelayHost) contains \".\", SrcDomain, \"\"),\n DvcDomainType = iff(isnotempty(DvcDomain), \"FQDN\", \"\"),\n EventEndTime = EventStartTime,\n EventCount = int(1),\n EventVendor = \"Proofpoint\",\n EventProduct = \"POD MailLog\",\n EventSchema = \"EmailEvent\",\n EventSchemaVersion = \"1.0.0\",\n EventType = \"EmailDelivery\",\n EventOriginalType = \"maillog\",\n EmailDirection = \"Unknown\",\n EventOriginalResultDetails = strcat(Dsn, iff(isnotempty(Dsn) and isnotempty(DeliveryStatus), \"; \", \"\"), DeliveryStatus),\n EventUid = iff(isnotempty(id), strcat(\"ProofpointPODMailLog_CL:\", id), \"\"),\n DvcEntityKey = \"\",\n AdditionalEntities = dynamic([]),\n AdditionalFields = iff(pack, bag_pack(\"sm\", sm, \"tls\", tls, \"pps\", pps, \"metadata\", metadata), dynamic({})),\n Type = \"ProofpointPODMailLog_CL\"\n | extend SrcDomainType = iff(isnotempty(SrcDomain), \"FQDN\", \"\"), IpAddr = SrcIpAddr\n | project-rename EventOriginalUid = id, EventMessage = data\n | project\n TimeGenerated, Type, EventStartTime, EventEndTime, EventCount,\n EventVendor, EventProduct, EventSchema, EventSchemaVersion,\n EventType, EventSubType, EventOriginalType, EventResult,\n EventOriginalResultDetails, EventOriginalUid, EventUid, EventMessage,\n Dvc, DvcHostname, DvcDomain, DvcDomainType, DvcFQDN, DvcAction, DvcEntityKey,\n Src, SrcIpAddr, SrcHostname, SrcDomain, SrcDomainType, SrcFQDN, Dst, DstIpAddr, IpAddr,\n EmailMailFromAddress, EmailMailFromDomain, EmailSizeBytes, EmailProtocolName, EmailDirection,\n AdditionalFields, AdditionalEntities\n};\nparser(starttime=starttime, endtime=endtime, ipaddr_has_any_prefix=ipaddr_has_any_prefix,\n emailfromaddress_has_any=emailfromaddress_has_any, emailrecipient_has_any=emailrecipient_has_any,\n emailsubject_has_any=emailsubject_has_any, emaildirection_in=emaildirection_in,\n emaildeliverylocation_in=emaildeliverylocation_in, eventresult=eventresult, disabled=disabled, pack=pack)\n", + "version": 1, + "functionParameters": "starttime:datetime=datetime(null),endtime:datetime=datetime(null),ipaddr_has_any_prefix:dynamic=dynamic([]),emailfromaddress_has_any:dynamic=dynamic([]),emailrecipient_has_any:dynamic=dynamic([]),emailsubject_has_any:dynamic=dynamic([]),emaildirection_in:dynamic=dynamic([]),emaildeliverylocation_in:dynamic=dynamic([]),eventresult:string='*',disabled:bool=False,pack:bool=False" + } + } + ] +} \ No newline at end of file diff --git a/Parsers/ASimEmailEvent/CHANGELOG/ASimEmailEvent.md b/Parsers/ASimEmailEvent/CHANGELOG/ASimEmailEvent.md index e0675b9d4df..dc90b2ba63c 100644 --- a/Parsers/ASimEmailEvent/CHANGELOG/ASimEmailEvent.md +++ b/Parsers/ASimEmailEvent/CHANGELOG/ASimEmailEvent.md @@ -1,5 +1,10 @@ # Changelog for ASimEmailEvent.yaml +## Version 1.0.1 + +- (2026-10-21) Registered the Proofpoint POD MailLog source parser and forwarded packing and disable controls. +- No pull request has been created for this change. + ## Version 0.1.0 - (2026-09-09) Added new Email Event schema - [PR #15090](https://github.com/Azure/Azure-Sentinel/pull/15090) \ No newline at end of file diff --git a/Parsers/ASimEmailEvent/CHANGELOG/ASimEmailEventProofpointPODMailLog.md b/Parsers/ASimEmailEvent/CHANGELOG/ASimEmailEventProofpointPODMailLog.md new file mode 100644 index 00000000000..7ea1021118d --- /dev/null +++ b/Parsers/ASimEmailEvent/CHANGELOG/ASimEmailEventProofpointPODMailLog.md @@ -0,0 +1,7 @@ +# Changelog for ASimEmailEventProofpointPODMailLog.yaml + +## Version 0.1.0 + +- (2026-10-21) Added a Proofpoint POD MailLog EmailEvent parser using the CCP table schema and repository samples. +- Preserves individual SMTP receipt and delivery records without correlation or recipient expansion. +- Live schema and data validation are deferred. No pull request has been created for this change. diff --git a/Parsers/ASimEmailEvent/CHANGELOG/imEmailEvent.md b/Parsers/ASimEmailEvent/CHANGELOG/imEmailEvent.md index 7bcdd53e2f8..226597f7c0e 100644 --- a/Parsers/ASimEmailEvent/CHANGELOG/imEmailEvent.md +++ b/Parsers/ASimEmailEvent/CHANGELOG/imEmailEvent.md @@ -1,5 +1,10 @@ # Changelog for imEmailEvent.yaml +## Version 1.0.1 + +- (2026-10-21) Registered the Proofpoint POD MailLog filtering parser and forwarded all EmailEvent filters and disable controls. +- No pull request has been created for this change. + ## Version 0.1.0 - (2026-09-09) Added new Email Event schema - [PR #15090](https://github.com/Azure/Azure-Sentinel/pull/15090) \ No newline at end of file diff --git a/Parsers/ASimEmailEvent/CHANGELOG/vimEmailEventProofpointPODMailLog.md b/Parsers/ASimEmailEvent/CHANGELOG/vimEmailEventProofpointPODMailLog.md new file mode 100644 index 00000000000..f752b95a5fc --- /dev/null +++ b/Parsers/ASimEmailEvent/CHANGELOG/vimEmailEventProofpointPODMailLog.md @@ -0,0 +1,7 @@ +# Changelog for vimEmailEventProofpointPODMailLog.yaml + +## Version 0.1.0 + +- (2026-10-21) Added a Proofpoint POD MailLog EmailEvent filtering parser with time, IPv4 prefix, direction and result filters. +- Nonempty header sender, header recipient, subject and delivery-location filters return no rows because these fields are unavailable. +- Live schema, data and filter validation are deferred. No pull request has been created for this change. diff --git a/Parsers/ASimEmailEvent/Parsers/ASimEmailEvent.yaml b/Parsers/ASimEmailEvent/Parsers/ASimEmailEvent.yaml index fd0fcebb1ad..b0d2de1bca7 100644 --- a/Parsers/ASimEmailEvent/Parsers/ASimEmailEvent.yaml +++ b/Parsers/ASimEmailEvent/Parsers/ASimEmailEvent.yaml @@ -1,7 +1,7 @@ Parser: Title: Email Event ASIM parser - Version: '1.0.0' - LastUpdated: Sep 09, 2026 + Version: '1.0.1' + LastUpdated: Oct 21, 2026 Product: Name: Source agnostic Normalization: @@ -18,6 +18,7 @@ ParserName: ASimEmailEvent EquivalentBuiltInParser: _ASim_EmailEvent Parsers: - _Im_EmailEvent_Empty + - _ASim_EmailEvent_ProofpointPODMailLog ParserParams: - Name: pack Type: bool @@ -27,6 +28,7 @@ ParserQuery: | let ASimBuiltInDisabled=toscalar('ExcludeASimEmailEvent' in (DisabledParsers) or 'Any' in (DisabledParsers)); let parser=(pack:bool=false){ union isfuzzy=true - vimEmailEventEmpty + vimEmailEventEmpty, + ASimEmailEventProofpointPODMailLog(pack=pack, disabled=(ASimBuiltInDisabled or ('ExcludeASimEmailEventProofpointPODMailLog' in (DisabledParsers)))) }; parser (pack=pack) \ No newline at end of file diff --git a/Parsers/ASimEmailEvent/Parsers/ASimEmailEventProofpointPODMailLog.yaml b/Parsers/ASimEmailEvent/Parsers/ASimEmailEventProofpointPODMailLog.yaml new file mode 100644 index 00000000000..f4f1deaa558 --- /dev/null +++ b/Parsers/ASimEmailEvent/Parsers/ASimEmailEventProofpointPODMailLog.yaml @@ -0,0 +1,96 @@ +Parser: + Title: Email Event ASIM parser for Proofpoint POD MailLog + Version: '0.1.0' + LastUpdated: Oct 21, 2026 +Product: + Name: Proofpoint POD MailLog +Normalization: + Schema: EmailEvent + Version: '1.0.0' +References: +- Title: ASIM Email Event Schema + Link: https://aka.ms/ASimEmailEventDoc +- Title: ASIM + Link: https://aka.ms/AboutASIM +- Title: Proofpoint on Demand Log API + Link: https://proofpoint.my.site.com/community/s/article/Proofpoint-on-Demand-Pod-Log-API +Description: | + Normalizes ProofpointPODMailLog_CL SMTP receipt and delivery records to EmailEvent. + Uses the CCP table schema and repository samples; live validation is deferred. + Envelope recipients are retained in AdditionalFields, not mapped to header recipients. +ParserName: ASimEmailEventProofpointPODMailLog +EquivalentBuiltInParser: _ASim_EmailEvent_ProofpointPODMailLog +ParserParams: + - Name: disabled + Type: bool + Default: false + - Name: pack + Type: bool + Default: false +ParserQuery: | + let parser = (disabled:bool=false, pack:bool=false) { + ProofpointPODMailLog_CL + | where not(disabled) + | where isnotempty(tostring(sm.qid)) and (bag_has_key(sm, "from") or array_length(sm["to"]) > 0) + | extend + IsDelivery = coalesce(array_length(sm["to"]), 0) > 0, + Relay = tostring(sm.relay), + Dvc = coalesce(tostring(pps.agent), tostring(metadata.origin.data.agent)), + EventStartTime = coalesce(eventTime, TimeGenerated), + EmailMailFromAddress = trim(@"[<>\s]", tostring(sm["from"])), + EmailSizeBytes = tolong(sm.sizeBytes), + Dsn = tostring(sm.dsn), + DeliveryStatus = tostring(sm.stat) + | parse Relay with RelayHost " [" RelayIp "]" * + | extend + EmailMailFromAddress = iff(EmailMailFromAddress contains "@", EmailMailFromAddress, ""), + RelayHost = trim_end(@"\.", coalesce(RelayHost, Relay)), + RelayIp = iff(isnotnull(parse_ipv4(RelayIp)) or isnotempty(parse_ipv6(RelayIp)), RelayIp, ""), + EventResult = case(Dsn startswith "2.", "Success", Dsn startswith "4." or Dsn startswith "5.", "Failure", "NA"), + EmailProtocolName = case(tostring(sm.proto) =~ "SMTP" or tostring(sm.mailer) in~ ("smtp", "esmtp"), "SMTP", ""), + EventSubType = iff(IsDelivery, "EmailSent", "EmailReceived") + | extend + SrcIpAddr = iff(IsDelivery, "", RelayIp), + DstIpAddr = iff(IsDelivery, RelayIp, ""), + Src = iff(IsDelivery, Dvc, coalesce(RelayIp, RelayHost)), + Dst = iff(IsDelivery, coalesce(RelayIp, RelayHost), Dvc), + SrcFQDN = iff(not(IsDelivery) and RelayHost contains ".", RelayHost, ""), + SrcHostname = iff(IsDelivery, tostring(split(Dvc, ".")[0]), tostring(split(RelayHost, ".")[0])), + SrcDomain = iff(IsDelivery, substring(Dvc, indexof(Dvc, ".") + 1), substring(RelayHost, indexof(RelayHost, ".") + 1)), + DvcHostname = tostring(split(Dvc, ".")[0]), + DvcDomain = iff(Dvc contains ".", substring(Dvc, indexof(Dvc, ".") + 1), ""), + DvcFQDN = iff(Dvc contains ".", Dvc, ""), + EmailMailFromDomain = iff(EmailMailFromAddress contains "@", tostring(split(EmailMailFromAddress, "@")[1]), ""), + DvcAction = iff(IsDelivery and EventResult == "Success" and DeliveryStatus startswith "Sent", "Delivered", "NA") + | extend + SrcFQDN = iff(IsDelivery, DvcFQDN, SrcFQDN), + SrcDomain = iff(iff(IsDelivery, Dvc, RelayHost) contains ".", SrcDomain, ""), + DvcDomainType = iff(isnotempty(DvcDomain), "FQDN", ""), + EventEndTime = EventStartTime, + EventCount = int(1), + EventVendor = "Proofpoint", + EventProduct = "POD MailLog", + EventSchema = "EmailEvent", + EventSchemaVersion = "1.0.0", + EventType = "EmailDelivery", + EventOriginalType = "maillog", + EmailDirection = "Unknown", + EventOriginalResultDetails = strcat(Dsn, iff(isnotempty(Dsn) and isnotempty(DeliveryStatus), "; ", ""), DeliveryStatus), + EventUid = iff(isnotempty(id), strcat("ProofpointPODMailLog_CL:", id), ""), + DvcEntityKey = "", + AdditionalEntities = dynamic([]), + AdditionalFields = iff(pack, bag_pack("sm", sm, "tls", tls, "pps", pps, "metadata", metadata), dynamic({})), + Type = "ProofpointPODMailLog_CL" + | extend SrcDomainType = iff(isnotempty(SrcDomain), "FQDN", ""), IpAddr = SrcIpAddr + | project-rename EventOriginalUid = id, EventMessage = data + | project + TimeGenerated, Type, EventStartTime, EventEndTime, EventCount, + EventVendor, EventProduct, EventSchema, EventSchemaVersion, + EventType, EventSubType, EventOriginalType, EventResult, + EventOriginalResultDetails, EventOriginalUid, EventUid, EventMessage, + Dvc, DvcHostname, DvcDomain, DvcDomainType, DvcFQDN, DvcAction, DvcEntityKey, + Src, SrcIpAddr, SrcHostname, SrcDomain, SrcDomainType, SrcFQDN, Dst, DstIpAddr, IpAddr, + EmailMailFromAddress, EmailMailFromDomain, EmailSizeBytes, EmailProtocolName, EmailDirection, + AdditionalFields, AdditionalEntities + }; + parser(disabled=disabled, pack=pack) diff --git a/Parsers/ASimEmailEvent/Parsers/imEmailEvent.yaml b/Parsers/ASimEmailEvent/Parsers/imEmailEvent.yaml index b719bcf7657..bfc7d7690eb 100644 --- a/Parsers/ASimEmailEvent/Parsers/imEmailEvent.yaml +++ b/Parsers/ASimEmailEvent/Parsers/imEmailEvent.yaml @@ -1,7 +1,7 @@ Parser: Title: Email Event ASIM filtering parser - Version: '1.0.0' - LastUpdated: Sep 09, 2026 + Version: '1.0.1' + LastUpdated: Oct 21, 2026 Product: Name: Source agnostic Normalization: @@ -18,6 +18,7 @@ ParserName: imEmailEvent EquivalentBuiltInParser: _Im_EmailEvent Parsers: - _Im_EmailEvent_Empty + - _Im_EmailEvent_ProofpointPODMailLog ParserParams: - Name: starttime Type: datetime @@ -65,6 +66,12 @@ ParserQuery: | pack:bool=false) { union isfuzzy=true - vimEmailEventEmpty + vimEmailEventEmpty, + vimEmailEventProofpointPODMailLog( + starttime=starttime, endtime=endtime, ipaddr_has_any_prefix=ipaddr_has_any_prefix, + emailfromaddress_has_any=emailfromaddress_has_any, emailrecipient_has_any=emailrecipient_has_any, + emailsubject_has_any=emailsubject_has_any, emaildirection_in=emaildirection_in, + emaildeliverylocation_in=emaildeliverylocation_in, eventresult=eventresult, pack=pack, + disabled=(imBuiltInDisabled or ('ExcludevimEmailEventProofpointPODMailLog' in (DisabledParsers)))) }; parser (starttime=starttime, endtime=endtime, ipaddr_has_any_prefix=ipaddr_has_any_prefix, emailfromaddress_has_any=emailfromaddress_has_any, emailrecipient_has_any=emailrecipient_has_any, emailsubject_has_any=emailsubject_has_any, emaildirection_in=emaildirection_in, emaildeliverylocation_in=emaildeliverylocation_in, eventresult=eventresult, pack=pack) \ No newline at end of file diff --git a/Parsers/ASimEmailEvent/Parsers/vimEmailEventProofpointPODMailLog.yaml b/Parsers/ASimEmailEvent/Parsers/vimEmailEventProofpointPODMailLog.yaml new file mode 100644 index 00000000000..3e77e485b95 --- /dev/null +++ b/Parsers/ASimEmailEvent/Parsers/vimEmailEventProofpointPODMailLog.yaml @@ -0,0 +1,149 @@ +Parser: + Title: Email Event ASIM filtering parser for Proofpoint POD MailLog + Version: '0.1.0' + LastUpdated: Oct 21, 2026 +Product: + Name: Proofpoint POD MailLog +Normalization: + Schema: EmailEvent + Version: '1.0.0' +References: +- Title: ASIM Email Event Schema + Link: https://aka.ms/ASimEmailEventDoc +- Title: ASIM + Link: https://aka.ms/AboutASIM +- Title: Proofpoint on Demand Log API + Link: https://proofpoint.my.site.com/community/s/article/Proofpoint-on-Demand-Pod-Log-API +Description: | + Filters and normalizes ProofpointPODMailLog_CL SMTP receipt and delivery records to EmailEvent. + Uses the CCP table schema and repository samples; live validation is deferred. + Header sender, header recipients, subject and delivery location are unavailable; + nonempty filters for these fields return no records. +ParserName: vimEmailEventProofpointPODMailLog +EquivalentBuiltInParser: _Im_EmailEvent_ProofpointPODMailLog +ParserParams: + - Name: starttime + Type: datetime + Default: datetime(null) + - Name: endtime + Type: datetime + Default: datetime(null) + - Name: ipaddr_has_any_prefix + Type: dynamic + Default: dynamic([]) + - Name: emailfromaddress_has_any + Type: dynamic + Default: dynamic([]) + - Name: emailrecipient_has_any + Type: dynamic + Default: dynamic([]) + - Name: emailsubject_has_any + Type: dynamic + Default: dynamic([]) + - Name: emaildirection_in + Type: dynamic + Default: dynamic([]) + - Name: emaildeliverylocation_in + Type: dynamic + Default: dynamic([]) + - Name: eventresult + Type: string + Default: '*' + - Name: disabled + Type: bool + Default: false + - Name: pack + Type: bool + Default: false +ParserQuery: | + let parser = ( + starttime:datetime=datetime(null), + endtime:datetime=datetime(null), + ipaddr_has_any_prefix:dynamic=dynamic([]), + emailfromaddress_has_any:dynamic=dynamic([]), + emailrecipient_has_any:dynamic=dynamic([]), + emailsubject_has_any:dynamic=dynamic([]), + emaildirection_in:dynamic=dynamic([]), + emaildeliverylocation_in:dynamic=dynamic([]), + eventresult:string='*', + disabled:bool=false, + pack:bool=false) + { + ProofpointPODMailLog_CL + | where not(disabled) + | where (isnull(starttime) or eventTime >= starttime or (isnull(eventTime) and TimeGenerated >= starttime)) + and (isnull(endtime) or eventTime <= endtime or (isnull(eventTime) and TimeGenerated <= endtime)) + | where array_length(emailfromaddress_has_any) == 0 and array_length(emailrecipient_has_any) == 0 + and array_length(emailsubject_has_any) == 0 and array_length(emaildeliverylocation_in) == 0 + | where array_length(emaildirection_in) == 0 or "Unknown" in (emaildirection_in) + | where isnotempty(tostring(sm.qid)) and (bag_has_key(sm, "from") or array_length(sm["to"]) > 0) + | extend + IsDelivery = coalesce(array_length(sm["to"]), 0) > 0, + Dsn = tostring(sm.dsn) + | extend EventResult = case(Dsn startswith "2.", "Success", Dsn startswith "4." or Dsn startswith "5.", "Failure", "NA") + | where eventresult == "*" or EventResult == eventresult + | extend + Relay = tostring(sm.relay), + Dvc = coalesce(tostring(pps.agent), tostring(metadata.origin.data.agent)), + EventStartTime = coalesce(eventTime, TimeGenerated), + EmailMailFromAddress = trim(@"[<>\s]", tostring(sm["from"])), + EmailSizeBytes = tolong(sm.sizeBytes), + DeliveryStatus = tostring(sm.stat) + | parse Relay with RelayHost " [" RelayIp "]" * + | extend + EmailMailFromAddress = iff(EmailMailFromAddress contains "@", EmailMailFromAddress, ""), + RelayHost = trim_end(@"\.", coalesce(RelayHost, Relay)), + RelayIp = iff(isnotnull(parse_ipv4(RelayIp)) or isnotempty(parse_ipv6(RelayIp)), RelayIp, ""), + EmailProtocolName = case(tostring(sm.proto) =~ "SMTP" or tostring(sm.mailer) in~ ("smtp", "esmtp"), "SMTP", ""), + EventSubType = iff(IsDelivery, "EmailSent", "EmailReceived") + | extend + SrcIpAddr = iff(IsDelivery, "", RelayIp), + DstIpAddr = iff(IsDelivery, RelayIp, ""), + Src = iff(IsDelivery, Dvc, coalesce(RelayIp, RelayHost)), + Dst = iff(IsDelivery, coalesce(RelayIp, RelayHost), Dvc), + SrcFQDN = iff(not(IsDelivery) and RelayHost contains ".", RelayHost, ""), + SrcHostname = iff(IsDelivery, tostring(split(Dvc, ".")[0]), tostring(split(RelayHost, ".")[0])), + SrcDomain = iff(IsDelivery, substring(Dvc, indexof(Dvc, ".") + 1), substring(RelayHost, indexof(RelayHost, ".") + 1)), + DvcHostname = tostring(split(Dvc, ".")[0]), + DvcDomain = iff(Dvc contains ".", substring(Dvc, indexof(Dvc, ".") + 1), ""), + DvcFQDN = iff(Dvc contains ".", Dvc, ""), + EmailMailFromDomain = iff(EmailMailFromAddress contains "@", tostring(split(EmailMailFromAddress, "@")[1]), ""), + DvcAction = iff(IsDelivery and EventResult == "Success" and DeliveryStatus startswith "Sent", "Delivered", "NA") + | where array_length(ipaddr_has_any_prefix) == 0 + or has_any_ipv4_prefix(SrcIpAddr, ipaddr_has_any_prefix) + or has_any_ipv4_prefix(DstIpAddr, ipaddr_has_any_prefix) + | extend + SrcFQDN = iff(IsDelivery, DvcFQDN, SrcFQDN), + SrcDomain = iff(iff(IsDelivery, Dvc, RelayHost) contains ".", SrcDomain, ""), + DvcDomainType = iff(isnotempty(DvcDomain), "FQDN", ""), + EventEndTime = EventStartTime, + EventCount = int(1), + EventVendor = "Proofpoint", + EventProduct = "POD MailLog", + EventSchema = "EmailEvent", + EventSchemaVersion = "1.0.0", + EventType = "EmailDelivery", + EventOriginalType = "maillog", + EmailDirection = "Unknown", + EventOriginalResultDetails = strcat(Dsn, iff(isnotempty(Dsn) and isnotempty(DeliveryStatus), "; ", ""), DeliveryStatus), + EventUid = iff(isnotempty(id), strcat("ProofpointPODMailLog_CL:", id), ""), + DvcEntityKey = "", + AdditionalEntities = dynamic([]), + AdditionalFields = iff(pack, bag_pack("sm", sm, "tls", tls, "pps", pps, "metadata", metadata), dynamic({})), + Type = "ProofpointPODMailLog_CL" + | extend SrcDomainType = iff(isnotempty(SrcDomain), "FQDN", ""), IpAddr = SrcIpAddr + | project-rename EventOriginalUid = id, EventMessage = data + | project + TimeGenerated, Type, EventStartTime, EventEndTime, EventCount, + EventVendor, EventProduct, EventSchema, EventSchemaVersion, + EventType, EventSubType, EventOriginalType, EventResult, + EventOriginalResultDetails, EventOriginalUid, EventUid, EventMessage, + Dvc, DvcHostname, DvcDomain, DvcDomainType, DvcFQDN, DvcAction, DvcEntityKey, + Src, SrcIpAddr, SrcHostname, SrcDomain, SrcDomainType, SrcFQDN, Dst, DstIpAddr, IpAddr, + EmailMailFromAddress, EmailMailFromDomain, EmailSizeBytes, EmailProtocolName, EmailDirection, + AdditionalFields, AdditionalEntities + }; + parser(starttime=starttime, endtime=endtime, ipaddr_has_any_prefix=ipaddr_has_any_prefix, + emailfromaddress_has_any=emailfromaddress_has_any, emailrecipient_has_any=emailrecipient_has_any, + emailsubject_has_any=emailsubject_has_any, emaildirection_in=emaildirection_in, + emaildeliverylocation_in=emaildeliverylocation_in, eventresult=eventresult, disabled=disabled, pack=pack) diff --git a/Parsers/ASimEmailEvent/README.md b/Parsers/ASimEmailEvent/README.md index b4298540428..526b4b6f4d5 100644 --- a/Parsers/ASimEmailEvent/README.md +++ b/Parsers/ASimEmailEvent/README.md @@ -8,4 +8,91 @@ For more information, see: - [Normalization and the Advanced Security Information Model (ASIM)](https://aka.ms/AboutASIM) - [Deploy all of ASIM](https://aka.ms/DeployASIM) -- [ASIM Email Event normalization schema reference](https://aka.ms/ASimEmailEventDoc) \ No newline at end of file +- [ASIM Email Event normalization schema reference](https://aka.ms/ASimEmailEventDoc) + +## Proofpoint POD MailLog + +`ASimEmailEventProofpointPODMailLog` and `vimEmailEventProofpointPODMailLog` +normalize the CCP table `ProofpointPODMailLog_CL` to EmailEvent 1.0.0. +The legacy `ProofpointPOD_maillog_CL` table and `ProofpointPODMessage_CL` are +not inputs to these functions. Both parsers read the source once and emit at +most one row for each source row. They select message-related sendmail records +with a queue ID and either an envelope sender property or nonempty envelope +recipients; TLS-only and other non-message records are not normalized. + +### Evidence and mappings + +The source contract is the solution's +[CCP table definition](../../Solutions/Proofpoint%20On%20demand%28POD%29%20Email%20Security/Data%20Connectors/ProofPointEmailSecurity_CCP/ProofpointPOD_Table_MailLog.json) +and [DCR transform](../../Solutions/Proofpoint%20On%20demand%28POD%29%20Email%20Security/Data%20Connectors/ProofPointEmailSecurity_CCP/ProofpointPOD_DCR.json). +The [20 existing raw samples](../../Sample%20Data/Custom/ProofpointPOD_maillog_CL.json) +provide the observed `sm` and device fields. The ASIM fixture +`Sample Data/ASIM/Proofpoint_POD MailLog_EmailEvent_IngestedLogs.csv` is a +local transformation of those samples into the CCP shape, not a workspace export. +Its timestamps use the source `ts` value for reproducibility; the real DCR +sets `TimeGenerated` to ingestion time and `eventTime` to `todatetime(ts)`. + +| Source | ASIM output | +| --- | --- | +| `TimeGenerated` | `TimeGenerated` | +| `eventTime` (fallback: `TimeGenerated`) | `EventStartTime`, `EventEndTime` | +| `id` | `EventOriginalUid`; namespaced as `EventUid` | +| `data` | `EventMessage` | +| `pps.agent` (fallback: `metadata.origin.data.agent`) | `Dvc` and device hostname/domain fields | +| Presence of `sm.from` or nonempty `sm.to`, with `sm.qid` | Select receipt/delivery records; nonempty `sm.to` identifies `EmailSent`, otherwise `EmailReceived` | +| `sm.relay` | Remote host/IP; source on receipt, destination on delivery | +| Reporting device | Destination on receipt, source on delivery | +| `sm.from` | `EmailMailFromAddress`, `EmailMailFromDomain` after removing surrounding whitespace/angle brackets | +| `sm.sizeBytes` | `EmailSizeBytes` as a long | +| `sm.proto`, `sm.mailer` | `EmailProtocolName` (`SMTP` for SMTP/esmtp) | +| `sm.dsn` | `EventResult`: `2.x.x` success, `4.x.x`/`5.x.x` failure, otherwise `NA` | +| `sm.dsn`, `sm.stat` | `EventOriginalResultDetails` | +| `sm.stat` starting with `Sent`, with successful DSN | `DvcAction=Delivered` for transport handoff, not proof of inbox placement | +| `sm`, `tls`, `pps`, `metadata` | Original objects in `AdditionalFields` when `pack=true`; empty dictionary otherwise | + +The reporting device may be either endpoint of an SMTP hop; `EmailDirection` +is therefore `Unknown`, not an inferred organizational inbound/outbound direction. +Receipt rows do not contain a delivery result and return `EventResult=NA`. +The parser does not repair the truncated sender present in one repository sample: +sender strings without `@` are left unmapped, with the original in +`AdditionalFields.sm` when packing is enabled. This is not full RFC mailbox validation. + +`sm.from` and `sm.to` are **envelope** addresses, not `From`/`To` header fields. +The parser deliberately leaves header sender/recipient, subject, Reply-To, +delivery location, attachment counts and SPF/DKIM/DMARC results unmapped. +Envelope recipients and queue/message identifiers remain available in +`AdditionalFields.sm`; the parser does not choose one recipient, expand arrays +or correlate receipt and delivery rows. TLS verification is not an email +authentication result. + +### Filtering and validation status + +The filtering parser supports event-time bounds (with ingestion-time fallback +only when `eventTime` is null), source/destination IPv4 prefixes, normalized +result and the known direction `Unknown`. Both endpoints are checked for the +IPv4-prefix filter; IPv6 prefix matching is not implemented. +Nonempty `emailfromaddress_has_any`, `emailrecipient_has_any`, +`emailsubject_has_any` or `emaildeliverylocation_in` filters return no rows: +the corresponding normalized header/location fields are unavailable, even +when envelope addresses are present. All parameters are forwarded by +`imEmailEvent`. `disabled=true` returns no rows. + +Live sampling, KQL execution, `ASimSchemaTester`, `ASimDataTester` and filter +tests are **deferred by user request**, not passed. The supplied workspace +did not contain this table. No table creation, ingestion or deployment has +been performed. The Proofpoint API reference requires sign-in; task-specific +Azure DevOps acceptance criteria were also unavailable. SMTP status meanings +follow [RFC 3463](https://www.rfc-editor.org/rfc/rfc3463). +Before release, verify the source shape and mixed receipt/delivery mappings +against real CCP records, run both ASIM testers and filter tests, and add the +actual pull-request links to the four affected changelogs. Missing recommended +header/security fields are intentional source limitations, not evidence of +empty headers or zero attachments. + +The repository's focused `Validate_ParsersFunctions_HaveValidKql` tests currently +fail with `KS204`: their custom-table catalog does not define +`ProofpointPODMailLog_CL`. The standalone offline compiler checks use the +connector's declared schema and pass, but do not replace those repository tests. +A maintainer must add the CCP table definition to the protected validation +catalog before those tests can pass; this parser task does not modify tooling +or test implementation. \ No newline at end of file diff --git a/Sample Data/ASIM/ProofpointPODMailLog_CL_Schema.csv b/Sample Data/ASIM/ProofpointPODMailLog_CL_Schema.csv new file mode 100644 index 00000000000..c193100e7fd --- /dev/null +++ b/Sample Data/ASIM/ProofpointPODMailLog_CL_Schema.csv @@ -0,0 +1,9 @@ +ColumnName,ColumnType +TimeGenerated,datetime +id,string +data,string +eventTime,datetime +metadata,dynamic +pps,dynamic +sm,dynamic +tls,dynamic diff --git a/Sample Data/ASIM/Proofpoint_POD MailLog_EmailEvent_IngestedLogs.csv b/Sample Data/ASIM/Proofpoint_POD MailLog_EmailEvent_IngestedLogs.csv new file mode 100644 index 00000000000..3036eb2b5fb --- /dev/null +++ b/Sample Data/ASIM/Proofpoint_POD MailLog_EmailEvent_IngestedLogs.csv @@ -0,0 +1,21 @@ +TimeGenerated,id,data,eventTime,metadata,pps,sm,tls,Type +2020-11-05T12:59:43.881767+01:00,rGWIupM+g6R4PWIeqf6SMA,"2020-11-05T12:59:43.881767+01:00 t1234555 sendmail[18160]: 0A0BqJgv000000: to=, delay=00:00:00, xdelay=00:00:00, mailer=esmtp, tls_verify=OK, pri=154217, relay=relay.mail.net. [10.10.10.10], dsn=2.0.0, stat=Sent ( [InternalId=3444563792735, Hostname=eurprd01.prod.exchangelabs.com] 45616 bytes in 0.073, 610.105 KB/sec Queued mail for delivery)",2020-11-05T12:59:43.881767+01:00,"{""origin"":{""data"":{""agent"":""t1234555.ppops.net"",""cid"":""CID""}}}","{""cid"":""CID"",""agent"":""t1234555.popps.net""}","{""mailer"":""esmtp"",""xdelay"":""00:00:00"",""delay"":""00:00:00"",""tls"":{""verify"":""OK""},""dsn"":""2.0.0"",""to"":[""""],""relay"":""relay.mail.net [10.10.10.10]"",""stat"":""Sent ( [InternalId"",""pri"":""154217"",""qid"":""0A0BqJgv000000""}",null,ProofpointPODMailLog_CL +2020-11-05T12:59:44.487319+01:00,tnsgrBLZ9hdWbGRd3HMS9A,"2020-11-05T12:59:44.487319+01:00 t1234555 sendmail[18157]: 0A0BqJgv000000: to=, delay=00:00:03, xdelay=00:00:03, mailer=esmtp, tls_verify=OK, pri=2242435, relay=relay.mail.net. [10.10.10.10], dsn=2.0.0, stat=Sent ( [InternalId=18167711674865, Hostname=eurprd01.prod.exchangelabs.com] 2159936 bytes in 0.641, 3288.805 KB/sec Queued mail for delivery)",2020-11-05T12:59:44.487319+01:00,"{""origin"":{""data"":{""agent"":""t1234555.ppops.net"",""cid"":""CID""}}}","{""cid"":""CID"",""agent"":""t1234555.ppops.net""}","{""delay"":""00:00:03"",""dsn"":""2.0.0"",""tls"":{""verify"":""OK""},""xdelay"":""00:00:03"",""mailer"":""esmtp"",""qid"":""0A0BqJgv000000"",""pri"":""2242435"",""relay"":""relay.mail.net [10.10.10.10]"",""to"":[""""],""stat"":""Sent ( [InternalId""}",null,ProofpointPODMailLog_CL +2020-11-05T12:59:45.217049+01:00,Rv4coMMYHs/yWaGyMlR49g,"2020-11-05T12:59:45.217049+01:00 t1234555 sendmail[30248]: 0A5BqKs4030248: from=, size=8255, class=0, nrcpts=1, msgid=, proto=SMTP, daemon=MTA, tls_verify=NONE, auth=NONE, relay=t1234555.ppops.net [127.0.0.1]",2020-11-05T12:59:45.217049+01:00,"{""origin"":{""data"":{""agent"":""t1234555.ppops.net"",""cid"":""CID""}}}","{""agent"":""t1234555.ppops.net"",""cid"":""CID""}","{""from"":"""",""auth"":""NONE"",""class"":""0"",""proto"":""SMTP"",""daemon"":""MTA"",""qid"":""0A5BqKs4030248"",""sizeBytes"":""8255"",""nrcpts"":""1"",""relay"":""t1234555.ppops.net [127.0.0.1]""}",null,ProofpointPODMailLog_CL +2020-11-05T12:59:46.045430+01:00,wYvxXQ8YHRvWgpopQeLskA,"2020-11-05T12:59:46.045430+01:00 t1234555 sendmail[18165]: 0A5BqKs4030248: to=, delay=00:00:01, xdelay=00:00:01, mailer=esmtp, tls_verify=OK, pri=128255, relay=externals-adidasgr...ction.outlook.com. [10.10.10.10], dsn=2.0.0, stat=Sent ( [InternalId=...B6347.eurprd01.prod.exchangelabs.com] 19101 bytes in 0.071, 260.062 KB/sec Queued mail for delivery)",2020-11-05T12:59:46.045430+01:00,"{""origin"":{""data"":{""agent"":""t1234555.ppops.net"",""cid"":""CID""}}}","{""cid"":""CID"",""agent"":""t1234555.ppops.net""}","{""pri"":""128255"",""qid"":""0A5BqKs4030248"",""stat"":""Sent ( [InternalId"",""relay"":""relay.mail.net [10.10.10.10]"",""to"":[""""],""tls"":{""verify"":""OK""},""dsn"":""2.0.0"",""delay"":""00:00:01"",""xdelay"":""00:00:01"",""mailer"":""esmtp""}",null,ProofpointPODMailLog_CL +2020-11-05T12:59:49.375433+01:00,S57nYRhfj7qb0FUTymKhfg,"2020-11-05T12:59:49.375433+01:00 t1234555 sendmail[30448]: 0A5BqZtL030448: from=, size=148995, class=0, nrcpts=1, msgid=, proto=SMTP, daemon=MTA, tls_verify=NONE, auth=NONE, relay=t1234555.ppops.net [127.0.0.1]",2020-11-05T12:59:49.375433+01:00,"{""origin"":{""data"":{""agent"":""t1234555.ppops.net"",""cid"":""CID""}}}","{""agent"":""t1234555.ppops.net"",""cid"":""CID""}","{""relay"":""relay.mail.net [10.10.10.10]"",""nrcpts"":""1"",""daemon"":""MTA"",""sizeBytes"":""148995"",""qid"":""0A5BqZtL030448"",""proto"":""SMTP"",""auth"":""NONE"",""class"":""0"",""msgid"":"""",""tls"":{""verify"":""NONE""},""from"":""""}",null,ProofpointPODMailLog_CL +2020-11-05T12:59:50.261397+01:00,KmK6ghZhp+8N+k+y5M/ZrA,"2020-11-05T12:59:50.261397+01:00 t1234555 sendmail[30240]: 0A5BqJBl030240: from=, size=560625, class=0, nrcpts=1, msgid=, proto=SMTP, daemon=MTA, tls_verify=NONE, auth=NONE, relay=t1234555.ppops.net [127.0.0.1]",2020-11-05T12:59:50.261397+01:00,"{""origin"":{""data"":{""agent"":""t1234555.ppops.net"",""cid"":""CID""}}}","{""agent"":""t1234555.ppops.net"",""cid"":""CID""}","{""relay"":""relay.mail.net [10.10.10.10]"",""nrcpts"":""1"",""daemon"":""MTA"",""sizeBytes"":""560625"",""qid"":""0A5BqJBl030240"",""proto"":""SMTP"",""auth"":""NONE"",""class"":""0"",""tls"":{""verify"":""NONE""},""msgid"":"""",""from"":""""}",null,ProofpointPODMailLog_CL +2020-11-05T12:59:50.356977+01:00,6o9hWtB++VRhC2HZCU0/mQ,"2020-11-05T12:59:50.356977+01:00 t1234555 sendmail[30427]: 0A5BqWEW030427: from=, size=24805, class=0, nrcpts=1, msgid=, proto=SMTP, daemon=MTA, tls_verify=NONE, auth=NONE, relay=t1234555.ppops.net [127.0.0.1]",2020-11-05T12:59:50.356977+01:00,"{""origin"":{""data"":{""agent"":""t1234555.ppops.net"",""cid"":""CID""}}}","{""cid"":""CID"",""agent"":""t1234555.ppops.net""}","{""nrcpts"":""1"",""relay"":""relay.mail.net [10.10.10.10]"",""daemon"":""MTA"",""qid"":""0A5BqWEW030427"",""sizeBytes"":""24805"",""class"":""0"",""auth"":""NONE"",""proto"":""SMTP"",""from"":"""",""msgid"":"""",""tls"":{""verify"":""NONE""}}",null,ProofpointPODMailLog_CL +2020-11-05T12:59:51.698428+01:00,jOtbUua8OnKtoWNglHt4GA,"2020-11-05T12:59:51.698428+01:00 t1234555 sendmail[18173]: 0A5BqZtL030448: to=, delay=00:00:02, xdelay=00:00:02, mailer=esmtp, tls_verify=OK, pri=268995, relay=reebok-com.mail.protection.outlook.com. [104.47.1.36], dsn=2.0.0, stat=Sent ( [InternalId=8723078591673, Hostname=AM0PR...281.eurprd01.prod.exchangelabs.com] 165052 bytes in 0.159, 1010.909 KB/sec Queued mail for delivery)",2020-11-05T12:59:51.698428+01:00,"{""origin"":{""data"":{""agent"":""t1234555.ppops.net"",""cid"":""CID""}}}","{""cid"":""CID"",""agent"":""t1234555.ppops.net""}","{""mailer"":""esmtp"",""xdelay"":""00:00:02"",""delay"":""00:00:02"",""tls"":{""verify"":""OK""},""dsn"":""2.0.0"",""to"":[""""],""relay"":""relay.mail.net [10.10.10.10]"",""stat"":""Sent ( [InternalId"",""qid"":""0A5BqZtL030448"",""pri"":""268995""}",null,ProofpointPODMailLog_CL +2020-11-05T12:59:51.739825+01:00,c5Fw7jOLm46PeqG8UJEWbg,"2020-11-05T12:59:51.739825+01:00 t1234555 sendmail[18175]: 0A5BqJBl030240: to=, delay=00:00:01, xdelay=00:00:01, mailer=esmtp, tls_verify=OK, pri=680625, relay=relay.mail.net. [10.10.10.10], dsn=2.0.0, stat=Sent ( [InternalId=2...285.eurprd01.prod.exchangelabs.com] 583928 bytes in 0.236, 2413.571 KB/sec Queued mail for delivery)",2020-11-05T12:59:51.739825+01:00,"{""origin"":{""data"":{""agent"":""t1234555.ppops.net"",""cid"":""CID""}}}","{""cid"":""CID"",""agent"":""t1234555.ppops.net""}","{""stat"":""Sent ( [InternalId"",""relay"":""relay.mail.net [10.10.10.10]"",""to"":[""""],""qid"":""0A5BqJBl030240"",""pri"":""680625"",""xdelay"":""00:00:01"",""mailer"":""esmtp"",""tls"":{""verify"":""OK""},""dsn"":""2.0.0"",""delay"":""00:00:01""}",null,ProofpointPODMailLog_CL +2020-11-05T12:59:51.822918+01:00,/ALzUxRLM2Tz0YJbdoltTQ,"2020-11-05T12:59:51.822918+01:00 t1234555 sendmail[18177]: 0A5BqWEW030427: to=, delay=00:00:01, xdelay=00:00:01, mailer=esmtp, tls_verify=OK, pri=144805, relay=relay.mail.net. [10.10.10.10], dsn=2.0.0, stat=Sent ( [InternalI...B2686.eurprd01.prod.exchangelabs.com] 36596 bytes in 0.149, 239.832 KB/sec Queued mail for delivery)",2020-11-05T12:59:51.822918+01:00,"{""origin"":{""data"":{""agent"":""t1234555.ppops.net"",""cid"":""CID""}}}","{""cid"":""CID"",""agent"":""t1234555.ppops.net""}","{""qid"":""0A5BqWEW030427"",""pri"":""144805"",""stat"":""Sent ( [InternalI...B2686.eurprd01.prod.exchangelabs.com] 36596 bytes in 0.149"",""relay"":""relay.mail.net [10.10.10.10]"",""to"":[""""],""tls"":{""verify"":""OK""},""dsn"":""2.0.0"",""delay"":""00:00:01"",""xdelay"":""00:00:01"",""mailer"":""esmtp""}",null,ProofpointPODMailLog_CL +2020-11-05T12:59:52.368451+01:00,z++bC1rsPj/CWSn4ZScnTg,"2020-11-05T12:59:52.368451+01:00 t1234555 sendmail[30243]: 0A5BqJgw030243: from=, size=301268, class=0, nrcpts=2, msgid=, proto=SMTP, daemon=MTA, tls_verify=NONE, auth=NONE, relay=t1234555.ppops.net [127.0.0.1]",2020-11-05T12:59:52.368451+01:00,"{""origin"":{""data"":{""agent"":""t1234555.ppops.net"",""cid"":""CID""}}}","{""agent"":""t1234555.ppops.net"",""cid"":""CID""}","{""daemon"":""MTA"",""qid"":""0A5BqJgw030243"",""sizeBytes"":""301268"",""nrcpts"":""2"",""relay"":""relay.mail.net [10.10.10.10]"",""from"":"""",""msgid"":"""",""tls"":{""verify"":""NONE""},""auth"":""NONE"",""class"":""0"",""proto"":""SMTP""}",null,ProofpointPODMailLog_CL +2020-11-05T12:59:53.128685+01:00,EdOJdJWaYM6H2KYMjj0A8w,"2020-11-05T12:59:53.128685+01:00 t1234555 sendmail[30240]: 0A5BqJBm030240: from=, size=301692, class=0, nrcpts=1, msgid=, proto=SMTP, daemon=MTA, tls_verify=NONE, auth=NONE, relay=t1234555.ppops.net [127.0.0.1]",2020-11-05T12:59:53.128685+01:00,"{""origin"":{""data"":{""agent"":""t1234555.ppops.net"",""cid"":""CID""}}}","{""agent"":""t1234555.ppops.net"",""cid"":""CID""}","{""class"":""0"",""auth"":""NONE"",""proto"":""SMTP"",""from"":"""",""msgid"":"""",""tls"":{""verify"":""NONE""},""nrcpts"":""1"",""relay"":""relay.mail.net [10.10.10.10]"",""qid"":""0A5BqJBm030240"",""sizeBytes"":""301692"",""daemon"":""MTA""}",null,ProofpointPODMailLog_CL +2020-11-05T12:59:54.447139+01:00,o1LSqlKOOCILO8WDfvM+SQ,"2020-11-05T12:59:54.447139+01:00 t1234555 sendmail[30233]: 0A5BqHS0030233: from=, size=65592, class=-60, nrcpts=1, msgid=, proto=SMTP, daemon=MTA, tls_verify=NONE, auth=NONE, relay=t1234555.ppops.net [127.0.0.1]",2020-11-05T12:59:54.447139+01:00,"{""origin"":{""data"":{""agent"":""t1234555.ppops.net"",""cid"":""CID""}}}","{""agent"":""t1234555.ppops.net"",""cid"":""CID""}","{""auth"":""NONE"",""class"":""-60"",""proto"":""SMTP"",""from"":"""",""msgid"":"""",""tls"":{""verify"":""NONE""},""nrcpts"":""1"",""relay"":""relay.mail.net [10.10.10.10]"",""qid"":""0A5BqHS0030233"",""sizeBytes"":""65592"",""daemon"":""MTA""}",null,ProofpointPODMailLog_CL +2020-11-05T12:59:54.882947+01:00,nSWY7WY+wg3vqLBEV9rlFA,"2020-11-05T12:59:54.882947+01:00 t1234555 sendmail[30448]: 0A5BqZtM030448: from=, size=1060602, class=0, nrcpts=1, msgid=, proto=SMTP, daemon=MTA, tls_verify=NONE, auth=NONE, relay=t1234555.ppops.net [127.0.0.1]",2020-11-05T12:59:54.882947+01:00,"{""origin"":{""data"":{""agent"":""t1234555.ppops.net"",""cid"":""CID""}}}","{""cid"":""CID"",""agent"":""t1234555.ppops.net""}","{""from"":"""",""tls"":{""verify"":""NONE""},""msgid"":"""",""class"":""0"",""auth"":""NONE"",""proto"":""SMTP"",""qid"":""0A5BqZtM030448"",""sizeBytes"":""1060602"",""daemon"":""MTA"",""nrcpts"":""1"",""relay"":""t1234555.ppops.net [127.0.0.1]""}",null,ProofpointPODMailLog_CL +2020-11-05T12:59:55.581745+01:00,nQsGhy3nue3O9PXE9cyZ4A,"2020-11-05T12:59:55.581745+01:00 t1234555 sendmail[30464]: 0A5Bqbi3030464: from=, size=67596, class=-60, nrcpts=4, msgid=, proto=SMTP, daemon=MTA, tls_verify=NONE, auth=NONE, relay=t1234555.ppops.net [127.0.0.1]",2020-11-05T12:59:55.581745+01:00,"{""origin"":{""data"":{""agent"":""t1234555.ppops.net"",""cid"":""CID""}}}","{""cid"":""CID"",""agent"":""t1234555.ppops.net""}","{""class"":""-60"",""auth"":""NONE"",""proto"":""SMTP"",""from"":"""",""tls"":{""verify"":""NONE""},""msgid"":"""",""nrcpts"":""4"",""relay"":""relay.mail.net [10.10.10.10]"",""daemon"":""MTA"",""qid"":""0A5Bqbi3030464"",""sizeBytes"":""67596""}",null,ProofpointPODMailLog_CL +2020-11-05T12:59:55.702601+01:00,TLgvfc+EnwlgpGstT4gKKg,"2020-11-05T12:59:55.702601+01:00 t1234555 sendmail[18188]: 0A5BqJBm030240: to=, delay=00:00:02, xdelay=00:00:02, mailer=esmtp, tls_verify=OK, pri=421692, relay=relay.mail.net. [104.47.0.36], dsn=2.0.0, stat=Sent ( [InternalId=4...537.eurprd01.prod.exchangelabs.com] 319576 bytes in 0.197, 1576.411 KB/sec Queued mail for delivery)",2020-11-05T12:59:55.702601+01:00,"{""origin"":{""data"":{""agent"":""t1234555.ppops.net"",""cid"":""CID""}}}","{""cid"":""CID"",""agent"":""t1234555.ppops.net""}","{""delay"":""00:00:02"",""dsn"":""2.0.0"",""tls"":{""verify"":""OK""},""mailer"":""esmtp"",""xdelay"":""00:00:02"",""pri"":""421692"",""qid"":""0A5BqJBm030240"",""to"":[""""],""relay"":""relay.mail.net [10.10.10.10]"",""stat"":""Sent ( [InternalId""}",null,ProofpointPODMailLog_CL +2020-11-05T12:59:25.616354+01:00,4k8PCXVfzzmB3MOLxMJzow,"2020-11-05T12:59:25.616354+01:00 m0000001 sendmail[25015]: 0A5Bs6pD012719: to=, delay=00:00:02, xdelay=00:00:02, mailer=esmtp, tls_verify=OK, pri=172457, relay=relay.mail.net. [104.47.0.36], dsn=2.0.0, stat=Sent ( [InternalId=57157424788340, Ho...MB5237.eurprd01.prod.exchangelabs.com] 63604 bytes in 1.095, 56.722 KB/sec Queued mail for delivery)",2020-11-05T12:59:25.616354+01:00,"{""origin"":{""data"":{""agent"":""m0000001.ppops.net"",""cid"":""CID""}}}","{""cid"":""CID"",""agent"":""m0000001.ppops.net""}","{""relay"":""relay.mail.net [10.10.10.10]"",""to"":[""""],""dsn"":""2.0.0"",""qid"":""0A5Bs6pD012719"",""tls"":{""verify"":""OK""},""mailer"":""esmtp"",""stat"":""Sent ( [InternalId"",""xdelay"":""00:00:02"",""delay"":""00:00:02"",""pri"":""172457""}",null,ProofpointPODMailLog_CL +2020-11-05T12:59:25.782683+01:00,yHgmo29ReWNG+wG9z2081w,"2020-11-05T12:59:25.782683+01:00 m0000001 sendmail[12716]: 0A5Bs66u012716: from=, size=133518, class=0, nrcpts=1, msgid=, proto=SMTP, daemon=MTA, tls_verify=NONE, auth=NONE, relay=m0000001.ppops.net [127.0.0.1]",2020-11-05T12:59:25.782683+01:00,"{""origin"":{""data"":{""agent"":""m0000001.ppops.net"",""cid"":""CID""}}}","{""cid"":""CID"",""agent"":""m0000001.ppops.net""}","{""daemon"":""MTA"",""sizeBytes"":""133518"",""from"":"""",""nrcpts"":""1"",""tls"":{""verify"":""NONE""},""class"":""0"",""qid"":""0A5Bs66u012716"",""relay"":""relay.mail.net [10.10.10.10]"",""proto"":""SMTP"",""auth"":""NONE"",""msgid"":""""}",null,ProofpointPODMailLog_CL +2020-11-05T12:59:27.214776+01:00,/7mPV7fork8xQ2FLmmV5gA,"2020-11-05T12:59:27.214776+01:00 m0000001 sendmail[25035]: 0A5Bs66u012716: to=, delay=00:00:02, xdelay=00:00:02, mailer=esmtp, tls_verify=OK, pri=253518, relay=relay.mail.net. [104.47.0.36], dsn=2.0.0, stat=Sent ( [InternalId=50349901633126, Hos...245.eurprd01.prod.exchangelabs.com] 147801 bytes in 0.083, 1737.387 KB/sec Queued mail for delivery)",2020-11-05T12:59:27.214776+01:00,"{""origin"":{""data"":{""agent"":""m0000001.ppops.net"",""cid"":""CID""}}}","{""cid"":""CID"",""agent"":""m0000001.ppops.net""}","{""xdelay"":""00:00:02"",""stat"":""Sent ( [InternalId"",""delay"":""00:00:02"",""pri"":""253518"",""dsn"":""2.0.0"",""to"":[""""],""relay"":""relay.mail.net [10.10.10.10]"",""mailer"":""esmtp"",""qid"":""0A5Bs66u012716"",""tls"":{""verify"":""OK""}}",null,ProofpointPODMailLog_CL +2020-11-05T12:59:27.672144+01:00,kcFh9j7CGJ5CnjZ1l/0fYw,"2020-11-05T12:59:27.672144+01:00 m0000001 sendmail[12713]: 0A5Bs5V2012713: from=, size=20387, class=0, nrcpts=1, msgid=, proto=SMTP, daemon=MTA, tls_verify=NONE, auth=NONE, relay=m0000001.ppops.net [127.0.0.1]",2020-11-05T12:59:27.672144+01:00,"{""origin"":{""data"":{""agent"":""m0000001.ppops.net"",""cid"":""CID""}}}","{""cid"":""CID"",""agent"":""m0000001.ppops.net""}","{""nrcpts"":""1"",""from"":"""",""sizeBytes"":""20387"",""daemon"":""MTA"",""class"":""0"",""tls"":{""verify"":""NONE""},""qid"":""0A5Bs5V2012713"",""auth"":""NONE"",""msgid"":"""",""proto"":""SMTP"",""relay"":""m0000001.ppops.net [127.0.0.1]""}",null,ProofpointPODMailLog_CL From ec3fe2f11b4125e19506664c8310146207049f34 Mon Sep 17 00:00:00 2001 From: Aditi Bhoge Date: Fri, 23 Oct 2026 10:05:52 -0700 Subject: [PATCH 02/10] Link MailLog parser changelogs to draft PR 15410 Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 321517f1-a1c5-4242-be85-2d144a9cc295 --- Parsers/ASimEmailEvent/CHANGELOG/ASimEmailEvent.md | 3 +-- .../CHANGELOG/ASimEmailEventProofpointPODMailLog.md | 4 ++-- Parsers/ASimEmailEvent/CHANGELOG/imEmailEvent.md | 3 +-- .../CHANGELOG/vimEmailEventProofpointPODMailLog.md | 4 ++-- Parsers/ASimEmailEvent/README.md | 4 ++-- 5 files changed, 8 insertions(+), 10 deletions(-) diff --git a/Parsers/ASimEmailEvent/CHANGELOG/ASimEmailEvent.md b/Parsers/ASimEmailEvent/CHANGELOG/ASimEmailEvent.md index dc90b2ba63c..5f4d42d65b6 100644 --- a/Parsers/ASimEmailEvent/CHANGELOG/ASimEmailEvent.md +++ b/Parsers/ASimEmailEvent/CHANGELOG/ASimEmailEvent.md @@ -2,8 +2,7 @@ ## Version 1.0.1 -- (2026-10-21) Registered the Proofpoint POD MailLog source parser and forwarded packing and disable controls. -- No pull request has been created for this change. +- (2026-10-21) Registered the Proofpoint POD MailLog source parser and forwarded packing and disable controls - [PR #15410](https://github.com/Azure/Azure-Sentinel/pull/15410) ## Version 0.1.0 diff --git a/Parsers/ASimEmailEvent/CHANGELOG/ASimEmailEventProofpointPODMailLog.md b/Parsers/ASimEmailEvent/CHANGELOG/ASimEmailEventProofpointPODMailLog.md index 7ea1021118d..f98462a80d1 100644 --- a/Parsers/ASimEmailEvent/CHANGELOG/ASimEmailEventProofpointPODMailLog.md +++ b/Parsers/ASimEmailEvent/CHANGELOG/ASimEmailEventProofpointPODMailLog.md @@ -2,6 +2,6 @@ ## Version 0.1.0 -- (2026-10-21) Added a Proofpoint POD MailLog EmailEvent parser using the CCP table schema and repository samples. +- (2026-10-21) Added a Proofpoint POD MailLog EmailEvent parser using the CCP table schema and repository samples - [PR #15410](https://github.com/Azure/Azure-Sentinel/pull/15410) - Preserves individual SMTP receipt and delivery records without correlation or recipient expansion. -- Live schema and data validation are deferred. No pull request has been created for this change. +- Live schema and data validation are deferred. diff --git a/Parsers/ASimEmailEvent/CHANGELOG/imEmailEvent.md b/Parsers/ASimEmailEvent/CHANGELOG/imEmailEvent.md index 226597f7c0e..3e955e3edcf 100644 --- a/Parsers/ASimEmailEvent/CHANGELOG/imEmailEvent.md +++ b/Parsers/ASimEmailEvent/CHANGELOG/imEmailEvent.md @@ -2,8 +2,7 @@ ## Version 1.0.1 -- (2026-10-21) Registered the Proofpoint POD MailLog filtering parser and forwarded all EmailEvent filters and disable controls. -- No pull request has been created for this change. +- (2026-10-21) Registered the Proofpoint POD MailLog filtering parser and forwarded all EmailEvent filters and disable controls - [PR #15410](https://github.com/Azure/Azure-Sentinel/pull/15410) ## Version 0.1.0 diff --git a/Parsers/ASimEmailEvent/CHANGELOG/vimEmailEventProofpointPODMailLog.md b/Parsers/ASimEmailEvent/CHANGELOG/vimEmailEventProofpointPODMailLog.md index f752b95a5fc..8ad339bf38d 100644 --- a/Parsers/ASimEmailEvent/CHANGELOG/vimEmailEventProofpointPODMailLog.md +++ b/Parsers/ASimEmailEvent/CHANGELOG/vimEmailEventProofpointPODMailLog.md @@ -2,6 +2,6 @@ ## Version 0.1.0 -- (2026-10-21) Added a Proofpoint POD MailLog EmailEvent filtering parser with time, IPv4 prefix, direction and result filters. +- (2026-10-21) Added a Proofpoint POD MailLog EmailEvent filtering parser with time, IPv4 prefix, direction and result filters - [PR #15410](https://github.com/Azure/Azure-Sentinel/pull/15410) - Nonempty header sender, header recipient, subject and delivery-location filters return no rows because these fields are unavailable. -- Live schema, data and filter validation are deferred. No pull request has been created for this change. +- Live schema, data and filter validation are deferred. diff --git a/Parsers/ASimEmailEvent/README.md b/Parsers/ASimEmailEvent/README.md index 526b4b6f4d5..3ccda235b56 100644 --- a/Parsers/ASimEmailEvent/README.md +++ b/Parsers/ASimEmailEvent/README.md @@ -84,8 +84,8 @@ been performed. The Proofpoint API reference requires sign-in; task-specific Azure DevOps acceptance criteria were also unavailable. SMTP status meanings follow [RFC 3463](https://www.rfc-editor.org/rfc/rfc3463). Before release, verify the source shape and mixed receipt/delivery mappings -against real CCP records, run both ASIM testers and filter tests, and add the -actual pull-request links to the four affected changelogs. Missing recommended +against real CCP records and run both ASIM testers and filter tests. The change +is tracked in [draft PR #15410](https://github.com/Azure/Azure-Sentinel/pull/15410). Missing recommended header/security fields are intentional source limitations, not evidence of empty headers or zero attachments. From 998a1bfb9920c93995fff83b7e33ef676591d5e5 Mon Sep 17 00:00:00 2001 From: Aditi Bhoge Date: Fri, 23 Oct 2026 10:55:48 -0700 Subject: [PATCH 03/10] Remove generated ARM artifacts from MailLog parser PR Leave ARM generation to GitHub Actions; preserve parser YAML, registrations, fixtures and changelogs. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 321517f1-a1c5-4242-be85-2d144a9cc295 --- .../ARM/ASimEmailEvent/ASimEmailEvent.json | 2 +- .../ASimEmailEventProofpointPODMailLog.json | 36 ----------------- .../README.md | 23 ----------- .../ARM/FullDeploymentEmailEvent.json | 40 ------------------- .../ARM/imEmailEvent/imEmailEvent.json | 2 +- .../README.md | 24 ----------- .../vimEmailEventProofpointPODMailLog.json | 36 ----------------- Parsers/ASimEmailEvent/README.md | 3 ++ 8 files changed, 5 insertions(+), 161 deletions(-) delete mode 100644 Parsers/ASimEmailEvent/ARM/ASimEmailEventProofpointPODMailLog/ASimEmailEventProofpointPODMailLog.json delete mode 100644 Parsers/ASimEmailEvent/ARM/ASimEmailEventProofpointPODMailLog/README.md delete mode 100644 Parsers/ASimEmailEvent/ARM/vimEmailEventProofpointPODMailLog/README.md delete mode 100644 Parsers/ASimEmailEvent/ARM/vimEmailEventProofpointPODMailLog/vimEmailEventProofpointPODMailLog.json diff --git a/Parsers/ASimEmailEvent/ARM/ASimEmailEvent/ASimEmailEvent.json b/Parsers/ASimEmailEvent/ARM/ASimEmailEvent/ASimEmailEvent.json index 8468dbff75b..b1cd990b7ce 100644 --- a/Parsers/ASimEmailEvent/ARM/ASimEmailEvent/ASimEmailEvent.json +++ b/Parsers/ASimEmailEvent/ARM/ASimEmailEvent/ASimEmailEvent.json @@ -27,7 +27,7 @@ "displayName": "Email Event ASIM parser", "category": "ASIM", "FunctionAlias": "ASimEmailEvent", - "query": "let DisabledParsers=materialize(_GetWatchlist('ASimDisabledParsers') | where SearchKey in ('Any', 'ExcludeASimEmailEvent') | extend SourceSpecificParser=column_ifexists('SourceSpecificParser','') | distinct SourceSpecificParser| where isnotempty(SourceSpecificParser));\nlet ASimBuiltInDisabled=toscalar('ExcludeASimEmailEvent' in (DisabledParsers) or 'Any' in (DisabledParsers)); \nlet parser=(pack:bool=false){\nunion isfuzzy=true\n vimEmailEventEmpty,\n ASimEmailEventProofpointPODMailLog(pack=pack, disabled=(ASimBuiltInDisabled or ('ExcludeASimEmailEventProofpointPODMailLog' in (DisabledParsers))))\n}; \nparser (pack=pack)", + "query": "let DisabledParsers=materialize(_GetWatchlist('ASimDisabledParsers') | where SearchKey in ('Any', 'ExcludeASimEmailEvent') | extend SourceSpecificParser=column_ifexists('SourceSpecificParser','') | distinct SourceSpecificParser| where isnotempty(SourceSpecificParser));\nlet ASimBuiltInDisabled=toscalar('ExcludeASimEmailEvent' in (DisabledParsers) or 'Any' in (DisabledParsers)); \nlet parser=(pack:bool=false){\nunion isfuzzy=true\n vimEmailEventEmpty\n}; \nparser (pack=pack)", "version": 1, "functionParameters": "pack:bool=False" } diff --git a/Parsers/ASimEmailEvent/ARM/ASimEmailEventProofpointPODMailLog/ASimEmailEventProofpointPODMailLog.json b/Parsers/ASimEmailEvent/ARM/ASimEmailEventProofpointPODMailLog/ASimEmailEventProofpointPODMailLog.json deleted file mode 100644 index 36932c82b74..00000000000 --- a/Parsers/ASimEmailEvent/ARM/ASimEmailEventProofpointPODMailLog/ASimEmailEventProofpointPODMailLog.json +++ /dev/null @@ -1,36 +0,0 @@ -{ - "$schema": "https://schema.management.azure.com/schemas/2019-08-01/deploymentTemplate.json#", - "contentVersion": "1.0.0.0", - "parameters": { - "Workspace": { - "type": "string", - "metadata": { - "description": "The Microsoft Sentinel workspace into which the function will be deployed. Has to be in the selected Resource Group." - } - }, - "WorkspaceRegion": { - "type": "string", - "defaultValue": "[resourceGroup().location]", - "metadata": { - "description": "The region of the selected workspace. The default value will use the Region selection above." - } - } - }, - "resources": [ - { - "type": "Microsoft.OperationalInsights/workspaces/savedSearches", - "apiVersion": "2020-08-01", - "name": "[concat(parameters('Workspace'), '/ASimEmailEventProofpointPODMailLog')]", - "location": "[parameters('WorkspaceRegion')]", - "properties": { - "etag": "*", - "displayName": "Email Event ASIM parser for Proofpoint POD MailLog", - "category": "ASIM", - "FunctionAlias": "ASimEmailEventProofpointPODMailLog", - "query": "let parser = (disabled:bool=false, pack:bool=false) {\n ProofpointPODMailLog_CL\n | where not(disabled)\n | where isnotempty(tostring(sm.qid)) and (bag_has_key(sm, \"from\") or array_length(sm[\"to\"]) > 0)\n | extend\n IsDelivery = coalesce(array_length(sm[\"to\"]), 0) > 0,\n Relay = tostring(sm.relay),\n Dvc = coalesce(tostring(pps.agent), tostring(metadata.origin.data.agent)),\n EventStartTime = coalesce(eventTime, TimeGenerated),\n EmailMailFromAddress = trim(@\"[<>\\s]\", tostring(sm[\"from\"])),\n EmailSizeBytes = tolong(sm.sizeBytes),\n Dsn = tostring(sm.dsn),\n DeliveryStatus = tostring(sm.stat)\n | parse Relay with RelayHost \" [\" RelayIp \"]\" *\n | extend\n EmailMailFromAddress = iff(EmailMailFromAddress contains \"@\", EmailMailFromAddress, \"\"),\n RelayHost = trim_end(@\"\\.\", coalesce(RelayHost, Relay)),\n RelayIp = iff(isnotnull(parse_ipv4(RelayIp)) or isnotempty(parse_ipv6(RelayIp)), RelayIp, \"\"),\n EventResult = case(Dsn startswith \"2.\", \"Success\", Dsn startswith \"4.\" or Dsn startswith \"5.\", \"Failure\", \"NA\"),\n EmailProtocolName = case(tostring(sm.proto) =~ \"SMTP\" or tostring(sm.mailer) in~ (\"smtp\", \"esmtp\"), \"SMTP\", \"\"),\n EventSubType = iff(IsDelivery, \"EmailSent\", \"EmailReceived\")\n | extend\n SrcIpAddr = iff(IsDelivery, \"\", RelayIp),\n DstIpAddr = iff(IsDelivery, RelayIp, \"\"),\n Src = iff(IsDelivery, Dvc, coalesce(RelayIp, RelayHost)),\n Dst = iff(IsDelivery, coalesce(RelayIp, RelayHost), Dvc),\n SrcFQDN = iff(not(IsDelivery) and RelayHost contains \".\", RelayHost, \"\"),\n SrcHostname = iff(IsDelivery, tostring(split(Dvc, \".\")[0]), tostring(split(RelayHost, \".\")[0])),\n SrcDomain = iff(IsDelivery, substring(Dvc, indexof(Dvc, \".\") + 1), substring(RelayHost, indexof(RelayHost, \".\") + 1)),\n DvcHostname = tostring(split(Dvc, \".\")[0]),\n DvcDomain = iff(Dvc contains \".\", substring(Dvc, indexof(Dvc, \".\") + 1), \"\"),\n DvcFQDN = iff(Dvc contains \".\", Dvc, \"\"),\n EmailMailFromDomain = iff(EmailMailFromAddress contains \"@\", tostring(split(EmailMailFromAddress, \"@\")[1]), \"\"),\n DvcAction = iff(IsDelivery and EventResult == \"Success\" and DeliveryStatus startswith \"Sent\", \"Delivered\", \"NA\")\n | extend\n SrcFQDN = iff(IsDelivery, DvcFQDN, SrcFQDN),\n SrcDomain = iff(iff(IsDelivery, Dvc, RelayHost) contains \".\", SrcDomain, \"\"),\n DvcDomainType = iff(isnotempty(DvcDomain), \"FQDN\", \"\"),\n EventEndTime = EventStartTime,\n EventCount = int(1),\n EventVendor = \"Proofpoint\",\n EventProduct = \"POD MailLog\",\n EventSchema = \"EmailEvent\",\n EventSchemaVersion = \"1.0.0\",\n EventType = \"EmailDelivery\",\n EventOriginalType = \"maillog\",\n EmailDirection = \"Unknown\",\n EventOriginalResultDetails = strcat(Dsn, iff(isnotempty(Dsn) and isnotempty(DeliveryStatus), \"; \", \"\"), DeliveryStatus),\n EventUid = iff(isnotempty(id), strcat(\"ProofpointPODMailLog_CL:\", id), \"\"),\n DvcEntityKey = \"\",\n AdditionalEntities = dynamic([]),\n AdditionalFields = iff(pack, bag_pack(\"sm\", sm, \"tls\", tls, \"pps\", pps, \"metadata\", metadata), dynamic({})),\n Type = \"ProofpointPODMailLog_CL\"\n | extend SrcDomainType = iff(isnotempty(SrcDomain), \"FQDN\", \"\"), IpAddr = SrcIpAddr\n | project-rename EventOriginalUid = id, EventMessage = data\n | project\n TimeGenerated, Type, EventStartTime, EventEndTime, EventCount,\n EventVendor, EventProduct, EventSchema, EventSchemaVersion,\n EventType, EventSubType, EventOriginalType, EventResult,\n EventOriginalResultDetails, EventOriginalUid, EventUid, EventMessage,\n Dvc, DvcHostname, DvcDomain, DvcDomainType, DvcFQDN, DvcAction, DvcEntityKey,\n Src, SrcIpAddr, SrcHostname, SrcDomain, SrcDomainType, SrcFQDN, Dst, DstIpAddr, IpAddr,\n EmailMailFromAddress, EmailMailFromDomain, EmailSizeBytes, EmailProtocolName, EmailDirection,\n AdditionalFields, AdditionalEntities\n};\nparser(disabled=disabled, pack=pack)\n", - "version": 1, - "functionParameters": "disabled:bool=False,pack:bool=False" - } - } - ] -} \ No newline at end of file diff --git a/Parsers/ASimEmailEvent/ARM/ASimEmailEventProofpointPODMailLog/README.md b/Parsers/ASimEmailEvent/ARM/ASimEmailEventProofpointPODMailLog/README.md deleted file mode 100644 index 742834533b9..00000000000 --- a/Parsers/ASimEmailEvent/ARM/ASimEmailEventProofpointPODMailLog/README.md +++ /dev/null @@ -1,23 +0,0 @@ -# Proofpoint POD MailLog ASIM EmailEvent Normalization Parser - -ARM template for ASIM EmailEvent schema parser for Proofpoint POD MailLog. - -Normalizes ProofpointPODMailLog_CL SMTP receipt and delivery records to EmailEvent. -Uses the CCP table schema and repository samples; live validation is deferred. -Envelope recipients are retained in AdditionalFields, not mapped to header recipients. - - -The Advanced Security Information Model (ASIM) enables you to use and create source-agnostic content, simplifying your analysis of the data in your Microsoft Sentinel workspace. - -For more information, see: - -- [Normalization and the Advanced Security Information Model (ASIM)](https://aka.ms/AboutASIM) -- [Deploy all of ASIM](https://aka.ms/DeployASIM) -- [ASIM EmailEvent normalization schema reference](https://aka.ms/ASimEmailEventDoc) - -For the changelog, see: -- [CHANGELOG](https://github.com/Azure/Azure-Sentinel/blob/master/Parsers/ASimEmailEvent/CHANGELOG/ASimEmailEventProofpointPODMailLog.md) - -
- -[![Deploy to Azure](https://aka.ms/deploytoazurebutton)](https://portal.azure.com/#create/Microsoft.Template/uri/https%3A%2F%2Fraw.githubusercontent.com%2FAzure%2FAzure-Sentinel%2Fmaster%2FParsers%2FASimEmailEvent%2FARM%2FASimEmailEventProofpointPODMailLog%2FASimEmailEventProofpointPODMailLog.json) [![Deploy to Azure Gov](https://aka.ms/deploytoazuregovernbutton)](https://portal.azure.us/#create/Microsoft.Template/uri/https%3A%2F%2Fraw.githubusercontent.com%2FAzure%2FAzure-Sentinel%2Fmaster%2FParsers%2FASimEmailEvent%2FARM%2FASimEmailEventProofpointPODMailLog%2FASimEmailEventProofpointPODMailLog.json) diff --git a/Parsers/ASimEmailEvent/ARM/FullDeploymentEmailEvent.json b/Parsers/ASimEmailEvent/ARM/FullDeploymentEmailEvent.json index b5e89e872ec..0cec41ce310 100644 --- a/Parsers/ASimEmailEvent/ARM/FullDeploymentEmailEvent.json +++ b/Parsers/ASimEmailEvent/ARM/FullDeploymentEmailEvent.json @@ -38,26 +38,6 @@ } } }, - { - "type": "Microsoft.Resources/deployments", - "apiVersion": "2020-10-01", - "name": "linkedASimEmailEventProofpointPODMailLog", - "properties": { - "mode": "Incremental", - "templateLink": { - "uri": "https://raw.githubusercontent.com/Azure/Azure-Sentinel/master/Parsers/ASimEmailEvent/ARM/ASimEmailEventProofpointPODMailLog/ASimEmailEventProofpointPODMailLog.json", - "contentVersion": "1.0.0.0" - }, - "parameters": { - "Workspace": { - "value": "[parameters('Workspace')]" - }, - "WorkspaceRegion": { - "value": "[parameters('WorkspaceRegion')]" - } - } - } - }, { "type": "Microsoft.Resources/deployments", "apiVersion": "2020-10-01", @@ -97,26 +77,6 @@ } } } - }, - { - "type": "Microsoft.Resources/deployments", - "apiVersion": "2020-10-01", - "name": "linkedvimEmailEventProofpointPODMailLog", - "properties": { - "mode": "Incremental", - "templateLink": { - "uri": "https://raw.githubusercontent.com/Azure/Azure-Sentinel/master/Parsers/ASimEmailEvent/ARM/vimEmailEventProofpointPODMailLog/vimEmailEventProofpointPODMailLog.json", - "contentVersion": "1.0.0.0" - }, - "parameters": { - "Workspace": { - "value": "[parameters('Workspace')]" - }, - "WorkspaceRegion": { - "value": "[parameters('WorkspaceRegion')]" - } - } - } } ], "outputs": {} diff --git a/Parsers/ASimEmailEvent/ARM/imEmailEvent/imEmailEvent.json b/Parsers/ASimEmailEvent/ARM/imEmailEvent/imEmailEvent.json index 59e3e47e991..f598cd155ef 100644 --- a/Parsers/ASimEmailEvent/ARM/imEmailEvent/imEmailEvent.json +++ b/Parsers/ASimEmailEvent/ARM/imEmailEvent/imEmailEvent.json @@ -27,7 +27,7 @@ "displayName": "Email Event ASIM filtering parser", "category": "ASIM", "FunctionAlias": "imEmailEvent", - "query": "let DisabledParsers=materialize(_GetWatchlist('ASimDisabledParsers') | where SearchKey in ('Any', 'ExcludeimEmailEvent') | extend SourceSpecificParser=column_ifexists('SourceSpecificParser','') | distinct SourceSpecificParser | where isnotempty(SourceSpecificParser));\nlet imBuiltInDisabled=toscalar('ExcludeimEmailEvent' in (DisabledParsers) or 'Any' in (DisabledParsers)); \nlet parser=(\n starttime: datetime=datetime(null), \n endtime: datetime=datetime(null), \n ipaddr_has_any_prefix: dynamic=dynamic([]),\n emailfromaddress_has_any: dynamic=dynamic([]),\n emailrecipient_has_any: dynamic=dynamic([]),\n emailsubject_has_any: dynamic=dynamic([]),\n emaildirection_in: dynamic=dynamic([]),\n emaildeliverylocation_in: dynamic=dynamic([]),\n eventresult: string='*',\n pack:bool=false)\n{\nunion isfuzzy=true\n vimEmailEventEmpty,\n vimEmailEventProofpointPODMailLog(\n starttime=starttime, endtime=endtime, ipaddr_has_any_prefix=ipaddr_has_any_prefix,\n emailfromaddress_has_any=emailfromaddress_has_any, emailrecipient_has_any=emailrecipient_has_any,\n emailsubject_has_any=emailsubject_has_any, emaildirection_in=emaildirection_in,\n emaildeliverylocation_in=emaildeliverylocation_in, eventresult=eventresult, pack=pack,\n disabled=(imBuiltInDisabled or ('ExcludevimEmailEventProofpointPODMailLog' in (DisabledParsers))))\n};\nparser (starttime=starttime, endtime=endtime, ipaddr_has_any_prefix=ipaddr_has_any_prefix, emailfromaddress_has_any=emailfromaddress_has_any, emailrecipient_has_any=emailrecipient_has_any, emailsubject_has_any=emailsubject_has_any, emaildirection_in=emaildirection_in, emaildeliverylocation_in=emaildeliverylocation_in, eventresult=eventresult, pack=pack)", + "query": "let DisabledParsers=materialize(_GetWatchlist('ASimDisabledParsers') | where SearchKey in ('Any', 'ExcludeimEmailEvent') | extend SourceSpecificParser=column_ifexists('SourceSpecificParser','') | distinct SourceSpecificParser | where isnotempty(SourceSpecificParser));\nlet imBuiltInDisabled=toscalar('ExcludeimEmailEvent' in (DisabledParsers) or 'Any' in (DisabledParsers)); \nlet parser=(\n starttime: datetime=datetime(null), \n endtime: datetime=datetime(null), \n ipaddr_has_any_prefix: dynamic=dynamic([]),\n emailfromaddress_has_any: dynamic=dynamic([]),\n emailrecipient_has_any: dynamic=dynamic([]),\n emailsubject_has_any: dynamic=dynamic([]),\n emaildirection_in: dynamic=dynamic([]),\n emaildeliverylocation_in: dynamic=dynamic([]),\n eventresult: string='*',\n pack:bool=false)\n{\nunion isfuzzy=true\n vimEmailEventEmpty\n};\nparser (starttime=starttime, endtime=endtime, ipaddr_has_any_prefix=ipaddr_has_any_prefix, emailfromaddress_has_any=emailfromaddress_has_any, emailrecipient_has_any=emailrecipient_has_any, emailsubject_has_any=emailsubject_has_any, emaildirection_in=emaildirection_in, emaildeliverylocation_in=emaildeliverylocation_in, eventresult=eventresult, pack=pack)", "version": 1, "functionParameters": "starttime:datetime=datetime(null),endtime:datetime=datetime(null),ipaddr_has_any_prefix:dynamic=dynamic([]),emailfromaddress_has_any:dynamic=dynamic([]),emailrecipient_has_any:dynamic=dynamic([]),emailsubject_has_any:dynamic=dynamic([]),emaildirection_in:dynamic=dynamic([]),emaildeliverylocation_in:dynamic=dynamic([]),eventresult:string='*',pack:bool=False" } diff --git a/Parsers/ASimEmailEvent/ARM/vimEmailEventProofpointPODMailLog/README.md b/Parsers/ASimEmailEvent/ARM/vimEmailEventProofpointPODMailLog/README.md deleted file mode 100644 index 0433444dd2c..00000000000 --- a/Parsers/ASimEmailEvent/ARM/vimEmailEventProofpointPODMailLog/README.md +++ /dev/null @@ -1,24 +0,0 @@ -# Proofpoint POD MailLog ASIM EmailEvent Normalization Parser - -ARM template for ASIM EmailEvent schema parser for Proofpoint POD MailLog. - -Filters and normalizes ProofpointPODMailLog_CL SMTP receipt and delivery records to EmailEvent. -Uses the CCP table schema and repository samples; live validation is deferred. -Header sender, header recipients, subject and delivery location are unavailable; -nonempty filters for these fields return no records. - - -The Advanced Security Information Model (ASIM) enables you to use and create source-agnostic content, simplifying your analysis of the data in your Microsoft Sentinel workspace. - -For more information, see: - -- [Normalization and the Advanced Security Information Model (ASIM)](https://aka.ms/AboutASIM) -- [Deploy all of ASIM](https://aka.ms/DeployASIM) -- [ASIM EmailEvent normalization schema reference](https://aka.ms/ASimEmailEventDoc) - -For the changelog, see: -- [CHANGELOG](https://github.com/Azure/Azure-Sentinel/blob/master/Parsers/ASimEmailEvent/CHANGELOG/vimEmailEventProofpointPODMailLog.md) - -
- -[![Deploy to Azure](https://aka.ms/deploytoazurebutton)](https://portal.azure.com/#create/Microsoft.Template/uri/https%3A%2F%2Fraw.githubusercontent.com%2FAzure%2FAzure-Sentinel%2Fmaster%2FParsers%2FASimEmailEvent%2FARM%2FvimEmailEventProofpointPODMailLog%2FvimEmailEventProofpointPODMailLog.json) [![Deploy to Azure Gov](https://aka.ms/deploytoazuregovernbutton)](https://portal.azure.us/#create/Microsoft.Template/uri/https%3A%2F%2Fraw.githubusercontent.com%2FAzure%2FAzure-Sentinel%2Fmaster%2FParsers%2FASimEmailEvent%2FARM%2FvimEmailEventProofpointPODMailLog%2FvimEmailEventProofpointPODMailLog.json) diff --git a/Parsers/ASimEmailEvent/ARM/vimEmailEventProofpointPODMailLog/vimEmailEventProofpointPODMailLog.json b/Parsers/ASimEmailEvent/ARM/vimEmailEventProofpointPODMailLog/vimEmailEventProofpointPODMailLog.json deleted file mode 100644 index 1427541ca7e..00000000000 --- a/Parsers/ASimEmailEvent/ARM/vimEmailEventProofpointPODMailLog/vimEmailEventProofpointPODMailLog.json +++ /dev/null @@ -1,36 +0,0 @@ -{ - "$schema": "https://schema.management.azure.com/schemas/2019-08-01/deploymentTemplate.json#", - "contentVersion": "1.0.0.0", - "parameters": { - "Workspace": { - "type": "string", - "metadata": { - "description": "The Microsoft Sentinel workspace into which the function will be deployed. Has to be in the selected Resource Group." - } - }, - "WorkspaceRegion": { - "type": "string", - "defaultValue": "[resourceGroup().location]", - "metadata": { - "description": "The region of the selected workspace. The default value will use the Region selection above." - } - } - }, - "resources": [ - { - "type": "Microsoft.OperationalInsights/workspaces/savedSearches", - "apiVersion": "2020-08-01", - "name": "[concat(parameters('Workspace'), '/vimEmailEventProofpointPODMailLog')]", - "location": "[parameters('WorkspaceRegion')]", - "properties": { - "etag": "*", - "displayName": "Email Event ASIM filtering parser for Proofpoint POD MailLog", - "category": "ASIM", - "FunctionAlias": "vimEmailEventProofpointPODMailLog", - "query": "let parser = (\n starttime:datetime=datetime(null),\n endtime:datetime=datetime(null),\n ipaddr_has_any_prefix:dynamic=dynamic([]),\n emailfromaddress_has_any:dynamic=dynamic([]),\n emailrecipient_has_any:dynamic=dynamic([]),\n emailsubject_has_any:dynamic=dynamic([]),\n emaildirection_in:dynamic=dynamic([]),\n emaildeliverylocation_in:dynamic=dynamic([]),\n eventresult:string='*',\n disabled:bool=false,\n pack:bool=false)\n{\n ProofpointPODMailLog_CL\n | where not(disabled)\n | where (isnull(starttime) or eventTime >= starttime or (isnull(eventTime) and TimeGenerated >= starttime))\n and (isnull(endtime) or eventTime <= endtime or (isnull(eventTime) and TimeGenerated <= endtime))\n | where array_length(emailfromaddress_has_any) == 0 and array_length(emailrecipient_has_any) == 0\n and array_length(emailsubject_has_any) == 0 and array_length(emaildeliverylocation_in) == 0\n | where array_length(emaildirection_in) == 0 or \"Unknown\" in (emaildirection_in)\n | where isnotempty(tostring(sm.qid)) and (bag_has_key(sm, \"from\") or array_length(sm[\"to\"]) > 0)\n | extend\n IsDelivery = coalesce(array_length(sm[\"to\"]), 0) > 0,\n Dsn = tostring(sm.dsn)\n | extend EventResult = case(Dsn startswith \"2.\", \"Success\", Dsn startswith \"4.\" or Dsn startswith \"5.\", \"Failure\", \"NA\")\n | where eventresult == \"*\" or EventResult == eventresult\n | extend\n Relay = tostring(sm.relay),\n Dvc = coalesce(tostring(pps.agent), tostring(metadata.origin.data.agent)),\n EventStartTime = coalesce(eventTime, TimeGenerated),\n EmailMailFromAddress = trim(@\"[<>\\s]\", tostring(sm[\"from\"])),\n EmailSizeBytes = tolong(sm.sizeBytes),\n DeliveryStatus = tostring(sm.stat)\n | parse Relay with RelayHost \" [\" RelayIp \"]\" *\n | extend\n EmailMailFromAddress = iff(EmailMailFromAddress contains \"@\", EmailMailFromAddress, \"\"),\n RelayHost = trim_end(@\"\\.\", coalesce(RelayHost, Relay)),\n RelayIp = iff(isnotnull(parse_ipv4(RelayIp)) or isnotempty(parse_ipv6(RelayIp)), RelayIp, \"\"),\n EmailProtocolName = case(tostring(sm.proto) =~ \"SMTP\" or tostring(sm.mailer) in~ (\"smtp\", \"esmtp\"), \"SMTP\", \"\"),\n EventSubType = iff(IsDelivery, \"EmailSent\", \"EmailReceived\")\n | extend\n SrcIpAddr = iff(IsDelivery, \"\", RelayIp),\n DstIpAddr = iff(IsDelivery, RelayIp, \"\"),\n Src = iff(IsDelivery, Dvc, coalesce(RelayIp, RelayHost)),\n Dst = iff(IsDelivery, coalesce(RelayIp, RelayHost), Dvc),\n SrcFQDN = iff(not(IsDelivery) and RelayHost contains \".\", RelayHost, \"\"),\n SrcHostname = iff(IsDelivery, tostring(split(Dvc, \".\")[0]), tostring(split(RelayHost, \".\")[0])),\n SrcDomain = iff(IsDelivery, substring(Dvc, indexof(Dvc, \".\") + 1), substring(RelayHost, indexof(RelayHost, \".\") + 1)),\n DvcHostname = tostring(split(Dvc, \".\")[0]),\n DvcDomain = iff(Dvc contains \".\", substring(Dvc, indexof(Dvc, \".\") + 1), \"\"),\n DvcFQDN = iff(Dvc contains \".\", Dvc, \"\"),\n EmailMailFromDomain = iff(EmailMailFromAddress contains \"@\", tostring(split(EmailMailFromAddress, \"@\")[1]), \"\"),\n DvcAction = iff(IsDelivery and EventResult == \"Success\" and DeliveryStatus startswith \"Sent\", \"Delivered\", \"NA\")\n | where array_length(ipaddr_has_any_prefix) == 0\n or has_any_ipv4_prefix(SrcIpAddr, ipaddr_has_any_prefix)\n or has_any_ipv4_prefix(DstIpAddr, ipaddr_has_any_prefix)\n | extend\n SrcFQDN = iff(IsDelivery, DvcFQDN, SrcFQDN),\n SrcDomain = iff(iff(IsDelivery, Dvc, RelayHost) contains \".\", SrcDomain, \"\"),\n DvcDomainType = iff(isnotempty(DvcDomain), \"FQDN\", \"\"),\n EventEndTime = EventStartTime,\n EventCount = int(1),\n EventVendor = \"Proofpoint\",\n EventProduct = \"POD MailLog\",\n EventSchema = \"EmailEvent\",\n EventSchemaVersion = \"1.0.0\",\n EventType = \"EmailDelivery\",\n EventOriginalType = \"maillog\",\n EmailDirection = \"Unknown\",\n EventOriginalResultDetails = strcat(Dsn, iff(isnotempty(Dsn) and isnotempty(DeliveryStatus), \"; \", \"\"), DeliveryStatus),\n EventUid = iff(isnotempty(id), strcat(\"ProofpointPODMailLog_CL:\", id), \"\"),\n DvcEntityKey = \"\",\n AdditionalEntities = dynamic([]),\n AdditionalFields = iff(pack, bag_pack(\"sm\", sm, \"tls\", tls, \"pps\", pps, \"metadata\", metadata), dynamic({})),\n Type = \"ProofpointPODMailLog_CL\"\n | extend SrcDomainType = iff(isnotempty(SrcDomain), \"FQDN\", \"\"), IpAddr = SrcIpAddr\n | project-rename EventOriginalUid = id, EventMessage = data\n | project\n TimeGenerated, Type, EventStartTime, EventEndTime, EventCount,\n EventVendor, EventProduct, EventSchema, EventSchemaVersion,\n EventType, EventSubType, EventOriginalType, EventResult,\n EventOriginalResultDetails, EventOriginalUid, EventUid, EventMessage,\n Dvc, DvcHostname, DvcDomain, DvcDomainType, DvcFQDN, DvcAction, DvcEntityKey,\n Src, SrcIpAddr, SrcHostname, SrcDomain, SrcDomainType, SrcFQDN, Dst, DstIpAddr, IpAddr,\n EmailMailFromAddress, EmailMailFromDomain, EmailSizeBytes, EmailProtocolName, EmailDirection,\n AdditionalFields, AdditionalEntities\n};\nparser(starttime=starttime, endtime=endtime, ipaddr_has_any_prefix=ipaddr_has_any_prefix,\n emailfromaddress_has_any=emailfromaddress_has_any, emailrecipient_has_any=emailrecipient_has_any,\n emailsubject_has_any=emailsubject_has_any, emaildirection_in=emaildirection_in,\n emaildeliverylocation_in=emaildeliverylocation_in, eventresult=eventresult, disabled=disabled, pack=pack)\n", - "version": 1, - "functionParameters": "starttime:datetime=datetime(null),endtime:datetime=datetime(null),ipaddr_has_any_prefix:dynamic=dynamic([]),emailfromaddress_has_any:dynamic=dynamic([]),emailrecipient_has_any:dynamic=dynamic([]),emailsubject_has_any:dynamic=dynamic([]),emaildirection_in:dynamic=dynamic([]),emaildeliverylocation_in:dynamic=dynamic([]),eventresult:string='*',disabled:bool=False,pack:bool=False" - } - } - ] -} \ No newline at end of file diff --git a/Parsers/ASimEmailEvent/README.md b/Parsers/ASimEmailEvent/README.md index 3ccda235b56..ee6295ece96 100644 --- a/Parsers/ASimEmailEvent/README.md +++ b/Parsers/ASimEmailEvent/README.md @@ -20,6 +20,9 @@ most one row for each source row. They select message-related sendmail records with a queue ID and either an envelope sender property or nonempty envelope recipients; TLS-only and other non-message records are not normalized. +ARM templates are generated by the repository GitHub Action from the parser +YAML; this change does not include manually generated ARM artifacts. + ### Evidence and mappings The source contract is the solution's From 2054ea952cb3f6de5f584974b4c9e6405d963140 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <> Date: Fri, 23 Oct 2026 18:00:43 +0000 Subject: [PATCH 04/10] [ASIM Parsers] Generate deployable ARM templates from KQL function YAML files. --- .../ARM/ASimEmailEvent/ASimEmailEvent.json | 2 +- .../ASimEmailEventProofpointPODMailLog.json | 36 +++++++++++++++++ .../README.md | 23 +++++++++++ .../ARM/FullDeploymentEmailEvent.json | 40 +++++++++++++++++++ .../ARM/imEmailEvent/imEmailEvent.json | 2 +- .../README.md | 24 +++++++++++ .../vimEmailEventProofpointPODMailLog.json | 36 +++++++++++++++++ 7 files changed, 161 insertions(+), 2 deletions(-) create mode 100644 Parsers/ASimEmailEvent/ARM/ASimEmailEventProofpointPODMailLog/ASimEmailEventProofpointPODMailLog.json create mode 100644 Parsers/ASimEmailEvent/ARM/ASimEmailEventProofpointPODMailLog/README.md create mode 100644 Parsers/ASimEmailEvent/ARM/vimEmailEventProofpointPODMailLog/README.md create mode 100644 Parsers/ASimEmailEvent/ARM/vimEmailEventProofpointPODMailLog/vimEmailEventProofpointPODMailLog.json diff --git a/Parsers/ASimEmailEvent/ARM/ASimEmailEvent/ASimEmailEvent.json b/Parsers/ASimEmailEvent/ARM/ASimEmailEvent/ASimEmailEvent.json index b1cd990b7ce..8468dbff75b 100644 --- a/Parsers/ASimEmailEvent/ARM/ASimEmailEvent/ASimEmailEvent.json +++ b/Parsers/ASimEmailEvent/ARM/ASimEmailEvent/ASimEmailEvent.json @@ -27,7 +27,7 @@ "displayName": "Email Event ASIM parser", "category": "ASIM", "FunctionAlias": "ASimEmailEvent", - "query": "let DisabledParsers=materialize(_GetWatchlist('ASimDisabledParsers') | where SearchKey in ('Any', 'ExcludeASimEmailEvent') | extend SourceSpecificParser=column_ifexists('SourceSpecificParser','') | distinct SourceSpecificParser| where isnotempty(SourceSpecificParser));\nlet ASimBuiltInDisabled=toscalar('ExcludeASimEmailEvent' in (DisabledParsers) or 'Any' in (DisabledParsers)); \nlet parser=(pack:bool=false){\nunion isfuzzy=true\n vimEmailEventEmpty\n}; \nparser (pack=pack)", + "query": "let DisabledParsers=materialize(_GetWatchlist('ASimDisabledParsers') | where SearchKey in ('Any', 'ExcludeASimEmailEvent') | extend SourceSpecificParser=column_ifexists('SourceSpecificParser','') | distinct SourceSpecificParser| where isnotempty(SourceSpecificParser));\nlet ASimBuiltInDisabled=toscalar('ExcludeASimEmailEvent' in (DisabledParsers) or 'Any' in (DisabledParsers)); \nlet parser=(pack:bool=false){\nunion isfuzzy=true\n vimEmailEventEmpty,\n ASimEmailEventProofpointPODMailLog(pack=pack, disabled=(ASimBuiltInDisabled or ('ExcludeASimEmailEventProofpointPODMailLog' in (DisabledParsers))))\n}; \nparser (pack=pack)", "version": 1, "functionParameters": "pack:bool=False" } diff --git a/Parsers/ASimEmailEvent/ARM/ASimEmailEventProofpointPODMailLog/ASimEmailEventProofpointPODMailLog.json b/Parsers/ASimEmailEvent/ARM/ASimEmailEventProofpointPODMailLog/ASimEmailEventProofpointPODMailLog.json new file mode 100644 index 00000000000..36932c82b74 --- /dev/null +++ b/Parsers/ASimEmailEvent/ARM/ASimEmailEventProofpointPODMailLog/ASimEmailEventProofpointPODMailLog.json @@ -0,0 +1,36 @@ +{ + "$schema": "https://schema.management.azure.com/schemas/2019-08-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "Workspace": { + "type": "string", + "metadata": { + "description": "The Microsoft Sentinel workspace into which the function will be deployed. Has to be in the selected Resource Group." + } + }, + "WorkspaceRegion": { + "type": "string", + "defaultValue": "[resourceGroup().location]", + "metadata": { + "description": "The region of the selected workspace. The default value will use the Region selection above." + } + } + }, + "resources": [ + { + "type": "Microsoft.OperationalInsights/workspaces/savedSearches", + "apiVersion": "2020-08-01", + "name": "[concat(parameters('Workspace'), '/ASimEmailEventProofpointPODMailLog')]", + "location": "[parameters('WorkspaceRegion')]", + "properties": { + "etag": "*", + "displayName": "Email Event ASIM parser for Proofpoint POD MailLog", + "category": "ASIM", + "FunctionAlias": "ASimEmailEventProofpointPODMailLog", + "query": "let parser = (disabled:bool=false, pack:bool=false) {\n ProofpointPODMailLog_CL\n | where not(disabled)\n | where isnotempty(tostring(sm.qid)) and (bag_has_key(sm, \"from\") or array_length(sm[\"to\"]) > 0)\n | extend\n IsDelivery = coalesce(array_length(sm[\"to\"]), 0) > 0,\n Relay = tostring(sm.relay),\n Dvc = coalesce(tostring(pps.agent), tostring(metadata.origin.data.agent)),\n EventStartTime = coalesce(eventTime, TimeGenerated),\n EmailMailFromAddress = trim(@\"[<>\\s]\", tostring(sm[\"from\"])),\n EmailSizeBytes = tolong(sm.sizeBytes),\n Dsn = tostring(sm.dsn),\n DeliveryStatus = tostring(sm.stat)\n | parse Relay with RelayHost \" [\" RelayIp \"]\" *\n | extend\n EmailMailFromAddress = iff(EmailMailFromAddress contains \"@\", EmailMailFromAddress, \"\"),\n RelayHost = trim_end(@\"\\.\", coalesce(RelayHost, Relay)),\n RelayIp = iff(isnotnull(parse_ipv4(RelayIp)) or isnotempty(parse_ipv6(RelayIp)), RelayIp, \"\"),\n EventResult = case(Dsn startswith \"2.\", \"Success\", Dsn startswith \"4.\" or Dsn startswith \"5.\", \"Failure\", \"NA\"),\n EmailProtocolName = case(tostring(sm.proto) =~ \"SMTP\" or tostring(sm.mailer) in~ (\"smtp\", \"esmtp\"), \"SMTP\", \"\"),\n EventSubType = iff(IsDelivery, \"EmailSent\", \"EmailReceived\")\n | extend\n SrcIpAddr = iff(IsDelivery, \"\", RelayIp),\n DstIpAddr = iff(IsDelivery, RelayIp, \"\"),\n Src = iff(IsDelivery, Dvc, coalesce(RelayIp, RelayHost)),\n Dst = iff(IsDelivery, coalesce(RelayIp, RelayHost), Dvc),\n SrcFQDN = iff(not(IsDelivery) and RelayHost contains \".\", RelayHost, \"\"),\n SrcHostname = iff(IsDelivery, tostring(split(Dvc, \".\")[0]), tostring(split(RelayHost, \".\")[0])),\n SrcDomain = iff(IsDelivery, substring(Dvc, indexof(Dvc, \".\") + 1), substring(RelayHost, indexof(RelayHost, \".\") + 1)),\n DvcHostname = tostring(split(Dvc, \".\")[0]),\n DvcDomain = iff(Dvc contains \".\", substring(Dvc, indexof(Dvc, \".\") + 1), \"\"),\n DvcFQDN = iff(Dvc contains \".\", Dvc, \"\"),\n EmailMailFromDomain = iff(EmailMailFromAddress contains \"@\", tostring(split(EmailMailFromAddress, \"@\")[1]), \"\"),\n DvcAction = iff(IsDelivery and EventResult == \"Success\" and DeliveryStatus startswith \"Sent\", \"Delivered\", \"NA\")\n | extend\n SrcFQDN = iff(IsDelivery, DvcFQDN, SrcFQDN),\n SrcDomain = iff(iff(IsDelivery, Dvc, RelayHost) contains \".\", SrcDomain, \"\"),\n DvcDomainType = iff(isnotempty(DvcDomain), \"FQDN\", \"\"),\n EventEndTime = EventStartTime,\n EventCount = int(1),\n EventVendor = \"Proofpoint\",\n EventProduct = \"POD MailLog\",\n EventSchema = \"EmailEvent\",\n EventSchemaVersion = \"1.0.0\",\n EventType = \"EmailDelivery\",\n EventOriginalType = \"maillog\",\n EmailDirection = \"Unknown\",\n EventOriginalResultDetails = strcat(Dsn, iff(isnotempty(Dsn) and isnotempty(DeliveryStatus), \"; \", \"\"), DeliveryStatus),\n EventUid = iff(isnotempty(id), strcat(\"ProofpointPODMailLog_CL:\", id), \"\"),\n DvcEntityKey = \"\",\n AdditionalEntities = dynamic([]),\n AdditionalFields = iff(pack, bag_pack(\"sm\", sm, \"tls\", tls, \"pps\", pps, \"metadata\", metadata), dynamic({})),\n Type = \"ProofpointPODMailLog_CL\"\n | extend SrcDomainType = iff(isnotempty(SrcDomain), \"FQDN\", \"\"), IpAddr = SrcIpAddr\n | project-rename EventOriginalUid = id, EventMessage = data\n | project\n TimeGenerated, Type, EventStartTime, EventEndTime, EventCount,\n EventVendor, EventProduct, EventSchema, EventSchemaVersion,\n EventType, EventSubType, EventOriginalType, EventResult,\n EventOriginalResultDetails, EventOriginalUid, EventUid, EventMessage,\n Dvc, DvcHostname, DvcDomain, DvcDomainType, DvcFQDN, DvcAction, DvcEntityKey,\n Src, SrcIpAddr, SrcHostname, SrcDomain, SrcDomainType, SrcFQDN, Dst, DstIpAddr, IpAddr,\n EmailMailFromAddress, EmailMailFromDomain, EmailSizeBytes, EmailProtocolName, EmailDirection,\n AdditionalFields, AdditionalEntities\n};\nparser(disabled=disabled, pack=pack)\n", + "version": 1, + "functionParameters": "disabled:bool=False,pack:bool=False" + } + } + ] +} \ No newline at end of file diff --git a/Parsers/ASimEmailEvent/ARM/ASimEmailEventProofpointPODMailLog/README.md b/Parsers/ASimEmailEvent/ARM/ASimEmailEventProofpointPODMailLog/README.md new file mode 100644 index 00000000000..742834533b9 --- /dev/null +++ b/Parsers/ASimEmailEvent/ARM/ASimEmailEventProofpointPODMailLog/README.md @@ -0,0 +1,23 @@ +# Proofpoint POD MailLog ASIM EmailEvent Normalization Parser + +ARM template for ASIM EmailEvent schema parser for Proofpoint POD MailLog. + +Normalizes ProofpointPODMailLog_CL SMTP receipt and delivery records to EmailEvent. +Uses the CCP table schema and repository samples; live validation is deferred. +Envelope recipients are retained in AdditionalFields, not mapped to header recipients. + + +The Advanced Security Information Model (ASIM) enables you to use and create source-agnostic content, simplifying your analysis of the data in your Microsoft Sentinel workspace. + +For more information, see: + +- [Normalization and the Advanced Security Information Model (ASIM)](https://aka.ms/AboutASIM) +- [Deploy all of ASIM](https://aka.ms/DeployASIM) +- [ASIM EmailEvent normalization schema reference](https://aka.ms/ASimEmailEventDoc) + +For the changelog, see: +- [CHANGELOG](https://github.com/Azure/Azure-Sentinel/blob/master/Parsers/ASimEmailEvent/CHANGELOG/ASimEmailEventProofpointPODMailLog.md) + +
+ +[![Deploy to Azure](https://aka.ms/deploytoazurebutton)](https://portal.azure.com/#create/Microsoft.Template/uri/https%3A%2F%2Fraw.githubusercontent.com%2FAzure%2FAzure-Sentinel%2Fmaster%2FParsers%2FASimEmailEvent%2FARM%2FASimEmailEventProofpointPODMailLog%2FASimEmailEventProofpointPODMailLog.json) [![Deploy to Azure Gov](https://aka.ms/deploytoazuregovernbutton)](https://portal.azure.us/#create/Microsoft.Template/uri/https%3A%2F%2Fraw.githubusercontent.com%2FAzure%2FAzure-Sentinel%2Fmaster%2FParsers%2FASimEmailEvent%2FARM%2FASimEmailEventProofpointPODMailLog%2FASimEmailEventProofpointPODMailLog.json) diff --git a/Parsers/ASimEmailEvent/ARM/FullDeploymentEmailEvent.json b/Parsers/ASimEmailEvent/ARM/FullDeploymentEmailEvent.json index 0cec41ce310..b5e89e872ec 100644 --- a/Parsers/ASimEmailEvent/ARM/FullDeploymentEmailEvent.json +++ b/Parsers/ASimEmailEvent/ARM/FullDeploymentEmailEvent.json @@ -38,6 +38,26 @@ } } }, + { + "type": "Microsoft.Resources/deployments", + "apiVersion": "2020-10-01", + "name": "linkedASimEmailEventProofpointPODMailLog", + "properties": { + "mode": "Incremental", + "templateLink": { + "uri": "https://raw.githubusercontent.com/Azure/Azure-Sentinel/master/Parsers/ASimEmailEvent/ARM/ASimEmailEventProofpointPODMailLog/ASimEmailEventProofpointPODMailLog.json", + "contentVersion": "1.0.0.0" + }, + "parameters": { + "Workspace": { + "value": "[parameters('Workspace')]" + }, + "WorkspaceRegion": { + "value": "[parameters('WorkspaceRegion')]" + } + } + } + }, { "type": "Microsoft.Resources/deployments", "apiVersion": "2020-10-01", @@ -77,6 +97,26 @@ } } } + }, + { + "type": "Microsoft.Resources/deployments", + "apiVersion": "2020-10-01", + "name": "linkedvimEmailEventProofpointPODMailLog", + "properties": { + "mode": "Incremental", + "templateLink": { + "uri": "https://raw.githubusercontent.com/Azure/Azure-Sentinel/master/Parsers/ASimEmailEvent/ARM/vimEmailEventProofpointPODMailLog/vimEmailEventProofpointPODMailLog.json", + "contentVersion": "1.0.0.0" + }, + "parameters": { + "Workspace": { + "value": "[parameters('Workspace')]" + }, + "WorkspaceRegion": { + "value": "[parameters('WorkspaceRegion')]" + } + } + } } ], "outputs": {} diff --git a/Parsers/ASimEmailEvent/ARM/imEmailEvent/imEmailEvent.json b/Parsers/ASimEmailEvent/ARM/imEmailEvent/imEmailEvent.json index f598cd155ef..59e3e47e991 100644 --- a/Parsers/ASimEmailEvent/ARM/imEmailEvent/imEmailEvent.json +++ b/Parsers/ASimEmailEvent/ARM/imEmailEvent/imEmailEvent.json @@ -27,7 +27,7 @@ "displayName": "Email Event ASIM filtering parser", "category": "ASIM", "FunctionAlias": "imEmailEvent", - "query": "let DisabledParsers=materialize(_GetWatchlist('ASimDisabledParsers') | where SearchKey in ('Any', 'ExcludeimEmailEvent') | extend SourceSpecificParser=column_ifexists('SourceSpecificParser','') | distinct SourceSpecificParser | where isnotempty(SourceSpecificParser));\nlet imBuiltInDisabled=toscalar('ExcludeimEmailEvent' in (DisabledParsers) or 'Any' in (DisabledParsers)); \nlet parser=(\n starttime: datetime=datetime(null), \n endtime: datetime=datetime(null), \n ipaddr_has_any_prefix: dynamic=dynamic([]),\n emailfromaddress_has_any: dynamic=dynamic([]),\n emailrecipient_has_any: dynamic=dynamic([]),\n emailsubject_has_any: dynamic=dynamic([]),\n emaildirection_in: dynamic=dynamic([]),\n emaildeliverylocation_in: dynamic=dynamic([]),\n eventresult: string='*',\n pack:bool=false)\n{\nunion isfuzzy=true\n vimEmailEventEmpty\n};\nparser (starttime=starttime, endtime=endtime, ipaddr_has_any_prefix=ipaddr_has_any_prefix, emailfromaddress_has_any=emailfromaddress_has_any, emailrecipient_has_any=emailrecipient_has_any, emailsubject_has_any=emailsubject_has_any, emaildirection_in=emaildirection_in, emaildeliverylocation_in=emaildeliverylocation_in, eventresult=eventresult, pack=pack)", + "query": "let DisabledParsers=materialize(_GetWatchlist('ASimDisabledParsers') | where SearchKey in ('Any', 'ExcludeimEmailEvent') | extend SourceSpecificParser=column_ifexists('SourceSpecificParser','') | distinct SourceSpecificParser | where isnotempty(SourceSpecificParser));\nlet imBuiltInDisabled=toscalar('ExcludeimEmailEvent' in (DisabledParsers) or 'Any' in (DisabledParsers)); \nlet parser=(\n starttime: datetime=datetime(null), \n endtime: datetime=datetime(null), \n ipaddr_has_any_prefix: dynamic=dynamic([]),\n emailfromaddress_has_any: dynamic=dynamic([]),\n emailrecipient_has_any: dynamic=dynamic([]),\n emailsubject_has_any: dynamic=dynamic([]),\n emaildirection_in: dynamic=dynamic([]),\n emaildeliverylocation_in: dynamic=dynamic([]),\n eventresult: string='*',\n pack:bool=false)\n{\nunion isfuzzy=true\n vimEmailEventEmpty,\n vimEmailEventProofpointPODMailLog(\n starttime=starttime, endtime=endtime, ipaddr_has_any_prefix=ipaddr_has_any_prefix,\n emailfromaddress_has_any=emailfromaddress_has_any, emailrecipient_has_any=emailrecipient_has_any,\n emailsubject_has_any=emailsubject_has_any, emaildirection_in=emaildirection_in,\n emaildeliverylocation_in=emaildeliverylocation_in, eventresult=eventresult, pack=pack,\n disabled=(imBuiltInDisabled or ('ExcludevimEmailEventProofpointPODMailLog' in (DisabledParsers))))\n};\nparser (starttime=starttime, endtime=endtime, ipaddr_has_any_prefix=ipaddr_has_any_prefix, emailfromaddress_has_any=emailfromaddress_has_any, emailrecipient_has_any=emailrecipient_has_any, emailsubject_has_any=emailsubject_has_any, emaildirection_in=emaildirection_in, emaildeliverylocation_in=emaildeliverylocation_in, eventresult=eventresult, pack=pack)", "version": 1, "functionParameters": "starttime:datetime=datetime(null),endtime:datetime=datetime(null),ipaddr_has_any_prefix:dynamic=dynamic([]),emailfromaddress_has_any:dynamic=dynamic([]),emailrecipient_has_any:dynamic=dynamic([]),emailsubject_has_any:dynamic=dynamic([]),emaildirection_in:dynamic=dynamic([]),emaildeliverylocation_in:dynamic=dynamic([]),eventresult:string='*',pack:bool=False" } diff --git a/Parsers/ASimEmailEvent/ARM/vimEmailEventProofpointPODMailLog/README.md b/Parsers/ASimEmailEvent/ARM/vimEmailEventProofpointPODMailLog/README.md new file mode 100644 index 00000000000..0433444dd2c --- /dev/null +++ b/Parsers/ASimEmailEvent/ARM/vimEmailEventProofpointPODMailLog/README.md @@ -0,0 +1,24 @@ +# Proofpoint POD MailLog ASIM EmailEvent Normalization Parser + +ARM template for ASIM EmailEvent schema parser for Proofpoint POD MailLog. + +Filters and normalizes ProofpointPODMailLog_CL SMTP receipt and delivery records to EmailEvent. +Uses the CCP table schema and repository samples; live validation is deferred. +Header sender, header recipients, subject and delivery location are unavailable; +nonempty filters for these fields return no records. + + +The Advanced Security Information Model (ASIM) enables you to use and create source-agnostic content, simplifying your analysis of the data in your Microsoft Sentinel workspace. + +For more information, see: + +- [Normalization and the Advanced Security Information Model (ASIM)](https://aka.ms/AboutASIM) +- [Deploy all of ASIM](https://aka.ms/DeployASIM) +- [ASIM EmailEvent normalization schema reference](https://aka.ms/ASimEmailEventDoc) + +For the changelog, see: +- [CHANGELOG](https://github.com/Azure/Azure-Sentinel/blob/master/Parsers/ASimEmailEvent/CHANGELOG/vimEmailEventProofpointPODMailLog.md) + +
+ +[![Deploy to Azure](https://aka.ms/deploytoazurebutton)](https://portal.azure.com/#create/Microsoft.Template/uri/https%3A%2F%2Fraw.githubusercontent.com%2FAzure%2FAzure-Sentinel%2Fmaster%2FParsers%2FASimEmailEvent%2FARM%2FvimEmailEventProofpointPODMailLog%2FvimEmailEventProofpointPODMailLog.json) [![Deploy to Azure Gov](https://aka.ms/deploytoazuregovernbutton)](https://portal.azure.us/#create/Microsoft.Template/uri/https%3A%2F%2Fraw.githubusercontent.com%2FAzure%2FAzure-Sentinel%2Fmaster%2FParsers%2FASimEmailEvent%2FARM%2FvimEmailEventProofpointPODMailLog%2FvimEmailEventProofpointPODMailLog.json) diff --git a/Parsers/ASimEmailEvent/ARM/vimEmailEventProofpointPODMailLog/vimEmailEventProofpointPODMailLog.json b/Parsers/ASimEmailEvent/ARM/vimEmailEventProofpointPODMailLog/vimEmailEventProofpointPODMailLog.json new file mode 100644 index 00000000000..1427541ca7e --- /dev/null +++ b/Parsers/ASimEmailEvent/ARM/vimEmailEventProofpointPODMailLog/vimEmailEventProofpointPODMailLog.json @@ -0,0 +1,36 @@ +{ + "$schema": "https://schema.management.azure.com/schemas/2019-08-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "Workspace": { + "type": "string", + "metadata": { + "description": "The Microsoft Sentinel workspace into which the function will be deployed. Has to be in the selected Resource Group." + } + }, + "WorkspaceRegion": { + "type": "string", + "defaultValue": "[resourceGroup().location]", + "metadata": { + "description": "The region of the selected workspace. The default value will use the Region selection above." + } + } + }, + "resources": [ + { + "type": "Microsoft.OperationalInsights/workspaces/savedSearches", + "apiVersion": "2020-08-01", + "name": "[concat(parameters('Workspace'), '/vimEmailEventProofpointPODMailLog')]", + "location": "[parameters('WorkspaceRegion')]", + "properties": { + "etag": "*", + "displayName": "Email Event ASIM filtering parser for Proofpoint POD MailLog", + "category": "ASIM", + "FunctionAlias": "vimEmailEventProofpointPODMailLog", + "query": "let parser = (\n starttime:datetime=datetime(null),\n endtime:datetime=datetime(null),\n ipaddr_has_any_prefix:dynamic=dynamic([]),\n emailfromaddress_has_any:dynamic=dynamic([]),\n emailrecipient_has_any:dynamic=dynamic([]),\n emailsubject_has_any:dynamic=dynamic([]),\n emaildirection_in:dynamic=dynamic([]),\n emaildeliverylocation_in:dynamic=dynamic([]),\n eventresult:string='*',\n disabled:bool=false,\n pack:bool=false)\n{\n ProofpointPODMailLog_CL\n | where not(disabled)\n | where (isnull(starttime) or eventTime >= starttime or (isnull(eventTime) and TimeGenerated >= starttime))\n and (isnull(endtime) or eventTime <= endtime or (isnull(eventTime) and TimeGenerated <= endtime))\n | where array_length(emailfromaddress_has_any) == 0 and array_length(emailrecipient_has_any) == 0\n and array_length(emailsubject_has_any) == 0 and array_length(emaildeliverylocation_in) == 0\n | where array_length(emaildirection_in) == 0 or \"Unknown\" in (emaildirection_in)\n | where isnotempty(tostring(sm.qid)) and (bag_has_key(sm, \"from\") or array_length(sm[\"to\"]) > 0)\n | extend\n IsDelivery = coalesce(array_length(sm[\"to\"]), 0) > 0,\n Dsn = tostring(sm.dsn)\n | extend EventResult = case(Dsn startswith \"2.\", \"Success\", Dsn startswith \"4.\" or Dsn startswith \"5.\", \"Failure\", \"NA\")\n | where eventresult == \"*\" or EventResult == eventresult\n | extend\n Relay = tostring(sm.relay),\n Dvc = coalesce(tostring(pps.agent), tostring(metadata.origin.data.agent)),\n EventStartTime = coalesce(eventTime, TimeGenerated),\n EmailMailFromAddress = trim(@\"[<>\\s]\", tostring(sm[\"from\"])),\n EmailSizeBytes = tolong(sm.sizeBytes),\n DeliveryStatus = tostring(sm.stat)\n | parse Relay with RelayHost \" [\" RelayIp \"]\" *\n | extend\n EmailMailFromAddress = iff(EmailMailFromAddress contains \"@\", EmailMailFromAddress, \"\"),\n RelayHost = trim_end(@\"\\.\", coalesce(RelayHost, Relay)),\n RelayIp = iff(isnotnull(parse_ipv4(RelayIp)) or isnotempty(parse_ipv6(RelayIp)), RelayIp, \"\"),\n EmailProtocolName = case(tostring(sm.proto) =~ \"SMTP\" or tostring(sm.mailer) in~ (\"smtp\", \"esmtp\"), \"SMTP\", \"\"),\n EventSubType = iff(IsDelivery, \"EmailSent\", \"EmailReceived\")\n | extend\n SrcIpAddr = iff(IsDelivery, \"\", RelayIp),\n DstIpAddr = iff(IsDelivery, RelayIp, \"\"),\n Src = iff(IsDelivery, Dvc, coalesce(RelayIp, RelayHost)),\n Dst = iff(IsDelivery, coalesce(RelayIp, RelayHost), Dvc),\n SrcFQDN = iff(not(IsDelivery) and RelayHost contains \".\", RelayHost, \"\"),\n SrcHostname = iff(IsDelivery, tostring(split(Dvc, \".\")[0]), tostring(split(RelayHost, \".\")[0])),\n SrcDomain = iff(IsDelivery, substring(Dvc, indexof(Dvc, \".\") + 1), substring(RelayHost, indexof(RelayHost, \".\") + 1)),\n DvcHostname = tostring(split(Dvc, \".\")[0]),\n DvcDomain = iff(Dvc contains \".\", substring(Dvc, indexof(Dvc, \".\") + 1), \"\"),\n DvcFQDN = iff(Dvc contains \".\", Dvc, \"\"),\n EmailMailFromDomain = iff(EmailMailFromAddress contains \"@\", tostring(split(EmailMailFromAddress, \"@\")[1]), \"\"),\n DvcAction = iff(IsDelivery and EventResult == \"Success\" and DeliveryStatus startswith \"Sent\", \"Delivered\", \"NA\")\n | where array_length(ipaddr_has_any_prefix) == 0\n or has_any_ipv4_prefix(SrcIpAddr, ipaddr_has_any_prefix)\n or has_any_ipv4_prefix(DstIpAddr, ipaddr_has_any_prefix)\n | extend\n SrcFQDN = iff(IsDelivery, DvcFQDN, SrcFQDN),\n SrcDomain = iff(iff(IsDelivery, Dvc, RelayHost) contains \".\", SrcDomain, \"\"),\n DvcDomainType = iff(isnotempty(DvcDomain), \"FQDN\", \"\"),\n EventEndTime = EventStartTime,\n EventCount = int(1),\n EventVendor = \"Proofpoint\",\n EventProduct = \"POD MailLog\",\n EventSchema = \"EmailEvent\",\n EventSchemaVersion = \"1.0.0\",\n EventType = \"EmailDelivery\",\n EventOriginalType = \"maillog\",\n EmailDirection = \"Unknown\",\n EventOriginalResultDetails = strcat(Dsn, iff(isnotempty(Dsn) and isnotempty(DeliveryStatus), \"; \", \"\"), DeliveryStatus),\n EventUid = iff(isnotempty(id), strcat(\"ProofpointPODMailLog_CL:\", id), \"\"),\n DvcEntityKey = \"\",\n AdditionalEntities = dynamic([]),\n AdditionalFields = iff(pack, bag_pack(\"sm\", sm, \"tls\", tls, \"pps\", pps, \"metadata\", metadata), dynamic({})),\n Type = \"ProofpointPODMailLog_CL\"\n | extend SrcDomainType = iff(isnotempty(SrcDomain), \"FQDN\", \"\"), IpAddr = SrcIpAddr\n | project-rename EventOriginalUid = id, EventMessage = data\n | project\n TimeGenerated, Type, EventStartTime, EventEndTime, EventCount,\n EventVendor, EventProduct, EventSchema, EventSchemaVersion,\n EventType, EventSubType, EventOriginalType, EventResult,\n EventOriginalResultDetails, EventOriginalUid, EventUid, EventMessage,\n Dvc, DvcHostname, DvcDomain, DvcDomainType, DvcFQDN, DvcAction, DvcEntityKey,\n Src, SrcIpAddr, SrcHostname, SrcDomain, SrcDomainType, SrcFQDN, Dst, DstIpAddr, IpAddr,\n EmailMailFromAddress, EmailMailFromDomain, EmailSizeBytes, EmailProtocolName, EmailDirection,\n AdditionalFields, AdditionalEntities\n};\nparser(starttime=starttime, endtime=endtime, ipaddr_has_any_prefix=ipaddr_has_any_prefix,\n emailfromaddress_has_any=emailfromaddress_has_any, emailrecipient_has_any=emailrecipient_has_any,\n emailsubject_has_any=emailsubject_has_any, emaildirection_in=emaildirection_in,\n emaildeliverylocation_in=emaildeliverylocation_in, eventresult=eventresult, disabled=disabled, pack=pack)\n", + "version": 1, + "functionParameters": "starttime:datetime=datetime(null),endtime:datetime=datetime(null),ipaddr_has_any_prefix:dynamic=dynamic([]),emailfromaddress_has_any:dynamic=dynamic([]),emailrecipient_has_any:dynamic=dynamic([]),emailsubject_has_any:dynamic=dynamic([]),emaildirection_in:dynamic=dynamic([]),emaildeliverylocation_in:dynamic=dynamic([]),eventresult:string='*',disabled:bool=False,pack:bool=False" + } + } + ] +} \ No newline at end of file From af9ac0ad021c66486d4e1057f784e2e7d819a50f Mon Sep 17 00:00:00 2001 From: Aditi Bhoge Date: Fri, 23 Oct 2026 11:21:40 -0700 Subject: [PATCH 05/10] Register MailLog CCP schema and handle absent optional source IDs Match the connector schema in CI KQL catalog. ASIM sample ingestion excludes id; preserve actual connector IDs when present and leave optional event IDs empty when absent, without changing validation gates. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 321517f1-a1c5-4242-be85-2d144a9cc295 --- .../CustomTables/ProofpointPODMailLog_CL.json | 13 ++++++++++++ .../ASimEmailEventProofpointPODMailLog.md | 5 +++++ .../vimEmailEventProofpointPODMailLog.md | 5 +++++ .../ASimEmailEventProofpointPODMailLog.yaml | 12 ++++++----- .../vimEmailEventProofpointPODMailLog.yaml | 12 ++++++----- Parsers/ASimEmailEvent/README.md | 20 ++++++++++++------- 6 files changed, 50 insertions(+), 17 deletions(-) create mode 100644 .script/tests/KqlvalidationsTests/CustomTables/ProofpointPODMailLog_CL.json diff --git a/.script/tests/KqlvalidationsTests/CustomTables/ProofpointPODMailLog_CL.json b/.script/tests/KqlvalidationsTests/CustomTables/ProofpointPODMailLog_CL.json new file mode 100644 index 00000000000..8cf6bcc1b1d --- /dev/null +++ b/.script/tests/KqlvalidationsTests/CustomTables/ProofpointPODMailLog_CL.json @@ -0,0 +1,13 @@ +{ + "Name": "ProofpointPODMailLog_CL", + "Properties": [ + { "Name": "TimeGenerated", "Type": "DateTime" }, + { "Name": "id", "Type": "String" }, + { "Name": "data", "Type": "String" }, + { "Name": "eventTime", "Type": "DateTime" }, + { "Name": "metadata", "Type": "Dynamic" }, + { "Name": "pps", "Type": "Dynamic" }, + { "Name": "sm", "Type": "Dynamic" }, + { "Name": "tls", "Type": "Dynamic" } + ] +} diff --git a/Parsers/ASimEmailEvent/CHANGELOG/ASimEmailEventProofpointPODMailLog.md b/Parsers/ASimEmailEvent/CHANGELOG/ASimEmailEventProofpointPODMailLog.md index f98462a80d1..74b0c651f42 100644 --- a/Parsers/ASimEmailEvent/CHANGELOG/ASimEmailEventProofpointPODMailLog.md +++ b/Parsers/ASimEmailEvent/CHANGELOG/ASimEmailEventProofpointPODMailLog.md @@ -1,5 +1,10 @@ # Changelog for ASimEmailEventProofpointPODMailLog.yaml +## Version 0.1.1 + +- (2026-10-23) Handle missing optional source IDs in CI-ingested records while preserving connector-provided IDs - [PR #15410](https://github.com/Azure/Azure-Sentinel/pull/15410) +- Register the current CCP table schema in the KQL validation catalog. + ## Version 0.1.0 - (2026-10-21) Added a Proofpoint POD MailLog EmailEvent parser using the CCP table schema and repository samples - [PR #15410](https://github.com/Azure/Azure-Sentinel/pull/15410) diff --git a/Parsers/ASimEmailEvent/CHANGELOG/vimEmailEventProofpointPODMailLog.md b/Parsers/ASimEmailEvent/CHANGELOG/vimEmailEventProofpointPODMailLog.md index 8ad339bf38d..56c68ed2d32 100644 --- a/Parsers/ASimEmailEvent/CHANGELOG/vimEmailEventProofpointPODMailLog.md +++ b/Parsers/ASimEmailEvent/CHANGELOG/vimEmailEventProofpointPODMailLog.md @@ -1,5 +1,10 @@ # Changelog for vimEmailEventProofpointPODMailLog.yaml +## Version 0.1.1 + +- (2026-10-23) Handle missing optional source IDs in CI-ingested records without changing filtering or inventing event identifiers - [PR #15410](https://github.com/Azure/Azure-Sentinel/pull/15410) +- Register the current CCP table schema in the KQL validation catalog. + ## Version 0.1.0 - (2026-10-21) Added a Proofpoint POD MailLog EmailEvent filtering parser with time, IPv4 prefix, direction and result filters - [PR #15410](https://github.com/Azure/Azure-Sentinel/pull/15410) diff --git a/Parsers/ASimEmailEvent/Parsers/ASimEmailEventProofpointPODMailLog.yaml b/Parsers/ASimEmailEvent/Parsers/ASimEmailEventProofpointPODMailLog.yaml index f4f1deaa558..002ad44a980 100644 --- a/Parsers/ASimEmailEvent/Parsers/ASimEmailEventProofpointPODMailLog.yaml +++ b/Parsers/ASimEmailEvent/Parsers/ASimEmailEventProofpointPODMailLog.yaml @@ -1,7 +1,7 @@ Parser: Title: Email Event ASIM parser for Proofpoint POD MailLog - Version: '0.1.0' - LastUpdated: Oct 21, 2026 + Version: '0.1.1' + LastUpdated: Oct 23, 2026 Product: Name: Proofpoint POD MailLog Normalization: @@ -76,13 +76,15 @@ ParserQuery: | EventOriginalType = "maillog", EmailDirection = "Unknown", EventOriginalResultDetails = strcat(Dsn, iff(isnotempty(Dsn) and isnotempty(DeliveryStatus), "; ", ""), DeliveryStatus), - EventUid = iff(isnotempty(id), strcat("ProofpointPODMailLog_CL:", id), ""), + EventOriginalUid = tostring(column_ifexists("id", "")), DvcEntityKey = "", AdditionalEntities = dynamic([]), AdditionalFields = iff(pack, bag_pack("sm", sm, "tls", tls, "pps", pps, "metadata", metadata), dynamic({})), Type = "ProofpointPODMailLog_CL" - | extend SrcDomainType = iff(isnotempty(SrcDomain), "FQDN", ""), IpAddr = SrcIpAddr - | project-rename EventOriginalUid = id, EventMessage = data + | extend + EventUid = iff(isnotempty(EventOriginalUid), strcat("ProofpointPODMailLog_CL:", EventOriginalUid), ""), + SrcDomainType = iff(isnotempty(SrcDomain), "FQDN", ""), IpAddr = SrcIpAddr + | project-rename EventMessage = data | project TimeGenerated, Type, EventStartTime, EventEndTime, EventCount, EventVendor, EventProduct, EventSchema, EventSchemaVersion, diff --git a/Parsers/ASimEmailEvent/Parsers/vimEmailEventProofpointPODMailLog.yaml b/Parsers/ASimEmailEvent/Parsers/vimEmailEventProofpointPODMailLog.yaml index 3e77e485b95..929457c6597 100644 --- a/Parsers/ASimEmailEvent/Parsers/vimEmailEventProofpointPODMailLog.yaml +++ b/Parsers/ASimEmailEvent/Parsers/vimEmailEventProofpointPODMailLog.yaml @@ -1,7 +1,7 @@ Parser: Title: Email Event ASIM filtering parser for Proofpoint POD MailLog - Version: '0.1.0' - LastUpdated: Oct 21, 2026 + Version: '0.1.1' + LastUpdated: Oct 23, 2026 Product: Name: Proofpoint POD MailLog Normalization: @@ -126,13 +126,15 @@ ParserQuery: | EventOriginalType = "maillog", EmailDirection = "Unknown", EventOriginalResultDetails = strcat(Dsn, iff(isnotempty(Dsn) and isnotempty(DeliveryStatus), "; ", ""), DeliveryStatus), - EventUid = iff(isnotempty(id), strcat("ProofpointPODMailLog_CL:", id), ""), + EventOriginalUid = tostring(column_ifexists("id", "")), DvcEntityKey = "", AdditionalEntities = dynamic([]), AdditionalFields = iff(pack, bag_pack("sm", sm, "tls", tls, "pps", pps, "metadata", metadata), dynamic({})), Type = "ProofpointPODMailLog_CL" - | extend SrcDomainType = iff(isnotempty(SrcDomain), "FQDN", ""), IpAddr = SrcIpAddr - | project-rename EventOriginalUid = id, EventMessage = data + | extend + EventUid = iff(isnotempty(EventOriginalUid), strcat("ProofpointPODMailLog_CL:", EventOriginalUid), ""), + SrcDomainType = iff(isnotempty(SrcDomain), "FQDN", ""), IpAddr = SrcIpAddr + | project-rename EventMessage = data | project TimeGenerated, Type, EventStartTime, EventEndTime, EventCount, EventVendor, EventProduct, EventSchema, EventSchemaVersion, diff --git a/Parsers/ASimEmailEvent/README.md b/Parsers/ASimEmailEvent/README.md index ee6295ece96..8d0b5ba65f2 100644 --- a/Parsers/ASimEmailEvent/README.md +++ b/Parsers/ASimEmailEvent/README.md @@ -92,10 +92,16 @@ is tracked in [draft PR #15410](https://github.com/Azure/Azure-Sentinel/pull/154 header/security fields are intentional source limitations, not evidence of empty headers or zero attachments. -The repository's focused `Validate_ParsersFunctions_HaveValidKql` tests currently -fail with `KS204`: their custom-table catalog does not define -`ProofpointPODMailLog_CL`. The standalone offline compiler checks use the -connector's declared schema and pass, but do not replace those repository tests. -A maintainer must add the CCP table definition to the protected validation -catalog before those tests can pass; this parser task does not modify tooling -or test implementation. \ No newline at end of file +The current CCP table schema is registered in +`.script/tests/KqlvalidationsTests/CustomTables/ProofpointPODMailLog_CL.json`, +separately from the legacy flattened table. Missing catalog registration caused +the earlier KQL validation `KS204` errors. + +ASIM CI's sample-ingestion script excludes reserved columns including `id` +from the table schema and payload. This caused the earlier schema/data and +filtering jobs to fail with `SEM0100` when compiling references to `id`. +Both parsers now read this optional identifier with `column_ifexists`: +connector-provided IDs are preserved, while missing IDs produce empty +`EventOriginalUid` and `EventUid` rather than fabricated identifiers. Required +source columns remain required. The ingestion script and validation gates +are unchanged; user-workspace live validation remains deferred. \ No newline at end of file From 71ae0b66e8fda99a965eb5023d109da5f26f0138 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <> Date: Fri, 23 Oct 2026 18:24:35 +0000 Subject: [PATCH 06/10] [ASIM Parsers] Generate deployable ARM templates from KQL function YAML files. --- .../ASimEmailEventProofpointPODMailLog.json | 2 +- .../vimEmailEventProofpointPODMailLog.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Parsers/ASimEmailEvent/ARM/ASimEmailEventProofpointPODMailLog/ASimEmailEventProofpointPODMailLog.json b/Parsers/ASimEmailEvent/ARM/ASimEmailEventProofpointPODMailLog/ASimEmailEventProofpointPODMailLog.json index 36932c82b74..47bf4db2af6 100644 --- a/Parsers/ASimEmailEvent/ARM/ASimEmailEventProofpointPODMailLog/ASimEmailEventProofpointPODMailLog.json +++ b/Parsers/ASimEmailEvent/ARM/ASimEmailEventProofpointPODMailLog/ASimEmailEventProofpointPODMailLog.json @@ -27,7 +27,7 @@ "displayName": "Email Event ASIM parser for Proofpoint POD MailLog", "category": "ASIM", "FunctionAlias": "ASimEmailEventProofpointPODMailLog", - "query": "let parser = (disabled:bool=false, pack:bool=false) {\n ProofpointPODMailLog_CL\n | where not(disabled)\n | where isnotempty(tostring(sm.qid)) and (bag_has_key(sm, \"from\") or array_length(sm[\"to\"]) > 0)\n | extend\n IsDelivery = coalesce(array_length(sm[\"to\"]), 0) > 0,\n Relay = tostring(sm.relay),\n Dvc = coalesce(tostring(pps.agent), tostring(metadata.origin.data.agent)),\n EventStartTime = coalesce(eventTime, TimeGenerated),\n EmailMailFromAddress = trim(@\"[<>\\s]\", tostring(sm[\"from\"])),\n EmailSizeBytes = tolong(sm.sizeBytes),\n Dsn = tostring(sm.dsn),\n DeliveryStatus = tostring(sm.stat)\n | parse Relay with RelayHost \" [\" RelayIp \"]\" *\n | extend\n EmailMailFromAddress = iff(EmailMailFromAddress contains \"@\", EmailMailFromAddress, \"\"),\n RelayHost = trim_end(@\"\\.\", coalesce(RelayHost, Relay)),\n RelayIp = iff(isnotnull(parse_ipv4(RelayIp)) or isnotempty(parse_ipv6(RelayIp)), RelayIp, \"\"),\n EventResult = case(Dsn startswith \"2.\", \"Success\", Dsn startswith \"4.\" or Dsn startswith \"5.\", \"Failure\", \"NA\"),\n EmailProtocolName = case(tostring(sm.proto) =~ \"SMTP\" or tostring(sm.mailer) in~ (\"smtp\", \"esmtp\"), \"SMTP\", \"\"),\n EventSubType = iff(IsDelivery, \"EmailSent\", \"EmailReceived\")\n | extend\n SrcIpAddr = iff(IsDelivery, \"\", RelayIp),\n DstIpAddr = iff(IsDelivery, RelayIp, \"\"),\n Src = iff(IsDelivery, Dvc, coalesce(RelayIp, RelayHost)),\n Dst = iff(IsDelivery, coalesce(RelayIp, RelayHost), Dvc),\n SrcFQDN = iff(not(IsDelivery) and RelayHost contains \".\", RelayHost, \"\"),\n SrcHostname = iff(IsDelivery, tostring(split(Dvc, \".\")[0]), tostring(split(RelayHost, \".\")[0])),\n SrcDomain = iff(IsDelivery, substring(Dvc, indexof(Dvc, \".\") + 1), substring(RelayHost, indexof(RelayHost, \".\") + 1)),\n DvcHostname = tostring(split(Dvc, \".\")[0]),\n DvcDomain = iff(Dvc contains \".\", substring(Dvc, indexof(Dvc, \".\") + 1), \"\"),\n DvcFQDN = iff(Dvc contains \".\", Dvc, \"\"),\n EmailMailFromDomain = iff(EmailMailFromAddress contains \"@\", tostring(split(EmailMailFromAddress, \"@\")[1]), \"\"),\n DvcAction = iff(IsDelivery and EventResult == \"Success\" and DeliveryStatus startswith \"Sent\", \"Delivered\", \"NA\")\n | extend\n SrcFQDN = iff(IsDelivery, DvcFQDN, SrcFQDN),\n SrcDomain = iff(iff(IsDelivery, Dvc, RelayHost) contains \".\", SrcDomain, \"\"),\n DvcDomainType = iff(isnotempty(DvcDomain), \"FQDN\", \"\"),\n EventEndTime = EventStartTime,\n EventCount = int(1),\n EventVendor = \"Proofpoint\",\n EventProduct = \"POD MailLog\",\n EventSchema = \"EmailEvent\",\n EventSchemaVersion = \"1.0.0\",\n EventType = \"EmailDelivery\",\n EventOriginalType = \"maillog\",\n EmailDirection = \"Unknown\",\n EventOriginalResultDetails = strcat(Dsn, iff(isnotempty(Dsn) and isnotempty(DeliveryStatus), \"; \", \"\"), DeliveryStatus),\n EventUid = iff(isnotempty(id), strcat(\"ProofpointPODMailLog_CL:\", id), \"\"),\n DvcEntityKey = \"\",\n AdditionalEntities = dynamic([]),\n AdditionalFields = iff(pack, bag_pack(\"sm\", sm, \"tls\", tls, \"pps\", pps, \"metadata\", metadata), dynamic({})),\n Type = \"ProofpointPODMailLog_CL\"\n | extend SrcDomainType = iff(isnotempty(SrcDomain), \"FQDN\", \"\"), IpAddr = SrcIpAddr\n | project-rename EventOriginalUid = id, EventMessage = data\n | project\n TimeGenerated, Type, EventStartTime, EventEndTime, EventCount,\n EventVendor, EventProduct, EventSchema, EventSchemaVersion,\n EventType, EventSubType, EventOriginalType, EventResult,\n EventOriginalResultDetails, EventOriginalUid, EventUid, EventMessage,\n Dvc, DvcHostname, DvcDomain, DvcDomainType, DvcFQDN, DvcAction, DvcEntityKey,\n Src, SrcIpAddr, SrcHostname, SrcDomain, SrcDomainType, SrcFQDN, Dst, DstIpAddr, IpAddr,\n EmailMailFromAddress, EmailMailFromDomain, EmailSizeBytes, EmailProtocolName, EmailDirection,\n AdditionalFields, AdditionalEntities\n};\nparser(disabled=disabled, pack=pack)\n", + "query": "let parser = (disabled:bool=false, pack:bool=false) {\n ProofpointPODMailLog_CL\n | where not(disabled)\n | where isnotempty(tostring(sm.qid)) and (bag_has_key(sm, \"from\") or array_length(sm[\"to\"]) > 0)\n | extend\n IsDelivery = coalesce(array_length(sm[\"to\"]), 0) > 0,\n Relay = tostring(sm.relay),\n Dvc = coalesce(tostring(pps.agent), tostring(metadata.origin.data.agent)),\n EventStartTime = coalesce(eventTime, TimeGenerated),\n EmailMailFromAddress = trim(@\"[<>\\s]\", tostring(sm[\"from\"])),\n EmailSizeBytes = tolong(sm.sizeBytes),\n Dsn = tostring(sm.dsn),\n DeliveryStatus = tostring(sm.stat)\n | parse Relay with RelayHost \" [\" RelayIp \"]\" *\n | extend\n EmailMailFromAddress = iff(EmailMailFromAddress contains \"@\", EmailMailFromAddress, \"\"),\n RelayHost = trim_end(@\"\\.\", coalesce(RelayHost, Relay)),\n RelayIp = iff(isnotnull(parse_ipv4(RelayIp)) or isnotempty(parse_ipv6(RelayIp)), RelayIp, \"\"),\n EventResult = case(Dsn startswith \"2.\", \"Success\", Dsn startswith \"4.\" or Dsn startswith \"5.\", \"Failure\", \"NA\"),\n EmailProtocolName = case(tostring(sm.proto) =~ \"SMTP\" or tostring(sm.mailer) in~ (\"smtp\", \"esmtp\"), \"SMTP\", \"\"),\n EventSubType = iff(IsDelivery, \"EmailSent\", \"EmailReceived\")\n | extend\n SrcIpAddr = iff(IsDelivery, \"\", RelayIp),\n DstIpAddr = iff(IsDelivery, RelayIp, \"\"),\n Src = iff(IsDelivery, Dvc, coalesce(RelayIp, RelayHost)),\n Dst = iff(IsDelivery, coalesce(RelayIp, RelayHost), Dvc),\n SrcFQDN = iff(not(IsDelivery) and RelayHost contains \".\", RelayHost, \"\"),\n SrcHostname = iff(IsDelivery, tostring(split(Dvc, \".\")[0]), tostring(split(RelayHost, \".\")[0])),\n SrcDomain = iff(IsDelivery, substring(Dvc, indexof(Dvc, \".\") + 1), substring(RelayHost, indexof(RelayHost, \".\") + 1)),\n DvcHostname = tostring(split(Dvc, \".\")[0]),\n DvcDomain = iff(Dvc contains \".\", substring(Dvc, indexof(Dvc, \".\") + 1), \"\"),\n DvcFQDN = iff(Dvc contains \".\", Dvc, \"\"),\n EmailMailFromDomain = iff(EmailMailFromAddress contains \"@\", tostring(split(EmailMailFromAddress, \"@\")[1]), \"\"),\n DvcAction = iff(IsDelivery and EventResult == \"Success\" and DeliveryStatus startswith \"Sent\", \"Delivered\", \"NA\")\n | extend\n SrcFQDN = iff(IsDelivery, DvcFQDN, SrcFQDN),\n SrcDomain = iff(iff(IsDelivery, Dvc, RelayHost) contains \".\", SrcDomain, \"\"),\n DvcDomainType = iff(isnotempty(DvcDomain), \"FQDN\", \"\"),\n EventEndTime = EventStartTime,\n EventCount = int(1),\n EventVendor = \"Proofpoint\",\n EventProduct = \"POD MailLog\",\n EventSchema = \"EmailEvent\",\n EventSchemaVersion = \"1.0.0\",\n EventType = \"EmailDelivery\",\n EventOriginalType = \"maillog\",\n EmailDirection = \"Unknown\",\n EventOriginalResultDetails = strcat(Dsn, iff(isnotempty(Dsn) and isnotempty(DeliveryStatus), \"; \", \"\"), DeliveryStatus),\n EventOriginalUid = tostring(column_ifexists(\"id\", \"\")),\n DvcEntityKey = \"\",\n AdditionalEntities = dynamic([]),\n AdditionalFields = iff(pack, bag_pack(\"sm\", sm, \"tls\", tls, \"pps\", pps, \"metadata\", metadata), dynamic({})),\n Type = \"ProofpointPODMailLog_CL\"\n | extend\n EventUid = iff(isnotempty(EventOriginalUid), strcat(\"ProofpointPODMailLog_CL:\", EventOriginalUid), \"\"),\n SrcDomainType = iff(isnotempty(SrcDomain), \"FQDN\", \"\"), IpAddr = SrcIpAddr\n | project-rename EventMessage = data\n | project\n TimeGenerated, Type, EventStartTime, EventEndTime, EventCount,\n EventVendor, EventProduct, EventSchema, EventSchemaVersion,\n EventType, EventSubType, EventOriginalType, EventResult,\n EventOriginalResultDetails, EventOriginalUid, EventUid, EventMessage,\n Dvc, DvcHostname, DvcDomain, DvcDomainType, DvcFQDN, DvcAction, DvcEntityKey,\n Src, SrcIpAddr, SrcHostname, SrcDomain, SrcDomainType, SrcFQDN, Dst, DstIpAddr, IpAddr,\n EmailMailFromAddress, EmailMailFromDomain, EmailSizeBytes, EmailProtocolName, EmailDirection,\n AdditionalFields, AdditionalEntities\n};\nparser(disabled=disabled, pack=pack)\n", "version": 1, "functionParameters": "disabled:bool=False,pack:bool=False" } diff --git a/Parsers/ASimEmailEvent/ARM/vimEmailEventProofpointPODMailLog/vimEmailEventProofpointPODMailLog.json b/Parsers/ASimEmailEvent/ARM/vimEmailEventProofpointPODMailLog/vimEmailEventProofpointPODMailLog.json index 1427541ca7e..1a0d89b9759 100644 --- a/Parsers/ASimEmailEvent/ARM/vimEmailEventProofpointPODMailLog/vimEmailEventProofpointPODMailLog.json +++ b/Parsers/ASimEmailEvent/ARM/vimEmailEventProofpointPODMailLog/vimEmailEventProofpointPODMailLog.json @@ -27,7 +27,7 @@ "displayName": "Email Event ASIM filtering parser for Proofpoint POD MailLog", "category": "ASIM", "FunctionAlias": "vimEmailEventProofpointPODMailLog", - "query": "let parser = (\n starttime:datetime=datetime(null),\n endtime:datetime=datetime(null),\n ipaddr_has_any_prefix:dynamic=dynamic([]),\n emailfromaddress_has_any:dynamic=dynamic([]),\n emailrecipient_has_any:dynamic=dynamic([]),\n emailsubject_has_any:dynamic=dynamic([]),\n emaildirection_in:dynamic=dynamic([]),\n emaildeliverylocation_in:dynamic=dynamic([]),\n eventresult:string='*',\n disabled:bool=false,\n pack:bool=false)\n{\n ProofpointPODMailLog_CL\n | where not(disabled)\n | where (isnull(starttime) or eventTime >= starttime or (isnull(eventTime) and TimeGenerated >= starttime))\n and (isnull(endtime) or eventTime <= endtime or (isnull(eventTime) and TimeGenerated <= endtime))\n | where array_length(emailfromaddress_has_any) == 0 and array_length(emailrecipient_has_any) == 0\n and array_length(emailsubject_has_any) == 0 and array_length(emaildeliverylocation_in) == 0\n | where array_length(emaildirection_in) == 0 or \"Unknown\" in (emaildirection_in)\n | where isnotempty(tostring(sm.qid)) and (bag_has_key(sm, \"from\") or array_length(sm[\"to\"]) > 0)\n | extend\n IsDelivery = coalesce(array_length(sm[\"to\"]), 0) > 0,\n Dsn = tostring(sm.dsn)\n | extend EventResult = case(Dsn startswith \"2.\", \"Success\", Dsn startswith \"4.\" or Dsn startswith \"5.\", \"Failure\", \"NA\")\n | where eventresult == \"*\" or EventResult == eventresult\n | extend\n Relay = tostring(sm.relay),\n Dvc = coalesce(tostring(pps.agent), tostring(metadata.origin.data.agent)),\n EventStartTime = coalesce(eventTime, TimeGenerated),\n EmailMailFromAddress = trim(@\"[<>\\s]\", tostring(sm[\"from\"])),\n EmailSizeBytes = tolong(sm.sizeBytes),\n DeliveryStatus = tostring(sm.stat)\n | parse Relay with RelayHost \" [\" RelayIp \"]\" *\n | extend\n EmailMailFromAddress = iff(EmailMailFromAddress contains \"@\", EmailMailFromAddress, \"\"),\n RelayHost = trim_end(@\"\\.\", coalesce(RelayHost, Relay)),\n RelayIp = iff(isnotnull(parse_ipv4(RelayIp)) or isnotempty(parse_ipv6(RelayIp)), RelayIp, \"\"),\n EmailProtocolName = case(tostring(sm.proto) =~ \"SMTP\" or tostring(sm.mailer) in~ (\"smtp\", \"esmtp\"), \"SMTP\", \"\"),\n EventSubType = iff(IsDelivery, \"EmailSent\", \"EmailReceived\")\n | extend\n SrcIpAddr = iff(IsDelivery, \"\", RelayIp),\n DstIpAddr = iff(IsDelivery, RelayIp, \"\"),\n Src = iff(IsDelivery, Dvc, coalesce(RelayIp, RelayHost)),\n Dst = iff(IsDelivery, coalesce(RelayIp, RelayHost), Dvc),\n SrcFQDN = iff(not(IsDelivery) and RelayHost contains \".\", RelayHost, \"\"),\n SrcHostname = iff(IsDelivery, tostring(split(Dvc, \".\")[0]), tostring(split(RelayHost, \".\")[0])),\n SrcDomain = iff(IsDelivery, substring(Dvc, indexof(Dvc, \".\") + 1), substring(RelayHost, indexof(RelayHost, \".\") + 1)),\n DvcHostname = tostring(split(Dvc, \".\")[0]),\n DvcDomain = iff(Dvc contains \".\", substring(Dvc, indexof(Dvc, \".\") + 1), \"\"),\n DvcFQDN = iff(Dvc contains \".\", Dvc, \"\"),\n EmailMailFromDomain = iff(EmailMailFromAddress contains \"@\", tostring(split(EmailMailFromAddress, \"@\")[1]), \"\"),\n DvcAction = iff(IsDelivery and EventResult == \"Success\" and DeliveryStatus startswith \"Sent\", \"Delivered\", \"NA\")\n | where array_length(ipaddr_has_any_prefix) == 0\n or has_any_ipv4_prefix(SrcIpAddr, ipaddr_has_any_prefix)\n or has_any_ipv4_prefix(DstIpAddr, ipaddr_has_any_prefix)\n | extend\n SrcFQDN = iff(IsDelivery, DvcFQDN, SrcFQDN),\n SrcDomain = iff(iff(IsDelivery, Dvc, RelayHost) contains \".\", SrcDomain, \"\"),\n DvcDomainType = iff(isnotempty(DvcDomain), \"FQDN\", \"\"),\n EventEndTime = EventStartTime,\n EventCount = int(1),\n EventVendor = \"Proofpoint\",\n EventProduct = \"POD MailLog\",\n EventSchema = \"EmailEvent\",\n EventSchemaVersion = \"1.0.0\",\n EventType = \"EmailDelivery\",\n EventOriginalType = \"maillog\",\n EmailDirection = \"Unknown\",\n EventOriginalResultDetails = strcat(Dsn, iff(isnotempty(Dsn) and isnotempty(DeliveryStatus), \"; \", \"\"), DeliveryStatus),\n EventUid = iff(isnotempty(id), strcat(\"ProofpointPODMailLog_CL:\", id), \"\"),\n DvcEntityKey = \"\",\n AdditionalEntities = dynamic([]),\n AdditionalFields = iff(pack, bag_pack(\"sm\", sm, \"tls\", tls, \"pps\", pps, \"metadata\", metadata), dynamic({})),\n Type = \"ProofpointPODMailLog_CL\"\n | extend SrcDomainType = iff(isnotempty(SrcDomain), \"FQDN\", \"\"), IpAddr = SrcIpAddr\n | project-rename EventOriginalUid = id, EventMessage = data\n | project\n TimeGenerated, Type, EventStartTime, EventEndTime, EventCount,\n EventVendor, EventProduct, EventSchema, EventSchemaVersion,\n EventType, EventSubType, EventOriginalType, EventResult,\n EventOriginalResultDetails, EventOriginalUid, EventUid, EventMessage,\n Dvc, DvcHostname, DvcDomain, DvcDomainType, DvcFQDN, DvcAction, DvcEntityKey,\n Src, SrcIpAddr, SrcHostname, SrcDomain, SrcDomainType, SrcFQDN, Dst, DstIpAddr, IpAddr,\n EmailMailFromAddress, EmailMailFromDomain, EmailSizeBytes, EmailProtocolName, EmailDirection,\n AdditionalFields, AdditionalEntities\n};\nparser(starttime=starttime, endtime=endtime, ipaddr_has_any_prefix=ipaddr_has_any_prefix,\n emailfromaddress_has_any=emailfromaddress_has_any, emailrecipient_has_any=emailrecipient_has_any,\n emailsubject_has_any=emailsubject_has_any, emaildirection_in=emaildirection_in,\n emaildeliverylocation_in=emaildeliverylocation_in, eventresult=eventresult, disabled=disabled, pack=pack)\n", + "query": "let parser = (\n starttime:datetime=datetime(null),\n endtime:datetime=datetime(null),\n ipaddr_has_any_prefix:dynamic=dynamic([]),\n emailfromaddress_has_any:dynamic=dynamic([]),\n emailrecipient_has_any:dynamic=dynamic([]),\n emailsubject_has_any:dynamic=dynamic([]),\n emaildirection_in:dynamic=dynamic([]),\n emaildeliverylocation_in:dynamic=dynamic([]),\n eventresult:string='*',\n disabled:bool=false,\n pack:bool=false)\n{\n ProofpointPODMailLog_CL\n | where not(disabled)\n | where (isnull(starttime) or eventTime >= starttime or (isnull(eventTime) and TimeGenerated >= starttime))\n and (isnull(endtime) or eventTime <= endtime or (isnull(eventTime) and TimeGenerated <= endtime))\n | where array_length(emailfromaddress_has_any) == 0 and array_length(emailrecipient_has_any) == 0\n and array_length(emailsubject_has_any) == 0 and array_length(emaildeliverylocation_in) == 0\n | where array_length(emaildirection_in) == 0 or \"Unknown\" in (emaildirection_in)\n | where isnotempty(tostring(sm.qid)) and (bag_has_key(sm, \"from\") or array_length(sm[\"to\"]) > 0)\n | extend\n IsDelivery = coalesce(array_length(sm[\"to\"]), 0) > 0,\n Dsn = tostring(sm.dsn)\n | extend EventResult = case(Dsn startswith \"2.\", \"Success\", Dsn startswith \"4.\" or Dsn startswith \"5.\", \"Failure\", \"NA\")\n | where eventresult == \"*\" or EventResult == eventresult\n | extend\n Relay = tostring(sm.relay),\n Dvc = coalesce(tostring(pps.agent), tostring(metadata.origin.data.agent)),\n EventStartTime = coalesce(eventTime, TimeGenerated),\n EmailMailFromAddress = trim(@\"[<>\\s]\", tostring(sm[\"from\"])),\n EmailSizeBytes = tolong(sm.sizeBytes),\n DeliveryStatus = tostring(sm.stat)\n | parse Relay with RelayHost \" [\" RelayIp \"]\" *\n | extend\n EmailMailFromAddress = iff(EmailMailFromAddress contains \"@\", EmailMailFromAddress, \"\"),\n RelayHost = trim_end(@\"\\.\", coalesce(RelayHost, Relay)),\n RelayIp = iff(isnotnull(parse_ipv4(RelayIp)) or isnotempty(parse_ipv6(RelayIp)), RelayIp, \"\"),\n EmailProtocolName = case(tostring(sm.proto) =~ \"SMTP\" or tostring(sm.mailer) in~ (\"smtp\", \"esmtp\"), \"SMTP\", \"\"),\n EventSubType = iff(IsDelivery, \"EmailSent\", \"EmailReceived\")\n | extend\n SrcIpAddr = iff(IsDelivery, \"\", RelayIp),\n DstIpAddr = iff(IsDelivery, RelayIp, \"\"),\n Src = iff(IsDelivery, Dvc, coalesce(RelayIp, RelayHost)),\n Dst = iff(IsDelivery, coalesce(RelayIp, RelayHost), Dvc),\n SrcFQDN = iff(not(IsDelivery) and RelayHost contains \".\", RelayHost, \"\"),\n SrcHostname = iff(IsDelivery, tostring(split(Dvc, \".\")[0]), tostring(split(RelayHost, \".\")[0])),\n SrcDomain = iff(IsDelivery, substring(Dvc, indexof(Dvc, \".\") + 1), substring(RelayHost, indexof(RelayHost, \".\") + 1)),\n DvcHostname = tostring(split(Dvc, \".\")[0]),\n DvcDomain = iff(Dvc contains \".\", substring(Dvc, indexof(Dvc, \".\") + 1), \"\"),\n DvcFQDN = iff(Dvc contains \".\", Dvc, \"\"),\n EmailMailFromDomain = iff(EmailMailFromAddress contains \"@\", tostring(split(EmailMailFromAddress, \"@\")[1]), \"\"),\n DvcAction = iff(IsDelivery and EventResult == \"Success\" and DeliveryStatus startswith \"Sent\", \"Delivered\", \"NA\")\n | where array_length(ipaddr_has_any_prefix) == 0\n or has_any_ipv4_prefix(SrcIpAddr, ipaddr_has_any_prefix)\n or has_any_ipv4_prefix(DstIpAddr, ipaddr_has_any_prefix)\n | extend\n SrcFQDN = iff(IsDelivery, DvcFQDN, SrcFQDN),\n SrcDomain = iff(iff(IsDelivery, Dvc, RelayHost) contains \".\", SrcDomain, \"\"),\n DvcDomainType = iff(isnotempty(DvcDomain), \"FQDN\", \"\"),\n EventEndTime = EventStartTime,\n EventCount = int(1),\n EventVendor = \"Proofpoint\",\n EventProduct = \"POD MailLog\",\n EventSchema = \"EmailEvent\",\n EventSchemaVersion = \"1.0.0\",\n EventType = \"EmailDelivery\",\n EventOriginalType = \"maillog\",\n EmailDirection = \"Unknown\",\n EventOriginalResultDetails = strcat(Dsn, iff(isnotempty(Dsn) and isnotempty(DeliveryStatus), \"; \", \"\"), DeliveryStatus),\n EventOriginalUid = tostring(column_ifexists(\"id\", \"\")),\n DvcEntityKey = \"\",\n AdditionalEntities = dynamic([]),\n AdditionalFields = iff(pack, bag_pack(\"sm\", sm, \"tls\", tls, \"pps\", pps, \"metadata\", metadata), dynamic({})),\n Type = \"ProofpointPODMailLog_CL\"\n | extend\n EventUid = iff(isnotempty(EventOriginalUid), strcat(\"ProofpointPODMailLog_CL:\", EventOriginalUid), \"\"),\n SrcDomainType = iff(isnotempty(SrcDomain), \"FQDN\", \"\"), IpAddr = SrcIpAddr\n | project-rename EventMessage = data\n | project\n TimeGenerated, Type, EventStartTime, EventEndTime, EventCount,\n EventVendor, EventProduct, EventSchema, EventSchemaVersion,\n EventType, EventSubType, EventOriginalType, EventResult,\n EventOriginalResultDetails, EventOriginalUid, EventUid, EventMessage,\n Dvc, DvcHostname, DvcDomain, DvcDomainType, DvcFQDN, DvcAction, DvcEntityKey,\n Src, SrcIpAddr, SrcHostname, SrcDomain, SrcDomainType, SrcFQDN, Dst, DstIpAddr, IpAddr,\n EmailMailFromAddress, EmailMailFromDomain, EmailSizeBytes, EmailProtocolName, EmailDirection,\n AdditionalFields, AdditionalEntities\n};\nparser(starttime=starttime, endtime=endtime, ipaddr_has_any_prefix=ipaddr_has_any_prefix,\n emailfromaddress_has_any=emailfromaddress_has_any, emailrecipient_has_any=emailrecipient_has_any,\n emailsubject_has_any=emailsubject_has_any, emaildirection_in=emaildirection_in,\n emaildeliverylocation_in=emaildeliverylocation_in, eventresult=eventresult, disabled=disabled, pack=pack)\n", "version": 1, "functionParameters": "starttime:datetime=datetime(null),endtime:datetime=datetime(null),ipaddr_has_any_prefix:dynamic=dynamic([]),emailfromaddress_has_any:dynamic=dynamic([]),emailrecipient_has_any:dynamic=dynamic([]),emailsubject_has_any:dynamic=dynamic([]),emaildirection_in:dynamic=dynamic([]),emaildeliverylocation_in:dynamic=dynamic([]),eventresult:string='*',disabled:bool=False,pack:bool=False" } From 233f62ff58622253bb55b7da2cc58d8781f480af Mon Sep 17 00:00:00 2001 From: adbhogeGit Date: Fri, 23 Oct 2026 23:43:07 -0600 Subject: [PATCH 07/10] Refactor RelayIp handling and extend email parsing Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- .../Parsers/vimEmailEventProofpointPODMailLog.yaml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/Parsers/ASimEmailEvent/Parsers/vimEmailEventProofpointPODMailLog.yaml b/Parsers/ASimEmailEvent/Parsers/vimEmailEventProofpointPODMailLog.yaml index 929457c6597..5230cc5b7c0 100644 --- a/Parsers/ASimEmailEvent/Parsers/vimEmailEventProofpointPODMailLog.yaml +++ b/Parsers/ASimEmailEvent/Parsers/vimEmailEventProofpointPODMailLog.yaml @@ -90,10 +90,12 @@ ParserQuery: | EmailSizeBytes = tolong(sm.sizeBytes), DeliveryStatus = tostring(sm.stat) | parse Relay with RelayHost " [" RelayIp "]" * + | extend RelayIp = iff(isnotnull(parse_ipv4(RelayIp)) or isnotempty(parse_ipv6(RelayIp)), RelayIp, "") + | where array_length(ipaddr_has_any_prefix) == 0 + or has_any_ipv4_prefix(RelayIp, ipaddr_has_any_prefix) | extend EmailMailFromAddress = iff(EmailMailFromAddress contains "@", EmailMailFromAddress, ""), RelayHost = trim_end(@"\.", coalesce(RelayHost, Relay)), - RelayIp = iff(isnotnull(parse_ipv4(RelayIp)) or isnotempty(parse_ipv6(RelayIp)), RelayIp, ""), EmailProtocolName = case(tostring(sm.proto) =~ "SMTP" or tostring(sm.mailer) in~ ("smtp", "esmtp"), "SMTP", ""), EventSubType = iff(IsDelivery, "EmailSent", "EmailReceived") | extend @@ -109,9 +111,6 @@ ParserQuery: | DvcFQDN = iff(Dvc contains ".", Dvc, ""), EmailMailFromDomain = iff(EmailMailFromAddress contains "@", tostring(split(EmailMailFromAddress, "@")[1]), ""), DvcAction = iff(IsDelivery and EventResult == "Success" and DeliveryStatus startswith "Sent", "Delivered", "NA") - | where array_length(ipaddr_has_any_prefix) == 0 - or has_any_ipv4_prefix(SrcIpAddr, ipaddr_has_any_prefix) - or has_any_ipv4_prefix(DstIpAddr, ipaddr_has_any_prefix) | extend SrcFQDN = iff(IsDelivery, DvcFQDN, SrcFQDN), SrcDomain = iff(iff(IsDelivery, Dvc, RelayHost) contains ".", SrcDomain, ""), From a88bfb0ebec305be24fc9d154d62cb1e39363df9 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <> Date: Sat, 24 Oct 2026 05:47:01 +0000 Subject: [PATCH 08/10] [ASIM Parsers] Generate deployable ARM templates from KQL function YAML files. --- .../vimEmailEventProofpointPODMailLog.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Parsers/ASimEmailEvent/ARM/vimEmailEventProofpointPODMailLog/vimEmailEventProofpointPODMailLog.json b/Parsers/ASimEmailEvent/ARM/vimEmailEventProofpointPODMailLog/vimEmailEventProofpointPODMailLog.json index 1a0d89b9759..ae193859940 100644 --- a/Parsers/ASimEmailEvent/ARM/vimEmailEventProofpointPODMailLog/vimEmailEventProofpointPODMailLog.json +++ b/Parsers/ASimEmailEvent/ARM/vimEmailEventProofpointPODMailLog/vimEmailEventProofpointPODMailLog.json @@ -27,7 +27,7 @@ "displayName": "Email Event ASIM filtering parser for Proofpoint POD MailLog", "category": "ASIM", "FunctionAlias": "vimEmailEventProofpointPODMailLog", - "query": "let parser = (\n starttime:datetime=datetime(null),\n endtime:datetime=datetime(null),\n ipaddr_has_any_prefix:dynamic=dynamic([]),\n emailfromaddress_has_any:dynamic=dynamic([]),\n emailrecipient_has_any:dynamic=dynamic([]),\n emailsubject_has_any:dynamic=dynamic([]),\n emaildirection_in:dynamic=dynamic([]),\n emaildeliverylocation_in:dynamic=dynamic([]),\n eventresult:string='*',\n disabled:bool=false,\n pack:bool=false)\n{\n ProofpointPODMailLog_CL\n | where not(disabled)\n | where (isnull(starttime) or eventTime >= starttime or (isnull(eventTime) and TimeGenerated >= starttime))\n and (isnull(endtime) or eventTime <= endtime or (isnull(eventTime) and TimeGenerated <= endtime))\n | where array_length(emailfromaddress_has_any) == 0 and array_length(emailrecipient_has_any) == 0\n and array_length(emailsubject_has_any) == 0 and array_length(emaildeliverylocation_in) == 0\n | where array_length(emaildirection_in) == 0 or \"Unknown\" in (emaildirection_in)\n | where isnotempty(tostring(sm.qid)) and (bag_has_key(sm, \"from\") or array_length(sm[\"to\"]) > 0)\n | extend\n IsDelivery = coalesce(array_length(sm[\"to\"]), 0) > 0,\n Dsn = tostring(sm.dsn)\n | extend EventResult = case(Dsn startswith \"2.\", \"Success\", Dsn startswith \"4.\" or Dsn startswith \"5.\", \"Failure\", \"NA\")\n | where eventresult == \"*\" or EventResult == eventresult\n | extend\n Relay = tostring(sm.relay),\n Dvc = coalesce(tostring(pps.agent), tostring(metadata.origin.data.agent)),\n EventStartTime = coalesce(eventTime, TimeGenerated),\n EmailMailFromAddress = trim(@\"[<>\\s]\", tostring(sm[\"from\"])),\n EmailSizeBytes = tolong(sm.sizeBytes),\n DeliveryStatus = tostring(sm.stat)\n | parse Relay with RelayHost \" [\" RelayIp \"]\" *\n | extend\n EmailMailFromAddress = iff(EmailMailFromAddress contains \"@\", EmailMailFromAddress, \"\"),\n RelayHost = trim_end(@\"\\.\", coalesce(RelayHost, Relay)),\n RelayIp = iff(isnotnull(parse_ipv4(RelayIp)) or isnotempty(parse_ipv6(RelayIp)), RelayIp, \"\"),\n EmailProtocolName = case(tostring(sm.proto) =~ \"SMTP\" or tostring(sm.mailer) in~ (\"smtp\", \"esmtp\"), \"SMTP\", \"\"),\n EventSubType = iff(IsDelivery, \"EmailSent\", \"EmailReceived\")\n | extend\n SrcIpAddr = iff(IsDelivery, \"\", RelayIp),\n DstIpAddr = iff(IsDelivery, RelayIp, \"\"),\n Src = iff(IsDelivery, Dvc, coalesce(RelayIp, RelayHost)),\n Dst = iff(IsDelivery, coalesce(RelayIp, RelayHost), Dvc),\n SrcFQDN = iff(not(IsDelivery) and RelayHost contains \".\", RelayHost, \"\"),\n SrcHostname = iff(IsDelivery, tostring(split(Dvc, \".\")[0]), tostring(split(RelayHost, \".\")[0])),\n SrcDomain = iff(IsDelivery, substring(Dvc, indexof(Dvc, \".\") + 1), substring(RelayHost, indexof(RelayHost, \".\") + 1)),\n DvcHostname = tostring(split(Dvc, \".\")[0]),\n DvcDomain = iff(Dvc contains \".\", substring(Dvc, indexof(Dvc, \".\") + 1), \"\"),\n DvcFQDN = iff(Dvc contains \".\", Dvc, \"\"),\n EmailMailFromDomain = iff(EmailMailFromAddress contains \"@\", tostring(split(EmailMailFromAddress, \"@\")[1]), \"\"),\n DvcAction = iff(IsDelivery and EventResult == \"Success\" and DeliveryStatus startswith \"Sent\", \"Delivered\", \"NA\")\n | where array_length(ipaddr_has_any_prefix) == 0\n or has_any_ipv4_prefix(SrcIpAddr, ipaddr_has_any_prefix)\n or has_any_ipv4_prefix(DstIpAddr, ipaddr_has_any_prefix)\n | extend\n SrcFQDN = iff(IsDelivery, DvcFQDN, SrcFQDN),\n SrcDomain = iff(iff(IsDelivery, Dvc, RelayHost) contains \".\", SrcDomain, \"\"),\n DvcDomainType = iff(isnotempty(DvcDomain), \"FQDN\", \"\"),\n EventEndTime = EventStartTime,\n EventCount = int(1),\n EventVendor = \"Proofpoint\",\n EventProduct = \"POD MailLog\",\n EventSchema = \"EmailEvent\",\n EventSchemaVersion = \"1.0.0\",\n EventType = \"EmailDelivery\",\n EventOriginalType = \"maillog\",\n EmailDirection = \"Unknown\",\n EventOriginalResultDetails = strcat(Dsn, iff(isnotempty(Dsn) and isnotempty(DeliveryStatus), \"; \", \"\"), DeliveryStatus),\n EventOriginalUid = tostring(column_ifexists(\"id\", \"\")),\n DvcEntityKey = \"\",\n AdditionalEntities = dynamic([]),\n AdditionalFields = iff(pack, bag_pack(\"sm\", sm, \"tls\", tls, \"pps\", pps, \"metadata\", metadata), dynamic({})),\n Type = \"ProofpointPODMailLog_CL\"\n | extend\n EventUid = iff(isnotempty(EventOriginalUid), strcat(\"ProofpointPODMailLog_CL:\", EventOriginalUid), \"\"),\n SrcDomainType = iff(isnotempty(SrcDomain), \"FQDN\", \"\"), IpAddr = SrcIpAddr\n | project-rename EventMessage = data\n | project\n TimeGenerated, Type, EventStartTime, EventEndTime, EventCount,\n EventVendor, EventProduct, EventSchema, EventSchemaVersion,\n EventType, EventSubType, EventOriginalType, EventResult,\n EventOriginalResultDetails, EventOriginalUid, EventUid, EventMessage,\n Dvc, DvcHostname, DvcDomain, DvcDomainType, DvcFQDN, DvcAction, DvcEntityKey,\n Src, SrcIpAddr, SrcHostname, SrcDomain, SrcDomainType, SrcFQDN, Dst, DstIpAddr, IpAddr,\n EmailMailFromAddress, EmailMailFromDomain, EmailSizeBytes, EmailProtocolName, EmailDirection,\n AdditionalFields, AdditionalEntities\n};\nparser(starttime=starttime, endtime=endtime, ipaddr_has_any_prefix=ipaddr_has_any_prefix,\n emailfromaddress_has_any=emailfromaddress_has_any, emailrecipient_has_any=emailrecipient_has_any,\n emailsubject_has_any=emailsubject_has_any, emaildirection_in=emaildirection_in,\n emaildeliverylocation_in=emaildeliverylocation_in, eventresult=eventresult, disabled=disabled, pack=pack)\n", + "query": "let parser = (\n starttime:datetime=datetime(null),\n endtime:datetime=datetime(null),\n ipaddr_has_any_prefix:dynamic=dynamic([]),\n emailfromaddress_has_any:dynamic=dynamic([]),\n emailrecipient_has_any:dynamic=dynamic([]),\n emailsubject_has_any:dynamic=dynamic([]),\n emaildirection_in:dynamic=dynamic([]),\n emaildeliverylocation_in:dynamic=dynamic([]),\n eventresult:string='*',\n disabled:bool=false,\n pack:bool=false)\n{\n ProofpointPODMailLog_CL\n | where not(disabled)\n | where (isnull(starttime) or eventTime >= starttime or (isnull(eventTime) and TimeGenerated >= starttime))\n and (isnull(endtime) or eventTime <= endtime or (isnull(eventTime) and TimeGenerated <= endtime))\n | where array_length(emailfromaddress_has_any) == 0 and array_length(emailrecipient_has_any) == 0\n and array_length(emailsubject_has_any) == 0 and array_length(emaildeliverylocation_in) == 0\n | where array_length(emaildirection_in) == 0 or \"Unknown\" in (emaildirection_in)\n | where isnotempty(tostring(sm.qid)) and (bag_has_key(sm, \"from\") or array_length(sm[\"to\"]) > 0)\n | extend\n IsDelivery = coalesce(array_length(sm[\"to\"]), 0) > 0,\n Dsn = tostring(sm.dsn)\n | extend EventResult = case(Dsn startswith \"2.\", \"Success\", Dsn startswith \"4.\" or Dsn startswith \"5.\", \"Failure\", \"NA\")\n | where eventresult == \"*\" or EventResult == eventresult\n | extend\n Relay = tostring(sm.relay),\n Dvc = coalesce(tostring(pps.agent), tostring(metadata.origin.data.agent)),\n EventStartTime = coalesce(eventTime, TimeGenerated),\n EmailMailFromAddress = trim(@\"[<>\\s]\", tostring(sm[\"from\"])),\n EmailSizeBytes = tolong(sm.sizeBytes),\n DeliveryStatus = tostring(sm.stat)\n | parse Relay with RelayHost \" [\" RelayIp \"]\" *\n | extend RelayIp = iff(isnotnull(parse_ipv4(RelayIp)) or isnotempty(parse_ipv6(RelayIp)), RelayIp, \"\")\n | where array_length(ipaddr_has_any_prefix) == 0\n or has_any_ipv4_prefix(RelayIp, ipaddr_has_any_prefix)\n | extend\n EmailMailFromAddress = iff(EmailMailFromAddress contains \"@\", EmailMailFromAddress, \"\"),\n RelayHost = trim_end(@\"\\.\", coalesce(RelayHost, Relay)),\n EmailProtocolName = case(tostring(sm.proto) =~ \"SMTP\" or tostring(sm.mailer) in~ (\"smtp\", \"esmtp\"), \"SMTP\", \"\"),\n EventSubType = iff(IsDelivery, \"EmailSent\", \"EmailReceived\")\n | extend\n SrcIpAddr = iff(IsDelivery, \"\", RelayIp),\n DstIpAddr = iff(IsDelivery, RelayIp, \"\"),\n Src = iff(IsDelivery, Dvc, coalesce(RelayIp, RelayHost)),\n Dst = iff(IsDelivery, coalesce(RelayIp, RelayHost), Dvc),\n SrcFQDN = iff(not(IsDelivery) and RelayHost contains \".\", RelayHost, \"\"),\n SrcHostname = iff(IsDelivery, tostring(split(Dvc, \".\")[0]), tostring(split(RelayHost, \".\")[0])),\n SrcDomain = iff(IsDelivery, substring(Dvc, indexof(Dvc, \".\") + 1), substring(RelayHost, indexof(RelayHost, \".\") + 1)),\n DvcHostname = tostring(split(Dvc, \".\")[0]),\n DvcDomain = iff(Dvc contains \".\", substring(Dvc, indexof(Dvc, \".\") + 1), \"\"),\n DvcFQDN = iff(Dvc contains \".\", Dvc, \"\"),\n EmailMailFromDomain = iff(EmailMailFromAddress contains \"@\", tostring(split(EmailMailFromAddress, \"@\")[1]), \"\"),\n DvcAction = iff(IsDelivery and EventResult == \"Success\" and DeliveryStatus startswith \"Sent\", \"Delivered\", \"NA\")\n | extend\n SrcFQDN = iff(IsDelivery, DvcFQDN, SrcFQDN),\n SrcDomain = iff(iff(IsDelivery, Dvc, RelayHost) contains \".\", SrcDomain, \"\"),\n DvcDomainType = iff(isnotempty(DvcDomain), \"FQDN\", \"\"),\n EventEndTime = EventStartTime,\n EventCount = int(1),\n EventVendor = \"Proofpoint\",\n EventProduct = \"POD MailLog\",\n EventSchema = \"EmailEvent\",\n EventSchemaVersion = \"1.0.0\",\n EventType = \"EmailDelivery\",\n EventOriginalType = \"maillog\",\n EmailDirection = \"Unknown\",\n EventOriginalResultDetails = strcat(Dsn, iff(isnotempty(Dsn) and isnotempty(DeliveryStatus), \"; \", \"\"), DeliveryStatus),\n EventOriginalUid = tostring(column_ifexists(\"id\", \"\")),\n DvcEntityKey = \"\",\n AdditionalEntities = dynamic([]),\n AdditionalFields = iff(pack, bag_pack(\"sm\", sm, \"tls\", tls, \"pps\", pps, \"metadata\", metadata), dynamic({})),\n Type = \"ProofpointPODMailLog_CL\"\n | extend\n EventUid = iff(isnotempty(EventOriginalUid), strcat(\"ProofpointPODMailLog_CL:\", EventOriginalUid), \"\"),\n SrcDomainType = iff(isnotempty(SrcDomain), \"FQDN\", \"\"), IpAddr = SrcIpAddr\n | project-rename EventMessage = data\n | project\n TimeGenerated, Type, EventStartTime, EventEndTime, EventCount,\n EventVendor, EventProduct, EventSchema, EventSchemaVersion,\n EventType, EventSubType, EventOriginalType, EventResult,\n EventOriginalResultDetails, EventOriginalUid, EventUid, EventMessage,\n Dvc, DvcHostname, DvcDomain, DvcDomainType, DvcFQDN, DvcAction, DvcEntityKey,\n Src, SrcIpAddr, SrcHostname, SrcDomain, SrcDomainType, SrcFQDN, Dst, DstIpAddr, IpAddr,\n EmailMailFromAddress, EmailMailFromDomain, EmailSizeBytes, EmailProtocolName, EmailDirection,\n AdditionalFields, AdditionalEntities\n};\nparser(starttime=starttime, endtime=endtime, ipaddr_has_any_prefix=ipaddr_has_any_prefix,\n emailfromaddress_has_any=emailfromaddress_has_any, emailrecipient_has_any=emailrecipient_has_any,\n emailsubject_has_any=emailsubject_has_any, emaildirection_in=emaildirection_in,\n emaildeliverylocation_in=emaildeliverylocation_in, eventresult=eventresult, disabled=disabled, pack=pack)\n", "version": 1, "functionParameters": "starttime:datetime=datetime(null),endtime:datetime=datetime(null),ipaddr_has_any_prefix:dynamic=dynamic([]),emailfromaddress_has_any:dynamic=dynamic([]),emailrecipient_has_any:dynamic=dynamic([]),emailsubject_has_any:dynamic=dynamic([]),emaildirection_in:dynamic=dynamic([]),emaildeliverylocation_in:dynamic=dynamic([]),eventresult:string='*',disabled:bool=False,pack:bool=False" } From 96dcd18ca32dc4f8f2a2412d9926a0529267015a Mon Sep 17 00:00:00 2001 From: Aditi Bhoge Date: Fri, 23 Oct 2026 23:07:55 -0700 Subject: [PATCH 09/10] Fix Proofpoint MailLog local endpoint normalization Normalize mailbox-form relays to their host component and use the reporting device for relay-less local delivery. Add synthetic regression fixture rows. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 321517f1-a1c5-4242-be85-2d144a9cc295 --- .../CHANGELOG/ASimEmailEventProofpointPODMailLog.md | 5 +++++ .../CHANGELOG/vimEmailEventProofpointPODMailLog.md | 5 +++++ .../Parsers/ASimEmailEventProofpointPODMailLog.yaml | 8 ++++---- .../Parsers/vimEmailEventProofpointPODMailLog.yaml | 8 ++++---- Parsers/ASimEmailEvent/README.md | 12 +++++++++++- ...roofpoint_POD MailLog_EmailEvent_IngestedLogs.csv | 3 +++ 6 files changed, 32 insertions(+), 9 deletions(-) diff --git a/Parsers/ASimEmailEvent/CHANGELOG/ASimEmailEventProofpointPODMailLog.md b/Parsers/ASimEmailEvent/CHANGELOG/ASimEmailEventProofpointPODMailLog.md index 74b0c651f42..f835d1672e4 100644 --- a/Parsers/ASimEmailEvent/CHANGELOG/ASimEmailEventProofpointPODMailLog.md +++ b/Parsers/ASimEmailEvent/CHANGELOG/ASimEmailEventProofpointPODMailLog.md @@ -1,5 +1,10 @@ # Changelog for ASimEmailEventProofpointPODMailLog.yaml +## Version 0.1.2 + +- (2026-10-24) Normalize mailbox-form relay hosts and identify relay-less local delivery destinations - [PR #15410](https://github.com/Azure/Azure-Sentinel/pull/15410) +- Extract the host from relay values such as `pps@localhost`; use the reporting device as destination only when `sm.mailer=local` and no relay endpoint is available. + ## Version 0.1.1 - (2026-10-23) Handle missing optional source IDs in CI-ingested records while preserving connector-provided IDs - [PR #15410](https://github.com/Azure/Azure-Sentinel/pull/15410) diff --git a/Parsers/ASimEmailEvent/CHANGELOG/vimEmailEventProofpointPODMailLog.md b/Parsers/ASimEmailEvent/CHANGELOG/vimEmailEventProofpointPODMailLog.md index 56c68ed2d32..e53b6248ad9 100644 --- a/Parsers/ASimEmailEvent/CHANGELOG/vimEmailEventProofpointPODMailLog.md +++ b/Parsers/ASimEmailEvent/CHANGELOG/vimEmailEventProofpointPODMailLog.md @@ -1,5 +1,10 @@ # Changelog for vimEmailEventProofpointPODMailLog.yaml +## Version 0.1.2 + +- (2026-10-24) Normalize mailbox-form relay hosts and identify relay-less local delivery destinations - [PR #15410](https://github.com/Azure/Azure-Sentinel/pull/15410) +- Preserve filtering while extracting the relay host and using the reporting device for local delivery without a relay endpoint. + ## Version 0.1.1 - (2026-10-23) Handle missing optional source IDs in CI-ingested records without changing filtering or inventing event identifiers - [PR #15410](https://github.com/Azure/Azure-Sentinel/pull/15410) diff --git a/Parsers/ASimEmailEvent/Parsers/ASimEmailEventProofpointPODMailLog.yaml b/Parsers/ASimEmailEvent/Parsers/ASimEmailEventProofpointPODMailLog.yaml index 002ad44a980..7ff102f4530 100644 --- a/Parsers/ASimEmailEvent/Parsers/ASimEmailEventProofpointPODMailLog.yaml +++ b/Parsers/ASimEmailEvent/Parsers/ASimEmailEventProofpointPODMailLog.yaml @@ -1,7 +1,7 @@ Parser: Title: Email Event ASIM parser for Proofpoint POD MailLog - Version: '0.1.1' - LastUpdated: Oct 23, 2026 + Version: '0.1.2' + LastUpdated: Oct 24, 2026 Product: Name: Proofpoint POD MailLog Normalization: @@ -44,7 +44,7 @@ ParserQuery: | | parse Relay with RelayHost " [" RelayIp "]" * | extend EmailMailFromAddress = iff(EmailMailFromAddress contains "@", EmailMailFromAddress, ""), - RelayHost = trim_end(@"\.", coalesce(RelayHost, Relay)), + RelayHost = trim_end(@"\.", tostring(split(coalesce(RelayHost, Relay), "@")[-1])), RelayIp = iff(isnotnull(parse_ipv4(RelayIp)) or isnotempty(parse_ipv6(RelayIp)), RelayIp, ""), EventResult = case(Dsn startswith "2.", "Success", Dsn startswith "4." or Dsn startswith "5.", "Failure", "NA"), EmailProtocolName = case(tostring(sm.proto) =~ "SMTP" or tostring(sm.mailer) in~ ("smtp", "esmtp"), "SMTP", ""), @@ -53,7 +53,7 @@ ParserQuery: | SrcIpAddr = iff(IsDelivery, "", RelayIp), DstIpAddr = iff(IsDelivery, RelayIp, ""), Src = iff(IsDelivery, Dvc, coalesce(RelayIp, RelayHost)), - Dst = iff(IsDelivery, coalesce(RelayIp, RelayHost), Dvc), + Dst = iff(IsDelivery, coalesce(RelayIp, RelayHost, iff(tostring(sm.mailer) =~ "local", Dvc, "")), Dvc), SrcFQDN = iff(not(IsDelivery) and RelayHost contains ".", RelayHost, ""), SrcHostname = iff(IsDelivery, tostring(split(Dvc, ".")[0]), tostring(split(RelayHost, ".")[0])), SrcDomain = iff(IsDelivery, substring(Dvc, indexof(Dvc, ".") + 1), substring(RelayHost, indexof(RelayHost, ".") + 1)), diff --git a/Parsers/ASimEmailEvent/Parsers/vimEmailEventProofpointPODMailLog.yaml b/Parsers/ASimEmailEvent/Parsers/vimEmailEventProofpointPODMailLog.yaml index 5230cc5b7c0..47a64521d83 100644 --- a/Parsers/ASimEmailEvent/Parsers/vimEmailEventProofpointPODMailLog.yaml +++ b/Parsers/ASimEmailEvent/Parsers/vimEmailEventProofpointPODMailLog.yaml @@ -1,7 +1,7 @@ Parser: Title: Email Event ASIM filtering parser for Proofpoint POD MailLog - Version: '0.1.1' - LastUpdated: Oct 23, 2026 + Version: '0.1.2' + LastUpdated: Oct 24, 2026 Product: Name: Proofpoint POD MailLog Normalization: @@ -95,14 +95,14 @@ ParserQuery: | or has_any_ipv4_prefix(RelayIp, ipaddr_has_any_prefix) | extend EmailMailFromAddress = iff(EmailMailFromAddress contains "@", EmailMailFromAddress, ""), - RelayHost = trim_end(@"\.", coalesce(RelayHost, Relay)), + RelayHost = trim_end(@"\.", tostring(split(coalesce(RelayHost, Relay), "@")[-1])), EmailProtocolName = case(tostring(sm.proto) =~ "SMTP" or tostring(sm.mailer) in~ ("smtp", "esmtp"), "SMTP", ""), EventSubType = iff(IsDelivery, "EmailSent", "EmailReceived") | extend SrcIpAddr = iff(IsDelivery, "", RelayIp), DstIpAddr = iff(IsDelivery, RelayIp, ""), Src = iff(IsDelivery, Dvc, coalesce(RelayIp, RelayHost)), - Dst = iff(IsDelivery, coalesce(RelayIp, RelayHost), Dvc), + Dst = iff(IsDelivery, coalesce(RelayIp, RelayHost, iff(tostring(sm.mailer) =~ "local", Dvc, "")), Dvc), SrcFQDN = iff(not(IsDelivery) and RelayHost contains ".", RelayHost, ""), SrcHostname = iff(IsDelivery, tostring(split(Dvc, ".")[0]), tostring(split(RelayHost, ".")[0])), SrcDomain = iff(IsDelivery, substring(Dvc, indexof(Dvc, ".") + 1), substring(RelayHost, indexof(RelayHost, ".") + 1)), diff --git a/Parsers/ASimEmailEvent/README.md b/Parsers/ASimEmailEvent/README.md index 8d0b5ba65f2..e4c59bdf962 100644 --- a/Parsers/ASimEmailEvent/README.md +++ b/Parsers/ASimEmailEvent/README.md @@ -31,7 +31,9 @@ and [DCR transform](../../Solutions/Proofpoint%20On%20demand%28POD%29%20Email%20 The [20 existing raw samples](../../Sample%20Data/Custom/ProofpointPOD_maillog_CL.json) provide the observed `sm` and device fields. The ASIM fixture `Sample Data/ASIM/Proofpoint_POD MailLog_EmailEvent_IngestedLogs.csv` is a -local transformation of those samples into the CCP shape, not a workspace export. +local transformation of those samples into the CCP shape, not a workspace export, +plus three explicitly synthetic local-endpoint regression rows based on the +user-provided relay shapes (23 rows total). Its timestamps use the source `ts` value for reproducibility; the real DCR sets `TimeGenerated` to ingestion time and `eventTime` to `todatetime(ts)`. @@ -45,6 +47,8 @@ sets `TimeGenerated` to ingestion time and `eventTime` to `todatetime(ts)`. | Presence of `sm.from` or nonempty `sm.to`, with `sm.qid` | Select receipt/delivery records; nonempty `sm.to` identifies `EmailSent`, otherwise `EmailReceived` | | `sm.relay` | Remote host/IP; source on receipt, destination on delivery | | Reporting device | Destination on receipt, source on delivery | +| Mailbox-form `sm.relay` (for example, `pps@localhost`) | Host portion (`localhost`), not the mailbox, in endpoint hostname fields | +| `sm.mailer=local` with no relay endpoint | Reporting device as `Dst`; no destination IP is inferred | | `sm.from` | `EmailMailFromAddress`, `EmailMailFromDomain` after removing surrounding whitespace/angle brackets | | `sm.sizeBytes` | `EmailSizeBytes` as a long | | `sm.proto`, `sm.mailer` | `EmailProtocolName` (`SMTP` for SMTP/esmtp) | @@ -56,6 +60,12 @@ sets `TimeGenerated` to ingestion time and `eventTime` to `todatetime(ts)`. The reporting device may be either endpoint of an SMTP hop; `EmailDirection` is therefore `Unknown`, not an inferred organizational inbound/outbound direction. Receipt rows do not contain a delivery result and return `EventResult=NA`. +User-provided validation samples also show local submissions with mailbox-form +relay values and local deliveries without a relay. The parser extracts the host +portion of a mailbox-form relay. Only the explicit `local` mailer permits a +missing delivery endpoint to fall back to the reporting device; missing remote +endpoints remain unmapped. Original relay values remain in `AdditionalFields.sm` +when packing is enabled. The parser does not repair the truncated sender present in one repository sample: sender strings without `@` are left unmapped, with the original in `AdditionalFields.sm` when packing is enabled. This is not full RFC mailbox validation. diff --git a/Sample Data/ASIM/Proofpoint_POD MailLog_EmailEvent_IngestedLogs.csv b/Sample Data/ASIM/Proofpoint_POD MailLog_EmailEvent_IngestedLogs.csv index 3036eb2b5fb..b503c4936ca 100644 --- a/Sample Data/ASIM/Proofpoint_POD MailLog_EmailEvent_IngestedLogs.csv +++ b/Sample Data/ASIM/Proofpoint_POD MailLog_EmailEvent_IngestedLogs.csv @@ -1,4 +1,7 @@ TimeGenerated,id,data,eventTime,metadata,pps,sm,tls,Type +2020-11-05T12:00:00Z,local-receipt-1,Synthetic mailbox-form relay receipt,2020-11-05T12:00:00Z,{},"{""agent"":""mail.example.test""}","{""qid"":""local-receipt-1"",""from"":""<>"",""relay"":""pps@localhost""}",{},ProofpointPODMailLog_CL +2020-11-05T12:00:01Z,local-receipt-2,Synthetic mailbox-form relay receipt,2020-11-05T12:00:01Z,{},"{""agent"":""mail.example.test""}","{""qid"":""local-receipt-2"",""from"":""<>"",""relay"":""support@localhost""}",{},ProofpointPODMailLog_CL +2020-11-05T12:00:02Z,local-delivery,Synthetic relay-less local delivery,2020-11-05T12:00:02Z,{},"{""agent"":""mail.example.test""}","{""qid"":""local-delivery"",""mailer"":""local"",""to"":[""""],""stat"":""Sent"",""dsn"":""2.0.0""}",{},ProofpointPODMailLog_CL 2020-11-05T12:59:43.881767+01:00,rGWIupM+g6R4PWIeqf6SMA,"2020-11-05T12:59:43.881767+01:00 t1234555 sendmail[18160]: 0A0BqJgv000000: to=, delay=00:00:00, xdelay=00:00:00, mailer=esmtp, tls_verify=OK, pri=154217, relay=relay.mail.net. [10.10.10.10], dsn=2.0.0, stat=Sent ( [InternalId=3444563792735, Hostname=eurprd01.prod.exchangelabs.com] 45616 bytes in 0.073, 610.105 KB/sec Queued mail for delivery)",2020-11-05T12:59:43.881767+01:00,"{""origin"":{""data"":{""agent"":""t1234555.ppops.net"",""cid"":""CID""}}}","{""cid"":""CID"",""agent"":""t1234555.popps.net""}","{""mailer"":""esmtp"",""xdelay"":""00:00:00"",""delay"":""00:00:00"",""tls"":{""verify"":""OK""},""dsn"":""2.0.0"",""to"":[""""],""relay"":""relay.mail.net [10.10.10.10]"",""stat"":""Sent ( [InternalId"",""pri"":""154217"",""qid"":""0A0BqJgv000000""}",null,ProofpointPODMailLog_CL 2020-11-05T12:59:44.487319+01:00,tnsgrBLZ9hdWbGRd3HMS9A,"2020-11-05T12:59:44.487319+01:00 t1234555 sendmail[18157]: 0A0BqJgv000000: to=, delay=00:00:03, xdelay=00:00:03, mailer=esmtp, tls_verify=OK, pri=2242435, relay=relay.mail.net. [10.10.10.10], dsn=2.0.0, stat=Sent ( [InternalId=18167711674865, Hostname=eurprd01.prod.exchangelabs.com] 2159936 bytes in 0.641, 3288.805 KB/sec Queued mail for delivery)",2020-11-05T12:59:44.487319+01:00,"{""origin"":{""data"":{""agent"":""t1234555.ppops.net"",""cid"":""CID""}}}","{""cid"":""CID"",""agent"":""t1234555.ppops.net""}","{""delay"":""00:00:03"",""dsn"":""2.0.0"",""tls"":{""verify"":""OK""},""xdelay"":""00:00:03"",""mailer"":""esmtp"",""qid"":""0A0BqJgv000000"",""pri"":""2242435"",""relay"":""relay.mail.net [10.10.10.10]"",""to"":[""""],""stat"":""Sent ( [InternalId""}",null,ProofpointPODMailLog_CL 2020-11-05T12:59:45.217049+01:00,Rv4coMMYHs/yWaGyMlR49g,"2020-11-05T12:59:45.217049+01:00 t1234555 sendmail[30248]: 0A5BqKs4030248: from=, size=8255, class=0, nrcpts=1, msgid=, proto=SMTP, daemon=MTA, tls_verify=NONE, auth=NONE, relay=t1234555.ppops.net [127.0.0.1]",2020-11-05T12:59:45.217049+01:00,"{""origin"":{""data"":{""agent"":""t1234555.ppops.net"",""cid"":""CID""}}}","{""agent"":""t1234555.ppops.net"",""cid"":""CID""}","{""from"":"""",""auth"":""NONE"",""class"":""0"",""proto"":""SMTP"",""daemon"":""MTA"",""qid"":""0A5BqKs4030248"",""sizeBytes"":""8255"",""nrcpts"":""1"",""relay"":""t1234555.ppops.net [127.0.0.1]""}",null,ProofpointPODMailLog_CL From 0c7129c9a73de44e015386cb6437df5f81b15c26 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <> Date: Sat, 24 Oct 2026 06:11:31 +0000 Subject: [PATCH 10/10] [ASIM Parsers] Generate deployable ARM templates from KQL function YAML files. --- .../ASimEmailEventProofpointPODMailLog.json | 2 +- .../vimEmailEventProofpointPODMailLog.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Parsers/ASimEmailEvent/ARM/ASimEmailEventProofpointPODMailLog/ASimEmailEventProofpointPODMailLog.json b/Parsers/ASimEmailEvent/ARM/ASimEmailEventProofpointPODMailLog/ASimEmailEventProofpointPODMailLog.json index 47bf4db2af6..d452fd7e49a 100644 --- a/Parsers/ASimEmailEvent/ARM/ASimEmailEventProofpointPODMailLog/ASimEmailEventProofpointPODMailLog.json +++ b/Parsers/ASimEmailEvent/ARM/ASimEmailEventProofpointPODMailLog/ASimEmailEventProofpointPODMailLog.json @@ -27,7 +27,7 @@ "displayName": "Email Event ASIM parser for Proofpoint POD MailLog", "category": "ASIM", "FunctionAlias": "ASimEmailEventProofpointPODMailLog", - "query": "let parser = (disabled:bool=false, pack:bool=false) {\n ProofpointPODMailLog_CL\n | where not(disabled)\n | where isnotempty(tostring(sm.qid)) and (bag_has_key(sm, \"from\") or array_length(sm[\"to\"]) > 0)\n | extend\n IsDelivery = coalesce(array_length(sm[\"to\"]), 0) > 0,\n Relay = tostring(sm.relay),\n Dvc = coalesce(tostring(pps.agent), tostring(metadata.origin.data.agent)),\n EventStartTime = coalesce(eventTime, TimeGenerated),\n EmailMailFromAddress = trim(@\"[<>\\s]\", tostring(sm[\"from\"])),\n EmailSizeBytes = tolong(sm.sizeBytes),\n Dsn = tostring(sm.dsn),\n DeliveryStatus = tostring(sm.stat)\n | parse Relay with RelayHost \" [\" RelayIp \"]\" *\n | extend\n EmailMailFromAddress = iff(EmailMailFromAddress contains \"@\", EmailMailFromAddress, \"\"),\n RelayHost = trim_end(@\"\\.\", coalesce(RelayHost, Relay)),\n RelayIp = iff(isnotnull(parse_ipv4(RelayIp)) or isnotempty(parse_ipv6(RelayIp)), RelayIp, \"\"),\n EventResult = case(Dsn startswith \"2.\", \"Success\", Dsn startswith \"4.\" or Dsn startswith \"5.\", \"Failure\", \"NA\"),\n EmailProtocolName = case(tostring(sm.proto) =~ \"SMTP\" or tostring(sm.mailer) in~ (\"smtp\", \"esmtp\"), \"SMTP\", \"\"),\n EventSubType = iff(IsDelivery, \"EmailSent\", \"EmailReceived\")\n | extend\n SrcIpAddr = iff(IsDelivery, \"\", RelayIp),\n DstIpAddr = iff(IsDelivery, RelayIp, \"\"),\n Src = iff(IsDelivery, Dvc, coalesce(RelayIp, RelayHost)),\n Dst = iff(IsDelivery, coalesce(RelayIp, RelayHost), Dvc),\n SrcFQDN = iff(not(IsDelivery) and RelayHost contains \".\", RelayHost, \"\"),\n SrcHostname = iff(IsDelivery, tostring(split(Dvc, \".\")[0]), tostring(split(RelayHost, \".\")[0])),\n SrcDomain = iff(IsDelivery, substring(Dvc, indexof(Dvc, \".\") + 1), substring(RelayHost, indexof(RelayHost, \".\") + 1)),\n DvcHostname = tostring(split(Dvc, \".\")[0]),\n DvcDomain = iff(Dvc contains \".\", substring(Dvc, indexof(Dvc, \".\") + 1), \"\"),\n DvcFQDN = iff(Dvc contains \".\", Dvc, \"\"),\n EmailMailFromDomain = iff(EmailMailFromAddress contains \"@\", tostring(split(EmailMailFromAddress, \"@\")[1]), \"\"),\n DvcAction = iff(IsDelivery and EventResult == \"Success\" and DeliveryStatus startswith \"Sent\", \"Delivered\", \"NA\")\n | extend\n SrcFQDN = iff(IsDelivery, DvcFQDN, SrcFQDN),\n SrcDomain = iff(iff(IsDelivery, Dvc, RelayHost) contains \".\", SrcDomain, \"\"),\n DvcDomainType = iff(isnotempty(DvcDomain), \"FQDN\", \"\"),\n EventEndTime = EventStartTime,\n EventCount = int(1),\n EventVendor = \"Proofpoint\",\n EventProduct = \"POD MailLog\",\n EventSchema = \"EmailEvent\",\n EventSchemaVersion = \"1.0.0\",\n EventType = \"EmailDelivery\",\n EventOriginalType = \"maillog\",\n EmailDirection = \"Unknown\",\n EventOriginalResultDetails = strcat(Dsn, iff(isnotempty(Dsn) and isnotempty(DeliveryStatus), \"; \", \"\"), DeliveryStatus),\n EventOriginalUid = tostring(column_ifexists(\"id\", \"\")),\n DvcEntityKey = \"\",\n AdditionalEntities = dynamic([]),\n AdditionalFields = iff(pack, bag_pack(\"sm\", sm, \"tls\", tls, \"pps\", pps, \"metadata\", metadata), dynamic({})),\n Type = \"ProofpointPODMailLog_CL\"\n | extend\n EventUid = iff(isnotempty(EventOriginalUid), strcat(\"ProofpointPODMailLog_CL:\", EventOriginalUid), \"\"),\n SrcDomainType = iff(isnotempty(SrcDomain), \"FQDN\", \"\"), IpAddr = SrcIpAddr\n | project-rename EventMessage = data\n | project\n TimeGenerated, Type, EventStartTime, EventEndTime, EventCount,\n EventVendor, EventProduct, EventSchema, EventSchemaVersion,\n EventType, EventSubType, EventOriginalType, EventResult,\n EventOriginalResultDetails, EventOriginalUid, EventUid, EventMessage,\n Dvc, DvcHostname, DvcDomain, DvcDomainType, DvcFQDN, DvcAction, DvcEntityKey,\n Src, SrcIpAddr, SrcHostname, SrcDomain, SrcDomainType, SrcFQDN, Dst, DstIpAddr, IpAddr,\n EmailMailFromAddress, EmailMailFromDomain, EmailSizeBytes, EmailProtocolName, EmailDirection,\n AdditionalFields, AdditionalEntities\n};\nparser(disabled=disabled, pack=pack)\n", + "query": "let parser = (disabled:bool=false, pack:bool=false) {\n ProofpointPODMailLog_CL\n | where not(disabled)\n | where isnotempty(tostring(sm.qid)) and (bag_has_key(sm, \"from\") or array_length(sm[\"to\"]) > 0)\n | extend\n IsDelivery = coalesce(array_length(sm[\"to\"]), 0) > 0,\n Relay = tostring(sm.relay),\n Dvc = coalesce(tostring(pps.agent), tostring(metadata.origin.data.agent)),\n EventStartTime = coalesce(eventTime, TimeGenerated),\n EmailMailFromAddress = trim(@\"[<>\\s]\", tostring(sm[\"from\"])),\n EmailSizeBytes = tolong(sm.sizeBytes),\n Dsn = tostring(sm.dsn),\n DeliveryStatus = tostring(sm.stat)\n | parse Relay with RelayHost \" [\" RelayIp \"]\" *\n | extend\n EmailMailFromAddress = iff(EmailMailFromAddress contains \"@\", EmailMailFromAddress, \"\"),\n RelayHost = trim_end(@\"\\.\", tostring(split(coalesce(RelayHost, Relay), \"@\")[-1])),\n RelayIp = iff(isnotnull(parse_ipv4(RelayIp)) or isnotempty(parse_ipv6(RelayIp)), RelayIp, \"\"),\n EventResult = case(Dsn startswith \"2.\", \"Success\", Dsn startswith \"4.\" or Dsn startswith \"5.\", \"Failure\", \"NA\"),\n EmailProtocolName = case(tostring(sm.proto) =~ \"SMTP\" or tostring(sm.mailer) in~ (\"smtp\", \"esmtp\"), \"SMTP\", \"\"),\n EventSubType = iff(IsDelivery, \"EmailSent\", \"EmailReceived\")\n | extend\n SrcIpAddr = iff(IsDelivery, \"\", RelayIp),\n DstIpAddr = iff(IsDelivery, RelayIp, \"\"),\n Src = iff(IsDelivery, Dvc, coalesce(RelayIp, RelayHost)),\n Dst = iff(IsDelivery, coalesce(RelayIp, RelayHost, iff(tostring(sm.mailer) =~ \"local\", Dvc, \"\")), Dvc),\n SrcFQDN = iff(not(IsDelivery) and RelayHost contains \".\", RelayHost, \"\"),\n SrcHostname = iff(IsDelivery, tostring(split(Dvc, \".\")[0]), tostring(split(RelayHost, \".\")[0])),\n SrcDomain = iff(IsDelivery, substring(Dvc, indexof(Dvc, \".\") + 1), substring(RelayHost, indexof(RelayHost, \".\") + 1)),\n DvcHostname = tostring(split(Dvc, \".\")[0]),\n DvcDomain = iff(Dvc contains \".\", substring(Dvc, indexof(Dvc, \".\") + 1), \"\"),\n DvcFQDN = iff(Dvc contains \".\", Dvc, \"\"),\n EmailMailFromDomain = iff(EmailMailFromAddress contains \"@\", tostring(split(EmailMailFromAddress, \"@\")[1]), \"\"),\n DvcAction = iff(IsDelivery and EventResult == \"Success\" and DeliveryStatus startswith \"Sent\", \"Delivered\", \"NA\")\n | extend\n SrcFQDN = iff(IsDelivery, DvcFQDN, SrcFQDN),\n SrcDomain = iff(iff(IsDelivery, Dvc, RelayHost) contains \".\", SrcDomain, \"\"),\n DvcDomainType = iff(isnotempty(DvcDomain), \"FQDN\", \"\"),\n EventEndTime = EventStartTime,\n EventCount = int(1),\n EventVendor = \"Proofpoint\",\n EventProduct = \"POD MailLog\",\n EventSchema = \"EmailEvent\",\n EventSchemaVersion = \"1.0.0\",\n EventType = \"EmailDelivery\",\n EventOriginalType = \"maillog\",\n EmailDirection = \"Unknown\",\n EventOriginalResultDetails = strcat(Dsn, iff(isnotempty(Dsn) and isnotempty(DeliveryStatus), \"; \", \"\"), DeliveryStatus),\n EventOriginalUid = tostring(column_ifexists(\"id\", \"\")),\n DvcEntityKey = \"\",\n AdditionalEntities = dynamic([]),\n AdditionalFields = iff(pack, bag_pack(\"sm\", sm, \"tls\", tls, \"pps\", pps, \"metadata\", metadata), dynamic({})),\n Type = \"ProofpointPODMailLog_CL\"\n | extend\n EventUid = iff(isnotempty(EventOriginalUid), strcat(\"ProofpointPODMailLog_CL:\", EventOriginalUid), \"\"),\n SrcDomainType = iff(isnotempty(SrcDomain), \"FQDN\", \"\"), IpAddr = SrcIpAddr\n | project-rename EventMessage = data\n | project\n TimeGenerated, Type, EventStartTime, EventEndTime, EventCount,\n EventVendor, EventProduct, EventSchema, EventSchemaVersion,\n EventType, EventSubType, EventOriginalType, EventResult,\n EventOriginalResultDetails, EventOriginalUid, EventUid, EventMessage,\n Dvc, DvcHostname, DvcDomain, DvcDomainType, DvcFQDN, DvcAction, DvcEntityKey,\n Src, SrcIpAddr, SrcHostname, SrcDomain, SrcDomainType, SrcFQDN, Dst, DstIpAddr, IpAddr,\n EmailMailFromAddress, EmailMailFromDomain, EmailSizeBytes, EmailProtocolName, EmailDirection,\n AdditionalFields, AdditionalEntities\n};\nparser(disabled=disabled, pack=pack)\n", "version": 1, "functionParameters": "disabled:bool=False,pack:bool=False" } diff --git a/Parsers/ASimEmailEvent/ARM/vimEmailEventProofpointPODMailLog/vimEmailEventProofpointPODMailLog.json b/Parsers/ASimEmailEvent/ARM/vimEmailEventProofpointPODMailLog/vimEmailEventProofpointPODMailLog.json index ae193859940..3b928ddba92 100644 --- a/Parsers/ASimEmailEvent/ARM/vimEmailEventProofpointPODMailLog/vimEmailEventProofpointPODMailLog.json +++ b/Parsers/ASimEmailEvent/ARM/vimEmailEventProofpointPODMailLog/vimEmailEventProofpointPODMailLog.json @@ -27,7 +27,7 @@ "displayName": "Email Event ASIM filtering parser for Proofpoint POD MailLog", "category": "ASIM", "FunctionAlias": "vimEmailEventProofpointPODMailLog", - "query": "let parser = (\n starttime:datetime=datetime(null),\n endtime:datetime=datetime(null),\n ipaddr_has_any_prefix:dynamic=dynamic([]),\n emailfromaddress_has_any:dynamic=dynamic([]),\n emailrecipient_has_any:dynamic=dynamic([]),\n emailsubject_has_any:dynamic=dynamic([]),\n emaildirection_in:dynamic=dynamic([]),\n emaildeliverylocation_in:dynamic=dynamic([]),\n eventresult:string='*',\n disabled:bool=false,\n pack:bool=false)\n{\n ProofpointPODMailLog_CL\n | where not(disabled)\n | where (isnull(starttime) or eventTime >= starttime or (isnull(eventTime) and TimeGenerated >= starttime))\n and (isnull(endtime) or eventTime <= endtime or (isnull(eventTime) and TimeGenerated <= endtime))\n | where array_length(emailfromaddress_has_any) == 0 and array_length(emailrecipient_has_any) == 0\n and array_length(emailsubject_has_any) == 0 and array_length(emaildeliverylocation_in) == 0\n | where array_length(emaildirection_in) == 0 or \"Unknown\" in (emaildirection_in)\n | where isnotempty(tostring(sm.qid)) and (bag_has_key(sm, \"from\") or array_length(sm[\"to\"]) > 0)\n | extend\n IsDelivery = coalesce(array_length(sm[\"to\"]), 0) > 0,\n Dsn = tostring(sm.dsn)\n | extend EventResult = case(Dsn startswith \"2.\", \"Success\", Dsn startswith \"4.\" or Dsn startswith \"5.\", \"Failure\", \"NA\")\n | where eventresult == \"*\" or EventResult == eventresult\n | extend\n Relay = tostring(sm.relay),\n Dvc = coalesce(tostring(pps.agent), tostring(metadata.origin.data.agent)),\n EventStartTime = coalesce(eventTime, TimeGenerated),\n EmailMailFromAddress = trim(@\"[<>\\s]\", tostring(sm[\"from\"])),\n EmailSizeBytes = tolong(sm.sizeBytes),\n DeliveryStatus = tostring(sm.stat)\n | parse Relay with RelayHost \" [\" RelayIp \"]\" *\n | extend RelayIp = iff(isnotnull(parse_ipv4(RelayIp)) or isnotempty(parse_ipv6(RelayIp)), RelayIp, \"\")\n | where array_length(ipaddr_has_any_prefix) == 0\n or has_any_ipv4_prefix(RelayIp, ipaddr_has_any_prefix)\n | extend\n EmailMailFromAddress = iff(EmailMailFromAddress contains \"@\", EmailMailFromAddress, \"\"),\n RelayHost = trim_end(@\"\\.\", coalesce(RelayHost, Relay)),\n EmailProtocolName = case(tostring(sm.proto) =~ \"SMTP\" or tostring(sm.mailer) in~ (\"smtp\", \"esmtp\"), \"SMTP\", \"\"),\n EventSubType = iff(IsDelivery, \"EmailSent\", \"EmailReceived\")\n | extend\n SrcIpAddr = iff(IsDelivery, \"\", RelayIp),\n DstIpAddr = iff(IsDelivery, RelayIp, \"\"),\n Src = iff(IsDelivery, Dvc, coalesce(RelayIp, RelayHost)),\n Dst = iff(IsDelivery, coalesce(RelayIp, RelayHost), Dvc),\n SrcFQDN = iff(not(IsDelivery) and RelayHost contains \".\", RelayHost, \"\"),\n SrcHostname = iff(IsDelivery, tostring(split(Dvc, \".\")[0]), tostring(split(RelayHost, \".\")[0])),\n SrcDomain = iff(IsDelivery, substring(Dvc, indexof(Dvc, \".\") + 1), substring(RelayHost, indexof(RelayHost, \".\") + 1)),\n DvcHostname = tostring(split(Dvc, \".\")[0]),\n DvcDomain = iff(Dvc contains \".\", substring(Dvc, indexof(Dvc, \".\") + 1), \"\"),\n DvcFQDN = iff(Dvc contains \".\", Dvc, \"\"),\n EmailMailFromDomain = iff(EmailMailFromAddress contains \"@\", tostring(split(EmailMailFromAddress, \"@\")[1]), \"\"),\n DvcAction = iff(IsDelivery and EventResult == \"Success\" and DeliveryStatus startswith \"Sent\", \"Delivered\", \"NA\")\n | extend\n SrcFQDN = iff(IsDelivery, DvcFQDN, SrcFQDN),\n SrcDomain = iff(iff(IsDelivery, Dvc, RelayHost) contains \".\", SrcDomain, \"\"),\n DvcDomainType = iff(isnotempty(DvcDomain), \"FQDN\", \"\"),\n EventEndTime = EventStartTime,\n EventCount = int(1),\n EventVendor = \"Proofpoint\",\n EventProduct = \"POD MailLog\",\n EventSchema = \"EmailEvent\",\n EventSchemaVersion = \"1.0.0\",\n EventType = \"EmailDelivery\",\n EventOriginalType = \"maillog\",\n EmailDirection = \"Unknown\",\n EventOriginalResultDetails = strcat(Dsn, iff(isnotempty(Dsn) and isnotempty(DeliveryStatus), \"; \", \"\"), DeliveryStatus),\n EventOriginalUid = tostring(column_ifexists(\"id\", \"\")),\n DvcEntityKey = \"\",\n AdditionalEntities = dynamic([]),\n AdditionalFields = iff(pack, bag_pack(\"sm\", sm, \"tls\", tls, \"pps\", pps, \"metadata\", metadata), dynamic({})),\n Type = \"ProofpointPODMailLog_CL\"\n | extend\n EventUid = iff(isnotempty(EventOriginalUid), strcat(\"ProofpointPODMailLog_CL:\", EventOriginalUid), \"\"),\n SrcDomainType = iff(isnotempty(SrcDomain), \"FQDN\", \"\"), IpAddr = SrcIpAddr\n | project-rename EventMessage = data\n | project\n TimeGenerated, Type, EventStartTime, EventEndTime, EventCount,\n EventVendor, EventProduct, EventSchema, EventSchemaVersion,\n EventType, EventSubType, EventOriginalType, EventResult,\n EventOriginalResultDetails, EventOriginalUid, EventUid, EventMessage,\n Dvc, DvcHostname, DvcDomain, DvcDomainType, DvcFQDN, DvcAction, DvcEntityKey,\n Src, SrcIpAddr, SrcHostname, SrcDomain, SrcDomainType, SrcFQDN, Dst, DstIpAddr, IpAddr,\n EmailMailFromAddress, EmailMailFromDomain, EmailSizeBytes, EmailProtocolName, EmailDirection,\n AdditionalFields, AdditionalEntities\n};\nparser(starttime=starttime, endtime=endtime, ipaddr_has_any_prefix=ipaddr_has_any_prefix,\n emailfromaddress_has_any=emailfromaddress_has_any, emailrecipient_has_any=emailrecipient_has_any,\n emailsubject_has_any=emailsubject_has_any, emaildirection_in=emaildirection_in,\n emaildeliverylocation_in=emaildeliverylocation_in, eventresult=eventresult, disabled=disabled, pack=pack)\n", + "query": "let parser = (\n starttime:datetime=datetime(null),\n endtime:datetime=datetime(null),\n ipaddr_has_any_prefix:dynamic=dynamic([]),\n emailfromaddress_has_any:dynamic=dynamic([]),\n emailrecipient_has_any:dynamic=dynamic([]),\n emailsubject_has_any:dynamic=dynamic([]),\n emaildirection_in:dynamic=dynamic([]),\n emaildeliverylocation_in:dynamic=dynamic([]),\n eventresult:string='*',\n disabled:bool=false,\n pack:bool=false)\n{\n ProofpointPODMailLog_CL\n | where not(disabled)\n | where (isnull(starttime) or eventTime >= starttime or (isnull(eventTime) and TimeGenerated >= starttime))\n and (isnull(endtime) or eventTime <= endtime or (isnull(eventTime) and TimeGenerated <= endtime))\n | where array_length(emailfromaddress_has_any) == 0 and array_length(emailrecipient_has_any) == 0\n and array_length(emailsubject_has_any) == 0 and array_length(emaildeliverylocation_in) == 0\n | where array_length(emaildirection_in) == 0 or \"Unknown\" in (emaildirection_in)\n | where isnotempty(tostring(sm.qid)) and (bag_has_key(sm, \"from\") or array_length(sm[\"to\"]) > 0)\n | extend\n IsDelivery = coalesce(array_length(sm[\"to\"]), 0) > 0,\n Dsn = tostring(sm.dsn)\n | extend EventResult = case(Dsn startswith \"2.\", \"Success\", Dsn startswith \"4.\" or Dsn startswith \"5.\", \"Failure\", \"NA\")\n | where eventresult == \"*\" or EventResult == eventresult\n | extend\n Relay = tostring(sm.relay),\n Dvc = coalesce(tostring(pps.agent), tostring(metadata.origin.data.agent)),\n EventStartTime = coalesce(eventTime, TimeGenerated),\n EmailMailFromAddress = trim(@\"[<>\\s]\", tostring(sm[\"from\"])),\n EmailSizeBytes = tolong(sm.sizeBytes),\n DeliveryStatus = tostring(sm.stat)\n | parse Relay with RelayHost \" [\" RelayIp \"]\" *\n | extend RelayIp = iff(isnotnull(parse_ipv4(RelayIp)) or isnotempty(parse_ipv6(RelayIp)), RelayIp, \"\")\n | where array_length(ipaddr_has_any_prefix) == 0\n or has_any_ipv4_prefix(RelayIp, ipaddr_has_any_prefix)\n | extend\n EmailMailFromAddress = iff(EmailMailFromAddress contains \"@\", EmailMailFromAddress, \"\"),\n RelayHost = trim_end(@\"\\.\", tostring(split(coalesce(RelayHost, Relay), \"@\")[-1])),\n EmailProtocolName = case(tostring(sm.proto) =~ \"SMTP\" or tostring(sm.mailer) in~ (\"smtp\", \"esmtp\"), \"SMTP\", \"\"),\n EventSubType = iff(IsDelivery, \"EmailSent\", \"EmailReceived\")\n | extend\n SrcIpAddr = iff(IsDelivery, \"\", RelayIp),\n DstIpAddr = iff(IsDelivery, RelayIp, \"\"),\n Src = iff(IsDelivery, Dvc, coalesce(RelayIp, RelayHost)),\n Dst = iff(IsDelivery, coalesce(RelayIp, RelayHost, iff(tostring(sm.mailer) =~ \"local\", Dvc, \"\")), Dvc),\n SrcFQDN = iff(not(IsDelivery) and RelayHost contains \".\", RelayHost, \"\"),\n SrcHostname = iff(IsDelivery, tostring(split(Dvc, \".\")[0]), tostring(split(RelayHost, \".\")[0])),\n SrcDomain = iff(IsDelivery, substring(Dvc, indexof(Dvc, \".\") + 1), substring(RelayHost, indexof(RelayHost, \".\") + 1)),\n DvcHostname = tostring(split(Dvc, \".\")[0]),\n DvcDomain = iff(Dvc contains \".\", substring(Dvc, indexof(Dvc, \".\") + 1), \"\"),\n DvcFQDN = iff(Dvc contains \".\", Dvc, \"\"),\n EmailMailFromDomain = iff(EmailMailFromAddress contains \"@\", tostring(split(EmailMailFromAddress, \"@\")[1]), \"\"),\n DvcAction = iff(IsDelivery and EventResult == \"Success\" and DeliveryStatus startswith \"Sent\", \"Delivered\", \"NA\")\n | extend\n SrcFQDN = iff(IsDelivery, DvcFQDN, SrcFQDN),\n SrcDomain = iff(iff(IsDelivery, Dvc, RelayHost) contains \".\", SrcDomain, \"\"),\n DvcDomainType = iff(isnotempty(DvcDomain), \"FQDN\", \"\"),\n EventEndTime = EventStartTime,\n EventCount = int(1),\n EventVendor = \"Proofpoint\",\n EventProduct = \"POD MailLog\",\n EventSchema = \"EmailEvent\",\n EventSchemaVersion = \"1.0.0\",\n EventType = \"EmailDelivery\",\n EventOriginalType = \"maillog\",\n EmailDirection = \"Unknown\",\n EventOriginalResultDetails = strcat(Dsn, iff(isnotempty(Dsn) and isnotempty(DeliveryStatus), \"; \", \"\"), DeliveryStatus),\n EventOriginalUid = tostring(column_ifexists(\"id\", \"\")),\n DvcEntityKey = \"\",\n AdditionalEntities = dynamic([]),\n AdditionalFields = iff(pack, bag_pack(\"sm\", sm, \"tls\", tls, \"pps\", pps, \"metadata\", metadata), dynamic({})),\n Type = \"ProofpointPODMailLog_CL\"\n | extend\n EventUid = iff(isnotempty(EventOriginalUid), strcat(\"ProofpointPODMailLog_CL:\", EventOriginalUid), \"\"),\n SrcDomainType = iff(isnotempty(SrcDomain), \"FQDN\", \"\"), IpAddr = SrcIpAddr\n | project-rename EventMessage = data\n | project\n TimeGenerated, Type, EventStartTime, EventEndTime, EventCount,\n EventVendor, EventProduct, EventSchema, EventSchemaVersion,\n EventType, EventSubType, EventOriginalType, EventResult,\n EventOriginalResultDetails, EventOriginalUid, EventUid, EventMessage,\n Dvc, DvcHostname, DvcDomain, DvcDomainType, DvcFQDN, DvcAction, DvcEntityKey,\n Src, SrcIpAddr, SrcHostname, SrcDomain, SrcDomainType, SrcFQDN, Dst, DstIpAddr, IpAddr,\n EmailMailFromAddress, EmailMailFromDomain, EmailSizeBytes, EmailProtocolName, EmailDirection,\n AdditionalFields, AdditionalEntities\n};\nparser(starttime=starttime, endtime=endtime, ipaddr_has_any_prefix=ipaddr_has_any_prefix,\n emailfromaddress_has_any=emailfromaddress_has_any, emailrecipient_has_any=emailrecipient_has_any,\n emailsubject_has_any=emailsubject_has_any, emaildirection_in=emaildirection_in,\n emaildeliverylocation_in=emaildeliverylocation_in, eventresult=eventresult, disabled=disabled, pack=pack)\n", "version": 1, "functionParameters": "starttime:datetime=datetime(null),endtime:datetime=datetime(null),ipaddr_has_any_prefix:dynamic=dynamic([]),emailfromaddress_has_any:dynamic=dynamic([]),emailrecipient_has_any:dynamic=dynamic([]),emailsubject_has_any:dynamic=dynamic([]),emaildirection_in:dynamic=dynamic([]),emaildeliverylocation_in:dynamic=dynamic([]),eventresult:string='*',disabled:bool=False,pack:bool=False" }