ASIM: Add Proofpoint POD MailLog EmailEvent parsers

.script/tests/KqlvalidationsTests/CustomTables/ProofpointPODMailLog_CL.json

@@ -0,0 +1,13 @@
+{
+  "Name": "ProofpointPODMailLog_CL",
+  "Properties": [
+    { "Name": "TimeGenerated", "Type": "DateTime" },
+    { "Name": "id", "Type": "String" },
+    { "Name": "data", "Type": "String" },
+    { "Name": "eventTime", "Type": "DateTime" },
+    { "Name": "metadata", "Type": "Dynamic" },
+    { "Name": "pps", "Type": "Dynamic" },
+    { "Name": "sm", "Type": "Dynamic" },
+    { "Name": "tls", "Type": "Dynamic" }
+  ]
+}

Parsers/ASimEmailEvent/ARM/ASimEmailEvent/ASimEmailEvent.json

@@ -27,7 +27,7 @@
         "displayName": "Email Event ASIM parser",
         "category": "ASIM",
         "FunctionAlias": "ASimEmailEvent",
-        "query": "let DisabledParsers=materialize(_GetWatchlist('ASimDisabledParsers') | where SearchKey in ('Any', 'ExcludeASimEmailEvent') | extend SourceSpecificParser=column_ifexists('SourceSpecificParser','') | distinct SourceSpecificParser| where isnotempty(SourceSpecificParser));\nlet ASimBuiltInDisabled=toscalar('ExcludeASimEmailEvent' in (DisabledParsers) or 'Any' in (DisabledParsers)); \nlet parser=(pack:bool=false){\nunion isfuzzy=true\n  vimEmailEventEmpty\n}; \nparser (pack=pack)",
+        "query": "let DisabledParsers=materialize(_GetWatchlist('ASimDisabledParsers') | where SearchKey in ('Any', 'ExcludeASimEmailEvent') | extend SourceSpecificParser=column_ifexists('SourceSpecificParser','') | distinct SourceSpecificParser| where isnotempty(SourceSpecificParser));\nlet ASimBuiltInDisabled=toscalar('ExcludeASimEmailEvent' in (DisabledParsers) or 'Any' in (DisabledParsers)); \nlet parser=(pack:bool=false){\nunion isfuzzy=true\n  vimEmailEventEmpty,\n  ASimEmailEventProofpointPODMailLog(pack=pack, disabled=(ASimBuiltInDisabled or ('ExcludeASimEmailEventProofpointPODMailLog' in (DisabledParsers))))\n}; \nparser (pack=pack)",
         "version": 1,
         "functionParameters": "pack:bool=False"
       }

Parsers/ASimEmailEvent/ARM/ASimEmailEventProofpointPODMailLog/ASimEmailEventProofpointPODMailLog.json

@@ -0,0 +1,36 @@
+{
+  "$schema": "https://schema.management.azure.com/schemas/2019-08-01/deploymentTemplate.json#",
+  "contentVersion": "1.0.0.0",
+  "parameters": {
+    "Workspace": {
+      "type": "string",
+      "metadata": {
+        "description": "The Microsoft Sentinel workspace into which the function will be deployed. Has to be in the selected Resource Group."
+      }
+    },
+    "WorkspaceRegion": {
+      "type": "string",
+      "defaultValue": "[resourceGroup().location]",
+      "metadata": {
+        "description": "The region of the selected workspace. The default value will use the Region selection above."
+      }
+    }
+  },
+  "resources": [
+    {
+      "type": "Microsoft.OperationalInsights/workspaces/savedSearches",
+      "apiVersion": "2020-08-01",
+      "name": "[concat(parameters('Workspace'), '/ASimEmailEventProofpointPODMailLog')]",
+      "location": "[parameters('WorkspaceRegion')]",
+      "properties": {
+        "etag": "*",
+        "displayName": "Email Event ASIM parser for Proofpoint POD MailLog",
+        "category": "ASIM",
+        "FunctionAlias": "ASimEmailEventProofpointPODMailLog",
+        "query": "let parser = (disabled:bool=false, pack:bool=false) {\n  ProofpointPODMailLog_CL\n  | where not(disabled)\n  | where isnotempty(tostring(sm.qid)) and (bag_has_key(sm, \"from\") or array_length(sm[\"to\"]) > 0)\n  | extend\n      IsDelivery = coalesce(array_length(sm[\"to\"]), 0) > 0,\n      Relay = tostring(sm.relay),\n      Dvc = coalesce(tostring(pps.agent), tostring(metadata.origin.data.agent)),\n      EventStartTime = coalesce(eventTime, TimeGenerated),\n      EmailMailFromAddress = trim(@\"[<>\\s]\", tostring(sm[\"from\"])),\n      EmailSizeBytes = tolong(sm.sizeBytes),\n      Dsn = tostring(sm.dsn),\n      DeliveryStatus = tostring(sm.stat)\n  | parse Relay with RelayHost \" [\" RelayIp \"]\" *\n  | extend\n      EmailMailFromAddress = iff(EmailMailFromAddress contains \"@\", EmailMailFromAddress, \"\"),\n      RelayHost = trim_end(@\"\\.\", tostring(split(coalesce(RelayHost, Relay), \"@\")[-1])),\n      RelayIp = iff(isnotnull(parse_ipv4(RelayIp)) or isnotempty(parse_ipv6(RelayIp)), RelayIp, \"\"),\n      EventResult = case(Dsn startswith \"2.\", \"Success\", Dsn startswith \"4.\" or Dsn startswith \"5.\", \"Failure\", \"NA\"),\n      EmailProtocolName = case(tostring(sm.proto) =~ \"SMTP\" or tostring(sm.mailer) in~ (\"smtp\", \"esmtp\"), \"SMTP\", \"\"),\n      EventSubType = iff(IsDelivery, \"EmailSent\", \"EmailReceived\")\n  | extend\n      SrcIpAddr = iff(IsDelivery, \"\", RelayIp),\n      DstIpAddr = iff(IsDelivery, RelayIp, \"\"),\n      Src = iff(IsDelivery, Dvc, coalesce(RelayIp, RelayHost)),\n      Dst = iff(IsDelivery, coalesce(RelayIp, RelayHost, iff(tostring(sm.mailer) =~ \"local\", Dvc, \"\")), Dvc),\n      SrcFQDN = iff(not(IsDelivery) and RelayHost contains \".\", RelayHost, \"\"),\n      SrcHostname = iff(IsDelivery, tostring(split(Dvc, \".\")[0]), tostring(split(RelayHost, \".\")[0])),\n      SrcDomain = iff(IsDelivery, substring(Dvc, indexof(Dvc, \".\") + 1), substring(RelayHost, indexof(RelayHost, \".\") + 1)),\n      DvcHostname = tostring(split(Dvc, \".\")[0]),\n      DvcDomain = iff(Dvc contains \".\", substring(Dvc, indexof(Dvc, \".\") + 1), \"\"),\n      DvcFQDN = iff(Dvc contains \".\", Dvc, \"\"),\n      EmailMailFromDomain = iff(EmailMailFromAddress contains \"@\", tostring(split(EmailMailFromAddress, \"@\")[1]), \"\"),\n      DvcAction = iff(IsDelivery and EventResult == \"Success\" and DeliveryStatus startswith \"Sent\", \"Delivered\", \"NA\")\n  | extend\n      SrcFQDN = iff(IsDelivery, DvcFQDN, SrcFQDN),\n      SrcDomain = iff(iff(IsDelivery, Dvc, RelayHost) contains \".\", SrcDomain, \"\"),\n      DvcDomainType = iff(isnotempty(DvcDomain), \"FQDN\", \"\"),\n      EventEndTime = EventStartTime,\n      EventCount = int(1),\n      EventVendor = \"Proofpoint\",\n      EventProduct = \"POD MailLog\",\n      EventSchema = \"EmailEvent\",\n      EventSchemaVersion = \"1.0.0\",\n      EventType = \"EmailDelivery\",\n      EventOriginalType = \"maillog\",\n      EmailDirection = \"Unknown\",\n      EventOriginalResultDetails = strcat(Dsn, iff(isnotempty(Dsn) and isnotempty(DeliveryStatus), \"; \", \"\"), DeliveryStatus),\n      EventOriginalUid = tostring(column_ifexists(\"id\", \"\")),\n      DvcEntityKey = \"\",\n      AdditionalEntities = dynamic([]),\n      AdditionalFields = iff(pack, bag_pack(\"sm\", sm, \"tls\", tls, \"pps\", pps, \"metadata\", metadata), dynamic({})),\n      Type = \"ProofpointPODMailLog_CL\"\n  | extend\n      EventUid = iff(isnotempty(EventOriginalUid), strcat(\"ProofpointPODMailLog_CL:\", EventOriginalUid), \"\"),\n      SrcDomainType = iff(isnotempty(SrcDomain), \"FQDN\", \"\"), IpAddr = SrcIpAddr\n  | project-rename EventMessage = data\n  | project\n      TimeGenerated, Type, EventStartTime, EventEndTime, EventCount,\n      EventVendor, EventProduct, EventSchema, EventSchemaVersion,\n      EventType, EventSubType, EventOriginalType, EventResult,\n      EventOriginalResultDetails, EventOriginalUid, EventUid, EventMessage,\n      Dvc, DvcHostname, DvcDomain, DvcDomainType, DvcFQDN, DvcAction, DvcEntityKey,\n      Src, SrcIpAddr, SrcHostname, SrcDomain, SrcDomainType, SrcFQDN, Dst, DstIpAddr, IpAddr,\n      EmailMailFromAddress, EmailMailFromDomain, EmailSizeBytes, EmailProtocolName, EmailDirection,\n      AdditionalFields, AdditionalEntities\n};\nparser(disabled=disabled, pack=pack)\n",
+        "version": 1,
+        "functionParameters": "disabled:bool=False,pack:bool=False"
+      }
+    }
+  ]
+}

Parsers/ASimEmailEvent/ARM/ASimEmailEventProofpointPODMailLog/README.md

@@ -0,0 +1,23 @@
+# Proofpoint POD MailLog ASIM EmailEvent Normalization Parser
+
+ARM template for ASIM EmailEvent schema parser for Proofpoint POD MailLog.
+
+Normalizes ProofpointPODMailLog_CL SMTP receipt and delivery records to EmailEvent.
+Uses the CCP table schema and repository samples; live validation is deferred.
+Envelope recipients are retained in AdditionalFields, not mapped to header recipients.
+
+
+The Advanced Security Information Model (ASIM) enables you to use and create source-agnostic content, simplifying your analysis of the data in your Microsoft Sentinel workspace.
+
+For more information, see:
+
+- [Normalization and the Advanced Security Information Model (ASIM)](https://aka.ms/AboutASIM)
+- [Deploy all of ASIM](https://aka.ms/DeployASIM)
+- [ASIM EmailEvent normalization schema reference](https://aka.ms/ASimEmailEventDoc)
+
+For the changelog, see:
+- [CHANGELOG](https://github.com/Azure/Azure-Sentinel/blob/master/Parsers/ASimEmailEvent/CHANGELOG/ASimEmailEventProofpointPODMailLog.md)
+
+<br>
+
+[![Deploy to Azure](https://aka.ms/deploytoazurebutton)](https://portal.azure.com/#create/Microsoft.Template/uri/https%3A%2F%2Fraw.githubusercontent.com%2FAzure%2FAzure-Sentinel%2Fmaster%2FParsers%2FASimEmailEvent%2FARM%2FASimEmailEventProofpointPODMailLog%2FASimEmailEventProofpointPODMailLog.json) [![Deploy to Azure Gov](https://aka.ms/deploytoazuregovernbutton)](https://portal.azure.us/#create/Microsoft.Template/uri/https%3A%2F%2Fraw.githubusercontent.com%2FAzure%2FAzure-Sentinel%2Fmaster%2FParsers%2FASimEmailEvent%2FARM%2FASimEmailEventProofpointPODMailLog%2FASimEmailEventProofpointPODMailLog.json)

Parsers/ASimEmailEvent/ARM/FullDeploymentEmailEvent.json

@@ -38,6 +38,26 @@
         }
       }
     },
+    {
+      "type": "Microsoft.Resources/deployments",
+      "apiVersion": "2020-10-01",
+      "name": "linkedASimEmailEventProofpointPODMailLog",
+      "properties": {
+        "mode": "Incremental",
+        "templateLink": {
+          "uri": "https://raw.githubusercontent.com/Azure/Azure-Sentinel/master/Parsers/ASimEmailEvent/ARM/ASimEmailEventProofpointPODMailLog/ASimEmailEventProofpointPODMailLog.json",
+          "contentVersion": "1.0.0.0"
+        },
+        "parameters": {
+          "Workspace": {
+            "value": "[parameters('Workspace')]"
+          },
+          "WorkspaceRegion": {
+            "value": "[parameters('WorkspaceRegion')]"
+          }
+        }
+      }
+    },
     {
       "type": "Microsoft.Resources/deployments",
       "apiVersion": "2020-10-01",
@@ -77,6 +97,26 @@
           }
         }
       }
+    },
+    {
+      "type": "Microsoft.Resources/deployments",
+      "apiVersion": "2020-10-01",
+      "name": "linkedvimEmailEventProofpointPODMailLog",
+      "properties": {
+        "mode": "Incremental",
+        "templateLink": {
+          "uri": "https://raw.githubusercontent.com/Azure/Azure-Sentinel/master/Parsers/ASimEmailEvent/ARM/vimEmailEventProofpointPODMailLog/vimEmailEventProofpointPODMailLog.json",
+          "contentVersion": "1.0.0.0"
+        },
+        "parameters": {
+          "Workspace": {
+            "value": "[parameters('Workspace')]"
+          },
+          "WorkspaceRegion": {
+            "value": "[parameters('WorkspaceRegion')]"
+          }
+        }
+      }
     }
   ],
   "outputs": {}

Parsers/ASimEmailEvent/ARM/imEmailEvent/imEmailEvent.json

@@ -27,7 +27,7 @@
         "displayName": "Email Event ASIM filtering parser",
         "category": "ASIM",
         "FunctionAlias": "imEmailEvent",
-        "query": "let DisabledParsers=materialize(_GetWatchlist('ASimDisabledParsers') | where SearchKey in ('Any', 'ExcludeimEmailEvent') | extend SourceSpecificParser=column_ifexists('SourceSpecificParser','') | distinct SourceSpecificParser | where isnotempty(SourceSpecificParser));\nlet imBuiltInDisabled=toscalar('ExcludeimEmailEvent' in (DisabledParsers) or 'Any' in (DisabledParsers)); \nlet parser=(\n  starttime: datetime=datetime(null), \n  endtime: datetime=datetime(null), \n  ipaddr_has_any_prefix: dynamic=dynamic([]),\n  emailfromaddress_has_any: dynamic=dynamic([]),\n  emailrecipient_has_any: dynamic=dynamic([]),\n  emailsubject_has_any: dynamic=dynamic([]),\n  emaildirection_in: dynamic=dynamic([]),\n  emaildeliverylocation_in: dynamic=dynamic([]),\n  eventresult: string='*',\n  pack:bool=false)\n{\nunion isfuzzy=true\n  vimEmailEventEmpty\n};\nparser (starttime=starttime, endtime=endtime, ipaddr_has_any_prefix=ipaddr_has_any_prefix, emailfromaddress_has_any=emailfromaddress_has_any, emailrecipient_has_any=emailrecipient_has_any, emailsubject_has_any=emailsubject_has_any, emaildirection_in=emaildirection_in, emaildeliverylocation_in=emaildeliverylocation_in, eventresult=eventresult, pack=pack)",
+        "query": "let DisabledParsers=materialize(_GetWatchlist('ASimDisabledParsers') | where SearchKey in ('Any', 'ExcludeimEmailEvent') | extend SourceSpecificParser=column_ifexists('SourceSpecificParser','') | distinct SourceSpecificParser | where isnotempty(SourceSpecificParser));\nlet imBuiltInDisabled=toscalar('ExcludeimEmailEvent' in (DisabledParsers) or 'Any' in (DisabledParsers)); \nlet parser=(\n  starttime: datetime=datetime(null), \n  endtime: datetime=datetime(null), \n  ipaddr_has_any_prefix: dynamic=dynamic([]),\n  emailfromaddress_has_any: dynamic=dynamic([]),\n  emailrecipient_has_any: dynamic=dynamic([]),\n  emailsubject_has_any: dynamic=dynamic([]),\n  emaildirection_in: dynamic=dynamic([]),\n  emaildeliverylocation_in: dynamic=dynamic([]),\n  eventresult: string='*',\n  pack:bool=false)\n{\nunion isfuzzy=true\n  vimEmailEventEmpty,\n  vimEmailEventProofpointPODMailLog(\n    starttime=starttime, endtime=endtime, ipaddr_has_any_prefix=ipaddr_has_any_prefix,\n    emailfromaddress_has_any=emailfromaddress_has_any, emailrecipient_has_any=emailrecipient_has_any,\n    emailsubject_has_any=emailsubject_has_any, emaildirection_in=emaildirection_in,\n    emaildeliverylocation_in=emaildeliverylocation_in, eventresult=eventresult, pack=pack,\n    disabled=(imBuiltInDisabled or ('ExcludevimEmailEventProofpointPODMailLog' in (DisabledParsers))))\n};\nparser (starttime=starttime, endtime=endtime, ipaddr_has_any_prefix=ipaddr_has_any_prefix, emailfromaddress_has_any=emailfromaddress_has_any, emailrecipient_has_any=emailrecipient_has_any, emailsubject_has_any=emailsubject_has_any, emaildirection_in=emaildirection_in, emaildeliverylocation_in=emaildeliverylocation_in, eventresult=eventresult, pack=pack)",
         "version": 1,
         "functionParameters": "starttime:datetime=datetime(null),endtime:datetime=datetime(null),ipaddr_has_any_prefix:dynamic=dynamic([]),emailfromaddress_has_any:dynamic=dynamic([]),emailrecipient_has_any:dynamic=dynamic([]),emailsubject_has_any:dynamic=dynamic([]),emaildirection_in:dynamic=dynamic([]),emaildeliverylocation_in:dynamic=dynamic([]),eventresult:string='*',pack:bool=False"
       }

Parsers/ASimEmailEvent/ARM/vimEmailEventProofpointPODMailLog/README.md

@@ -0,0 +1,24 @@
+# Proofpoint POD MailLog ASIM EmailEvent Normalization Parser
+
+ARM template for ASIM EmailEvent schema parser for Proofpoint POD MailLog.
+
+Filters and normalizes ProofpointPODMailLog_CL SMTP receipt and delivery records to EmailEvent.
+Uses the CCP table schema and repository samples; live validation is deferred.
+Header sender, header recipients, subject and delivery location are unavailable;
+nonempty filters for these fields return no records.
+
+
+The Advanced Security Information Model (ASIM) enables you to use and create source-agnostic content, simplifying your analysis of the data in your Microsoft Sentinel workspace.
+
+For more information, see:
+
+- [Normalization and the Advanced Security Information Model (ASIM)](https://aka.ms/AboutASIM)
+- [Deploy all of ASIM](https://aka.ms/DeployASIM)
+- [ASIM EmailEvent normalization schema reference](https://aka.ms/ASimEmailEventDoc)
+
+For the changelog, see:
+- [CHANGELOG](https://github.com/Azure/Azure-Sentinel/blob/master/Parsers/ASimEmailEvent/CHANGELOG/vimEmailEventProofpointPODMailLog.md)
+
+<br>
+
+[![Deploy to Azure](https://aka.ms/deploytoazurebutton)](https://portal.azure.com/#create/Microsoft.Template/uri/https%3A%2F%2Fraw.githubusercontent.com%2FAzure%2FAzure-Sentinel%2Fmaster%2FParsers%2FASimEmailEvent%2FARM%2FvimEmailEventProofpointPODMailLog%2FvimEmailEventProofpointPODMailLog.json) [![Deploy to Azure Gov](https://aka.ms/deploytoazuregovernbutton)](https://portal.azure.us/#create/Microsoft.Template/uri/https%3A%2F%2Fraw.githubusercontent.com%2FAzure%2FAzure-Sentinel%2Fmaster%2FParsers%2FASimEmailEvent%2FARM%2FvimEmailEventProofpointPODMailLog%2FvimEmailEventProofpointPODMailLog.json)